drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2022 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10 #include <unistd.h>
  11
  12 #include <rte_ip.h>
  13 #include <rte_mbuf.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_malloc.h>
  16 #include <rte_memcpy.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_cycles.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_dev.h>
  21 #include <cryptodev_pmd.h>
  22 #include <rte_common.h>
  23 #include <rte_fslmc.h>
  24 #include <fslmc_vfio.h>
  25 #include <dpaa2_hw_pvt.h>
  26 #include <dpaa2_hw_dpio.h>
  27 #include <dpaa2_hw_mempool.h>
  28 #include <fsl_dpopr.h>
  29 #include <fsl_dpseci.h>
  30 #include <fsl_mc_sys.h>
  31 #include <rte_hexdump.h>
  32
  33 #include "dpaa2_sec_priv.h"
  34 #include "dpaa2_sec_event.h"
  35 #include "dpaa2_sec_logs.h"
  36
  37 /* RTA header files */
  38 #include <desc/ipsec.h>
  39 #include <desc/pdcp.h>
  40 #include <desc/sdap.h>
  41 #include <desc/algo.h>
  42
  43 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
  44  * a pointer to the shared descriptor
  45  */
  46 #define MIN_JOB_DESC_SIZE       (CAAM_CMD_SZ + CAAM_PTR_SZ)
  47 #define FSL_VENDOR_ID           0x1957
  48 #define FSL_DEVICE_ID           0x410
  49 #define FSL_SUBSYSTEM_SEC       1
  50 #define FSL_MC_DPSECI_DEVID     3
  51
  52 #define NO_PREFETCH 0
  53
  54 #define DRIVER_DUMP_MODE "drv_dump_mode"
  55 #define DRIVER_STRICT_ORDER "drv_strict_order"
  56
  57 /* DPAA2_SEC_DP_DUMP levels */
  58 enum dpaa2_sec_dump_levels {
  59         DPAA2_SEC_DP_NO_DUMP,
  60         DPAA2_SEC_DP_ERR_DUMP,
  61         DPAA2_SEC_DP_FULL_DUMP
  62 };
  63
  64 uint8_t cryptodev_driver_id;
  65 uint8_t dpaa2_sec_dp_dump = DPAA2_SEC_DP_ERR_DUMP;
  66
  67 static inline void
  68 free_fle(const struct qbman_fd *fd, struct dpaa2_sec_qp *qp)
  69 {
  70         struct qbman_fle *fle;
  71         struct rte_crypto_op *op;
  72
  73 #ifdef RTE_LIB_SECURITY
  74         if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
  75                 return;
  76 #endif
  77         fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
  78         op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
  79         /* free the fle memory */
  80         if (likely(rte_pktmbuf_is_contiguous(op->sym->m_src)))
  81                 rte_mempool_put(qp->fle_pool, (void *)(fle-1));
  82         else
  83                 rte_free((void *)(fle-1));
  84 }
  85
  86 #ifdef RTE_LIB_SECURITY
  87 static inline int
  88 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
  89                            struct rte_crypto_op *op,
  90                            struct qbman_fd *fd, uint16_t bpid)
  91 {
  92         struct rte_crypto_sym_op *sym_op = op->sym;
  93         struct ctxt_priv *priv = sess->ctxt;
  94         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
  95         struct sec_flow_context *flc;
  96         struct rte_mbuf *mbuf;
  97         uint32_t in_len = 0, out_len = 0;
  98
  99         if (sym_op->m_dst)
 100                 mbuf = sym_op->m_dst;
 101         else
 102                 mbuf = sym_op->m_src;
 103
 104         /* first FLE entry used to store mbuf and session ctxt */
 105         fle = (struct qbman_fle *)rte_malloc(NULL,
 106                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 107                         RTE_CACHE_LINE_SIZE);
 108         if (unlikely(!fle)) {
 109                 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
 110                 return -ENOMEM;
 111         }
 112         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 113         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 114         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 115
 116         /* Save the shared descriptor */
 117         flc = &priv->flc_desc[0].flc;
 118
 119         op_fle = fle + 1;
 120         ip_fle = fle + 2;
 121         sge = fle + 3;
 122
 123         if (likely(bpid < MAX_BPID)) {
 124                 DPAA2_SET_FD_BPID(fd, bpid);
 125                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 126                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 127         } else {
 128                 DPAA2_SET_FD_IVP(fd);
 129                 DPAA2_SET_FLE_IVP(op_fle);
 130                 DPAA2_SET_FLE_IVP(ip_fle);
 131         }
 132
 133         /* Configure FD as a FRAME LIST */
 134         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 135         DPAA2_SET_FD_COMPOUND_FMT(fd);
 136         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 137
 138         /* Configure Output FLE with Scatter/Gather Entry */
 139         DPAA2_SET_FLE_SG_EXT(op_fle);
 140         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 141
 142         /* Configure Output SGE for Encap/Decap */
 143         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 144         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 145         /* o/p segs */
 146         while (mbuf->next) {
 147                 sge->length = mbuf->data_len;
 148                 out_len += sge->length;
 149                 sge++;
 150                 mbuf = mbuf->next;
 151                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 152                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 153         }
 154         /* using buf_len for last buf - so that extra data can be added */
 155         sge->length = mbuf->buf_len - mbuf->data_off;
 156         out_len += sge->length;
 157
 158         DPAA2_SET_FLE_FIN(sge);
 159         op_fle->length = out_len;
 160
 161         sge++;
 162         mbuf = sym_op->m_src;
 163
 164         /* Configure Input FLE with Scatter/Gather Entry */
 165         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 166         DPAA2_SET_FLE_SG_EXT(ip_fle);
 167         DPAA2_SET_FLE_FIN(ip_fle);
 168
 169         /* Configure input SGE for Encap/Decap */
 170         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 171         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 172         sge->length = mbuf->data_len;
 173         in_len += sge->length;
 174
 175         mbuf = mbuf->next;
 176         /* i/p segs */
 177         while (mbuf) {
 178                 sge++;
 179                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 180                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 181                 sge->length = mbuf->data_len;
 182                 in_len += sge->length;
 183                 mbuf = mbuf->next;
 184         }
 185         ip_fle->length = in_len;
 186         DPAA2_SET_FLE_FIN(sge);
 187
 188         /* In case of PDCP, per packet HFN is stored in
 189          * mbuf priv after sym_op.
 190          */
 191         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 192                 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
 193                                         sess->pdcp.hfn_ovd_offset);
 194                 /*enable HFN override override */
 195                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 196                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 197                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 198         }
 199         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 200
 201         return 0;
 202 }
 203
 204 static inline int
 205 build_proto_compound_fd(dpaa2_sec_session *sess,
 206                struct rte_crypto_op *op,
 207                struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
 208 {
 209         struct rte_crypto_sym_op *sym_op = op->sym;
 210         struct ctxt_priv *priv = sess->ctxt;
 211         struct qbman_fle *fle, *ip_fle, *op_fle;
 212         struct sec_flow_context *flc;
 213         struct rte_mbuf *src_mbuf = sym_op->m_src;
 214         struct rte_mbuf *dst_mbuf = sym_op->m_dst;
 215         int retval;
 216
 217         if (!dst_mbuf)
 218                 dst_mbuf = src_mbuf;
 219
 220         /* Save the shared descriptor */
 221         flc = &priv->flc_desc[0].flc;
 222
 223         /* we are using the first FLE entry to store Mbuf */
 224         retval = rte_mempool_get(qp->fle_pool, (void **)(&fle));
 225         if (retval) {
 226                 DPAA2_SEC_DP_DEBUG("Proto: Memory alloc failed");
 227                 return -ENOMEM;
 228         }
 229         memset(fle, 0, FLE_POOL_BUF_SIZE);
 230         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 231         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 232
 233         op_fle = fle + 1;
 234         ip_fle = fle + 2;
 235
 236         if (likely(bpid < MAX_BPID)) {
 237                 DPAA2_SET_FD_BPID(fd, bpid);
 238                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 239                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 240         } else {
 241                 DPAA2_SET_FD_IVP(fd);
 242                 DPAA2_SET_FLE_IVP(op_fle);
 243                 DPAA2_SET_FLE_IVP(ip_fle);
 244         }
 245
 246         /* Configure FD as a FRAME LIST */
 247         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 248         DPAA2_SET_FD_COMPOUND_FMT(fd);
 249         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 250
 251         /* Configure Output FLE with dst mbuf data  */
 252         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
 253         DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
 254         DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
 255
 256         /* Configure Input FLE with src mbuf data */
 257         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
 258         DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
 259         DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
 260
 261         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 262         DPAA2_SET_FLE_FIN(ip_fle);
 263
 264         /* In case of PDCP, per packet HFN is stored in
 265          * mbuf priv after sym_op.
 266          */
 267         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 268                 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
 269                                         sess->pdcp.hfn_ovd_offset);
 270                 /*enable HFN override override */
 271                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 272                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 273                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 274         }
 275
 276         return 0;
 277
 278 }
 279
 280 static inline int
 281 build_proto_fd(dpaa2_sec_session *sess,
 282                struct rte_crypto_op *op,
 283                struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
 284 {
 285         struct rte_crypto_sym_op *sym_op = op->sym;
 286         if (sym_op->m_dst)
 287                 return build_proto_compound_fd(sess, op, fd, bpid, qp);
 288
 289         struct ctxt_priv *priv = sess->ctxt;
 290         struct sec_flow_context *flc;
 291         struct rte_mbuf *mbuf = sym_op->m_src;
 292
 293         if (likely(bpid < MAX_BPID))
 294                 DPAA2_SET_FD_BPID(fd, bpid);
 295         else
 296                 DPAA2_SET_FD_IVP(fd);
 297
 298         /* Save the shared descriptor */
 299         flc = &priv->flc_desc[0].flc;
 300
 301         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 302         DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
 303         DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
 304         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 305
 306         /* save physical address of mbuf */
 307         op->sym->aead.digest.phys_addr = mbuf->buf_iova;
 308         mbuf->buf_iova = (size_t)op;
 309
 310         return 0;
 311 }
 312 #endif
 313
 314 static inline int
 315 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
 316                  struct rte_crypto_op *op,
 317                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 318 {
 319         struct rte_crypto_sym_op *sym_op = op->sym;
 320         struct ctxt_priv *priv = sess->ctxt;
 321         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 322         struct sec_flow_context *flc;
 323         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 324         int icv_len = sess->digest_length;
 325         uint8_t *old_icv;
 326         struct rte_mbuf *mbuf;
 327         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 328                         sess->iv.offset);
 329
 330         if (sym_op->m_dst)
 331                 mbuf = sym_op->m_dst;
 332         else
 333                 mbuf = sym_op->m_src;
 334
 335         /* first FLE entry used to store mbuf and session ctxt */
 336         fle = (struct qbman_fle *)rte_malloc(NULL,
 337                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 338                         RTE_CACHE_LINE_SIZE);
 339         if (unlikely(!fle)) {
 340                 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
 341                 return -ENOMEM;
 342         }
 343         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 344         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 345         DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
 346
 347         op_fle = fle + 1;
 348         ip_fle = fle + 2;
 349         sge = fle + 3;
 350
 351         /* Save the shared descriptor */
 352         flc = &priv->flc_desc[0].flc;
 353
 354         /* Configure FD as a FRAME LIST */
 355         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 356         DPAA2_SET_FD_COMPOUND_FMT(fd);
 357         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 358
 359         DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 360                    "iv-len=%d data_off: 0x%x\n",
 361                    sym_op->aead.data.offset,
 362                    sym_op->aead.data.length,
 363                    sess->digest_length,
 364                    sess->iv.length,
 365                    sym_op->m_src->data_off);
 366
 367         /* Configure Output FLE with Scatter/Gather Entry */
 368         DPAA2_SET_FLE_SG_EXT(op_fle);
 369         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 370
 371         if (auth_only_len)
 372                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 373
 374         op_fle->length = (sess->dir == DIR_ENC) ?
 375                         (sym_op->aead.data.length + icv_len) :
 376                         sym_op->aead.data.length;
 377
 378         /* Configure Output SGE for Encap/Decap */
 379         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 380         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
 381         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 382
 383         mbuf = mbuf->next;
 384         /* o/p segs */
 385         while (mbuf) {
 386                 sge++;
 387                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 388                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 389                 sge->length = mbuf->data_len;
 390                 mbuf = mbuf->next;
 391         }
 392         sge->length -= icv_len;
 393
 394         if (sess->dir == DIR_ENC) {
 395                 sge++;
 396                 DPAA2_SET_FLE_ADDR(sge,
 397                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 398                 sge->length = icv_len;
 399         }
 400         DPAA2_SET_FLE_FIN(sge);
 401
 402         sge++;
 403         mbuf = sym_op->m_src;
 404
 405         /* Configure Input FLE with Scatter/Gather Entry */
 406         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 407         DPAA2_SET_FLE_SG_EXT(ip_fle);
 408         DPAA2_SET_FLE_FIN(ip_fle);
 409         ip_fle->length = (sess->dir == DIR_ENC) ?
 410                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 411                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 412                  icv_len);
 413
 414         /* Configure Input SGE for Encap/Decap */
 415         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 416         sge->length = sess->iv.length;
 417
 418         sge++;
 419         if (auth_only_len) {
 420                 DPAA2_SET_FLE_ADDR(sge,
 421                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 422                 sge->length = auth_only_len;
 423                 sge++;
 424         }
 425
 426         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 427         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 428                                 mbuf->data_off);
 429         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 430
 431         mbuf = mbuf->next;
 432         /* i/p segs */
 433         while (mbuf) {
 434                 sge++;
 435                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 436                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 437                 sge->length = mbuf->data_len;
 438                 mbuf = mbuf->next;
 439         }
 440
 441         if (sess->dir == DIR_DEC) {
 442                 sge++;
 443                 old_icv = (uint8_t *)(sge + 1);
 444                 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
 445                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 446                 sge->length = icv_len;
 447         }
 448
 449         DPAA2_SET_FLE_FIN(sge);
 450         if (auth_only_len) {
 451                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 452                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 453         }
 454         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 455
 456         return 0;
 457 }
 458
 459 static inline int
 460 build_authenc_gcm_fd(dpaa2_sec_session *sess,
 461                      struct rte_crypto_op *op,
 462                      struct qbman_fd *fd, uint16_t bpid,
 463                      struct dpaa2_sec_qp *qp)
 464 {
 465         struct rte_crypto_sym_op *sym_op = op->sym;
 466         struct ctxt_priv *priv = sess->ctxt;
 467         struct qbman_fle *fle, *sge;
 468         struct sec_flow_context *flc;
 469         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 470         int icv_len = sess->digest_length, retval;
 471         uint8_t *old_icv;
 472         struct rte_mbuf *dst;
 473         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 474                         sess->iv.offset);
 475
 476         if (sym_op->m_dst)
 477                 dst = sym_op->m_dst;
 478         else
 479                 dst = sym_op->m_src;
 480
 481         /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
 482          * Currently we donot know which FLE has the mbuf stored.
 483          * So while retreiving we can go back 1 FLE from the FD -ADDR
 484          * to get the MBUF Addr from the previous FLE.
 485          * We can have a better approach to use the inline Mbuf
 486          */
 487         retval = rte_mempool_get(qp->fle_pool, (void **)(&fle));
 488         if (retval) {
 489                 DPAA2_SEC_DP_DEBUG("GCM: no buffer available in fle pool");
 490                 return -ENOMEM;
 491         }
 492         memset(fle, 0, FLE_POOL_BUF_SIZE);
 493         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 494         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 495         fle = fle + 1;
 496         sge = fle + 2;
 497         if (likely(bpid < MAX_BPID)) {
 498                 DPAA2_SET_FD_BPID(fd, bpid);
 499                 DPAA2_SET_FLE_BPID(fle, bpid);
 500                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 501                 DPAA2_SET_FLE_BPID(sge, bpid);
 502                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 503                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 504                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 505         } else {
 506                 DPAA2_SET_FD_IVP(fd);
 507                 DPAA2_SET_FLE_IVP(fle);
 508                 DPAA2_SET_FLE_IVP((fle + 1));
 509                 DPAA2_SET_FLE_IVP(sge);
 510                 DPAA2_SET_FLE_IVP((sge + 1));
 511                 DPAA2_SET_FLE_IVP((sge + 2));
 512                 DPAA2_SET_FLE_IVP((sge + 3));
 513         }
 514
 515         /* Save the shared descriptor */
 516         flc = &priv->flc_desc[0].flc;
 517         /* Configure FD as a FRAME LIST */
 518         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 519         DPAA2_SET_FD_COMPOUND_FMT(fd);
 520         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 521
 522         DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
 523                    "iv-len=%d data_off: 0x%x\n",
 524                    sym_op->aead.data.offset,
 525                    sym_op->aead.data.length,
 526                    sess->digest_length,
 527                    sess->iv.length,
 528                    sym_op->m_src->data_off);
 529
 530         /* Configure Output FLE with Scatter/Gather Entry */
 531         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 532         if (auth_only_len)
 533                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 534         fle->length = (sess->dir == DIR_ENC) ?
 535                         (sym_op->aead.data.length + icv_len) :
 536                         sym_op->aead.data.length;
 537
 538         DPAA2_SET_FLE_SG_EXT(fle);
 539
 540         /* Configure Output SGE for Encap/Decap */
 541         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 542         DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
 543         sge->length = sym_op->aead.data.length;
 544
 545         if (sess->dir == DIR_ENC) {
 546                 sge++;
 547                 DPAA2_SET_FLE_ADDR(sge,
 548                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 549                 sge->length = sess->digest_length;
 550         }
 551         DPAA2_SET_FLE_FIN(sge);
 552
 553         sge++;
 554         fle++;
 555
 556         /* Configure Input FLE with Scatter/Gather Entry */
 557         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 558         DPAA2_SET_FLE_SG_EXT(fle);
 559         DPAA2_SET_FLE_FIN(fle);
 560         fle->length = (sess->dir == DIR_ENC) ?
 561                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 562                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 563                  sess->digest_length);
 564
 565         /* Configure Input SGE for Encap/Decap */
 566         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 567         sge->length = sess->iv.length;
 568         sge++;
 569         if (auth_only_len) {
 570                 DPAA2_SET_FLE_ADDR(sge,
 571                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 572                 sge->length = auth_only_len;
 573                 DPAA2_SET_FLE_BPID(sge, bpid);
 574                 sge++;
 575         }
 576
 577         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 578         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 579                                 sym_op->m_src->data_off);
 580         sge->length = sym_op->aead.data.length;
 581         if (sess->dir == DIR_DEC) {
 582                 sge++;
 583                 old_icv = (uint8_t *)(sge + 1);
 584                 memcpy(old_icv, sym_op->aead.digest.data,
 585                        sess->digest_length);
 586                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 587                 sge->length = sess->digest_length;
 588         }
 589         DPAA2_SET_FLE_FIN(sge);
 590
 591         if (auth_only_len) {
 592                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 593                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 594         }
 595
 596         DPAA2_SET_FD_LEN(fd, fle->length);
 597         return 0;
 598 }
 599
 600 static inline int
 601 build_authenc_sg_fd(dpaa2_sec_session *sess,
 602                  struct rte_crypto_op *op,
 603                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 604 {
 605         struct rte_crypto_sym_op *sym_op = op->sym;
 606         struct ctxt_priv *priv = sess->ctxt;
 607         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 608         struct sec_flow_context *flc;
 609         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 610                                 sym_op->auth.data.offset;
 611         uint16_t auth_tail_len = sym_op->auth.data.length -
 612                                 sym_op->cipher.data.length - auth_hdr_len;
 613         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 614         int icv_len = sess->digest_length;
 615         uint8_t *old_icv;
 616         struct rte_mbuf *mbuf;
 617         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 618                         sess->iv.offset);
 619
 620         if (sym_op->m_dst)
 621                 mbuf = sym_op->m_dst;
 622         else
 623                 mbuf = sym_op->m_src;
 624
 625         /* first FLE entry used to store mbuf and session ctxt */
 626         fle = (struct qbman_fle *)rte_malloc(NULL,
 627                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 628                         RTE_CACHE_LINE_SIZE);
 629         if (unlikely(!fle)) {
 630                 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
 631                 return -ENOMEM;
 632         }
 633         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 634         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 635         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 636
 637         op_fle = fle + 1;
 638         ip_fle = fle + 2;
 639         sge = fle + 3;
 640
 641         /* Save the shared descriptor */
 642         flc = &priv->flc_desc[0].flc;
 643
 644         /* Configure FD as a FRAME LIST */
 645         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 646         DPAA2_SET_FD_COMPOUND_FMT(fd);
 647         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 648
 649         DPAA2_SEC_DP_DEBUG(
 650                 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 651                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 652                 sym_op->auth.data.offset,
 653                 sym_op->auth.data.length,
 654                 sess->digest_length,
 655                 sym_op->cipher.data.offset,
 656                 sym_op->cipher.data.length,
 657                 sess->iv.length,
 658                 sym_op->m_src->data_off);
 659
 660         /* Configure Output FLE with Scatter/Gather Entry */
 661         DPAA2_SET_FLE_SG_EXT(op_fle);
 662         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 663
 664         if (auth_only_len)
 665                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 666
 667         op_fle->length = (sess->dir == DIR_ENC) ?
 668                         (sym_op->cipher.data.length + icv_len) :
 669                         sym_op->cipher.data.length;
 670
 671         /* Configure Output SGE for Encap/Decap */
 672         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 673         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
 674         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 675
 676         mbuf = mbuf->next;
 677         /* o/p segs */
 678         while (mbuf) {
 679                 sge++;
 680                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 681                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 682                 sge->length = mbuf->data_len;
 683                 mbuf = mbuf->next;
 684         }
 685         sge->length -= icv_len;
 686
 687         if (sess->dir == DIR_ENC) {
 688                 sge++;
 689                 DPAA2_SET_FLE_ADDR(sge,
 690                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 691                 sge->length = icv_len;
 692         }
 693         DPAA2_SET_FLE_FIN(sge);
 694
 695         sge++;
 696         mbuf = sym_op->m_src;
 697
 698         /* Configure Input FLE with Scatter/Gather Entry */
 699         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 700         DPAA2_SET_FLE_SG_EXT(ip_fle);
 701         DPAA2_SET_FLE_FIN(ip_fle);
 702         ip_fle->length = (sess->dir == DIR_ENC) ?
 703                         (sym_op->auth.data.length + sess->iv.length) :
 704                         (sym_op->auth.data.length + sess->iv.length +
 705                          icv_len);
 706
 707         /* Configure Input SGE for Encap/Decap */
 708         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 709         sge->length = sess->iv.length;
 710
 711         sge++;
 712         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 713         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 714                                 mbuf->data_off);
 715         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 716
 717         mbuf = mbuf->next;
 718         /* i/p segs */
 719         while (mbuf) {
 720                 sge++;
 721                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 722                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 723                 sge->length = mbuf->data_len;
 724                 mbuf = mbuf->next;
 725         }
 726         sge->length -= icv_len;
 727
 728         if (sess->dir == DIR_DEC) {
 729                 sge++;
 730                 old_icv = (uint8_t *)(sge + 1);
 731                 memcpy(old_icv, sym_op->auth.digest.data,
 732                        icv_len);
 733                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 734                 sge->length = icv_len;
 735         }
 736
 737         DPAA2_SET_FLE_FIN(sge);
 738         if (auth_only_len) {
 739                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 740                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 741         }
 742         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 743
 744         return 0;
 745 }
 746
 747 static inline int
 748 build_authenc_fd(dpaa2_sec_session *sess,
 749                  struct rte_crypto_op *op,
 750                  struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
 751 {
 752         struct rte_crypto_sym_op *sym_op = op->sym;
 753         struct ctxt_priv *priv = sess->ctxt;
 754         struct qbman_fle *fle, *sge;
 755         struct sec_flow_context *flc;
 756         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 757                                 sym_op->auth.data.offset;
 758         uint16_t auth_tail_len = sym_op->auth.data.length -
 759                                 sym_op->cipher.data.length - auth_hdr_len;
 760         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 761
 762         int icv_len = sess->digest_length, retval;
 763         uint8_t *old_icv;
 764         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 765                         sess->iv.offset);
 766         struct rte_mbuf *dst;
 767
 768         if (sym_op->m_dst)
 769                 dst = sym_op->m_dst;
 770         else
 771                 dst = sym_op->m_src;
 772
 773         /* we are using the first FLE entry to store Mbuf.
 774          * Currently we donot know which FLE has the mbuf stored.
 775          * So while retreiving we can go back 1 FLE from the FD -ADDR
 776          * to get the MBUF Addr from the previous FLE.
 777          * We can have a better approach to use the inline Mbuf
 778          */
 779         retval = rte_mempool_get(qp->fle_pool, (void **)(&fle));
 780         if (retval) {
 781                 DPAA2_SEC_DP_DEBUG("AUTHENC: no buffer available in fle pool");
 782                 return -ENOMEM;
 783         }
 784         memset(fle, 0, FLE_POOL_BUF_SIZE);
 785         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 786         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 787         fle = fle + 1;
 788         sge = fle + 2;
 789         if (likely(bpid < MAX_BPID)) {
 790                 DPAA2_SET_FD_BPID(fd, bpid);
 791                 DPAA2_SET_FLE_BPID(fle, bpid);
 792                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 793                 DPAA2_SET_FLE_BPID(sge, bpid);
 794                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 795                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 796                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 797         } else {
 798                 DPAA2_SET_FD_IVP(fd);
 799                 DPAA2_SET_FLE_IVP(fle);
 800                 DPAA2_SET_FLE_IVP((fle + 1));
 801                 DPAA2_SET_FLE_IVP(sge);
 802                 DPAA2_SET_FLE_IVP((sge + 1));
 803                 DPAA2_SET_FLE_IVP((sge + 2));
 804                 DPAA2_SET_FLE_IVP((sge + 3));
 805         }
 806
 807         /* Save the shared descriptor */
 808         flc = &priv->flc_desc[0].flc;
 809         /* Configure FD as a FRAME LIST */
 810         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 811         DPAA2_SET_FD_COMPOUND_FMT(fd);
 812         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 813
 814         DPAA2_SEC_DP_DEBUG(
 815                 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
 816                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 817                 sym_op->auth.data.offset,
 818                 sym_op->auth.data.length,
 819                 sess->digest_length,
 820                 sym_op->cipher.data.offset,
 821                 sym_op->cipher.data.length,
 822                 sess->iv.length,
 823                 sym_op->m_src->data_off);
 824
 825         /* Configure Output FLE with Scatter/Gather Entry */
 826         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 827         if (auth_only_len)
 828                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 829         fle->length = (sess->dir == DIR_ENC) ?
 830                         (sym_op->cipher.data.length + icv_len) :
 831                         sym_op->cipher.data.length;
 832
 833         DPAA2_SET_FLE_SG_EXT(fle);
 834
 835         /* Configure Output SGE for Encap/Decap */
 836         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 837         DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
 838                                 dst->data_off);
 839         sge->length = sym_op->cipher.data.length;
 840
 841         if (sess->dir == DIR_ENC) {
 842                 sge++;
 843                 DPAA2_SET_FLE_ADDR(sge,
 844                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 845                 sge->length = sess->digest_length;
 846                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 847                                         sess->iv.length));
 848         }
 849         DPAA2_SET_FLE_FIN(sge);
 850
 851         sge++;
 852         fle++;
 853
 854         /* Configure Input FLE with Scatter/Gather Entry */
 855         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 856         DPAA2_SET_FLE_SG_EXT(fle);
 857         DPAA2_SET_FLE_FIN(fle);
 858         fle->length = (sess->dir == DIR_ENC) ?
 859                         (sym_op->auth.data.length + sess->iv.length) :
 860                         (sym_op->auth.data.length + sess->iv.length +
 861                          sess->digest_length);
 862
 863         /* Configure Input SGE for Encap/Decap */
 864         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 865         sge->length = sess->iv.length;
 866         sge++;
 867
 868         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 869         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 870                                 sym_op->m_src->data_off);
 871         sge->length = sym_op->auth.data.length;
 872         if (sess->dir == DIR_DEC) {
 873                 sge++;
 874                 old_icv = (uint8_t *)(sge + 1);
 875                 memcpy(old_icv, sym_op->auth.digest.data,
 876                        sess->digest_length);
 877                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 878                 sge->length = sess->digest_length;
 879                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 880                                  sess->digest_length +
 881                                  sess->iv.length));
 882         }
 883         DPAA2_SET_FLE_FIN(sge);
 884         if (auth_only_len) {
 885                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 886                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 887         }
 888         return 0;
 889 }
 890
 891 static inline int build_auth_sg_fd(
 892                 dpaa2_sec_session *sess,
 893                 struct rte_crypto_op *op,
 894                 struct qbman_fd *fd,
 895                 __rte_unused uint16_t bpid)
 896 {
 897         struct rte_crypto_sym_op *sym_op = op->sym;
 898         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 899         struct sec_flow_context *flc;
 900         struct ctxt_priv *priv = sess->ctxt;
 901         int data_len, data_offset;
 902         uint8_t *old_digest;
 903         struct rte_mbuf *mbuf;
 904
 905         data_len = sym_op->auth.data.length;
 906         data_offset = sym_op->auth.data.offset;
 907
 908         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 909             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 910                 if ((data_len & 7) || (data_offset & 7)) {
 911                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
 912                         return -ENOTSUP;
 913                 }
 914
 915                 data_len = data_len >> 3;
 916                 data_offset = data_offset >> 3;
 917         }
 918
 919         mbuf = sym_op->m_src;
 920         fle = (struct qbman_fle *)rte_malloc(NULL,
 921                         FLE_SG_MEM_SIZE(mbuf->nb_segs),
 922                         RTE_CACHE_LINE_SIZE);
 923         if (unlikely(!fle)) {
 924                 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
 925                 return -ENOMEM;
 926         }
 927         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
 928         /* first FLE entry used to store mbuf and session ctxt */
 929         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 930         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 931         op_fle = fle + 1;
 932         ip_fle = fle + 2;
 933         sge = fle + 3;
 934
 935         flc = &priv->flc_desc[DESC_INITFINAL].flc;
 936         /* sg FD */
 937         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 938         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 939         DPAA2_SET_FD_COMPOUND_FMT(fd);
 940
 941         /* o/p fle */
 942         DPAA2_SET_FLE_ADDR(op_fle,
 943                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 944         op_fle->length = sess->digest_length;
 945
 946         /* i/p fle */
 947         DPAA2_SET_FLE_SG_EXT(ip_fle);
 948         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 949         ip_fle->length = data_len;
 950
 951         if (sess->iv.length) {
 952                 uint8_t *iv_ptr;
 953
 954                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 955                                                    sess->iv.offset);
 956
 957                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 958                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 959                         sge->length = 12;
 960                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 961                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 962                         sge->length = 8;
 963                 } else {
 964                         sge->length = sess->iv.length;
 965                 }
 966                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 967                 ip_fle->length += sge->length;
 968                 sge++;
 969         }
 970         /* i/p 1st seg */
 971         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 972         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
 973
 974         if (data_len <= (mbuf->data_len - data_offset)) {
 975                 sge->length = data_len;
 976                 data_len = 0;
 977         } else {
 978                 sge->length = mbuf->data_len - data_offset;
 979
 980                 /* remaining i/p segs */
 981                 while ((data_len = data_len - sge->length) &&
 982                        (mbuf = mbuf->next)) {
 983                         sge++;
 984                         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 985                         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 986                         if (data_len > mbuf->data_len)
 987                                 sge->length = mbuf->data_len;
 988                         else
 989                                 sge->length = data_len;
 990                 }
 991         }
 992
 993         if (sess->dir == DIR_DEC) {
 994                 /* Digest verification case */
 995                 sge++;
 996                 old_digest = (uint8_t *)(sge + 1);
 997                 rte_memcpy(old_digest, sym_op->auth.digest.data,
 998                            sess->digest_length);
 999                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1000                 sge->length = sess->digest_length;
1001                 ip_fle->length += sess->digest_length;
1002         }
1003         DPAA2_SET_FLE_FIN(sge);
1004         DPAA2_SET_FLE_FIN(ip_fle);
1005         DPAA2_SET_FD_LEN(fd, ip_fle->length);
1006
1007         return 0;
1008 }
1009
1010 static inline int
1011 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1012               struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
1013 {
1014         struct rte_crypto_sym_op *sym_op = op->sym;
1015         struct qbman_fle *fle, *sge;
1016         struct sec_flow_context *flc;
1017         struct ctxt_priv *priv = sess->ctxt;
1018         int data_len, data_offset;
1019         uint8_t *old_digest;
1020         int retval;
1021
1022         data_len = sym_op->auth.data.length;
1023         data_offset = sym_op->auth.data.offset;
1024
1025         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1026             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1027                 if ((data_len & 7) || (data_offset & 7)) {
1028                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1029                         return -ENOTSUP;
1030                 }
1031
1032                 data_len = data_len >> 3;
1033                 data_offset = data_offset >> 3;
1034         }
1035
1036         retval = rte_mempool_get(qp->fle_pool, (void **)(&fle));
1037         if (retval) {
1038                 DPAA2_SEC_DP_DEBUG("AUTH: no buffer available in fle pool");
1039                 return -ENOMEM;
1040         }
1041         memset(fle, 0, FLE_POOL_BUF_SIZE);
1042         /* TODO we are using the first FLE entry to store Mbuf.
1043          * Currently we donot know which FLE has the mbuf stored.
1044          * So while retreiving we can go back 1 FLE from the FD -ADDR
1045          * to get the MBUF Addr from the previous FLE.
1046          * We can have a better approach to use the inline Mbuf
1047          */
1048         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1049         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1050         fle = fle + 1;
1051         sge = fle + 2;
1052
1053         if (likely(bpid < MAX_BPID)) {
1054                 DPAA2_SET_FD_BPID(fd, bpid);
1055                 DPAA2_SET_FLE_BPID(fle, bpid);
1056                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1057                 DPAA2_SET_FLE_BPID(sge, bpid);
1058                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1059         } else {
1060                 DPAA2_SET_FD_IVP(fd);
1061                 DPAA2_SET_FLE_IVP(fle);
1062                 DPAA2_SET_FLE_IVP((fle + 1));
1063                 DPAA2_SET_FLE_IVP(sge);
1064                 DPAA2_SET_FLE_IVP((sge + 1));
1065         }
1066
1067         flc = &priv->flc_desc[DESC_INITFINAL].flc;
1068         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1069         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1070         DPAA2_SET_FD_COMPOUND_FMT(fd);
1071
1072         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1073         fle->length = sess->digest_length;
1074         fle++;
1075
1076         /* Setting input FLE */
1077         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1078         DPAA2_SET_FLE_SG_EXT(fle);
1079         fle->length = data_len;
1080
1081         if (sess->iv.length) {
1082                 uint8_t *iv_ptr;
1083
1084                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1085                                                    sess->iv.offset);
1086
1087                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1088                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1089                         sge->length = 12;
1090                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1091                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1092                         sge->length = 8;
1093                 } else {
1094                         sge->length = sess->iv.length;
1095                 }
1096
1097                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1098                 fle->length = fle->length + sge->length;
1099                 sge++;
1100         }
1101
1102         /* Setting data to authenticate */
1103         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1104         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1105         sge->length = data_len;
1106
1107         if (sess->dir == DIR_DEC) {
1108                 sge++;
1109                 old_digest = (uint8_t *)(sge + 1);
1110                 rte_memcpy(old_digest, sym_op->auth.digest.data,
1111                            sess->digest_length);
1112                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1113                 sge->length = sess->digest_length;
1114                 fle->length = fle->length + sess->digest_length;
1115         }
1116
1117         DPAA2_SET_FLE_FIN(sge);
1118         DPAA2_SET_FLE_FIN(fle);
1119         DPAA2_SET_FD_LEN(fd, fle->length);
1120
1121         return 0;
1122 }
1123
1124 static int
1125 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1126                 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1127 {
1128         struct rte_crypto_sym_op *sym_op = op->sym;
1129         struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1130         int data_len, data_offset;
1131         struct sec_flow_context *flc;
1132         struct ctxt_priv *priv = sess->ctxt;
1133         struct rte_mbuf *mbuf;
1134         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1135                         sess->iv.offset);
1136
1137         data_len = sym_op->cipher.data.length;
1138         data_offset = sym_op->cipher.data.offset;
1139
1140         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1141                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1142                 if ((data_len & 7) || (data_offset & 7)) {
1143                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1144                         return -ENOTSUP;
1145                 }
1146
1147                 data_len = data_len >> 3;
1148                 data_offset = data_offset >> 3;
1149         }
1150
1151         if (sym_op->m_dst)
1152                 mbuf = sym_op->m_dst;
1153         else
1154                 mbuf = sym_op->m_src;
1155
1156         /* first FLE entry used to store mbuf and session ctxt */
1157         fle = (struct qbman_fle *)rte_malloc(NULL,
1158                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1159                         RTE_CACHE_LINE_SIZE);
1160         if (!fle) {
1161                 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1162                 return -ENOMEM;
1163         }
1164         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1165         /* first FLE entry used to store mbuf and session ctxt */
1166         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1167         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1168
1169         op_fle = fle + 1;
1170         ip_fle = fle + 2;
1171         sge = fle + 3;
1172
1173         flc = &priv->flc_desc[0].flc;
1174
1175         DPAA2_SEC_DP_DEBUG(
1176                 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1177                 " data_off: 0x%x\n",
1178                 data_offset,
1179                 data_len,
1180                 sess->iv.length,
1181                 sym_op->m_src->data_off);
1182
1183         /* o/p fle */
1184         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1185         op_fle->length = data_len;
1186         DPAA2_SET_FLE_SG_EXT(op_fle);
1187
1188         /* o/p 1st seg */
1189         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1190         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1191         sge->length = mbuf->data_len - data_offset;
1192
1193         mbuf = mbuf->next;
1194         /* o/p segs */
1195         while (mbuf) {
1196                 sge++;
1197                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1198                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1199                 sge->length = mbuf->data_len;
1200                 mbuf = mbuf->next;
1201         }
1202         DPAA2_SET_FLE_FIN(sge);
1203
1204         DPAA2_SEC_DP_DEBUG(
1205                 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1206                 flc, fle, fle->addr_hi, fle->addr_lo,
1207                 fle->length);
1208
1209         /* i/p fle */
1210         mbuf = sym_op->m_src;
1211         sge++;
1212         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1213         ip_fle->length = sess->iv.length + data_len;
1214         DPAA2_SET_FLE_SG_EXT(ip_fle);
1215
1216         /* i/p IV */
1217         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1218         DPAA2_SET_FLE_OFFSET(sge, 0);
1219         sge->length = sess->iv.length;
1220
1221         sge++;
1222
1223         /* i/p 1st seg */
1224         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1225         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1226         sge->length = mbuf->data_len - data_offset;
1227
1228         mbuf = mbuf->next;
1229         /* i/p segs */
1230         while (mbuf) {
1231                 sge++;
1232                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1233                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1234                 sge->length = mbuf->data_len;
1235                 mbuf = mbuf->next;
1236         }
1237         DPAA2_SET_FLE_FIN(sge);
1238         DPAA2_SET_FLE_FIN(ip_fle);
1239
1240         /* sg fd */
1241         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1242         DPAA2_SET_FD_LEN(fd, ip_fle->length);
1243         DPAA2_SET_FD_COMPOUND_FMT(fd);
1244         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1245
1246         DPAA2_SEC_DP_DEBUG(
1247                 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1248                 " off =%d, len =%d\n",
1249                 DPAA2_GET_FD_ADDR(fd),
1250                 DPAA2_GET_FD_BPID(fd),
1251                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1252                 DPAA2_GET_FD_OFFSET(fd),
1253                 DPAA2_GET_FD_LEN(fd));
1254         return 0;
1255 }
1256
1257 static int
1258 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1259                 struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
1260 {
1261         struct rte_crypto_sym_op *sym_op = op->sym;
1262         struct qbman_fle *fle, *sge;
1263         int retval, data_len, data_offset;
1264         struct sec_flow_context *flc;
1265         struct ctxt_priv *priv = sess->ctxt;
1266         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1267                         sess->iv.offset);
1268         struct rte_mbuf *dst;
1269
1270         data_len = sym_op->cipher.data.length;
1271         data_offset = sym_op->cipher.data.offset;
1272
1273         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1274                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1275                 if ((data_len & 7) || (data_offset & 7)) {
1276                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1277                         return -ENOTSUP;
1278                 }
1279
1280                 data_len = data_len >> 3;
1281                 data_offset = data_offset >> 3;
1282         }
1283
1284         if (sym_op->m_dst)
1285                 dst = sym_op->m_dst;
1286         else
1287                 dst = sym_op->m_src;
1288
1289         retval = rte_mempool_get(qp->fle_pool, (void **)(&fle));
1290         if (retval) {
1291                 DPAA2_SEC_DP_DEBUG("CIPHER: no buffer available in fle pool");
1292                 return -ENOMEM;
1293         }
1294         memset(fle, 0, FLE_POOL_BUF_SIZE);
1295         /* TODO we are using the first FLE entry to store Mbuf.
1296          * Currently we donot know which FLE has the mbuf stored.
1297          * So while retreiving we can go back 1 FLE from the FD -ADDR
1298          * to get the MBUF Addr from the previous FLE.
1299          * We can have a better approach to use the inline Mbuf
1300          */
1301         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1302         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1303         fle = fle + 1;
1304         sge = fle + 2;
1305
1306         if (likely(bpid < MAX_BPID)) {
1307                 DPAA2_SET_FD_BPID(fd, bpid);
1308                 DPAA2_SET_FLE_BPID(fle, bpid);
1309                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1310                 DPAA2_SET_FLE_BPID(sge, bpid);
1311                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1312         } else {
1313                 DPAA2_SET_FD_IVP(fd);
1314                 DPAA2_SET_FLE_IVP(fle);
1315                 DPAA2_SET_FLE_IVP((fle + 1));
1316                 DPAA2_SET_FLE_IVP(sge);
1317                 DPAA2_SET_FLE_IVP((sge + 1));
1318         }
1319
1320         flc = &priv->flc_desc[0].flc;
1321         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1322         DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1323         DPAA2_SET_FD_COMPOUND_FMT(fd);
1324         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1325
1326         DPAA2_SEC_DP_DEBUG(
1327                 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1328                 " data_off: 0x%x\n",
1329                 data_offset,
1330                 data_len,
1331                 sess->iv.length,
1332                 sym_op->m_src->data_off);
1333
1334         DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1335         DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1336
1337         fle->length = data_len + sess->iv.length;
1338
1339         DPAA2_SEC_DP_DEBUG(
1340                 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1341                 flc, fle, fle->addr_hi, fle->addr_lo,
1342                 fle->length);
1343
1344         fle++;
1345
1346         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1347         fle->length = data_len + sess->iv.length;
1348
1349         DPAA2_SET_FLE_SG_EXT(fle);
1350
1351         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1352         sge->length = sess->iv.length;
1353
1354         sge++;
1355         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1356         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1357
1358         sge->length = data_len;
1359         DPAA2_SET_FLE_FIN(sge);
1360         DPAA2_SET_FLE_FIN(fle);
1361
1362         DPAA2_SEC_DP_DEBUG(
1363                 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1364                 " off =%d, len =%d\n",
1365                 DPAA2_GET_FD_ADDR(fd),
1366                 DPAA2_GET_FD_BPID(fd),
1367                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1368                 DPAA2_GET_FD_OFFSET(fd),
1369                 DPAA2_GET_FD_LEN(fd));
1370
1371         return 0;
1372 }
1373
1374 static inline int
1375 build_sec_fd(struct rte_crypto_op *op,
1376              struct qbman_fd *fd, uint16_t bpid, struct dpaa2_sec_qp *qp)
1377 {
1378         int ret = -1;
1379         dpaa2_sec_session *sess;
1380
1381         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1382                 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1383                                 op->sym->session, cryptodev_driver_id);
1384 #ifdef RTE_LIB_SECURITY
1385         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1386                 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1387                                 op->sym->sec_session);
1388 #endif
1389         else {
1390                 DPAA2_SEC_DP_ERR("Session type invalid\n");
1391                 return -ENOTSUP;
1392         }
1393
1394         if (!sess) {
1395                 DPAA2_SEC_DP_ERR("Session not available\n");
1396                 return -EINVAL;
1397         }
1398
1399         /* Any of the buffer is segmented*/
1400         if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1401                   ((op->sym->m_dst != NULL) &&
1402                    !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1403                 switch (sess->ctxt_type) {
1404                 case DPAA2_SEC_CIPHER:
1405                         ret = build_cipher_sg_fd(sess, op, fd, bpid);
1406                         break;
1407                 case DPAA2_SEC_AUTH:
1408                         ret = build_auth_sg_fd(sess, op, fd, bpid);
1409                         break;
1410                 case DPAA2_SEC_AEAD:
1411                         ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1412                         break;
1413                 case DPAA2_SEC_CIPHER_HASH:
1414                         ret = build_authenc_sg_fd(sess, op, fd, bpid);
1415                         break;
1416 #ifdef RTE_LIB_SECURITY
1417                 case DPAA2_SEC_IPSEC:
1418                 case DPAA2_SEC_PDCP:
1419                         ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1420                         break;
1421 #endif
1422                 case DPAA2_SEC_HASH_CIPHER:
1423                 default:
1424                         DPAA2_SEC_ERR("error: Unsupported session");
1425                 }
1426         } else {
1427                 switch (sess->ctxt_type) {
1428                 case DPAA2_SEC_CIPHER:
1429                         ret = build_cipher_fd(sess, op, fd, bpid, qp);
1430                         break;
1431                 case DPAA2_SEC_AUTH:
1432                         ret = build_auth_fd(sess, op, fd, bpid, qp);
1433                         break;
1434                 case DPAA2_SEC_AEAD:
1435                         ret = build_authenc_gcm_fd(sess, op, fd, bpid, qp);
1436                         break;
1437                 case DPAA2_SEC_CIPHER_HASH:
1438                         ret = build_authenc_fd(sess, op, fd, bpid, qp);
1439                         break;
1440 #ifdef RTE_LIB_SECURITY
1441                 case DPAA2_SEC_IPSEC:
1442                         ret = build_proto_fd(sess, op, fd, bpid, qp);
1443                         break;
1444                 case DPAA2_SEC_PDCP:
1445                         ret = build_proto_compound_fd(sess, op, fd, bpid, qp);
1446                         break;
1447 #endif
1448                 case DPAA2_SEC_HASH_CIPHER:
1449                 default:
1450                         DPAA2_SEC_ERR("error: Unsupported session");
1451                         ret = -ENOTSUP;
1452                 }
1453         }
1454         return ret;
1455 }
1456
1457 static uint16_t
1458 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1459                         uint16_t nb_ops)
1460 {
1461         /* Function to transmit the frames to given device and VQ*/
1462         uint32_t loop;
1463         int32_t ret;
1464         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1465         uint32_t frames_to_send, retry_count;
1466         struct qbman_eq_desc eqdesc;
1467         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1468         struct qbman_swp *swp;
1469         uint16_t num_tx = 0;
1470         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1471         /*todo - need to support multiple buffer pools */
1472         uint16_t bpid;
1473         struct rte_mempool *mb_pool;
1474
1475         if (unlikely(nb_ops == 0))
1476                 return 0;
1477
1478         if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1479                 DPAA2_SEC_ERR("sessionless crypto op not supported");
1480                 return 0;
1481         }
1482         /*Prepare enqueue descriptor*/
1483         qbman_eq_desc_clear(&eqdesc);
1484         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1485         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1486         qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1487
1488         if (!DPAA2_PER_LCORE_DPIO) {
1489                 ret = dpaa2_affine_qbman_swp();
1490                 if (ret) {
1491                         DPAA2_SEC_ERR(
1492                                 "Failed to allocate IO portal, tid: %d\n",
1493                                 rte_gettid());
1494                         return 0;
1495                 }
1496         }
1497         swp = DPAA2_PER_LCORE_PORTAL;
1498
1499         while (nb_ops) {
1500                 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1501                         dpaa2_eqcr_size : nb_ops;
1502
1503                 for (loop = 0; loop < frames_to_send; loop++) {
1504                         if (*dpaa2_seqn((*ops)->sym->m_src)) {
1505                                 if (*dpaa2_seqn((*ops)->sym->m_src) & QBMAN_ENQUEUE_FLAG_DCA) {
1506                                         DPAA2_PER_LCORE_DQRR_SIZE--;
1507                                         DPAA2_PER_LCORE_DQRR_HELD &= ~(1 <<
1508                                         *dpaa2_seqn((*ops)->sym->m_src) &
1509                                         QBMAN_EQCR_DCA_IDXMASK);
1510                                 }
1511                                 flags[loop] = *dpaa2_seqn((*ops)->sym->m_src);
1512                                 *dpaa2_seqn((*ops)->sym->m_src) = DPAA2_INVALID_MBUF_SEQN;
1513                         }
1514
1515                         /*Clear the unused FD fields before sending*/
1516                         memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1517                         mb_pool = (*ops)->sym->m_src->pool;
1518                         bpid = mempool_to_bpid(mb_pool);
1519                         ret = build_sec_fd(*ops, &fd_arr[loop], bpid, dpaa2_qp);
1520                         if (ret) {
1521                                 DPAA2_SEC_DP_DEBUG("FD build failed\n");
1522                                 goto skip_tx;
1523                         }
1524                         ops++;
1525                 }
1526
1527                 loop = 0;
1528                 retry_count = 0;
1529                 while (loop < frames_to_send) {
1530                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1531                                                          &fd_arr[loop],
1532                                                          &flags[loop],
1533                                                          frames_to_send - loop);
1534                         if (unlikely(ret < 0)) {
1535                                 retry_count++;
1536                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1537                                         num_tx += loop;
1538                                         nb_ops -= loop;
1539                                         DPAA2_SEC_DP_DEBUG("Enqueue fail\n");
1540                                         /* freeing the fle buffers */
1541                                         while (loop < frames_to_send) {
1542                                                 free_fle(&fd_arr[loop],
1543                                                                 dpaa2_qp);
1544                                                 loop++;
1545                                         }
1546                                         goto skip_tx;
1547                                 }
1548                         } else {
1549                                 loop += ret;
1550                                 retry_count = 0;
1551                         }
1552                 }
1553
1554                 num_tx += loop;
1555                 nb_ops -= loop;
1556         }
1557 skip_tx:
1558         dpaa2_qp->tx_vq.tx_pkts += num_tx;
1559         dpaa2_qp->tx_vq.err_pkts += nb_ops;
1560         return num_tx;
1561 }
1562
1563 #ifdef RTE_LIB_SECURITY
1564 static inline struct rte_crypto_op *
1565 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1566 {
1567         struct rte_crypto_op *op;
1568         uint16_t len = DPAA2_GET_FD_LEN(fd);
1569         int16_t diff = 0;
1570         dpaa2_sec_session *sess_priv __rte_unused;
1571
1572         if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1573                 DPAA2_SEC_ERR("error: non inline buffer");
1574                 return NULL;
1575         }
1576         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1577                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1578                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1579
1580         diff = len - mbuf->pkt_len;
1581         mbuf->pkt_len += diff;
1582         mbuf->data_len += diff;
1583         op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1584         mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1585         op->sym->aead.digest.phys_addr = 0L;
1586
1587         sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1588                                 op->sym->sec_session);
1589         if (sess_priv->dir == DIR_ENC)
1590                 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1591         else
1592                 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1593
1594         if (unlikely(fd->simple.frc)) {
1595                 DPAA2_SEC_ERR("SEC returned Error - %x",
1596                                 fd->simple.frc);
1597                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1598         } else {
1599                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1600         }
1601
1602         return op;
1603 }
1604 #endif
1605
1606 static inline struct rte_crypto_op *
1607 sec_fd_to_mbuf(const struct qbman_fd *fd, struct dpaa2_sec_qp *qp)
1608 {
1609         struct qbman_fle *fle;
1610         struct rte_crypto_op *op;
1611         struct rte_mbuf *dst, *src;
1612
1613 #ifdef RTE_LIB_SECURITY
1614         if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1615                 return sec_simple_fd_to_mbuf(fd);
1616 #endif
1617         fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1618
1619         DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1620                            fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1621
1622         /* we are using the first FLE entry to store Mbuf.
1623          * Currently we donot know which FLE has the mbuf stored.
1624          * So while retreiving we can go back 1 FLE from the FD -ADDR
1625          * to get the MBUF Addr from the previous FLE.
1626          * We can have a better approach to use the inline Mbuf
1627          */
1628
1629         op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1630
1631         /* Prefeth op */
1632         src = op->sym->m_src;
1633         rte_prefetch0(src);
1634
1635         if (op->sym->m_dst) {
1636                 dst = op->sym->m_dst;
1637                 rte_prefetch0(dst);
1638         } else
1639                 dst = src;
1640
1641 #ifdef RTE_LIB_SECURITY
1642         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1643                 uint16_t len = DPAA2_GET_FD_LEN(fd);
1644                 dst->pkt_len = len;
1645                 while (dst->next != NULL) {
1646                         len -= dst->data_len;
1647                         dst = dst->next;
1648                 }
1649                 dst->data_len = len;
1650         }
1651 #endif
1652         DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1653                 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1654                 (void *)dst,
1655                 dst->buf_addr,
1656                 DPAA2_GET_FD_ADDR(fd),
1657                 DPAA2_GET_FD_BPID(fd),
1658                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1659                 DPAA2_GET_FD_OFFSET(fd),
1660                 DPAA2_GET_FD_LEN(fd));
1661
1662         /* free the fle memory */
1663         if (likely(rte_pktmbuf_is_contiguous(src))) {
1664                 rte_mempool_put(qp->fle_pool, (void *)(fle-1));
1665         } else
1666                 rte_free((void *)(fle-1));
1667
1668         return op;
1669 }
1670
1671 static void
1672 dpaa2_sec_dump(struct rte_crypto_op *op)
1673 {
1674         int i;
1675         dpaa2_sec_session *sess = NULL;
1676         struct ctxt_priv *priv;
1677         uint8_t bufsize;
1678         struct rte_crypto_sym_op *sym_op;
1679
1680         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1681                 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1682                         op->sym->session, cryptodev_driver_id);
1683 #ifdef RTE_LIBRTE_SECURITY
1684         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1685                 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1686                         op->sym->sec_session);
1687 #endif
1688
1689         if (sess == NULL)
1690                 goto mbuf_dump;
1691
1692         priv = (struct ctxt_priv *)sess->ctxt;
1693         printf("\n****************************************\n"
1694                 "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n"
1695                 "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n"
1696                 "\tCipher key len:\t%zd\n", sess->ctxt_type,
1697                 (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC",
1698                 sess->cipher_alg, sess->auth_alg, sess->aead_alg,
1699                 sess->cipher_key.length);
1700                 rte_hexdump(stdout, "cipher key", sess->cipher_key.data,
1701                                 sess->cipher_key.length);
1702                 rte_hexdump(stdout, "auth key", sess->auth_key.data,
1703                                 sess->auth_key.length);
1704         printf("\tAuth key len:\t%zd\n\tIV len:\t\t%d\n\tIV offset:\t%d\n"
1705                 "\tdigest length:\t%d\n\tstatus:\t\t%d\n\taead auth only"
1706                 " len:\t%d\n\taead cipher text:\t%d\n",
1707                 sess->auth_key.length, sess->iv.length, sess->iv.offset,
1708                 sess->digest_length, sess->status,
1709                 sess->ext_params.aead_ctxt.auth_only_len,
1710                 sess->ext_params.aead_ctxt.auth_cipher_text);
1711 #ifdef RTE_LIBRTE_SECURITY
1712         printf("PDCP session params:\n"
1713                 "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:"
1714                 "\t%d\n\tsn_size:\t%d\n\thfn_ovd_offset:\t%d\n\thfn:\t\t%d\n"
1715                 "\thfn_threshold:\t0x%x\n", sess->pdcp.domain,
1716                 sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd,
1717                 sess->pdcp.sn_size, sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn,
1718                 sess->pdcp.hfn_threshold);
1719
1720 #endif
1721         bufsize = (uint8_t)priv->flc_desc[0].flc.word1_sdl;
1722         printf("Descriptor Dump:\n");
1723         for (i = 0; i < bufsize; i++)
1724                 printf("\tDESC[%d]:0x%x\n", i, priv->flc_desc[0].desc[i]);
1725
1726         printf("\n");
1727 mbuf_dump:
1728         sym_op = op->sym;
1729         if (sym_op->m_src) {
1730                 printf("Source mbuf:\n");
1731                 rte_pktmbuf_dump(stdout, sym_op->m_src, sym_op->m_src->data_len);
1732         }
1733         if (sym_op->m_dst) {
1734                 printf("Destination mbuf:\n");
1735                 rte_pktmbuf_dump(stdout, sym_op->m_dst, sym_op->m_dst->data_len);
1736         }
1737
1738         printf("Session address = %p\ncipher offset: %d, length: %d\n"
1739                 "auth offset: %d, length:  %d\n aead offset: %d, length: %d\n"
1740                 , sym_op->session,
1741                 sym_op->cipher.data.offset, sym_op->cipher.data.length,
1742                 sym_op->auth.data.offset, sym_op->auth.data.length,
1743                 sym_op->aead.data.offset, sym_op->aead.data.length);
1744         printf("\n");
1745
1746 }
1747
1748 static void
1749 dpaa2_sec_free_eqresp_buf(uint16_t eqresp_ci,
1750                           struct dpaa2_queue *dpaa2_q)
1751 {
1752         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1753         struct rte_crypto_op *op;
1754         struct qbman_fd *fd;
1755         struct dpaa2_sec_qp *dpaa2_qp;
1756
1757         dpaa2_qp = container_of(dpaa2_q, struct dpaa2_sec_qp, tx_vq);
1758         fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]);
1759         op = sec_fd_to_mbuf(fd, dpaa2_qp);
1760         /* Instead of freeing, enqueue it to the sec tx queue (sec->core)
1761          * after setting an error in FD. But this will have performance impact.
1762          */
1763         rte_pktmbuf_free(op->sym->m_src);
1764 }
1765
1766 static void
1767 dpaa2_sec_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q,
1768                              struct rte_mbuf *m,
1769                              struct qbman_eq_desc *eqdesc)
1770 {
1771         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1772         struct eqresp_metadata *eqresp_meta;
1773         struct dpaa2_sec_dev_private *priv = dpaa2_q->crypto_data->dev_private;
1774         uint16_t orpid, seqnum;
1775         uint8_t dq_idx;
1776
1777         if (*dpaa2_seqn(m) & DPAA2_ENQUEUE_FLAG_ORP) {
1778                 orpid = (*dpaa2_seqn(m) & DPAA2_EQCR_OPRID_MASK) >>
1779                         DPAA2_EQCR_OPRID_SHIFT;
1780                 seqnum = (*dpaa2_seqn(m) & DPAA2_EQCR_SEQNUM_MASK) >>
1781                         DPAA2_EQCR_SEQNUM_SHIFT;
1782
1783
1784                 if (!priv->en_loose_ordered) {
1785                         qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0);
1786                         qbman_eq_desc_set_response(eqdesc, (uint64_t)
1787                                 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[
1788                                 dpio_dev->eqresp_pi]), 1);
1789                         qbman_eq_desc_set_token(eqdesc, 1);
1790
1791                         eqresp_meta = &dpio_dev->eqresp_meta[dpio_dev->eqresp_pi];
1792                         eqresp_meta->dpaa2_q = dpaa2_q;
1793                         eqresp_meta->mp = m->pool;
1794
1795                         dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ?
1796                                 dpio_dev->eqresp_pi++ : (dpio_dev->eqresp_pi = 0);
1797                 } else {
1798                         qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0);
1799                 }
1800         } else {
1801                 dq_idx = *dpaa2_seqn(m) - 1;
1802                 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0);
1803                 DPAA2_PER_LCORE_DQRR_SIZE--;
1804                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx);
1805         }
1806         *dpaa2_seqn(m) = DPAA2_INVALID_MBUF_SEQN;
1807 }
1808
1809
1810 static uint16_t
1811 dpaa2_sec_enqueue_burst_ordered(void *qp, struct rte_crypto_op **ops,
1812                         uint16_t nb_ops)
1813 {
1814         /* Function to transmit the frames to given device and VQ*/
1815         uint32_t loop;
1816         int32_t ret;
1817         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1818         uint32_t frames_to_send, num_free_eq_desc, retry_count;
1819         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
1820         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1821         struct qbman_swp *swp;
1822         uint16_t num_tx = 0;
1823         uint16_t bpid;
1824         struct rte_mempool *mb_pool;
1825         struct dpaa2_sec_dev_private *priv =
1826                                 dpaa2_qp->tx_vq.crypto_data->dev_private;
1827
1828         if (unlikely(nb_ops == 0))
1829                 return 0;
1830
1831         if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1832                 DPAA2_SEC_ERR("sessionless crypto op not supported");
1833                 return 0;
1834         }
1835
1836         if (!DPAA2_PER_LCORE_DPIO) {
1837                 ret = dpaa2_affine_qbman_swp();
1838                 if (ret) {
1839                         DPAA2_SEC_ERR("Failure in affining portal");
1840                         return 0;
1841                 }
1842         }
1843         swp = DPAA2_PER_LCORE_PORTAL;
1844
1845         while (nb_ops) {
1846                 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1847                         dpaa2_eqcr_size : nb_ops;
1848
1849                 if (!priv->en_loose_ordered) {
1850                         if (*dpaa2_seqn((*ops)->sym->m_src)) {
1851                                 num_free_eq_desc = dpaa2_free_eq_descriptors();
1852                                 if (num_free_eq_desc < frames_to_send)
1853                                         frames_to_send = num_free_eq_desc;
1854                         }
1855                 }
1856
1857                 for (loop = 0; loop < frames_to_send; loop++) {
1858                         /*Prepare enqueue descriptor*/
1859                         qbman_eq_desc_clear(&eqdesc[loop]);
1860                         qbman_eq_desc_set_fq(&eqdesc[loop], dpaa2_qp->tx_vq.fqid);
1861
1862                         if (*dpaa2_seqn((*ops)->sym->m_src))
1863                                 dpaa2_sec_set_enqueue_descriptor(
1864                                                 &dpaa2_qp->tx_vq,
1865                                                 (*ops)->sym->m_src,
1866                                                 &eqdesc[loop]);
1867                         else
1868                                 qbman_eq_desc_set_no_orp(&eqdesc[loop],
1869                                                          DPAA2_EQ_RESP_ERR_FQ);
1870
1871                         /*Clear the unused FD fields before sending*/
1872                         memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1873                         mb_pool = (*ops)->sym->m_src->pool;
1874                         bpid = mempool_to_bpid(mb_pool);
1875                         ret = build_sec_fd(*ops, &fd_arr[loop], bpid, dpaa2_qp);
1876                         if (ret) {
1877                                 DPAA2_SEC_DP_DEBUG("FD build failed\n");
1878                                 goto skip_tx;
1879                         }
1880                         ops++;
1881                 }
1882
1883                 loop = 0;
1884                 retry_count = 0;
1885                 while (loop < frames_to_send) {
1886                         ret = qbman_swp_enqueue_multiple_desc(swp,
1887                                         &eqdesc[loop], &fd_arr[loop],
1888                                         frames_to_send - loop);
1889                         if (unlikely(ret < 0)) {
1890                                 retry_count++;
1891                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1892                                         num_tx += loop;
1893                                         nb_ops -= loop;
1894                                         DPAA2_SEC_DP_DEBUG("Enqueue fail\n");
1895                                         /* freeing the fle buffers */
1896                                         while (loop < frames_to_send) {
1897                                                 free_fle(&fd_arr[loop],
1898                                                                 dpaa2_qp);
1899                                                 loop++;
1900                                         }
1901                                         goto skip_tx;
1902                                 }
1903                         } else {
1904                                 loop += ret;
1905                                 retry_count = 0;
1906                         }
1907                 }
1908
1909                 num_tx += loop;
1910                 nb_ops -= loop;
1911         }
1912
1913 skip_tx:
1914         dpaa2_qp->tx_vq.tx_pkts += num_tx;
1915         dpaa2_qp->tx_vq.err_pkts += nb_ops;
1916         return num_tx;
1917 }
1918
1919 static uint16_t
1920 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1921                         uint16_t nb_ops)
1922 {
1923         /* Function is responsible to receive frames for a given device and VQ*/
1924         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1925         struct qbman_result *dq_storage;
1926         uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1927         int ret, num_rx = 0;
1928         uint8_t is_last = 0, status;
1929         struct qbman_swp *swp;
1930         const struct qbman_fd *fd;
1931         struct qbman_pull_desc pulldesc;
1932
1933         if (!DPAA2_PER_LCORE_DPIO) {
1934                 ret = dpaa2_affine_qbman_swp();
1935                 if (ret) {
1936                         DPAA2_SEC_ERR(
1937                                 "Failed to allocate IO portal, tid: %d\n",
1938                                 rte_gettid());
1939                         return 0;
1940                 }
1941         }
1942         swp = DPAA2_PER_LCORE_PORTAL;
1943         dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1944
1945         qbman_pull_desc_clear(&pulldesc);
1946         qbman_pull_desc_set_numframes(&pulldesc,
1947                                       (nb_ops > dpaa2_dqrr_size) ?
1948                                       dpaa2_dqrr_size : nb_ops);
1949         qbman_pull_desc_set_fq(&pulldesc, fqid);
1950         qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1951                                     (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1952                                     1);
1953
1954         /*Issue a volatile dequeue command. */
1955         while (1) {
1956                 if (qbman_swp_pull(swp, &pulldesc)) {
1957                         DPAA2_SEC_WARN(
1958                                 "SEC VDQ command is not issued : QBMAN busy");
1959                         /* Portal was busy, try again */
1960                         continue;
1961                 }
1962                 break;
1963         };
1964
1965         /* Receive the packets till Last Dequeue entry is found with
1966          * respect to the above issues PULL command.
1967          */
1968         while (!is_last) {
1969                 /* Check if the previous issued command is completed.
1970                  * Also seems like the SWP is shared between the Ethernet Driver
1971                  * and the SEC driver.
1972                  */
1973                 while (!qbman_check_command_complete(dq_storage))
1974                         ;
1975
1976                 /* Loop until the dq_storage is updated with
1977                  * new token by QBMAN
1978                  */
1979                 while (!qbman_check_new_result(dq_storage))
1980                         ;
1981                 /* Check whether Last Pull command is Expired and
1982                  * setting Condition for Loop termination
1983                  */
1984                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1985                         is_last = 1;
1986                         /* Check for valid frame. */
1987                         status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1988                         if (unlikely(
1989                                 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1990                                 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1991                                 continue;
1992                         }
1993                 }
1994
1995                 fd = qbman_result_DQ_fd(dq_storage);
1996                 ops[num_rx] = sec_fd_to_mbuf(fd, dpaa2_qp);
1997
1998                 if (unlikely(fd->simple.frc)) {
1999                         /* TODO Parse SEC errors */
2000                         if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_NO_DUMP) {
2001                                 DPAA2_SEC_DP_ERR("SEC returned Error - %x\n",
2002                                                  fd->simple.frc);
2003                                 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_ERR_DUMP)
2004                                         dpaa2_sec_dump(ops[num_rx]);
2005                         }
2006
2007                         dpaa2_qp->rx_vq.err_pkts += 1;
2008                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
2009                 } else {
2010                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
2011                 }
2012
2013                 num_rx++;
2014                 dq_storage++;
2015         } /* End of Packet Rx loop */
2016
2017         dpaa2_qp->rx_vq.rx_pkts += num_rx;
2018
2019         DPAA2_SEC_DP_DEBUG("SEC RX pkts %d err pkts %" PRIu64 "\n", num_rx,
2020                                 dpaa2_qp->rx_vq.err_pkts);
2021         /*Return the total number of packets received to DPAA2 app*/
2022         return num_rx;
2023 }
2024
2025 /** Release queue pair */
2026 static int
2027 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
2028 {
2029         struct dpaa2_sec_qp *qp =
2030                 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
2031
2032         PMD_INIT_FUNC_TRACE();
2033
2034         if (qp->rx_vq.q_storage) {
2035                 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
2036                 rte_free(qp->rx_vq.q_storage);
2037         }
2038         rte_mempool_free(qp->fle_pool);
2039         rte_free(qp);
2040
2041         dev->data->queue_pairs[queue_pair_id] = NULL;
2042
2043         return 0;
2044 }
2045
2046 /** Setup a queue pair */
2047 static int
2048 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
2049                 const struct rte_cryptodev_qp_conf *qp_conf,
2050                 __rte_unused int socket_id)
2051 {
2052         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
2053         struct dpaa2_sec_qp *qp;
2054         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
2055         struct dpseci_rx_queue_cfg cfg;
2056         int32_t retcode;
2057         char str[30];
2058
2059         PMD_INIT_FUNC_TRACE();
2060
2061         /* If qp is already in use free ring memory and qp metadata. */
2062         if (dev->data->queue_pairs[qp_id] != NULL) {
2063                 DPAA2_SEC_INFO("QP already setup");
2064                 return 0;
2065         }
2066
2067         DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
2068                     dev, qp_id, qp_conf);
2069
2070         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
2071
2072         qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
2073                         RTE_CACHE_LINE_SIZE);
2074         if (!qp) {
2075                 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
2076                 return -ENOMEM;
2077         }
2078
2079         qp->rx_vq.crypto_data = dev->data;
2080         qp->tx_vq.crypto_data = dev->data;
2081         qp->rx_vq.q_storage = rte_malloc("sec dq storage",
2082                 sizeof(struct queue_storage_info_t),
2083                 RTE_CACHE_LINE_SIZE);
2084         if (!qp->rx_vq.q_storage) {
2085                 DPAA2_SEC_ERR("malloc failed for q_storage");
2086                 return -ENOMEM;
2087         }
2088         memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
2089
2090         if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
2091                 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
2092                 return -ENOMEM;
2093         }
2094
2095         dev->data->queue_pairs[qp_id] = qp;
2096
2097         snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d_%d",
2098                         getpid(), dev->data->dev_id, qp_id);
2099         qp->fle_pool = rte_mempool_create((const char *)str,
2100                         qp_conf->nb_descriptors,
2101                         FLE_POOL_BUF_SIZE,
2102                         FLE_POOL_CACHE_SIZE, 0,
2103                         NULL, NULL, NULL, NULL,
2104                         SOCKET_ID_ANY, MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET);
2105         if (!qp->fle_pool) {
2106                 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
2107                 return -ENOMEM;
2108         }
2109
2110         cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
2111         cfg.user_ctx = (size_t)(&qp->rx_vq);
2112         retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
2113                                       qp_id, &cfg);
2114         return retcode;
2115 }
2116
2117 /** Returns the size of the aesni gcm session structure */
2118 static unsigned int
2119 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
2120 {
2121         PMD_INIT_FUNC_TRACE();
2122
2123         return sizeof(dpaa2_sec_session);
2124 }
2125
2126 static int
2127 dpaa2_sec_cipher_init(struct rte_crypto_sym_xform *xform,
2128                       dpaa2_sec_session *session)
2129 {
2130         struct alginfo cipherdata;
2131         int bufsize, ret = 0;
2132         struct ctxt_priv *priv;
2133         struct sec_flow_context *flc;
2134
2135         PMD_INIT_FUNC_TRACE();
2136
2137         /* For SEC CIPHER only one descriptor is required. */
2138         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2139                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2140                         RTE_CACHE_LINE_SIZE);
2141         if (priv == NULL) {
2142                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2143                 return -ENOMEM;
2144         }
2145
2146         flc = &priv->flc_desc[0].flc;
2147
2148         session->ctxt_type = DPAA2_SEC_CIPHER;
2149         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
2150                         RTE_CACHE_LINE_SIZE);
2151         if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
2152                 DPAA2_SEC_ERR("No Memory for cipher key");
2153                 rte_free(priv);
2154                 return -ENOMEM;
2155         }
2156         session->cipher_key.length = xform->cipher.key.length;
2157
2158         memcpy(session->cipher_key.data, xform->cipher.key.data,
2159                xform->cipher.key.length);
2160         cipherdata.key = (size_t)session->cipher_key.data;
2161         cipherdata.keylen = session->cipher_key.length;
2162         cipherdata.key_enc_flags = 0;
2163         cipherdata.key_type = RTA_DATA_IMM;
2164
2165         /* Set IV parameters */
2166         session->iv.offset = xform->cipher.iv.offset;
2167         session->iv.length = xform->cipher.iv.length;
2168         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2169                                 DIR_ENC : DIR_DEC;
2170
2171         switch (xform->cipher.algo) {
2172         case RTE_CRYPTO_CIPHER_AES_CBC:
2173                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2174                 cipherdata.algmode = OP_ALG_AAI_CBC;
2175                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2176                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
2177                                                 SHR_NEVER, &cipherdata,
2178                                                 session->iv.length,
2179                                                 session->dir);
2180                 break;
2181         case RTE_CRYPTO_CIPHER_3DES_CBC:
2182                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2183                 cipherdata.algmode = OP_ALG_AAI_CBC;
2184                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2185                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
2186                                                 SHR_NEVER, &cipherdata,
2187                                                 session->iv.length,
2188                                                 session->dir);
2189                 break;
2190         case RTE_CRYPTO_CIPHER_DES_CBC:
2191                 cipherdata.algtype = OP_ALG_ALGSEL_DES;
2192                 cipherdata.algmode = OP_ALG_AAI_CBC;
2193                 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
2194                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
2195                                                 SHR_NEVER, &cipherdata,
2196                                                 session->iv.length,
2197                                                 session->dir);
2198                 break;
2199         case RTE_CRYPTO_CIPHER_AES_CTR:
2200                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2201                 cipherdata.algmode = OP_ALG_AAI_CTR;
2202                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2203                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
2204                                                 SHR_NEVER, &cipherdata,
2205                                                 session->iv.length,
2206                                                 session->dir);
2207                 break;
2208         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2209                 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
2210                 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
2211                 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
2212                                               &cipherdata,
2213                                               session->dir);
2214                 break;
2215         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2216                 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
2217                 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
2218                 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
2219                                               &cipherdata,
2220                                               session->dir);
2221                 break;
2222         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2223         case RTE_CRYPTO_CIPHER_AES_F8:
2224         case RTE_CRYPTO_CIPHER_AES_ECB:
2225         case RTE_CRYPTO_CIPHER_3DES_ECB:
2226         case RTE_CRYPTO_CIPHER_3DES_CTR:
2227         case RTE_CRYPTO_CIPHER_AES_XTS:
2228         case RTE_CRYPTO_CIPHER_ARC4:
2229         case RTE_CRYPTO_CIPHER_NULL:
2230                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2231                         xform->cipher.algo);
2232                 ret = -ENOTSUP;
2233                 goto error_out;
2234         default:
2235                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2236                         xform->cipher.algo);
2237                 ret = -ENOTSUP;
2238                 goto error_out;
2239         }
2240
2241         if (bufsize < 0) {
2242                 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
2243                 ret = -EINVAL;
2244                 goto error_out;
2245         }
2246
2247         flc->word1_sdl = (uint8_t)bufsize;
2248         session->ctxt = priv;
2249
2250 #ifdef CAAM_DESC_DEBUG
2251         int i;
2252         for (i = 0; i < bufsize; i++)
2253                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
2254 #endif
2255         return ret;
2256
2257 error_out:
2258         rte_free(session->cipher_key.data);
2259         rte_free(priv);
2260         return ret;
2261 }
2262
2263 static int
2264 dpaa2_sec_auth_init(struct rte_crypto_sym_xform *xform,
2265                     dpaa2_sec_session *session)
2266 {
2267         struct alginfo authdata;
2268         int bufsize, ret = 0;
2269         struct ctxt_priv *priv;
2270         struct sec_flow_context *flc;
2271
2272         PMD_INIT_FUNC_TRACE();
2273
2274         /* For SEC AUTH three descriptors are required for various stages */
2275         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2276                         sizeof(struct ctxt_priv) + 3 *
2277                         sizeof(struct sec_flc_desc),
2278                         RTE_CACHE_LINE_SIZE);
2279         if (priv == NULL) {
2280                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2281                 return -ENOMEM;
2282         }
2283
2284         flc = &priv->flc_desc[DESC_INITFINAL].flc;
2285
2286         session->ctxt_type = DPAA2_SEC_AUTH;
2287         session->auth_key.length = xform->auth.key.length;
2288         if (xform->auth.key.length) {
2289                 session->auth_key.data = rte_zmalloc(NULL,
2290                         xform->auth.key.length,
2291                         RTE_CACHE_LINE_SIZE);
2292                 if (session->auth_key.data == NULL) {
2293                         DPAA2_SEC_ERR("Unable to allocate memory for auth key");
2294                         rte_free(priv);
2295                         return -ENOMEM;
2296                 }
2297                 memcpy(session->auth_key.data, xform->auth.key.data,
2298                        xform->auth.key.length);
2299                 authdata.key = (size_t)session->auth_key.data;
2300                 authdata.key_enc_flags = 0;
2301                 authdata.key_type = RTA_DATA_IMM;
2302         }
2303         authdata.keylen = session->auth_key.length;
2304
2305         session->digest_length = xform->auth.digest_length;
2306         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2307                                 DIR_ENC : DIR_DEC;
2308
2309         switch (xform->auth.algo) {
2310         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2311                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2312                 authdata.algmode = OP_ALG_AAI_HMAC;
2313                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2314                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2315                                            1, 0, SHR_NEVER, &authdata,
2316                                            !session->dir,
2317                                            session->digest_length);
2318                 break;
2319         case RTE_CRYPTO_AUTH_MD5_HMAC:
2320                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2321                 authdata.algmode = OP_ALG_AAI_HMAC;
2322                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2323                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2324                                            1, 0, SHR_NEVER, &authdata,
2325                                            !session->dir,
2326                                            session->digest_length);
2327                 break;
2328         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2329                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2330                 authdata.algmode = OP_ALG_AAI_HMAC;
2331                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2332                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2333                                            1, 0, SHR_NEVER, &authdata,
2334                                            !session->dir,
2335                                            session->digest_length);
2336                 break;
2337         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2338                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2339                 authdata.algmode = OP_ALG_AAI_HMAC;
2340                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2341                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2342                                            1, 0, SHR_NEVER, &authdata,
2343                                            !session->dir,
2344                                            session->digest_length);
2345                 break;
2346         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2347                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2348                 authdata.algmode = OP_ALG_AAI_HMAC;
2349                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2350                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2351                                            1, 0, SHR_NEVER, &authdata,
2352                                            !session->dir,
2353                                            session->digest_length);
2354                 break;
2355         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2356                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2357                 authdata.algmode = OP_ALG_AAI_HMAC;
2358                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2359                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2360                                            1, 0, SHR_NEVER, &authdata,
2361                                            !session->dir,
2362                                            session->digest_length);
2363                 break;
2364         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2365                 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2366                 authdata.algmode = OP_ALG_AAI_F9;
2367                 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2368                 session->iv.offset = xform->auth.iv.offset;
2369                 session->iv.length = xform->auth.iv.length;
2370                 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2371                                               1, 0, &authdata,
2372                                               !session->dir,
2373                                               session->digest_length);
2374                 break;
2375         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2376                 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2377                 authdata.algmode = OP_ALG_AAI_F9;
2378                 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2379                 session->iv.offset = xform->auth.iv.offset;
2380                 session->iv.length = xform->auth.iv.length;
2381                 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2382                                            1, 0, &authdata,
2383                                            !session->dir,
2384                                            session->digest_length);
2385                 break;
2386         case RTE_CRYPTO_AUTH_SHA1:
2387                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2388                 authdata.algmode = OP_ALG_AAI_HASH;
2389                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1;
2390                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2391                                            1, 0, SHR_NEVER, &authdata,
2392                                            !session->dir,
2393                                            session->digest_length);
2394                 break;
2395         case RTE_CRYPTO_AUTH_MD5:
2396                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2397                 authdata.algmode = OP_ALG_AAI_HASH;
2398                 session->auth_alg = RTE_CRYPTO_AUTH_MD5;
2399                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2400                                            1, 0, SHR_NEVER, &authdata,
2401                                            !session->dir,
2402                                            session->digest_length);
2403                 break;
2404         case RTE_CRYPTO_AUTH_SHA256:
2405                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2406                 authdata.algmode = OP_ALG_AAI_HASH;
2407                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256;
2408                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2409                                            1, 0, SHR_NEVER, &authdata,
2410                                            !session->dir,
2411                                            session->digest_length);
2412                 break;
2413         case RTE_CRYPTO_AUTH_SHA384:
2414                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2415                 authdata.algmode = OP_ALG_AAI_HASH;
2416                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384;
2417                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2418                                            1, 0, SHR_NEVER, &authdata,
2419                                            !session->dir,
2420                                            session->digest_length);
2421                 break;
2422         case RTE_CRYPTO_AUTH_SHA512:
2423                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2424                 authdata.algmode = OP_ALG_AAI_HASH;
2425                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512;
2426                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2427                                            1, 0, SHR_NEVER, &authdata,
2428                                            !session->dir,
2429                                            session->digest_length);
2430                 break;
2431         case RTE_CRYPTO_AUTH_SHA224:
2432                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2433                 authdata.algmode = OP_ALG_AAI_HASH;
2434                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224;
2435                 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2436                                            1, 0, SHR_NEVER, &authdata,
2437                                            !session->dir,
2438                                            session->digest_length);
2439                 break;
2440         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2441                 authdata.algtype = OP_ALG_ALGSEL_AES;
2442                 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2443                 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2444                 bufsize = cnstr_shdsc_aes_mac(
2445                                         priv->flc_desc[DESC_INITFINAL].desc,
2446                                         1, 0, SHR_NEVER, &authdata,
2447                                         !session->dir,
2448                                         session->digest_length);
2449                 break;
2450         case RTE_CRYPTO_AUTH_AES_CMAC:
2451                 authdata.algtype = OP_ALG_ALGSEL_AES;
2452                 authdata.algmode = OP_ALG_AAI_CMAC;
2453                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2454                 bufsize = cnstr_shdsc_aes_mac(
2455                                            priv->flc_desc[DESC_INITFINAL].desc,
2456                                            1, 0, SHR_NEVER, &authdata,
2457                                            !session->dir,
2458                                            session->digest_length);
2459                 break;
2460         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2461         case RTE_CRYPTO_AUTH_AES_GMAC:
2462         case RTE_CRYPTO_AUTH_KASUMI_F9:
2463         case RTE_CRYPTO_AUTH_NULL:
2464                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2465                               xform->auth.algo);
2466                 ret = -ENOTSUP;
2467                 goto error_out;
2468         default:
2469                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2470                               xform->auth.algo);
2471                 ret = -ENOTSUP;
2472                 goto error_out;
2473         }
2474
2475         if (bufsize < 0) {
2476                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2477                 ret = -EINVAL;
2478                 goto error_out;
2479         }
2480
2481         flc->word1_sdl = (uint8_t)bufsize;
2482         session->ctxt = priv;
2483 #ifdef CAAM_DESC_DEBUG
2484         int i;
2485         for (i = 0; i < bufsize; i++)
2486                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2487                                 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2488 #endif
2489
2490         return ret;
2491
2492 error_out:
2493         rte_free(session->auth_key.data);
2494         rte_free(priv);
2495         return ret;
2496 }
2497
2498 static int
2499 dpaa2_sec_aead_init(struct rte_crypto_sym_xform *xform,
2500                     dpaa2_sec_session *session)
2501 {
2502         struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2503         struct alginfo aeaddata;
2504         int bufsize;
2505         struct ctxt_priv *priv;
2506         struct sec_flow_context *flc;
2507         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2508         int err, ret = 0;
2509
2510         PMD_INIT_FUNC_TRACE();
2511
2512         /* Set IV parameters */
2513         session->iv.offset = aead_xform->iv.offset;
2514         session->iv.length = aead_xform->iv.length;
2515         session->ctxt_type = DPAA2_SEC_AEAD;
2516
2517         /* For SEC AEAD only one descriptor is required */
2518         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2519                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2520                         RTE_CACHE_LINE_SIZE);
2521         if (priv == NULL) {
2522                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2523                 return -ENOMEM;
2524         }
2525
2526         flc = &priv->flc_desc[0].flc;
2527
2528         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2529                                                RTE_CACHE_LINE_SIZE);
2530         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2531                 DPAA2_SEC_ERR("No Memory for aead key");
2532                 rte_free(priv);
2533                 return -ENOMEM;
2534         }
2535         memcpy(session->aead_key.data, aead_xform->key.data,
2536                aead_xform->key.length);
2537
2538         session->digest_length = aead_xform->digest_length;
2539         session->aead_key.length = aead_xform->key.length;
2540         ctxt->auth_only_len = aead_xform->aad_length;
2541
2542         aeaddata.key = (size_t)session->aead_key.data;
2543         aeaddata.keylen = session->aead_key.length;
2544         aeaddata.key_enc_flags = 0;
2545         aeaddata.key_type = RTA_DATA_IMM;
2546
2547         switch (aead_xform->algo) {
2548         case RTE_CRYPTO_AEAD_AES_GCM:
2549                 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2550                 aeaddata.algmode = OP_ALG_AAI_GCM;
2551                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2552                 break;
2553         case RTE_CRYPTO_AEAD_AES_CCM:
2554                 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2555                               aead_xform->algo);
2556                 ret = -ENOTSUP;
2557                 goto error_out;
2558         default:
2559                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2560                               aead_xform->algo);
2561                 ret = -ENOTSUP;
2562                 goto error_out;
2563         }
2564         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2565                                 DIR_ENC : DIR_DEC;
2566
2567         priv->flc_desc[0].desc[0] = aeaddata.keylen;
2568         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2569                                DESC_JOB_IO_LEN,
2570                                (unsigned int *)priv->flc_desc[0].desc,
2571                                &priv->flc_desc[0].desc[1], 1);
2572
2573         if (err < 0) {
2574                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2575                 ret = -EINVAL;
2576                 goto error_out;
2577         }
2578         if (priv->flc_desc[0].desc[1] & 1) {
2579                 aeaddata.key_type = RTA_DATA_IMM;
2580         } else {
2581                 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2582                 aeaddata.key_type = RTA_DATA_PTR;
2583         }
2584         priv->flc_desc[0].desc[0] = 0;
2585         priv->flc_desc[0].desc[1] = 0;
2586
2587         if (session->dir == DIR_ENC)
2588                 bufsize = cnstr_shdsc_gcm_encap(
2589                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2590                                 &aeaddata, session->iv.length,
2591                                 session->digest_length);
2592         else
2593                 bufsize = cnstr_shdsc_gcm_decap(
2594                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2595                                 &aeaddata, session->iv.length,
2596                                 session->digest_length);
2597         if (bufsize < 0) {
2598                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2599                 ret = -EINVAL;
2600                 goto error_out;
2601         }
2602
2603         flc->word1_sdl = (uint8_t)bufsize;
2604         session->ctxt = priv;
2605 #ifdef CAAM_DESC_DEBUG
2606         int i;
2607         for (i = 0; i < bufsize; i++)
2608                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2609                             i, priv->flc_desc[0].desc[i]);
2610 #endif
2611         return ret;
2612
2613 error_out:
2614         rte_free(session->aead_key.data);
2615         rte_free(priv);
2616         return ret;
2617 }
2618
2619
2620 static int
2621 dpaa2_sec_aead_chain_init(struct rte_crypto_sym_xform *xform,
2622                     dpaa2_sec_session *session)
2623 {
2624         struct alginfo authdata, cipherdata;
2625         int bufsize;
2626         struct ctxt_priv *priv;
2627         struct sec_flow_context *flc;
2628         struct rte_crypto_cipher_xform *cipher_xform;
2629         struct rte_crypto_auth_xform *auth_xform;
2630         int err, ret = 0;
2631
2632         PMD_INIT_FUNC_TRACE();
2633
2634         if (session->ext_params.aead_ctxt.auth_cipher_text) {
2635                 cipher_xform = &xform->cipher;
2636                 auth_xform = &xform->next->auth;
2637                 session->ctxt_type =
2638                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2639                         DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2640         } else {
2641                 cipher_xform = &xform->next->cipher;
2642                 auth_xform = &xform->auth;
2643                 session->ctxt_type =
2644                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2645                         DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2646         }
2647
2648         /* Set IV parameters */
2649         session->iv.offset = cipher_xform->iv.offset;
2650         session->iv.length = cipher_xform->iv.length;
2651
2652         /* For SEC AEAD only one descriptor is required */
2653         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2654                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2655                         RTE_CACHE_LINE_SIZE);
2656         if (priv == NULL) {
2657                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2658                 return -ENOMEM;
2659         }
2660
2661         flc = &priv->flc_desc[0].flc;
2662
2663         session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2664                                                RTE_CACHE_LINE_SIZE);
2665         if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2666                 DPAA2_SEC_ERR("No Memory for cipher key");
2667                 rte_free(priv);
2668                 return -ENOMEM;
2669         }
2670         session->cipher_key.length = cipher_xform->key.length;
2671         session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2672                                              RTE_CACHE_LINE_SIZE);
2673         if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2674                 DPAA2_SEC_ERR("No Memory for auth key");
2675                 rte_free(session->cipher_key.data);
2676                 rte_free(priv);
2677                 return -ENOMEM;
2678         }
2679         session->auth_key.length = auth_xform->key.length;
2680         memcpy(session->cipher_key.data, cipher_xform->key.data,
2681                cipher_xform->key.length);
2682         memcpy(session->auth_key.data, auth_xform->key.data,
2683                auth_xform->key.length);
2684
2685         authdata.key = (size_t)session->auth_key.data;
2686         authdata.keylen = session->auth_key.length;
2687         authdata.key_enc_flags = 0;
2688         authdata.key_type = RTA_DATA_IMM;
2689
2690         session->digest_length = auth_xform->digest_length;
2691
2692         switch (auth_xform->algo) {
2693         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2694                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2695                 authdata.algmode = OP_ALG_AAI_HMAC;
2696                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2697                 break;
2698         case RTE_CRYPTO_AUTH_MD5_HMAC:
2699                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2700                 authdata.algmode = OP_ALG_AAI_HMAC;
2701                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2702                 break;
2703         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2704                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2705                 authdata.algmode = OP_ALG_AAI_HMAC;
2706                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2707                 break;
2708         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2709                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2710                 authdata.algmode = OP_ALG_AAI_HMAC;
2711                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2712                 break;
2713         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2714                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2715                 authdata.algmode = OP_ALG_AAI_HMAC;
2716                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2717                 break;
2718         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2719                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2720                 authdata.algmode = OP_ALG_AAI_HMAC;
2721                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2722                 break;
2723         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2724                 authdata.algtype = OP_ALG_ALGSEL_AES;
2725                 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2726                 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2727                 break;
2728         case RTE_CRYPTO_AUTH_AES_CMAC:
2729                 authdata.algtype = OP_ALG_ALGSEL_AES;
2730                 authdata.algmode = OP_ALG_AAI_CMAC;
2731                 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2732                 break;
2733         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2734         case RTE_CRYPTO_AUTH_AES_GMAC:
2735         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2736         case RTE_CRYPTO_AUTH_NULL:
2737         case RTE_CRYPTO_AUTH_SHA1:
2738         case RTE_CRYPTO_AUTH_SHA256:
2739         case RTE_CRYPTO_AUTH_SHA512:
2740         case RTE_CRYPTO_AUTH_SHA224:
2741         case RTE_CRYPTO_AUTH_SHA384:
2742         case RTE_CRYPTO_AUTH_MD5:
2743         case RTE_CRYPTO_AUTH_KASUMI_F9:
2744         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2745                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2746                               auth_xform->algo);
2747                 ret = -ENOTSUP;
2748                 goto error_out;
2749         default:
2750                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2751                               auth_xform->algo);
2752                 ret = -ENOTSUP;
2753                 goto error_out;
2754         }
2755         cipherdata.key = (size_t)session->cipher_key.data;
2756         cipherdata.keylen = session->cipher_key.length;
2757         cipherdata.key_enc_flags = 0;
2758         cipherdata.key_type = RTA_DATA_IMM;
2759
2760         switch (cipher_xform->algo) {
2761         case RTE_CRYPTO_CIPHER_AES_CBC:
2762                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2763                 cipherdata.algmode = OP_ALG_AAI_CBC;
2764                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2765                 break;
2766         case RTE_CRYPTO_CIPHER_3DES_CBC:
2767                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2768                 cipherdata.algmode = OP_ALG_AAI_CBC;
2769                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2770                 break;
2771         case RTE_CRYPTO_CIPHER_DES_CBC:
2772                 cipherdata.algtype = OP_ALG_ALGSEL_DES;
2773                 cipherdata.algmode = OP_ALG_AAI_CBC;
2774                 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
2775                 break;
2776         case RTE_CRYPTO_CIPHER_AES_CTR:
2777                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2778                 cipherdata.algmode = OP_ALG_AAI_CTR;
2779                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2780                 break;
2781         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2782         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2783         case RTE_CRYPTO_CIPHER_NULL:
2784         case RTE_CRYPTO_CIPHER_3DES_ECB:
2785         case RTE_CRYPTO_CIPHER_3DES_CTR:
2786         case RTE_CRYPTO_CIPHER_AES_ECB:
2787         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2788                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2789                               cipher_xform->algo);
2790                 ret = -ENOTSUP;
2791                 goto error_out;
2792         default:
2793                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2794                               cipher_xform->algo);
2795                 ret = -ENOTSUP;
2796                 goto error_out;
2797         }
2798         session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2799                                 DIR_ENC : DIR_DEC;
2800
2801         priv->flc_desc[0].desc[0] = cipherdata.keylen;
2802         priv->flc_desc[0].desc[1] = authdata.keylen;
2803         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2804                                DESC_JOB_IO_LEN,
2805                                (unsigned int *)priv->flc_desc[0].desc,
2806                                &priv->flc_desc[0].desc[2], 2);
2807
2808         if (err < 0) {
2809                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2810                 ret = -EINVAL;
2811                 goto error_out;
2812         }
2813         if (priv->flc_desc[0].desc[2] & 1) {
2814                 cipherdata.key_type = RTA_DATA_IMM;
2815         } else {
2816                 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2817                 cipherdata.key_type = RTA_DATA_PTR;
2818         }
2819         if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2820                 authdata.key_type = RTA_DATA_IMM;
2821         } else {
2822                 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2823                 authdata.key_type = RTA_DATA_PTR;
2824         }
2825         priv->flc_desc[0].desc[0] = 0;
2826         priv->flc_desc[0].desc[1] = 0;
2827         priv->flc_desc[0].desc[2] = 0;
2828
2829         if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2830                 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2831                                               0, SHR_SERIAL,
2832                                               &cipherdata, &authdata,
2833                                               session->iv.length,
2834                                               session->digest_length,
2835                                               session->dir);
2836                 if (bufsize < 0) {
2837                         DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2838                         ret = -EINVAL;
2839                         goto error_out;
2840                 }
2841         } else {
2842                 DPAA2_SEC_ERR("Hash before cipher not supported");
2843                 ret = -ENOTSUP;
2844                 goto error_out;
2845         }
2846
2847         flc->word1_sdl = (uint8_t)bufsize;
2848         session->ctxt = priv;
2849 #ifdef CAAM_DESC_DEBUG
2850         int i;
2851         for (i = 0; i < bufsize; i++)
2852                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2853                             i, priv->flc_desc[0].desc[i]);
2854 #endif
2855
2856         return ret;
2857
2858 error_out:
2859         rte_free(session->cipher_key.data);
2860         rte_free(session->auth_key.data);
2861         rte_free(priv);
2862         return ret;
2863 }
2864
2865 static int
2866 dpaa2_sec_set_session_parameters(struct rte_crypto_sym_xform *xform, void *sess)
2867 {
2868         dpaa2_sec_session *session = sess;
2869         int ret;
2870
2871         PMD_INIT_FUNC_TRACE();
2872
2873         if (unlikely(sess == NULL)) {
2874                 DPAA2_SEC_ERR("Invalid session struct");
2875                 return -EINVAL;
2876         }
2877
2878         memset(session, 0, sizeof(dpaa2_sec_session));
2879         /* Default IV length = 0 */
2880         session->iv.length = 0;
2881
2882         /* Cipher Only */
2883         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2884                 ret = dpaa2_sec_cipher_init(xform, session);
2885
2886         /* Authentication Only */
2887         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2888                    xform->next == NULL) {
2889                 ret = dpaa2_sec_auth_init(xform, session);
2890
2891         /* Cipher then Authenticate */
2892         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2893                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2894                 session->ext_params.aead_ctxt.auth_cipher_text = true;
2895                 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2896                         ret = dpaa2_sec_auth_init(xform, session);
2897                 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2898                         ret = dpaa2_sec_cipher_init(xform, session);
2899                 else
2900                         ret = dpaa2_sec_aead_chain_init(xform, session);
2901         /* Authenticate then Cipher */
2902         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2903                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2904                 session->ext_params.aead_ctxt.auth_cipher_text = false;
2905                 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2906                         ret = dpaa2_sec_cipher_init(xform, session);
2907                 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2908                         ret = dpaa2_sec_auth_init(xform, session);
2909                 else
2910                         ret = dpaa2_sec_aead_chain_init(xform, session);
2911         /* AEAD operation for AES-GCM kind of Algorithms */
2912         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2913                    xform->next == NULL) {
2914                 ret = dpaa2_sec_aead_init(xform, session);
2915
2916         } else {
2917                 DPAA2_SEC_ERR("Invalid crypto type");
2918                 return -EINVAL;
2919         }
2920
2921         return ret;
2922 }
2923
2924 #ifdef RTE_LIB_SECURITY
2925 static int
2926 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2927                         dpaa2_sec_session *session,
2928                         struct alginfo *aeaddata)
2929 {
2930         PMD_INIT_FUNC_TRACE();
2931
2932         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2933                                                RTE_CACHE_LINE_SIZE);
2934         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2935                 DPAA2_SEC_ERR("No Memory for aead key");
2936                 return -ENOMEM;
2937         }
2938         memcpy(session->aead_key.data, aead_xform->key.data,
2939                aead_xform->key.length);
2940
2941         session->digest_length = aead_xform->digest_length;
2942         session->aead_key.length = aead_xform->key.length;
2943
2944         aeaddata->key = (size_t)session->aead_key.data;
2945         aeaddata->keylen = session->aead_key.length;
2946         aeaddata->key_enc_flags = 0;
2947         aeaddata->key_type = RTA_DATA_IMM;
2948
2949         switch (aead_xform->algo) {
2950         case RTE_CRYPTO_AEAD_AES_GCM:
2951                 switch (session->digest_length) {
2952                 case 8:
2953                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2954                         break;
2955                 case 12:
2956                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2957                         break;
2958                 case 16:
2959                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2960                         break;
2961                 default:
2962                         DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2963                                       session->digest_length);
2964                         return -EINVAL;
2965                 }
2966                 aeaddata->algmode = OP_ALG_AAI_GCM;
2967                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2968                 break;
2969         case RTE_CRYPTO_AEAD_AES_CCM:
2970                 switch (session->digest_length) {
2971                 case 8:
2972                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2973                         break;
2974                 case 12:
2975                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2976                         break;
2977                 case 16:
2978                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2979                         break;
2980                 default:
2981                         DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2982                                       session->digest_length);
2983                         return -EINVAL;
2984                 }
2985                 aeaddata->algmode = OP_ALG_AAI_CCM;
2986                 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2987                 break;
2988         default:
2989                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2990                               aead_xform->algo);
2991                 return -ENOTSUP;
2992         }
2993         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2994                                 DIR_ENC : DIR_DEC;
2995
2996         return 0;
2997 }
2998
2999 static int
3000 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
3001         struct rte_crypto_auth_xform *auth_xform,
3002         dpaa2_sec_session *session,
3003         struct alginfo *cipherdata,
3004         struct alginfo *authdata)
3005 {
3006         if (cipher_xform) {
3007                 session->cipher_key.data = rte_zmalloc(NULL,
3008                                                        cipher_xform->key.length,
3009                                                        RTE_CACHE_LINE_SIZE);
3010                 if (session->cipher_key.data == NULL &&
3011                                 cipher_xform->key.length > 0) {
3012                         DPAA2_SEC_ERR("No Memory for cipher key");
3013                         return -ENOMEM;
3014                 }
3015
3016                 session->cipher_key.length = cipher_xform->key.length;
3017                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3018                                 cipher_xform->key.length);
3019                 session->cipher_alg = cipher_xform->algo;
3020         } else {
3021                 session->cipher_key.data = NULL;
3022                 session->cipher_key.length = 0;
3023                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3024         }
3025
3026         if (auth_xform) {
3027                 session->auth_key.data = rte_zmalloc(NULL,
3028                                                 auth_xform->key.length,
3029                                                 RTE_CACHE_LINE_SIZE);
3030                 if (session->auth_key.data == NULL &&
3031                                 auth_xform->key.length > 0) {
3032                         DPAA2_SEC_ERR("No Memory for auth key");
3033                         return -ENOMEM;
3034                 }
3035                 session->auth_key.length = auth_xform->key.length;
3036                 memcpy(session->auth_key.data, auth_xform->key.data,
3037                                 auth_xform->key.length);
3038                 session->auth_alg = auth_xform->algo;
3039                 session->digest_length = auth_xform->digest_length;
3040         } else {
3041                 session->auth_key.data = NULL;
3042                 session->auth_key.length = 0;
3043                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
3044         }
3045
3046         authdata->key = (size_t)session->auth_key.data;
3047         authdata->keylen = session->auth_key.length;
3048         authdata->key_enc_flags = 0;
3049         authdata->key_type = RTA_DATA_IMM;
3050         switch (session->auth_alg) {
3051         case RTE_CRYPTO_AUTH_SHA1_HMAC:
3052                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
3053                 authdata->algmode = OP_ALG_AAI_HMAC;
3054                 break;
3055         case RTE_CRYPTO_AUTH_MD5_HMAC:
3056                 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
3057                 authdata->algmode = OP_ALG_AAI_HMAC;
3058                 break;
3059         case RTE_CRYPTO_AUTH_SHA256_HMAC:
3060                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
3061                 authdata->algmode = OP_ALG_AAI_HMAC;
3062                 if (session->digest_length != 16)
3063                         DPAA2_SEC_WARN(
3064                         "+++Using sha256-hmac truncated len is non-standard,"
3065                         "it will not work with lookaside proto");
3066                 break;
3067         case RTE_CRYPTO_AUTH_SHA384_HMAC:
3068                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
3069                 authdata->algmode = OP_ALG_AAI_HMAC;
3070                 break;
3071         case RTE_CRYPTO_AUTH_SHA512_HMAC:
3072                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
3073                 authdata->algmode = OP_ALG_AAI_HMAC;
3074                 break;
3075         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
3076                 authdata->algtype = OP_PCL_IPSEC_AES_XCBC_MAC_96;
3077                 authdata->algmode = OP_ALG_AAI_XCBC_MAC;
3078                 break;
3079         case RTE_CRYPTO_AUTH_AES_CMAC:
3080                 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
3081                 authdata->algmode = OP_ALG_AAI_CMAC;
3082                 break;
3083         case RTE_CRYPTO_AUTH_NULL:
3084                 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
3085                 break;
3086         case RTE_CRYPTO_AUTH_SHA224_HMAC:
3087         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3088         case RTE_CRYPTO_AUTH_SHA1:
3089         case RTE_CRYPTO_AUTH_SHA256:
3090         case RTE_CRYPTO_AUTH_SHA512:
3091         case RTE_CRYPTO_AUTH_SHA224:
3092         case RTE_CRYPTO_AUTH_SHA384:
3093         case RTE_CRYPTO_AUTH_MD5:
3094         case RTE_CRYPTO_AUTH_AES_GMAC:
3095         case RTE_CRYPTO_AUTH_KASUMI_F9:
3096         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
3097         case RTE_CRYPTO_AUTH_ZUC_EIA3:
3098                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3099                               session->auth_alg);
3100                 return -ENOTSUP;
3101         default:
3102                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
3103                               session->auth_alg);
3104                 return -ENOTSUP;
3105         }
3106         cipherdata->key = (size_t)session->cipher_key.data;
3107         cipherdata->keylen = session->cipher_key.length;
3108         cipherdata->key_enc_flags = 0;
3109         cipherdata->key_type = RTA_DATA_IMM;
3110
3111         switch (session->cipher_alg) {
3112         case RTE_CRYPTO_CIPHER_AES_CBC:
3113                 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
3114                 cipherdata->algmode = OP_ALG_AAI_CBC;
3115                 break;
3116         case RTE_CRYPTO_CIPHER_3DES_CBC:
3117                 cipherdata->algtype = OP_PCL_IPSEC_3DES;
3118                 cipherdata->algmode = OP_ALG_AAI_CBC;
3119                 break;
3120         case RTE_CRYPTO_CIPHER_DES_CBC:
3121                 cipherdata->algtype = OP_PCL_IPSEC_DES;
3122                 cipherdata->algmode = OP_ALG_AAI_CBC;
3123                 break;
3124         case RTE_CRYPTO_CIPHER_AES_CTR:
3125                 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
3126                 cipherdata->algmode = OP_ALG_AAI_CTR;
3127                 break;
3128         case RTE_CRYPTO_CIPHER_NULL:
3129                 cipherdata->algtype = OP_PCL_IPSEC_NULL;
3130                 break;
3131         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3132         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3133         case RTE_CRYPTO_CIPHER_3DES_ECB:
3134         case RTE_CRYPTO_CIPHER_3DES_CTR:
3135         case RTE_CRYPTO_CIPHER_AES_ECB:
3136         case RTE_CRYPTO_CIPHER_KASUMI_F8:
3137                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
3138                               session->cipher_alg);
3139                 return -ENOTSUP;
3140         default:
3141                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3142                               session->cipher_alg);
3143                 return -ENOTSUP;
3144         }
3145
3146         return 0;
3147 }
3148
3149 static int
3150 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
3151                             struct rte_security_session_conf *conf,
3152                             void *sess)
3153 {
3154         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
3155         struct rte_crypto_cipher_xform *cipher_xform = NULL;
3156         struct rte_crypto_auth_xform *auth_xform = NULL;
3157         struct rte_crypto_aead_xform *aead_xform = NULL;
3158         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3159         struct ctxt_priv *priv;
3160         struct alginfo authdata, cipherdata;
3161         int bufsize;
3162         struct sec_flow_context *flc;
3163         int ret = -1;
3164
3165         PMD_INIT_FUNC_TRACE();
3166
3167         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3168                                 sizeof(struct ctxt_priv) +
3169                                 sizeof(struct sec_flc_desc),
3170                                 RTE_CACHE_LINE_SIZE);
3171
3172         if (priv == NULL) {
3173                 DPAA2_SEC_ERR("No memory for priv CTXT");
3174                 return -ENOMEM;
3175         }
3176
3177         flc = &priv->flc_desc[0].flc;
3178
3179         if (ipsec_xform->life.bytes_hard_limit != 0 ||
3180             ipsec_xform->life.bytes_soft_limit != 0 ||
3181             ipsec_xform->life.packets_hard_limit != 0 ||
3182             ipsec_xform->life.packets_soft_limit != 0)
3183                 return -ENOTSUP;
3184
3185         memset(session, 0, sizeof(dpaa2_sec_session));
3186
3187         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3188                 cipher_xform = &conf->crypto_xform->cipher;
3189                 if (conf->crypto_xform->next)
3190                         auth_xform = &conf->crypto_xform->next->auth;
3191                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
3192                                         session, &cipherdata, &authdata);
3193         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3194                 auth_xform = &conf->crypto_xform->auth;
3195                 if (conf->crypto_xform->next)
3196                         cipher_xform = &conf->crypto_xform->next->cipher;
3197                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
3198                                         session, &cipherdata, &authdata);
3199         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
3200                 aead_xform = &conf->crypto_xform->aead;
3201                 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
3202                                         session, &cipherdata);
3203                 authdata.keylen = 0;
3204                 authdata.algtype = 0;
3205         } else {
3206                 DPAA2_SEC_ERR("XFORM not specified");
3207                 ret = -EINVAL;
3208                 goto out;
3209         }
3210         if (ret) {
3211                 DPAA2_SEC_ERR("Failed to process xform");
3212                 goto out;
3213         }
3214
3215         session->ctxt_type = DPAA2_SEC_IPSEC;
3216         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
3217                 uint8_t *hdr = NULL;
3218                 struct ip ip4_hdr;
3219                 struct rte_ipv6_hdr ip6_hdr;
3220                 struct ipsec_encap_pdb encap_pdb;
3221
3222                 flc->dhr = SEC_FLC_DHR_OUTBOUND;
3223                 /* For Sec Proto only one descriptor is required. */
3224                 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
3225
3226                 /* copy algo specific data to PDB */
3227                 switch (cipherdata.algtype) {
3228                 case OP_PCL_IPSEC_AES_CTR:
3229                         encap_pdb.ctr.ctr_initial = 0x00000001;
3230                         encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3231                         break;
3232                 case OP_PCL_IPSEC_AES_GCM8:
3233                 case OP_PCL_IPSEC_AES_GCM12:
3234                 case OP_PCL_IPSEC_AES_GCM16:
3235                         memcpy(encap_pdb.gcm.salt,
3236                                 (uint8_t *)&(ipsec_xform->salt), 4);
3237                         break;
3238                 }
3239
3240                 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
3241                         PDBOPTS_ESP_OIHI_PDB_INL |
3242                         PDBOPTS_ESP_IVSRC |
3243                         PDBHMO_ESP_SNR;
3244                 if (ipsec_xform->options.dec_ttl)
3245                         encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL;
3246                 if (ipsec_xform->options.esn)
3247                         encap_pdb.options |= PDBOPTS_ESP_ESN;
3248                 encap_pdb.spi = ipsec_xform->spi;
3249                 session->dir = DIR_ENC;
3250                 if (ipsec_xform->tunnel.type ==
3251                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
3252                         encap_pdb.ip_hdr_len = sizeof(struct ip);
3253                         ip4_hdr.ip_v = IPVERSION;
3254                         ip4_hdr.ip_hl = 5;
3255                         ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
3256                         ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
3257                         ip4_hdr.ip_id = 0;
3258                         ip4_hdr.ip_off = 0;
3259                         ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
3260                         ip4_hdr.ip_p = IPPROTO_ESP;
3261                         ip4_hdr.ip_sum = 0;
3262                         ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
3263                         ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
3264                         ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
3265                                         &ip4_hdr, sizeof(struct ip));
3266                         hdr = (uint8_t *)&ip4_hdr;
3267                 } else if (ipsec_xform->tunnel.type ==
3268                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
3269                         ip6_hdr.vtc_flow = rte_cpu_to_be_32(
3270                                 DPAA2_IPv6_DEFAULT_VTC_FLOW |
3271                                 ((ipsec_xform->tunnel.ipv6.dscp <<
3272                                         RTE_IPV6_HDR_TC_SHIFT) &
3273                                         RTE_IPV6_HDR_TC_MASK) |
3274                                 ((ipsec_xform->tunnel.ipv6.flabel <<
3275                                         RTE_IPV6_HDR_FL_SHIFT) &
3276                                         RTE_IPV6_HDR_FL_MASK));
3277                         /* Payload length will be updated by HW */
3278                         ip6_hdr.payload_len = 0;
3279                         ip6_hdr.hop_limits =
3280                                         ipsec_xform->tunnel.ipv6.hlimit;
3281                         ip6_hdr.proto = (ipsec_xform->proto ==
3282                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
3283                                         IPPROTO_ESP : IPPROTO_AH;
3284                         memcpy(&ip6_hdr.src_addr,
3285                                 &ipsec_xform->tunnel.ipv6.src_addr, 16);
3286                         memcpy(&ip6_hdr.dst_addr,
3287                                 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
3288                         encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
3289                         hdr = (uint8_t *)&ip6_hdr;
3290                 }
3291
3292                 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
3293                                 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3294                                 SHR_WAIT : SHR_SERIAL, &encap_pdb,
3295                                 hdr, &cipherdata, &authdata);
3296         } else if (ipsec_xform->direction ==
3297                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
3298                 struct ipsec_decap_pdb decap_pdb;
3299
3300                 flc->dhr = SEC_FLC_DHR_INBOUND;
3301                 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
3302                 /* copy algo specific data to PDB */
3303                 switch (cipherdata.algtype) {
3304                 case OP_PCL_IPSEC_AES_CTR:
3305                         decap_pdb.ctr.ctr_initial = 0x00000001;
3306                         decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3307                         break;
3308                 case OP_PCL_IPSEC_AES_GCM8:
3309                 case OP_PCL_IPSEC_AES_GCM12:
3310                 case OP_PCL_IPSEC_AES_GCM16:
3311                         memcpy(decap_pdb.gcm.salt,
3312                                 (uint8_t *)&(ipsec_xform->salt), 4);
3313                         break;
3314                 }
3315
3316                 decap_pdb.options = (ipsec_xform->tunnel.type ==
3317                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
3318                                 sizeof(struct ip) << 16 :
3319                                 sizeof(struct rte_ipv6_hdr) << 16;
3320                 if (ipsec_xform->options.esn)
3321                         decap_pdb.options |= PDBOPTS_ESP_ESN;
3322
3323                 if (ipsec_xform->replay_win_sz) {
3324                         uint32_t win_sz;
3325                         win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
3326
3327                         if (rta_sec_era < RTA_SEC_ERA_10 && win_sz > 128) {
3328                                 DPAA2_SEC_INFO("Max Anti replay Win sz = 128");
3329                                 win_sz = 128;
3330                         }
3331                         switch (win_sz) {
3332                         case 1:
3333                         case 2:
3334                         case 4:
3335                         case 8:
3336                         case 16:
3337                         case 32:
3338                                 decap_pdb.options |= PDBOPTS_ESP_ARS32;
3339                                 break;
3340                         case 64:
3341                                 decap_pdb.options |= PDBOPTS_ESP_ARS64;
3342                                 break;
3343                         case 256:
3344                                 decap_pdb.options |= PDBOPTS_ESP_ARS256;
3345                                 break;
3346                         case 512:
3347                                 decap_pdb.options |= PDBOPTS_ESP_ARS512;
3348                                 break;
3349                         case 1024:
3350                                 decap_pdb.options |= PDBOPTS_ESP_ARS1024;
3351                                 break;
3352                         case 128:
3353                         default:
3354                                 decap_pdb.options |= PDBOPTS_ESP_ARS128;
3355                         }
3356                 }
3357                 session->dir = DIR_DEC;
3358                 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
3359                                 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3360                                 SHR_WAIT : SHR_SERIAL,
3361                                 &decap_pdb, &cipherdata, &authdata);
3362         } else
3363                 goto out;
3364
3365         if (bufsize < 0) {
3366                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3367                 goto out;
3368         }
3369
3370         flc->word1_sdl = (uint8_t)bufsize;
3371
3372         /* Enable the stashing control bit */
3373         DPAA2_SET_FLC_RSC(flc);
3374         flc->word2_rflc_31_0 = lower_32_bits(
3375                         (size_t)&(((struct dpaa2_sec_qp *)
3376                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
3377         flc->word3_rflc_63_32 = upper_32_bits(
3378                         (size_t)&(((struct dpaa2_sec_qp *)
3379                         dev->data->queue_pairs[0])->rx_vq));
3380
3381         /* Set EWS bit i.e. enable write-safe */
3382         DPAA2_SET_FLC_EWS(flc);
3383         /* Set BS = 1 i.e reuse input buffers as output buffers */
3384         DPAA2_SET_FLC_REUSE_BS(flc);
3385         /* Set FF = 10; reuse input buffers if they provide sufficient space */
3386         DPAA2_SET_FLC_REUSE_FF(flc);
3387
3388         session->ctxt = priv;
3389
3390         return 0;
3391 out:
3392         rte_free(session->auth_key.data);
3393         rte_free(session->cipher_key.data);
3394         rte_free(priv);
3395         return ret;
3396 }
3397
3398 static int
3399 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
3400                            struct rte_security_session_conf *conf,
3401                            void *sess)
3402 {
3403         struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3404         struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3405         struct rte_crypto_auth_xform *auth_xform = NULL;
3406         struct rte_crypto_cipher_xform *cipher_xform = NULL;
3407         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3408         struct ctxt_priv *priv;
3409         struct alginfo authdata, cipherdata;
3410         struct alginfo *p_authdata = NULL;
3411         int bufsize = -1;
3412         struct sec_flow_context *flc;
3413 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3414         int swap = true;
3415 #else
3416         int swap = false;
3417 #endif
3418
3419         PMD_INIT_FUNC_TRACE();
3420
3421         memset(session, 0, sizeof(dpaa2_sec_session));
3422
3423         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3424                                 sizeof(struct ctxt_priv) +
3425                                 sizeof(struct sec_flc_desc),
3426                                 RTE_CACHE_LINE_SIZE);
3427
3428         if (priv == NULL) {
3429                 DPAA2_SEC_ERR("No memory for priv CTXT");
3430                 return -ENOMEM;
3431         }
3432
3433         flc = &priv->flc_desc[0].flc;
3434
3435         /* find xfrm types */
3436         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3437                 cipher_xform = &xform->cipher;
3438                 if (xform->next != NULL &&
3439                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3440                         session->ext_params.aead_ctxt.auth_cipher_text = true;
3441                         auth_xform = &xform->next->auth;
3442                 }
3443         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3444                 auth_xform = &xform->auth;
3445                 if (xform->next != NULL &&
3446                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3447                         session->ext_params.aead_ctxt.auth_cipher_text = false;
3448                         cipher_xform = &xform->next->cipher;
3449                 }
3450         } else {
3451                 DPAA2_SEC_ERR("Invalid crypto type");
3452                 return -EINVAL;
3453         }
3454
3455         session->ctxt_type = DPAA2_SEC_PDCP;
3456         if (cipher_xform) {
3457                 session->cipher_key.data = rte_zmalloc(NULL,
3458                                                cipher_xform->key.length,
3459                                                RTE_CACHE_LINE_SIZE);
3460                 if (session->cipher_key.data == NULL &&
3461                                 cipher_xform->key.length > 0) {
3462                         DPAA2_SEC_ERR("No Memory for cipher key");
3463                         rte_free(priv);
3464                         return -ENOMEM;
3465                 }
3466                 session->cipher_key.length = cipher_xform->key.length;
3467                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3468                         cipher_xform->key.length);
3469                 session->dir =
3470                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3471                                         DIR_ENC : DIR_DEC;
3472                 session->cipher_alg = cipher_xform->algo;
3473         } else {
3474                 session->cipher_key.data = NULL;
3475                 session->cipher_key.length = 0;
3476                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3477                 session->dir = DIR_ENC;
3478         }
3479
3480         session->pdcp.domain = pdcp_xform->domain;
3481         session->pdcp.bearer = pdcp_xform->bearer;
3482         session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3483         session->pdcp.sn_size = pdcp_xform->sn_size;
3484         session->pdcp.hfn = pdcp_xform->hfn;
3485         session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3486         session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3487         /* hfv ovd offset location is stored in iv.offset value*/
3488         if (cipher_xform)
3489                 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3490
3491         cipherdata.key = (size_t)session->cipher_key.data;
3492         cipherdata.keylen = session->cipher_key.length;
3493         cipherdata.key_enc_flags = 0;
3494         cipherdata.key_type = RTA_DATA_IMM;
3495
3496         switch (session->cipher_alg) {
3497         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3498                 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3499                 break;
3500         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3501                 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3502                 break;
3503         case RTE_CRYPTO_CIPHER_AES_CTR:
3504                 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3505                 break;
3506         case RTE_CRYPTO_CIPHER_NULL:
3507                 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3508                 break;
3509         default:
3510                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3511                               session->cipher_alg);
3512                 goto out;
3513         }
3514
3515         if (auth_xform) {
3516                 session->auth_key.data = rte_zmalloc(NULL,
3517                                                      auth_xform->key.length,
3518                                                      RTE_CACHE_LINE_SIZE);
3519                 if (!session->auth_key.data &&
3520                     auth_xform->key.length > 0) {
3521                         DPAA2_SEC_ERR("No Memory for auth key");
3522                         rte_free(session->cipher_key.data);
3523                         rte_free(priv);
3524                         return -ENOMEM;
3525                 }
3526                 session->auth_key.length = auth_xform->key.length;
3527                 memcpy(session->auth_key.data, auth_xform->key.data,
3528                        auth_xform->key.length);
3529                 session->auth_alg = auth_xform->algo;
3530         } else {
3531                 session->auth_key.data = NULL;
3532                 session->auth_key.length = 0;
3533                 session->auth_alg = 0;
3534         }
3535         authdata.key = (size_t)session->auth_key.data;
3536         authdata.keylen = session->auth_key.length;
3537         authdata.key_enc_flags = 0;
3538         authdata.key_type = RTA_DATA_IMM;
3539
3540         if (session->auth_alg) {
3541                 switch (session->auth_alg) {
3542                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3543                         authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3544                         break;
3545                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3546                         authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3547                         break;
3548                 case RTE_CRYPTO_AUTH_AES_CMAC:
3549                         authdata.algtype = PDCP_AUTH_TYPE_AES;
3550                         break;
3551                 case RTE_CRYPTO_AUTH_NULL:
3552                         authdata.algtype = PDCP_AUTH_TYPE_NULL;
3553                         break;
3554                 default:
3555                         DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3556                                       session->auth_alg);
3557                         goto out;
3558                 }
3559
3560                 p_authdata = &authdata;
3561         } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3562                 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3563                 goto out;
3564         }
3565
3566         if (pdcp_xform->sdap_enabled) {
3567                 int nb_keys_to_inline =
3568                         rta_inline_pdcp_sdap_query(authdata.algtype,
3569                                         cipherdata.algtype,
3570                                         session->pdcp.sn_size,
3571                                         session->pdcp.hfn_ovd);
3572                 if (nb_keys_to_inline >= 1) {
3573                         cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3574                         cipherdata.key_type = RTA_DATA_PTR;
3575                 }
3576                 if (nb_keys_to_inline >= 2) {
3577                         authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
3578                         authdata.key_type = RTA_DATA_PTR;
3579                 }
3580         } else {
3581                 if (rta_inline_pdcp_query(authdata.algtype,
3582                                         cipherdata.algtype,
3583                                         session->pdcp.sn_size,
3584                                         session->pdcp.hfn_ovd)) {
3585                         cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3586                         cipherdata.key_type = RTA_DATA_PTR;
3587                 }
3588         }
3589
3590         if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3591                 if (session->dir == DIR_ENC)
3592                         bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3593                                         priv->flc_desc[0].desc, 1, swap,
3594                                         pdcp_xform->hfn,
3595                                         session->pdcp.sn_size,
3596                                         pdcp_xform->bearer,
3597                                         pdcp_xform->pkt_dir,
3598                                         pdcp_xform->hfn_threshold,
3599                                         &cipherdata, &authdata);
3600                 else if (session->dir == DIR_DEC)
3601                         bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3602                                         priv->flc_desc[0].desc, 1, swap,
3603                                         pdcp_xform->hfn,
3604                                         session->pdcp.sn_size,
3605                                         pdcp_xform->bearer,
3606                                         pdcp_xform->pkt_dir,
3607                                         pdcp_xform->hfn_threshold,
3608                                         &cipherdata, &authdata);
3609
3610         } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) {
3611                 bufsize = cnstr_shdsc_pdcp_short_mac(priv->flc_desc[0].desc,
3612                                                      1, swap, &authdata);
3613         } else {
3614                 if (session->dir == DIR_ENC) {
3615                         if (pdcp_xform->sdap_enabled)
3616                                 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_encap(
3617                                         priv->flc_desc[0].desc, 1, swap,
3618                                         session->pdcp.sn_size,
3619                                         pdcp_xform->hfn,
3620                                         pdcp_xform->bearer,
3621                                         pdcp_xform->pkt_dir,
3622                                         pdcp_xform->hfn_threshold,
3623                                         &cipherdata, p_authdata);
3624                         else
3625                                 bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3626                                         priv->flc_desc[0].desc, 1, swap,
3627                                         session->pdcp.sn_size,
3628                                         pdcp_xform->hfn,
3629                                         pdcp_xform->bearer,
3630                                         pdcp_xform->pkt_dir,
3631                                         pdcp_xform->hfn_threshold,
3632                                         &cipherdata, p_authdata);
3633                 } else if (session->dir == DIR_DEC) {
3634                         if (pdcp_xform->sdap_enabled)
3635                                 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_decap(
3636                                         priv->flc_desc[0].desc, 1, swap,
3637                                         session->pdcp.sn_size,
3638                                         pdcp_xform->hfn,
3639                                         pdcp_xform->bearer,
3640                                         pdcp_xform->pkt_dir,
3641                                         pdcp_xform->hfn_threshold,
3642                                         &cipherdata, p_authdata);
3643                         else
3644                                 bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3645                                         priv->flc_desc[0].desc, 1, swap,
3646                                         session->pdcp.sn_size,
3647                                         pdcp_xform->hfn,
3648                                         pdcp_xform->bearer,
3649                                         pdcp_xform->pkt_dir,
3650                                         pdcp_xform->hfn_threshold,
3651                                         &cipherdata, p_authdata);
3652                 }
3653         }
3654
3655         if (bufsize < 0) {
3656                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3657                 goto out;
3658         }
3659
3660         /* Enable the stashing control bit */
3661         DPAA2_SET_FLC_RSC(flc);
3662         flc->word2_rflc_31_0 = lower_32_bits(
3663                         (size_t)&(((struct dpaa2_sec_qp *)
3664                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
3665         flc->word3_rflc_63_32 = upper_32_bits(
3666                         (size_t)&(((struct dpaa2_sec_qp *)
3667                         dev->data->queue_pairs[0])->rx_vq));
3668
3669         flc->word1_sdl = (uint8_t)bufsize;
3670
3671         /* TODO - check the perf impact or
3672          * align as per descriptor type
3673          * Set EWS bit i.e. enable write-safe
3674          * DPAA2_SET_FLC_EWS(flc);
3675          */
3676
3677         /* Set BS = 1 i.e reuse input buffers as output buffers */
3678         DPAA2_SET_FLC_REUSE_BS(flc);
3679         /* Set FF = 10; reuse input buffers if they provide sufficient space */
3680         DPAA2_SET_FLC_REUSE_FF(flc);
3681
3682         session->ctxt = priv;
3683
3684         return 0;
3685 out:
3686         rte_free(session->auth_key.data);
3687         rte_free(session->cipher_key.data);
3688         rte_free(priv);
3689         return -EINVAL;
3690 }
3691
3692 static int
3693 dpaa2_sec_security_session_create(void *dev,
3694                                   struct rte_security_session_conf *conf,
3695                                   struct rte_security_session *sess,
3696                                   struct rte_mempool *mempool)
3697 {
3698         void *sess_private_data;
3699         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3700         int ret;
3701
3702         if (rte_mempool_get(mempool, &sess_private_data)) {
3703                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3704                 return -ENOMEM;
3705         }
3706
3707         switch (conf->protocol) {
3708         case RTE_SECURITY_PROTOCOL_IPSEC:
3709                 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3710                                 sess_private_data);
3711                 break;
3712         case RTE_SECURITY_PROTOCOL_MACSEC:
3713                 return -ENOTSUP;
3714         case RTE_SECURITY_PROTOCOL_PDCP:
3715                 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3716                                 sess_private_data);
3717                 break;
3718         default:
3719                 return -EINVAL;
3720         }
3721         if (ret != 0) {
3722                 DPAA2_SEC_ERR("Failed to configure session parameters");
3723                 /* Return session to mempool */
3724                 rte_mempool_put(mempool, sess_private_data);
3725                 return ret;
3726         }
3727
3728         set_sec_session_private_data(sess, sess_private_data);
3729
3730         return ret;
3731 }
3732
3733 /** Clear the memory of session so it doesn't leave key material behind */
3734 static int
3735 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3736                 struct rte_security_session *sess)
3737 {
3738         PMD_INIT_FUNC_TRACE();
3739         void *sess_priv = get_sec_session_private_data(sess);
3740
3741         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3742
3743         if (sess_priv) {
3744                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3745
3746                 rte_free(s->ctxt);
3747                 rte_free(s->cipher_key.data);
3748                 rte_free(s->auth_key.data);
3749                 memset(s, 0, sizeof(dpaa2_sec_session));
3750                 set_sec_session_private_data(sess, NULL);
3751                 rte_mempool_put(sess_mp, sess_priv);
3752         }
3753         return 0;
3754 }
3755 #endif
3756 static int
3757 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3758                 struct rte_crypto_sym_xform *xform,
3759                 struct rte_cryptodev_sym_session *sess,
3760                 struct rte_mempool *mempool)
3761 {
3762         void *sess_private_data;
3763         int ret;
3764
3765         if (rte_mempool_get(mempool, &sess_private_data)) {
3766                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3767                 return -ENOMEM;
3768         }
3769
3770         ret = dpaa2_sec_set_session_parameters(xform, sess_private_data);
3771         if (ret != 0) {
3772                 DPAA2_SEC_ERR("Failed to configure session parameters");
3773                 /* Return session to mempool */
3774                 rte_mempool_put(mempool, sess_private_data);
3775                 return ret;
3776         }
3777
3778         set_sym_session_private_data(sess, dev->driver_id,
3779                 sess_private_data);
3780
3781         return 0;
3782 }
3783
3784 /** Clear the memory of session so it doesn't leave key material behind */
3785 static void
3786 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3787                 struct rte_cryptodev_sym_session *sess)
3788 {
3789         PMD_INIT_FUNC_TRACE();
3790         uint8_t index = dev->driver_id;
3791         void *sess_priv = get_sym_session_private_data(sess, index);
3792         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3793
3794         if (sess_priv) {
3795                 rte_free(s->ctxt);
3796                 rte_free(s->cipher_key.data);
3797                 rte_free(s->auth_key.data);
3798                 memset(s, 0, sizeof(dpaa2_sec_session));
3799                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3800                 set_sym_session_private_data(sess, index, NULL);
3801                 rte_mempool_put(sess_mp, sess_priv);
3802         }
3803 }
3804
3805 static int
3806 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3807                         struct rte_cryptodev_config *config __rte_unused)
3808 {
3809         PMD_INIT_FUNC_TRACE();
3810
3811         return 0;
3812 }
3813
3814 static int
3815 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3816 {
3817         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3818         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3819         struct dpseci_attr attr;
3820         struct dpaa2_queue *dpaa2_q;
3821         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3822                                         dev->data->queue_pairs;
3823         struct dpseci_rx_queue_attr rx_attr;
3824         struct dpseci_tx_queue_attr tx_attr;
3825         int ret, i;
3826
3827         PMD_INIT_FUNC_TRACE();
3828
3829         /* Change the tx burst function if ordered queues are used */
3830         if (priv->en_ordered)
3831                 dev->enqueue_burst = dpaa2_sec_enqueue_burst_ordered;
3832
3833         memset(&attr, 0, sizeof(struct dpseci_attr));
3834
3835         ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3836         if (ret) {
3837                 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3838                               priv->hw_id);
3839                 goto get_attr_failure;
3840         }
3841         ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3842         if (ret) {
3843                 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3844                 goto get_attr_failure;
3845         }
3846         for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3847                 dpaa2_q = &qp[i]->rx_vq;
3848                 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3849                                     &rx_attr);
3850                 dpaa2_q->fqid = rx_attr.fqid;
3851                 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3852         }
3853         for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3854                 dpaa2_q = &qp[i]->tx_vq;
3855                 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3856                                     &tx_attr);
3857                 dpaa2_q->fqid = tx_attr.fqid;
3858                 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3859         }
3860
3861         return 0;
3862 get_attr_failure:
3863         dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3864         return -1;
3865 }
3866
3867 static void
3868 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3869 {
3870         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3871         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3872         int ret;
3873
3874         PMD_INIT_FUNC_TRACE();
3875
3876         ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3877         if (ret) {
3878                 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3879                              priv->hw_id);
3880                 return;
3881         }
3882
3883         ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3884         if (ret < 0) {
3885                 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3886                 return;
3887         }
3888 }
3889
3890 static int
3891 dpaa2_sec_dev_close(struct rte_cryptodev *dev __rte_unused)
3892 {
3893         PMD_INIT_FUNC_TRACE();
3894
3895         return 0;
3896 }
3897
3898 static void
3899 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3900                         struct rte_cryptodev_info *info)
3901 {
3902         struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3903
3904         PMD_INIT_FUNC_TRACE();
3905         if (info != NULL) {
3906                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3907                 info->feature_flags = dev->feature_flags;
3908                 info->capabilities = dpaa2_sec_capabilities;
3909                 /* No limit of number of sessions */
3910                 info->sym.max_nb_sessions = 0;
3911                 info->driver_id = cryptodev_driver_id;
3912         }
3913 }
3914
3915 static
3916 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3917                          struct rte_cryptodev_stats *stats)
3918 {
3919         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3920         struct fsl_mc_io dpseci;
3921         struct dpseci_sec_counters counters = {0};
3922         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3923                                         dev->data->queue_pairs;
3924         int ret, i;
3925
3926         PMD_INIT_FUNC_TRACE();
3927         if (stats == NULL) {
3928                 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3929                 return;
3930         }
3931         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3932                 if (qp == NULL || qp[i] == NULL) {
3933                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3934                         continue;
3935                 }
3936
3937                 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3938                 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3939                 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3940                 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3941         }
3942
3943         /* In case as secondary process access stats, MCP portal in priv-hw
3944          * may have primary process address. Need the secondary process
3945          * based MCP portal address for this object.
3946          */
3947         dpseci.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3948         ret = dpseci_get_sec_counters(&dpseci, CMD_PRI_LOW, priv->token,
3949                                       &counters);
3950         if (ret) {
3951                 DPAA2_SEC_ERR("SEC counters failed");
3952         } else {
3953                 DPAA2_SEC_INFO("dpseci hardware stats:"
3954                             "\n\tNum of Requests Dequeued = %" PRIu64
3955                             "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3956                             "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3957                             "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3958                             "\n\tNum of Outbound Bytes Protected = %" PRIu64
3959                             "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3960                             "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3961                             counters.dequeued_requests,
3962                             counters.ob_enc_requests,
3963                             counters.ib_dec_requests,
3964                             counters.ob_enc_bytes,
3965                             counters.ob_prot_bytes,
3966                             counters.ib_dec_bytes,
3967                             counters.ib_valid_bytes);
3968         }
3969 }
3970
3971 static
3972 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3973 {
3974         int i;
3975         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3976                                    (dev->data->queue_pairs);
3977
3978         PMD_INIT_FUNC_TRACE();
3979
3980         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3981                 if (qp[i] == NULL) {
3982                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3983                         continue;
3984                 }
3985                 qp[i]->tx_vq.rx_pkts = 0;
3986                 qp[i]->tx_vq.tx_pkts = 0;
3987                 qp[i]->tx_vq.err_pkts = 0;
3988                 qp[i]->rx_vq.rx_pkts = 0;
3989                 qp[i]->rx_vq.tx_pkts = 0;
3990                 qp[i]->rx_vq.err_pkts = 0;
3991         }
3992 }
3993
3994 static void __rte_hot
3995 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3996                                  const struct qbman_fd *fd,
3997                                  const struct qbman_result *dq,
3998                                  struct dpaa2_queue *rxq,
3999                                  struct rte_event *ev)
4000 {
4001         struct dpaa2_sec_qp *qp;
4002         /* Prefetching mbuf */
4003         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
4004                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
4005
4006         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
4007         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
4008
4009         qp = container_of(rxq, struct dpaa2_sec_qp, rx_vq);
4010         ev->flow_id = rxq->ev.flow_id;
4011         ev->sub_event_type = rxq->ev.sub_event_type;
4012         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
4013         ev->op = RTE_EVENT_OP_NEW;
4014         ev->sched_type = rxq->ev.sched_type;
4015         ev->queue_id = rxq->ev.queue_id;
4016         ev->priority = rxq->ev.priority;
4017         ev->event_ptr = sec_fd_to_mbuf(fd, qp);
4018
4019         qbman_swp_dqrr_consume(swp, dq);
4020 }
4021 static void
4022 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
4023                                  const struct qbman_fd *fd,
4024                                  const struct qbman_result *dq,
4025                                  struct dpaa2_queue *rxq,
4026                                  struct rte_event *ev)
4027 {
4028         uint8_t dqrr_index;
4029         struct dpaa2_sec_qp *qp;
4030         struct rte_crypto_op *crypto_op;
4031         /* Prefetching mbuf */
4032         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
4033                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
4034
4035         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
4036         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
4037
4038         qp = container_of(rxq, struct dpaa2_sec_qp, rx_vq);
4039         ev->flow_id = rxq->ev.flow_id;
4040         ev->sub_event_type = rxq->ev.sub_event_type;
4041         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
4042         ev->op = RTE_EVENT_OP_NEW;
4043         ev->sched_type = rxq->ev.sched_type;
4044         ev->queue_id = rxq->ev.queue_id;
4045         ev->priority = rxq->ev.priority;
4046
4047         crypto_op = sec_fd_to_mbuf(fd, qp);
4048         dqrr_index = qbman_get_dqrr_idx(dq);
4049         *dpaa2_seqn(crypto_op->sym->m_src) = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
4050         DPAA2_PER_LCORE_DQRR_SIZE++;
4051         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
4052         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
4053         ev->event_ptr = crypto_op;
4054 }
4055
4056 static void __rte_hot
4057 dpaa2_sec_process_ordered_event(struct qbman_swp *swp,
4058                                 const struct qbman_fd *fd,
4059                                 const struct qbman_result *dq,
4060                                 struct dpaa2_queue *rxq,
4061                                 struct rte_event *ev)
4062 {
4063         struct rte_crypto_op *crypto_op;
4064         struct dpaa2_sec_qp *qp;
4065
4066         /* Prefetching mbuf */
4067         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
4068                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
4069
4070         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
4071         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
4072
4073         qp = container_of(rxq, struct dpaa2_sec_qp, rx_vq);
4074         ev->flow_id = rxq->ev.flow_id;
4075         ev->sub_event_type = rxq->ev.sub_event_type;
4076         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
4077         ev->op = RTE_EVENT_OP_NEW;
4078         ev->sched_type = rxq->ev.sched_type;
4079         ev->queue_id = rxq->ev.queue_id;
4080         ev->priority = rxq->ev.priority;
4081         crypto_op = sec_fd_to_mbuf(fd, qp);
4082
4083         *dpaa2_seqn(crypto_op->sym->m_src) = DPAA2_ENQUEUE_FLAG_ORP;
4084         *dpaa2_seqn(crypto_op->sym->m_src) |= qbman_result_DQ_odpid(dq) <<
4085                 DPAA2_EQCR_OPRID_SHIFT;
4086         *dpaa2_seqn(crypto_op->sym->m_src) |= qbman_result_DQ_seqnum(dq) <<
4087                 DPAA2_EQCR_SEQNUM_SHIFT;
4088
4089         qbman_swp_dqrr_consume(swp, dq);
4090         ev->event_ptr = crypto_op;
4091 }
4092
4093 int
4094 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
4095                 int qp_id,
4096                 struct dpaa2_dpcon_dev *dpcon,
4097                 const struct rte_event *event)
4098 {
4099         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
4100         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
4101         struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
4102         struct dpseci_rx_queue_cfg cfg;
4103         uint8_t priority;
4104         int ret;
4105
4106         if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
4107                 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
4108         else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
4109                 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
4110         else if (event->sched_type == RTE_SCHED_TYPE_ORDERED)
4111                 qp->rx_vq.cb = dpaa2_sec_process_ordered_event;
4112         else
4113                 return -EINVAL;
4114
4115         priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
4116                    (dpcon->num_priorities - 1);
4117
4118         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
4119         cfg.options = DPSECI_QUEUE_OPT_DEST;
4120         cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
4121         cfg.dest_cfg.dest_id = dpcon->dpcon_id;
4122         cfg.dest_cfg.priority = priority;
4123
4124         cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
4125         cfg.user_ctx = (size_t)(qp);
4126         if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
4127                 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
4128                 cfg.order_preservation_en = 1;
4129         }
4130
4131         if (event->sched_type == RTE_SCHED_TYPE_ORDERED) {
4132                 struct opr_cfg ocfg;
4133
4134                 /* Restoration window size = 256 frames */
4135                 ocfg.oprrws = 3;
4136                 /* Restoration window size = 512 frames for LX2 */
4137                 if (dpaa2_svr_family == SVR_LX2160A)
4138                         ocfg.oprrws = 4;
4139                 /* Auto advance NESN window enabled */
4140                 ocfg.oa = 1;
4141                 /* Late arrival window size disabled */
4142                 ocfg.olws = 0;
4143                 /* ORL resource exhaustaion advance NESN disabled */
4144                 ocfg.oeane = 0;
4145
4146                 if (priv->en_loose_ordered)
4147                         ocfg.oloe = 1;
4148                 else
4149                         ocfg.oloe = 0;
4150
4151                 ret = dpseci_set_opr(dpseci, CMD_PRI_LOW, priv->token,
4152                                    qp_id, OPR_OPT_CREATE, &ocfg);
4153                 if (ret) {
4154                         RTE_LOG(ERR, PMD, "Error setting opr: ret: %d\n", ret);
4155                         return ret;
4156                 }
4157                 qp->tx_vq.cb_eqresp_free = dpaa2_sec_free_eqresp_buf;
4158                 priv->en_ordered = 1;
4159         }
4160
4161         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
4162                                   qp_id, &cfg);
4163         if (ret) {
4164                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
4165                 return ret;
4166         }
4167
4168         memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
4169
4170         return 0;
4171 }
4172
4173 int
4174 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
4175                         int qp_id)
4176 {
4177         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
4178         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
4179         struct dpseci_rx_queue_cfg cfg;
4180         int ret;
4181
4182         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
4183         cfg.options = DPSECI_QUEUE_OPT_DEST;
4184         cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
4185
4186         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
4187                                   qp_id, &cfg);
4188         if (ret)
4189                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
4190
4191         return ret;
4192 }
4193
4194 static struct rte_cryptodev_ops crypto_ops = {
4195         .dev_configure        = dpaa2_sec_dev_configure,
4196         .dev_start            = dpaa2_sec_dev_start,
4197         .dev_stop             = dpaa2_sec_dev_stop,
4198         .dev_close            = dpaa2_sec_dev_close,
4199         .dev_infos_get        = dpaa2_sec_dev_infos_get,
4200         .stats_get            = dpaa2_sec_stats_get,
4201         .stats_reset          = dpaa2_sec_stats_reset,
4202         .queue_pair_setup     = dpaa2_sec_queue_pair_setup,
4203         .queue_pair_release   = dpaa2_sec_queue_pair_release,
4204         .sym_session_get_size     = dpaa2_sec_sym_session_get_size,
4205         .sym_session_configure    = dpaa2_sec_sym_session_configure,
4206         .sym_session_clear        = dpaa2_sec_sym_session_clear,
4207         /* Raw data-path API related operations */
4208         .sym_get_raw_dp_ctx_size = dpaa2_sec_get_dp_ctx_size,
4209         .sym_configure_raw_dp_ctx = dpaa2_sec_configure_raw_dp_ctx,
4210 };
4211
4212 #ifdef RTE_LIB_SECURITY
4213 static const struct rte_security_capability *
4214 dpaa2_sec_capabilities_get(void *device __rte_unused)
4215 {
4216         return dpaa2_sec_security_cap;
4217 }
4218
4219 static const struct rte_security_ops dpaa2_sec_security_ops = {
4220         .session_create = dpaa2_sec_security_session_create,
4221         .session_update = NULL,
4222         .session_stats_get = NULL,
4223         .session_destroy = dpaa2_sec_security_session_destroy,
4224         .set_pkt_metadata = NULL,
4225         .capabilities_get = dpaa2_sec_capabilities_get
4226 };
4227 #endif
4228
4229 static int
4230 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
4231 {
4232         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
4233         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
4234         int ret;
4235
4236         PMD_INIT_FUNC_TRACE();
4237
4238         /* Function is reverse of dpaa2_sec_dev_init.
4239          * It does the following:
4240          * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
4241          * 2. Close the DPSECI device
4242          * 3. Free the allocated resources.
4243          */
4244
4245         /*Close the device at underlying layer*/
4246         ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
4247         if (ret) {
4248                 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
4249                 return -1;
4250         }
4251
4252         /*Free the allocated memory for ethernet private data and dpseci*/
4253         priv->hw = NULL;
4254         rte_free(dpseci);
4255         rte_free(dev->security_ctx);
4256
4257         DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
4258                        dev->data->name, rte_socket_id());
4259
4260         return 0;
4261 }
4262
4263 static int
4264 check_devargs_handler(const char *key, const char *value,
4265                       void *opaque)
4266 {
4267         struct rte_cryptodev *dev = (struct rte_cryptodev *)opaque;
4268         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
4269
4270         if (!strcmp(key, "drv_strict_order")) {
4271                 priv->en_loose_ordered = false;
4272         } else if (!strcmp(key, "drv_dump_mode")) {
4273                 dpaa2_sec_dp_dump = atoi(value);
4274                 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_FULL_DUMP) {
4275                         DPAA2_SEC_WARN("WARN: DPAA2_SEC_DP_DUMP_LEVEL is not "
4276                                       "supported, changing to FULL error"
4277                                       " prints\n");
4278                         dpaa2_sec_dp_dump = DPAA2_SEC_DP_FULL_DUMP;
4279                 }
4280         } else
4281                 return -1;
4282
4283         return 0;
4284 }
4285
4286 static void
4287 dpaa2_sec_get_devargs(struct rte_cryptodev *cryptodev, const char *key)
4288 {
4289         struct rte_kvargs *kvlist;
4290         struct rte_devargs *devargs;
4291
4292         devargs = cryptodev->device->devargs;
4293         if (!devargs)
4294                 return;
4295
4296         kvlist = rte_kvargs_parse(devargs->args, NULL);
4297         if (!kvlist)
4298                 return;
4299
4300         if (!rte_kvargs_count(kvlist, key)) {
4301                 rte_kvargs_free(kvlist);
4302                 return;
4303         }
4304
4305         rte_kvargs_process(kvlist, key,
4306                         check_devargs_handler, (void *)cryptodev);
4307         rte_kvargs_free(kvlist);
4308 }
4309
4310 static int
4311 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
4312 {
4313         struct dpaa2_sec_dev_private *internals;
4314         struct rte_device *dev = cryptodev->device;
4315         struct rte_dpaa2_device *dpaa2_dev;
4316 #ifdef RTE_LIB_SECURITY
4317         struct rte_security_ctx *security_instance;
4318 #endif
4319         struct fsl_mc_io *dpseci;
4320         uint16_t token;
4321         struct dpseci_attr attr;
4322         int retcode, hw_id;
4323
4324         PMD_INIT_FUNC_TRACE();
4325         dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
4326         hw_id = dpaa2_dev->object_id;
4327
4328         cryptodev->driver_id = cryptodev_driver_id;
4329         cryptodev->dev_ops = &crypto_ops;
4330
4331         cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
4332         cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
4333         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
4334                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
4335                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
4336                         RTE_CRYPTODEV_FF_SECURITY |
4337                         RTE_CRYPTODEV_FF_SYM_RAW_DP |
4338                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
4339                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
4340                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
4341                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
4342                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
4343
4344         internals = cryptodev->data->dev_private;
4345
4346         /*
4347          * For secondary processes, we don't initialise any further as primary
4348          * has already done this work. Only check we don't need a different
4349          * RX function
4350          */
4351         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
4352                 DPAA2_SEC_DEBUG("Device already init by primary process");
4353                 return 0;
4354         }
4355 #ifdef RTE_LIB_SECURITY
4356         /* Initialize security_ctx only for primary process*/
4357         security_instance = rte_malloc("rte_security_instances_ops",
4358                                 sizeof(struct rte_security_ctx), 0);
4359         if (security_instance == NULL)
4360                 return -ENOMEM;
4361         security_instance->device = (void *)cryptodev;
4362         security_instance->ops = &dpaa2_sec_security_ops;
4363         security_instance->sess_cnt = 0;
4364         cryptodev->security_ctx = security_instance;
4365 #endif
4366         /*Open the rte device via MC and save the handle for further use*/
4367         dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
4368                                 sizeof(struct fsl_mc_io), 0);
4369         if (!dpseci) {
4370                 DPAA2_SEC_ERR(
4371                         "Error in allocating the memory for dpsec object");
4372                 return -ENOMEM;
4373         }
4374         dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
4375
4376         retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
4377         if (retcode != 0) {
4378                 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
4379                               retcode);
4380                 goto init_error;
4381         }
4382         retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
4383         if (retcode != 0) {
4384                 DPAA2_SEC_ERR(
4385                              "Cannot get dpsec device attributed: Error = %x",
4386                              retcode);
4387                 goto init_error;
4388         }
4389         snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
4390                         "dpsec-%u", hw_id);
4391
4392         internals->max_nb_queue_pairs = attr.num_tx_queues;
4393         cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
4394         internals->hw = dpseci;
4395         internals->token = token;
4396         internals->en_loose_ordered = true;
4397
4398         dpaa2_sec_get_devargs(cryptodev, DRIVER_DUMP_MODE);
4399         dpaa2_sec_get_devargs(cryptodev, DRIVER_STRICT_ORDER);
4400         DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
4401         return 0;
4402
4403 init_error:
4404         DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
4405
4406         /* dpaa2_sec_uninit(crypto_dev_name); */
4407         return -EFAULT;
4408 }
4409
4410 static int
4411 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
4412                           struct rte_dpaa2_device *dpaa2_dev)
4413 {
4414         struct rte_cryptodev *cryptodev;
4415         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
4416
4417         int retval;
4418
4419         snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
4420                         dpaa2_dev->object_id);
4421
4422         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
4423         if (cryptodev == NULL)
4424                 return -ENOMEM;
4425
4426         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4427                 cryptodev->data->dev_private = rte_zmalloc_socket(
4428                                         "cryptodev private structure",
4429                                         sizeof(struct dpaa2_sec_dev_private),
4430                                         RTE_CACHE_LINE_SIZE,
4431                                         rte_socket_id());
4432
4433                 if (cryptodev->data->dev_private == NULL)
4434                         rte_panic("Cannot allocate memzone for private "
4435                                   "device data");
4436         }
4437
4438         dpaa2_dev->cryptodev = cryptodev;
4439         cryptodev->device = &dpaa2_dev->device;
4440
4441         /* init user callbacks */
4442         TAILQ_INIT(&(cryptodev->link_intr_cbs));
4443
4444         if (dpaa2_svr_family == SVR_LX2160A)
4445                 rta_set_sec_era(RTA_SEC_ERA_10);
4446         else
4447                 rta_set_sec_era(RTA_SEC_ERA_8);
4448
4449         DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
4450
4451         /* Invoke PMD device initialization function */
4452         retval = dpaa2_sec_dev_init(cryptodev);
4453         if (retval == 0) {
4454                 rte_cryptodev_pmd_probing_finish(cryptodev);
4455                 return 0;
4456         }
4457
4458         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
4459                 rte_free(cryptodev->data->dev_private);
4460
4461         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
4462
4463         return -ENXIO;
4464 }
4465
4466 static int
4467 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
4468 {
4469         struct rte_cryptodev *cryptodev;
4470         int ret;
4471
4472         cryptodev = dpaa2_dev->cryptodev;
4473         if (cryptodev == NULL)
4474                 return -ENODEV;
4475
4476         ret = dpaa2_sec_uninit(cryptodev);
4477         if (ret)
4478                 return ret;
4479
4480         return rte_cryptodev_pmd_destroy(cryptodev);
4481 }
4482
4483 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
4484         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
4485         .drv_type = DPAA2_CRYPTO,
4486         .driver = {
4487                 .name = "DPAA2 SEC PMD"
4488         },
4489         .probe = cryptodev_dpaa2_sec_probe,
4490         .remove = cryptodev_dpaa2_sec_remove,
4491 };
4492
4493 static struct cryptodev_driver dpaa2_sec_crypto_drv;
4494
4495 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
4496 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
4497                 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
4498 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA2_SEC_PMD,
4499                 DRIVER_STRICT_ORDER "=<int>"
4500                 DRIVER_DUMP_MODE "=<int>");
4501 RTE_LOG_REGISTER(dpaa2_logtype_sec, pmd.crypto.dpaa2, NOTICE);