drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2019 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10 #include <unistd.h>
  11
  12 #include <rte_ip.h>
  13 #include <rte_mbuf.h>
  14 #include <rte_cryptodev.h>
  15 #include <rte_malloc.h>
  16 #include <rte_memcpy.h>
  17 #include <rte_string_fns.h>
  18 #include <rte_cycles.h>
  19 #include <rte_kvargs.h>
  20 #include <rte_dev.h>
  21 #include <rte_cryptodev_pmd.h>
  22 #include <rte_common.h>
  23 #include <rte_fslmc.h>
  24 #include <fslmc_vfio.h>
  25 #include <dpaa2_hw_pvt.h>
  26 #include <dpaa2_hw_dpio.h>
  27 #include <dpaa2_hw_mempool.h>
  28 #include <fsl_dpopr.h>
  29 #include <fsl_dpseci.h>
  30 #include <fsl_mc_sys.h>
  31
  32 #include "dpaa2_sec_priv.h"
  33 #include "dpaa2_sec_event.h"
  34 #include "dpaa2_sec_logs.h"
  35
  36 /* RTA header files */
  37 #include <desc/ipsec.h>
  38 #include <desc/pdcp.h>
  39 #include <desc/algo.h>
  40
  41 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
  42  * a pointer to the shared descriptor
  43  */
  44 #define MIN_JOB_DESC_SIZE       (CAAM_CMD_SZ + CAAM_PTR_SZ)
  45 #define FSL_VENDOR_ID           0x1957
  46 #define FSL_DEVICE_ID           0x410
  47 #define FSL_SUBSYSTEM_SEC       1
  48 #define FSL_MC_DPSECI_DEVID     3
  49
  50 #define NO_PREFETCH 0
  51 /* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */
  52 #define FLE_POOL_NUM_BUFS       32000
  53 #define FLE_POOL_BUF_SIZE       256
  54 #define FLE_POOL_CACHE_SIZE     512
  55 #define FLE_SG_MEM_SIZE(num)    (FLE_POOL_BUF_SIZE + ((num) * 32))
  56 #define SEC_FLC_DHR_OUTBOUND    -114
  57 #define SEC_FLC_DHR_INBOUND     0
  58
  59 enum rta_sec_era rta_sec_era = RTA_SEC_ERA_8;
  60
  61 static uint8_t cryptodev_driver_id;
  62
  63 int dpaa2_logtype_sec;
  64
  65 #ifdef RTE_LIBRTE_SECURITY
  66 static inline int
  67 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
  68                            struct rte_crypto_op *op,
  69                            struct qbman_fd *fd, uint16_t bpid)
  70 {
  71         struct rte_crypto_sym_op *sym_op = op->sym;
  72         struct ctxt_priv *priv = sess->ctxt;
  73         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
  74         struct sec_flow_context *flc;
  75         struct rte_mbuf *mbuf;
  76         uint32_t in_len = 0, out_len = 0;
  77
  78         if (sym_op->m_dst)
  79                 mbuf = sym_op->m_dst;
  80         else
  81                 mbuf = sym_op->m_src;
  82
  83         /* first FLE entry used to store mbuf and session ctxt */
  84         fle = (struct qbman_fle *)rte_malloc(NULL,
  85                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
  86                         RTE_CACHE_LINE_SIZE);
  87         if (unlikely(!fle)) {
  88                 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
  89                 return -ENOMEM;
  90         }
  91         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
  92         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
  93         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
  94
  95         /* Save the shared descriptor */
  96         flc = &priv->flc_desc[0].flc;
  97
  98         op_fle = fle + 1;
  99         ip_fle = fle + 2;
 100         sge = fle + 3;
 101
 102         if (likely(bpid < MAX_BPID)) {
 103                 DPAA2_SET_FD_BPID(fd, bpid);
 104                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 105                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 106         } else {
 107                 DPAA2_SET_FD_IVP(fd);
 108                 DPAA2_SET_FLE_IVP(op_fle);
 109                 DPAA2_SET_FLE_IVP(ip_fle);
 110         }
 111
 112         /* Configure FD as a FRAME LIST */
 113         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 114         DPAA2_SET_FD_COMPOUND_FMT(fd);
 115         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 116
 117         /* Configure Output FLE with Scatter/Gather Entry */
 118         DPAA2_SET_FLE_SG_EXT(op_fle);
 119         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 120
 121         /* Configure Output SGE for Encap/Decap */
 122         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 123         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 124         /* o/p segs */
 125         while (mbuf->next) {
 126                 sge->length = mbuf->data_len;
 127                 out_len += sge->length;
 128                 sge++;
 129                 mbuf = mbuf->next;
 130                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 131                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 132         }
 133         /* using buf_len for last buf - so that extra data can be added */
 134         sge->length = mbuf->buf_len - mbuf->data_off;
 135         out_len += sge->length;
 136
 137         DPAA2_SET_FLE_FIN(sge);
 138         op_fle->length = out_len;
 139
 140         sge++;
 141         mbuf = sym_op->m_src;
 142
 143         /* Configure Input FLE with Scatter/Gather Entry */
 144         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 145         DPAA2_SET_FLE_SG_EXT(ip_fle);
 146         DPAA2_SET_FLE_FIN(ip_fle);
 147
 148         /* Configure input SGE for Encap/Decap */
 149         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 150         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 151         sge->length = mbuf->data_len;
 152         in_len += sge->length;
 153
 154         mbuf = mbuf->next;
 155         /* i/p segs */
 156         while (mbuf) {
 157                 sge++;
 158                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 159                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 160                 sge->length = mbuf->data_len;
 161                 in_len += sge->length;
 162                 mbuf = mbuf->next;
 163         }
 164         ip_fle->length = in_len;
 165         DPAA2_SET_FLE_FIN(sge);
 166
 167         /* In case of PDCP, per packet HFN is stored in
 168          * mbuf priv after sym_op.
 169          */
 170         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 171                 uint32_t hfn_ovd = *((uint8_t *)op + sess->pdcp.hfn_ovd_offset);
 172                 /*enable HFN override override */
 173                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 174                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 175                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 176         }
 177         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 178
 179         return 0;
 180 }
 181
 182 static inline int
 183 build_proto_compound_fd(dpaa2_sec_session *sess,
 184                struct rte_crypto_op *op,
 185                struct qbman_fd *fd, uint16_t bpid)
 186 {
 187         struct rte_crypto_sym_op *sym_op = op->sym;
 188         struct ctxt_priv *priv = sess->ctxt;
 189         struct qbman_fle *fle, *ip_fle, *op_fle;
 190         struct sec_flow_context *flc;
 191         struct rte_mbuf *src_mbuf = sym_op->m_src;
 192         struct rte_mbuf *dst_mbuf = sym_op->m_dst;
 193         int retval;
 194
 195         if (!dst_mbuf)
 196                 dst_mbuf = src_mbuf;
 197
 198         /* Save the shared descriptor */
 199         flc = &priv->flc_desc[0].flc;
 200
 201         /* we are using the first FLE entry to store Mbuf */
 202         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 203         if (retval) {
 204                 DPAA2_SEC_DP_ERR("Memory alloc failed");
 205                 return -ENOMEM;
 206         }
 207         memset(fle, 0, FLE_POOL_BUF_SIZE);
 208         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 209         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 210
 211         op_fle = fle + 1;
 212         ip_fle = fle + 2;
 213
 214         if (likely(bpid < MAX_BPID)) {
 215                 DPAA2_SET_FD_BPID(fd, bpid);
 216                 DPAA2_SET_FLE_BPID(op_fle, bpid);
 217                 DPAA2_SET_FLE_BPID(ip_fle, bpid);
 218         } else {
 219                 DPAA2_SET_FD_IVP(fd);
 220                 DPAA2_SET_FLE_IVP(op_fle);
 221                 DPAA2_SET_FLE_IVP(ip_fle);
 222         }
 223
 224         /* Configure FD as a FRAME LIST */
 225         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 226         DPAA2_SET_FD_COMPOUND_FMT(fd);
 227         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 228
 229         /* Configure Output FLE with dst mbuf data  */
 230         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
 231         DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
 232         DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
 233
 234         /* Configure Input FLE with src mbuf data */
 235         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
 236         DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
 237         DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
 238
 239         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 240         DPAA2_SET_FLE_FIN(ip_fle);
 241
 242         /* In case of PDCP, per packet HFN is stored in
 243          * mbuf priv after sym_op.
 244          */
 245         if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
 246                 uint32_t hfn_ovd = *((uint8_t *)op + sess->pdcp.hfn_ovd_offset);
 247                 /*enable HFN override override */
 248                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
 249                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
 250                 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
 251         }
 252
 253         return 0;
 254
 255 }
 256
 257 static inline int
 258 build_proto_fd(dpaa2_sec_session *sess,
 259                struct rte_crypto_op *op,
 260                struct qbman_fd *fd, uint16_t bpid)
 261 {
 262         struct rte_crypto_sym_op *sym_op = op->sym;
 263         if (sym_op->m_dst)
 264                 return build_proto_compound_fd(sess, op, fd, bpid);
 265
 266         struct ctxt_priv *priv = sess->ctxt;
 267         struct sec_flow_context *flc;
 268         struct rte_mbuf *mbuf = sym_op->m_src;
 269
 270         if (likely(bpid < MAX_BPID))
 271                 DPAA2_SET_FD_BPID(fd, bpid);
 272         else
 273                 DPAA2_SET_FD_IVP(fd);
 274
 275         /* Save the shared descriptor */
 276         flc = &priv->flc_desc[0].flc;
 277
 278         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 279         DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
 280         DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
 281         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 282
 283         /* save physical address of mbuf */
 284         op->sym->aead.digest.phys_addr = mbuf->buf_iova;
 285         mbuf->buf_iova = (size_t)op;
 286
 287         return 0;
 288 }
 289 #endif
 290
 291 static inline int
 292 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
 293                  struct rte_crypto_op *op,
 294                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 295 {
 296         struct rte_crypto_sym_op *sym_op = op->sym;
 297         struct ctxt_priv *priv = sess->ctxt;
 298         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 299         struct sec_flow_context *flc;
 300         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 301         int icv_len = sess->digest_length;
 302         uint8_t *old_icv;
 303         struct rte_mbuf *mbuf;
 304         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 305                         sess->iv.offset);
 306
 307         if (sym_op->m_dst)
 308                 mbuf = sym_op->m_dst;
 309         else
 310                 mbuf = sym_op->m_src;
 311
 312         /* first FLE entry used to store mbuf and session ctxt */
 313         fle = (struct qbman_fle *)rte_malloc(NULL,
 314                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 315                         RTE_CACHE_LINE_SIZE);
 316         if (unlikely(!fle)) {
 317                 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
 318                 return -ENOMEM;
 319         }
 320         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 321         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 322         DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
 323
 324         op_fle = fle + 1;
 325         ip_fle = fle + 2;
 326         sge = fle + 3;
 327
 328         /* Save the shared descriptor */
 329         flc = &priv->flc_desc[0].flc;
 330
 331         /* Configure FD as a FRAME LIST */
 332         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 333         DPAA2_SET_FD_COMPOUND_FMT(fd);
 334         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 335
 336         DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 337                    "iv-len=%d data_off: 0x%x\n",
 338                    sym_op->aead.data.offset,
 339                    sym_op->aead.data.length,
 340                    sess->digest_length,
 341                    sess->iv.length,
 342                    sym_op->m_src->data_off);
 343
 344         /* Configure Output FLE with Scatter/Gather Entry */
 345         DPAA2_SET_FLE_SG_EXT(op_fle);
 346         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 347
 348         if (auth_only_len)
 349                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 350
 351         op_fle->length = (sess->dir == DIR_ENC) ?
 352                         (sym_op->aead.data.length + icv_len) :
 353                         sym_op->aead.data.length;
 354
 355         /* Configure Output SGE for Encap/Decap */
 356         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 357         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
 358         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 359
 360         mbuf = mbuf->next;
 361         /* o/p segs */
 362         while (mbuf) {
 363                 sge++;
 364                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 365                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 366                 sge->length = mbuf->data_len;
 367                 mbuf = mbuf->next;
 368         }
 369         sge->length -= icv_len;
 370
 371         if (sess->dir == DIR_ENC) {
 372                 sge++;
 373                 DPAA2_SET_FLE_ADDR(sge,
 374                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 375                 sge->length = icv_len;
 376         }
 377         DPAA2_SET_FLE_FIN(sge);
 378
 379         sge++;
 380         mbuf = sym_op->m_src;
 381
 382         /* Configure Input FLE with Scatter/Gather Entry */
 383         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 384         DPAA2_SET_FLE_SG_EXT(ip_fle);
 385         DPAA2_SET_FLE_FIN(ip_fle);
 386         ip_fle->length = (sess->dir == DIR_ENC) ?
 387                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 388                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 389                  icv_len);
 390
 391         /* Configure Input SGE for Encap/Decap */
 392         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 393         sge->length = sess->iv.length;
 394
 395         sge++;
 396         if (auth_only_len) {
 397                 DPAA2_SET_FLE_ADDR(sge,
 398                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 399                 sge->length = auth_only_len;
 400                 sge++;
 401         }
 402
 403         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 404         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 405                                 mbuf->data_off);
 406         sge->length = mbuf->data_len - sym_op->aead.data.offset;
 407
 408         mbuf = mbuf->next;
 409         /* i/p segs */
 410         while (mbuf) {
 411                 sge++;
 412                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 413                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 414                 sge->length = mbuf->data_len;
 415                 mbuf = mbuf->next;
 416         }
 417
 418         if (sess->dir == DIR_DEC) {
 419                 sge++;
 420                 old_icv = (uint8_t *)(sge + 1);
 421                 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
 422                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 423                 sge->length = icv_len;
 424         }
 425
 426         DPAA2_SET_FLE_FIN(sge);
 427         if (auth_only_len) {
 428                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 429                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 430         }
 431         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 432
 433         return 0;
 434 }
 435
 436 static inline int
 437 build_authenc_gcm_fd(dpaa2_sec_session *sess,
 438                      struct rte_crypto_op *op,
 439                      struct qbman_fd *fd, uint16_t bpid)
 440 {
 441         struct rte_crypto_sym_op *sym_op = op->sym;
 442         struct ctxt_priv *priv = sess->ctxt;
 443         struct qbman_fle *fle, *sge;
 444         struct sec_flow_context *flc;
 445         uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
 446         int icv_len = sess->digest_length, retval;
 447         uint8_t *old_icv;
 448         struct rte_mbuf *dst;
 449         uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 450                         sess->iv.offset);
 451
 452         if (sym_op->m_dst)
 453                 dst = sym_op->m_dst;
 454         else
 455                 dst = sym_op->m_src;
 456
 457         /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
 458          * Currently we donot know which FLE has the mbuf stored.
 459          * So while retreiving we can go back 1 FLE from the FD -ADDR
 460          * to get the MBUF Addr from the previous FLE.
 461          * We can have a better approach to use the inline Mbuf
 462          */
 463         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 464         if (retval) {
 465                 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE");
 466                 return -ENOMEM;
 467         }
 468         memset(fle, 0, FLE_POOL_BUF_SIZE);
 469         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 470         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 471         fle = fle + 1;
 472         sge = fle + 2;
 473         if (likely(bpid < MAX_BPID)) {
 474                 DPAA2_SET_FD_BPID(fd, bpid);
 475                 DPAA2_SET_FLE_BPID(fle, bpid);
 476                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 477                 DPAA2_SET_FLE_BPID(sge, bpid);
 478                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 479                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 480                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 481         } else {
 482                 DPAA2_SET_FD_IVP(fd);
 483                 DPAA2_SET_FLE_IVP(fle);
 484                 DPAA2_SET_FLE_IVP((fle + 1));
 485                 DPAA2_SET_FLE_IVP(sge);
 486                 DPAA2_SET_FLE_IVP((sge + 1));
 487                 DPAA2_SET_FLE_IVP((sge + 2));
 488                 DPAA2_SET_FLE_IVP((sge + 3));
 489         }
 490
 491         /* Save the shared descriptor */
 492         flc = &priv->flc_desc[0].flc;
 493         /* Configure FD as a FRAME LIST */
 494         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 495         DPAA2_SET_FD_COMPOUND_FMT(fd);
 496         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 497
 498         DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
 499                    "iv-len=%d data_off: 0x%x\n",
 500                    sym_op->aead.data.offset,
 501                    sym_op->aead.data.length,
 502                    sess->digest_length,
 503                    sess->iv.length,
 504                    sym_op->m_src->data_off);
 505
 506         /* Configure Output FLE with Scatter/Gather Entry */
 507         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 508         if (auth_only_len)
 509                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 510         fle->length = (sess->dir == DIR_ENC) ?
 511                         (sym_op->aead.data.length + icv_len) :
 512                         sym_op->aead.data.length;
 513
 514         DPAA2_SET_FLE_SG_EXT(fle);
 515
 516         /* Configure Output SGE for Encap/Decap */
 517         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 518         DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
 519         sge->length = sym_op->aead.data.length;
 520
 521         if (sess->dir == DIR_ENC) {
 522                 sge++;
 523                 DPAA2_SET_FLE_ADDR(sge,
 524                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
 525                 sge->length = sess->digest_length;
 526         }
 527         DPAA2_SET_FLE_FIN(sge);
 528
 529         sge++;
 530         fle++;
 531
 532         /* Configure Input FLE with Scatter/Gather Entry */
 533         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 534         DPAA2_SET_FLE_SG_EXT(fle);
 535         DPAA2_SET_FLE_FIN(fle);
 536         fle->length = (sess->dir == DIR_ENC) ?
 537                 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
 538                 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
 539                  sess->digest_length);
 540
 541         /* Configure Input SGE for Encap/Decap */
 542         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
 543         sge->length = sess->iv.length;
 544         sge++;
 545         if (auth_only_len) {
 546                 DPAA2_SET_FLE_ADDR(sge,
 547                                 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
 548                 sge->length = auth_only_len;
 549                 DPAA2_SET_FLE_BPID(sge, bpid);
 550                 sge++;
 551         }
 552
 553         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 554         DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
 555                                 sym_op->m_src->data_off);
 556         sge->length = sym_op->aead.data.length;
 557         if (sess->dir == DIR_DEC) {
 558                 sge++;
 559                 old_icv = (uint8_t *)(sge + 1);
 560                 memcpy(old_icv, sym_op->aead.digest.data,
 561                        sess->digest_length);
 562                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 563                 sge->length = sess->digest_length;
 564         }
 565         DPAA2_SET_FLE_FIN(sge);
 566
 567         if (auth_only_len) {
 568                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 569                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 570         }
 571
 572         DPAA2_SET_FD_LEN(fd, fle->length);
 573         return 0;
 574 }
 575
 576 static inline int
 577 build_authenc_sg_fd(dpaa2_sec_session *sess,
 578                  struct rte_crypto_op *op,
 579                  struct qbman_fd *fd, __rte_unused uint16_t bpid)
 580 {
 581         struct rte_crypto_sym_op *sym_op = op->sym;
 582         struct ctxt_priv *priv = sess->ctxt;
 583         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 584         struct sec_flow_context *flc;
 585         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 586                                 sym_op->auth.data.offset;
 587         uint16_t auth_tail_len = sym_op->auth.data.length -
 588                                 sym_op->cipher.data.length - auth_hdr_len;
 589         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 590         int icv_len = sess->digest_length;
 591         uint8_t *old_icv;
 592         struct rte_mbuf *mbuf;
 593         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 594                         sess->iv.offset);
 595
 596         if (sym_op->m_dst)
 597                 mbuf = sym_op->m_dst;
 598         else
 599                 mbuf = sym_op->m_src;
 600
 601         /* first FLE entry used to store mbuf and session ctxt */
 602         fle = (struct qbman_fle *)rte_malloc(NULL,
 603                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
 604                         RTE_CACHE_LINE_SIZE);
 605         if (unlikely(!fle)) {
 606                 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
 607                 return -ENOMEM;
 608         }
 609         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
 610         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 611         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 612
 613         op_fle = fle + 1;
 614         ip_fle = fle + 2;
 615         sge = fle + 3;
 616
 617         /* Save the shared descriptor */
 618         flc = &priv->flc_desc[0].flc;
 619
 620         /* Configure FD as a FRAME LIST */
 621         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 622         DPAA2_SET_FD_COMPOUND_FMT(fd);
 623         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 624
 625         DPAA2_SEC_DP_DEBUG(
 626                 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
 627                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 628                 sym_op->auth.data.offset,
 629                 sym_op->auth.data.length,
 630                 sess->digest_length,
 631                 sym_op->cipher.data.offset,
 632                 sym_op->cipher.data.length,
 633                 sess->iv.length,
 634                 sym_op->m_src->data_off);
 635
 636         /* Configure Output FLE with Scatter/Gather Entry */
 637         DPAA2_SET_FLE_SG_EXT(op_fle);
 638         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
 639
 640         if (auth_only_len)
 641                 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
 642
 643         op_fle->length = (sess->dir == DIR_ENC) ?
 644                         (sym_op->cipher.data.length + icv_len) :
 645                         sym_op->cipher.data.length;
 646
 647         /* Configure Output SGE for Encap/Decap */
 648         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 649         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
 650         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 651
 652         mbuf = mbuf->next;
 653         /* o/p segs */
 654         while (mbuf) {
 655                 sge++;
 656                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 657                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 658                 sge->length = mbuf->data_len;
 659                 mbuf = mbuf->next;
 660         }
 661         sge->length -= icv_len;
 662
 663         if (sess->dir == DIR_ENC) {
 664                 sge++;
 665                 DPAA2_SET_FLE_ADDR(sge,
 666                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 667                 sge->length = icv_len;
 668         }
 669         DPAA2_SET_FLE_FIN(sge);
 670
 671         sge++;
 672         mbuf = sym_op->m_src;
 673
 674         /* Configure Input FLE with Scatter/Gather Entry */
 675         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 676         DPAA2_SET_FLE_SG_EXT(ip_fle);
 677         DPAA2_SET_FLE_FIN(ip_fle);
 678         ip_fle->length = (sess->dir == DIR_ENC) ?
 679                         (sym_op->auth.data.length + sess->iv.length) :
 680                         (sym_op->auth.data.length + sess->iv.length +
 681                          icv_len);
 682
 683         /* Configure Input SGE for Encap/Decap */
 684         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 685         sge->length = sess->iv.length;
 686
 687         sge++;
 688         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 689         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 690                                 mbuf->data_off);
 691         sge->length = mbuf->data_len - sym_op->auth.data.offset;
 692
 693         mbuf = mbuf->next;
 694         /* i/p segs */
 695         while (mbuf) {
 696                 sge++;
 697                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 698                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 699                 sge->length = mbuf->data_len;
 700                 mbuf = mbuf->next;
 701         }
 702         sge->length -= icv_len;
 703
 704         if (sess->dir == DIR_DEC) {
 705                 sge++;
 706                 old_icv = (uint8_t *)(sge + 1);
 707                 memcpy(old_icv, sym_op->auth.digest.data,
 708                        icv_len);
 709                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 710                 sge->length = icv_len;
 711         }
 712
 713         DPAA2_SET_FLE_FIN(sge);
 714         if (auth_only_len) {
 715                 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
 716                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 717         }
 718         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 719
 720         return 0;
 721 }
 722
 723 static inline int
 724 build_authenc_fd(dpaa2_sec_session *sess,
 725                  struct rte_crypto_op *op,
 726                  struct qbman_fd *fd, uint16_t bpid)
 727 {
 728         struct rte_crypto_sym_op *sym_op = op->sym;
 729         struct ctxt_priv *priv = sess->ctxt;
 730         struct qbman_fle *fle, *sge;
 731         struct sec_flow_context *flc;
 732         uint16_t auth_hdr_len = sym_op->cipher.data.offset -
 733                                 sym_op->auth.data.offset;
 734         uint16_t auth_tail_len = sym_op->auth.data.length -
 735                                 sym_op->cipher.data.length - auth_hdr_len;
 736         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 737
 738         int icv_len = sess->digest_length, retval;
 739         uint8_t *old_icv;
 740         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 741                         sess->iv.offset);
 742         struct rte_mbuf *dst;
 743
 744         if (sym_op->m_dst)
 745                 dst = sym_op->m_dst;
 746         else
 747                 dst = sym_op->m_src;
 748
 749         /* we are using the first FLE entry to store Mbuf.
 750          * Currently we donot know which FLE has the mbuf stored.
 751          * So while retreiving we can go back 1 FLE from the FD -ADDR
 752          * to get the MBUF Addr from the previous FLE.
 753          * We can have a better approach to use the inline Mbuf
 754          */
 755         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
 756         if (retval) {
 757                 DPAA2_SEC_ERR("Memory alloc failed for SGE");
 758                 return -ENOMEM;
 759         }
 760         memset(fle, 0, FLE_POOL_BUF_SIZE);
 761         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 762         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 763         fle = fle + 1;
 764         sge = fle + 2;
 765         if (likely(bpid < MAX_BPID)) {
 766                 DPAA2_SET_FD_BPID(fd, bpid);
 767                 DPAA2_SET_FLE_BPID(fle, bpid);
 768                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
 769                 DPAA2_SET_FLE_BPID(sge, bpid);
 770                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
 771                 DPAA2_SET_FLE_BPID(sge + 2, bpid);
 772                 DPAA2_SET_FLE_BPID(sge + 3, bpid);
 773         } else {
 774                 DPAA2_SET_FD_IVP(fd);
 775                 DPAA2_SET_FLE_IVP(fle);
 776                 DPAA2_SET_FLE_IVP((fle + 1));
 777                 DPAA2_SET_FLE_IVP(sge);
 778                 DPAA2_SET_FLE_IVP((sge + 1));
 779                 DPAA2_SET_FLE_IVP((sge + 2));
 780                 DPAA2_SET_FLE_IVP((sge + 3));
 781         }
 782
 783         /* Save the shared descriptor */
 784         flc = &priv->flc_desc[0].flc;
 785         /* Configure FD as a FRAME LIST */
 786         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 787         DPAA2_SET_FD_COMPOUND_FMT(fd);
 788         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 789
 790         DPAA2_SEC_DP_DEBUG(
 791                 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
 792                 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
 793                 sym_op->auth.data.offset,
 794                 sym_op->auth.data.length,
 795                 sess->digest_length,
 796                 sym_op->cipher.data.offset,
 797                 sym_op->cipher.data.length,
 798                 sess->iv.length,
 799                 sym_op->m_src->data_off);
 800
 801         /* Configure Output FLE with Scatter/Gather Entry */
 802         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 803         if (auth_only_len)
 804                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 805         fle->length = (sess->dir == DIR_ENC) ?
 806                         (sym_op->cipher.data.length + icv_len) :
 807                         sym_op->cipher.data.length;
 808
 809         DPAA2_SET_FLE_SG_EXT(fle);
 810
 811         /* Configure Output SGE for Encap/Decap */
 812         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
 813         DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
 814                                 dst->data_off);
 815         sge->length = sym_op->cipher.data.length;
 816
 817         if (sess->dir == DIR_ENC) {
 818                 sge++;
 819                 DPAA2_SET_FLE_ADDR(sge,
 820                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 821                 sge->length = sess->digest_length;
 822                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 823                                         sess->iv.length));
 824         }
 825         DPAA2_SET_FLE_FIN(sge);
 826
 827         sge++;
 828         fle++;
 829
 830         /* Configure Input FLE with Scatter/Gather Entry */
 831         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
 832         DPAA2_SET_FLE_SG_EXT(fle);
 833         DPAA2_SET_FLE_FIN(fle);
 834         fle->length = (sess->dir == DIR_ENC) ?
 835                         (sym_op->auth.data.length + sess->iv.length) :
 836                         (sym_op->auth.data.length + sess->iv.length +
 837                          sess->digest_length);
 838
 839         /* Configure Input SGE for Encap/Decap */
 840         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 841         sge->length = sess->iv.length;
 842         sge++;
 843
 844         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
 845         DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
 846                                 sym_op->m_src->data_off);
 847         sge->length = sym_op->auth.data.length;
 848         if (sess->dir == DIR_DEC) {
 849                 sge++;
 850                 old_icv = (uint8_t *)(sge + 1);
 851                 memcpy(old_icv, sym_op->auth.digest.data,
 852                        sess->digest_length);
 853                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
 854                 sge->length = sess->digest_length;
 855                 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
 856                                  sess->digest_length +
 857                                  sess->iv.length));
 858         }
 859         DPAA2_SET_FLE_FIN(sge);
 860         if (auth_only_len) {
 861                 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
 862                 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
 863         }
 864         return 0;
 865 }
 866
 867 static inline int build_auth_sg_fd(
 868                 dpaa2_sec_session *sess,
 869                 struct rte_crypto_op *op,
 870                 struct qbman_fd *fd,
 871                 __rte_unused uint16_t bpid)
 872 {
 873         struct rte_crypto_sym_op *sym_op = op->sym;
 874         struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
 875         struct sec_flow_context *flc;
 876         struct ctxt_priv *priv = sess->ctxt;
 877         int data_len, data_offset;
 878         uint8_t *old_digest;
 879         struct rte_mbuf *mbuf;
 880
 881         data_len = sym_op->auth.data.length;
 882         data_offset = sym_op->auth.data.offset;
 883
 884         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 885             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 886                 if ((data_len & 7) || (data_offset & 7)) {
 887                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
 888                         return -ENOTSUP;
 889                 }
 890
 891                 data_len = data_len >> 3;
 892                 data_offset = data_offset >> 3;
 893         }
 894
 895         mbuf = sym_op->m_src;
 896         fle = (struct qbman_fle *)rte_malloc(NULL,
 897                         FLE_SG_MEM_SIZE(mbuf->nb_segs),
 898                         RTE_CACHE_LINE_SIZE);
 899         if (unlikely(!fle)) {
 900                 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
 901                 return -ENOMEM;
 902         }
 903         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
 904         /* first FLE entry used to store mbuf and session ctxt */
 905         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
 906         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
 907         op_fle = fle + 1;
 908         ip_fle = fle + 2;
 909         sge = fle + 3;
 910
 911         flc = &priv->flc_desc[DESC_INITFINAL].flc;
 912         /* sg FD */
 913         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
 914         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
 915         DPAA2_SET_FD_COMPOUND_FMT(fd);
 916
 917         /* o/p fle */
 918         DPAA2_SET_FLE_ADDR(op_fle,
 919                                 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
 920         op_fle->length = sess->digest_length;
 921
 922         /* i/p fle */
 923         DPAA2_SET_FLE_SG_EXT(ip_fle);
 924         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
 925         ip_fle->length = data_len;
 926
 927         if (sess->iv.length) {
 928                 uint8_t *iv_ptr;
 929
 930                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 931                                                    sess->iv.offset);
 932
 933                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 934                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 935                         sge->length = 12;
 936                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 937                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 938                         sge->length = 8;
 939                 } else {
 940                         sge->length = sess->iv.length;
 941                 }
 942                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
 943                 ip_fle->length += sge->length;
 944                 sge++;
 945         }
 946         /* i/p 1st seg */
 947         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 948         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
 949
 950         if (data_len <= (mbuf->data_len - data_offset)) {
 951                 sge->length = data_len;
 952                 data_len = 0;
 953         } else {
 954                 sge->length = mbuf->data_len - data_offset;
 955
 956                 /* remaining i/p segs */
 957                 while ((data_len = data_len - sge->length) &&
 958                        (mbuf = mbuf->next)) {
 959                         sge++;
 960                         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 961                         DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
 962                         if (data_len > mbuf->data_len)
 963                                 sge->length = mbuf->data_len;
 964                         else
 965                                 sge->length = data_len;
 966                 }
 967         }
 968
 969         if (sess->dir == DIR_DEC) {
 970                 /* Digest verification case */
 971                 sge++;
 972                 old_digest = (uint8_t *)(sge + 1);
 973                 rte_memcpy(old_digest, sym_op->auth.digest.data,
 974                            sess->digest_length);
 975                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
 976                 sge->length = sess->digest_length;
 977                 ip_fle->length += sess->digest_length;
 978         }
 979         DPAA2_SET_FLE_FIN(sge);
 980         DPAA2_SET_FLE_FIN(ip_fle);
 981         DPAA2_SET_FD_LEN(fd, ip_fle->length);
 982
 983         return 0;
 984 }
 985
 986 static inline int
 987 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
 988               struct qbman_fd *fd, uint16_t bpid)
 989 {
 990         struct rte_crypto_sym_op *sym_op = op->sym;
 991         struct qbman_fle *fle, *sge;
 992         struct sec_flow_context *flc;
 993         struct ctxt_priv *priv = sess->ctxt;
 994         int data_len, data_offset;
 995         uint8_t *old_digest;
 996         int retval;
 997
 998         data_len = sym_op->auth.data.length;
 999         data_offset = sym_op->auth.data.offset;
1000
1001         if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1002             sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1003                 if ((data_len & 7) || (data_offset & 7)) {
1004                         DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1005                         return -ENOTSUP;
1006                 }
1007
1008                 data_len = data_len >> 3;
1009                 data_offset = data_offset >> 3;
1010         }
1011
1012         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1013         if (retval) {
1014                 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE");
1015                 return -ENOMEM;
1016         }
1017         memset(fle, 0, FLE_POOL_BUF_SIZE);
1018         /* TODO we are using the first FLE entry to store Mbuf.
1019          * Currently we donot know which FLE has the mbuf stored.
1020          * So while retreiving we can go back 1 FLE from the FD -ADDR
1021          * to get the MBUF Addr from the previous FLE.
1022          * We can have a better approach to use the inline Mbuf
1023          */
1024         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1025         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1026         fle = fle + 1;
1027         sge = fle + 2;
1028
1029         if (likely(bpid < MAX_BPID)) {
1030                 DPAA2_SET_FD_BPID(fd, bpid);
1031                 DPAA2_SET_FLE_BPID(fle, bpid);
1032                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1033                 DPAA2_SET_FLE_BPID(sge, bpid);
1034                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1035         } else {
1036                 DPAA2_SET_FD_IVP(fd);
1037                 DPAA2_SET_FLE_IVP(fle);
1038                 DPAA2_SET_FLE_IVP((fle + 1));
1039                 DPAA2_SET_FLE_IVP(sge);
1040                 DPAA2_SET_FLE_IVP((sge + 1));
1041         }
1042
1043         flc = &priv->flc_desc[DESC_INITFINAL].flc;
1044         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1045         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1046         DPAA2_SET_FD_COMPOUND_FMT(fd);
1047
1048         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1049         fle->length = sess->digest_length;
1050         fle++;
1051
1052         /* Setting input FLE */
1053         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1054         DPAA2_SET_FLE_SG_EXT(fle);
1055         fle->length = data_len;
1056
1057         if (sess->iv.length) {
1058                 uint8_t *iv_ptr;
1059
1060                 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1061                                                    sess->iv.offset);
1062
1063                 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1064                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1065                         sge->length = 12;
1066                 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1067                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1068                         sge->length = 8;
1069                 } else {
1070                         sge->length = sess->iv.length;
1071                 }
1072
1073                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1074                 fle->length = fle->length + sge->length;
1075                 sge++;
1076         }
1077
1078         /* Setting data to authenticate */
1079         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1080         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1081         sge->length = data_len;
1082
1083         if (sess->dir == DIR_DEC) {
1084                 sge++;
1085                 old_digest = (uint8_t *)(sge + 1);
1086                 rte_memcpy(old_digest, sym_op->auth.digest.data,
1087                            sess->digest_length);
1088                 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1089                 sge->length = sess->digest_length;
1090                 fle->length = fle->length + sess->digest_length;
1091         }
1092
1093         DPAA2_SET_FLE_FIN(sge);
1094         DPAA2_SET_FLE_FIN(fle);
1095         DPAA2_SET_FD_LEN(fd, fle->length);
1096
1097         return 0;
1098 }
1099
1100 static int
1101 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1102                 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1103 {
1104         struct rte_crypto_sym_op *sym_op = op->sym;
1105         struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1106         int data_len, data_offset;
1107         struct sec_flow_context *flc;
1108         struct ctxt_priv *priv = sess->ctxt;
1109         struct rte_mbuf *mbuf;
1110         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1111                         sess->iv.offset);
1112
1113         data_len = sym_op->cipher.data.length;
1114         data_offset = sym_op->cipher.data.offset;
1115
1116         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1117                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1118                 if ((data_len & 7) || (data_offset & 7)) {
1119                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1120                         return -ENOTSUP;
1121                 }
1122
1123                 data_len = data_len >> 3;
1124                 data_offset = data_offset >> 3;
1125         }
1126
1127         if (sym_op->m_dst)
1128                 mbuf = sym_op->m_dst;
1129         else
1130                 mbuf = sym_op->m_src;
1131
1132         /* first FLE entry used to store mbuf and session ctxt */
1133         fle = (struct qbman_fle *)rte_malloc(NULL,
1134                         FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1135                         RTE_CACHE_LINE_SIZE);
1136         if (!fle) {
1137                 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1138                 return -ENOMEM;
1139         }
1140         memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1141         /* first FLE entry used to store mbuf and session ctxt */
1142         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1143         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1144
1145         op_fle = fle + 1;
1146         ip_fle = fle + 2;
1147         sge = fle + 3;
1148
1149         flc = &priv->flc_desc[0].flc;
1150
1151         DPAA2_SEC_DP_DEBUG(
1152                 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1153                 " data_off: 0x%x\n",
1154                 data_offset,
1155                 data_len,
1156                 sess->iv.length,
1157                 sym_op->m_src->data_off);
1158
1159         /* o/p fle */
1160         DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1161         op_fle->length = data_len;
1162         DPAA2_SET_FLE_SG_EXT(op_fle);
1163
1164         /* o/p 1st seg */
1165         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1166         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1167         sge->length = mbuf->data_len - data_offset;
1168
1169         mbuf = mbuf->next;
1170         /* o/p segs */
1171         while (mbuf) {
1172                 sge++;
1173                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1174                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1175                 sge->length = mbuf->data_len;
1176                 mbuf = mbuf->next;
1177         }
1178         DPAA2_SET_FLE_FIN(sge);
1179
1180         DPAA2_SEC_DP_DEBUG(
1181                 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1182                 flc, fle, fle->addr_hi, fle->addr_lo,
1183                 fle->length);
1184
1185         /* i/p fle */
1186         mbuf = sym_op->m_src;
1187         sge++;
1188         DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1189         ip_fle->length = sess->iv.length + data_len;
1190         DPAA2_SET_FLE_SG_EXT(ip_fle);
1191
1192         /* i/p IV */
1193         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1194         DPAA2_SET_FLE_OFFSET(sge, 0);
1195         sge->length = sess->iv.length;
1196
1197         sge++;
1198
1199         /* i/p 1st seg */
1200         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1201         DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1202         sge->length = mbuf->data_len - data_offset;
1203
1204         mbuf = mbuf->next;
1205         /* i/p segs */
1206         while (mbuf) {
1207                 sge++;
1208                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1209                 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1210                 sge->length = mbuf->data_len;
1211                 mbuf = mbuf->next;
1212         }
1213         DPAA2_SET_FLE_FIN(sge);
1214         DPAA2_SET_FLE_FIN(ip_fle);
1215
1216         /* sg fd */
1217         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1218         DPAA2_SET_FD_LEN(fd, ip_fle->length);
1219         DPAA2_SET_FD_COMPOUND_FMT(fd);
1220         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1221
1222         DPAA2_SEC_DP_DEBUG(
1223                 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1224                 " off =%d, len =%d\n",
1225                 DPAA2_GET_FD_ADDR(fd),
1226                 DPAA2_GET_FD_BPID(fd),
1227                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1228                 DPAA2_GET_FD_OFFSET(fd),
1229                 DPAA2_GET_FD_LEN(fd));
1230         return 0;
1231 }
1232
1233 static int
1234 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1235                 struct qbman_fd *fd, uint16_t bpid)
1236 {
1237         struct rte_crypto_sym_op *sym_op = op->sym;
1238         struct qbman_fle *fle, *sge;
1239         int retval, data_len, data_offset;
1240         struct sec_flow_context *flc;
1241         struct ctxt_priv *priv = sess->ctxt;
1242         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1243                         sess->iv.offset);
1244         struct rte_mbuf *dst;
1245
1246         data_len = sym_op->cipher.data.length;
1247         data_offset = sym_op->cipher.data.offset;
1248
1249         if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1250                 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1251                 if ((data_len & 7) || (data_offset & 7)) {
1252                         DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1253                         return -ENOTSUP;
1254                 }
1255
1256                 data_len = data_len >> 3;
1257                 data_offset = data_offset >> 3;
1258         }
1259
1260         if (sym_op->m_dst)
1261                 dst = sym_op->m_dst;
1262         else
1263                 dst = sym_op->m_src;
1264
1265         retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1266         if (retval) {
1267                 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE");
1268                 return -ENOMEM;
1269         }
1270         memset(fle, 0, FLE_POOL_BUF_SIZE);
1271         /* TODO we are using the first FLE entry to store Mbuf.
1272          * Currently we donot know which FLE has the mbuf stored.
1273          * So while retreiving we can go back 1 FLE from the FD -ADDR
1274          * to get the MBUF Addr from the previous FLE.
1275          * We can have a better approach to use the inline Mbuf
1276          */
1277         DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1278         DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1279         fle = fle + 1;
1280         sge = fle + 2;
1281
1282         if (likely(bpid < MAX_BPID)) {
1283                 DPAA2_SET_FD_BPID(fd, bpid);
1284                 DPAA2_SET_FLE_BPID(fle, bpid);
1285                 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1286                 DPAA2_SET_FLE_BPID(sge, bpid);
1287                 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1288         } else {
1289                 DPAA2_SET_FD_IVP(fd);
1290                 DPAA2_SET_FLE_IVP(fle);
1291                 DPAA2_SET_FLE_IVP((fle + 1));
1292                 DPAA2_SET_FLE_IVP(sge);
1293                 DPAA2_SET_FLE_IVP((sge + 1));
1294         }
1295
1296         flc = &priv->flc_desc[0].flc;
1297         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1298         DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1299         DPAA2_SET_FD_COMPOUND_FMT(fd);
1300         DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1301
1302         DPAA2_SEC_DP_DEBUG(
1303                 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1304                 " data_off: 0x%x\n",
1305                 data_offset,
1306                 data_len,
1307                 sess->iv.length,
1308                 sym_op->m_src->data_off);
1309
1310         DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1311         DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1312
1313         fle->length = data_len + sess->iv.length;
1314
1315         DPAA2_SEC_DP_DEBUG(
1316                 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1317                 flc, fle, fle->addr_hi, fle->addr_lo,
1318                 fle->length);
1319
1320         fle++;
1321
1322         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1323         fle->length = data_len + sess->iv.length;
1324
1325         DPAA2_SET_FLE_SG_EXT(fle);
1326
1327         DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1328         sge->length = sess->iv.length;
1329
1330         sge++;
1331         DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1332         DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1333
1334         sge->length = data_len;
1335         DPAA2_SET_FLE_FIN(sge);
1336         DPAA2_SET_FLE_FIN(fle);
1337
1338         DPAA2_SEC_DP_DEBUG(
1339                 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1340                 " off =%d, len =%d\n",
1341                 DPAA2_GET_FD_ADDR(fd),
1342                 DPAA2_GET_FD_BPID(fd),
1343                 rte_dpaa2_bpid_info[bpid].meta_data_size,
1344                 DPAA2_GET_FD_OFFSET(fd),
1345                 DPAA2_GET_FD_LEN(fd));
1346
1347         return 0;
1348 }
1349
1350 static inline int
1351 build_sec_fd(struct rte_crypto_op *op,
1352              struct qbman_fd *fd, uint16_t bpid)
1353 {
1354         int ret = -1;
1355         dpaa2_sec_session *sess;
1356
1357         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1358                 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1359                                 op->sym->session, cryptodev_driver_id);
1360 #ifdef RTE_LIBRTE_SECURITY
1361         else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1362                 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1363                                 op->sym->sec_session);
1364 #endif
1365         else
1366                 return -ENOTSUP;
1367
1368         if (!sess)
1369                 return -EINVAL;
1370
1371         /* Any of the buffer is segmented*/
1372         if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1373                   ((op->sym->m_dst != NULL) &&
1374                    !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1375                 switch (sess->ctxt_type) {
1376                 case DPAA2_SEC_CIPHER:
1377                         ret = build_cipher_sg_fd(sess, op, fd, bpid);
1378                         break;
1379                 case DPAA2_SEC_AUTH:
1380                         ret = build_auth_sg_fd(sess, op, fd, bpid);
1381                         break;
1382                 case DPAA2_SEC_AEAD:
1383                         ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1384                         break;
1385                 case DPAA2_SEC_CIPHER_HASH:
1386                         ret = build_authenc_sg_fd(sess, op, fd, bpid);
1387                         break;
1388 #ifdef RTE_LIBRTE_SECURITY
1389                 case DPAA2_SEC_IPSEC:
1390                 case DPAA2_SEC_PDCP:
1391                         ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1392                         break;
1393 #endif
1394                 case DPAA2_SEC_HASH_CIPHER:
1395                 default:
1396                         DPAA2_SEC_ERR("error: Unsupported session");
1397                 }
1398         } else {
1399                 switch (sess->ctxt_type) {
1400                 case DPAA2_SEC_CIPHER:
1401                         ret = build_cipher_fd(sess, op, fd, bpid);
1402                         break;
1403                 case DPAA2_SEC_AUTH:
1404                         ret = build_auth_fd(sess, op, fd, bpid);
1405                         break;
1406                 case DPAA2_SEC_AEAD:
1407                         ret = build_authenc_gcm_fd(sess, op, fd, bpid);
1408                         break;
1409                 case DPAA2_SEC_CIPHER_HASH:
1410                         ret = build_authenc_fd(sess, op, fd, bpid);
1411                         break;
1412 #ifdef RTE_LIBRTE_SECURITY
1413                 case DPAA2_SEC_IPSEC:
1414                         ret = build_proto_fd(sess, op, fd, bpid);
1415                         break;
1416                 case DPAA2_SEC_PDCP:
1417                         ret = build_proto_compound_fd(sess, op, fd, bpid);
1418                         break;
1419 #endif
1420                 case DPAA2_SEC_HASH_CIPHER:
1421                 default:
1422                         DPAA2_SEC_ERR("error: Unsupported session");
1423                         ret = -ENOTSUP;
1424                 }
1425         }
1426         return ret;
1427 }
1428
1429 static uint16_t
1430 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1431                         uint16_t nb_ops)
1432 {
1433         /* Function to transmit the frames to given device and VQ*/
1434         uint32_t loop;
1435         int32_t ret;
1436         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1437         uint32_t frames_to_send, retry_count;
1438         struct qbman_eq_desc eqdesc;
1439         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1440         struct qbman_swp *swp;
1441         uint16_t num_tx = 0;
1442         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1443         /*todo - need to support multiple buffer pools */
1444         uint16_t bpid;
1445         struct rte_mempool *mb_pool;
1446
1447         if (unlikely(nb_ops == 0))
1448                 return 0;
1449
1450         if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1451                 DPAA2_SEC_ERR("sessionless crypto op not supported");
1452                 return 0;
1453         }
1454         /*Prepare enqueue descriptor*/
1455         qbman_eq_desc_clear(&eqdesc);
1456         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1457         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1458         qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1459
1460         if (!DPAA2_PER_LCORE_DPIO) {
1461                 ret = dpaa2_affine_qbman_swp();
1462                 if (ret) {
1463                         DPAA2_SEC_ERR(
1464                                 "Failed to allocate IO portal, tid: %d\n",
1465                                 rte_gettid());
1466                         return 0;
1467                 }
1468         }
1469         swp = DPAA2_PER_LCORE_PORTAL;
1470
1471         while (nb_ops) {
1472                 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1473                         dpaa2_eqcr_size : nb_ops;
1474
1475                 for (loop = 0; loop < frames_to_send; loop++) {
1476                         if ((*ops)->sym->m_src->seqn) {
1477                          uint8_t dqrr_index = (*ops)->sym->m_src->seqn - 1;
1478
1479                          flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
1480                          DPAA2_PER_LCORE_DQRR_SIZE--;
1481                          DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1482                          (*ops)->sym->m_src->seqn = DPAA2_INVALID_MBUF_SEQN;
1483                         }
1484
1485                         /*Clear the unused FD fields before sending*/
1486                         memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1487                         mb_pool = (*ops)->sym->m_src->pool;
1488                         bpid = mempool_to_bpid(mb_pool);
1489                         ret = build_sec_fd(*ops, &fd_arr[loop], bpid);
1490                         if (ret) {
1491                                 DPAA2_SEC_ERR("error: Improper packet contents"
1492                                               " for crypto operation");
1493                                 goto skip_tx;
1494                         }
1495                         ops++;
1496                 }
1497
1498                 loop = 0;
1499                 retry_count = 0;
1500                 while (loop < frames_to_send) {
1501                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1502                                                          &fd_arr[loop],
1503                                                          &flags[loop],
1504                                                          frames_to_send - loop);
1505                         if (unlikely(ret < 0)) {
1506                                 retry_count++;
1507                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1508                                         num_tx += loop;
1509                                         nb_ops -= loop;
1510                                         goto skip_tx;
1511                                 }
1512                         } else {
1513                                 loop += ret;
1514                                 retry_count = 0;
1515                         }
1516                 }
1517
1518                 num_tx += loop;
1519                 nb_ops -= loop;
1520         }
1521 skip_tx:
1522         dpaa2_qp->tx_vq.tx_pkts += num_tx;
1523         dpaa2_qp->tx_vq.err_pkts += nb_ops;
1524         return num_tx;
1525 }
1526
1527 #ifdef RTE_LIBRTE_SECURITY
1528 static inline struct rte_crypto_op *
1529 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1530 {
1531         struct rte_crypto_op *op;
1532         uint16_t len = DPAA2_GET_FD_LEN(fd);
1533         int16_t diff = 0;
1534         dpaa2_sec_session *sess_priv __rte_unused;
1535
1536         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1537                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1538                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1539
1540         diff = len - mbuf->pkt_len;
1541         mbuf->pkt_len += diff;
1542         mbuf->data_len += diff;
1543         op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1544         mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1545         op->sym->aead.digest.phys_addr = 0L;
1546
1547         sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1548                                 op->sym->sec_session);
1549         if (sess_priv->dir == DIR_ENC)
1550                 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1551         else
1552                 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1553
1554         return op;
1555 }
1556 #endif
1557
1558 static inline struct rte_crypto_op *
1559 sec_fd_to_mbuf(const struct qbman_fd *fd)
1560 {
1561         struct qbman_fle *fle;
1562         struct rte_crypto_op *op;
1563         struct ctxt_priv *priv;
1564         struct rte_mbuf *dst, *src;
1565
1566 #ifdef RTE_LIBRTE_SECURITY
1567         if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1568                 return sec_simple_fd_to_mbuf(fd);
1569 #endif
1570         fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1571
1572         DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1573                            fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1574
1575         /* we are using the first FLE entry to store Mbuf.
1576          * Currently we donot know which FLE has the mbuf stored.
1577          * So while retreiving we can go back 1 FLE from the FD -ADDR
1578          * to get the MBUF Addr from the previous FLE.
1579          * We can have a better approach to use the inline Mbuf
1580          */
1581
1582         if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1583                 /* TODO complete it. */
1584                 DPAA2_SEC_ERR("error: non inline buffer");
1585                 return NULL;
1586         }
1587         op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1588
1589         /* Prefeth op */
1590         src = op->sym->m_src;
1591         rte_prefetch0(src);
1592
1593         if (op->sym->m_dst) {
1594                 dst = op->sym->m_dst;
1595                 rte_prefetch0(dst);
1596         } else
1597                 dst = src;
1598
1599 #ifdef RTE_LIBRTE_SECURITY
1600         if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1601                 uint16_t len = DPAA2_GET_FD_LEN(fd);
1602                 dst->pkt_len = len;
1603                 while (dst->next != NULL) {
1604                         len -= dst->data_len;
1605                         dst = dst->next;
1606                 }
1607                 dst->data_len = len;
1608         }
1609 #endif
1610         DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1611                 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1612                 (void *)dst,
1613                 dst->buf_addr,
1614                 DPAA2_GET_FD_ADDR(fd),
1615                 DPAA2_GET_FD_BPID(fd),
1616                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1617                 DPAA2_GET_FD_OFFSET(fd),
1618                 DPAA2_GET_FD_LEN(fd));
1619
1620         /* free the fle memory */
1621         if (likely(rte_pktmbuf_is_contiguous(src))) {
1622                 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1);
1623                 rte_mempool_put(priv->fle_pool, (void *)(fle-1));
1624         } else
1625                 rte_free((void *)(fle-1));
1626
1627         return op;
1628 }
1629
1630 static uint16_t
1631 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1632                         uint16_t nb_ops)
1633 {
1634         /* Function is responsible to receive frames for a given device and VQ*/
1635         struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1636         struct qbman_result *dq_storage;
1637         uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1638         int ret, num_rx = 0;
1639         uint8_t is_last = 0, status;
1640         struct qbman_swp *swp;
1641         const struct qbman_fd *fd;
1642         struct qbman_pull_desc pulldesc;
1643
1644         if (!DPAA2_PER_LCORE_DPIO) {
1645                 ret = dpaa2_affine_qbman_swp();
1646                 if (ret) {
1647                         DPAA2_SEC_ERR(
1648                                 "Failed to allocate IO portal, tid: %d\n",
1649                                 rte_gettid());
1650                         return 0;
1651                 }
1652         }
1653         swp = DPAA2_PER_LCORE_PORTAL;
1654         dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1655
1656         qbman_pull_desc_clear(&pulldesc);
1657         qbman_pull_desc_set_numframes(&pulldesc,
1658                                       (nb_ops > dpaa2_dqrr_size) ?
1659                                       dpaa2_dqrr_size : nb_ops);
1660         qbman_pull_desc_set_fq(&pulldesc, fqid);
1661         qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1662                                     (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1663                                     1);
1664
1665         /*Issue a volatile dequeue command. */
1666         while (1) {
1667                 if (qbman_swp_pull(swp, &pulldesc)) {
1668                         DPAA2_SEC_WARN(
1669                                 "SEC VDQ command is not issued : QBMAN busy");
1670                         /* Portal was busy, try again */
1671                         continue;
1672                 }
1673                 break;
1674         };
1675
1676         /* Receive the packets till Last Dequeue entry is found with
1677          * respect to the above issues PULL command.
1678          */
1679         while (!is_last) {
1680                 /* Check if the previous issued command is completed.
1681                  * Also seems like the SWP is shared between the Ethernet Driver
1682                  * and the SEC driver.
1683                  */
1684                 while (!qbman_check_command_complete(dq_storage))
1685                         ;
1686
1687                 /* Loop until the dq_storage is updated with
1688                  * new token by QBMAN
1689                  */
1690                 while (!qbman_check_new_result(dq_storage))
1691                         ;
1692                 /* Check whether Last Pull command is Expired and
1693                  * setting Condition for Loop termination
1694                  */
1695                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1696                         is_last = 1;
1697                         /* Check for valid frame. */
1698                         status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1699                         if (unlikely(
1700                                 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1701                                 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1702                                 continue;
1703                         }
1704                 }
1705
1706                 fd = qbman_result_DQ_fd(dq_storage);
1707                 ops[num_rx] = sec_fd_to_mbuf(fd);
1708
1709                 if (unlikely(fd->simple.frc)) {
1710                         /* TODO Parse SEC errors */
1711                         DPAA2_SEC_ERR("SEC returned Error - %x",
1712                                       fd->simple.frc);
1713                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
1714                 } else {
1715                         ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1716                 }
1717
1718                 num_rx++;
1719                 dq_storage++;
1720         } /* End of Packet Rx loop */
1721
1722         dpaa2_qp->rx_vq.rx_pkts += num_rx;
1723
1724         DPAA2_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1725         /*Return the total number of packets received to DPAA2 app*/
1726         return num_rx;
1727 }
1728
1729 /** Release queue pair */
1730 static int
1731 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
1732 {
1733         struct dpaa2_sec_qp *qp =
1734                 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
1735
1736         PMD_INIT_FUNC_TRACE();
1737
1738         if (qp->rx_vq.q_storage) {
1739                 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
1740                 rte_free(qp->rx_vq.q_storage);
1741         }
1742         rte_free(qp);
1743
1744         dev->data->queue_pairs[queue_pair_id] = NULL;
1745
1746         return 0;
1747 }
1748
1749 /** Setup a queue pair */
1750 static int
1751 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1752                 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1753                 __rte_unused int socket_id)
1754 {
1755         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
1756         struct dpaa2_sec_qp *qp;
1757         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
1758         struct dpseci_rx_queue_cfg cfg;
1759         int32_t retcode;
1760
1761         PMD_INIT_FUNC_TRACE();
1762
1763         /* If qp is already in use free ring memory and qp metadata. */
1764         if (dev->data->queue_pairs[qp_id] != NULL) {
1765                 DPAA2_SEC_INFO("QP already setup");
1766                 return 0;
1767         }
1768
1769         DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
1770                     dev, qp_id, qp_conf);
1771
1772         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
1773
1774         qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
1775                         RTE_CACHE_LINE_SIZE);
1776         if (!qp) {
1777                 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
1778                 return -ENOMEM;
1779         }
1780
1781         qp->rx_vq.crypto_data = dev->data;
1782         qp->tx_vq.crypto_data = dev->data;
1783         qp->rx_vq.q_storage = rte_malloc("sec dq storage",
1784                 sizeof(struct queue_storage_info_t),
1785                 RTE_CACHE_LINE_SIZE);
1786         if (!qp->rx_vq.q_storage) {
1787                 DPAA2_SEC_ERR("malloc failed for q_storage");
1788                 return -ENOMEM;
1789         }
1790         memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
1791
1792         if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
1793                 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
1794                 return -ENOMEM;
1795         }
1796
1797         dev->data->queue_pairs[qp_id] = qp;
1798
1799         cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
1800         cfg.user_ctx = (size_t)(&qp->rx_vq);
1801         retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
1802                                       qp_id, &cfg);
1803         return retcode;
1804 }
1805
1806 /** Returns the size of the aesni gcm session structure */
1807 static unsigned int
1808 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1809 {
1810         PMD_INIT_FUNC_TRACE();
1811
1812         return sizeof(dpaa2_sec_session);
1813 }
1814
1815 static int
1816 dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
1817                       struct rte_crypto_sym_xform *xform,
1818                       dpaa2_sec_session *session)
1819 {
1820         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1821         struct alginfo cipherdata;
1822         int bufsize, ret = 0;
1823         struct ctxt_priv *priv;
1824         struct sec_flow_context *flc;
1825
1826         PMD_INIT_FUNC_TRACE();
1827
1828         /* For SEC CIPHER only one descriptor is required. */
1829         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1830                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
1831                         RTE_CACHE_LINE_SIZE);
1832         if (priv == NULL) {
1833                 DPAA2_SEC_ERR("No Memory for priv CTXT");
1834                 return -ENOMEM;
1835         }
1836
1837         priv->fle_pool = dev_priv->fle_pool;
1838
1839         flc = &priv->flc_desc[0].flc;
1840
1841         session->ctxt_type = DPAA2_SEC_CIPHER;
1842         session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1843                         RTE_CACHE_LINE_SIZE);
1844         if (session->cipher_key.data == NULL) {
1845                 DPAA2_SEC_ERR("No Memory for cipher key");
1846                 rte_free(priv);
1847                 return -ENOMEM;
1848         }
1849         session->cipher_key.length = xform->cipher.key.length;
1850
1851         memcpy(session->cipher_key.data, xform->cipher.key.data,
1852                xform->cipher.key.length);
1853         cipherdata.key = (size_t)session->cipher_key.data;
1854         cipherdata.keylen = session->cipher_key.length;
1855         cipherdata.key_enc_flags = 0;
1856         cipherdata.key_type = RTA_DATA_IMM;
1857
1858         /* Set IV parameters */
1859         session->iv.offset = xform->cipher.iv.offset;
1860         session->iv.length = xform->cipher.iv.length;
1861         session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1862                                 DIR_ENC : DIR_DEC;
1863
1864         switch (xform->cipher.algo) {
1865         case RTE_CRYPTO_CIPHER_AES_CBC:
1866                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1867                 cipherdata.algmode = OP_ALG_AAI_CBC;
1868                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1869                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1870                                                 SHR_NEVER, &cipherdata,
1871                                                 session->iv.length,
1872                                                 session->dir);
1873                 break;
1874         case RTE_CRYPTO_CIPHER_3DES_CBC:
1875                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1876                 cipherdata.algmode = OP_ALG_AAI_CBC;
1877                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1878                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1879                                                 SHR_NEVER, &cipherdata,
1880                                                 session->iv.length,
1881                                                 session->dir);
1882                 break;
1883         case RTE_CRYPTO_CIPHER_AES_CTR:
1884                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1885                 cipherdata.algmode = OP_ALG_AAI_CTR;
1886                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1887                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1888                                                 SHR_NEVER, &cipherdata,
1889                                                 session->iv.length,
1890                                                 session->dir);
1891                 break;
1892         case RTE_CRYPTO_CIPHER_3DES_CTR:
1893                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1894                 cipherdata.algmode = OP_ALG_AAI_CTR;
1895                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CTR;
1896                 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1897                                                 SHR_NEVER, &cipherdata,
1898                                                 session->iv.length,
1899                                                 session->dir);
1900                 break;
1901         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1902                 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
1903                 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
1904                 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
1905                                               &cipherdata,
1906                                               session->dir);
1907                 break;
1908         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
1909                 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
1910                 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
1911                 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
1912                                               &cipherdata,
1913                                               session->dir);
1914                 break;
1915         case RTE_CRYPTO_CIPHER_KASUMI_F8:
1916         case RTE_CRYPTO_CIPHER_AES_F8:
1917         case RTE_CRYPTO_CIPHER_AES_ECB:
1918         case RTE_CRYPTO_CIPHER_3DES_ECB:
1919         case RTE_CRYPTO_CIPHER_AES_XTS:
1920         case RTE_CRYPTO_CIPHER_ARC4:
1921         case RTE_CRYPTO_CIPHER_NULL:
1922                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
1923                         xform->cipher.algo);
1924                 ret = -ENOTSUP;
1925                 goto error_out;
1926         default:
1927                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
1928                         xform->cipher.algo);
1929                 ret = -ENOTSUP;
1930                 goto error_out;
1931         }
1932
1933         if (bufsize < 0) {
1934                 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
1935                 ret = -EINVAL;
1936                 goto error_out;
1937         }
1938
1939         flc->word1_sdl = (uint8_t)bufsize;
1940         session->ctxt = priv;
1941
1942 #ifdef CAAM_DESC_DEBUG
1943         int i;
1944         for (i = 0; i < bufsize; i++)
1945                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
1946 #endif
1947         return ret;
1948
1949 error_out:
1950         rte_free(session->cipher_key.data);
1951         rte_free(priv);
1952         return ret;
1953 }
1954
1955 static int
1956 dpaa2_sec_auth_init(struct rte_cryptodev *dev,
1957                     struct rte_crypto_sym_xform *xform,
1958                     dpaa2_sec_session *session)
1959 {
1960         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1961         struct alginfo authdata;
1962         int bufsize, ret = 0;
1963         struct ctxt_priv *priv;
1964         struct sec_flow_context *flc;
1965
1966         PMD_INIT_FUNC_TRACE();
1967
1968         /* For SEC AUTH three descriptors are required for various stages */
1969         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1970                         sizeof(struct ctxt_priv) + 3 *
1971                         sizeof(struct sec_flc_desc),
1972                         RTE_CACHE_LINE_SIZE);
1973         if (priv == NULL) {
1974                 DPAA2_SEC_ERR("No Memory for priv CTXT");
1975                 return -ENOMEM;
1976         }
1977
1978         priv->fle_pool = dev_priv->fle_pool;
1979         flc = &priv->flc_desc[DESC_INITFINAL].flc;
1980
1981         session->ctxt_type = DPAA2_SEC_AUTH;
1982         session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1983                         RTE_CACHE_LINE_SIZE);
1984         if (session->auth_key.data == NULL) {
1985                 DPAA2_SEC_ERR("Unable to allocate memory for auth key");
1986                 rte_free(priv);
1987                 return -ENOMEM;
1988         }
1989         session->auth_key.length = xform->auth.key.length;
1990
1991         memcpy(session->auth_key.data, xform->auth.key.data,
1992                xform->auth.key.length);
1993         authdata.key = (size_t)session->auth_key.data;
1994         authdata.keylen = session->auth_key.length;
1995         authdata.key_enc_flags = 0;
1996         authdata.key_type = RTA_DATA_IMM;
1997
1998         session->digest_length = xform->auth.digest_length;
1999         session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2000                                 DIR_ENC : DIR_DEC;
2001
2002         switch (xform->auth.algo) {
2003         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2004                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2005                 authdata.algmode = OP_ALG_AAI_HMAC;
2006                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2007                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2008                                            1, 0, SHR_NEVER, &authdata,
2009                                            !session->dir,
2010                                            session->digest_length);
2011                 break;
2012         case RTE_CRYPTO_AUTH_MD5_HMAC:
2013                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2014                 authdata.algmode = OP_ALG_AAI_HMAC;
2015                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2016                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2017                                            1, 0, SHR_NEVER, &authdata,
2018                                            !session->dir,
2019                                            session->digest_length);
2020                 break;
2021         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2022                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2023                 authdata.algmode = OP_ALG_AAI_HMAC;
2024                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2025                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2026                                            1, 0, SHR_NEVER, &authdata,
2027                                            !session->dir,
2028                                            session->digest_length);
2029                 break;
2030         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2031                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2032                 authdata.algmode = OP_ALG_AAI_HMAC;
2033                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2034                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2035                                            1, 0, SHR_NEVER, &authdata,
2036                                            !session->dir,
2037                                            session->digest_length);
2038                 break;
2039         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2040                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2041                 authdata.algmode = OP_ALG_AAI_HMAC;
2042                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2043                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2044                                            1, 0, SHR_NEVER, &authdata,
2045                                            !session->dir,
2046                                            session->digest_length);
2047                 break;
2048         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2049                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2050                 authdata.algmode = OP_ALG_AAI_HMAC;
2051                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2052                 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2053                                            1, 0, SHR_NEVER, &authdata,
2054                                            !session->dir,
2055                                            session->digest_length);
2056                 break;
2057         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2058                 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2059                 authdata.algmode = OP_ALG_AAI_F9;
2060                 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2061                 session->iv.offset = xform->auth.iv.offset;
2062                 session->iv.length = xform->auth.iv.length;
2063                 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2064                                               1, 0, &authdata,
2065                                               !session->dir,
2066                                               session->digest_length);
2067                 break;
2068         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2069                 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2070                 authdata.algmode = OP_ALG_AAI_F9;
2071                 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2072                 session->iv.offset = xform->auth.iv.offset;
2073                 session->iv.length = xform->auth.iv.length;
2074                 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2075                                            1, 0, &authdata,
2076                                            !session->dir,
2077                                            session->digest_length);
2078                 break;
2079         case RTE_CRYPTO_AUTH_KASUMI_F9:
2080         case RTE_CRYPTO_AUTH_NULL:
2081         case RTE_CRYPTO_AUTH_SHA1:
2082         case RTE_CRYPTO_AUTH_SHA256:
2083         case RTE_CRYPTO_AUTH_SHA512:
2084         case RTE_CRYPTO_AUTH_SHA224:
2085         case RTE_CRYPTO_AUTH_SHA384:
2086         case RTE_CRYPTO_AUTH_MD5:
2087         case RTE_CRYPTO_AUTH_AES_GMAC:
2088         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2089         case RTE_CRYPTO_AUTH_AES_CMAC:
2090         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2091                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2092                               xform->auth.algo);
2093                 ret = -ENOTSUP;
2094                 goto error_out;
2095         default:
2096                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2097                               xform->auth.algo);
2098                 ret = -ENOTSUP;
2099                 goto error_out;
2100         }
2101
2102         if (bufsize < 0) {
2103                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2104                 ret = -EINVAL;
2105                 goto error_out;
2106         }
2107
2108         flc->word1_sdl = (uint8_t)bufsize;
2109         session->ctxt = priv;
2110 #ifdef CAAM_DESC_DEBUG
2111         int i;
2112         for (i = 0; i < bufsize; i++)
2113                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2114                                 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2115 #endif
2116
2117         return ret;
2118
2119 error_out:
2120         rte_free(session->auth_key.data);
2121         rte_free(priv);
2122         return ret;
2123 }
2124
2125 static int
2126 dpaa2_sec_aead_init(struct rte_cryptodev *dev,
2127                     struct rte_crypto_sym_xform *xform,
2128                     dpaa2_sec_session *session)
2129 {
2130         struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2131         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2132         struct alginfo aeaddata;
2133         int bufsize;
2134         struct ctxt_priv *priv;
2135         struct sec_flow_context *flc;
2136         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2137         int err, ret = 0;
2138
2139         PMD_INIT_FUNC_TRACE();
2140
2141         /* Set IV parameters */
2142         session->iv.offset = aead_xform->iv.offset;
2143         session->iv.length = aead_xform->iv.length;
2144         session->ctxt_type = DPAA2_SEC_AEAD;
2145
2146         /* For SEC AEAD only one descriptor is required */
2147         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2148                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2149                         RTE_CACHE_LINE_SIZE);
2150         if (priv == NULL) {
2151                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2152                 return -ENOMEM;
2153         }
2154
2155         priv->fle_pool = dev_priv->fle_pool;
2156         flc = &priv->flc_desc[0].flc;
2157
2158         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2159                                                RTE_CACHE_LINE_SIZE);
2160         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2161                 DPAA2_SEC_ERR("No Memory for aead key");
2162                 rte_free(priv);
2163                 return -ENOMEM;
2164         }
2165         memcpy(session->aead_key.data, aead_xform->key.data,
2166                aead_xform->key.length);
2167
2168         session->digest_length = aead_xform->digest_length;
2169         session->aead_key.length = aead_xform->key.length;
2170         ctxt->auth_only_len = aead_xform->aad_length;
2171
2172         aeaddata.key = (size_t)session->aead_key.data;
2173         aeaddata.keylen = session->aead_key.length;
2174         aeaddata.key_enc_flags = 0;
2175         aeaddata.key_type = RTA_DATA_IMM;
2176
2177         switch (aead_xform->algo) {
2178         case RTE_CRYPTO_AEAD_AES_GCM:
2179                 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2180                 aeaddata.algmode = OP_ALG_AAI_GCM;
2181                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2182                 break;
2183         case RTE_CRYPTO_AEAD_AES_CCM:
2184                 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2185                               aead_xform->algo);
2186                 ret = -ENOTSUP;
2187                 goto error_out;
2188         default:
2189                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2190                               aead_xform->algo);
2191                 ret = -ENOTSUP;
2192                 goto error_out;
2193         }
2194         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2195                                 DIR_ENC : DIR_DEC;
2196
2197         priv->flc_desc[0].desc[0] = aeaddata.keylen;
2198         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2199                                MIN_JOB_DESC_SIZE,
2200                                (unsigned int *)priv->flc_desc[0].desc,
2201                                &priv->flc_desc[0].desc[1], 1);
2202
2203         if (err < 0) {
2204                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2205                 ret = -EINVAL;
2206                 goto error_out;
2207         }
2208         if (priv->flc_desc[0].desc[1] & 1) {
2209                 aeaddata.key_type = RTA_DATA_IMM;
2210         } else {
2211                 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2212                 aeaddata.key_type = RTA_DATA_PTR;
2213         }
2214         priv->flc_desc[0].desc[0] = 0;
2215         priv->flc_desc[0].desc[1] = 0;
2216
2217         if (session->dir == DIR_ENC)
2218                 bufsize = cnstr_shdsc_gcm_encap(
2219                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2220                                 &aeaddata, session->iv.length,
2221                                 session->digest_length);
2222         else
2223                 bufsize = cnstr_shdsc_gcm_decap(
2224                                 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2225                                 &aeaddata, session->iv.length,
2226                                 session->digest_length);
2227         if (bufsize < 0) {
2228                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2229                 ret = -EINVAL;
2230                 goto error_out;
2231         }
2232
2233         flc->word1_sdl = (uint8_t)bufsize;
2234         session->ctxt = priv;
2235 #ifdef CAAM_DESC_DEBUG
2236         int i;
2237         for (i = 0; i < bufsize; i++)
2238                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2239                             i, priv->flc_desc[0].desc[i]);
2240 #endif
2241         return ret;
2242
2243 error_out:
2244         rte_free(session->aead_key.data);
2245         rte_free(priv);
2246         return ret;
2247 }
2248
2249
2250 static int
2251 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
2252                     struct rte_crypto_sym_xform *xform,
2253                     dpaa2_sec_session *session)
2254 {
2255         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2256         struct alginfo authdata, cipherdata;
2257         int bufsize;
2258         struct ctxt_priv *priv;
2259         struct sec_flow_context *flc;
2260         struct rte_crypto_cipher_xform *cipher_xform;
2261         struct rte_crypto_auth_xform *auth_xform;
2262         int err, ret = 0;
2263
2264         PMD_INIT_FUNC_TRACE();
2265
2266         if (session->ext_params.aead_ctxt.auth_cipher_text) {
2267                 cipher_xform = &xform->cipher;
2268                 auth_xform = &xform->next->auth;
2269                 session->ctxt_type =
2270                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2271                         DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2272         } else {
2273                 cipher_xform = &xform->next->cipher;
2274                 auth_xform = &xform->auth;
2275                 session->ctxt_type =
2276                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2277                         DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2278         }
2279
2280         /* Set IV parameters */
2281         session->iv.offset = cipher_xform->iv.offset;
2282         session->iv.length = cipher_xform->iv.length;
2283
2284         /* For SEC AEAD only one descriptor is required */
2285         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2286                         sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2287                         RTE_CACHE_LINE_SIZE);
2288         if (priv == NULL) {
2289                 DPAA2_SEC_ERR("No Memory for priv CTXT");
2290                 return -ENOMEM;
2291         }
2292
2293         priv->fle_pool = dev_priv->fle_pool;
2294         flc = &priv->flc_desc[0].flc;
2295
2296         session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2297                                                RTE_CACHE_LINE_SIZE);
2298         if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2299                 DPAA2_SEC_ERR("No Memory for cipher key");
2300                 rte_free(priv);
2301                 return -ENOMEM;
2302         }
2303         session->cipher_key.length = cipher_xform->key.length;
2304         session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2305                                              RTE_CACHE_LINE_SIZE);
2306         if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2307                 DPAA2_SEC_ERR("No Memory for auth key");
2308                 rte_free(session->cipher_key.data);
2309                 rte_free(priv);
2310                 return -ENOMEM;
2311         }
2312         session->auth_key.length = auth_xform->key.length;
2313         memcpy(session->cipher_key.data, cipher_xform->key.data,
2314                cipher_xform->key.length);
2315         memcpy(session->auth_key.data, auth_xform->key.data,
2316                auth_xform->key.length);
2317
2318         authdata.key = (size_t)session->auth_key.data;
2319         authdata.keylen = session->auth_key.length;
2320         authdata.key_enc_flags = 0;
2321         authdata.key_type = RTA_DATA_IMM;
2322
2323         session->digest_length = auth_xform->digest_length;
2324
2325         switch (auth_xform->algo) {
2326         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2327                 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2328                 authdata.algmode = OP_ALG_AAI_HMAC;
2329                 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2330                 break;
2331         case RTE_CRYPTO_AUTH_MD5_HMAC:
2332                 authdata.algtype = OP_ALG_ALGSEL_MD5;
2333                 authdata.algmode = OP_ALG_AAI_HMAC;
2334                 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2335                 break;
2336         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2337                 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2338                 authdata.algmode = OP_ALG_AAI_HMAC;
2339                 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2340                 break;
2341         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2342                 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2343                 authdata.algmode = OP_ALG_AAI_HMAC;
2344                 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2345                 break;
2346         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2347                 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2348                 authdata.algmode = OP_ALG_AAI_HMAC;
2349                 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2350                 break;
2351         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2352                 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2353                 authdata.algmode = OP_ALG_AAI_HMAC;
2354                 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2355                 break;
2356         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2357         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2358         case RTE_CRYPTO_AUTH_NULL:
2359         case RTE_CRYPTO_AUTH_SHA1:
2360         case RTE_CRYPTO_AUTH_SHA256:
2361         case RTE_CRYPTO_AUTH_SHA512:
2362         case RTE_CRYPTO_AUTH_SHA224:
2363         case RTE_CRYPTO_AUTH_SHA384:
2364         case RTE_CRYPTO_AUTH_MD5:
2365         case RTE_CRYPTO_AUTH_AES_GMAC:
2366         case RTE_CRYPTO_AUTH_KASUMI_F9:
2367         case RTE_CRYPTO_AUTH_AES_CMAC:
2368         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2369         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2370                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2371                               auth_xform->algo);
2372                 ret = -ENOTSUP;
2373                 goto error_out;
2374         default:
2375                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2376                               auth_xform->algo);
2377                 ret = -ENOTSUP;
2378                 goto error_out;
2379         }
2380         cipherdata.key = (size_t)session->cipher_key.data;
2381         cipherdata.keylen = session->cipher_key.length;
2382         cipherdata.key_enc_flags = 0;
2383         cipherdata.key_type = RTA_DATA_IMM;
2384
2385         switch (cipher_xform->algo) {
2386         case RTE_CRYPTO_CIPHER_AES_CBC:
2387                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2388                 cipherdata.algmode = OP_ALG_AAI_CBC;
2389                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2390                 break;
2391         case RTE_CRYPTO_CIPHER_3DES_CBC:
2392                 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2393                 cipherdata.algmode = OP_ALG_AAI_CBC;
2394                 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2395                 break;
2396         case RTE_CRYPTO_CIPHER_AES_CTR:
2397                 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2398                 cipherdata.algmode = OP_ALG_AAI_CTR;
2399                 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2400                 break;
2401         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2402         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2403         case RTE_CRYPTO_CIPHER_NULL:
2404         case RTE_CRYPTO_CIPHER_3DES_ECB:
2405         case RTE_CRYPTO_CIPHER_AES_ECB:
2406         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2407                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2408                               cipher_xform->algo);
2409                 ret = -ENOTSUP;
2410                 goto error_out;
2411         default:
2412                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2413                               cipher_xform->algo);
2414                 ret = -ENOTSUP;
2415                 goto error_out;
2416         }
2417         session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2418                                 DIR_ENC : DIR_DEC;
2419
2420         priv->flc_desc[0].desc[0] = cipherdata.keylen;
2421         priv->flc_desc[0].desc[1] = authdata.keylen;
2422         err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2423                                MIN_JOB_DESC_SIZE,
2424                                (unsigned int *)priv->flc_desc[0].desc,
2425                                &priv->flc_desc[0].desc[2], 2);
2426
2427         if (err < 0) {
2428                 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2429                 ret = -EINVAL;
2430                 goto error_out;
2431         }
2432         if (priv->flc_desc[0].desc[2] & 1) {
2433                 cipherdata.key_type = RTA_DATA_IMM;
2434         } else {
2435                 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2436                 cipherdata.key_type = RTA_DATA_PTR;
2437         }
2438         if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2439                 authdata.key_type = RTA_DATA_IMM;
2440         } else {
2441                 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2442                 authdata.key_type = RTA_DATA_PTR;
2443         }
2444         priv->flc_desc[0].desc[0] = 0;
2445         priv->flc_desc[0].desc[1] = 0;
2446         priv->flc_desc[0].desc[2] = 0;
2447
2448         if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2449                 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2450                                               0, SHR_SERIAL,
2451                                               &cipherdata, &authdata,
2452                                               session->iv.length,
2453                                               session->digest_length,
2454                                               session->dir);
2455                 if (bufsize < 0) {
2456                         DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2457                         ret = -EINVAL;
2458                         goto error_out;
2459                 }
2460         } else {
2461                 DPAA2_SEC_ERR("Hash before cipher not supported");
2462                 ret = -ENOTSUP;
2463                 goto error_out;
2464         }
2465
2466         flc->word1_sdl = (uint8_t)bufsize;
2467         session->ctxt = priv;
2468 #ifdef CAAM_DESC_DEBUG
2469         int i;
2470         for (i = 0; i < bufsize; i++)
2471                 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2472                             i, priv->flc_desc[0].desc[i]);
2473 #endif
2474
2475         return ret;
2476
2477 error_out:
2478         rte_free(session->cipher_key.data);
2479         rte_free(session->auth_key.data);
2480         rte_free(priv);
2481         return ret;
2482 }
2483
2484 static int
2485 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
2486                             struct rte_crypto_sym_xform *xform, void *sess)
2487 {
2488         dpaa2_sec_session *session = sess;
2489         int ret;
2490
2491         PMD_INIT_FUNC_TRACE();
2492
2493         if (unlikely(sess == NULL)) {
2494                 DPAA2_SEC_ERR("Invalid session struct");
2495                 return -EINVAL;
2496         }
2497
2498         memset(session, 0, sizeof(dpaa2_sec_session));
2499         /* Default IV length = 0 */
2500         session->iv.length = 0;
2501
2502         /* Cipher Only */
2503         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2504                 ret = dpaa2_sec_cipher_init(dev, xform, session);
2505
2506         /* Authentication Only */
2507         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2508                    xform->next == NULL) {
2509                 ret = dpaa2_sec_auth_init(dev, xform, session);
2510
2511         /* Cipher then Authenticate */
2512         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2513                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2514                 session->ext_params.aead_ctxt.auth_cipher_text = true;
2515                 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2516                         ret = dpaa2_sec_auth_init(dev, xform, session);
2517                 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2518                         ret = dpaa2_sec_cipher_init(dev, xform, session);
2519                 else
2520                         ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2521         /* Authenticate then Cipher */
2522         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2523                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2524                 session->ext_params.aead_ctxt.auth_cipher_text = false;
2525                 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2526                         ret = dpaa2_sec_cipher_init(dev, xform, session);
2527                 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2528                         ret = dpaa2_sec_auth_init(dev, xform, session);
2529                 else
2530                         ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2531         /* AEAD operation for AES-GCM kind of Algorithms */
2532         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2533                    xform->next == NULL) {
2534                 ret = dpaa2_sec_aead_init(dev, xform, session);
2535
2536         } else {
2537                 DPAA2_SEC_ERR("Invalid crypto type");
2538                 return -EINVAL;
2539         }
2540
2541         return ret;
2542 }
2543
2544 #ifdef RTE_LIBRTE_SECURITY
2545 static int
2546 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2547                         dpaa2_sec_session *session,
2548                         struct alginfo *aeaddata)
2549 {
2550         PMD_INIT_FUNC_TRACE();
2551
2552         session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2553                                                RTE_CACHE_LINE_SIZE);
2554         if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2555                 DPAA2_SEC_ERR("No Memory for aead key");
2556                 return -ENOMEM;
2557         }
2558         memcpy(session->aead_key.data, aead_xform->key.data,
2559                aead_xform->key.length);
2560
2561         session->digest_length = aead_xform->digest_length;
2562         session->aead_key.length = aead_xform->key.length;
2563
2564         aeaddata->key = (size_t)session->aead_key.data;
2565         aeaddata->keylen = session->aead_key.length;
2566         aeaddata->key_enc_flags = 0;
2567         aeaddata->key_type = RTA_DATA_IMM;
2568
2569         switch (aead_xform->algo) {
2570         case RTE_CRYPTO_AEAD_AES_GCM:
2571                 switch (session->digest_length) {
2572                 case 8:
2573                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2574                         break;
2575                 case 12:
2576                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2577                         break;
2578                 case 16:
2579                         aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2580                         break;
2581                 default:
2582                         DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2583                                       session->digest_length);
2584                         return -EINVAL;
2585                 }
2586                 aeaddata->algmode = OP_ALG_AAI_GCM;
2587                 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2588                 break;
2589         case RTE_CRYPTO_AEAD_AES_CCM:
2590                 switch (session->digest_length) {
2591                 case 8:
2592                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2593                         break;
2594                 case 12:
2595                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2596                         break;
2597                 case 16:
2598                         aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2599                         break;
2600                 default:
2601                         DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2602                                       session->digest_length);
2603                         return -EINVAL;
2604                 }
2605                 aeaddata->algmode = OP_ALG_AAI_CCM;
2606                 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2607                 break;
2608         default:
2609                 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2610                               aead_xform->algo);
2611                 return -ENOTSUP;
2612         }
2613         session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2614                                 DIR_ENC : DIR_DEC;
2615
2616         return 0;
2617 }
2618
2619 static int
2620 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2621         struct rte_crypto_auth_xform *auth_xform,
2622         dpaa2_sec_session *session,
2623         struct alginfo *cipherdata,
2624         struct alginfo *authdata)
2625 {
2626         if (cipher_xform) {
2627                 session->cipher_key.data = rte_zmalloc(NULL,
2628                                                        cipher_xform->key.length,
2629                                                        RTE_CACHE_LINE_SIZE);
2630                 if (session->cipher_key.data == NULL &&
2631                                 cipher_xform->key.length > 0) {
2632                         DPAA2_SEC_ERR("No Memory for cipher key");
2633                         return -ENOMEM;
2634                 }
2635
2636                 session->cipher_key.length = cipher_xform->key.length;
2637                 memcpy(session->cipher_key.data, cipher_xform->key.data,
2638                                 cipher_xform->key.length);
2639                 session->cipher_alg = cipher_xform->algo;
2640         } else {
2641                 session->cipher_key.data = NULL;
2642                 session->cipher_key.length = 0;
2643                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2644         }
2645
2646         if (auth_xform) {
2647                 session->auth_key.data = rte_zmalloc(NULL,
2648                                                 auth_xform->key.length,
2649                                                 RTE_CACHE_LINE_SIZE);
2650                 if (session->auth_key.data == NULL &&
2651                                 auth_xform->key.length > 0) {
2652                         DPAA2_SEC_ERR("No Memory for auth key");
2653                         return -ENOMEM;
2654                 }
2655                 session->auth_key.length = auth_xform->key.length;
2656                 memcpy(session->auth_key.data, auth_xform->key.data,
2657                                 auth_xform->key.length);
2658                 session->auth_alg = auth_xform->algo;
2659                 session->digest_length = auth_xform->digest_length;
2660         } else {
2661                 session->auth_key.data = NULL;
2662                 session->auth_key.length = 0;
2663                 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2664         }
2665
2666         authdata->key = (size_t)session->auth_key.data;
2667         authdata->keylen = session->auth_key.length;
2668         authdata->key_enc_flags = 0;
2669         authdata->key_type = RTA_DATA_IMM;
2670         switch (session->auth_alg) {
2671         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2672                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
2673                 authdata->algmode = OP_ALG_AAI_HMAC;
2674                 break;
2675         case RTE_CRYPTO_AUTH_MD5_HMAC:
2676                 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
2677                 authdata->algmode = OP_ALG_AAI_HMAC;
2678                 break;
2679         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2680                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2681                 authdata->algmode = OP_ALG_AAI_HMAC;
2682                 if (session->digest_length != 16)
2683                         DPAA2_SEC_WARN(
2684                         "+++Using sha256-hmac truncated len is non-standard,"
2685                         "it will not work with lookaside proto");
2686                 break;
2687         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2688                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2689                 authdata->algmode = OP_ALG_AAI_HMAC;
2690                 break;
2691         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2692                 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2693                 authdata->algmode = OP_ALG_AAI_HMAC;
2694                 break;
2695         case RTE_CRYPTO_AUTH_AES_CMAC:
2696                 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
2697                 break;
2698         case RTE_CRYPTO_AUTH_NULL:
2699                 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
2700                 break;
2701         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2702         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2703         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2704         case RTE_CRYPTO_AUTH_SHA1:
2705         case RTE_CRYPTO_AUTH_SHA256:
2706         case RTE_CRYPTO_AUTH_SHA512:
2707         case RTE_CRYPTO_AUTH_SHA224:
2708         case RTE_CRYPTO_AUTH_SHA384:
2709         case RTE_CRYPTO_AUTH_MD5:
2710         case RTE_CRYPTO_AUTH_AES_GMAC:
2711         case RTE_CRYPTO_AUTH_KASUMI_F9:
2712         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2713         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2714                 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2715                               session->auth_alg);
2716                 return -ENOTSUP;
2717         default:
2718                 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2719                               session->auth_alg);
2720                 return -ENOTSUP;
2721         }
2722         cipherdata->key = (size_t)session->cipher_key.data;
2723         cipherdata->keylen = session->cipher_key.length;
2724         cipherdata->key_enc_flags = 0;
2725         cipherdata->key_type = RTA_DATA_IMM;
2726
2727         switch (session->cipher_alg) {
2728         case RTE_CRYPTO_CIPHER_AES_CBC:
2729                 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
2730                 cipherdata->algmode = OP_ALG_AAI_CBC;
2731                 break;
2732         case RTE_CRYPTO_CIPHER_3DES_CBC:
2733                 cipherdata->algtype = OP_PCL_IPSEC_3DES;
2734                 cipherdata->algmode = OP_ALG_AAI_CBC;
2735                 break;
2736         case RTE_CRYPTO_CIPHER_AES_CTR:
2737                 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
2738                 cipherdata->algmode = OP_ALG_AAI_CTR;
2739                 break;
2740         case RTE_CRYPTO_CIPHER_NULL:
2741                 cipherdata->algtype = OP_PCL_IPSEC_NULL;
2742                 break;
2743         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2744         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2745         case RTE_CRYPTO_CIPHER_3DES_ECB:
2746         case RTE_CRYPTO_CIPHER_AES_ECB:
2747         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2748                 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2749                               session->cipher_alg);
2750                 return -ENOTSUP;
2751         default:
2752                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2753                               session->cipher_alg);
2754                 return -ENOTSUP;
2755         }
2756
2757         return 0;
2758 }
2759
2760 #ifdef RTE_LIBRTE_SECURITY_TEST
2761 static uint8_t aes_cbc_iv[] = {
2762         0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
2763         0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
2764 #endif
2765
2766 static int
2767 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
2768                             struct rte_security_session_conf *conf,
2769                             void *sess)
2770 {
2771         struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2772         struct rte_crypto_cipher_xform *cipher_xform = NULL;
2773         struct rte_crypto_auth_xform *auth_xform = NULL;
2774         struct rte_crypto_aead_xform *aead_xform = NULL;
2775         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
2776         struct ctxt_priv *priv;
2777         struct alginfo authdata, cipherdata;
2778         int bufsize;
2779         struct sec_flow_context *flc;
2780         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2781         int ret = -1;
2782
2783         PMD_INIT_FUNC_TRACE();
2784
2785         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2786                                 sizeof(struct ctxt_priv) +
2787                                 sizeof(struct sec_flc_desc),
2788                                 RTE_CACHE_LINE_SIZE);
2789
2790         if (priv == NULL) {
2791                 DPAA2_SEC_ERR("No memory for priv CTXT");
2792                 return -ENOMEM;
2793         }
2794
2795         priv->fle_pool = dev_priv->fle_pool;
2796         flc = &priv->flc_desc[0].flc;
2797
2798         memset(session, 0, sizeof(dpaa2_sec_session));
2799
2800         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2801                 cipher_xform = &conf->crypto_xform->cipher;
2802                 if (conf->crypto_xform->next)
2803                         auth_xform = &conf->crypto_xform->next->auth;
2804                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2805                                         session, &cipherdata, &authdata);
2806         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2807                 auth_xform = &conf->crypto_xform->auth;
2808                 if (conf->crypto_xform->next)
2809                         cipher_xform = &conf->crypto_xform->next->cipher;
2810                 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2811                                         session, &cipherdata, &authdata);
2812         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2813                 aead_xform = &conf->crypto_xform->aead;
2814                 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
2815                                         session, &cipherdata);
2816                 authdata.keylen = 0;
2817                 authdata.algtype = 0;
2818         } else {
2819                 DPAA2_SEC_ERR("XFORM not specified");
2820                 ret = -EINVAL;
2821                 goto out;
2822         }
2823         if (ret) {
2824                 DPAA2_SEC_ERR("Failed to process xform");
2825                 goto out;
2826         }
2827
2828         session->ctxt_type = DPAA2_SEC_IPSEC;
2829         if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2830                 uint8_t *hdr = NULL;
2831                 struct ip ip4_hdr;
2832                 struct rte_ipv6_hdr ip6_hdr;
2833                 struct ipsec_encap_pdb encap_pdb;
2834
2835                 flc->dhr = SEC_FLC_DHR_OUTBOUND;
2836                 /* For Sec Proto only one descriptor is required. */
2837                 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
2838
2839                 /* copy algo specific data to PDB */
2840                 switch (cipherdata.algtype) {
2841                 case OP_PCL_IPSEC_AES_CTR:
2842                         encap_pdb.ctr.ctr_initial = 0x00000001;
2843                         encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2844                         break;
2845                 case OP_PCL_IPSEC_AES_GCM8:
2846                 case OP_PCL_IPSEC_AES_GCM12:
2847                 case OP_PCL_IPSEC_AES_GCM16:
2848                         memcpy(encap_pdb.gcm.salt,
2849                                 (uint8_t *)&(ipsec_xform->salt), 4);
2850                         break;
2851                 }
2852
2853                 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2854                         PDBOPTS_ESP_OIHI_PDB_INL |
2855                         PDBOPTS_ESP_IVSRC |
2856                         PDBHMO_ESP_ENCAP_DTTL |
2857                         PDBHMO_ESP_SNR;
2858                 if (ipsec_xform->options.esn)
2859                         encap_pdb.options |= PDBOPTS_ESP_ESN;
2860                 encap_pdb.spi = ipsec_xform->spi;
2861                 session->dir = DIR_ENC;
2862                 if (ipsec_xform->tunnel.type ==
2863                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
2864                         encap_pdb.ip_hdr_len = sizeof(struct ip);
2865                         ip4_hdr.ip_v = IPVERSION;
2866                         ip4_hdr.ip_hl = 5;
2867                         ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
2868                         ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2869                         ip4_hdr.ip_id = 0;
2870                         ip4_hdr.ip_off = 0;
2871                         ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2872                         ip4_hdr.ip_p = IPPROTO_ESP;
2873                         ip4_hdr.ip_sum = 0;
2874                         ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2875                         ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2876                         ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
2877                                         &ip4_hdr, sizeof(struct ip));
2878                         hdr = (uint8_t *)&ip4_hdr;
2879                 } else if (ipsec_xform->tunnel.type ==
2880                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
2881                         ip6_hdr.vtc_flow = rte_cpu_to_be_32(
2882                                 DPAA2_IPv6_DEFAULT_VTC_FLOW |
2883                                 ((ipsec_xform->tunnel.ipv6.dscp <<
2884                                         RTE_IPV6_HDR_TC_SHIFT) &
2885                                         RTE_IPV6_HDR_TC_MASK) |
2886                                 ((ipsec_xform->tunnel.ipv6.flabel <<
2887                                         RTE_IPV6_HDR_FL_SHIFT) &
2888                                         RTE_IPV6_HDR_FL_MASK));
2889                         /* Payload length will be updated by HW */
2890                         ip6_hdr.payload_len = 0;
2891                         ip6_hdr.hop_limits =
2892                                         ipsec_xform->tunnel.ipv6.hlimit;
2893                         ip6_hdr.proto = (ipsec_xform->proto ==
2894                                         RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
2895                                         IPPROTO_ESP : IPPROTO_AH;
2896                         memcpy(&ip6_hdr.src_addr,
2897                                 &ipsec_xform->tunnel.ipv6.src_addr, 16);
2898                         memcpy(&ip6_hdr.dst_addr,
2899                                 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
2900                         encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
2901                         hdr = (uint8_t *)&ip6_hdr;
2902                 }
2903
2904                 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
2905                                 1, 0, SHR_SERIAL, &encap_pdb,
2906                                 hdr, &cipherdata, &authdata);
2907         } else if (ipsec_xform->direction ==
2908                         RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2909                 struct ipsec_decap_pdb decap_pdb;
2910
2911                 flc->dhr = SEC_FLC_DHR_INBOUND;
2912                 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2913                 /* copy algo specific data to PDB */
2914                 switch (cipherdata.algtype) {
2915                 case OP_PCL_IPSEC_AES_CTR:
2916                         decap_pdb.ctr.ctr_initial = 0x00000001;
2917                         decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2918                         break;
2919                 case OP_PCL_IPSEC_AES_GCM8:
2920                 case OP_PCL_IPSEC_AES_GCM12:
2921                 case OP_PCL_IPSEC_AES_GCM16:
2922                         memcpy(decap_pdb.gcm.salt,
2923                                 (uint8_t *)&(ipsec_xform->salt), 4);
2924                         break;
2925                 }
2926
2927                 decap_pdb.options = (ipsec_xform->tunnel.type ==
2928                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
2929                                 sizeof(struct ip) << 16 :
2930                                 sizeof(struct rte_ipv6_hdr) << 16;
2931                 if (ipsec_xform->options.esn)
2932                         decap_pdb.options |= PDBOPTS_ESP_ESN;
2933
2934                 if (ipsec_xform->replay_win_sz) {
2935                         uint32_t win_sz;
2936                         win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
2937
2938                         switch (win_sz) {
2939                         case 1:
2940                         case 2:
2941                         case 4:
2942                         case 8:
2943                         case 16:
2944                         case 32:
2945                                 decap_pdb.options |= PDBOPTS_ESP_ARS32;
2946                                 break;
2947                         case 64:
2948                                 decap_pdb.options |= PDBOPTS_ESP_ARS64;
2949                                 break;
2950                         default:
2951                                 decap_pdb.options |= PDBOPTS_ESP_ARS128;
2952                         }
2953                 }
2954                 session->dir = DIR_DEC;
2955                 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
2956                                 1, 0, SHR_SERIAL,
2957                                 &decap_pdb, &cipherdata, &authdata);
2958         } else
2959                 goto out;
2960
2961         if (bufsize < 0) {
2962                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2963                 goto out;
2964         }
2965
2966         flc->word1_sdl = (uint8_t)bufsize;
2967
2968         /* Enable the stashing control bit */
2969         DPAA2_SET_FLC_RSC(flc);
2970         flc->word2_rflc_31_0 = lower_32_bits(
2971                         (size_t)&(((struct dpaa2_sec_qp *)
2972                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
2973         flc->word3_rflc_63_32 = upper_32_bits(
2974                         (size_t)&(((struct dpaa2_sec_qp *)
2975                         dev->data->queue_pairs[0])->rx_vq));
2976
2977         /* Set EWS bit i.e. enable write-safe */
2978         DPAA2_SET_FLC_EWS(flc);
2979         /* Set BS = 1 i.e reuse input buffers as output buffers */
2980         DPAA2_SET_FLC_REUSE_BS(flc);
2981         /* Set FF = 10; reuse input buffers if they provide sufficient space */
2982         DPAA2_SET_FLC_REUSE_FF(flc);
2983
2984         session->ctxt = priv;
2985
2986         return 0;
2987 out:
2988         rte_free(session->auth_key.data);
2989         rte_free(session->cipher_key.data);
2990         rte_free(priv);
2991         return ret;
2992 }
2993
2994 static int
2995 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
2996                            struct rte_security_session_conf *conf,
2997                            void *sess)
2998 {
2999         struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3000         struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3001         struct rte_crypto_auth_xform *auth_xform = NULL;
3002         struct rte_crypto_cipher_xform *cipher_xform;
3003         dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3004         struct ctxt_priv *priv;
3005         struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
3006         struct alginfo authdata, cipherdata;
3007         struct alginfo *p_authdata = NULL;
3008         int bufsize = -1;
3009         struct sec_flow_context *flc;
3010 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3011         int swap = true;
3012 #else
3013         int swap = false;
3014 #endif
3015
3016         PMD_INIT_FUNC_TRACE();
3017
3018         memset(session, 0, sizeof(dpaa2_sec_session));
3019
3020         priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3021                                 sizeof(struct ctxt_priv) +
3022                                 sizeof(struct sec_flc_desc),
3023                                 RTE_CACHE_LINE_SIZE);
3024
3025         if (priv == NULL) {
3026                 DPAA2_SEC_ERR("No memory for priv CTXT");
3027                 return -ENOMEM;
3028         }
3029
3030         priv->fle_pool = dev_priv->fle_pool;
3031         flc = &priv->flc_desc[0].flc;
3032
3033         /* find xfrm types */
3034         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
3035                 cipher_xform = &xform->cipher;
3036         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
3037                    xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3038                 session->ext_params.aead_ctxt.auth_cipher_text = true;
3039                 cipher_xform = &xform->cipher;
3040                 auth_xform = &xform->next->auth;
3041         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
3042                    xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3043                 session->ext_params.aead_ctxt.auth_cipher_text = false;
3044                 cipher_xform = &xform->next->cipher;
3045                 auth_xform = &xform->auth;
3046         } else {
3047                 DPAA2_SEC_ERR("Invalid crypto type");
3048                 return -EINVAL;
3049         }
3050
3051         session->ctxt_type = DPAA2_SEC_PDCP;
3052         if (cipher_xform) {
3053                 session->cipher_key.data = rte_zmalloc(NULL,
3054                                                cipher_xform->key.length,
3055                                                RTE_CACHE_LINE_SIZE);
3056                 if (session->cipher_key.data == NULL &&
3057                                 cipher_xform->key.length > 0) {
3058                         DPAA2_SEC_ERR("No Memory for cipher key");
3059                         rte_free(priv);
3060                         return -ENOMEM;
3061                 }
3062                 session->cipher_key.length = cipher_xform->key.length;
3063                 memcpy(session->cipher_key.data, cipher_xform->key.data,
3064                         cipher_xform->key.length);
3065                 session->dir =
3066                         (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3067                                         DIR_ENC : DIR_DEC;
3068                 session->cipher_alg = cipher_xform->algo;
3069         } else {
3070                 session->cipher_key.data = NULL;
3071                 session->cipher_key.length = 0;
3072                 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3073                 session->dir = DIR_ENC;
3074         }
3075
3076         session->pdcp.domain = pdcp_xform->domain;
3077         session->pdcp.bearer = pdcp_xform->bearer;
3078         session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3079         session->pdcp.sn_size = pdcp_xform->sn_size;
3080         session->pdcp.hfn = pdcp_xform->hfn;
3081         session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3082         session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3083         /* hfv ovd offset location is stored in iv.offset value*/
3084         session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3085
3086         cipherdata.key = (size_t)session->cipher_key.data;
3087         cipherdata.keylen = session->cipher_key.length;
3088         cipherdata.key_enc_flags = 0;
3089         cipherdata.key_type = RTA_DATA_IMM;
3090
3091         switch (session->cipher_alg) {
3092         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3093                 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3094                 break;
3095         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3096                 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3097                 break;
3098         case RTE_CRYPTO_CIPHER_AES_CTR:
3099                 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3100                 break;
3101         case RTE_CRYPTO_CIPHER_NULL:
3102                 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3103                 break;
3104         default:
3105                 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3106                               session->cipher_alg);
3107                 goto out;
3108         }
3109
3110         if (auth_xform) {
3111                 session->auth_key.data = rte_zmalloc(NULL,
3112                                                      auth_xform->key.length,
3113                                                      RTE_CACHE_LINE_SIZE);
3114                 if (!session->auth_key.data &&
3115                     auth_xform->key.length > 0) {
3116                         DPAA2_SEC_ERR("No Memory for auth key");
3117                         rte_free(session->cipher_key.data);
3118                         rte_free(priv);
3119                         return -ENOMEM;
3120                 }
3121                 session->auth_key.length = auth_xform->key.length;
3122                 memcpy(session->auth_key.data, auth_xform->key.data,
3123                        auth_xform->key.length);
3124                 session->auth_alg = auth_xform->algo;
3125         } else {
3126                 session->auth_key.data = NULL;
3127                 session->auth_key.length = 0;
3128                 session->auth_alg = 0;
3129         }
3130         authdata.key = (size_t)session->auth_key.data;
3131         authdata.keylen = session->auth_key.length;
3132         authdata.key_enc_flags = 0;
3133         authdata.key_type = RTA_DATA_IMM;
3134
3135         if (session->auth_alg) {
3136                 switch (session->auth_alg) {
3137                 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3138                         authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3139                         break;
3140                 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3141                         authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3142                         break;
3143                 case RTE_CRYPTO_AUTH_AES_CMAC:
3144                         authdata.algtype = PDCP_AUTH_TYPE_AES;
3145                         break;
3146                 case RTE_CRYPTO_AUTH_NULL:
3147                         authdata.algtype = PDCP_AUTH_TYPE_NULL;
3148                         break;
3149                 default:
3150                         DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3151                                       session->auth_alg);
3152                         goto out;
3153                 }
3154
3155                 p_authdata = &authdata;
3156         } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3157                 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3158                 goto out;
3159         }
3160
3161         if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3162                 if (session->dir == DIR_ENC)
3163                         bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3164                                         priv->flc_desc[0].desc, 1, swap,
3165                                         pdcp_xform->hfn,
3166                                         session->pdcp.sn_size,
3167                                         pdcp_xform->bearer,
3168                                         pdcp_xform->pkt_dir,
3169                                         pdcp_xform->hfn_threshold,
3170                                         &cipherdata, &authdata,
3171                                         0);
3172                 else if (session->dir == DIR_DEC)
3173                         bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3174                                         priv->flc_desc[0].desc, 1, swap,
3175                                         pdcp_xform->hfn,
3176                                         session->pdcp.sn_size,
3177                                         pdcp_xform->bearer,
3178                                         pdcp_xform->pkt_dir,
3179                                         pdcp_xform->hfn_threshold,
3180                                         &cipherdata, &authdata,
3181                                         0);
3182         } else {
3183                 if (session->dir == DIR_ENC)
3184                         bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3185                                         priv->flc_desc[0].desc, 1, swap,
3186                                         session->pdcp.sn_size,
3187                                         pdcp_xform->hfn,
3188                                         pdcp_xform->bearer,
3189                                         pdcp_xform->pkt_dir,
3190                                         pdcp_xform->hfn_threshold,
3191                                         &cipherdata, p_authdata, 0);
3192                 else if (session->dir == DIR_DEC)
3193                         bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3194                                         priv->flc_desc[0].desc, 1, swap,
3195                                         session->pdcp.sn_size,
3196                                         pdcp_xform->hfn,
3197                                         pdcp_xform->bearer,
3198                                         pdcp_xform->pkt_dir,
3199                                         pdcp_xform->hfn_threshold,
3200                                         &cipherdata, p_authdata, 0);
3201         }
3202
3203         if (bufsize < 0) {
3204                 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3205                 goto out;
3206         }
3207
3208         /* Enable the stashing control bit */
3209         DPAA2_SET_FLC_RSC(flc);
3210         flc->word2_rflc_31_0 = lower_32_bits(
3211                         (size_t)&(((struct dpaa2_sec_qp *)
3212                         dev->data->queue_pairs[0])->rx_vq) | 0x14);
3213         flc->word3_rflc_63_32 = upper_32_bits(
3214                         (size_t)&(((struct dpaa2_sec_qp *)
3215                         dev->data->queue_pairs[0])->rx_vq));
3216
3217         flc->word1_sdl = (uint8_t)bufsize;
3218
3219         /* TODO - check the perf impact or
3220          * align as per descriptor type
3221          * Set EWS bit i.e. enable write-safe
3222          * DPAA2_SET_FLC_EWS(flc);
3223          */
3224
3225         /* Set BS = 1 i.e reuse input buffers as output buffers */
3226         DPAA2_SET_FLC_REUSE_BS(flc);
3227         /* Set FF = 10; reuse input buffers if they provide sufficient space */
3228         DPAA2_SET_FLC_REUSE_FF(flc);
3229
3230         session->ctxt = priv;
3231
3232         return 0;
3233 out:
3234         rte_free(session->auth_key.data);
3235         rte_free(session->cipher_key.data);
3236         rte_free(priv);
3237         return -EINVAL;
3238 }
3239
3240 static int
3241 dpaa2_sec_security_session_create(void *dev,
3242                                   struct rte_security_session_conf *conf,
3243                                   struct rte_security_session *sess,
3244                                   struct rte_mempool *mempool)
3245 {
3246         void *sess_private_data;
3247         struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3248         int ret;
3249
3250         if (rte_mempool_get(mempool, &sess_private_data)) {
3251                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3252                 return -ENOMEM;
3253         }
3254
3255         switch (conf->protocol) {
3256         case RTE_SECURITY_PROTOCOL_IPSEC:
3257                 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3258                                 sess_private_data);
3259                 break;
3260         case RTE_SECURITY_PROTOCOL_MACSEC:
3261                 return -ENOTSUP;
3262         case RTE_SECURITY_PROTOCOL_PDCP:
3263                 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3264                                 sess_private_data);
3265                 break;
3266         default:
3267                 return -EINVAL;
3268         }
3269         if (ret != 0) {
3270                 DPAA2_SEC_ERR("Failed to configure session parameters");
3271                 /* Return session to mempool */
3272                 rte_mempool_put(mempool, sess_private_data);
3273                 return ret;
3274         }
3275
3276         set_sec_session_private_data(sess, sess_private_data);
3277
3278         return ret;
3279 }
3280
3281 /** Clear the memory of session so it doesn't leave key material behind */
3282 static int
3283 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3284                 struct rte_security_session *sess)
3285 {
3286         PMD_INIT_FUNC_TRACE();
3287         void *sess_priv = get_sec_session_private_data(sess);
3288
3289         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3290
3291         if (sess_priv) {
3292                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3293
3294                 rte_free(s->ctxt);
3295                 rte_free(s->cipher_key.data);
3296                 rte_free(s->auth_key.data);
3297                 memset(s, 0, sizeof(dpaa2_sec_session));
3298                 set_sec_session_private_data(sess, NULL);
3299                 rte_mempool_put(sess_mp, sess_priv);
3300         }
3301         return 0;
3302 }
3303 #endif
3304 static int
3305 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3306                 struct rte_crypto_sym_xform *xform,
3307                 struct rte_cryptodev_sym_session *sess,
3308                 struct rte_mempool *mempool)
3309 {
3310         void *sess_private_data;
3311         int ret;
3312
3313         if (rte_mempool_get(mempool, &sess_private_data)) {
3314                 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3315                 return -ENOMEM;
3316         }
3317
3318         ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data);
3319         if (ret != 0) {
3320                 DPAA2_SEC_ERR("Failed to configure session parameters");
3321                 /* Return session to mempool */
3322                 rte_mempool_put(mempool, sess_private_data);
3323                 return ret;
3324         }
3325
3326         set_sym_session_private_data(sess, dev->driver_id,
3327                 sess_private_data);
3328
3329         return 0;
3330 }
3331
3332 /** Clear the memory of session so it doesn't leave key material behind */
3333 static void
3334 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3335                 struct rte_cryptodev_sym_session *sess)
3336 {
3337         PMD_INIT_FUNC_TRACE();
3338         uint8_t index = dev->driver_id;
3339         void *sess_priv = get_sym_session_private_data(sess, index);
3340         dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3341
3342         if (sess_priv) {
3343                 rte_free(s->ctxt);
3344                 rte_free(s->cipher_key.data);
3345                 rte_free(s->auth_key.data);
3346                 memset(s, 0, sizeof(dpaa2_sec_session));
3347                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3348                 set_sym_session_private_data(sess, index, NULL);
3349                 rte_mempool_put(sess_mp, sess_priv);
3350         }
3351 }
3352
3353 static int
3354 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3355                         struct rte_cryptodev_config *config __rte_unused)
3356 {
3357         PMD_INIT_FUNC_TRACE();
3358
3359         return 0;
3360 }
3361
3362 static int
3363 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3364 {
3365         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3366         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3367         struct dpseci_attr attr;
3368         struct dpaa2_queue *dpaa2_q;
3369         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3370                                         dev->data->queue_pairs;
3371         struct dpseci_rx_queue_attr rx_attr;
3372         struct dpseci_tx_queue_attr tx_attr;
3373         int ret, i;
3374
3375         PMD_INIT_FUNC_TRACE();
3376
3377         memset(&attr, 0, sizeof(struct dpseci_attr));
3378
3379         ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3380         if (ret) {
3381                 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3382                               priv->hw_id);
3383                 goto get_attr_failure;
3384         }
3385         ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3386         if (ret) {
3387                 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3388                 goto get_attr_failure;
3389         }
3390         for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3391                 dpaa2_q = &qp[i]->rx_vq;
3392                 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3393                                     &rx_attr);
3394                 dpaa2_q->fqid = rx_attr.fqid;
3395                 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3396         }
3397         for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3398                 dpaa2_q = &qp[i]->tx_vq;
3399                 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3400                                     &tx_attr);
3401                 dpaa2_q->fqid = tx_attr.fqid;
3402                 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3403         }
3404
3405         return 0;
3406 get_attr_failure:
3407         dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3408         return -1;
3409 }
3410
3411 static void
3412 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3413 {
3414         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3415         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3416         int ret;
3417
3418         PMD_INIT_FUNC_TRACE();
3419
3420         ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3421         if (ret) {
3422                 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3423                              priv->hw_id);
3424                 return;
3425         }
3426
3427         ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3428         if (ret < 0) {
3429                 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3430                 return;
3431         }
3432 }
3433
3434 static int
3435 dpaa2_sec_dev_close(struct rte_cryptodev *dev)
3436 {
3437         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3438         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3439         int ret;
3440
3441         PMD_INIT_FUNC_TRACE();
3442
3443         /* Function is reverse of dpaa2_sec_dev_init.
3444          * It does the following:
3445          * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
3446          * 2. Close the DPSECI device
3447          * 3. Free the allocated resources.
3448          */
3449
3450         /*Close the device at underlying layer*/
3451         ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
3452         if (ret) {
3453                 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
3454                 return -1;
3455         }
3456
3457         /*Free the allocated memory for ethernet private data and dpseci*/
3458         priv->hw = NULL;
3459         rte_free(dpseci);
3460
3461         return 0;
3462 }
3463
3464 static void
3465 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3466                         struct rte_cryptodev_info *info)
3467 {
3468         struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3469
3470         PMD_INIT_FUNC_TRACE();
3471         if (info != NULL) {
3472                 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3473                 info->feature_flags = dev->feature_flags;
3474                 info->capabilities = dpaa2_sec_capabilities;
3475                 /* No limit of number of sessions */
3476                 info->sym.max_nb_sessions = 0;
3477                 info->driver_id = cryptodev_driver_id;
3478         }
3479 }
3480
3481 static
3482 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3483                          struct rte_cryptodev_stats *stats)
3484 {
3485         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3486         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3487         struct dpseci_sec_counters counters = {0};
3488         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3489                                         dev->data->queue_pairs;
3490         int ret, i;
3491
3492         PMD_INIT_FUNC_TRACE();
3493         if (stats == NULL) {
3494                 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3495                 return;
3496         }
3497         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3498                 if (qp[i] == NULL) {
3499                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3500                         continue;
3501                 }
3502
3503                 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3504                 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3505                 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3506                 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3507         }
3508
3509         ret = dpseci_get_sec_counters(dpseci, CMD_PRI_LOW, priv->token,
3510                                       &counters);
3511         if (ret) {
3512                 DPAA2_SEC_ERR("SEC counters failed");
3513         } else {
3514                 DPAA2_SEC_INFO("dpseci hardware stats:"
3515                             "\n\tNum of Requests Dequeued = %" PRIu64
3516                             "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3517                             "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3518                             "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3519                             "\n\tNum of Outbound Bytes Protected = %" PRIu64
3520                             "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3521                             "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3522                             counters.dequeued_requests,
3523                             counters.ob_enc_requests,
3524                             counters.ib_dec_requests,
3525                             counters.ob_enc_bytes,
3526                             counters.ob_prot_bytes,
3527                             counters.ib_dec_bytes,
3528                             counters.ib_valid_bytes);
3529         }
3530 }
3531
3532 static
3533 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3534 {
3535         int i;
3536         struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3537                                    (dev->data->queue_pairs);
3538
3539         PMD_INIT_FUNC_TRACE();
3540
3541         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3542                 if (qp[i] == NULL) {
3543                         DPAA2_SEC_DEBUG("Uninitialised queue pair");
3544                         continue;
3545                 }
3546                 qp[i]->tx_vq.rx_pkts = 0;
3547                 qp[i]->tx_vq.tx_pkts = 0;
3548                 qp[i]->tx_vq.err_pkts = 0;
3549                 qp[i]->rx_vq.rx_pkts = 0;
3550                 qp[i]->rx_vq.tx_pkts = 0;
3551                 qp[i]->rx_vq.err_pkts = 0;
3552         }
3553 }
3554
3555 static void __rte_hot
3556 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3557                                  const struct qbman_fd *fd,
3558                                  const struct qbman_result *dq,
3559                                  struct dpaa2_queue *rxq,
3560                                  struct rte_event *ev)
3561 {
3562         /* Prefetching mbuf */
3563         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3564                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3565
3566         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3567         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3568
3569         ev->flow_id = rxq->ev.flow_id;
3570         ev->sub_event_type = rxq->ev.sub_event_type;
3571         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3572         ev->op = RTE_EVENT_OP_NEW;
3573         ev->sched_type = rxq->ev.sched_type;
3574         ev->queue_id = rxq->ev.queue_id;
3575         ev->priority = rxq->ev.priority;
3576         ev->event_ptr = sec_fd_to_mbuf(fd);
3577
3578         qbman_swp_dqrr_consume(swp, dq);
3579 }
3580 static void
3581 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
3582                                  const struct qbman_fd *fd,
3583                                  const struct qbman_result *dq,
3584                                  struct dpaa2_queue *rxq,
3585                                  struct rte_event *ev)
3586 {
3587         uint8_t dqrr_index;
3588         struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
3589         /* Prefetching mbuf */
3590         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3591                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3592
3593         /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3594         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3595
3596         ev->flow_id = rxq->ev.flow_id;
3597         ev->sub_event_type = rxq->ev.sub_event_type;
3598         ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3599         ev->op = RTE_EVENT_OP_NEW;
3600         ev->sched_type = rxq->ev.sched_type;
3601         ev->queue_id = rxq->ev.queue_id;
3602         ev->priority = rxq->ev.priority;
3603
3604         ev->event_ptr = sec_fd_to_mbuf(fd);
3605         dqrr_index = qbman_get_dqrr_idx(dq);
3606         crypto_op->sym->m_src->seqn = dqrr_index + 1;
3607         DPAA2_PER_LCORE_DQRR_SIZE++;
3608         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
3609         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
3610 }
3611
3612 int
3613 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
3614                 int qp_id,
3615                 struct dpaa2_dpcon_dev *dpcon,
3616                 const struct rte_event *event)
3617 {
3618         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3619         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3620         struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
3621         struct dpseci_rx_queue_cfg cfg;
3622         uint8_t priority;
3623         int ret;
3624
3625         if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
3626                 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
3627         else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
3628                 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
3629         else
3630                 return -EINVAL;
3631
3632         priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
3633                    (dpcon->num_priorities - 1);
3634
3635         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3636         cfg.options = DPSECI_QUEUE_OPT_DEST;
3637         cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
3638         cfg.dest_cfg.dest_id = dpcon->dpcon_id;
3639         cfg.dest_cfg.priority = priority;
3640
3641         cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
3642         cfg.user_ctx = (size_t)(qp);
3643         if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
3644                 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
3645                 cfg.order_preservation_en = 1;
3646         }
3647         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3648                                   qp_id, &cfg);
3649         if (ret) {
3650                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3651                 return ret;
3652         }
3653
3654         memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
3655
3656         return 0;
3657 }
3658
3659 int
3660 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
3661                         int qp_id)
3662 {
3663         struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3664         struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3665         struct dpseci_rx_queue_cfg cfg;
3666         int ret;
3667
3668         memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3669         cfg.options = DPSECI_QUEUE_OPT_DEST;
3670         cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
3671
3672         ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3673                                   qp_id, &cfg);
3674         if (ret)
3675                 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3676
3677         return ret;
3678 }
3679
3680 static struct rte_cryptodev_ops crypto_ops = {
3681         .dev_configure        = dpaa2_sec_dev_configure,
3682         .dev_start            = dpaa2_sec_dev_start,
3683         .dev_stop             = dpaa2_sec_dev_stop,
3684         .dev_close            = dpaa2_sec_dev_close,
3685         .dev_infos_get        = dpaa2_sec_dev_infos_get,
3686         .stats_get            = dpaa2_sec_stats_get,
3687         .stats_reset          = dpaa2_sec_stats_reset,
3688         .queue_pair_setup     = dpaa2_sec_queue_pair_setup,
3689         .queue_pair_release   = dpaa2_sec_queue_pair_release,
3690         .sym_session_get_size     = dpaa2_sec_sym_session_get_size,
3691         .sym_session_configure    = dpaa2_sec_sym_session_configure,
3692         .sym_session_clear        = dpaa2_sec_sym_session_clear,
3693 };
3694
3695 #ifdef RTE_LIBRTE_SECURITY
3696 static const struct rte_security_capability *
3697 dpaa2_sec_capabilities_get(void *device __rte_unused)
3698 {
3699         return dpaa2_sec_security_cap;
3700 }
3701
3702 static const struct rte_security_ops dpaa2_sec_security_ops = {
3703         .session_create = dpaa2_sec_security_session_create,
3704         .session_update = NULL,
3705         .session_stats_get = NULL,
3706         .session_destroy = dpaa2_sec_security_session_destroy,
3707         .set_pkt_metadata = NULL,
3708         .capabilities_get = dpaa2_sec_capabilities_get
3709 };
3710 #endif
3711
3712 static int
3713 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
3714 {
3715         struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3716
3717         rte_free(dev->security_ctx);
3718
3719         rte_mempool_free(internals->fle_pool);
3720
3721         DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
3722                        dev->data->name, rte_socket_id());
3723
3724         return 0;
3725 }
3726
3727 static int
3728 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
3729 {
3730         struct dpaa2_sec_dev_private *internals;
3731         struct rte_device *dev = cryptodev->device;
3732         struct rte_dpaa2_device *dpaa2_dev;
3733 #ifdef RTE_LIBRTE_SECURITY
3734         struct rte_security_ctx *security_instance;
3735 #endif
3736         struct fsl_mc_io *dpseci;
3737         uint16_t token;
3738         struct dpseci_attr attr;
3739         int retcode, hw_id;
3740         char str[30];
3741
3742         PMD_INIT_FUNC_TRACE();
3743         dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
3744         hw_id = dpaa2_dev->object_id;
3745
3746         cryptodev->driver_id = cryptodev_driver_id;
3747         cryptodev->dev_ops = &crypto_ops;
3748
3749         cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
3750         cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
3751         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3752                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
3753                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3754                         RTE_CRYPTODEV_FF_SECURITY |
3755                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3756                         RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
3757                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3758                         RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
3759                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
3760
3761         internals = cryptodev->data->dev_private;
3762
3763         /*
3764          * For secondary processes, we don't initialise any further as primary
3765          * has already done this work. Only check we don't need a different
3766          * RX function
3767          */
3768         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3769                 DPAA2_SEC_DEBUG("Device already init by primary process");
3770                 return 0;
3771         }
3772 #ifdef RTE_LIBRTE_SECURITY
3773         /* Initialize security_ctx only for primary process*/
3774         security_instance = rte_malloc("rte_security_instances_ops",
3775                                 sizeof(struct rte_security_ctx), 0);
3776         if (security_instance == NULL)
3777                 return -ENOMEM;
3778         security_instance->device = (void *)cryptodev;
3779         security_instance->ops = &dpaa2_sec_security_ops;
3780         security_instance->sess_cnt = 0;
3781         cryptodev->security_ctx = security_instance;
3782 #endif
3783         /*Open the rte device via MC and save the handle for further use*/
3784         dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
3785                                 sizeof(struct fsl_mc_io), 0);
3786         if (!dpseci) {
3787                 DPAA2_SEC_ERR(
3788                         "Error in allocating the memory for dpsec object");
3789                 return -ENOMEM;
3790         }
3791         dpseci->regs = rte_mcp_ptr_list[0];
3792
3793         retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
3794         if (retcode != 0) {
3795                 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
3796                               retcode);
3797                 goto init_error;
3798         }
3799         retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
3800         if (retcode != 0) {
3801                 DPAA2_SEC_ERR(
3802                              "Cannot get dpsec device attributed: Error = %x",
3803                              retcode);
3804                 goto init_error;
3805         }
3806         snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
3807                         "dpsec-%u", hw_id);
3808
3809         internals->max_nb_queue_pairs = attr.num_tx_queues;
3810         cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
3811         internals->hw = dpseci;
3812         internals->token = token;
3813
3814         snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d",
3815                         getpid(), cryptodev->data->dev_id);
3816         internals->fle_pool = rte_mempool_create((const char *)str,
3817                         FLE_POOL_NUM_BUFS,
3818                         FLE_POOL_BUF_SIZE,
3819                         FLE_POOL_CACHE_SIZE, 0,
3820                         NULL, NULL, NULL, NULL,
3821                         SOCKET_ID_ANY, 0);
3822         if (!internals->fle_pool) {
3823                 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
3824                 goto init_error;
3825         }
3826
3827         DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
3828         return 0;
3829
3830 init_error:
3831         DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
3832
3833         /* dpaa2_sec_uninit(crypto_dev_name); */
3834         return -EFAULT;
3835 }
3836
3837 static int
3838 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
3839                           struct rte_dpaa2_device *dpaa2_dev)
3840 {
3841         struct rte_cryptodev *cryptodev;
3842         char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
3843
3844         int retval;
3845
3846         snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
3847                         dpaa2_dev->object_id);
3848
3849         cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
3850         if (cryptodev == NULL)
3851                 return -ENOMEM;
3852
3853         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3854                 cryptodev->data->dev_private = rte_zmalloc_socket(
3855                                         "cryptodev private structure",
3856                                         sizeof(struct dpaa2_sec_dev_private),
3857                                         RTE_CACHE_LINE_SIZE,
3858                                         rte_socket_id());
3859
3860                 if (cryptodev->data->dev_private == NULL)
3861                         rte_panic("Cannot allocate memzone for private "
3862                                   "device data");
3863         }
3864
3865         dpaa2_dev->cryptodev = cryptodev;
3866         cryptodev->device = &dpaa2_dev->device;
3867
3868         /* init user callbacks */
3869         TAILQ_INIT(&(cryptodev->link_intr_cbs));
3870
3871         if (dpaa2_svr_family == SVR_LX2160A)
3872                 rta_set_sec_era(RTA_SEC_ERA_10);
3873
3874         DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
3875
3876         /* Invoke PMD device initialization function */
3877         retval = dpaa2_sec_dev_init(cryptodev);
3878         if (retval == 0)
3879                 return 0;
3880
3881         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
3882                 rte_free(cryptodev->data->dev_private);
3883
3884         cryptodev->attached = RTE_CRYPTODEV_DETACHED;
3885
3886         return -ENXIO;
3887 }
3888
3889 static int
3890 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
3891 {
3892         struct rte_cryptodev *cryptodev;
3893         int ret;
3894
3895         cryptodev = dpaa2_dev->cryptodev;
3896         if (cryptodev == NULL)
3897                 return -ENODEV;
3898
3899         ret = dpaa2_sec_uninit(cryptodev);
3900         if (ret)
3901                 return ret;
3902
3903         return rte_cryptodev_pmd_destroy(cryptodev);
3904 }
3905
3906 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
3907         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
3908         .drv_type = DPAA2_CRYPTO,
3909         .driver = {
3910                 .name = "DPAA2 SEC PMD"
3911         },
3912         .probe = cryptodev_dpaa2_sec_probe,
3913         .remove = cryptodev_dpaa2_sec_remove,
3914 };
3915
3916 static struct cryptodev_driver dpaa2_sec_crypto_drv;
3917
3918 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
3919 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
3920                 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
3921
3922 RTE_INIT(dpaa2_sec_init_log)
3923 {
3924         /* Bus level logs */
3925         dpaa2_logtype_sec = rte_log_register("pmd.crypto.dpaa2");
3926         if (dpaa2_logtype_sec >= 0)
3927                 rte_log_set_level(dpaa2_logtype_sec, RTE_LOG_NOTICE);
3928 }