1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4 * Copyright 2016-2021 NXP
14 #include <rte_cryptodev.h>
15 #include <rte_malloc.h>
16 #include <rte_memcpy.h>
17 #include <rte_string_fns.h>
18 #include <rte_cycles.h>
19 #include <rte_kvargs.h>
21 #include <cryptodev_pmd.h>
22 #include <rte_common.h>
23 #include <rte_fslmc.h>
24 #include <fslmc_vfio.h>
25 #include <dpaa2_hw_pvt.h>
26 #include <dpaa2_hw_dpio.h>
27 #include <dpaa2_hw_mempool.h>
28 #include <fsl_dpopr.h>
29 #include <fsl_dpseci.h>
30 #include <fsl_mc_sys.h>
31 #include <rte_hexdump.h>
33 #include "dpaa2_sec_priv.h"
34 #include "dpaa2_sec_event.h"
35 #include "dpaa2_sec_logs.h"
37 /* RTA header files */
38 #include <desc/ipsec.h>
39 #include <desc/pdcp.h>
40 #include <desc/sdap.h>
41 #include <desc/algo.h>
43 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
44 * a pointer to the shared descriptor
46 #define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ)
47 #define FSL_VENDOR_ID 0x1957
48 #define FSL_DEVICE_ID 0x410
49 #define FSL_SUBSYSTEM_SEC 1
50 #define FSL_MC_DPSECI_DEVID 3
54 #define DRIVER_DUMP_MODE "drv_dump_mode"
56 /* DPAA2_SEC_DP_DUMP levels */
57 enum dpaa2_sec_dump_levels {
59 DPAA2_SEC_DP_ERR_DUMP,
60 DPAA2_SEC_DP_FULL_DUMP
63 uint8_t cryptodev_driver_id;
64 uint8_t dpaa2_sec_dp_dump = DPAA2_SEC_DP_ERR_DUMP;
66 #ifdef RTE_LIB_SECURITY
68 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
69 struct rte_crypto_op *op,
70 struct qbman_fd *fd, uint16_t bpid)
72 struct rte_crypto_sym_op *sym_op = op->sym;
73 struct ctxt_priv *priv = sess->ctxt;
74 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
75 struct sec_flow_context *flc;
76 struct rte_mbuf *mbuf;
77 uint32_t in_len = 0, out_len = 0;
84 /* first FLE entry used to store mbuf and session ctxt */
85 fle = (struct qbman_fle *)rte_malloc(NULL,
86 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
89 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
92 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
93 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
94 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
96 /* Save the shared descriptor */
97 flc = &priv->flc_desc[0].flc;
103 if (likely(bpid < MAX_BPID)) {
104 DPAA2_SET_FD_BPID(fd, bpid);
105 DPAA2_SET_FLE_BPID(op_fle, bpid);
106 DPAA2_SET_FLE_BPID(ip_fle, bpid);
108 DPAA2_SET_FD_IVP(fd);
109 DPAA2_SET_FLE_IVP(op_fle);
110 DPAA2_SET_FLE_IVP(ip_fle);
113 /* Configure FD as a FRAME LIST */
114 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
115 DPAA2_SET_FD_COMPOUND_FMT(fd);
116 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
118 /* Configure Output FLE with Scatter/Gather Entry */
119 DPAA2_SET_FLE_SG_EXT(op_fle);
120 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
122 /* Configure Output SGE for Encap/Decap */
123 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
124 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
127 sge->length = mbuf->data_len;
128 out_len += sge->length;
131 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
132 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
134 /* using buf_len for last buf - so that extra data can be added */
135 sge->length = mbuf->buf_len - mbuf->data_off;
136 out_len += sge->length;
138 DPAA2_SET_FLE_FIN(sge);
139 op_fle->length = out_len;
142 mbuf = sym_op->m_src;
144 /* Configure Input FLE with Scatter/Gather Entry */
145 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
146 DPAA2_SET_FLE_SG_EXT(ip_fle);
147 DPAA2_SET_FLE_FIN(ip_fle);
149 /* Configure input SGE for Encap/Decap */
150 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
151 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
152 sge->length = mbuf->data_len;
153 in_len += sge->length;
159 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
160 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
161 sge->length = mbuf->data_len;
162 in_len += sge->length;
165 ip_fle->length = in_len;
166 DPAA2_SET_FLE_FIN(sge);
168 /* In case of PDCP, per packet HFN is stored in
169 * mbuf priv after sym_op.
171 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
172 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
173 sess->pdcp.hfn_ovd_offset);
174 /*enable HFN override override */
175 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
176 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
177 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
179 DPAA2_SET_FD_LEN(fd, ip_fle->length);
185 build_proto_compound_fd(dpaa2_sec_session *sess,
186 struct rte_crypto_op *op,
187 struct qbman_fd *fd, uint16_t bpid)
189 struct rte_crypto_sym_op *sym_op = op->sym;
190 struct ctxt_priv *priv = sess->ctxt;
191 struct qbman_fle *fle, *ip_fle, *op_fle;
192 struct sec_flow_context *flc;
193 struct rte_mbuf *src_mbuf = sym_op->m_src;
194 struct rte_mbuf *dst_mbuf = sym_op->m_dst;
200 /* Save the shared descriptor */
201 flc = &priv->flc_desc[0].flc;
203 /* we are using the first FLE entry to store Mbuf */
204 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
206 DPAA2_SEC_DP_ERR("Memory alloc failed");
209 memset(fle, 0, FLE_POOL_BUF_SIZE);
210 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
211 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
216 if (likely(bpid < MAX_BPID)) {
217 DPAA2_SET_FD_BPID(fd, bpid);
218 DPAA2_SET_FLE_BPID(op_fle, bpid);
219 DPAA2_SET_FLE_BPID(ip_fle, bpid);
221 DPAA2_SET_FD_IVP(fd);
222 DPAA2_SET_FLE_IVP(op_fle);
223 DPAA2_SET_FLE_IVP(ip_fle);
226 /* Configure FD as a FRAME LIST */
227 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
228 DPAA2_SET_FD_COMPOUND_FMT(fd);
229 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
231 /* Configure Output FLE with dst mbuf data */
232 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
233 DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
234 DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
236 /* Configure Input FLE with src mbuf data */
237 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
238 DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
239 DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
241 DPAA2_SET_FD_LEN(fd, ip_fle->length);
242 DPAA2_SET_FLE_FIN(ip_fle);
244 /* In case of PDCP, per packet HFN is stored in
245 * mbuf priv after sym_op.
247 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
248 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
249 sess->pdcp.hfn_ovd_offset);
250 /*enable HFN override override */
251 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
252 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
253 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
261 build_proto_fd(dpaa2_sec_session *sess,
262 struct rte_crypto_op *op,
263 struct qbman_fd *fd, uint16_t bpid)
265 struct rte_crypto_sym_op *sym_op = op->sym;
267 return build_proto_compound_fd(sess, op, fd, bpid);
269 struct ctxt_priv *priv = sess->ctxt;
270 struct sec_flow_context *flc;
271 struct rte_mbuf *mbuf = sym_op->m_src;
273 if (likely(bpid < MAX_BPID))
274 DPAA2_SET_FD_BPID(fd, bpid);
276 DPAA2_SET_FD_IVP(fd);
278 /* Save the shared descriptor */
279 flc = &priv->flc_desc[0].flc;
281 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
282 DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
283 DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
284 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
286 /* save physical address of mbuf */
287 op->sym->aead.digest.phys_addr = mbuf->buf_iova;
288 mbuf->buf_iova = (size_t)op;
295 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
296 struct rte_crypto_op *op,
297 struct qbman_fd *fd, __rte_unused uint16_t bpid)
299 struct rte_crypto_sym_op *sym_op = op->sym;
300 struct ctxt_priv *priv = sess->ctxt;
301 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
302 struct sec_flow_context *flc;
303 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
304 int icv_len = sess->digest_length;
306 struct rte_mbuf *mbuf;
307 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
311 mbuf = sym_op->m_dst;
313 mbuf = sym_op->m_src;
315 /* first FLE entry used to store mbuf and session ctxt */
316 fle = (struct qbman_fle *)rte_malloc(NULL,
317 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
318 RTE_CACHE_LINE_SIZE);
319 if (unlikely(!fle)) {
320 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
323 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
324 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
325 DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
331 /* Save the shared descriptor */
332 flc = &priv->flc_desc[0].flc;
334 /* Configure FD as a FRAME LIST */
335 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
336 DPAA2_SET_FD_COMPOUND_FMT(fd);
337 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
339 DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
340 "iv-len=%d data_off: 0x%x\n",
341 sym_op->aead.data.offset,
342 sym_op->aead.data.length,
345 sym_op->m_src->data_off);
347 /* Configure Output FLE with Scatter/Gather Entry */
348 DPAA2_SET_FLE_SG_EXT(op_fle);
349 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
352 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
354 op_fle->length = (sess->dir == DIR_ENC) ?
355 (sym_op->aead.data.length + icv_len) :
356 sym_op->aead.data.length;
358 /* Configure Output SGE for Encap/Decap */
359 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
360 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
361 sge->length = mbuf->data_len - sym_op->aead.data.offset;
367 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
368 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
369 sge->length = mbuf->data_len;
372 sge->length -= icv_len;
374 if (sess->dir == DIR_ENC) {
376 DPAA2_SET_FLE_ADDR(sge,
377 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
378 sge->length = icv_len;
380 DPAA2_SET_FLE_FIN(sge);
383 mbuf = sym_op->m_src;
385 /* Configure Input FLE with Scatter/Gather Entry */
386 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
387 DPAA2_SET_FLE_SG_EXT(ip_fle);
388 DPAA2_SET_FLE_FIN(ip_fle);
389 ip_fle->length = (sess->dir == DIR_ENC) ?
390 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
391 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
394 /* Configure Input SGE for Encap/Decap */
395 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
396 sge->length = sess->iv.length;
400 DPAA2_SET_FLE_ADDR(sge,
401 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
402 sge->length = auth_only_len;
406 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
407 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
409 sge->length = mbuf->data_len - sym_op->aead.data.offset;
415 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
416 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
417 sge->length = mbuf->data_len;
421 if (sess->dir == DIR_DEC) {
423 old_icv = (uint8_t *)(sge + 1);
424 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
425 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
426 sge->length = icv_len;
429 DPAA2_SET_FLE_FIN(sge);
431 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
432 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
434 DPAA2_SET_FD_LEN(fd, ip_fle->length);
440 build_authenc_gcm_fd(dpaa2_sec_session *sess,
441 struct rte_crypto_op *op,
442 struct qbman_fd *fd, uint16_t bpid)
444 struct rte_crypto_sym_op *sym_op = op->sym;
445 struct ctxt_priv *priv = sess->ctxt;
446 struct qbman_fle *fle, *sge;
447 struct sec_flow_context *flc;
448 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
449 int icv_len = sess->digest_length, retval;
451 struct rte_mbuf *dst;
452 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
460 /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
461 * Currently we donot know which FLE has the mbuf stored.
462 * So while retreiving we can go back 1 FLE from the FD -ADDR
463 * to get the MBUF Addr from the previous FLE.
464 * We can have a better approach to use the inline Mbuf
466 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
468 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE");
471 memset(fle, 0, FLE_POOL_BUF_SIZE);
472 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
473 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
476 if (likely(bpid < MAX_BPID)) {
477 DPAA2_SET_FD_BPID(fd, bpid);
478 DPAA2_SET_FLE_BPID(fle, bpid);
479 DPAA2_SET_FLE_BPID(fle + 1, bpid);
480 DPAA2_SET_FLE_BPID(sge, bpid);
481 DPAA2_SET_FLE_BPID(sge + 1, bpid);
482 DPAA2_SET_FLE_BPID(sge + 2, bpid);
483 DPAA2_SET_FLE_BPID(sge + 3, bpid);
485 DPAA2_SET_FD_IVP(fd);
486 DPAA2_SET_FLE_IVP(fle);
487 DPAA2_SET_FLE_IVP((fle + 1));
488 DPAA2_SET_FLE_IVP(sge);
489 DPAA2_SET_FLE_IVP((sge + 1));
490 DPAA2_SET_FLE_IVP((sge + 2));
491 DPAA2_SET_FLE_IVP((sge + 3));
494 /* Save the shared descriptor */
495 flc = &priv->flc_desc[0].flc;
496 /* Configure FD as a FRAME LIST */
497 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
498 DPAA2_SET_FD_COMPOUND_FMT(fd);
499 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
501 DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
502 "iv-len=%d data_off: 0x%x\n",
503 sym_op->aead.data.offset,
504 sym_op->aead.data.length,
507 sym_op->m_src->data_off);
509 /* Configure Output FLE with Scatter/Gather Entry */
510 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
512 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
513 fle->length = (sess->dir == DIR_ENC) ?
514 (sym_op->aead.data.length + icv_len) :
515 sym_op->aead.data.length;
517 DPAA2_SET_FLE_SG_EXT(fle);
519 /* Configure Output SGE for Encap/Decap */
520 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
521 DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
522 sge->length = sym_op->aead.data.length;
524 if (sess->dir == DIR_ENC) {
526 DPAA2_SET_FLE_ADDR(sge,
527 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
528 sge->length = sess->digest_length;
530 DPAA2_SET_FLE_FIN(sge);
535 /* Configure Input FLE with Scatter/Gather Entry */
536 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
537 DPAA2_SET_FLE_SG_EXT(fle);
538 DPAA2_SET_FLE_FIN(fle);
539 fle->length = (sess->dir == DIR_ENC) ?
540 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
541 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
542 sess->digest_length);
544 /* Configure Input SGE for Encap/Decap */
545 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
546 sge->length = sess->iv.length;
549 DPAA2_SET_FLE_ADDR(sge,
550 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
551 sge->length = auth_only_len;
552 DPAA2_SET_FLE_BPID(sge, bpid);
556 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
557 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
558 sym_op->m_src->data_off);
559 sge->length = sym_op->aead.data.length;
560 if (sess->dir == DIR_DEC) {
562 old_icv = (uint8_t *)(sge + 1);
563 memcpy(old_icv, sym_op->aead.digest.data,
564 sess->digest_length);
565 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
566 sge->length = sess->digest_length;
568 DPAA2_SET_FLE_FIN(sge);
571 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
572 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
575 DPAA2_SET_FD_LEN(fd, fle->length);
580 build_authenc_sg_fd(dpaa2_sec_session *sess,
581 struct rte_crypto_op *op,
582 struct qbman_fd *fd, __rte_unused uint16_t bpid)
584 struct rte_crypto_sym_op *sym_op = op->sym;
585 struct ctxt_priv *priv = sess->ctxt;
586 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
587 struct sec_flow_context *flc;
588 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
589 sym_op->auth.data.offset;
590 uint16_t auth_tail_len = sym_op->auth.data.length -
591 sym_op->cipher.data.length - auth_hdr_len;
592 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
593 int icv_len = sess->digest_length;
595 struct rte_mbuf *mbuf;
596 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
600 mbuf = sym_op->m_dst;
602 mbuf = sym_op->m_src;
604 /* first FLE entry used to store mbuf and session ctxt */
605 fle = (struct qbman_fle *)rte_malloc(NULL,
606 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
607 RTE_CACHE_LINE_SIZE);
608 if (unlikely(!fle)) {
609 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
612 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
613 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
614 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
620 /* Save the shared descriptor */
621 flc = &priv->flc_desc[0].flc;
623 /* Configure FD as a FRAME LIST */
624 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
625 DPAA2_SET_FD_COMPOUND_FMT(fd);
626 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
629 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
630 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
631 sym_op->auth.data.offset,
632 sym_op->auth.data.length,
634 sym_op->cipher.data.offset,
635 sym_op->cipher.data.length,
637 sym_op->m_src->data_off);
639 /* Configure Output FLE with Scatter/Gather Entry */
640 DPAA2_SET_FLE_SG_EXT(op_fle);
641 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
644 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
646 op_fle->length = (sess->dir == DIR_ENC) ?
647 (sym_op->cipher.data.length + icv_len) :
648 sym_op->cipher.data.length;
650 /* Configure Output SGE for Encap/Decap */
651 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
652 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
653 sge->length = mbuf->data_len - sym_op->auth.data.offset;
659 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
660 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
661 sge->length = mbuf->data_len;
664 sge->length -= icv_len;
666 if (sess->dir == DIR_ENC) {
668 DPAA2_SET_FLE_ADDR(sge,
669 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
670 sge->length = icv_len;
672 DPAA2_SET_FLE_FIN(sge);
675 mbuf = sym_op->m_src;
677 /* Configure Input FLE with Scatter/Gather Entry */
678 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
679 DPAA2_SET_FLE_SG_EXT(ip_fle);
680 DPAA2_SET_FLE_FIN(ip_fle);
681 ip_fle->length = (sess->dir == DIR_ENC) ?
682 (sym_op->auth.data.length + sess->iv.length) :
683 (sym_op->auth.data.length + sess->iv.length +
686 /* Configure Input SGE for Encap/Decap */
687 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
688 sge->length = sess->iv.length;
691 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
692 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
694 sge->length = mbuf->data_len - sym_op->auth.data.offset;
700 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
701 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
702 sge->length = mbuf->data_len;
705 sge->length -= icv_len;
707 if (sess->dir == DIR_DEC) {
709 old_icv = (uint8_t *)(sge + 1);
710 memcpy(old_icv, sym_op->auth.digest.data,
712 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
713 sge->length = icv_len;
716 DPAA2_SET_FLE_FIN(sge);
718 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
719 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
721 DPAA2_SET_FD_LEN(fd, ip_fle->length);
727 build_authenc_fd(dpaa2_sec_session *sess,
728 struct rte_crypto_op *op,
729 struct qbman_fd *fd, uint16_t bpid)
731 struct rte_crypto_sym_op *sym_op = op->sym;
732 struct ctxt_priv *priv = sess->ctxt;
733 struct qbman_fle *fle, *sge;
734 struct sec_flow_context *flc;
735 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
736 sym_op->auth.data.offset;
737 uint16_t auth_tail_len = sym_op->auth.data.length -
738 sym_op->cipher.data.length - auth_hdr_len;
739 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
741 int icv_len = sess->digest_length, retval;
743 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
745 struct rte_mbuf *dst;
752 /* we are using the first FLE entry to store Mbuf.
753 * Currently we donot know which FLE has the mbuf stored.
754 * So while retreiving we can go back 1 FLE from the FD -ADDR
755 * to get the MBUF Addr from the previous FLE.
756 * We can have a better approach to use the inline Mbuf
758 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
760 DPAA2_SEC_ERR("Memory alloc failed for SGE");
763 memset(fle, 0, FLE_POOL_BUF_SIZE);
764 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
765 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
768 if (likely(bpid < MAX_BPID)) {
769 DPAA2_SET_FD_BPID(fd, bpid);
770 DPAA2_SET_FLE_BPID(fle, bpid);
771 DPAA2_SET_FLE_BPID(fle + 1, bpid);
772 DPAA2_SET_FLE_BPID(sge, bpid);
773 DPAA2_SET_FLE_BPID(sge + 1, bpid);
774 DPAA2_SET_FLE_BPID(sge + 2, bpid);
775 DPAA2_SET_FLE_BPID(sge + 3, bpid);
777 DPAA2_SET_FD_IVP(fd);
778 DPAA2_SET_FLE_IVP(fle);
779 DPAA2_SET_FLE_IVP((fle + 1));
780 DPAA2_SET_FLE_IVP(sge);
781 DPAA2_SET_FLE_IVP((sge + 1));
782 DPAA2_SET_FLE_IVP((sge + 2));
783 DPAA2_SET_FLE_IVP((sge + 3));
786 /* Save the shared descriptor */
787 flc = &priv->flc_desc[0].flc;
788 /* Configure FD as a FRAME LIST */
789 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
790 DPAA2_SET_FD_COMPOUND_FMT(fd);
791 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
794 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
795 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
796 sym_op->auth.data.offset,
797 sym_op->auth.data.length,
799 sym_op->cipher.data.offset,
800 sym_op->cipher.data.length,
802 sym_op->m_src->data_off);
804 /* Configure Output FLE with Scatter/Gather Entry */
805 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
807 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
808 fle->length = (sess->dir == DIR_ENC) ?
809 (sym_op->cipher.data.length + icv_len) :
810 sym_op->cipher.data.length;
812 DPAA2_SET_FLE_SG_EXT(fle);
814 /* Configure Output SGE for Encap/Decap */
815 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
816 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
818 sge->length = sym_op->cipher.data.length;
820 if (sess->dir == DIR_ENC) {
822 DPAA2_SET_FLE_ADDR(sge,
823 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
824 sge->length = sess->digest_length;
825 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
828 DPAA2_SET_FLE_FIN(sge);
833 /* Configure Input FLE with Scatter/Gather Entry */
834 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
835 DPAA2_SET_FLE_SG_EXT(fle);
836 DPAA2_SET_FLE_FIN(fle);
837 fle->length = (sess->dir == DIR_ENC) ?
838 (sym_op->auth.data.length + sess->iv.length) :
839 (sym_op->auth.data.length + sess->iv.length +
840 sess->digest_length);
842 /* Configure Input SGE for Encap/Decap */
843 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
844 sge->length = sess->iv.length;
847 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
848 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
849 sym_op->m_src->data_off);
850 sge->length = sym_op->auth.data.length;
851 if (sess->dir == DIR_DEC) {
853 old_icv = (uint8_t *)(sge + 1);
854 memcpy(old_icv, sym_op->auth.digest.data,
855 sess->digest_length);
856 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
857 sge->length = sess->digest_length;
858 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
859 sess->digest_length +
862 DPAA2_SET_FLE_FIN(sge);
864 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
865 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
870 static inline int build_auth_sg_fd(
871 dpaa2_sec_session *sess,
872 struct rte_crypto_op *op,
874 __rte_unused uint16_t bpid)
876 struct rte_crypto_sym_op *sym_op = op->sym;
877 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
878 struct sec_flow_context *flc;
879 struct ctxt_priv *priv = sess->ctxt;
880 int data_len, data_offset;
882 struct rte_mbuf *mbuf;
884 data_len = sym_op->auth.data.length;
885 data_offset = sym_op->auth.data.offset;
887 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
888 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
889 if ((data_len & 7) || (data_offset & 7)) {
890 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
894 data_len = data_len >> 3;
895 data_offset = data_offset >> 3;
898 mbuf = sym_op->m_src;
899 fle = (struct qbman_fle *)rte_malloc(NULL,
900 FLE_SG_MEM_SIZE(mbuf->nb_segs),
901 RTE_CACHE_LINE_SIZE);
902 if (unlikely(!fle)) {
903 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
906 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
907 /* first FLE entry used to store mbuf and session ctxt */
908 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
909 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
914 flc = &priv->flc_desc[DESC_INITFINAL].flc;
916 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
917 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
918 DPAA2_SET_FD_COMPOUND_FMT(fd);
921 DPAA2_SET_FLE_ADDR(op_fle,
922 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
923 op_fle->length = sess->digest_length;
926 DPAA2_SET_FLE_SG_EXT(ip_fle);
927 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
928 ip_fle->length = data_len;
930 if (sess->iv.length) {
933 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
936 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
937 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
939 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
940 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
943 sge->length = sess->iv.length;
945 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
946 ip_fle->length += sge->length;
950 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
951 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
953 if (data_len <= (mbuf->data_len - data_offset)) {
954 sge->length = data_len;
957 sge->length = mbuf->data_len - data_offset;
959 /* remaining i/p segs */
960 while ((data_len = data_len - sge->length) &&
961 (mbuf = mbuf->next)) {
963 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
964 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
965 if (data_len > mbuf->data_len)
966 sge->length = mbuf->data_len;
968 sge->length = data_len;
972 if (sess->dir == DIR_DEC) {
973 /* Digest verification case */
975 old_digest = (uint8_t *)(sge + 1);
976 rte_memcpy(old_digest, sym_op->auth.digest.data,
977 sess->digest_length);
978 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
979 sge->length = sess->digest_length;
980 ip_fle->length += sess->digest_length;
982 DPAA2_SET_FLE_FIN(sge);
983 DPAA2_SET_FLE_FIN(ip_fle);
984 DPAA2_SET_FD_LEN(fd, ip_fle->length);
990 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
991 struct qbman_fd *fd, uint16_t bpid)
993 struct rte_crypto_sym_op *sym_op = op->sym;
994 struct qbman_fle *fle, *sge;
995 struct sec_flow_context *flc;
996 struct ctxt_priv *priv = sess->ctxt;
997 int data_len, data_offset;
1001 data_len = sym_op->auth.data.length;
1002 data_offset = sym_op->auth.data.offset;
1004 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1005 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1006 if ((data_len & 7) || (data_offset & 7)) {
1007 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1011 data_len = data_len >> 3;
1012 data_offset = data_offset >> 3;
1015 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1017 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE");
1020 memset(fle, 0, FLE_POOL_BUF_SIZE);
1021 /* TODO we are using the first FLE entry to store Mbuf.
1022 * Currently we donot know which FLE has the mbuf stored.
1023 * So while retreiving we can go back 1 FLE from the FD -ADDR
1024 * to get the MBUF Addr from the previous FLE.
1025 * We can have a better approach to use the inline Mbuf
1027 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1028 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1032 if (likely(bpid < MAX_BPID)) {
1033 DPAA2_SET_FD_BPID(fd, bpid);
1034 DPAA2_SET_FLE_BPID(fle, bpid);
1035 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1036 DPAA2_SET_FLE_BPID(sge, bpid);
1037 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1039 DPAA2_SET_FD_IVP(fd);
1040 DPAA2_SET_FLE_IVP(fle);
1041 DPAA2_SET_FLE_IVP((fle + 1));
1042 DPAA2_SET_FLE_IVP(sge);
1043 DPAA2_SET_FLE_IVP((sge + 1));
1046 flc = &priv->flc_desc[DESC_INITFINAL].flc;
1047 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1048 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1049 DPAA2_SET_FD_COMPOUND_FMT(fd);
1051 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1052 fle->length = sess->digest_length;
1055 /* Setting input FLE */
1056 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1057 DPAA2_SET_FLE_SG_EXT(fle);
1058 fle->length = data_len;
1060 if (sess->iv.length) {
1063 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1066 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1067 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1069 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1070 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1073 sge->length = sess->iv.length;
1076 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1077 fle->length = fle->length + sge->length;
1081 /* Setting data to authenticate */
1082 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1083 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1084 sge->length = data_len;
1086 if (sess->dir == DIR_DEC) {
1088 old_digest = (uint8_t *)(sge + 1);
1089 rte_memcpy(old_digest, sym_op->auth.digest.data,
1090 sess->digest_length);
1091 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1092 sge->length = sess->digest_length;
1093 fle->length = fle->length + sess->digest_length;
1096 DPAA2_SET_FLE_FIN(sge);
1097 DPAA2_SET_FLE_FIN(fle);
1098 DPAA2_SET_FD_LEN(fd, fle->length);
1104 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1105 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1107 struct rte_crypto_sym_op *sym_op = op->sym;
1108 struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1109 int data_len, data_offset;
1110 struct sec_flow_context *flc;
1111 struct ctxt_priv *priv = sess->ctxt;
1112 struct rte_mbuf *mbuf;
1113 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1116 data_len = sym_op->cipher.data.length;
1117 data_offset = sym_op->cipher.data.offset;
1119 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1120 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1121 if ((data_len & 7) || (data_offset & 7)) {
1122 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1126 data_len = data_len >> 3;
1127 data_offset = data_offset >> 3;
1131 mbuf = sym_op->m_dst;
1133 mbuf = sym_op->m_src;
1135 /* first FLE entry used to store mbuf and session ctxt */
1136 fle = (struct qbman_fle *)rte_malloc(NULL,
1137 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1138 RTE_CACHE_LINE_SIZE);
1140 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1143 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1144 /* first FLE entry used to store mbuf and session ctxt */
1145 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1146 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1152 flc = &priv->flc_desc[0].flc;
1155 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1156 " data_off: 0x%x\n",
1160 sym_op->m_src->data_off);
1163 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1164 op_fle->length = data_len;
1165 DPAA2_SET_FLE_SG_EXT(op_fle);
1168 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1169 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1170 sge->length = mbuf->data_len - data_offset;
1176 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1177 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1178 sge->length = mbuf->data_len;
1181 DPAA2_SET_FLE_FIN(sge);
1184 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1185 flc, fle, fle->addr_hi, fle->addr_lo,
1189 mbuf = sym_op->m_src;
1191 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1192 ip_fle->length = sess->iv.length + data_len;
1193 DPAA2_SET_FLE_SG_EXT(ip_fle);
1196 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1197 DPAA2_SET_FLE_OFFSET(sge, 0);
1198 sge->length = sess->iv.length;
1203 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1204 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1205 sge->length = mbuf->data_len - data_offset;
1211 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1212 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1213 sge->length = mbuf->data_len;
1216 DPAA2_SET_FLE_FIN(sge);
1217 DPAA2_SET_FLE_FIN(ip_fle);
1220 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1221 DPAA2_SET_FD_LEN(fd, ip_fle->length);
1222 DPAA2_SET_FD_COMPOUND_FMT(fd);
1223 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1226 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1227 " off =%d, len =%d\n",
1228 DPAA2_GET_FD_ADDR(fd),
1229 DPAA2_GET_FD_BPID(fd),
1230 rte_dpaa2_bpid_info[bpid].meta_data_size,
1231 DPAA2_GET_FD_OFFSET(fd),
1232 DPAA2_GET_FD_LEN(fd));
1237 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1238 struct qbman_fd *fd, uint16_t bpid)
1240 struct rte_crypto_sym_op *sym_op = op->sym;
1241 struct qbman_fle *fle, *sge;
1242 int retval, data_len, data_offset;
1243 struct sec_flow_context *flc;
1244 struct ctxt_priv *priv = sess->ctxt;
1245 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1247 struct rte_mbuf *dst;
1249 data_len = sym_op->cipher.data.length;
1250 data_offset = sym_op->cipher.data.offset;
1252 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1253 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1254 if ((data_len & 7) || (data_offset & 7)) {
1255 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1259 data_len = data_len >> 3;
1260 data_offset = data_offset >> 3;
1264 dst = sym_op->m_dst;
1266 dst = sym_op->m_src;
1268 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1270 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE");
1273 memset(fle, 0, FLE_POOL_BUF_SIZE);
1274 /* TODO we are using the first FLE entry to store Mbuf.
1275 * Currently we donot know which FLE has the mbuf stored.
1276 * So while retreiving we can go back 1 FLE from the FD -ADDR
1277 * to get the MBUF Addr from the previous FLE.
1278 * We can have a better approach to use the inline Mbuf
1280 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1281 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1285 if (likely(bpid < MAX_BPID)) {
1286 DPAA2_SET_FD_BPID(fd, bpid);
1287 DPAA2_SET_FLE_BPID(fle, bpid);
1288 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1289 DPAA2_SET_FLE_BPID(sge, bpid);
1290 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1292 DPAA2_SET_FD_IVP(fd);
1293 DPAA2_SET_FLE_IVP(fle);
1294 DPAA2_SET_FLE_IVP((fle + 1));
1295 DPAA2_SET_FLE_IVP(sge);
1296 DPAA2_SET_FLE_IVP((sge + 1));
1299 flc = &priv->flc_desc[0].flc;
1300 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1301 DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1302 DPAA2_SET_FD_COMPOUND_FMT(fd);
1303 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1306 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1307 " data_off: 0x%x\n",
1311 sym_op->m_src->data_off);
1313 DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1314 DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1316 fle->length = data_len + sess->iv.length;
1319 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1320 flc, fle, fle->addr_hi, fle->addr_lo,
1325 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1326 fle->length = data_len + sess->iv.length;
1328 DPAA2_SET_FLE_SG_EXT(fle);
1330 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1331 sge->length = sess->iv.length;
1334 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1335 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1337 sge->length = data_len;
1338 DPAA2_SET_FLE_FIN(sge);
1339 DPAA2_SET_FLE_FIN(fle);
1342 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1343 " off =%d, len =%d\n",
1344 DPAA2_GET_FD_ADDR(fd),
1345 DPAA2_GET_FD_BPID(fd),
1346 rte_dpaa2_bpid_info[bpid].meta_data_size,
1347 DPAA2_GET_FD_OFFSET(fd),
1348 DPAA2_GET_FD_LEN(fd));
1354 build_sec_fd(struct rte_crypto_op *op,
1355 struct qbman_fd *fd, uint16_t bpid)
1358 dpaa2_sec_session *sess;
1360 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1361 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1362 op->sym->session, cryptodev_driver_id);
1363 #ifdef RTE_LIB_SECURITY
1364 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1365 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1366 op->sym->sec_session);
1374 /* Any of the buffer is segmented*/
1375 if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1376 ((op->sym->m_dst != NULL) &&
1377 !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1378 switch (sess->ctxt_type) {
1379 case DPAA2_SEC_CIPHER:
1380 ret = build_cipher_sg_fd(sess, op, fd, bpid);
1382 case DPAA2_SEC_AUTH:
1383 ret = build_auth_sg_fd(sess, op, fd, bpid);
1385 case DPAA2_SEC_AEAD:
1386 ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1388 case DPAA2_SEC_CIPHER_HASH:
1389 ret = build_authenc_sg_fd(sess, op, fd, bpid);
1391 #ifdef RTE_LIB_SECURITY
1392 case DPAA2_SEC_IPSEC:
1393 case DPAA2_SEC_PDCP:
1394 ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1397 case DPAA2_SEC_HASH_CIPHER:
1399 DPAA2_SEC_ERR("error: Unsupported session");
1402 switch (sess->ctxt_type) {
1403 case DPAA2_SEC_CIPHER:
1404 ret = build_cipher_fd(sess, op, fd, bpid);
1406 case DPAA2_SEC_AUTH:
1407 ret = build_auth_fd(sess, op, fd, bpid);
1409 case DPAA2_SEC_AEAD:
1410 ret = build_authenc_gcm_fd(sess, op, fd, bpid);
1412 case DPAA2_SEC_CIPHER_HASH:
1413 ret = build_authenc_fd(sess, op, fd, bpid);
1415 #ifdef RTE_LIB_SECURITY
1416 case DPAA2_SEC_IPSEC:
1417 ret = build_proto_fd(sess, op, fd, bpid);
1419 case DPAA2_SEC_PDCP:
1420 ret = build_proto_compound_fd(sess, op, fd, bpid);
1423 case DPAA2_SEC_HASH_CIPHER:
1425 DPAA2_SEC_ERR("error: Unsupported session");
1433 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1436 /* Function to transmit the frames to given device and VQ*/
1439 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1440 uint32_t frames_to_send, retry_count;
1441 struct qbman_eq_desc eqdesc;
1442 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1443 struct qbman_swp *swp;
1444 uint16_t num_tx = 0;
1445 uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1446 /*todo - need to support multiple buffer pools */
1448 struct rte_mempool *mb_pool;
1450 if (unlikely(nb_ops == 0))
1453 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1454 DPAA2_SEC_ERR("sessionless crypto op not supported");
1457 /*Prepare enqueue descriptor*/
1458 qbman_eq_desc_clear(&eqdesc);
1459 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1460 qbman_eq_desc_set_response(&eqdesc, 0, 0);
1461 qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1463 if (!DPAA2_PER_LCORE_DPIO) {
1464 ret = dpaa2_affine_qbman_swp();
1467 "Failed to allocate IO portal, tid: %d\n",
1472 swp = DPAA2_PER_LCORE_PORTAL;
1475 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1476 dpaa2_eqcr_size : nb_ops;
1478 for (loop = 0; loop < frames_to_send; loop++) {
1479 if (*dpaa2_seqn((*ops)->sym->m_src)) {
1480 uint8_t dqrr_index =
1481 *dpaa2_seqn((*ops)->sym->m_src) - 1;
1483 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
1484 DPAA2_PER_LCORE_DQRR_SIZE--;
1485 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1486 *dpaa2_seqn((*ops)->sym->m_src) =
1487 DPAA2_INVALID_MBUF_SEQN;
1490 /*Clear the unused FD fields before sending*/
1491 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1492 mb_pool = (*ops)->sym->m_src->pool;
1493 bpid = mempool_to_bpid(mb_pool);
1494 ret = build_sec_fd(*ops, &fd_arr[loop], bpid);
1496 DPAA2_SEC_ERR("error: Improper packet contents"
1497 " for crypto operation");
1505 while (loop < frames_to_send) {
1506 ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1509 frames_to_send - loop);
1510 if (unlikely(ret < 0)) {
1512 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1527 dpaa2_qp->tx_vq.tx_pkts += num_tx;
1528 dpaa2_qp->tx_vq.err_pkts += nb_ops;
1532 #ifdef RTE_LIB_SECURITY
1533 static inline struct rte_crypto_op *
1534 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1536 struct rte_crypto_op *op;
1537 uint16_t len = DPAA2_GET_FD_LEN(fd);
1539 dpaa2_sec_session *sess_priv __rte_unused;
1541 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1542 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1543 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1545 diff = len - mbuf->pkt_len;
1546 mbuf->pkt_len += diff;
1547 mbuf->data_len += diff;
1548 op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1549 mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1550 op->sym->aead.digest.phys_addr = 0L;
1552 sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1553 op->sym->sec_session);
1554 if (sess_priv->dir == DIR_ENC)
1555 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1557 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1563 static inline struct rte_crypto_op *
1564 sec_fd_to_mbuf(const struct qbman_fd *fd)
1566 struct qbman_fle *fle;
1567 struct rte_crypto_op *op;
1568 struct ctxt_priv *priv;
1569 struct rte_mbuf *dst, *src;
1571 #ifdef RTE_LIB_SECURITY
1572 if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1573 return sec_simple_fd_to_mbuf(fd);
1575 fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1577 DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1578 fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1580 /* we are using the first FLE entry to store Mbuf.
1581 * Currently we donot know which FLE has the mbuf stored.
1582 * So while retreiving we can go back 1 FLE from the FD -ADDR
1583 * to get the MBUF Addr from the previous FLE.
1584 * We can have a better approach to use the inline Mbuf
1587 if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1588 /* TODO complete it. */
1589 DPAA2_SEC_ERR("error: non inline buffer");
1592 op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1595 src = op->sym->m_src;
1598 if (op->sym->m_dst) {
1599 dst = op->sym->m_dst;
1604 #ifdef RTE_LIB_SECURITY
1605 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1606 uint16_t len = DPAA2_GET_FD_LEN(fd);
1608 while (dst->next != NULL) {
1609 len -= dst->data_len;
1612 dst->data_len = len;
1615 DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1616 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1619 DPAA2_GET_FD_ADDR(fd),
1620 DPAA2_GET_FD_BPID(fd),
1621 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1622 DPAA2_GET_FD_OFFSET(fd),
1623 DPAA2_GET_FD_LEN(fd));
1625 /* free the fle memory */
1626 if (likely(rte_pktmbuf_is_contiguous(src))) {
1627 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1);
1628 rte_mempool_put(priv->fle_pool, (void *)(fle-1));
1630 rte_free((void *)(fle-1));
1636 dpaa2_sec_dump(struct rte_crypto_op *op)
1639 dpaa2_sec_session *sess = NULL;
1640 struct ctxt_priv *priv;
1642 struct rte_crypto_sym_op *sym_op;
1644 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1645 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1646 op->sym->session, cryptodev_driver_id);
1647 #ifdef RTE_LIBRTE_SECURITY
1648 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1649 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1650 op->sym->sec_session);
1656 priv = (struct ctxt_priv *)sess->ctxt;
1657 printf("\n****************************************\n"
1658 "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n"
1659 "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n"
1660 "\tCipher key len:\t%zd\n", sess->ctxt_type,
1661 (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC",
1662 sess->cipher_alg, sess->auth_alg, sess->aead_alg,
1663 sess->cipher_key.length);
1664 rte_hexdump(stdout, "cipher key", sess->cipher_key.data,
1665 sess->cipher_key.length);
1666 rte_hexdump(stdout, "auth key", sess->auth_key.data,
1667 sess->auth_key.length);
1668 printf("\tAuth key len:\t%zd\n\tIV len:\t\t%d\n\tIV offset:\t%d\n"
1669 "\tdigest length:\t%d\n\tstatus:\t\t%d\n\taead auth only"
1670 " len:\t%d\n\taead cipher text:\t%d\n",
1671 sess->auth_key.length, sess->iv.length, sess->iv.offset,
1672 sess->digest_length, sess->status,
1673 sess->ext_params.aead_ctxt.auth_only_len,
1674 sess->ext_params.aead_ctxt.auth_cipher_text);
1675 #ifdef RTE_LIBRTE_SECURITY
1676 printf("PDCP session params:\n"
1677 "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:"
1678 "\t%d\n\tsn_size:\t%d\n\thfn_ovd_offset:\t%d\n\thfn:\t\t%d\n"
1679 "\thfn_threshold:\t0x%x\n", sess->pdcp.domain,
1680 sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd,
1681 sess->pdcp.sn_size, sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn,
1682 sess->pdcp.hfn_threshold);
1685 bufsize = (uint8_t)priv->flc_desc[0].flc.word1_sdl;
1686 printf("Descriptor Dump:\n");
1687 for (i = 0; i < bufsize; i++)
1688 printf("\tDESC[%d]:0x%x\n", i, priv->flc_desc[0].desc[i]);
1693 if (sym_op->m_src) {
1694 printf("Source mbuf:\n");
1695 rte_pktmbuf_dump(stdout, sym_op->m_src, sym_op->m_src->data_len);
1697 if (sym_op->m_dst) {
1698 printf("Destination mbuf:\n");
1699 rte_pktmbuf_dump(stdout, sym_op->m_dst, sym_op->m_dst->data_len);
1702 printf("Session address = %p\ncipher offset: %d, length: %d\n"
1703 "auth offset: %d, length: %d\n aead offset: %d, length: %d\n"
1705 sym_op->cipher.data.offset, sym_op->cipher.data.length,
1706 sym_op->auth.data.offset, sym_op->auth.data.length,
1707 sym_op->aead.data.offset, sym_op->aead.data.length);
1713 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1716 /* Function is responsible to receive frames for a given device and VQ*/
1717 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1718 struct qbman_result *dq_storage;
1719 uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1720 int ret, num_rx = 0;
1721 uint8_t is_last = 0, status;
1722 struct qbman_swp *swp;
1723 const struct qbman_fd *fd;
1724 struct qbman_pull_desc pulldesc;
1726 if (!DPAA2_PER_LCORE_DPIO) {
1727 ret = dpaa2_affine_qbman_swp();
1730 "Failed to allocate IO portal, tid: %d\n",
1735 swp = DPAA2_PER_LCORE_PORTAL;
1736 dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1738 qbman_pull_desc_clear(&pulldesc);
1739 qbman_pull_desc_set_numframes(&pulldesc,
1740 (nb_ops > dpaa2_dqrr_size) ?
1741 dpaa2_dqrr_size : nb_ops);
1742 qbman_pull_desc_set_fq(&pulldesc, fqid);
1743 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1744 (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1747 /*Issue a volatile dequeue command. */
1749 if (qbman_swp_pull(swp, &pulldesc)) {
1751 "SEC VDQ command is not issued : QBMAN busy");
1752 /* Portal was busy, try again */
1758 /* Receive the packets till Last Dequeue entry is found with
1759 * respect to the above issues PULL command.
1762 /* Check if the previous issued command is completed.
1763 * Also seems like the SWP is shared between the Ethernet Driver
1764 * and the SEC driver.
1766 while (!qbman_check_command_complete(dq_storage))
1769 /* Loop until the dq_storage is updated with
1770 * new token by QBMAN
1772 while (!qbman_check_new_result(dq_storage))
1774 /* Check whether Last Pull command is Expired and
1775 * setting Condition for Loop termination
1777 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1779 /* Check for valid frame. */
1780 status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1782 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1783 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1788 fd = qbman_result_DQ_fd(dq_storage);
1789 ops[num_rx] = sec_fd_to_mbuf(fd);
1791 if (unlikely(fd->simple.frc)) {
1792 /* TODO Parse SEC errors */
1793 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_NO_DUMP) {
1794 DPAA2_SEC_DP_ERR("SEC returned Error - %x\n",
1796 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_ERR_DUMP)
1797 dpaa2_sec_dump(ops[num_rx]);
1800 dpaa2_qp->rx_vq.err_pkts += 1;
1801 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
1803 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1808 } /* End of Packet Rx loop */
1810 dpaa2_qp->rx_vq.rx_pkts += num_rx;
1812 DPAA2_SEC_DP_DEBUG("SEC RX pkts %d err pkts %" PRIu64 "\n", num_rx,
1813 dpaa2_qp->rx_vq.err_pkts);
1814 /*Return the total number of packets received to DPAA2 app*/
1818 /** Release queue pair */
1820 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
1822 struct dpaa2_sec_qp *qp =
1823 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
1825 PMD_INIT_FUNC_TRACE();
1827 if (qp->rx_vq.q_storage) {
1828 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
1829 rte_free(qp->rx_vq.q_storage);
1833 dev->data->queue_pairs[queue_pair_id] = NULL;
1838 /** Setup a queue pair */
1840 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1841 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1842 __rte_unused int socket_id)
1844 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
1845 struct dpaa2_sec_qp *qp;
1846 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
1847 struct dpseci_rx_queue_cfg cfg;
1850 PMD_INIT_FUNC_TRACE();
1852 /* If qp is already in use free ring memory and qp metadata. */
1853 if (dev->data->queue_pairs[qp_id] != NULL) {
1854 DPAA2_SEC_INFO("QP already setup");
1858 DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
1859 dev, qp_id, qp_conf);
1861 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
1863 qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
1864 RTE_CACHE_LINE_SIZE);
1866 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
1870 qp->rx_vq.crypto_data = dev->data;
1871 qp->tx_vq.crypto_data = dev->data;
1872 qp->rx_vq.q_storage = rte_malloc("sec dq storage",
1873 sizeof(struct queue_storage_info_t),
1874 RTE_CACHE_LINE_SIZE);
1875 if (!qp->rx_vq.q_storage) {
1876 DPAA2_SEC_ERR("malloc failed for q_storage");
1879 memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
1881 if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
1882 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
1886 dev->data->queue_pairs[qp_id] = qp;
1888 cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
1889 cfg.user_ctx = (size_t)(&qp->rx_vq);
1890 retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
1895 /** Returns the size of the aesni gcm session structure */
1897 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1899 PMD_INIT_FUNC_TRACE();
1901 return sizeof(dpaa2_sec_session);
1905 dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
1906 struct rte_crypto_sym_xform *xform,
1907 dpaa2_sec_session *session)
1909 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1910 struct alginfo cipherdata;
1911 int bufsize, ret = 0;
1912 struct ctxt_priv *priv;
1913 struct sec_flow_context *flc;
1915 PMD_INIT_FUNC_TRACE();
1917 /* For SEC CIPHER only one descriptor is required. */
1918 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1919 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
1920 RTE_CACHE_LINE_SIZE);
1922 DPAA2_SEC_ERR("No Memory for priv CTXT");
1926 priv->fle_pool = dev_priv->fle_pool;
1928 flc = &priv->flc_desc[0].flc;
1930 session->ctxt_type = DPAA2_SEC_CIPHER;
1931 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1932 RTE_CACHE_LINE_SIZE);
1933 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) {
1934 DPAA2_SEC_ERR("No Memory for cipher key");
1938 session->cipher_key.length = xform->cipher.key.length;
1940 memcpy(session->cipher_key.data, xform->cipher.key.data,
1941 xform->cipher.key.length);
1942 cipherdata.key = (size_t)session->cipher_key.data;
1943 cipherdata.keylen = session->cipher_key.length;
1944 cipherdata.key_enc_flags = 0;
1945 cipherdata.key_type = RTA_DATA_IMM;
1947 /* Set IV parameters */
1948 session->iv.offset = xform->cipher.iv.offset;
1949 session->iv.length = xform->cipher.iv.length;
1950 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1953 switch (xform->cipher.algo) {
1954 case RTE_CRYPTO_CIPHER_AES_CBC:
1955 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1956 cipherdata.algmode = OP_ALG_AAI_CBC;
1957 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1958 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1959 SHR_NEVER, &cipherdata,
1963 case RTE_CRYPTO_CIPHER_3DES_CBC:
1964 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1965 cipherdata.algmode = OP_ALG_AAI_CBC;
1966 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1967 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1968 SHR_NEVER, &cipherdata,
1972 case RTE_CRYPTO_CIPHER_DES_CBC:
1973 cipherdata.algtype = OP_ALG_ALGSEL_DES;
1974 cipherdata.algmode = OP_ALG_AAI_CBC;
1975 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
1976 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1977 SHR_NEVER, &cipherdata,
1981 case RTE_CRYPTO_CIPHER_AES_CTR:
1982 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1983 cipherdata.algmode = OP_ALG_AAI_CTR;
1984 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1985 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1986 SHR_NEVER, &cipherdata,
1990 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1991 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
1992 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
1993 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
1997 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
1998 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
1999 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
2000 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
2004 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2005 case RTE_CRYPTO_CIPHER_AES_F8:
2006 case RTE_CRYPTO_CIPHER_AES_ECB:
2007 case RTE_CRYPTO_CIPHER_3DES_ECB:
2008 case RTE_CRYPTO_CIPHER_3DES_CTR:
2009 case RTE_CRYPTO_CIPHER_AES_XTS:
2010 case RTE_CRYPTO_CIPHER_ARC4:
2011 case RTE_CRYPTO_CIPHER_NULL:
2012 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2013 xform->cipher.algo);
2017 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2018 xform->cipher.algo);
2024 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
2029 flc->word1_sdl = (uint8_t)bufsize;
2030 session->ctxt = priv;
2032 #ifdef CAAM_DESC_DEBUG
2034 for (i = 0; i < bufsize; i++)
2035 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
2040 rte_free(session->cipher_key.data);
2046 dpaa2_sec_auth_init(struct rte_cryptodev *dev,
2047 struct rte_crypto_sym_xform *xform,
2048 dpaa2_sec_session *session)
2050 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2051 struct alginfo authdata;
2052 int bufsize, ret = 0;
2053 struct ctxt_priv *priv;
2054 struct sec_flow_context *flc;
2056 PMD_INIT_FUNC_TRACE();
2058 /* For SEC AUTH three descriptors are required for various stages */
2059 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2060 sizeof(struct ctxt_priv) + 3 *
2061 sizeof(struct sec_flc_desc),
2062 RTE_CACHE_LINE_SIZE);
2064 DPAA2_SEC_ERR("No Memory for priv CTXT");
2068 priv->fle_pool = dev_priv->fle_pool;
2069 flc = &priv->flc_desc[DESC_INITFINAL].flc;
2071 session->ctxt_type = DPAA2_SEC_AUTH;
2072 session->auth_key.length = xform->auth.key.length;
2073 if (xform->auth.key.length) {
2074 session->auth_key.data = rte_zmalloc(NULL,
2075 xform->auth.key.length,
2076 RTE_CACHE_LINE_SIZE);
2077 if (session->auth_key.data == NULL) {
2078 DPAA2_SEC_ERR("Unable to allocate memory for auth key");
2082 memcpy(session->auth_key.data, xform->auth.key.data,
2083 xform->auth.key.length);
2084 authdata.key = (size_t)session->auth_key.data;
2085 authdata.key_enc_flags = 0;
2086 authdata.key_type = RTA_DATA_IMM;
2088 authdata.keylen = session->auth_key.length;
2090 session->digest_length = xform->auth.digest_length;
2091 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2094 switch (xform->auth.algo) {
2095 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2096 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2097 authdata.algmode = OP_ALG_AAI_HMAC;
2098 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2099 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2100 1, 0, SHR_NEVER, &authdata,
2102 session->digest_length);
2104 case RTE_CRYPTO_AUTH_MD5_HMAC:
2105 authdata.algtype = OP_ALG_ALGSEL_MD5;
2106 authdata.algmode = OP_ALG_AAI_HMAC;
2107 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2108 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2109 1, 0, SHR_NEVER, &authdata,
2111 session->digest_length);
2113 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2114 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2115 authdata.algmode = OP_ALG_AAI_HMAC;
2116 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2117 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2118 1, 0, SHR_NEVER, &authdata,
2120 session->digest_length);
2122 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2123 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2124 authdata.algmode = OP_ALG_AAI_HMAC;
2125 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2126 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2127 1, 0, SHR_NEVER, &authdata,
2129 session->digest_length);
2131 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2132 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2133 authdata.algmode = OP_ALG_AAI_HMAC;
2134 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2135 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2136 1, 0, SHR_NEVER, &authdata,
2138 session->digest_length);
2140 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2141 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2142 authdata.algmode = OP_ALG_AAI_HMAC;
2143 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2144 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2145 1, 0, SHR_NEVER, &authdata,
2147 session->digest_length);
2149 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2150 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2151 authdata.algmode = OP_ALG_AAI_F9;
2152 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2153 session->iv.offset = xform->auth.iv.offset;
2154 session->iv.length = xform->auth.iv.length;
2155 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2158 session->digest_length);
2160 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2161 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2162 authdata.algmode = OP_ALG_AAI_F9;
2163 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2164 session->iv.offset = xform->auth.iv.offset;
2165 session->iv.length = xform->auth.iv.length;
2166 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2169 session->digest_length);
2171 case RTE_CRYPTO_AUTH_SHA1:
2172 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2173 authdata.algmode = OP_ALG_AAI_HASH;
2174 session->auth_alg = RTE_CRYPTO_AUTH_SHA1;
2175 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2176 1, 0, SHR_NEVER, &authdata,
2178 session->digest_length);
2180 case RTE_CRYPTO_AUTH_MD5:
2181 authdata.algtype = OP_ALG_ALGSEL_MD5;
2182 authdata.algmode = OP_ALG_AAI_HASH;
2183 session->auth_alg = RTE_CRYPTO_AUTH_MD5;
2184 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2185 1, 0, SHR_NEVER, &authdata,
2187 session->digest_length);
2189 case RTE_CRYPTO_AUTH_SHA256:
2190 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2191 authdata.algmode = OP_ALG_AAI_HASH;
2192 session->auth_alg = RTE_CRYPTO_AUTH_SHA256;
2193 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2194 1, 0, SHR_NEVER, &authdata,
2196 session->digest_length);
2198 case RTE_CRYPTO_AUTH_SHA384:
2199 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2200 authdata.algmode = OP_ALG_AAI_HASH;
2201 session->auth_alg = RTE_CRYPTO_AUTH_SHA384;
2202 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2203 1, 0, SHR_NEVER, &authdata,
2205 session->digest_length);
2207 case RTE_CRYPTO_AUTH_SHA512:
2208 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2209 authdata.algmode = OP_ALG_AAI_HASH;
2210 session->auth_alg = RTE_CRYPTO_AUTH_SHA512;
2211 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2212 1, 0, SHR_NEVER, &authdata,
2214 session->digest_length);
2216 case RTE_CRYPTO_AUTH_SHA224:
2217 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2218 authdata.algmode = OP_ALG_AAI_HASH;
2219 session->auth_alg = RTE_CRYPTO_AUTH_SHA224;
2220 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2221 1, 0, SHR_NEVER, &authdata,
2223 session->digest_length);
2225 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2226 authdata.algtype = OP_ALG_ALGSEL_AES;
2227 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2228 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2229 bufsize = cnstr_shdsc_aes_mac(
2230 priv->flc_desc[DESC_INITFINAL].desc,
2231 1, 0, SHR_NEVER, &authdata,
2233 session->digest_length);
2235 case RTE_CRYPTO_AUTH_AES_CMAC:
2236 authdata.algtype = OP_ALG_ALGSEL_AES;
2237 authdata.algmode = OP_ALG_AAI_CMAC;
2238 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2239 bufsize = cnstr_shdsc_aes_mac(
2240 priv->flc_desc[DESC_INITFINAL].desc,
2241 1, 0, SHR_NEVER, &authdata,
2243 session->digest_length);
2245 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2246 case RTE_CRYPTO_AUTH_AES_GMAC:
2247 case RTE_CRYPTO_AUTH_KASUMI_F9:
2248 case RTE_CRYPTO_AUTH_NULL:
2249 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2254 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2261 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2266 flc->word1_sdl = (uint8_t)bufsize;
2267 session->ctxt = priv;
2268 #ifdef CAAM_DESC_DEBUG
2270 for (i = 0; i < bufsize; i++)
2271 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2272 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2278 rte_free(session->auth_key.data);
2284 dpaa2_sec_aead_init(struct rte_cryptodev *dev,
2285 struct rte_crypto_sym_xform *xform,
2286 dpaa2_sec_session *session)
2288 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2289 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2290 struct alginfo aeaddata;
2292 struct ctxt_priv *priv;
2293 struct sec_flow_context *flc;
2294 struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2297 PMD_INIT_FUNC_TRACE();
2299 /* Set IV parameters */
2300 session->iv.offset = aead_xform->iv.offset;
2301 session->iv.length = aead_xform->iv.length;
2302 session->ctxt_type = DPAA2_SEC_AEAD;
2304 /* For SEC AEAD only one descriptor is required */
2305 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2306 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2307 RTE_CACHE_LINE_SIZE);
2309 DPAA2_SEC_ERR("No Memory for priv CTXT");
2313 priv->fle_pool = dev_priv->fle_pool;
2314 flc = &priv->flc_desc[0].flc;
2316 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2317 RTE_CACHE_LINE_SIZE);
2318 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2319 DPAA2_SEC_ERR("No Memory for aead key");
2323 memcpy(session->aead_key.data, aead_xform->key.data,
2324 aead_xform->key.length);
2326 session->digest_length = aead_xform->digest_length;
2327 session->aead_key.length = aead_xform->key.length;
2328 ctxt->auth_only_len = aead_xform->aad_length;
2330 aeaddata.key = (size_t)session->aead_key.data;
2331 aeaddata.keylen = session->aead_key.length;
2332 aeaddata.key_enc_flags = 0;
2333 aeaddata.key_type = RTA_DATA_IMM;
2335 switch (aead_xform->algo) {
2336 case RTE_CRYPTO_AEAD_AES_GCM:
2337 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2338 aeaddata.algmode = OP_ALG_AAI_GCM;
2339 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2341 case RTE_CRYPTO_AEAD_AES_CCM:
2342 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2347 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2352 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2355 priv->flc_desc[0].desc[0] = aeaddata.keylen;
2356 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2358 (unsigned int *)priv->flc_desc[0].desc,
2359 &priv->flc_desc[0].desc[1], 1);
2362 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2366 if (priv->flc_desc[0].desc[1] & 1) {
2367 aeaddata.key_type = RTA_DATA_IMM;
2369 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2370 aeaddata.key_type = RTA_DATA_PTR;
2372 priv->flc_desc[0].desc[0] = 0;
2373 priv->flc_desc[0].desc[1] = 0;
2375 if (session->dir == DIR_ENC)
2376 bufsize = cnstr_shdsc_gcm_encap(
2377 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2378 &aeaddata, session->iv.length,
2379 session->digest_length);
2381 bufsize = cnstr_shdsc_gcm_decap(
2382 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2383 &aeaddata, session->iv.length,
2384 session->digest_length);
2386 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2391 flc->word1_sdl = (uint8_t)bufsize;
2392 session->ctxt = priv;
2393 #ifdef CAAM_DESC_DEBUG
2395 for (i = 0; i < bufsize; i++)
2396 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2397 i, priv->flc_desc[0].desc[i]);
2402 rte_free(session->aead_key.data);
2409 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
2410 struct rte_crypto_sym_xform *xform,
2411 dpaa2_sec_session *session)
2413 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2414 struct alginfo authdata, cipherdata;
2416 struct ctxt_priv *priv;
2417 struct sec_flow_context *flc;
2418 struct rte_crypto_cipher_xform *cipher_xform;
2419 struct rte_crypto_auth_xform *auth_xform;
2422 PMD_INIT_FUNC_TRACE();
2424 if (session->ext_params.aead_ctxt.auth_cipher_text) {
2425 cipher_xform = &xform->cipher;
2426 auth_xform = &xform->next->auth;
2427 session->ctxt_type =
2428 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2429 DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2431 cipher_xform = &xform->next->cipher;
2432 auth_xform = &xform->auth;
2433 session->ctxt_type =
2434 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2435 DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2438 /* Set IV parameters */
2439 session->iv.offset = cipher_xform->iv.offset;
2440 session->iv.length = cipher_xform->iv.length;
2442 /* For SEC AEAD only one descriptor is required */
2443 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2444 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2445 RTE_CACHE_LINE_SIZE);
2447 DPAA2_SEC_ERR("No Memory for priv CTXT");
2451 priv->fle_pool = dev_priv->fle_pool;
2452 flc = &priv->flc_desc[0].flc;
2454 session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2455 RTE_CACHE_LINE_SIZE);
2456 if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2457 DPAA2_SEC_ERR("No Memory for cipher key");
2461 session->cipher_key.length = cipher_xform->key.length;
2462 session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2463 RTE_CACHE_LINE_SIZE);
2464 if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2465 DPAA2_SEC_ERR("No Memory for auth key");
2466 rte_free(session->cipher_key.data);
2470 session->auth_key.length = auth_xform->key.length;
2471 memcpy(session->cipher_key.data, cipher_xform->key.data,
2472 cipher_xform->key.length);
2473 memcpy(session->auth_key.data, auth_xform->key.data,
2474 auth_xform->key.length);
2476 authdata.key = (size_t)session->auth_key.data;
2477 authdata.keylen = session->auth_key.length;
2478 authdata.key_enc_flags = 0;
2479 authdata.key_type = RTA_DATA_IMM;
2481 session->digest_length = auth_xform->digest_length;
2483 switch (auth_xform->algo) {
2484 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2485 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2486 authdata.algmode = OP_ALG_AAI_HMAC;
2487 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2489 case RTE_CRYPTO_AUTH_MD5_HMAC:
2490 authdata.algtype = OP_ALG_ALGSEL_MD5;
2491 authdata.algmode = OP_ALG_AAI_HMAC;
2492 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2494 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2495 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2496 authdata.algmode = OP_ALG_AAI_HMAC;
2497 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2499 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2500 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2501 authdata.algmode = OP_ALG_AAI_HMAC;
2502 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2504 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2505 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2506 authdata.algmode = OP_ALG_AAI_HMAC;
2507 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2509 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2510 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2511 authdata.algmode = OP_ALG_AAI_HMAC;
2512 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2514 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2515 authdata.algtype = OP_ALG_ALGSEL_AES;
2516 authdata.algmode = OP_ALG_AAI_XCBC_MAC;
2517 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC;
2519 case RTE_CRYPTO_AUTH_AES_CMAC:
2520 authdata.algtype = OP_ALG_ALGSEL_AES;
2521 authdata.algmode = OP_ALG_AAI_CMAC;
2522 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC;
2524 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2525 case RTE_CRYPTO_AUTH_AES_GMAC:
2526 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2527 case RTE_CRYPTO_AUTH_NULL:
2528 case RTE_CRYPTO_AUTH_SHA1:
2529 case RTE_CRYPTO_AUTH_SHA256:
2530 case RTE_CRYPTO_AUTH_SHA512:
2531 case RTE_CRYPTO_AUTH_SHA224:
2532 case RTE_CRYPTO_AUTH_SHA384:
2533 case RTE_CRYPTO_AUTH_MD5:
2534 case RTE_CRYPTO_AUTH_KASUMI_F9:
2535 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2536 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2541 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2546 cipherdata.key = (size_t)session->cipher_key.data;
2547 cipherdata.keylen = session->cipher_key.length;
2548 cipherdata.key_enc_flags = 0;
2549 cipherdata.key_type = RTA_DATA_IMM;
2551 switch (cipher_xform->algo) {
2552 case RTE_CRYPTO_CIPHER_AES_CBC:
2553 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2554 cipherdata.algmode = OP_ALG_AAI_CBC;
2555 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2557 case RTE_CRYPTO_CIPHER_3DES_CBC:
2558 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2559 cipherdata.algmode = OP_ALG_AAI_CBC;
2560 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2562 case RTE_CRYPTO_CIPHER_DES_CBC:
2563 cipherdata.algtype = OP_ALG_ALGSEL_DES;
2564 cipherdata.algmode = OP_ALG_AAI_CBC;
2565 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
2567 case RTE_CRYPTO_CIPHER_AES_CTR:
2568 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2569 cipherdata.algmode = OP_ALG_AAI_CTR;
2570 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2572 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2573 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2574 case RTE_CRYPTO_CIPHER_NULL:
2575 case RTE_CRYPTO_CIPHER_3DES_ECB:
2576 case RTE_CRYPTO_CIPHER_3DES_CTR:
2577 case RTE_CRYPTO_CIPHER_AES_ECB:
2578 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2579 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2580 cipher_xform->algo);
2584 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2585 cipher_xform->algo);
2589 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2592 priv->flc_desc[0].desc[0] = cipherdata.keylen;
2593 priv->flc_desc[0].desc[1] = authdata.keylen;
2594 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2596 (unsigned int *)priv->flc_desc[0].desc,
2597 &priv->flc_desc[0].desc[2], 2);
2600 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2604 if (priv->flc_desc[0].desc[2] & 1) {
2605 cipherdata.key_type = RTA_DATA_IMM;
2607 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2608 cipherdata.key_type = RTA_DATA_PTR;
2610 if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2611 authdata.key_type = RTA_DATA_IMM;
2613 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2614 authdata.key_type = RTA_DATA_PTR;
2616 priv->flc_desc[0].desc[0] = 0;
2617 priv->flc_desc[0].desc[1] = 0;
2618 priv->flc_desc[0].desc[2] = 0;
2620 if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2621 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2623 &cipherdata, &authdata,
2625 session->digest_length,
2628 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2633 DPAA2_SEC_ERR("Hash before cipher not supported");
2638 flc->word1_sdl = (uint8_t)bufsize;
2639 session->ctxt = priv;
2640 #ifdef CAAM_DESC_DEBUG
2642 for (i = 0; i < bufsize; i++)
2643 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2644 i, priv->flc_desc[0].desc[i]);
2650 rte_free(session->cipher_key.data);
2651 rte_free(session->auth_key.data);
2657 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
2658 struct rte_crypto_sym_xform *xform, void *sess)
2660 dpaa2_sec_session *session = sess;
2663 PMD_INIT_FUNC_TRACE();
2665 if (unlikely(sess == NULL)) {
2666 DPAA2_SEC_ERR("Invalid session struct");
2670 memset(session, 0, sizeof(dpaa2_sec_session));
2671 /* Default IV length = 0 */
2672 session->iv.length = 0;
2675 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2676 ret = dpaa2_sec_cipher_init(dev, xform, session);
2678 /* Authentication Only */
2679 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2680 xform->next == NULL) {
2681 ret = dpaa2_sec_auth_init(dev, xform, session);
2683 /* Cipher then Authenticate */
2684 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2685 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2686 session->ext_params.aead_ctxt.auth_cipher_text = true;
2687 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2688 ret = dpaa2_sec_auth_init(dev, xform, session);
2689 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2690 ret = dpaa2_sec_cipher_init(dev, xform, session);
2692 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2693 /* Authenticate then Cipher */
2694 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2695 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2696 session->ext_params.aead_ctxt.auth_cipher_text = false;
2697 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2698 ret = dpaa2_sec_cipher_init(dev, xform, session);
2699 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2700 ret = dpaa2_sec_auth_init(dev, xform, session);
2702 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2703 /* AEAD operation for AES-GCM kind of Algorithms */
2704 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2705 xform->next == NULL) {
2706 ret = dpaa2_sec_aead_init(dev, xform, session);
2709 DPAA2_SEC_ERR("Invalid crypto type");
2716 #ifdef RTE_LIB_SECURITY
2718 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2719 dpaa2_sec_session *session,
2720 struct alginfo *aeaddata)
2722 PMD_INIT_FUNC_TRACE();
2724 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2725 RTE_CACHE_LINE_SIZE);
2726 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2727 DPAA2_SEC_ERR("No Memory for aead key");
2730 memcpy(session->aead_key.data, aead_xform->key.data,
2731 aead_xform->key.length);
2733 session->digest_length = aead_xform->digest_length;
2734 session->aead_key.length = aead_xform->key.length;
2736 aeaddata->key = (size_t)session->aead_key.data;
2737 aeaddata->keylen = session->aead_key.length;
2738 aeaddata->key_enc_flags = 0;
2739 aeaddata->key_type = RTA_DATA_IMM;
2741 switch (aead_xform->algo) {
2742 case RTE_CRYPTO_AEAD_AES_GCM:
2743 switch (session->digest_length) {
2745 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2748 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2751 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2754 DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2755 session->digest_length);
2758 aeaddata->algmode = OP_ALG_AAI_GCM;
2759 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2761 case RTE_CRYPTO_AEAD_AES_CCM:
2762 switch (session->digest_length) {
2764 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2767 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2770 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2773 DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2774 session->digest_length);
2777 aeaddata->algmode = OP_ALG_AAI_CCM;
2778 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2781 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2785 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2792 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2793 struct rte_crypto_auth_xform *auth_xform,
2794 dpaa2_sec_session *session,
2795 struct alginfo *cipherdata,
2796 struct alginfo *authdata)
2799 session->cipher_key.data = rte_zmalloc(NULL,
2800 cipher_xform->key.length,
2801 RTE_CACHE_LINE_SIZE);
2802 if (session->cipher_key.data == NULL &&
2803 cipher_xform->key.length > 0) {
2804 DPAA2_SEC_ERR("No Memory for cipher key");
2808 session->cipher_key.length = cipher_xform->key.length;
2809 memcpy(session->cipher_key.data, cipher_xform->key.data,
2810 cipher_xform->key.length);
2811 session->cipher_alg = cipher_xform->algo;
2813 session->cipher_key.data = NULL;
2814 session->cipher_key.length = 0;
2815 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2819 session->auth_key.data = rte_zmalloc(NULL,
2820 auth_xform->key.length,
2821 RTE_CACHE_LINE_SIZE);
2822 if (session->auth_key.data == NULL &&
2823 auth_xform->key.length > 0) {
2824 DPAA2_SEC_ERR("No Memory for auth key");
2827 session->auth_key.length = auth_xform->key.length;
2828 memcpy(session->auth_key.data, auth_xform->key.data,
2829 auth_xform->key.length);
2830 session->auth_alg = auth_xform->algo;
2831 session->digest_length = auth_xform->digest_length;
2833 session->auth_key.data = NULL;
2834 session->auth_key.length = 0;
2835 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2838 authdata->key = (size_t)session->auth_key.data;
2839 authdata->keylen = session->auth_key.length;
2840 authdata->key_enc_flags = 0;
2841 authdata->key_type = RTA_DATA_IMM;
2842 switch (session->auth_alg) {
2843 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2844 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
2845 authdata->algmode = OP_ALG_AAI_HMAC;
2847 case RTE_CRYPTO_AUTH_MD5_HMAC:
2848 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
2849 authdata->algmode = OP_ALG_AAI_HMAC;
2851 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2852 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2853 authdata->algmode = OP_ALG_AAI_HMAC;
2854 if (session->digest_length != 16)
2856 "+++Using sha256-hmac truncated len is non-standard,"
2857 "it will not work with lookaside proto");
2859 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2860 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2861 authdata->algmode = OP_ALG_AAI_HMAC;
2863 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2864 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2865 authdata->algmode = OP_ALG_AAI_HMAC;
2867 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2868 authdata->algtype = OP_PCL_IPSEC_AES_XCBC_MAC_96;
2869 authdata->algmode = OP_ALG_AAI_XCBC_MAC;
2871 case RTE_CRYPTO_AUTH_AES_CMAC:
2872 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
2873 authdata->algmode = OP_ALG_AAI_CMAC;
2875 case RTE_CRYPTO_AUTH_NULL:
2876 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
2878 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2879 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2880 case RTE_CRYPTO_AUTH_SHA1:
2881 case RTE_CRYPTO_AUTH_SHA256:
2882 case RTE_CRYPTO_AUTH_SHA512:
2883 case RTE_CRYPTO_AUTH_SHA224:
2884 case RTE_CRYPTO_AUTH_SHA384:
2885 case RTE_CRYPTO_AUTH_MD5:
2886 case RTE_CRYPTO_AUTH_AES_GMAC:
2887 case RTE_CRYPTO_AUTH_KASUMI_F9:
2888 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2889 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2890 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2894 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2898 cipherdata->key = (size_t)session->cipher_key.data;
2899 cipherdata->keylen = session->cipher_key.length;
2900 cipherdata->key_enc_flags = 0;
2901 cipherdata->key_type = RTA_DATA_IMM;
2903 switch (session->cipher_alg) {
2904 case RTE_CRYPTO_CIPHER_AES_CBC:
2905 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
2906 cipherdata->algmode = OP_ALG_AAI_CBC;
2908 case RTE_CRYPTO_CIPHER_3DES_CBC:
2909 cipherdata->algtype = OP_PCL_IPSEC_3DES;
2910 cipherdata->algmode = OP_ALG_AAI_CBC;
2912 case RTE_CRYPTO_CIPHER_DES_CBC:
2913 cipherdata->algtype = OP_PCL_IPSEC_DES;
2914 cipherdata->algmode = OP_ALG_AAI_CBC;
2916 case RTE_CRYPTO_CIPHER_AES_CTR:
2917 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
2918 cipherdata->algmode = OP_ALG_AAI_CTR;
2920 case RTE_CRYPTO_CIPHER_NULL:
2921 cipherdata->algtype = OP_PCL_IPSEC_NULL;
2923 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2924 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2925 case RTE_CRYPTO_CIPHER_3DES_ECB:
2926 case RTE_CRYPTO_CIPHER_3DES_CTR:
2927 case RTE_CRYPTO_CIPHER_AES_ECB:
2928 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2929 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2930 session->cipher_alg);
2933 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2934 session->cipher_alg);
2942 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
2943 struct rte_security_session_conf *conf,
2946 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2947 struct rte_crypto_cipher_xform *cipher_xform = NULL;
2948 struct rte_crypto_auth_xform *auth_xform = NULL;
2949 struct rte_crypto_aead_xform *aead_xform = NULL;
2950 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
2951 struct ctxt_priv *priv;
2952 struct alginfo authdata, cipherdata;
2954 struct sec_flow_context *flc;
2955 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2958 PMD_INIT_FUNC_TRACE();
2960 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2961 sizeof(struct ctxt_priv) +
2962 sizeof(struct sec_flc_desc),
2963 RTE_CACHE_LINE_SIZE);
2966 DPAA2_SEC_ERR("No memory for priv CTXT");
2970 priv->fle_pool = dev_priv->fle_pool;
2971 flc = &priv->flc_desc[0].flc;
2973 if (ipsec_xform->life.bytes_hard_limit != 0 ||
2974 ipsec_xform->life.bytes_soft_limit != 0 ||
2975 ipsec_xform->life.packets_hard_limit != 0 ||
2976 ipsec_xform->life.packets_soft_limit != 0)
2979 memset(session, 0, sizeof(dpaa2_sec_session));
2981 if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2982 cipher_xform = &conf->crypto_xform->cipher;
2983 if (conf->crypto_xform->next)
2984 auth_xform = &conf->crypto_xform->next->auth;
2985 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2986 session, &cipherdata, &authdata);
2987 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2988 auth_xform = &conf->crypto_xform->auth;
2989 if (conf->crypto_xform->next)
2990 cipher_xform = &conf->crypto_xform->next->cipher;
2991 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2992 session, &cipherdata, &authdata);
2993 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2994 aead_xform = &conf->crypto_xform->aead;
2995 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
2996 session, &cipherdata);
2997 authdata.keylen = 0;
2998 authdata.algtype = 0;
3000 DPAA2_SEC_ERR("XFORM not specified");
3005 DPAA2_SEC_ERR("Failed to process xform");
3009 session->ctxt_type = DPAA2_SEC_IPSEC;
3010 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
3011 uint8_t *hdr = NULL;
3013 struct rte_ipv6_hdr ip6_hdr;
3014 struct ipsec_encap_pdb encap_pdb;
3016 flc->dhr = SEC_FLC_DHR_OUTBOUND;
3017 /* For Sec Proto only one descriptor is required. */
3018 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
3020 /* copy algo specific data to PDB */
3021 switch (cipherdata.algtype) {
3022 case OP_PCL_IPSEC_AES_CTR:
3023 encap_pdb.ctr.ctr_initial = 0x00000001;
3024 encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3026 case OP_PCL_IPSEC_AES_GCM8:
3027 case OP_PCL_IPSEC_AES_GCM12:
3028 case OP_PCL_IPSEC_AES_GCM16:
3029 memcpy(encap_pdb.gcm.salt,
3030 (uint8_t *)&(ipsec_xform->salt), 4);
3034 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
3035 PDBOPTS_ESP_OIHI_PDB_INL |
3038 if (ipsec_xform->options.dec_ttl)
3039 encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL;
3040 if (ipsec_xform->options.esn)
3041 encap_pdb.options |= PDBOPTS_ESP_ESN;
3042 encap_pdb.spi = ipsec_xform->spi;
3043 session->dir = DIR_ENC;
3044 if (ipsec_xform->tunnel.type ==
3045 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
3046 encap_pdb.ip_hdr_len = sizeof(struct ip);
3047 ip4_hdr.ip_v = IPVERSION;
3049 ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
3050 ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
3053 ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
3054 ip4_hdr.ip_p = IPPROTO_ESP;
3056 ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
3057 ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
3058 ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
3059 &ip4_hdr, sizeof(struct ip));
3060 hdr = (uint8_t *)&ip4_hdr;
3061 } else if (ipsec_xform->tunnel.type ==
3062 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
3063 ip6_hdr.vtc_flow = rte_cpu_to_be_32(
3064 DPAA2_IPv6_DEFAULT_VTC_FLOW |
3065 ((ipsec_xform->tunnel.ipv6.dscp <<
3066 RTE_IPV6_HDR_TC_SHIFT) &
3067 RTE_IPV6_HDR_TC_MASK) |
3068 ((ipsec_xform->tunnel.ipv6.flabel <<
3069 RTE_IPV6_HDR_FL_SHIFT) &
3070 RTE_IPV6_HDR_FL_MASK));
3071 /* Payload length will be updated by HW */
3072 ip6_hdr.payload_len = 0;
3073 ip6_hdr.hop_limits =
3074 ipsec_xform->tunnel.ipv6.hlimit;
3075 ip6_hdr.proto = (ipsec_xform->proto ==
3076 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
3077 IPPROTO_ESP : IPPROTO_AH;
3078 memcpy(&ip6_hdr.src_addr,
3079 &ipsec_xform->tunnel.ipv6.src_addr, 16);
3080 memcpy(&ip6_hdr.dst_addr,
3081 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
3082 encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
3083 hdr = (uint8_t *)&ip6_hdr;
3086 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
3087 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3088 SHR_WAIT : SHR_SERIAL, &encap_pdb,
3089 hdr, &cipherdata, &authdata);
3090 } else if (ipsec_xform->direction ==
3091 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
3092 struct ipsec_decap_pdb decap_pdb;
3094 flc->dhr = SEC_FLC_DHR_INBOUND;
3095 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
3096 /* copy algo specific data to PDB */
3097 switch (cipherdata.algtype) {
3098 case OP_PCL_IPSEC_AES_CTR:
3099 decap_pdb.ctr.ctr_initial = 0x00000001;
3100 decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
3102 case OP_PCL_IPSEC_AES_GCM8:
3103 case OP_PCL_IPSEC_AES_GCM12:
3104 case OP_PCL_IPSEC_AES_GCM16:
3105 memcpy(decap_pdb.gcm.salt,
3106 (uint8_t *)&(ipsec_xform->salt), 4);
3110 decap_pdb.options = (ipsec_xform->tunnel.type ==
3111 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
3112 sizeof(struct ip) << 16 :
3113 sizeof(struct rte_ipv6_hdr) << 16;
3114 if (ipsec_xform->options.esn)
3115 decap_pdb.options |= PDBOPTS_ESP_ESN;
3117 if (ipsec_xform->replay_win_sz) {
3119 win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
3121 if (rta_sec_era < RTA_SEC_ERA_10 && win_sz > 128) {
3122 DPAA2_SEC_INFO("Max Anti replay Win sz = 128");
3132 decap_pdb.options |= PDBOPTS_ESP_ARS32;
3135 decap_pdb.options |= PDBOPTS_ESP_ARS64;
3138 decap_pdb.options |= PDBOPTS_ESP_ARS256;
3141 decap_pdb.options |= PDBOPTS_ESP_ARS512;
3144 decap_pdb.options |= PDBOPTS_ESP_ARS1024;
3148 decap_pdb.options |= PDBOPTS_ESP_ARS128;
3151 session->dir = DIR_DEC;
3152 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
3153 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ?
3154 SHR_WAIT : SHR_SERIAL,
3155 &decap_pdb, &cipherdata, &authdata);
3160 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3164 flc->word1_sdl = (uint8_t)bufsize;
3166 /* Enable the stashing control bit */
3167 DPAA2_SET_FLC_RSC(flc);
3168 flc->word2_rflc_31_0 = lower_32_bits(
3169 (size_t)&(((struct dpaa2_sec_qp *)
3170 dev->data->queue_pairs[0])->rx_vq) | 0x14);
3171 flc->word3_rflc_63_32 = upper_32_bits(
3172 (size_t)&(((struct dpaa2_sec_qp *)
3173 dev->data->queue_pairs[0])->rx_vq));
3175 /* Set EWS bit i.e. enable write-safe */
3176 DPAA2_SET_FLC_EWS(flc);
3177 /* Set BS = 1 i.e reuse input buffers as output buffers */
3178 DPAA2_SET_FLC_REUSE_BS(flc);
3179 /* Set FF = 10; reuse input buffers if they provide sufficient space */
3180 DPAA2_SET_FLC_REUSE_FF(flc);
3182 session->ctxt = priv;
3186 rte_free(session->auth_key.data);
3187 rte_free(session->cipher_key.data);
3193 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
3194 struct rte_security_session_conf *conf,
3197 struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3198 struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3199 struct rte_crypto_auth_xform *auth_xform = NULL;
3200 struct rte_crypto_cipher_xform *cipher_xform = NULL;
3201 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3202 struct ctxt_priv *priv;
3203 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
3204 struct alginfo authdata, cipherdata;
3205 struct alginfo *p_authdata = NULL;
3207 struct sec_flow_context *flc;
3208 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3214 PMD_INIT_FUNC_TRACE();
3216 memset(session, 0, sizeof(dpaa2_sec_session));
3218 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3219 sizeof(struct ctxt_priv) +
3220 sizeof(struct sec_flc_desc),
3221 RTE_CACHE_LINE_SIZE);
3224 DPAA2_SEC_ERR("No memory for priv CTXT");
3228 priv->fle_pool = dev_priv->fle_pool;
3229 flc = &priv->flc_desc[0].flc;
3231 /* find xfrm types */
3232 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3233 cipher_xform = &xform->cipher;
3234 if (xform->next != NULL &&
3235 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3236 session->ext_params.aead_ctxt.auth_cipher_text = true;
3237 auth_xform = &xform->next->auth;
3239 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3240 auth_xform = &xform->auth;
3241 if (xform->next != NULL &&
3242 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3243 session->ext_params.aead_ctxt.auth_cipher_text = false;
3244 cipher_xform = &xform->next->cipher;
3247 DPAA2_SEC_ERR("Invalid crypto type");
3251 session->ctxt_type = DPAA2_SEC_PDCP;
3253 session->cipher_key.data = rte_zmalloc(NULL,
3254 cipher_xform->key.length,
3255 RTE_CACHE_LINE_SIZE);
3256 if (session->cipher_key.data == NULL &&
3257 cipher_xform->key.length > 0) {
3258 DPAA2_SEC_ERR("No Memory for cipher key");
3262 session->cipher_key.length = cipher_xform->key.length;
3263 memcpy(session->cipher_key.data, cipher_xform->key.data,
3264 cipher_xform->key.length);
3266 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3268 session->cipher_alg = cipher_xform->algo;
3270 session->cipher_key.data = NULL;
3271 session->cipher_key.length = 0;
3272 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3273 session->dir = DIR_ENC;
3276 session->pdcp.domain = pdcp_xform->domain;
3277 session->pdcp.bearer = pdcp_xform->bearer;
3278 session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3279 session->pdcp.sn_size = pdcp_xform->sn_size;
3280 session->pdcp.hfn = pdcp_xform->hfn;
3281 session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3282 session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3283 /* hfv ovd offset location is stored in iv.offset value*/
3285 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3287 cipherdata.key = (size_t)session->cipher_key.data;
3288 cipherdata.keylen = session->cipher_key.length;
3289 cipherdata.key_enc_flags = 0;
3290 cipherdata.key_type = RTA_DATA_IMM;
3292 switch (session->cipher_alg) {
3293 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3294 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3296 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3297 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3299 case RTE_CRYPTO_CIPHER_AES_CTR:
3300 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3302 case RTE_CRYPTO_CIPHER_NULL:
3303 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3306 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3307 session->cipher_alg);
3312 session->auth_key.data = rte_zmalloc(NULL,
3313 auth_xform->key.length,
3314 RTE_CACHE_LINE_SIZE);
3315 if (!session->auth_key.data &&
3316 auth_xform->key.length > 0) {
3317 DPAA2_SEC_ERR("No Memory for auth key");
3318 rte_free(session->cipher_key.data);
3322 session->auth_key.length = auth_xform->key.length;
3323 memcpy(session->auth_key.data, auth_xform->key.data,
3324 auth_xform->key.length);
3325 session->auth_alg = auth_xform->algo;
3327 session->auth_key.data = NULL;
3328 session->auth_key.length = 0;
3329 session->auth_alg = 0;
3331 authdata.key = (size_t)session->auth_key.data;
3332 authdata.keylen = session->auth_key.length;
3333 authdata.key_enc_flags = 0;
3334 authdata.key_type = RTA_DATA_IMM;
3336 if (session->auth_alg) {
3337 switch (session->auth_alg) {
3338 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3339 authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3341 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3342 authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3344 case RTE_CRYPTO_AUTH_AES_CMAC:
3345 authdata.algtype = PDCP_AUTH_TYPE_AES;
3347 case RTE_CRYPTO_AUTH_NULL:
3348 authdata.algtype = PDCP_AUTH_TYPE_NULL;
3351 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3356 p_authdata = &authdata;
3357 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3358 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3362 if (pdcp_xform->sdap_enabled) {
3363 int nb_keys_to_inline =
3364 rta_inline_pdcp_sdap_query(authdata.algtype,
3366 session->pdcp.sn_size,
3367 session->pdcp.hfn_ovd);
3368 if (nb_keys_to_inline >= 1) {
3369 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3370 cipherdata.key_type = RTA_DATA_PTR;
3372 if (nb_keys_to_inline >= 2) {
3373 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
3374 authdata.key_type = RTA_DATA_PTR;
3377 if (rta_inline_pdcp_query(authdata.algtype,
3379 session->pdcp.sn_size,
3380 session->pdcp.hfn_ovd)) {
3381 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3382 cipherdata.key_type = RTA_DATA_PTR;
3386 if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3387 if (session->dir == DIR_ENC)
3388 bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3389 priv->flc_desc[0].desc, 1, swap,
3391 session->pdcp.sn_size,
3393 pdcp_xform->pkt_dir,
3394 pdcp_xform->hfn_threshold,
3395 &cipherdata, &authdata);
3396 else if (session->dir == DIR_DEC)
3397 bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3398 priv->flc_desc[0].desc, 1, swap,
3400 session->pdcp.sn_size,
3402 pdcp_xform->pkt_dir,
3403 pdcp_xform->hfn_threshold,
3404 &cipherdata, &authdata);
3406 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) {
3407 bufsize = cnstr_shdsc_pdcp_short_mac(priv->flc_desc[0].desc,
3408 1, swap, &authdata);
3410 if (session->dir == DIR_ENC) {
3411 if (pdcp_xform->sdap_enabled)
3412 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_encap(
3413 priv->flc_desc[0].desc, 1, swap,
3414 session->pdcp.sn_size,
3417 pdcp_xform->pkt_dir,
3418 pdcp_xform->hfn_threshold,
3419 &cipherdata, p_authdata);
3421 bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3422 priv->flc_desc[0].desc, 1, swap,
3423 session->pdcp.sn_size,
3426 pdcp_xform->pkt_dir,
3427 pdcp_xform->hfn_threshold,
3428 &cipherdata, p_authdata);
3429 } else if (session->dir == DIR_DEC) {
3430 if (pdcp_xform->sdap_enabled)
3431 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_decap(
3432 priv->flc_desc[0].desc, 1, swap,
3433 session->pdcp.sn_size,
3436 pdcp_xform->pkt_dir,
3437 pdcp_xform->hfn_threshold,
3438 &cipherdata, p_authdata);
3440 bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3441 priv->flc_desc[0].desc, 1, swap,
3442 session->pdcp.sn_size,
3445 pdcp_xform->pkt_dir,
3446 pdcp_xform->hfn_threshold,
3447 &cipherdata, p_authdata);
3452 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3456 /* Enable the stashing control bit */
3457 DPAA2_SET_FLC_RSC(flc);
3458 flc->word2_rflc_31_0 = lower_32_bits(
3459 (size_t)&(((struct dpaa2_sec_qp *)
3460 dev->data->queue_pairs[0])->rx_vq) | 0x14);
3461 flc->word3_rflc_63_32 = upper_32_bits(
3462 (size_t)&(((struct dpaa2_sec_qp *)
3463 dev->data->queue_pairs[0])->rx_vq));
3465 flc->word1_sdl = (uint8_t)bufsize;
3467 /* TODO - check the perf impact or
3468 * align as per descriptor type
3469 * Set EWS bit i.e. enable write-safe
3470 * DPAA2_SET_FLC_EWS(flc);
3473 /* Set BS = 1 i.e reuse input buffers as output buffers */
3474 DPAA2_SET_FLC_REUSE_BS(flc);
3475 /* Set FF = 10; reuse input buffers if they provide sufficient space */
3476 DPAA2_SET_FLC_REUSE_FF(flc);
3478 session->ctxt = priv;
3482 rte_free(session->auth_key.data);
3483 rte_free(session->cipher_key.data);
3489 dpaa2_sec_security_session_create(void *dev,
3490 struct rte_security_session_conf *conf,
3491 struct rte_security_session *sess,
3492 struct rte_mempool *mempool)
3494 void *sess_private_data;
3495 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3498 if (rte_mempool_get(mempool, &sess_private_data)) {
3499 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3503 switch (conf->protocol) {
3504 case RTE_SECURITY_PROTOCOL_IPSEC:
3505 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3508 case RTE_SECURITY_PROTOCOL_MACSEC:
3510 case RTE_SECURITY_PROTOCOL_PDCP:
3511 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3518 DPAA2_SEC_ERR("Failed to configure session parameters");
3519 /* Return session to mempool */
3520 rte_mempool_put(mempool, sess_private_data);
3524 set_sec_session_private_data(sess, sess_private_data);
3529 /** Clear the memory of session so it doesn't leave key material behind */
3531 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3532 struct rte_security_session *sess)
3534 PMD_INIT_FUNC_TRACE();
3535 void *sess_priv = get_sec_session_private_data(sess);
3537 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3540 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3543 rte_free(s->cipher_key.data);
3544 rte_free(s->auth_key.data);
3545 memset(s, 0, sizeof(dpaa2_sec_session));
3546 set_sec_session_private_data(sess, NULL);
3547 rte_mempool_put(sess_mp, sess_priv);
3553 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3554 struct rte_crypto_sym_xform *xform,
3555 struct rte_cryptodev_sym_session *sess,
3556 struct rte_mempool *mempool)
3558 void *sess_private_data;
3561 if (rte_mempool_get(mempool, &sess_private_data)) {
3562 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3566 ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data);
3568 DPAA2_SEC_ERR("Failed to configure session parameters");
3569 /* Return session to mempool */
3570 rte_mempool_put(mempool, sess_private_data);
3574 set_sym_session_private_data(sess, dev->driver_id,
3580 /** Clear the memory of session so it doesn't leave key material behind */
3582 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3583 struct rte_cryptodev_sym_session *sess)
3585 PMD_INIT_FUNC_TRACE();
3586 uint8_t index = dev->driver_id;
3587 void *sess_priv = get_sym_session_private_data(sess, index);
3588 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3592 rte_free(s->cipher_key.data);
3593 rte_free(s->auth_key.data);
3594 memset(s, 0, sizeof(dpaa2_sec_session));
3595 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3596 set_sym_session_private_data(sess, index, NULL);
3597 rte_mempool_put(sess_mp, sess_priv);
3602 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3603 struct rte_cryptodev_config *config __rte_unused)
3605 PMD_INIT_FUNC_TRACE();
3611 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3613 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3614 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3615 struct dpseci_attr attr;
3616 struct dpaa2_queue *dpaa2_q;
3617 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3618 dev->data->queue_pairs;
3619 struct dpseci_rx_queue_attr rx_attr;
3620 struct dpseci_tx_queue_attr tx_attr;
3623 PMD_INIT_FUNC_TRACE();
3625 memset(&attr, 0, sizeof(struct dpseci_attr));
3627 ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3629 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3631 goto get_attr_failure;
3633 ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3635 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3636 goto get_attr_failure;
3638 for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3639 dpaa2_q = &qp[i]->rx_vq;
3640 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3642 dpaa2_q->fqid = rx_attr.fqid;
3643 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3645 for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3646 dpaa2_q = &qp[i]->tx_vq;
3647 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3649 dpaa2_q->fqid = tx_attr.fqid;
3650 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3655 dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3660 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3662 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3663 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3666 PMD_INIT_FUNC_TRACE();
3668 ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3670 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3675 ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3677 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3683 dpaa2_sec_dev_close(struct rte_cryptodev *dev __rte_unused)
3685 PMD_INIT_FUNC_TRACE();
3691 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3692 struct rte_cryptodev_info *info)
3694 struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3696 PMD_INIT_FUNC_TRACE();
3698 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3699 info->feature_flags = dev->feature_flags;
3700 info->capabilities = dpaa2_sec_capabilities;
3701 /* No limit of number of sessions */
3702 info->sym.max_nb_sessions = 0;
3703 info->driver_id = cryptodev_driver_id;
3708 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3709 struct rte_cryptodev_stats *stats)
3711 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3712 struct fsl_mc_io dpseci;
3713 struct dpseci_sec_counters counters = {0};
3714 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3715 dev->data->queue_pairs;
3718 PMD_INIT_FUNC_TRACE();
3719 if (stats == NULL) {
3720 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3723 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3724 if (qp == NULL || qp[i] == NULL) {
3725 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3729 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3730 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3731 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3732 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3735 /* In case as secondary process access stats, MCP portal in priv-hw
3736 * may have primary process address. Need the secondary process
3737 * based MCP portal address for this object.
3739 dpseci.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3740 ret = dpseci_get_sec_counters(&dpseci, CMD_PRI_LOW, priv->token,
3743 DPAA2_SEC_ERR("SEC counters failed");
3745 DPAA2_SEC_INFO("dpseci hardware stats:"
3746 "\n\tNum of Requests Dequeued = %" PRIu64
3747 "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3748 "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3749 "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3750 "\n\tNum of Outbound Bytes Protected = %" PRIu64
3751 "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3752 "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3753 counters.dequeued_requests,
3754 counters.ob_enc_requests,
3755 counters.ib_dec_requests,
3756 counters.ob_enc_bytes,
3757 counters.ob_prot_bytes,
3758 counters.ib_dec_bytes,
3759 counters.ib_valid_bytes);
3764 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3767 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3768 (dev->data->queue_pairs);
3770 PMD_INIT_FUNC_TRACE();
3772 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3773 if (qp[i] == NULL) {
3774 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3777 qp[i]->tx_vq.rx_pkts = 0;
3778 qp[i]->tx_vq.tx_pkts = 0;
3779 qp[i]->tx_vq.err_pkts = 0;
3780 qp[i]->rx_vq.rx_pkts = 0;
3781 qp[i]->rx_vq.tx_pkts = 0;
3782 qp[i]->rx_vq.err_pkts = 0;
3786 static void __rte_hot
3787 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3788 const struct qbman_fd *fd,
3789 const struct qbman_result *dq,
3790 struct dpaa2_queue *rxq,
3791 struct rte_event *ev)
3793 /* Prefetching mbuf */
3794 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3795 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3797 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3798 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3800 ev->flow_id = rxq->ev.flow_id;
3801 ev->sub_event_type = rxq->ev.sub_event_type;
3802 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3803 ev->op = RTE_EVENT_OP_NEW;
3804 ev->sched_type = rxq->ev.sched_type;
3805 ev->queue_id = rxq->ev.queue_id;
3806 ev->priority = rxq->ev.priority;
3807 ev->event_ptr = sec_fd_to_mbuf(fd);
3809 qbman_swp_dqrr_consume(swp, dq);
3812 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
3813 const struct qbman_fd *fd,
3814 const struct qbman_result *dq,
3815 struct dpaa2_queue *rxq,
3816 struct rte_event *ev)
3819 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
3820 /* Prefetching mbuf */
3821 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3822 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3824 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3825 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3827 ev->flow_id = rxq->ev.flow_id;
3828 ev->sub_event_type = rxq->ev.sub_event_type;
3829 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3830 ev->op = RTE_EVENT_OP_NEW;
3831 ev->sched_type = rxq->ev.sched_type;
3832 ev->queue_id = rxq->ev.queue_id;
3833 ev->priority = rxq->ev.priority;
3835 ev->event_ptr = sec_fd_to_mbuf(fd);
3836 dqrr_index = qbman_get_dqrr_idx(dq);
3837 *dpaa2_seqn(crypto_op->sym->m_src) = dqrr_index + 1;
3838 DPAA2_PER_LCORE_DQRR_SIZE++;
3839 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
3840 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
3844 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
3846 struct dpaa2_dpcon_dev *dpcon,
3847 const struct rte_event *event)
3849 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3850 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3851 struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
3852 struct dpseci_rx_queue_cfg cfg;
3856 if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
3857 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
3858 else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
3859 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
3863 priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
3864 (dpcon->num_priorities - 1);
3866 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3867 cfg.options = DPSECI_QUEUE_OPT_DEST;
3868 cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
3869 cfg.dest_cfg.dest_id = dpcon->dpcon_id;
3870 cfg.dest_cfg.priority = priority;
3872 cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
3873 cfg.user_ctx = (size_t)(qp);
3874 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
3875 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
3876 cfg.order_preservation_en = 1;
3878 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3881 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3885 memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
3891 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
3894 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3895 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3896 struct dpseci_rx_queue_cfg cfg;
3899 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3900 cfg.options = DPSECI_QUEUE_OPT_DEST;
3901 cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
3903 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3906 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3911 static struct rte_cryptodev_ops crypto_ops = {
3912 .dev_configure = dpaa2_sec_dev_configure,
3913 .dev_start = dpaa2_sec_dev_start,
3914 .dev_stop = dpaa2_sec_dev_stop,
3915 .dev_close = dpaa2_sec_dev_close,
3916 .dev_infos_get = dpaa2_sec_dev_infos_get,
3917 .stats_get = dpaa2_sec_stats_get,
3918 .stats_reset = dpaa2_sec_stats_reset,
3919 .queue_pair_setup = dpaa2_sec_queue_pair_setup,
3920 .queue_pair_release = dpaa2_sec_queue_pair_release,
3921 .sym_session_get_size = dpaa2_sec_sym_session_get_size,
3922 .sym_session_configure = dpaa2_sec_sym_session_configure,
3923 .sym_session_clear = dpaa2_sec_sym_session_clear,
3924 /* Raw data-path API related operations */
3925 .sym_get_raw_dp_ctx_size = dpaa2_sec_get_dp_ctx_size,
3926 .sym_configure_raw_dp_ctx = dpaa2_sec_configure_raw_dp_ctx,
3929 #ifdef RTE_LIB_SECURITY
3930 static const struct rte_security_capability *
3931 dpaa2_sec_capabilities_get(void *device __rte_unused)
3933 return dpaa2_sec_security_cap;
3936 static const struct rte_security_ops dpaa2_sec_security_ops = {
3937 .session_create = dpaa2_sec_security_session_create,
3938 .session_update = NULL,
3939 .session_stats_get = NULL,
3940 .session_destroy = dpaa2_sec_security_session_destroy,
3941 .set_pkt_metadata = NULL,
3942 .capabilities_get = dpaa2_sec_capabilities_get
3947 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
3949 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3950 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3953 PMD_INIT_FUNC_TRACE();
3955 /* Function is reverse of dpaa2_sec_dev_init.
3956 * It does the following:
3957 * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
3958 * 2. Close the DPSECI device
3959 * 3. Free the allocated resources.
3962 /*Close the device at underlying layer*/
3963 ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
3965 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
3969 /*Free the allocated memory for ethernet private data and dpseci*/
3972 rte_free(dev->security_ctx);
3973 rte_mempool_free(priv->fle_pool);
3975 DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
3976 dev->data->name, rte_socket_id());
3982 check_devargs_handler(__rte_unused const char *key, const char *value,
3983 __rte_unused void *opaque)
3985 dpaa2_sec_dp_dump = atoi(value);
3986 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_FULL_DUMP) {
3987 DPAA2_SEC_WARN("WARN: DPAA2_SEC_DP_DUMP_LEVEL is not "
3988 "supported, changing to FULL error prints\n");
3989 dpaa2_sec_dp_dump = DPAA2_SEC_DP_FULL_DUMP;
3996 dpaa2_sec_get_devargs(struct rte_devargs *devargs, const char *key)
3998 struct rte_kvargs *kvlist;
4003 kvlist = rte_kvargs_parse(devargs->args, NULL);
4007 if (!rte_kvargs_count(kvlist, key)) {
4008 rte_kvargs_free(kvlist);
4012 rte_kvargs_process(kvlist, key,
4013 check_devargs_handler, NULL);
4014 rte_kvargs_free(kvlist);
4018 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
4020 struct dpaa2_sec_dev_private *internals;
4021 struct rte_device *dev = cryptodev->device;
4022 struct rte_dpaa2_device *dpaa2_dev;
4023 #ifdef RTE_LIB_SECURITY
4024 struct rte_security_ctx *security_instance;
4026 struct fsl_mc_io *dpseci;
4028 struct dpseci_attr attr;
4032 PMD_INIT_FUNC_TRACE();
4033 dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
4034 hw_id = dpaa2_dev->object_id;
4036 cryptodev->driver_id = cryptodev_driver_id;
4037 cryptodev->dev_ops = &crypto_ops;
4039 cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
4040 cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
4041 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
4042 RTE_CRYPTODEV_FF_HW_ACCELERATED |
4043 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
4044 RTE_CRYPTODEV_FF_SECURITY |
4045 RTE_CRYPTODEV_FF_SYM_RAW_DP |
4046 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
4047 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
4048 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
4049 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
4050 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
4052 internals = cryptodev->data->dev_private;
4055 * For secondary processes, we don't initialise any further as primary
4056 * has already done this work. Only check we don't need a different
4059 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
4060 DPAA2_SEC_DEBUG("Device already init by primary process");
4063 #ifdef RTE_LIB_SECURITY
4064 /* Initialize security_ctx only for primary process*/
4065 security_instance = rte_malloc("rte_security_instances_ops",
4066 sizeof(struct rte_security_ctx), 0);
4067 if (security_instance == NULL)
4069 security_instance->device = (void *)cryptodev;
4070 security_instance->ops = &dpaa2_sec_security_ops;
4071 security_instance->sess_cnt = 0;
4072 cryptodev->security_ctx = security_instance;
4074 /*Open the rte device via MC and save the handle for further use*/
4075 dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
4076 sizeof(struct fsl_mc_io), 0);
4079 "Error in allocating the memory for dpsec object");
4082 dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
4084 retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
4086 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
4090 retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
4093 "Cannot get dpsec device attributed: Error = %x",
4097 snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
4100 internals->max_nb_queue_pairs = attr.num_tx_queues;
4101 cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
4102 internals->hw = dpseci;
4103 internals->token = token;
4105 snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d",
4106 getpid(), cryptodev->data->dev_id);
4107 internals->fle_pool = rte_mempool_create((const char *)str,
4110 FLE_POOL_CACHE_SIZE, 0,
4111 NULL, NULL, NULL, NULL,
4113 if (!internals->fle_pool) {
4114 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
4118 dpaa2_sec_get_devargs(cryptodev->device->devargs, DRIVER_DUMP_MODE);
4119 DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
4123 DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
4125 /* dpaa2_sec_uninit(crypto_dev_name); */
4130 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
4131 struct rte_dpaa2_device *dpaa2_dev)
4133 struct rte_cryptodev *cryptodev;
4134 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
4138 snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
4139 dpaa2_dev->object_id);
4141 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
4142 if (cryptodev == NULL)
4145 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4146 cryptodev->data->dev_private = rte_zmalloc_socket(
4147 "cryptodev private structure",
4148 sizeof(struct dpaa2_sec_dev_private),
4149 RTE_CACHE_LINE_SIZE,
4152 if (cryptodev->data->dev_private == NULL)
4153 rte_panic("Cannot allocate memzone for private "
4157 dpaa2_dev->cryptodev = cryptodev;
4158 cryptodev->device = &dpaa2_dev->device;
4160 /* init user callbacks */
4161 TAILQ_INIT(&(cryptodev->link_intr_cbs));
4163 if (dpaa2_svr_family == SVR_LX2160A)
4164 rta_set_sec_era(RTA_SEC_ERA_10);
4166 rta_set_sec_era(RTA_SEC_ERA_8);
4168 DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
4170 /* Invoke PMD device initialization function */
4171 retval = dpaa2_sec_dev_init(cryptodev);
4173 rte_cryptodev_pmd_probing_finish(cryptodev);
4177 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
4178 rte_free(cryptodev->data->dev_private);
4180 cryptodev->attached = RTE_CRYPTODEV_DETACHED;
4186 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
4188 struct rte_cryptodev *cryptodev;
4191 cryptodev = dpaa2_dev->cryptodev;
4192 if (cryptodev == NULL)
4195 ret = dpaa2_sec_uninit(cryptodev);
4199 return rte_cryptodev_pmd_destroy(cryptodev);
4202 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
4203 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
4204 .drv_type = DPAA2_CRYPTO,
4206 .name = "DPAA2 SEC PMD"
4208 .probe = cryptodev_dpaa2_sec_probe,
4209 .remove = cryptodev_dpaa2_sec_remove,
4212 static struct cryptodev_driver dpaa2_sec_crypto_drv;
4214 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
4215 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
4216 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
4217 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA2_SEC_PMD,
4218 DRIVER_DUMP_MODE "=<int>");
4219 RTE_LOG_REGISTER(dpaa2_logtype_sec, pmd.crypto.dpaa2, NOTICE);