1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4 * Copyright 2016-2019 NXP
14 #include <rte_cryptodev.h>
15 #include <rte_malloc.h>
16 #include <rte_memcpy.h>
17 #include <rte_string_fns.h>
18 #include <rte_cycles.h>
19 #include <rte_kvargs.h>
21 #include <rte_cryptodev_pmd.h>
22 #include <rte_common.h>
23 #include <rte_fslmc.h>
24 #include <fslmc_vfio.h>
25 #include <dpaa2_hw_pvt.h>
26 #include <dpaa2_hw_dpio.h>
27 #include <dpaa2_hw_mempool.h>
28 #include <fsl_dpopr.h>
29 #include <fsl_dpseci.h>
30 #include <fsl_mc_sys.h>
32 #include "dpaa2_sec_priv.h"
33 #include "dpaa2_sec_event.h"
34 #include "dpaa2_sec_logs.h"
36 /* RTA header files */
37 #include <desc/ipsec.h>
38 #include <desc/pdcp.h>
39 #include <desc/algo.h>
41 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
42 * a pointer to the shared descriptor
44 #define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ)
45 #define FSL_VENDOR_ID 0x1957
46 #define FSL_DEVICE_ID 0x410
47 #define FSL_SUBSYSTEM_SEC 1
48 #define FSL_MC_DPSECI_DEVID 3
51 /* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */
52 #define FLE_POOL_NUM_BUFS 32000
53 #define FLE_POOL_BUF_SIZE 256
54 #define FLE_POOL_CACHE_SIZE 512
55 #define FLE_SG_MEM_SIZE(num) (FLE_POOL_BUF_SIZE + ((num) * 32))
56 #define SEC_FLC_DHR_OUTBOUND -114
57 #define SEC_FLC_DHR_INBOUND 0
59 static uint8_t cryptodev_driver_id;
61 int dpaa2_logtype_sec;
63 #ifdef RTE_LIBRTE_SECURITY
65 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
66 struct rte_crypto_op *op,
67 struct qbman_fd *fd, uint16_t bpid)
69 struct rte_crypto_sym_op *sym_op = op->sym;
70 struct ctxt_priv *priv = sess->ctxt;
71 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
72 struct sec_flow_context *flc;
73 struct rte_mbuf *mbuf;
74 uint32_t in_len = 0, out_len = 0;
81 /* first FLE entry used to store mbuf and session ctxt */
82 fle = (struct qbman_fle *)rte_malloc(NULL,
83 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
86 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
89 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
90 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
91 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
93 /* Save the shared descriptor */
94 flc = &priv->flc_desc[0].flc;
100 if (likely(bpid < MAX_BPID)) {
101 DPAA2_SET_FD_BPID(fd, bpid);
102 DPAA2_SET_FLE_BPID(op_fle, bpid);
103 DPAA2_SET_FLE_BPID(ip_fle, bpid);
105 DPAA2_SET_FD_IVP(fd);
106 DPAA2_SET_FLE_IVP(op_fle);
107 DPAA2_SET_FLE_IVP(ip_fle);
110 /* Configure FD as a FRAME LIST */
111 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
112 DPAA2_SET_FD_COMPOUND_FMT(fd);
113 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
115 /* Configure Output FLE with Scatter/Gather Entry */
116 DPAA2_SET_FLE_SG_EXT(op_fle);
117 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
119 /* Configure Output SGE for Encap/Decap */
120 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
121 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
124 sge->length = mbuf->data_len;
125 out_len += sge->length;
128 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
129 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
131 /* using buf_len for last buf - so that extra data can be added */
132 sge->length = mbuf->buf_len - mbuf->data_off;
133 out_len += sge->length;
135 DPAA2_SET_FLE_FIN(sge);
136 op_fle->length = out_len;
139 mbuf = sym_op->m_src;
141 /* Configure Input FLE with Scatter/Gather Entry */
142 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
143 DPAA2_SET_FLE_SG_EXT(ip_fle);
144 DPAA2_SET_FLE_FIN(ip_fle);
146 /* Configure input SGE for Encap/Decap */
147 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
148 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
149 sge->length = mbuf->data_len;
150 in_len += sge->length;
156 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
157 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
158 sge->length = mbuf->data_len;
159 in_len += sge->length;
162 ip_fle->length = in_len;
163 DPAA2_SET_FLE_FIN(sge);
165 /* In case of PDCP, per packet HFN is stored in
166 * mbuf priv after sym_op.
168 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
169 uint32_t hfn_ovd = *((uint8_t *)op + sess->pdcp.hfn_ovd_offset);
170 /*enable HFN override override */
171 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
172 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
173 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
175 DPAA2_SET_FD_LEN(fd, ip_fle->length);
181 build_proto_compound_fd(dpaa2_sec_session *sess,
182 struct rte_crypto_op *op,
183 struct qbman_fd *fd, uint16_t bpid)
185 struct rte_crypto_sym_op *sym_op = op->sym;
186 struct ctxt_priv *priv = sess->ctxt;
187 struct qbman_fle *fle, *ip_fle, *op_fle;
188 struct sec_flow_context *flc;
189 struct rte_mbuf *src_mbuf = sym_op->m_src;
190 struct rte_mbuf *dst_mbuf = sym_op->m_dst;
196 /* Save the shared descriptor */
197 flc = &priv->flc_desc[0].flc;
199 /* we are using the first FLE entry to store Mbuf */
200 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
202 DPAA2_SEC_DP_ERR("Memory alloc failed");
205 memset(fle, 0, FLE_POOL_BUF_SIZE);
206 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
207 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
212 if (likely(bpid < MAX_BPID)) {
213 DPAA2_SET_FD_BPID(fd, bpid);
214 DPAA2_SET_FLE_BPID(op_fle, bpid);
215 DPAA2_SET_FLE_BPID(ip_fle, bpid);
217 DPAA2_SET_FD_IVP(fd);
218 DPAA2_SET_FLE_IVP(op_fle);
219 DPAA2_SET_FLE_IVP(ip_fle);
222 /* Configure FD as a FRAME LIST */
223 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
224 DPAA2_SET_FD_COMPOUND_FMT(fd);
225 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
227 /* Configure Output FLE with dst mbuf data */
228 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
229 DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
230 DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
232 /* Configure Input FLE with src mbuf data */
233 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
234 DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
235 DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
237 DPAA2_SET_FD_LEN(fd, ip_fle->length);
238 DPAA2_SET_FLE_FIN(ip_fle);
240 /* In case of PDCP, per packet HFN is stored in
241 * mbuf priv after sym_op.
243 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
244 uint32_t hfn_ovd = *((uint8_t *)op + sess->pdcp.hfn_ovd_offset);
245 /*enable HFN override override */
246 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
247 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
248 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
256 build_proto_fd(dpaa2_sec_session *sess,
257 struct rte_crypto_op *op,
258 struct qbman_fd *fd, uint16_t bpid)
260 struct rte_crypto_sym_op *sym_op = op->sym;
262 return build_proto_compound_fd(sess, op, fd, bpid);
264 struct ctxt_priv *priv = sess->ctxt;
265 struct sec_flow_context *flc;
266 struct rte_mbuf *mbuf = sym_op->m_src;
268 if (likely(bpid < MAX_BPID))
269 DPAA2_SET_FD_BPID(fd, bpid);
271 DPAA2_SET_FD_IVP(fd);
273 /* Save the shared descriptor */
274 flc = &priv->flc_desc[0].flc;
276 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
277 DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
278 DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
279 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
281 /* save physical address of mbuf */
282 op->sym->aead.digest.phys_addr = mbuf->buf_iova;
283 mbuf->buf_iova = (size_t)op;
290 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
291 struct rte_crypto_op *op,
292 struct qbman_fd *fd, __rte_unused uint16_t bpid)
294 struct rte_crypto_sym_op *sym_op = op->sym;
295 struct ctxt_priv *priv = sess->ctxt;
296 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
297 struct sec_flow_context *flc;
298 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
299 int icv_len = sess->digest_length;
301 struct rte_mbuf *mbuf;
302 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
306 mbuf = sym_op->m_dst;
308 mbuf = sym_op->m_src;
310 /* first FLE entry used to store mbuf and session ctxt */
311 fle = (struct qbman_fle *)rte_malloc(NULL,
312 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
313 RTE_CACHE_LINE_SIZE);
314 if (unlikely(!fle)) {
315 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
318 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
319 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
320 DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
326 /* Save the shared descriptor */
327 flc = &priv->flc_desc[0].flc;
329 /* Configure FD as a FRAME LIST */
330 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
331 DPAA2_SET_FD_COMPOUND_FMT(fd);
332 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
334 DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
335 "iv-len=%d data_off: 0x%x\n",
336 sym_op->aead.data.offset,
337 sym_op->aead.data.length,
340 sym_op->m_src->data_off);
342 /* Configure Output FLE with Scatter/Gather Entry */
343 DPAA2_SET_FLE_SG_EXT(op_fle);
344 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
347 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
349 op_fle->length = (sess->dir == DIR_ENC) ?
350 (sym_op->aead.data.length + icv_len) :
351 sym_op->aead.data.length;
353 /* Configure Output SGE for Encap/Decap */
354 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
355 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
356 sge->length = mbuf->data_len - sym_op->aead.data.offset;
362 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
363 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
364 sge->length = mbuf->data_len;
367 sge->length -= icv_len;
369 if (sess->dir == DIR_ENC) {
371 DPAA2_SET_FLE_ADDR(sge,
372 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
373 sge->length = icv_len;
375 DPAA2_SET_FLE_FIN(sge);
378 mbuf = sym_op->m_src;
380 /* Configure Input FLE with Scatter/Gather Entry */
381 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
382 DPAA2_SET_FLE_SG_EXT(ip_fle);
383 DPAA2_SET_FLE_FIN(ip_fle);
384 ip_fle->length = (sess->dir == DIR_ENC) ?
385 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
386 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
389 /* Configure Input SGE for Encap/Decap */
390 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
391 sge->length = sess->iv.length;
395 DPAA2_SET_FLE_ADDR(sge,
396 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
397 sge->length = auth_only_len;
401 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
402 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
404 sge->length = mbuf->data_len - sym_op->aead.data.offset;
410 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
411 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
412 sge->length = mbuf->data_len;
416 if (sess->dir == DIR_DEC) {
418 old_icv = (uint8_t *)(sge + 1);
419 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
420 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
421 sge->length = icv_len;
424 DPAA2_SET_FLE_FIN(sge);
426 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
427 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
429 DPAA2_SET_FD_LEN(fd, ip_fle->length);
435 build_authenc_gcm_fd(dpaa2_sec_session *sess,
436 struct rte_crypto_op *op,
437 struct qbman_fd *fd, uint16_t bpid)
439 struct rte_crypto_sym_op *sym_op = op->sym;
440 struct ctxt_priv *priv = sess->ctxt;
441 struct qbman_fle *fle, *sge;
442 struct sec_flow_context *flc;
443 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
444 int icv_len = sess->digest_length, retval;
446 struct rte_mbuf *dst;
447 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
455 /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
456 * Currently we donot know which FLE has the mbuf stored.
457 * So while retreiving we can go back 1 FLE from the FD -ADDR
458 * to get the MBUF Addr from the previous FLE.
459 * We can have a better approach to use the inline Mbuf
461 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
463 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE");
466 memset(fle, 0, FLE_POOL_BUF_SIZE);
467 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
468 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
471 if (likely(bpid < MAX_BPID)) {
472 DPAA2_SET_FD_BPID(fd, bpid);
473 DPAA2_SET_FLE_BPID(fle, bpid);
474 DPAA2_SET_FLE_BPID(fle + 1, bpid);
475 DPAA2_SET_FLE_BPID(sge, bpid);
476 DPAA2_SET_FLE_BPID(sge + 1, bpid);
477 DPAA2_SET_FLE_BPID(sge + 2, bpid);
478 DPAA2_SET_FLE_BPID(sge + 3, bpid);
480 DPAA2_SET_FD_IVP(fd);
481 DPAA2_SET_FLE_IVP(fle);
482 DPAA2_SET_FLE_IVP((fle + 1));
483 DPAA2_SET_FLE_IVP(sge);
484 DPAA2_SET_FLE_IVP((sge + 1));
485 DPAA2_SET_FLE_IVP((sge + 2));
486 DPAA2_SET_FLE_IVP((sge + 3));
489 /* Save the shared descriptor */
490 flc = &priv->flc_desc[0].flc;
491 /* Configure FD as a FRAME LIST */
492 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
493 DPAA2_SET_FD_COMPOUND_FMT(fd);
494 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
496 DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
497 "iv-len=%d data_off: 0x%x\n",
498 sym_op->aead.data.offset,
499 sym_op->aead.data.length,
502 sym_op->m_src->data_off);
504 /* Configure Output FLE with Scatter/Gather Entry */
505 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
507 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
508 fle->length = (sess->dir == DIR_ENC) ?
509 (sym_op->aead.data.length + icv_len) :
510 sym_op->aead.data.length;
512 DPAA2_SET_FLE_SG_EXT(fle);
514 /* Configure Output SGE for Encap/Decap */
515 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
516 DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
517 sge->length = sym_op->aead.data.length;
519 if (sess->dir == DIR_ENC) {
521 DPAA2_SET_FLE_ADDR(sge,
522 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
523 sge->length = sess->digest_length;
525 DPAA2_SET_FLE_FIN(sge);
530 /* Configure Input FLE with Scatter/Gather Entry */
531 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
532 DPAA2_SET_FLE_SG_EXT(fle);
533 DPAA2_SET_FLE_FIN(fle);
534 fle->length = (sess->dir == DIR_ENC) ?
535 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
536 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
537 sess->digest_length);
539 /* Configure Input SGE for Encap/Decap */
540 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
541 sge->length = sess->iv.length;
544 DPAA2_SET_FLE_ADDR(sge,
545 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
546 sge->length = auth_only_len;
547 DPAA2_SET_FLE_BPID(sge, bpid);
551 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
552 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
553 sym_op->m_src->data_off);
554 sge->length = sym_op->aead.data.length;
555 if (sess->dir == DIR_DEC) {
557 old_icv = (uint8_t *)(sge + 1);
558 memcpy(old_icv, sym_op->aead.digest.data,
559 sess->digest_length);
560 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
561 sge->length = sess->digest_length;
563 DPAA2_SET_FLE_FIN(sge);
566 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
567 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
570 DPAA2_SET_FD_LEN(fd, fle->length);
575 build_authenc_sg_fd(dpaa2_sec_session *sess,
576 struct rte_crypto_op *op,
577 struct qbman_fd *fd, __rte_unused uint16_t bpid)
579 struct rte_crypto_sym_op *sym_op = op->sym;
580 struct ctxt_priv *priv = sess->ctxt;
581 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
582 struct sec_flow_context *flc;
583 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
584 sym_op->auth.data.offset;
585 uint16_t auth_tail_len = sym_op->auth.data.length -
586 sym_op->cipher.data.length - auth_hdr_len;
587 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
588 int icv_len = sess->digest_length;
590 struct rte_mbuf *mbuf;
591 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
595 mbuf = sym_op->m_dst;
597 mbuf = sym_op->m_src;
599 /* first FLE entry used to store mbuf and session ctxt */
600 fle = (struct qbman_fle *)rte_malloc(NULL,
601 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
602 RTE_CACHE_LINE_SIZE);
603 if (unlikely(!fle)) {
604 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
607 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
608 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
609 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
615 /* Save the shared descriptor */
616 flc = &priv->flc_desc[0].flc;
618 /* Configure FD as a FRAME LIST */
619 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
620 DPAA2_SET_FD_COMPOUND_FMT(fd);
621 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
624 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
625 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
626 sym_op->auth.data.offset,
627 sym_op->auth.data.length,
629 sym_op->cipher.data.offset,
630 sym_op->cipher.data.length,
632 sym_op->m_src->data_off);
634 /* Configure Output FLE with Scatter/Gather Entry */
635 DPAA2_SET_FLE_SG_EXT(op_fle);
636 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
639 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
641 op_fle->length = (sess->dir == DIR_ENC) ?
642 (sym_op->cipher.data.length + icv_len) :
643 sym_op->cipher.data.length;
645 /* Configure Output SGE for Encap/Decap */
646 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
647 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
648 sge->length = mbuf->data_len - sym_op->auth.data.offset;
654 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
655 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
656 sge->length = mbuf->data_len;
659 sge->length -= icv_len;
661 if (sess->dir == DIR_ENC) {
663 DPAA2_SET_FLE_ADDR(sge,
664 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
665 sge->length = icv_len;
667 DPAA2_SET_FLE_FIN(sge);
670 mbuf = sym_op->m_src;
672 /* Configure Input FLE with Scatter/Gather Entry */
673 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
674 DPAA2_SET_FLE_SG_EXT(ip_fle);
675 DPAA2_SET_FLE_FIN(ip_fle);
676 ip_fle->length = (sess->dir == DIR_ENC) ?
677 (sym_op->auth.data.length + sess->iv.length) :
678 (sym_op->auth.data.length + sess->iv.length +
681 /* Configure Input SGE for Encap/Decap */
682 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
683 sge->length = sess->iv.length;
686 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
687 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
689 sge->length = mbuf->data_len - sym_op->auth.data.offset;
695 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
696 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
697 sge->length = mbuf->data_len;
700 sge->length -= icv_len;
702 if (sess->dir == DIR_DEC) {
704 old_icv = (uint8_t *)(sge + 1);
705 memcpy(old_icv, sym_op->auth.digest.data,
707 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
708 sge->length = icv_len;
711 DPAA2_SET_FLE_FIN(sge);
713 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
714 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
716 DPAA2_SET_FD_LEN(fd, ip_fle->length);
722 build_authenc_fd(dpaa2_sec_session *sess,
723 struct rte_crypto_op *op,
724 struct qbman_fd *fd, uint16_t bpid)
726 struct rte_crypto_sym_op *sym_op = op->sym;
727 struct ctxt_priv *priv = sess->ctxt;
728 struct qbman_fle *fle, *sge;
729 struct sec_flow_context *flc;
730 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
731 sym_op->auth.data.offset;
732 uint16_t auth_tail_len = sym_op->auth.data.length -
733 sym_op->cipher.data.length - auth_hdr_len;
734 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
736 int icv_len = sess->digest_length, retval;
738 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
740 struct rte_mbuf *dst;
747 /* we are using the first FLE entry to store Mbuf.
748 * Currently we donot know which FLE has the mbuf stored.
749 * So while retreiving we can go back 1 FLE from the FD -ADDR
750 * to get the MBUF Addr from the previous FLE.
751 * We can have a better approach to use the inline Mbuf
753 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
755 DPAA2_SEC_ERR("Memory alloc failed for SGE");
758 memset(fle, 0, FLE_POOL_BUF_SIZE);
759 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
760 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
763 if (likely(bpid < MAX_BPID)) {
764 DPAA2_SET_FD_BPID(fd, bpid);
765 DPAA2_SET_FLE_BPID(fle, bpid);
766 DPAA2_SET_FLE_BPID(fle + 1, bpid);
767 DPAA2_SET_FLE_BPID(sge, bpid);
768 DPAA2_SET_FLE_BPID(sge + 1, bpid);
769 DPAA2_SET_FLE_BPID(sge + 2, bpid);
770 DPAA2_SET_FLE_BPID(sge + 3, bpid);
772 DPAA2_SET_FD_IVP(fd);
773 DPAA2_SET_FLE_IVP(fle);
774 DPAA2_SET_FLE_IVP((fle + 1));
775 DPAA2_SET_FLE_IVP(sge);
776 DPAA2_SET_FLE_IVP((sge + 1));
777 DPAA2_SET_FLE_IVP((sge + 2));
778 DPAA2_SET_FLE_IVP((sge + 3));
781 /* Save the shared descriptor */
782 flc = &priv->flc_desc[0].flc;
783 /* Configure FD as a FRAME LIST */
784 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
785 DPAA2_SET_FD_COMPOUND_FMT(fd);
786 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
789 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
790 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
791 sym_op->auth.data.offset,
792 sym_op->auth.data.length,
794 sym_op->cipher.data.offset,
795 sym_op->cipher.data.length,
797 sym_op->m_src->data_off);
799 /* Configure Output FLE with Scatter/Gather Entry */
800 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
802 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
803 fle->length = (sess->dir == DIR_ENC) ?
804 (sym_op->cipher.data.length + icv_len) :
805 sym_op->cipher.data.length;
807 DPAA2_SET_FLE_SG_EXT(fle);
809 /* Configure Output SGE for Encap/Decap */
810 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
811 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
813 sge->length = sym_op->cipher.data.length;
815 if (sess->dir == DIR_ENC) {
817 DPAA2_SET_FLE_ADDR(sge,
818 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
819 sge->length = sess->digest_length;
820 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
823 DPAA2_SET_FLE_FIN(sge);
828 /* Configure Input FLE with Scatter/Gather Entry */
829 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
830 DPAA2_SET_FLE_SG_EXT(fle);
831 DPAA2_SET_FLE_FIN(fle);
832 fle->length = (sess->dir == DIR_ENC) ?
833 (sym_op->auth.data.length + sess->iv.length) :
834 (sym_op->auth.data.length + sess->iv.length +
835 sess->digest_length);
837 /* Configure Input SGE for Encap/Decap */
838 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
839 sge->length = sess->iv.length;
842 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
843 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
844 sym_op->m_src->data_off);
845 sge->length = sym_op->auth.data.length;
846 if (sess->dir == DIR_DEC) {
848 old_icv = (uint8_t *)(sge + 1);
849 memcpy(old_icv, sym_op->auth.digest.data,
850 sess->digest_length);
851 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
852 sge->length = sess->digest_length;
853 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
854 sess->digest_length +
857 DPAA2_SET_FLE_FIN(sge);
859 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
860 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
865 static inline int build_auth_sg_fd(
866 dpaa2_sec_session *sess,
867 struct rte_crypto_op *op,
869 __rte_unused uint16_t bpid)
871 struct rte_crypto_sym_op *sym_op = op->sym;
872 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
873 struct sec_flow_context *flc;
874 struct ctxt_priv *priv = sess->ctxt;
875 int data_len, data_offset;
877 struct rte_mbuf *mbuf;
879 data_len = sym_op->auth.data.length;
880 data_offset = sym_op->auth.data.offset;
882 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
883 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
884 if ((data_len & 7) || (data_offset & 7)) {
885 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
889 data_len = data_len >> 3;
890 data_offset = data_offset >> 3;
893 mbuf = sym_op->m_src;
894 fle = (struct qbman_fle *)rte_malloc(NULL,
895 FLE_SG_MEM_SIZE(mbuf->nb_segs),
896 RTE_CACHE_LINE_SIZE);
897 if (unlikely(!fle)) {
898 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
901 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
902 /* first FLE entry used to store mbuf and session ctxt */
903 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
904 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
909 flc = &priv->flc_desc[DESC_INITFINAL].flc;
911 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
912 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
913 DPAA2_SET_FD_COMPOUND_FMT(fd);
916 DPAA2_SET_FLE_ADDR(op_fle,
917 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
918 op_fle->length = sess->digest_length;
921 DPAA2_SET_FLE_SG_EXT(ip_fle);
922 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
923 ip_fle->length = data_len;
925 if (sess->iv.length) {
928 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
931 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
932 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
934 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
935 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
938 sge->length = sess->iv.length;
940 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
941 ip_fle->length += sge->length;
945 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
946 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
948 if (data_len <= (mbuf->data_len - data_offset)) {
949 sge->length = data_len;
952 sge->length = mbuf->data_len - data_offset;
954 /* remaining i/p segs */
955 while ((data_len = data_len - sge->length) &&
956 (mbuf = mbuf->next)) {
958 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
959 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
960 if (data_len > mbuf->data_len)
961 sge->length = mbuf->data_len;
963 sge->length = data_len;
967 if (sess->dir == DIR_DEC) {
968 /* Digest verification case */
970 old_digest = (uint8_t *)(sge + 1);
971 rte_memcpy(old_digest, sym_op->auth.digest.data,
972 sess->digest_length);
973 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
974 sge->length = sess->digest_length;
975 ip_fle->length += sess->digest_length;
977 DPAA2_SET_FLE_FIN(sge);
978 DPAA2_SET_FLE_FIN(ip_fle);
979 DPAA2_SET_FD_LEN(fd, ip_fle->length);
985 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
986 struct qbman_fd *fd, uint16_t bpid)
988 struct rte_crypto_sym_op *sym_op = op->sym;
989 struct qbman_fle *fle, *sge;
990 struct sec_flow_context *flc;
991 struct ctxt_priv *priv = sess->ctxt;
992 int data_len, data_offset;
996 data_len = sym_op->auth.data.length;
997 data_offset = sym_op->auth.data.offset;
999 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1000 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1001 if ((data_len & 7) || (data_offset & 7)) {
1002 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1006 data_len = data_len >> 3;
1007 data_offset = data_offset >> 3;
1010 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1012 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE");
1015 memset(fle, 0, FLE_POOL_BUF_SIZE);
1016 /* TODO we are using the first FLE entry to store Mbuf.
1017 * Currently we donot know which FLE has the mbuf stored.
1018 * So while retreiving we can go back 1 FLE from the FD -ADDR
1019 * to get the MBUF Addr from the previous FLE.
1020 * We can have a better approach to use the inline Mbuf
1022 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1023 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1027 if (likely(bpid < MAX_BPID)) {
1028 DPAA2_SET_FD_BPID(fd, bpid);
1029 DPAA2_SET_FLE_BPID(fle, bpid);
1030 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1031 DPAA2_SET_FLE_BPID(sge, bpid);
1032 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1034 DPAA2_SET_FD_IVP(fd);
1035 DPAA2_SET_FLE_IVP(fle);
1036 DPAA2_SET_FLE_IVP((fle + 1));
1037 DPAA2_SET_FLE_IVP(sge);
1038 DPAA2_SET_FLE_IVP((sge + 1));
1041 flc = &priv->flc_desc[DESC_INITFINAL].flc;
1042 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1043 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1044 DPAA2_SET_FD_COMPOUND_FMT(fd);
1046 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1047 fle->length = sess->digest_length;
1050 /* Setting input FLE */
1051 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1052 DPAA2_SET_FLE_SG_EXT(fle);
1053 fle->length = data_len;
1055 if (sess->iv.length) {
1058 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1061 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1062 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1064 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1065 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1068 sge->length = sess->iv.length;
1071 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1072 fle->length = fle->length + sge->length;
1076 /* Setting data to authenticate */
1077 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1078 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1079 sge->length = data_len;
1081 if (sess->dir == DIR_DEC) {
1083 old_digest = (uint8_t *)(sge + 1);
1084 rte_memcpy(old_digest, sym_op->auth.digest.data,
1085 sess->digest_length);
1086 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1087 sge->length = sess->digest_length;
1088 fle->length = fle->length + sess->digest_length;
1091 DPAA2_SET_FLE_FIN(sge);
1092 DPAA2_SET_FLE_FIN(fle);
1093 DPAA2_SET_FD_LEN(fd, fle->length);
1099 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1100 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1102 struct rte_crypto_sym_op *sym_op = op->sym;
1103 struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1104 int data_len, data_offset;
1105 struct sec_flow_context *flc;
1106 struct ctxt_priv *priv = sess->ctxt;
1107 struct rte_mbuf *mbuf;
1108 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1111 data_len = sym_op->cipher.data.length;
1112 data_offset = sym_op->cipher.data.offset;
1114 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1115 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1116 if ((data_len & 7) || (data_offset & 7)) {
1117 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1121 data_len = data_len >> 3;
1122 data_offset = data_offset >> 3;
1126 mbuf = sym_op->m_dst;
1128 mbuf = sym_op->m_src;
1130 /* first FLE entry used to store mbuf and session ctxt */
1131 fle = (struct qbman_fle *)rte_malloc(NULL,
1132 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1133 RTE_CACHE_LINE_SIZE);
1135 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1138 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1139 /* first FLE entry used to store mbuf and session ctxt */
1140 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1141 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1147 flc = &priv->flc_desc[0].flc;
1150 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1151 " data_off: 0x%x\n",
1155 sym_op->m_src->data_off);
1158 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1159 op_fle->length = data_len;
1160 DPAA2_SET_FLE_SG_EXT(op_fle);
1163 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1164 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1165 sge->length = mbuf->data_len - data_offset;
1171 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1172 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1173 sge->length = mbuf->data_len;
1176 DPAA2_SET_FLE_FIN(sge);
1179 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1180 flc, fle, fle->addr_hi, fle->addr_lo,
1184 mbuf = sym_op->m_src;
1186 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1187 ip_fle->length = sess->iv.length + data_len;
1188 DPAA2_SET_FLE_SG_EXT(ip_fle);
1191 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1192 DPAA2_SET_FLE_OFFSET(sge, 0);
1193 sge->length = sess->iv.length;
1198 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1199 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1200 sge->length = mbuf->data_len - data_offset;
1206 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1207 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1208 sge->length = mbuf->data_len;
1211 DPAA2_SET_FLE_FIN(sge);
1212 DPAA2_SET_FLE_FIN(ip_fle);
1215 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1216 DPAA2_SET_FD_LEN(fd, ip_fle->length);
1217 DPAA2_SET_FD_COMPOUND_FMT(fd);
1218 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1221 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1222 " off =%d, len =%d\n",
1223 DPAA2_GET_FD_ADDR(fd),
1224 DPAA2_GET_FD_BPID(fd),
1225 rte_dpaa2_bpid_info[bpid].meta_data_size,
1226 DPAA2_GET_FD_OFFSET(fd),
1227 DPAA2_GET_FD_LEN(fd));
1232 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1233 struct qbman_fd *fd, uint16_t bpid)
1235 struct rte_crypto_sym_op *sym_op = op->sym;
1236 struct qbman_fle *fle, *sge;
1237 int retval, data_len, data_offset;
1238 struct sec_flow_context *flc;
1239 struct ctxt_priv *priv = sess->ctxt;
1240 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1242 struct rte_mbuf *dst;
1244 data_len = sym_op->cipher.data.length;
1245 data_offset = sym_op->cipher.data.offset;
1247 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1248 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1249 if ((data_len & 7) || (data_offset & 7)) {
1250 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1254 data_len = data_len >> 3;
1255 data_offset = data_offset >> 3;
1259 dst = sym_op->m_dst;
1261 dst = sym_op->m_src;
1263 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1265 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE");
1268 memset(fle, 0, FLE_POOL_BUF_SIZE);
1269 /* TODO we are using the first FLE entry to store Mbuf.
1270 * Currently we donot know which FLE has the mbuf stored.
1271 * So while retreiving we can go back 1 FLE from the FD -ADDR
1272 * to get the MBUF Addr from the previous FLE.
1273 * We can have a better approach to use the inline Mbuf
1275 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1276 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1280 if (likely(bpid < MAX_BPID)) {
1281 DPAA2_SET_FD_BPID(fd, bpid);
1282 DPAA2_SET_FLE_BPID(fle, bpid);
1283 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1284 DPAA2_SET_FLE_BPID(sge, bpid);
1285 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1287 DPAA2_SET_FD_IVP(fd);
1288 DPAA2_SET_FLE_IVP(fle);
1289 DPAA2_SET_FLE_IVP((fle + 1));
1290 DPAA2_SET_FLE_IVP(sge);
1291 DPAA2_SET_FLE_IVP((sge + 1));
1294 flc = &priv->flc_desc[0].flc;
1295 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1296 DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1297 DPAA2_SET_FD_COMPOUND_FMT(fd);
1298 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1301 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1302 " data_off: 0x%x\n",
1306 sym_op->m_src->data_off);
1308 DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1309 DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1311 fle->length = data_len + sess->iv.length;
1314 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1315 flc, fle, fle->addr_hi, fle->addr_lo,
1320 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1321 fle->length = data_len + sess->iv.length;
1323 DPAA2_SET_FLE_SG_EXT(fle);
1325 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1326 sge->length = sess->iv.length;
1329 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1330 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1332 sge->length = data_len;
1333 DPAA2_SET_FLE_FIN(sge);
1334 DPAA2_SET_FLE_FIN(fle);
1337 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1338 " off =%d, len =%d\n",
1339 DPAA2_GET_FD_ADDR(fd),
1340 DPAA2_GET_FD_BPID(fd),
1341 rte_dpaa2_bpid_info[bpid].meta_data_size,
1342 DPAA2_GET_FD_OFFSET(fd),
1343 DPAA2_GET_FD_LEN(fd));
1349 build_sec_fd(struct rte_crypto_op *op,
1350 struct qbman_fd *fd, uint16_t bpid)
1353 dpaa2_sec_session *sess;
1355 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1356 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1357 op->sym->session, cryptodev_driver_id);
1358 #ifdef RTE_LIBRTE_SECURITY
1359 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1360 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1361 op->sym->sec_session);
1369 /* Any of the buffer is segmented*/
1370 if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1371 ((op->sym->m_dst != NULL) &&
1372 !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1373 switch (sess->ctxt_type) {
1374 case DPAA2_SEC_CIPHER:
1375 ret = build_cipher_sg_fd(sess, op, fd, bpid);
1377 case DPAA2_SEC_AUTH:
1378 ret = build_auth_sg_fd(sess, op, fd, bpid);
1380 case DPAA2_SEC_AEAD:
1381 ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1383 case DPAA2_SEC_CIPHER_HASH:
1384 ret = build_authenc_sg_fd(sess, op, fd, bpid);
1386 #ifdef RTE_LIBRTE_SECURITY
1387 case DPAA2_SEC_IPSEC:
1388 case DPAA2_SEC_PDCP:
1389 ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1392 case DPAA2_SEC_HASH_CIPHER:
1394 DPAA2_SEC_ERR("error: Unsupported session");
1397 switch (sess->ctxt_type) {
1398 case DPAA2_SEC_CIPHER:
1399 ret = build_cipher_fd(sess, op, fd, bpid);
1401 case DPAA2_SEC_AUTH:
1402 ret = build_auth_fd(sess, op, fd, bpid);
1404 case DPAA2_SEC_AEAD:
1405 ret = build_authenc_gcm_fd(sess, op, fd, bpid);
1407 case DPAA2_SEC_CIPHER_HASH:
1408 ret = build_authenc_fd(sess, op, fd, bpid);
1410 #ifdef RTE_LIBRTE_SECURITY
1411 case DPAA2_SEC_IPSEC:
1412 ret = build_proto_fd(sess, op, fd, bpid);
1414 case DPAA2_SEC_PDCP:
1415 ret = build_proto_compound_fd(sess, op, fd, bpid);
1418 case DPAA2_SEC_HASH_CIPHER:
1420 DPAA2_SEC_ERR("error: Unsupported session");
1428 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1431 /* Function to transmit the frames to given device and VQ*/
1434 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1435 uint32_t frames_to_send, retry_count;
1436 struct qbman_eq_desc eqdesc;
1437 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1438 struct qbman_swp *swp;
1439 uint16_t num_tx = 0;
1440 uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1441 /*todo - need to support multiple buffer pools */
1443 struct rte_mempool *mb_pool;
1445 if (unlikely(nb_ops == 0))
1448 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1449 DPAA2_SEC_ERR("sessionless crypto op not supported");
1452 /*Prepare enqueue descriptor*/
1453 qbman_eq_desc_clear(&eqdesc);
1454 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1455 qbman_eq_desc_set_response(&eqdesc, 0, 0);
1456 qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1458 if (!DPAA2_PER_LCORE_DPIO) {
1459 ret = dpaa2_affine_qbman_swp();
1462 "Failed to allocate IO portal, tid: %d\n",
1467 swp = DPAA2_PER_LCORE_PORTAL;
1470 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1471 dpaa2_eqcr_size : nb_ops;
1473 for (loop = 0; loop < frames_to_send; loop++) {
1474 if ((*ops)->sym->m_src->seqn) {
1475 uint8_t dqrr_index = (*ops)->sym->m_src->seqn - 1;
1477 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
1478 DPAA2_PER_LCORE_DQRR_SIZE--;
1479 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1480 (*ops)->sym->m_src->seqn = DPAA2_INVALID_MBUF_SEQN;
1483 /*Clear the unused FD fields before sending*/
1484 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1485 mb_pool = (*ops)->sym->m_src->pool;
1486 bpid = mempool_to_bpid(mb_pool);
1487 ret = build_sec_fd(*ops, &fd_arr[loop], bpid);
1489 DPAA2_SEC_ERR("error: Improper packet contents"
1490 " for crypto operation");
1498 while (loop < frames_to_send) {
1499 ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1502 frames_to_send - loop);
1503 if (unlikely(ret < 0)) {
1505 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1520 dpaa2_qp->tx_vq.tx_pkts += num_tx;
1521 dpaa2_qp->tx_vq.err_pkts += nb_ops;
1525 #ifdef RTE_LIBRTE_SECURITY
1526 static inline struct rte_crypto_op *
1527 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1529 struct rte_crypto_op *op;
1530 uint16_t len = DPAA2_GET_FD_LEN(fd);
1532 dpaa2_sec_session *sess_priv __rte_unused;
1534 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1535 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1536 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1538 diff = len - mbuf->pkt_len;
1539 mbuf->pkt_len += diff;
1540 mbuf->data_len += diff;
1541 op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1542 mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1543 op->sym->aead.digest.phys_addr = 0L;
1545 sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1546 op->sym->sec_session);
1547 if (sess_priv->dir == DIR_ENC)
1548 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1550 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1556 static inline struct rte_crypto_op *
1557 sec_fd_to_mbuf(const struct qbman_fd *fd)
1559 struct qbman_fle *fle;
1560 struct rte_crypto_op *op;
1561 struct ctxt_priv *priv;
1562 struct rte_mbuf *dst, *src;
1564 #ifdef RTE_LIBRTE_SECURITY
1565 if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1566 return sec_simple_fd_to_mbuf(fd);
1568 fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1570 DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1571 fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1573 /* we are using the first FLE entry to store Mbuf.
1574 * Currently we donot know which FLE has the mbuf stored.
1575 * So while retreiving we can go back 1 FLE from the FD -ADDR
1576 * to get the MBUF Addr from the previous FLE.
1577 * We can have a better approach to use the inline Mbuf
1580 if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1581 /* TODO complete it. */
1582 DPAA2_SEC_ERR("error: non inline buffer");
1585 op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1588 src = op->sym->m_src;
1591 if (op->sym->m_dst) {
1592 dst = op->sym->m_dst;
1597 #ifdef RTE_LIBRTE_SECURITY
1598 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1599 uint16_t len = DPAA2_GET_FD_LEN(fd);
1601 while (dst->next != NULL) {
1602 len -= dst->data_len;
1605 dst->data_len = len;
1608 DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1609 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1612 DPAA2_GET_FD_ADDR(fd),
1613 DPAA2_GET_FD_BPID(fd),
1614 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1615 DPAA2_GET_FD_OFFSET(fd),
1616 DPAA2_GET_FD_LEN(fd));
1618 /* free the fle memory */
1619 if (likely(rte_pktmbuf_is_contiguous(src))) {
1620 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1);
1621 rte_mempool_put(priv->fle_pool, (void *)(fle-1));
1623 rte_free((void *)(fle-1));
1629 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1632 /* Function is responsible to receive frames for a given device and VQ*/
1633 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1634 struct qbman_result *dq_storage;
1635 uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1636 int ret, num_rx = 0;
1637 uint8_t is_last = 0, status;
1638 struct qbman_swp *swp;
1639 const struct qbman_fd *fd;
1640 struct qbman_pull_desc pulldesc;
1642 if (!DPAA2_PER_LCORE_DPIO) {
1643 ret = dpaa2_affine_qbman_swp();
1646 "Failed to allocate IO portal, tid: %d\n",
1651 swp = DPAA2_PER_LCORE_PORTAL;
1652 dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1654 qbman_pull_desc_clear(&pulldesc);
1655 qbman_pull_desc_set_numframes(&pulldesc,
1656 (nb_ops > dpaa2_dqrr_size) ?
1657 dpaa2_dqrr_size : nb_ops);
1658 qbman_pull_desc_set_fq(&pulldesc, fqid);
1659 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1660 (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1663 /*Issue a volatile dequeue command. */
1665 if (qbman_swp_pull(swp, &pulldesc)) {
1667 "SEC VDQ command is not issued : QBMAN busy");
1668 /* Portal was busy, try again */
1674 /* Receive the packets till Last Dequeue entry is found with
1675 * respect to the above issues PULL command.
1678 /* Check if the previous issued command is completed.
1679 * Also seems like the SWP is shared between the Ethernet Driver
1680 * and the SEC driver.
1682 while (!qbman_check_command_complete(dq_storage))
1685 /* Loop until the dq_storage is updated with
1686 * new token by QBMAN
1688 while (!qbman_check_new_result(dq_storage))
1690 /* Check whether Last Pull command is Expired and
1691 * setting Condition for Loop termination
1693 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1695 /* Check for valid frame. */
1696 status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1698 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1699 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1704 fd = qbman_result_DQ_fd(dq_storage);
1705 ops[num_rx] = sec_fd_to_mbuf(fd);
1707 if (unlikely(fd->simple.frc)) {
1708 /* TODO Parse SEC errors */
1709 DPAA2_SEC_ERR("SEC returned Error - %x",
1711 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
1713 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1718 } /* End of Packet Rx loop */
1720 dpaa2_qp->rx_vq.rx_pkts += num_rx;
1722 DPAA2_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1723 /*Return the total number of packets received to DPAA2 app*/
1727 /** Release queue pair */
1729 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
1731 struct dpaa2_sec_qp *qp =
1732 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
1734 PMD_INIT_FUNC_TRACE();
1736 if (qp->rx_vq.q_storage) {
1737 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
1738 rte_free(qp->rx_vq.q_storage);
1742 dev->data->queue_pairs[queue_pair_id] = NULL;
1747 /** Setup a queue pair */
1749 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1750 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1751 __rte_unused int socket_id)
1753 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
1754 struct dpaa2_sec_qp *qp;
1755 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
1756 struct dpseci_rx_queue_cfg cfg;
1759 PMD_INIT_FUNC_TRACE();
1761 /* If qp is already in use free ring memory and qp metadata. */
1762 if (dev->data->queue_pairs[qp_id] != NULL) {
1763 DPAA2_SEC_INFO("QP already setup");
1767 DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
1768 dev, qp_id, qp_conf);
1770 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
1772 qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
1773 RTE_CACHE_LINE_SIZE);
1775 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
1779 qp->rx_vq.crypto_data = dev->data;
1780 qp->tx_vq.crypto_data = dev->data;
1781 qp->rx_vq.q_storage = rte_malloc("sec dq storage",
1782 sizeof(struct queue_storage_info_t),
1783 RTE_CACHE_LINE_SIZE);
1784 if (!qp->rx_vq.q_storage) {
1785 DPAA2_SEC_ERR("malloc failed for q_storage");
1788 memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
1790 if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
1791 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
1795 dev->data->queue_pairs[qp_id] = qp;
1797 cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
1798 cfg.user_ctx = (size_t)(&qp->rx_vq);
1799 retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
1804 /** Returns the size of the aesni gcm session structure */
1806 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1808 PMD_INIT_FUNC_TRACE();
1810 return sizeof(dpaa2_sec_session);
1814 dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
1815 struct rte_crypto_sym_xform *xform,
1816 dpaa2_sec_session *session)
1818 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1819 struct alginfo cipherdata;
1820 int bufsize, ret = 0;
1821 struct ctxt_priv *priv;
1822 struct sec_flow_context *flc;
1824 PMD_INIT_FUNC_TRACE();
1826 /* For SEC CIPHER only one descriptor is required. */
1827 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1828 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
1829 RTE_CACHE_LINE_SIZE);
1831 DPAA2_SEC_ERR("No Memory for priv CTXT");
1835 priv->fle_pool = dev_priv->fle_pool;
1837 flc = &priv->flc_desc[0].flc;
1839 session->ctxt_type = DPAA2_SEC_CIPHER;
1840 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1841 RTE_CACHE_LINE_SIZE);
1842 if (session->cipher_key.data == NULL) {
1843 DPAA2_SEC_ERR("No Memory for cipher key");
1847 session->cipher_key.length = xform->cipher.key.length;
1849 memcpy(session->cipher_key.data, xform->cipher.key.data,
1850 xform->cipher.key.length);
1851 cipherdata.key = (size_t)session->cipher_key.data;
1852 cipherdata.keylen = session->cipher_key.length;
1853 cipherdata.key_enc_flags = 0;
1854 cipherdata.key_type = RTA_DATA_IMM;
1856 /* Set IV parameters */
1857 session->iv.offset = xform->cipher.iv.offset;
1858 session->iv.length = xform->cipher.iv.length;
1859 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1862 switch (xform->cipher.algo) {
1863 case RTE_CRYPTO_CIPHER_AES_CBC:
1864 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1865 cipherdata.algmode = OP_ALG_AAI_CBC;
1866 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1867 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1868 SHR_NEVER, &cipherdata,
1872 case RTE_CRYPTO_CIPHER_3DES_CBC:
1873 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1874 cipherdata.algmode = OP_ALG_AAI_CBC;
1875 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1876 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1877 SHR_NEVER, &cipherdata,
1881 case RTE_CRYPTO_CIPHER_AES_CTR:
1882 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1883 cipherdata.algmode = OP_ALG_AAI_CTR;
1884 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1885 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1886 SHR_NEVER, &cipherdata,
1890 case RTE_CRYPTO_CIPHER_3DES_CTR:
1891 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1892 cipherdata.algmode = OP_ALG_AAI_CTR;
1893 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CTR;
1894 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1895 SHR_NEVER, &cipherdata,
1899 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1900 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
1901 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
1902 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
1906 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
1907 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
1908 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
1909 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
1913 case RTE_CRYPTO_CIPHER_KASUMI_F8:
1914 case RTE_CRYPTO_CIPHER_AES_F8:
1915 case RTE_CRYPTO_CIPHER_AES_ECB:
1916 case RTE_CRYPTO_CIPHER_3DES_ECB:
1917 case RTE_CRYPTO_CIPHER_AES_XTS:
1918 case RTE_CRYPTO_CIPHER_ARC4:
1919 case RTE_CRYPTO_CIPHER_NULL:
1920 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
1921 xform->cipher.algo);
1925 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
1926 xform->cipher.algo);
1932 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
1937 flc->word1_sdl = (uint8_t)bufsize;
1938 session->ctxt = priv;
1940 #ifdef CAAM_DESC_DEBUG
1942 for (i = 0; i < bufsize; i++)
1943 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
1948 rte_free(session->cipher_key.data);
1954 dpaa2_sec_auth_init(struct rte_cryptodev *dev,
1955 struct rte_crypto_sym_xform *xform,
1956 dpaa2_sec_session *session)
1958 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1959 struct alginfo authdata;
1960 int bufsize, ret = 0;
1961 struct ctxt_priv *priv;
1962 struct sec_flow_context *flc;
1964 PMD_INIT_FUNC_TRACE();
1966 /* For SEC AUTH three descriptors are required for various stages */
1967 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1968 sizeof(struct ctxt_priv) + 3 *
1969 sizeof(struct sec_flc_desc),
1970 RTE_CACHE_LINE_SIZE);
1972 DPAA2_SEC_ERR("No Memory for priv CTXT");
1976 priv->fle_pool = dev_priv->fle_pool;
1977 flc = &priv->flc_desc[DESC_INITFINAL].flc;
1979 session->ctxt_type = DPAA2_SEC_AUTH;
1980 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length,
1981 RTE_CACHE_LINE_SIZE);
1982 if (session->auth_key.data == NULL) {
1983 DPAA2_SEC_ERR("Unable to allocate memory for auth key");
1987 session->auth_key.length = xform->auth.key.length;
1989 memcpy(session->auth_key.data, xform->auth.key.data,
1990 xform->auth.key.length);
1991 authdata.key = (size_t)session->auth_key.data;
1992 authdata.keylen = session->auth_key.length;
1993 authdata.key_enc_flags = 0;
1994 authdata.key_type = RTA_DATA_IMM;
1996 session->digest_length = xform->auth.digest_length;
1997 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2000 switch (xform->auth.algo) {
2001 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2002 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2003 authdata.algmode = OP_ALG_AAI_HMAC;
2004 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2005 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2006 1, 0, SHR_NEVER, &authdata,
2008 session->digest_length);
2010 case RTE_CRYPTO_AUTH_MD5_HMAC:
2011 authdata.algtype = OP_ALG_ALGSEL_MD5;
2012 authdata.algmode = OP_ALG_AAI_HMAC;
2013 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2014 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2015 1, 0, SHR_NEVER, &authdata,
2017 session->digest_length);
2019 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2020 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2021 authdata.algmode = OP_ALG_AAI_HMAC;
2022 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2023 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2024 1, 0, SHR_NEVER, &authdata,
2026 session->digest_length);
2028 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2029 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2030 authdata.algmode = OP_ALG_AAI_HMAC;
2031 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2032 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2033 1, 0, SHR_NEVER, &authdata,
2035 session->digest_length);
2037 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2038 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2039 authdata.algmode = OP_ALG_AAI_HMAC;
2040 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2041 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2042 1, 0, SHR_NEVER, &authdata,
2044 session->digest_length);
2046 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2047 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2048 authdata.algmode = OP_ALG_AAI_HMAC;
2049 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2050 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2051 1, 0, SHR_NEVER, &authdata,
2053 session->digest_length);
2055 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2056 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2057 authdata.algmode = OP_ALG_AAI_F9;
2058 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2059 session->iv.offset = xform->auth.iv.offset;
2060 session->iv.length = xform->auth.iv.length;
2061 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2064 session->digest_length);
2066 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2067 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2068 authdata.algmode = OP_ALG_AAI_F9;
2069 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2070 session->iv.offset = xform->auth.iv.offset;
2071 session->iv.length = xform->auth.iv.length;
2072 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2075 session->digest_length);
2077 case RTE_CRYPTO_AUTH_KASUMI_F9:
2078 case RTE_CRYPTO_AUTH_NULL:
2079 case RTE_CRYPTO_AUTH_SHA1:
2080 case RTE_CRYPTO_AUTH_SHA256:
2081 case RTE_CRYPTO_AUTH_SHA512:
2082 case RTE_CRYPTO_AUTH_SHA224:
2083 case RTE_CRYPTO_AUTH_SHA384:
2084 case RTE_CRYPTO_AUTH_MD5:
2085 case RTE_CRYPTO_AUTH_AES_GMAC:
2086 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2087 case RTE_CRYPTO_AUTH_AES_CMAC:
2088 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2089 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2094 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2101 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2106 flc->word1_sdl = (uint8_t)bufsize;
2107 session->ctxt = priv;
2108 #ifdef CAAM_DESC_DEBUG
2110 for (i = 0; i < bufsize; i++)
2111 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2112 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2118 rte_free(session->auth_key.data);
2124 dpaa2_sec_aead_init(struct rte_cryptodev *dev,
2125 struct rte_crypto_sym_xform *xform,
2126 dpaa2_sec_session *session)
2128 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2129 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2130 struct alginfo aeaddata;
2132 struct ctxt_priv *priv;
2133 struct sec_flow_context *flc;
2134 struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2137 PMD_INIT_FUNC_TRACE();
2139 /* Set IV parameters */
2140 session->iv.offset = aead_xform->iv.offset;
2141 session->iv.length = aead_xform->iv.length;
2142 session->ctxt_type = DPAA2_SEC_AEAD;
2144 /* For SEC AEAD only one descriptor is required */
2145 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2146 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2147 RTE_CACHE_LINE_SIZE);
2149 DPAA2_SEC_ERR("No Memory for priv CTXT");
2153 priv->fle_pool = dev_priv->fle_pool;
2154 flc = &priv->flc_desc[0].flc;
2156 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2157 RTE_CACHE_LINE_SIZE);
2158 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2159 DPAA2_SEC_ERR("No Memory for aead key");
2163 memcpy(session->aead_key.data, aead_xform->key.data,
2164 aead_xform->key.length);
2166 session->digest_length = aead_xform->digest_length;
2167 session->aead_key.length = aead_xform->key.length;
2168 ctxt->auth_only_len = aead_xform->aad_length;
2170 aeaddata.key = (size_t)session->aead_key.data;
2171 aeaddata.keylen = session->aead_key.length;
2172 aeaddata.key_enc_flags = 0;
2173 aeaddata.key_type = RTA_DATA_IMM;
2175 switch (aead_xform->algo) {
2176 case RTE_CRYPTO_AEAD_AES_GCM:
2177 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2178 aeaddata.algmode = OP_ALG_AAI_GCM;
2179 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2181 case RTE_CRYPTO_AEAD_AES_CCM:
2182 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2187 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2192 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2195 priv->flc_desc[0].desc[0] = aeaddata.keylen;
2196 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2198 (unsigned int *)priv->flc_desc[0].desc,
2199 &priv->flc_desc[0].desc[1], 1);
2202 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2206 if (priv->flc_desc[0].desc[1] & 1) {
2207 aeaddata.key_type = RTA_DATA_IMM;
2209 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2210 aeaddata.key_type = RTA_DATA_PTR;
2212 priv->flc_desc[0].desc[0] = 0;
2213 priv->flc_desc[0].desc[1] = 0;
2215 if (session->dir == DIR_ENC)
2216 bufsize = cnstr_shdsc_gcm_encap(
2217 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2218 &aeaddata, session->iv.length,
2219 session->digest_length);
2221 bufsize = cnstr_shdsc_gcm_decap(
2222 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2223 &aeaddata, session->iv.length,
2224 session->digest_length);
2226 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2231 flc->word1_sdl = (uint8_t)bufsize;
2232 session->ctxt = priv;
2233 #ifdef CAAM_DESC_DEBUG
2235 for (i = 0; i < bufsize; i++)
2236 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2237 i, priv->flc_desc[0].desc[i]);
2242 rte_free(session->aead_key.data);
2249 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
2250 struct rte_crypto_sym_xform *xform,
2251 dpaa2_sec_session *session)
2253 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2254 struct alginfo authdata, cipherdata;
2256 struct ctxt_priv *priv;
2257 struct sec_flow_context *flc;
2258 struct rte_crypto_cipher_xform *cipher_xform;
2259 struct rte_crypto_auth_xform *auth_xform;
2262 PMD_INIT_FUNC_TRACE();
2264 if (session->ext_params.aead_ctxt.auth_cipher_text) {
2265 cipher_xform = &xform->cipher;
2266 auth_xform = &xform->next->auth;
2267 session->ctxt_type =
2268 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2269 DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2271 cipher_xform = &xform->next->cipher;
2272 auth_xform = &xform->auth;
2273 session->ctxt_type =
2274 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2275 DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2278 /* Set IV parameters */
2279 session->iv.offset = cipher_xform->iv.offset;
2280 session->iv.length = cipher_xform->iv.length;
2282 /* For SEC AEAD only one descriptor is required */
2283 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2284 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2285 RTE_CACHE_LINE_SIZE);
2287 DPAA2_SEC_ERR("No Memory for priv CTXT");
2291 priv->fle_pool = dev_priv->fle_pool;
2292 flc = &priv->flc_desc[0].flc;
2294 session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2295 RTE_CACHE_LINE_SIZE);
2296 if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2297 DPAA2_SEC_ERR("No Memory for cipher key");
2301 session->cipher_key.length = cipher_xform->key.length;
2302 session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2303 RTE_CACHE_LINE_SIZE);
2304 if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2305 DPAA2_SEC_ERR("No Memory for auth key");
2306 rte_free(session->cipher_key.data);
2310 session->auth_key.length = auth_xform->key.length;
2311 memcpy(session->cipher_key.data, cipher_xform->key.data,
2312 cipher_xform->key.length);
2313 memcpy(session->auth_key.data, auth_xform->key.data,
2314 auth_xform->key.length);
2316 authdata.key = (size_t)session->auth_key.data;
2317 authdata.keylen = session->auth_key.length;
2318 authdata.key_enc_flags = 0;
2319 authdata.key_type = RTA_DATA_IMM;
2321 session->digest_length = auth_xform->digest_length;
2323 switch (auth_xform->algo) {
2324 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2325 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2326 authdata.algmode = OP_ALG_AAI_HMAC;
2327 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2329 case RTE_CRYPTO_AUTH_MD5_HMAC:
2330 authdata.algtype = OP_ALG_ALGSEL_MD5;
2331 authdata.algmode = OP_ALG_AAI_HMAC;
2332 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2334 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2335 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2336 authdata.algmode = OP_ALG_AAI_HMAC;
2337 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2339 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2340 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2341 authdata.algmode = OP_ALG_AAI_HMAC;
2342 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2344 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2345 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2346 authdata.algmode = OP_ALG_AAI_HMAC;
2347 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2349 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2350 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2351 authdata.algmode = OP_ALG_AAI_HMAC;
2352 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2354 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2355 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2356 case RTE_CRYPTO_AUTH_NULL:
2357 case RTE_CRYPTO_AUTH_SHA1:
2358 case RTE_CRYPTO_AUTH_SHA256:
2359 case RTE_CRYPTO_AUTH_SHA512:
2360 case RTE_CRYPTO_AUTH_SHA224:
2361 case RTE_CRYPTO_AUTH_SHA384:
2362 case RTE_CRYPTO_AUTH_MD5:
2363 case RTE_CRYPTO_AUTH_AES_GMAC:
2364 case RTE_CRYPTO_AUTH_KASUMI_F9:
2365 case RTE_CRYPTO_AUTH_AES_CMAC:
2366 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2367 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2368 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2373 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2378 cipherdata.key = (size_t)session->cipher_key.data;
2379 cipherdata.keylen = session->cipher_key.length;
2380 cipherdata.key_enc_flags = 0;
2381 cipherdata.key_type = RTA_DATA_IMM;
2383 switch (cipher_xform->algo) {
2384 case RTE_CRYPTO_CIPHER_AES_CBC:
2385 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2386 cipherdata.algmode = OP_ALG_AAI_CBC;
2387 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2389 case RTE_CRYPTO_CIPHER_3DES_CBC:
2390 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2391 cipherdata.algmode = OP_ALG_AAI_CBC;
2392 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2394 case RTE_CRYPTO_CIPHER_AES_CTR:
2395 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2396 cipherdata.algmode = OP_ALG_AAI_CTR;
2397 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2399 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2400 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2401 case RTE_CRYPTO_CIPHER_NULL:
2402 case RTE_CRYPTO_CIPHER_3DES_ECB:
2403 case RTE_CRYPTO_CIPHER_AES_ECB:
2404 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2405 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2406 cipher_xform->algo);
2410 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2411 cipher_xform->algo);
2415 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2418 priv->flc_desc[0].desc[0] = cipherdata.keylen;
2419 priv->flc_desc[0].desc[1] = authdata.keylen;
2420 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2422 (unsigned int *)priv->flc_desc[0].desc,
2423 &priv->flc_desc[0].desc[2], 2);
2426 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2430 if (priv->flc_desc[0].desc[2] & 1) {
2431 cipherdata.key_type = RTA_DATA_IMM;
2433 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2434 cipherdata.key_type = RTA_DATA_PTR;
2436 if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2437 authdata.key_type = RTA_DATA_IMM;
2439 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2440 authdata.key_type = RTA_DATA_PTR;
2442 priv->flc_desc[0].desc[0] = 0;
2443 priv->flc_desc[0].desc[1] = 0;
2444 priv->flc_desc[0].desc[2] = 0;
2446 if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2447 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2449 &cipherdata, &authdata,
2451 session->digest_length,
2454 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2459 DPAA2_SEC_ERR("Hash before cipher not supported");
2464 flc->word1_sdl = (uint8_t)bufsize;
2465 session->ctxt = priv;
2466 #ifdef CAAM_DESC_DEBUG
2468 for (i = 0; i < bufsize; i++)
2469 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2470 i, priv->flc_desc[0].desc[i]);
2476 rte_free(session->cipher_key.data);
2477 rte_free(session->auth_key.data);
2483 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
2484 struct rte_crypto_sym_xform *xform, void *sess)
2486 dpaa2_sec_session *session = sess;
2489 PMD_INIT_FUNC_TRACE();
2491 if (unlikely(sess == NULL)) {
2492 DPAA2_SEC_ERR("Invalid session struct");
2496 memset(session, 0, sizeof(dpaa2_sec_session));
2497 /* Default IV length = 0 */
2498 session->iv.length = 0;
2501 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2502 ret = dpaa2_sec_cipher_init(dev, xform, session);
2504 /* Authentication Only */
2505 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2506 xform->next == NULL) {
2507 ret = dpaa2_sec_auth_init(dev, xform, session);
2509 /* Cipher then Authenticate */
2510 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2511 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2512 session->ext_params.aead_ctxt.auth_cipher_text = true;
2513 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2514 ret = dpaa2_sec_auth_init(dev, xform, session);
2515 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2516 ret = dpaa2_sec_cipher_init(dev, xform, session);
2518 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2519 /* Authenticate then Cipher */
2520 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2521 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2522 session->ext_params.aead_ctxt.auth_cipher_text = false;
2523 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2524 ret = dpaa2_sec_cipher_init(dev, xform, session);
2525 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2526 ret = dpaa2_sec_auth_init(dev, xform, session);
2528 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2529 /* AEAD operation for AES-GCM kind of Algorithms */
2530 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2531 xform->next == NULL) {
2532 ret = dpaa2_sec_aead_init(dev, xform, session);
2535 DPAA2_SEC_ERR("Invalid crypto type");
2542 #ifdef RTE_LIBRTE_SECURITY
2544 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2545 dpaa2_sec_session *session,
2546 struct alginfo *aeaddata)
2548 PMD_INIT_FUNC_TRACE();
2550 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2551 RTE_CACHE_LINE_SIZE);
2552 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2553 DPAA2_SEC_ERR("No Memory for aead key");
2556 memcpy(session->aead_key.data, aead_xform->key.data,
2557 aead_xform->key.length);
2559 session->digest_length = aead_xform->digest_length;
2560 session->aead_key.length = aead_xform->key.length;
2562 aeaddata->key = (size_t)session->aead_key.data;
2563 aeaddata->keylen = session->aead_key.length;
2564 aeaddata->key_enc_flags = 0;
2565 aeaddata->key_type = RTA_DATA_IMM;
2567 switch (aead_xform->algo) {
2568 case RTE_CRYPTO_AEAD_AES_GCM:
2569 switch (session->digest_length) {
2571 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2574 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2577 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2580 DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2581 session->digest_length);
2584 aeaddata->algmode = OP_ALG_AAI_GCM;
2585 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2587 case RTE_CRYPTO_AEAD_AES_CCM:
2588 switch (session->digest_length) {
2590 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2593 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2596 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2599 DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2600 session->digest_length);
2603 aeaddata->algmode = OP_ALG_AAI_CCM;
2604 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2607 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2611 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2618 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2619 struct rte_crypto_auth_xform *auth_xform,
2620 dpaa2_sec_session *session,
2621 struct alginfo *cipherdata,
2622 struct alginfo *authdata)
2625 session->cipher_key.data = rte_zmalloc(NULL,
2626 cipher_xform->key.length,
2627 RTE_CACHE_LINE_SIZE);
2628 if (session->cipher_key.data == NULL &&
2629 cipher_xform->key.length > 0) {
2630 DPAA2_SEC_ERR("No Memory for cipher key");
2634 session->cipher_key.length = cipher_xform->key.length;
2635 memcpy(session->cipher_key.data, cipher_xform->key.data,
2636 cipher_xform->key.length);
2637 session->cipher_alg = cipher_xform->algo;
2639 session->cipher_key.data = NULL;
2640 session->cipher_key.length = 0;
2641 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2645 session->auth_key.data = rte_zmalloc(NULL,
2646 auth_xform->key.length,
2647 RTE_CACHE_LINE_SIZE);
2648 if (session->auth_key.data == NULL &&
2649 auth_xform->key.length > 0) {
2650 DPAA2_SEC_ERR("No Memory for auth key");
2653 session->auth_key.length = auth_xform->key.length;
2654 memcpy(session->auth_key.data, auth_xform->key.data,
2655 auth_xform->key.length);
2656 session->auth_alg = auth_xform->algo;
2657 session->digest_length = auth_xform->digest_length;
2659 session->auth_key.data = NULL;
2660 session->auth_key.length = 0;
2661 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2664 authdata->key = (size_t)session->auth_key.data;
2665 authdata->keylen = session->auth_key.length;
2666 authdata->key_enc_flags = 0;
2667 authdata->key_type = RTA_DATA_IMM;
2668 switch (session->auth_alg) {
2669 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2670 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
2671 authdata->algmode = OP_ALG_AAI_HMAC;
2673 case RTE_CRYPTO_AUTH_MD5_HMAC:
2674 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
2675 authdata->algmode = OP_ALG_AAI_HMAC;
2677 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2678 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2679 authdata->algmode = OP_ALG_AAI_HMAC;
2680 if (session->digest_length != 16)
2682 "+++Using sha256-hmac truncated len is non-standard,"
2683 "it will not work with lookaside proto");
2685 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2686 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2687 authdata->algmode = OP_ALG_AAI_HMAC;
2689 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2690 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2691 authdata->algmode = OP_ALG_AAI_HMAC;
2693 case RTE_CRYPTO_AUTH_AES_CMAC:
2694 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
2696 case RTE_CRYPTO_AUTH_NULL:
2697 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
2699 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2700 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2701 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2702 case RTE_CRYPTO_AUTH_SHA1:
2703 case RTE_CRYPTO_AUTH_SHA256:
2704 case RTE_CRYPTO_AUTH_SHA512:
2705 case RTE_CRYPTO_AUTH_SHA224:
2706 case RTE_CRYPTO_AUTH_SHA384:
2707 case RTE_CRYPTO_AUTH_MD5:
2708 case RTE_CRYPTO_AUTH_AES_GMAC:
2709 case RTE_CRYPTO_AUTH_KASUMI_F9:
2710 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2711 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2712 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2716 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2720 cipherdata->key = (size_t)session->cipher_key.data;
2721 cipherdata->keylen = session->cipher_key.length;
2722 cipherdata->key_enc_flags = 0;
2723 cipherdata->key_type = RTA_DATA_IMM;
2725 switch (session->cipher_alg) {
2726 case RTE_CRYPTO_CIPHER_AES_CBC:
2727 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
2728 cipherdata->algmode = OP_ALG_AAI_CBC;
2730 case RTE_CRYPTO_CIPHER_3DES_CBC:
2731 cipherdata->algtype = OP_PCL_IPSEC_3DES;
2732 cipherdata->algmode = OP_ALG_AAI_CBC;
2734 case RTE_CRYPTO_CIPHER_AES_CTR:
2735 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
2736 cipherdata->algmode = OP_ALG_AAI_CTR;
2738 case RTE_CRYPTO_CIPHER_NULL:
2739 cipherdata->algtype = OP_PCL_IPSEC_NULL;
2741 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2742 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2743 case RTE_CRYPTO_CIPHER_3DES_ECB:
2744 case RTE_CRYPTO_CIPHER_AES_ECB:
2745 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2746 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2747 session->cipher_alg);
2750 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2751 session->cipher_alg);
2758 #ifdef RTE_LIBRTE_SECURITY_TEST
2759 static uint8_t aes_cbc_iv[] = {
2760 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
2761 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
2765 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
2766 struct rte_security_session_conf *conf,
2769 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2770 struct rte_crypto_cipher_xform *cipher_xform = NULL;
2771 struct rte_crypto_auth_xform *auth_xform = NULL;
2772 struct rte_crypto_aead_xform *aead_xform = NULL;
2773 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
2774 struct ctxt_priv *priv;
2775 struct alginfo authdata, cipherdata;
2777 struct sec_flow_context *flc;
2778 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2781 PMD_INIT_FUNC_TRACE();
2783 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2784 sizeof(struct ctxt_priv) +
2785 sizeof(struct sec_flc_desc),
2786 RTE_CACHE_LINE_SIZE);
2789 DPAA2_SEC_ERR("No memory for priv CTXT");
2793 priv->fle_pool = dev_priv->fle_pool;
2794 flc = &priv->flc_desc[0].flc;
2796 memset(session, 0, sizeof(dpaa2_sec_session));
2798 if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2799 cipher_xform = &conf->crypto_xform->cipher;
2800 if (conf->crypto_xform->next)
2801 auth_xform = &conf->crypto_xform->next->auth;
2802 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2803 session, &cipherdata, &authdata);
2804 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2805 auth_xform = &conf->crypto_xform->auth;
2806 if (conf->crypto_xform->next)
2807 cipher_xform = &conf->crypto_xform->next->cipher;
2808 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2809 session, &cipherdata, &authdata);
2810 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2811 aead_xform = &conf->crypto_xform->aead;
2812 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
2813 session, &cipherdata);
2814 authdata.keylen = 0;
2815 authdata.algtype = 0;
2817 DPAA2_SEC_ERR("XFORM not specified");
2822 DPAA2_SEC_ERR("Failed to process xform");
2826 session->ctxt_type = DPAA2_SEC_IPSEC;
2827 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2828 uint8_t *hdr = NULL;
2830 struct rte_ipv6_hdr ip6_hdr;
2831 struct ipsec_encap_pdb encap_pdb;
2833 flc->dhr = SEC_FLC_DHR_OUTBOUND;
2834 /* For Sec Proto only one descriptor is required. */
2835 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
2837 /* copy algo specific data to PDB */
2838 switch (cipherdata.algtype) {
2839 case OP_PCL_IPSEC_AES_CTR:
2840 encap_pdb.ctr.ctr_initial = 0x00000001;
2841 encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2843 case OP_PCL_IPSEC_AES_GCM8:
2844 case OP_PCL_IPSEC_AES_GCM12:
2845 case OP_PCL_IPSEC_AES_GCM16:
2846 memcpy(encap_pdb.gcm.salt,
2847 (uint8_t *)&(ipsec_xform->salt), 4);
2851 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2852 PDBOPTS_ESP_OIHI_PDB_INL |
2854 PDBHMO_ESP_ENCAP_DTTL |
2856 if (ipsec_xform->options.esn)
2857 encap_pdb.options |= PDBOPTS_ESP_ESN;
2858 encap_pdb.spi = ipsec_xform->spi;
2859 session->dir = DIR_ENC;
2860 if (ipsec_xform->tunnel.type ==
2861 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
2862 encap_pdb.ip_hdr_len = sizeof(struct ip);
2863 ip4_hdr.ip_v = IPVERSION;
2865 ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
2866 ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2869 ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2870 ip4_hdr.ip_p = IPPROTO_ESP;
2872 ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2873 ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2874 ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
2875 &ip4_hdr, sizeof(struct ip));
2876 hdr = (uint8_t *)&ip4_hdr;
2877 } else if (ipsec_xform->tunnel.type ==
2878 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
2879 ip6_hdr.vtc_flow = rte_cpu_to_be_32(
2880 DPAA2_IPv6_DEFAULT_VTC_FLOW |
2881 ((ipsec_xform->tunnel.ipv6.dscp <<
2882 RTE_IPV6_HDR_TC_SHIFT) &
2883 RTE_IPV6_HDR_TC_MASK) |
2884 ((ipsec_xform->tunnel.ipv6.flabel <<
2885 RTE_IPV6_HDR_FL_SHIFT) &
2886 RTE_IPV6_HDR_FL_MASK));
2887 /* Payload length will be updated by HW */
2888 ip6_hdr.payload_len = 0;
2889 ip6_hdr.hop_limits =
2890 ipsec_xform->tunnel.ipv6.hlimit;
2891 ip6_hdr.proto = (ipsec_xform->proto ==
2892 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
2893 IPPROTO_ESP : IPPROTO_AH;
2894 memcpy(&ip6_hdr.src_addr,
2895 &ipsec_xform->tunnel.ipv6.src_addr, 16);
2896 memcpy(&ip6_hdr.dst_addr,
2897 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
2898 encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
2899 hdr = (uint8_t *)&ip6_hdr;
2902 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
2903 1, 0, SHR_SERIAL, &encap_pdb,
2904 hdr, &cipherdata, &authdata);
2905 } else if (ipsec_xform->direction ==
2906 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2907 struct ipsec_decap_pdb decap_pdb;
2909 flc->dhr = SEC_FLC_DHR_INBOUND;
2910 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2911 /* copy algo specific data to PDB */
2912 switch (cipherdata.algtype) {
2913 case OP_PCL_IPSEC_AES_CTR:
2914 decap_pdb.ctr.ctr_initial = 0x00000001;
2915 decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2917 case OP_PCL_IPSEC_AES_GCM8:
2918 case OP_PCL_IPSEC_AES_GCM12:
2919 case OP_PCL_IPSEC_AES_GCM16:
2920 memcpy(decap_pdb.gcm.salt,
2921 (uint8_t *)&(ipsec_xform->salt), 4);
2925 decap_pdb.options = (ipsec_xform->tunnel.type ==
2926 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
2927 sizeof(struct ip) << 16 :
2928 sizeof(struct rte_ipv6_hdr) << 16;
2929 if (ipsec_xform->options.esn)
2930 decap_pdb.options |= PDBOPTS_ESP_ESN;
2932 if (ipsec_xform->replay_win_sz) {
2934 win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
2943 decap_pdb.options |= PDBOPTS_ESP_ARS32;
2946 decap_pdb.options |= PDBOPTS_ESP_ARS64;
2949 decap_pdb.options |= PDBOPTS_ESP_ARS128;
2952 session->dir = DIR_DEC;
2953 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
2955 &decap_pdb, &cipherdata, &authdata);
2960 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2964 flc->word1_sdl = (uint8_t)bufsize;
2966 /* Enable the stashing control bit */
2967 DPAA2_SET_FLC_RSC(flc);
2968 flc->word2_rflc_31_0 = lower_32_bits(
2969 (size_t)&(((struct dpaa2_sec_qp *)
2970 dev->data->queue_pairs[0])->rx_vq) | 0x14);
2971 flc->word3_rflc_63_32 = upper_32_bits(
2972 (size_t)&(((struct dpaa2_sec_qp *)
2973 dev->data->queue_pairs[0])->rx_vq));
2975 /* Set EWS bit i.e. enable write-safe */
2976 DPAA2_SET_FLC_EWS(flc);
2977 /* Set BS = 1 i.e reuse input buffers as output buffers */
2978 DPAA2_SET_FLC_REUSE_BS(flc);
2979 /* Set FF = 10; reuse input buffers if they provide sufficient space */
2980 DPAA2_SET_FLC_REUSE_FF(flc);
2982 session->ctxt = priv;
2986 rte_free(session->auth_key.data);
2987 rte_free(session->cipher_key.data);
2993 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
2994 struct rte_security_session_conf *conf,
2997 struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
2998 struct rte_crypto_sym_xform *xform = conf->crypto_xform;
2999 struct rte_crypto_auth_xform *auth_xform = NULL;
3000 struct rte_crypto_cipher_xform *cipher_xform;
3001 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3002 struct ctxt_priv *priv;
3003 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
3004 struct alginfo authdata, cipherdata;
3005 struct alginfo *p_authdata = NULL;
3007 struct sec_flow_context *flc;
3008 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3014 PMD_INIT_FUNC_TRACE();
3016 memset(session, 0, sizeof(dpaa2_sec_session));
3018 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3019 sizeof(struct ctxt_priv) +
3020 sizeof(struct sec_flc_desc),
3021 RTE_CACHE_LINE_SIZE);
3024 DPAA2_SEC_ERR("No memory for priv CTXT");
3028 priv->fle_pool = dev_priv->fle_pool;
3029 flc = &priv->flc_desc[0].flc;
3031 /* find xfrm types */
3032 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
3033 cipher_xform = &xform->cipher;
3034 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
3035 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3036 session->ext_params.aead_ctxt.auth_cipher_text = true;
3037 cipher_xform = &xform->cipher;
3038 auth_xform = &xform->next->auth;
3039 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
3040 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3041 session->ext_params.aead_ctxt.auth_cipher_text = false;
3042 cipher_xform = &xform->next->cipher;
3043 auth_xform = &xform->auth;
3045 DPAA2_SEC_ERR("Invalid crypto type");
3049 session->ctxt_type = DPAA2_SEC_PDCP;
3051 session->cipher_key.data = rte_zmalloc(NULL,
3052 cipher_xform->key.length,
3053 RTE_CACHE_LINE_SIZE);
3054 if (session->cipher_key.data == NULL &&
3055 cipher_xform->key.length > 0) {
3056 DPAA2_SEC_ERR("No Memory for cipher key");
3060 session->cipher_key.length = cipher_xform->key.length;
3061 memcpy(session->cipher_key.data, cipher_xform->key.data,
3062 cipher_xform->key.length);
3064 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3066 session->cipher_alg = cipher_xform->algo;
3068 session->cipher_key.data = NULL;
3069 session->cipher_key.length = 0;
3070 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3071 session->dir = DIR_ENC;
3074 session->pdcp.domain = pdcp_xform->domain;
3075 session->pdcp.bearer = pdcp_xform->bearer;
3076 session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3077 session->pdcp.sn_size = pdcp_xform->sn_size;
3078 session->pdcp.hfn = pdcp_xform->hfn;
3079 session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3080 session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3081 /* hfv ovd offset location is stored in iv.offset value*/
3082 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3084 cipherdata.key = (size_t)session->cipher_key.data;
3085 cipherdata.keylen = session->cipher_key.length;
3086 cipherdata.key_enc_flags = 0;
3087 cipherdata.key_type = RTA_DATA_IMM;
3089 switch (session->cipher_alg) {
3090 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3091 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3093 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3094 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3096 case RTE_CRYPTO_CIPHER_AES_CTR:
3097 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3099 case RTE_CRYPTO_CIPHER_NULL:
3100 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3103 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3104 session->cipher_alg);
3109 session->auth_key.data = rte_zmalloc(NULL,
3110 auth_xform->key.length,
3111 RTE_CACHE_LINE_SIZE);
3112 if (!session->auth_key.data &&
3113 auth_xform->key.length > 0) {
3114 DPAA2_SEC_ERR("No Memory for auth key");
3115 rte_free(session->cipher_key.data);
3119 session->auth_key.length = auth_xform->key.length;
3120 memcpy(session->auth_key.data, auth_xform->key.data,
3121 auth_xform->key.length);
3122 session->auth_alg = auth_xform->algo;
3124 session->auth_key.data = NULL;
3125 session->auth_key.length = 0;
3126 session->auth_alg = 0;
3128 authdata.key = (size_t)session->auth_key.data;
3129 authdata.keylen = session->auth_key.length;
3130 authdata.key_enc_flags = 0;
3131 authdata.key_type = RTA_DATA_IMM;
3133 if (session->auth_alg) {
3134 switch (session->auth_alg) {
3135 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3136 authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3138 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3139 authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3141 case RTE_CRYPTO_AUTH_AES_CMAC:
3142 authdata.algtype = PDCP_AUTH_TYPE_AES;
3144 case RTE_CRYPTO_AUTH_NULL:
3145 authdata.algtype = PDCP_AUTH_TYPE_NULL;
3148 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3153 p_authdata = &authdata;
3154 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3155 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3159 if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3160 if (session->dir == DIR_ENC)
3161 bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3162 priv->flc_desc[0].desc, 1, swap,
3164 session->pdcp.sn_size,
3166 pdcp_xform->pkt_dir,
3167 pdcp_xform->hfn_threshold,
3168 &cipherdata, &authdata,
3170 else if (session->dir == DIR_DEC)
3171 bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3172 priv->flc_desc[0].desc, 1, swap,
3174 session->pdcp.sn_size,
3176 pdcp_xform->pkt_dir,
3177 pdcp_xform->hfn_threshold,
3178 &cipherdata, &authdata,
3181 if (session->dir == DIR_ENC)
3182 bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3183 priv->flc_desc[0].desc, 1, swap,
3184 session->pdcp.sn_size,
3187 pdcp_xform->pkt_dir,
3188 pdcp_xform->hfn_threshold,
3189 &cipherdata, p_authdata, 0);
3190 else if (session->dir == DIR_DEC)
3191 bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3192 priv->flc_desc[0].desc, 1, swap,
3193 session->pdcp.sn_size,
3196 pdcp_xform->pkt_dir,
3197 pdcp_xform->hfn_threshold,
3198 &cipherdata, p_authdata, 0);
3202 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3206 /* Enable the stashing control bit */
3207 DPAA2_SET_FLC_RSC(flc);
3208 flc->word2_rflc_31_0 = lower_32_bits(
3209 (size_t)&(((struct dpaa2_sec_qp *)
3210 dev->data->queue_pairs[0])->rx_vq) | 0x14);
3211 flc->word3_rflc_63_32 = upper_32_bits(
3212 (size_t)&(((struct dpaa2_sec_qp *)
3213 dev->data->queue_pairs[0])->rx_vq));
3215 flc->word1_sdl = (uint8_t)bufsize;
3217 /* TODO - check the perf impact or
3218 * align as per descriptor type
3219 * Set EWS bit i.e. enable write-safe
3220 * DPAA2_SET_FLC_EWS(flc);
3223 /* Set BS = 1 i.e reuse input buffers as output buffers */
3224 DPAA2_SET_FLC_REUSE_BS(flc);
3225 /* Set FF = 10; reuse input buffers if they provide sufficient space */
3226 DPAA2_SET_FLC_REUSE_FF(flc);
3228 session->ctxt = priv;
3232 rte_free(session->auth_key.data);
3233 rte_free(session->cipher_key.data);
3239 dpaa2_sec_security_session_create(void *dev,
3240 struct rte_security_session_conf *conf,
3241 struct rte_security_session *sess,
3242 struct rte_mempool *mempool)
3244 void *sess_private_data;
3245 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3248 if (rte_mempool_get(mempool, &sess_private_data)) {
3249 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3253 switch (conf->protocol) {
3254 case RTE_SECURITY_PROTOCOL_IPSEC:
3255 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3258 case RTE_SECURITY_PROTOCOL_MACSEC:
3260 case RTE_SECURITY_PROTOCOL_PDCP:
3261 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3268 DPAA2_SEC_ERR("Failed to configure session parameters");
3269 /* Return session to mempool */
3270 rte_mempool_put(mempool, sess_private_data);
3274 set_sec_session_private_data(sess, sess_private_data);
3279 /** Clear the memory of session so it doesn't leave key material behind */
3281 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3282 struct rte_security_session *sess)
3284 PMD_INIT_FUNC_TRACE();
3285 void *sess_priv = get_sec_session_private_data(sess);
3287 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3290 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3293 rte_free(s->cipher_key.data);
3294 rte_free(s->auth_key.data);
3295 memset(s, 0, sizeof(dpaa2_sec_session));
3296 set_sec_session_private_data(sess, NULL);
3297 rte_mempool_put(sess_mp, sess_priv);
3303 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3304 struct rte_crypto_sym_xform *xform,
3305 struct rte_cryptodev_sym_session *sess,
3306 struct rte_mempool *mempool)
3308 void *sess_private_data;
3311 if (rte_mempool_get(mempool, &sess_private_data)) {
3312 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3316 ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data);
3318 DPAA2_SEC_ERR("Failed to configure session parameters");
3319 /* Return session to mempool */
3320 rte_mempool_put(mempool, sess_private_data);
3324 set_sym_session_private_data(sess, dev->driver_id,
3330 /** Clear the memory of session so it doesn't leave key material behind */
3332 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3333 struct rte_cryptodev_sym_session *sess)
3335 PMD_INIT_FUNC_TRACE();
3336 uint8_t index = dev->driver_id;
3337 void *sess_priv = get_sym_session_private_data(sess, index);
3338 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3342 rte_free(s->cipher_key.data);
3343 rte_free(s->auth_key.data);
3344 memset(s, 0, sizeof(dpaa2_sec_session));
3345 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3346 set_sym_session_private_data(sess, index, NULL);
3347 rte_mempool_put(sess_mp, sess_priv);
3352 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3353 struct rte_cryptodev_config *config __rte_unused)
3355 PMD_INIT_FUNC_TRACE();
3361 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3363 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3364 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3365 struct dpseci_attr attr;
3366 struct dpaa2_queue *dpaa2_q;
3367 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3368 dev->data->queue_pairs;
3369 struct dpseci_rx_queue_attr rx_attr;
3370 struct dpseci_tx_queue_attr tx_attr;
3373 PMD_INIT_FUNC_TRACE();
3375 memset(&attr, 0, sizeof(struct dpseci_attr));
3377 ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3379 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3381 goto get_attr_failure;
3383 ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3385 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3386 goto get_attr_failure;
3388 for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3389 dpaa2_q = &qp[i]->rx_vq;
3390 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3392 dpaa2_q->fqid = rx_attr.fqid;
3393 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3395 for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3396 dpaa2_q = &qp[i]->tx_vq;
3397 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3399 dpaa2_q->fqid = tx_attr.fqid;
3400 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3405 dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3410 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3412 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3413 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3416 PMD_INIT_FUNC_TRACE();
3418 ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3420 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3425 ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3427 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3433 dpaa2_sec_dev_close(struct rte_cryptodev *dev)
3435 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3436 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3439 PMD_INIT_FUNC_TRACE();
3441 /* Function is reverse of dpaa2_sec_dev_init.
3442 * It does the following:
3443 * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
3444 * 2. Close the DPSECI device
3445 * 3. Free the allocated resources.
3448 /*Close the device at underlying layer*/
3449 ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
3451 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
3455 /*Free the allocated memory for ethernet private data and dpseci*/
3463 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3464 struct rte_cryptodev_info *info)
3466 struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3468 PMD_INIT_FUNC_TRACE();
3470 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3471 info->feature_flags = dev->feature_flags;
3472 info->capabilities = dpaa2_sec_capabilities;
3473 /* No limit of number of sessions */
3474 info->sym.max_nb_sessions = 0;
3475 info->driver_id = cryptodev_driver_id;
3480 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3481 struct rte_cryptodev_stats *stats)
3483 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3484 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3485 struct dpseci_sec_counters counters = {0};
3486 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3487 dev->data->queue_pairs;
3490 PMD_INIT_FUNC_TRACE();
3491 if (stats == NULL) {
3492 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3495 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3496 if (qp[i] == NULL) {
3497 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3501 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3502 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3503 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3504 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3507 ret = dpseci_get_sec_counters(dpseci, CMD_PRI_LOW, priv->token,
3510 DPAA2_SEC_ERR("SEC counters failed");
3512 DPAA2_SEC_INFO("dpseci hardware stats:"
3513 "\n\tNum of Requests Dequeued = %" PRIu64
3514 "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3515 "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3516 "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3517 "\n\tNum of Outbound Bytes Protected = %" PRIu64
3518 "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3519 "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3520 counters.dequeued_requests,
3521 counters.ob_enc_requests,
3522 counters.ib_dec_requests,
3523 counters.ob_enc_bytes,
3524 counters.ob_prot_bytes,
3525 counters.ib_dec_bytes,
3526 counters.ib_valid_bytes);
3531 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3534 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3535 (dev->data->queue_pairs);
3537 PMD_INIT_FUNC_TRACE();
3539 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3540 if (qp[i] == NULL) {
3541 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3544 qp[i]->tx_vq.rx_pkts = 0;
3545 qp[i]->tx_vq.tx_pkts = 0;
3546 qp[i]->tx_vq.err_pkts = 0;
3547 qp[i]->rx_vq.rx_pkts = 0;
3548 qp[i]->rx_vq.tx_pkts = 0;
3549 qp[i]->rx_vq.err_pkts = 0;
3553 static void __rte_hot
3554 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3555 const struct qbman_fd *fd,
3556 const struct qbman_result *dq,
3557 struct dpaa2_queue *rxq,
3558 struct rte_event *ev)
3560 /* Prefetching mbuf */
3561 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3562 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3564 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3565 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3567 ev->flow_id = rxq->ev.flow_id;
3568 ev->sub_event_type = rxq->ev.sub_event_type;
3569 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3570 ev->op = RTE_EVENT_OP_NEW;
3571 ev->sched_type = rxq->ev.sched_type;
3572 ev->queue_id = rxq->ev.queue_id;
3573 ev->priority = rxq->ev.priority;
3574 ev->event_ptr = sec_fd_to_mbuf(fd);
3576 qbman_swp_dqrr_consume(swp, dq);
3579 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
3580 const struct qbman_fd *fd,
3581 const struct qbman_result *dq,
3582 struct dpaa2_queue *rxq,
3583 struct rte_event *ev)
3586 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
3587 /* Prefetching mbuf */
3588 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3589 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3591 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3592 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3594 ev->flow_id = rxq->ev.flow_id;
3595 ev->sub_event_type = rxq->ev.sub_event_type;
3596 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3597 ev->op = RTE_EVENT_OP_NEW;
3598 ev->sched_type = rxq->ev.sched_type;
3599 ev->queue_id = rxq->ev.queue_id;
3600 ev->priority = rxq->ev.priority;
3602 ev->event_ptr = sec_fd_to_mbuf(fd);
3603 dqrr_index = qbman_get_dqrr_idx(dq);
3604 crypto_op->sym->m_src->seqn = dqrr_index + 1;
3605 DPAA2_PER_LCORE_DQRR_SIZE++;
3606 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
3607 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
3611 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
3613 struct dpaa2_dpcon_dev *dpcon,
3614 const struct rte_event *event)
3616 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3617 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3618 struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
3619 struct dpseci_rx_queue_cfg cfg;
3623 if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
3624 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
3625 else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
3626 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
3630 priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
3631 (dpcon->num_priorities - 1);
3633 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3634 cfg.options = DPSECI_QUEUE_OPT_DEST;
3635 cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
3636 cfg.dest_cfg.dest_id = dpcon->dpcon_id;
3637 cfg.dest_cfg.priority = priority;
3639 cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
3640 cfg.user_ctx = (size_t)(qp);
3641 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
3642 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
3643 cfg.order_preservation_en = 1;
3645 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3648 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3652 memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
3658 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
3661 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3662 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3663 struct dpseci_rx_queue_cfg cfg;
3666 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3667 cfg.options = DPSECI_QUEUE_OPT_DEST;
3668 cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
3670 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3673 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3678 static struct rte_cryptodev_ops crypto_ops = {
3679 .dev_configure = dpaa2_sec_dev_configure,
3680 .dev_start = dpaa2_sec_dev_start,
3681 .dev_stop = dpaa2_sec_dev_stop,
3682 .dev_close = dpaa2_sec_dev_close,
3683 .dev_infos_get = dpaa2_sec_dev_infos_get,
3684 .stats_get = dpaa2_sec_stats_get,
3685 .stats_reset = dpaa2_sec_stats_reset,
3686 .queue_pair_setup = dpaa2_sec_queue_pair_setup,
3687 .queue_pair_release = dpaa2_sec_queue_pair_release,
3688 .sym_session_get_size = dpaa2_sec_sym_session_get_size,
3689 .sym_session_configure = dpaa2_sec_sym_session_configure,
3690 .sym_session_clear = dpaa2_sec_sym_session_clear,
3693 #ifdef RTE_LIBRTE_SECURITY
3694 static const struct rte_security_capability *
3695 dpaa2_sec_capabilities_get(void *device __rte_unused)
3697 return dpaa2_sec_security_cap;
3700 static const struct rte_security_ops dpaa2_sec_security_ops = {
3701 .session_create = dpaa2_sec_security_session_create,
3702 .session_update = NULL,
3703 .session_stats_get = NULL,
3704 .session_destroy = dpaa2_sec_security_session_destroy,
3705 .set_pkt_metadata = NULL,
3706 .capabilities_get = dpaa2_sec_capabilities_get
3711 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
3713 struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3715 rte_free(dev->security_ctx);
3717 rte_mempool_free(internals->fle_pool);
3719 DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
3720 dev->data->name, rte_socket_id());
3726 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
3728 struct dpaa2_sec_dev_private *internals;
3729 struct rte_device *dev = cryptodev->device;
3730 struct rte_dpaa2_device *dpaa2_dev;
3731 #ifdef RTE_LIBRTE_SECURITY
3732 struct rte_security_ctx *security_instance;
3734 struct fsl_mc_io *dpseci;
3736 struct dpseci_attr attr;
3740 PMD_INIT_FUNC_TRACE();
3741 dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
3742 hw_id = dpaa2_dev->object_id;
3744 cryptodev->driver_id = cryptodev_driver_id;
3745 cryptodev->dev_ops = &crypto_ops;
3747 cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
3748 cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
3749 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3750 RTE_CRYPTODEV_FF_HW_ACCELERATED |
3751 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3752 RTE_CRYPTODEV_FF_SECURITY |
3753 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3754 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
3755 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3756 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
3757 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
3759 internals = cryptodev->data->dev_private;
3762 * For secondary processes, we don't initialise any further as primary
3763 * has already done this work. Only check we don't need a different
3766 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3767 DPAA2_SEC_DEBUG("Device already init by primary process");
3770 #ifdef RTE_LIBRTE_SECURITY
3771 /* Initialize security_ctx only for primary process*/
3772 security_instance = rte_malloc("rte_security_instances_ops",
3773 sizeof(struct rte_security_ctx), 0);
3774 if (security_instance == NULL)
3776 security_instance->device = (void *)cryptodev;
3777 security_instance->ops = &dpaa2_sec_security_ops;
3778 security_instance->sess_cnt = 0;
3779 cryptodev->security_ctx = security_instance;
3781 /*Open the rte device via MC and save the handle for further use*/
3782 dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
3783 sizeof(struct fsl_mc_io), 0);
3786 "Error in allocating the memory for dpsec object");
3789 dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3791 retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
3793 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
3797 retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
3800 "Cannot get dpsec device attributed: Error = %x",
3804 snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
3807 internals->max_nb_queue_pairs = attr.num_tx_queues;
3808 cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
3809 internals->hw = dpseci;
3810 internals->token = token;
3812 snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d",
3813 getpid(), cryptodev->data->dev_id);
3814 internals->fle_pool = rte_mempool_create((const char *)str,
3817 FLE_POOL_CACHE_SIZE, 0,
3818 NULL, NULL, NULL, NULL,
3820 if (!internals->fle_pool) {
3821 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
3825 DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
3829 DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
3831 /* dpaa2_sec_uninit(crypto_dev_name); */
3836 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
3837 struct rte_dpaa2_device *dpaa2_dev)
3839 struct rte_cryptodev *cryptodev;
3840 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
3844 snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
3845 dpaa2_dev->object_id);
3847 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
3848 if (cryptodev == NULL)
3851 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3852 cryptodev->data->dev_private = rte_zmalloc_socket(
3853 "cryptodev private structure",
3854 sizeof(struct dpaa2_sec_dev_private),
3855 RTE_CACHE_LINE_SIZE,
3858 if (cryptodev->data->dev_private == NULL)
3859 rte_panic("Cannot allocate memzone for private "
3863 dpaa2_dev->cryptodev = cryptodev;
3864 cryptodev->device = &dpaa2_dev->device;
3866 /* init user callbacks */
3867 TAILQ_INIT(&(cryptodev->link_intr_cbs));
3869 if (dpaa2_svr_family == SVR_LX2160A)
3870 rta_set_sec_era(RTA_SEC_ERA_10);
3872 rta_set_sec_era(RTA_SEC_ERA_8);
3874 DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
3876 /* Invoke PMD device initialization function */
3877 retval = dpaa2_sec_dev_init(cryptodev);
3881 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
3882 rte_free(cryptodev->data->dev_private);
3884 cryptodev->attached = RTE_CRYPTODEV_DETACHED;
3890 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
3892 struct rte_cryptodev *cryptodev;
3895 cryptodev = dpaa2_dev->cryptodev;
3896 if (cryptodev == NULL)
3899 ret = dpaa2_sec_uninit(cryptodev);
3903 return rte_cryptodev_pmd_destroy(cryptodev);
3906 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
3907 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
3908 .drv_type = DPAA2_CRYPTO,
3910 .name = "DPAA2 SEC PMD"
3912 .probe = cryptodev_dpaa2_sec_probe,
3913 .remove = cryptodev_dpaa2_sec_remove,
3916 static struct cryptodev_driver dpaa2_sec_crypto_drv;
3918 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
3919 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
3920 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
3922 RTE_INIT(dpaa2_sec_init_log)
3924 /* Bus level logs */
3925 dpaa2_logtype_sec = rte_log_register("pmd.crypto.dpaa2");
3926 if (dpaa2_logtype_sec >= 0)
3927 rte_log_set_level(dpaa2_logtype_sec, RTE_LOG_NOTICE);