1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4 * Copyright 2016-2020 NXP
14 #include <rte_cryptodev.h>
15 #include <rte_malloc.h>
16 #include <rte_memcpy.h>
17 #include <rte_string_fns.h>
18 #include <rte_cycles.h>
19 #include <rte_kvargs.h>
21 #include <rte_cryptodev_pmd.h>
22 #include <rte_common.h>
23 #include <rte_fslmc.h>
24 #include <fslmc_vfio.h>
25 #include <dpaa2_hw_pvt.h>
26 #include <dpaa2_hw_dpio.h>
27 #include <dpaa2_hw_mempool.h>
28 #include <fsl_dpopr.h>
29 #include <fsl_dpseci.h>
30 #include <fsl_mc_sys.h>
32 #include "dpaa2_sec_priv.h"
33 #include "dpaa2_sec_event.h"
34 #include "dpaa2_sec_logs.h"
36 /* RTA header files */
37 #include <desc/ipsec.h>
38 #include <desc/pdcp.h>
39 #include <desc/algo.h>
41 /* Minimum job descriptor consists of a oneword job descriptor HEADER and
42 * a pointer to the shared descriptor
44 #define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ)
45 #define FSL_VENDOR_ID 0x1957
46 #define FSL_DEVICE_ID 0x410
47 #define FSL_SUBSYSTEM_SEC 1
48 #define FSL_MC_DPSECI_DEVID 3
51 /* FLE_POOL_NUM_BUFS is set as per the ipsec-secgw application */
52 #define FLE_POOL_NUM_BUFS 32000
53 #define FLE_POOL_BUF_SIZE 256
54 #define FLE_POOL_CACHE_SIZE 512
55 #define FLE_SG_MEM_SIZE(num) (FLE_POOL_BUF_SIZE + ((num) * 32))
56 #define SEC_FLC_DHR_OUTBOUND -114
57 #define SEC_FLC_DHR_INBOUND 0
59 static uint8_t cryptodev_driver_id;
61 #ifdef RTE_LIBRTE_SECURITY
63 build_proto_compound_sg_fd(dpaa2_sec_session *sess,
64 struct rte_crypto_op *op,
65 struct qbman_fd *fd, uint16_t bpid)
67 struct rte_crypto_sym_op *sym_op = op->sym;
68 struct ctxt_priv *priv = sess->ctxt;
69 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
70 struct sec_flow_context *flc;
71 struct rte_mbuf *mbuf;
72 uint32_t in_len = 0, out_len = 0;
79 /* first FLE entry used to store mbuf and session ctxt */
80 fle = (struct qbman_fle *)rte_malloc(NULL,
81 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
84 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE");
87 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
88 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
89 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
91 /* Save the shared descriptor */
92 flc = &priv->flc_desc[0].flc;
98 if (likely(bpid < MAX_BPID)) {
99 DPAA2_SET_FD_BPID(fd, bpid);
100 DPAA2_SET_FLE_BPID(op_fle, bpid);
101 DPAA2_SET_FLE_BPID(ip_fle, bpid);
103 DPAA2_SET_FD_IVP(fd);
104 DPAA2_SET_FLE_IVP(op_fle);
105 DPAA2_SET_FLE_IVP(ip_fle);
108 /* Configure FD as a FRAME LIST */
109 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
110 DPAA2_SET_FD_COMPOUND_FMT(fd);
111 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
113 /* Configure Output FLE with Scatter/Gather Entry */
114 DPAA2_SET_FLE_SG_EXT(op_fle);
115 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
117 /* Configure Output SGE for Encap/Decap */
118 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
119 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
122 sge->length = mbuf->data_len;
123 out_len += sge->length;
126 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
127 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
129 /* using buf_len for last buf - so that extra data can be added */
130 sge->length = mbuf->buf_len - mbuf->data_off;
131 out_len += sge->length;
133 DPAA2_SET_FLE_FIN(sge);
134 op_fle->length = out_len;
137 mbuf = sym_op->m_src;
139 /* Configure Input FLE with Scatter/Gather Entry */
140 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
141 DPAA2_SET_FLE_SG_EXT(ip_fle);
142 DPAA2_SET_FLE_FIN(ip_fle);
144 /* Configure input SGE for Encap/Decap */
145 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
146 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
147 sge->length = mbuf->data_len;
148 in_len += sge->length;
154 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
155 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
156 sge->length = mbuf->data_len;
157 in_len += sge->length;
160 ip_fle->length = in_len;
161 DPAA2_SET_FLE_FIN(sge);
163 /* In case of PDCP, per packet HFN is stored in
164 * mbuf priv after sym_op.
166 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
167 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
168 sess->pdcp.hfn_ovd_offset);
169 /*enable HFN override override */
170 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
171 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
172 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
174 DPAA2_SET_FD_LEN(fd, ip_fle->length);
180 build_proto_compound_fd(dpaa2_sec_session *sess,
181 struct rte_crypto_op *op,
182 struct qbman_fd *fd, uint16_t bpid)
184 struct rte_crypto_sym_op *sym_op = op->sym;
185 struct ctxt_priv *priv = sess->ctxt;
186 struct qbman_fle *fle, *ip_fle, *op_fle;
187 struct sec_flow_context *flc;
188 struct rte_mbuf *src_mbuf = sym_op->m_src;
189 struct rte_mbuf *dst_mbuf = sym_op->m_dst;
195 /* Save the shared descriptor */
196 flc = &priv->flc_desc[0].flc;
198 /* we are using the first FLE entry to store Mbuf */
199 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
201 DPAA2_SEC_DP_ERR("Memory alloc failed");
204 memset(fle, 0, FLE_POOL_BUF_SIZE);
205 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
206 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
211 if (likely(bpid < MAX_BPID)) {
212 DPAA2_SET_FD_BPID(fd, bpid);
213 DPAA2_SET_FLE_BPID(op_fle, bpid);
214 DPAA2_SET_FLE_BPID(ip_fle, bpid);
216 DPAA2_SET_FD_IVP(fd);
217 DPAA2_SET_FLE_IVP(op_fle);
218 DPAA2_SET_FLE_IVP(ip_fle);
221 /* Configure FD as a FRAME LIST */
222 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
223 DPAA2_SET_FD_COMPOUND_FMT(fd);
224 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
226 /* Configure Output FLE with dst mbuf data */
227 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf));
228 DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off);
229 DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len);
231 /* Configure Input FLE with src mbuf data */
232 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf));
233 DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off);
234 DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len);
236 DPAA2_SET_FD_LEN(fd, ip_fle->length);
237 DPAA2_SET_FLE_FIN(ip_fle);
239 /* In case of PDCP, per packet HFN is stored in
240 * mbuf priv after sym_op.
242 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) {
243 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op +
244 sess->pdcp.hfn_ovd_offset);
245 /*enable HFN override override */
246 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd);
247 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd);
248 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd);
256 build_proto_fd(dpaa2_sec_session *sess,
257 struct rte_crypto_op *op,
258 struct qbman_fd *fd, uint16_t bpid)
260 struct rte_crypto_sym_op *sym_op = op->sym;
262 return build_proto_compound_fd(sess, op, fd, bpid);
264 struct ctxt_priv *priv = sess->ctxt;
265 struct sec_flow_context *flc;
266 struct rte_mbuf *mbuf = sym_op->m_src;
268 if (likely(bpid < MAX_BPID))
269 DPAA2_SET_FD_BPID(fd, bpid);
271 DPAA2_SET_FD_IVP(fd);
273 /* Save the shared descriptor */
274 flc = &priv->flc_desc[0].flc;
276 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
277 DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off);
278 DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len);
279 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
281 /* save physical address of mbuf */
282 op->sym->aead.digest.phys_addr = mbuf->buf_iova;
283 mbuf->buf_iova = (size_t)op;
290 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess,
291 struct rte_crypto_op *op,
292 struct qbman_fd *fd, __rte_unused uint16_t bpid)
294 struct rte_crypto_sym_op *sym_op = op->sym;
295 struct ctxt_priv *priv = sess->ctxt;
296 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
297 struct sec_flow_context *flc;
298 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
299 int icv_len = sess->digest_length;
301 struct rte_mbuf *mbuf;
302 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
306 mbuf = sym_op->m_dst;
308 mbuf = sym_op->m_src;
310 /* first FLE entry used to store mbuf and session ctxt */
311 fle = (struct qbman_fle *)rte_malloc(NULL,
312 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
313 RTE_CACHE_LINE_SIZE);
314 if (unlikely(!fle)) {
315 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE");
318 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
319 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
320 DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv);
326 /* Save the shared descriptor */
327 flc = &priv->flc_desc[0].flc;
329 /* Configure FD as a FRAME LIST */
330 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
331 DPAA2_SET_FD_COMPOUND_FMT(fd);
332 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
334 DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n"
335 "iv-len=%d data_off: 0x%x\n",
336 sym_op->aead.data.offset,
337 sym_op->aead.data.length,
340 sym_op->m_src->data_off);
342 /* Configure Output FLE with Scatter/Gather Entry */
343 DPAA2_SET_FLE_SG_EXT(op_fle);
344 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
347 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
349 op_fle->length = (sess->dir == DIR_ENC) ?
350 (sym_op->aead.data.length + icv_len) :
351 sym_op->aead.data.length;
353 /* Configure Output SGE for Encap/Decap */
354 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
355 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset);
356 sge->length = mbuf->data_len - sym_op->aead.data.offset;
362 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
363 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
364 sge->length = mbuf->data_len;
367 sge->length -= icv_len;
369 if (sess->dir == DIR_ENC) {
371 DPAA2_SET_FLE_ADDR(sge,
372 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
373 sge->length = icv_len;
375 DPAA2_SET_FLE_FIN(sge);
378 mbuf = sym_op->m_src;
380 /* Configure Input FLE with Scatter/Gather Entry */
381 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
382 DPAA2_SET_FLE_SG_EXT(ip_fle);
383 DPAA2_SET_FLE_FIN(ip_fle);
384 ip_fle->length = (sess->dir == DIR_ENC) ?
385 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
386 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
389 /* Configure Input SGE for Encap/Decap */
390 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
391 sge->length = sess->iv.length;
395 DPAA2_SET_FLE_ADDR(sge,
396 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
397 sge->length = auth_only_len;
401 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
402 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
404 sge->length = mbuf->data_len - sym_op->aead.data.offset;
410 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
411 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
412 sge->length = mbuf->data_len;
416 if (sess->dir == DIR_DEC) {
418 old_icv = (uint8_t *)(sge + 1);
419 memcpy(old_icv, sym_op->aead.digest.data, icv_len);
420 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
421 sge->length = icv_len;
424 DPAA2_SET_FLE_FIN(sge);
426 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
427 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
429 DPAA2_SET_FD_LEN(fd, ip_fle->length);
435 build_authenc_gcm_fd(dpaa2_sec_session *sess,
436 struct rte_crypto_op *op,
437 struct qbman_fd *fd, uint16_t bpid)
439 struct rte_crypto_sym_op *sym_op = op->sym;
440 struct ctxt_priv *priv = sess->ctxt;
441 struct qbman_fle *fle, *sge;
442 struct sec_flow_context *flc;
443 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len;
444 int icv_len = sess->digest_length, retval;
446 struct rte_mbuf *dst;
447 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
455 /* TODO we are using the first FLE entry to store Mbuf and session ctxt.
456 * Currently we donot know which FLE has the mbuf stored.
457 * So while retreiving we can go back 1 FLE from the FD -ADDR
458 * to get the MBUF Addr from the previous FLE.
459 * We can have a better approach to use the inline Mbuf
461 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
463 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE");
466 memset(fle, 0, FLE_POOL_BUF_SIZE);
467 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
468 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
471 if (likely(bpid < MAX_BPID)) {
472 DPAA2_SET_FD_BPID(fd, bpid);
473 DPAA2_SET_FLE_BPID(fle, bpid);
474 DPAA2_SET_FLE_BPID(fle + 1, bpid);
475 DPAA2_SET_FLE_BPID(sge, bpid);
476 DPAA2_SET_FLE_BPID(sge + 1, bpid);
477 DPAA2_SET_FLE_BPID(sge + 2, bpid);
478 DPAA2_SET_FLE_BPID(sge + 3, bpid);
480 DPAA2_SET_FD_IVP(fd);
481 DPAA2_SET_FLE_IVP(fle);
482 DPAA2_SET_FLE_IVP((fle + 1));
483 DPAA2_SET_FLE_IVP(sge);
484 DPAA2_SET_FLE_IVP((sge + 1));
485 DPAA2_SET_FLE_IVP((sge + 2));
486 DPAA2_SET_FLE_IVP((sge + 3));
489 /* Save the shared descriptor */
490 flc = &priv->flc_desc[0].flc;
491 /* Configure FD as a FRAME LIST */
492 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
493 DPAA2_SET_FD_COMPOUND_FMT(fd);
494 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
496 DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n"
497 "iv-len=%d data_off: 0x%x\n",
498 sym_op->aead.data.offset,
499 sym_op->aead.data.length,
502 sym_op->m_src->data_off);
504 /* Configure Output FLE with Scatter/Gather Entry */
505 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
507 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
508 fle->length = (sess->dir == DIR_ENC) ?
509 (sym_op->aead.data.length + icv_len) :
510 sym_op->aead.data.length;
512 DPAA2_SET_FLE_SG_EXT(fle);
514 /* Configure Output SGE for Encap/Decap */
515 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
516 DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset);
517 sge->length = sym_op->aead.data.length;
519 if (sess->dir == DIR_ENC) {
521 DPAA2_SET_FLE_ADDR(sge,
522 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data));
523 sge->length = sess->digest_length;
525 DPAA2_SET_FLE_FIN(sge);
530 /* Configure Input FLE with Scatter/Gather Entry */
531 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
532 DPAA2_SET_FLE_SG_EXT(fle);
533 DPAA2_SET_FLE_FIN(fle);
534 fle->length = (sess->dir == DIR_ENC) ?
535 (sym_op->aead.data.length + sess->iv.length + auth_only_len) :
536 (sym_op->aead.data.length + sess->iv.length + auth_only_len +
537 sess->digest_length);
539 /* Configure Input SGE for Encap/Decap */
540 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr));
541 sge->length = sess->iv.length;
544 DPAA2_SET_FLE_ADDR(sge,
545 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data));
546 sge->length = auth_only_len;
547 DPAA2_SET_FLE_BPID(sge, bpid);
551 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
552 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset +
553 sym_op->m_src->data_off);
554 sge->length = sym_op->aead.data.length;
555 if (sess->dir == DIR_DEC) {
557 old_icv = (uint8_t *)(sge + 1);
558 memcpy(old_icv, sym_op->aead.digest.data,
559 sess->digest_length);
560 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
561 sge->length = sess->digest_length;
563 DPAA2_SET_FLE_FIN(sge);
566 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
567 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
570 DPAA2_SET_FD_LEN(fd, fle->length);
575 build_authenc_sg_fd(dpaa2_sec_session *sess,
576 struct rte_crypto_op *op,
577 struct qbman_fd *fd, __rte_unused uint16_t bpid)
579 struct rte_crypto_sym_op *sym_op = op->sym;
580 struct ctxt_priv *priv = sess->ctxt;
581 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
582 struct sec_flow_context *flc;
583 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
584 sym_op->auth.data.offset;
585 uint16_t auth_tail_len = sym_op->auth.data.length -
586 sym_op->cipher.data.length - auth_hdr_len;
587 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
588 int icv_len = sess->digest_length;
590 struct rte_mbuf *mbuf;
591 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
595 mbuf = sym_op->m_dst;
597 mbuf = sym_op->m_src;
599 /* first FLE entry used to store mbuf and session ctxt */
600 fle = (struct qbman_fle *)rte_malloc(NULL,
601 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
602 RTE_CACHE_LINE_SIZE);
603 if (unlikely(!fle)) {
604 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE");
607 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
608 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
609 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
615 /* Save the shared descriptor */
616 flc = &priv->flc_desc[0].flc;
618 /* Configure FD as a FRAME LIST */
619 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
620 DPAA2_SET_FD_COMPOUND_FMT(fd);
621 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
624 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n"
625 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
626 sym_op->auth.data.offset,
627 sym_op->auth.data.length,
629 sym_op->cipher.data.offset,
630 sym_op->cipher.data.length,
632 sym_op->m_src->data_off);
634 /* Configure Output FLE with Scatter/Gather Entry */
635 DPAA2_SET_FLE_SG_EXT(op_fle);
636 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
639 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len);
641 op_fle->length = (sess->dir == DIR_ENC) ?
642 (sym_op->cipher.data.length + icv_len) :
643 sym_op->cipher.data.length;
645 /* Configure Output SGE for Encap/Decap */
646 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
647 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset);
648 sge->length = mbuf->data_len - sym_op->auth.data.offset;
654 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
655 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
656 sge->length = mbuf->data_len;
659 sge->length -= icv_len;
661 if (sess->dir == DIR_ENC) {
663 DPAA2_SET_FLE_ADDR(sge,
664 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
665 sge->length = icv_len;
667 DPAA2_SET_FLE_FIN(sge);
670 mbuf = sym_op->m_src;
672 /* Configure Input FLE with Scatter/Gather Entry */
673 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
674 DPAA2_SET_FLE_SG_EXT(ip_fle);
675 DPAA2_SET_FLE_FIN(ip_fle);
676 ip_fle->length = (sess->dir == DIR_ENC) ?
677 (sym_op->auth.data.length + sess->iv.length) :
678 (sym_op->auth.data.length + sess->iv.length +
681 /* Configure Input SGE for Encap/Decap */
682 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
683 sge->length = sess->iv.length;
686 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
687 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
689 sge->length = mbuf->data_len - sym_op->auth.data.offset;
695 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
696 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
697 sge->length = mbuf->data_len;
700 sge->length -= icv_len;
702 if (sess->dir == DIR_DEC) {
704 old_icv = (uint8_t *)(sge + 1);
705 memcpy(old_icv, sym_op->auth.digest.data,
707 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
708 sge->length = icv_len;
711 DPAA2_SET_FLE_FIN(sge);
713 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len);
714 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
716 DPAA2_SET_FD_LEN(fd, ip_fle->length);
722 build_authenc_fd(dpaa2_sec_session *sess,
723 struct rte_crypto_op *op,
724 struct qbman_fd *fd, uint16_t bpid)
726 struct rte_crypto_sym_op *sym_op = op->sym;
727 struct ctxt_priv *priv = sess->ctxt;
728 struct qbman_fle *fle, *sge;
729 struct sec_flow_context *flc;
730 uint16_t auth_hdr_len = sym_op->cipher.data.offset -
731 sym_op->auth.data.offset;
732 uint16_t auth_tail_len = sym_op->auth.data.length -
733 sym_op->cipher.data.length - auth_hdr_len;
734 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
736 int icv_len = sess->digest_length, retval;
738 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
740 struct rte_mbuf *dst;
747 /* we are using the first FLE entry to store Mbuf.
748 * Currently we donot know which FLE has the mbuf stored.
749 * So while retreiving we can go back 1 FLE from the FD -ADDR
750 * to get the MBUF Addr from the previous FLE.
751 * We can have a better approach to use the inline Mbuf
753 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
755 DPAA2_SEC_ERR("Memory alloc failed for SGE");
758 memset(fle, 0, FLE_POOL_BUF_SIZE);
759 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
760 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
763 if (likely(bpid < MAX_BPID)) {
764 DPAA2_SET_FD_BPID(fd, bpid);
765 DPAA2_SET_FLE_BPID(fle, bpid);
766 DPAA2_SET_FLE_BPID(fle + 1, bpid);
767 DPAA2_SET_FLE_BPID(sge, bpid);
768 DPAA2_SET_FLE_BPID(sge + 1, bpid);
769 DPAA2_SET_FLE_BPID(sge + 2, bpid);
770 DPAA2_SET_FLE_BPID(sge + 3, bpid);
772 DPAA2_SET_FD_IVP(fd);
773 DPAA2_SET_FLE_IVP(fle);
774 DPAA2_SET_FLE_IVP((fle + 1));
775 DPAA2_SET_FLE_IVP(sge);
776 DPAA2_SET_FLE_IVP((sge + 1));
777 DPAA2_SET_FLE_IVP((sge + 2));
778 DPAA2_SET_FLE_IVP((sge + 3));
781 /* Save the shared descriptor */
782 flc = &priv->flc_desc[0].flc;
783 /* Configure FD as a FRAME LIST */
784 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
785 DPAA2_SET_FD_COMPOUND_FMT(fd);
786 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
789 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n"
790 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n",
791 sym_op->auth.data.offset,
792 sym_op->auth.data.length,
794 sym_op->cipher.data.offset,
795 sym_op->cipher.data.length,
797 sym_op->m_src->data_off);
799 /* Configure Output FLE with Scatter/Gather Entry */
800 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
802 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
803 fle->length = (sess->dir == DIR_ENC) ?
804 (sym_op->cipher.data.length + icv_len) :
805 sym_op->cipher.data.length;
807 DPAA2_SET_FLE_SG_EXT(fle);
809 /* Configure Output SGE for Encap/Decap */
810 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst));
811 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset +
813 sge->length = sym_op->cipher.data.length;
815 if (sess->dir == DIR_ENC) {
817 DPAA2_SET_FLE_ADDR(sge,
818 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
819 sge->length = sess->digest_length;
820 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
823 DPAA2_SET_FLE_FIN(sge);
828 /* Configure Input FLE with Scatter/Gather Entry */
829 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
830 DPAA2_SET_FLE_SG_EXT(fle);
831 DPAA2_SET_FLE_FIN(fle);
832 fle->length = (sess->dir == DIR_ENC) ?
833 (sym_op->auth.data.length + sess->iv.length) :
834 (sym_op->auth.data.length + sess->iv.length +
835 sess->digest_length);
837 /* Configure Input SGE for Encap/Decap */
838 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
839 sge->length = sess->iv.length;
842 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
843 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset +
844 sym_op->m_src->data_off);
845 sge->length = sym_op->auth.data.length;
846 if (sess->dir == DIR_DEC) {
848 old_icv = (uint8_t *)(sge + 1);
849 memcpy(old_icv, sym_op->auth.digest.data,
850 sess->digest_length);
851 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv));
852 sge->length = sess->digest_length;
853 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length +
854 sess->digest_length +
857 DPAA2_SET_FLE_FIN(sge);
859 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len);
860 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len);
865 static inline int build_auth_sg_fd(
866 dpaa2_sec_session *sess,
867 struct rte_crypto_op *op,
869 __rte_unused uint16_t bpid)
871 struct rte_crypto_sym_op *sym_op = op->sym;
872 struct qbman_fle *fle, *sge, *ip_fle, *op_fle;
873 struct sec_flow_context *flc;
874 struct ctxt_priv *priv = sess->ctxt;
875 int data_len, data_offset;
877 struct rte_mbuf *mbuf;
879 data_len = sym_op->auth.data.length;
880 data_offset = sym_op->auth.data.offset;
882 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
883 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
884 if ((data_len & 7) || (data_offset & 7)) {
885 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
889 data_len = data_len >> 3;
890 data_offset = data_offset >> 3;
893 mbuf = sym_op->m_src;
894 fle = (struct qbman_fle *)rte_malloc(NULL,
895 FLE_SG_MEM_SIZE(mbuf->nb_segs),
896 RTE_CACHE_LINE_SIZE);
897 if (unlikely(!fle)) {
898 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE");
901 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs));
902 /* first FLE entry used to store mbuf and session ctxt */
903 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
904 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
909 flc = &priv->flc_desc[DESC_INITFINAL].flc;
911 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
912 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
913 DPAA2_SET_FD_COMPOUND_FMT(fd);
916 DPAA2_SET_FLE_ADDR(op_fle,
917 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
918 op_fle->length = sess->digest_length;
921 DPAA2_SET_FLE_SG_EXT(ip_fle);
922 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
923 ip_fle->length = data_len;
925 if (sess->iv.length) {
928 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
931 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
932 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
934 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
935 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
938 sge->length = sess->iv.length;
940 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
941 ip_fle->length += sge->length;
945 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
946 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
948 if (data_len <= (mbuf->data_len - data_offset)) {
949 sge->length = data_len;
952 sge->length = mbuf->data_len - data_offset;
954 /* remaining i/p segs */
955 while ((data_len = data_len - sge->length) &&
956 (mbuf = mbuf->next)) {
958 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
959 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
960 if (data_len > mbuf->data_len)
961 sge->length = mbuf->data_len;
963 sge->length = data_len;
967 if (sess->dir == DIR_DEC) {
968 /* Digest verification case */
970 old_digest = (uint8_t *)(sge + 1);
971 rte_memcpy(old_digest, sym_op->auth.digest.data,
972 sess->digest_length);
973 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
974 sge->length = sess->digest_length;
975 ip_fle->length += sess->digest_length;
977 DPAA2_SET_FLE_FIN(sge);
978 DPAA2_SET_FLE_FIN(ip_fle);
979 DPAA2_SET_FD_LEN(fd, ip_fle->length);
985 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
986 struct qbman_fd *fd, uint16_t bpid)
988 struct rte_crypto_sym_op *sym_op = op->sym;
989 struct qbman_fle *fle, *sge;
990 struct sec_flow_context *flc;
991 struct ctxt_priv *priv = sess->ctxt;
992 int data_len, data_offset;
996 data_len = sym_op->auth.data.length;
997 data_offset = sym_op->auth.data.offset;
999 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
1000 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1001 if ((data_len & 7) || (data_offset & 7)) {
1002 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes");
1006 data_len = data_len >> 3;
1007 data_offset = data_offset >> 3;
1010 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1012 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE");
1015 memset(fle, 0, FLE_POOL_BUF_SIZE);
1016 /* TODO we are using the first FLE entry to store Mbuf.
1017 * Currently we donot know which FLE has the mbuf stored.
1018 * So while retreiving we can go back 1 FLE from the FD -ADDR
1019 * to get the MBUF Addr from the previous FLE.
1020 * We can have a better approach to use the inline Mbuf
1022 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1023 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1027 if (likely(bpid < MAX_BPID)) {
1028 DPAA2_SET_FD_BPID(fd, bpid);
1029 DPAA2_SET_FLE_BPID(fle, bpid);
1030 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1031 DPAA2_SET_FLE_BPID(sge, bpid);
1032 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1034 DPAA2_SET_FD_IVP(fd);
1035 DPAA2_SET_FLE_IVP(fle);
1036 DPAA2_SET_FLE_IVP((fle + 1));
1037 DPAA2_SET_FLE_IVP(sge);
1038 DPAA2_SET_FLE_IVP((sge + 1));
1041 flc = &priv->flc_desc[DESC_INITFINAL].flc;
1042 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1043 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1044 DPAA2_SET_FD_COMPOUND_FMT(fd);
1046 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data));
1047 fle->length = sess->digest_length;
1050 /* Setting input FLE */
1051 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1052 DPAA2_SET_FLE_SG_EXT(fle);
1053 fle->length = data_len;
1055 if (sess->iv.length) {
1058 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1061 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
1062 iv_ptr = conv_to_snow_f9_iv(iv_ptr);
1064 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
1065 iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
1068 sge->length = sess->iv.length;
1071 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1072 fle->length = fle->length + sge->length;
1076 /* Setting data to authenticate */
1077 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1078 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1079 sge->length = data_len;
1081 if (sess->dir == DIR_DEC) {
1083 old_digest = (uint8_t *)(sge + 1);
1084 rte_memcpy(old_digest, sym_op->auth.digest.data,
1085 sess->digest_length);
1086 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest));
1087 sge->length = sess->digest_length;
1088 fle->length = fle->length + sess->digest_length;
1091 DPAA2_SET_FLE_FIN(sge);
1092 DPAA2_SET_FLE_FIN(fle);
1093 DPAA2_SET_FD_LEN(fd, fle->length);
1099 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1100 struct qbman_fd *fd, __rte_unused uint16_t bpid)
1102 struct rte_crypto_sym_op *sym_op = op->sym;
1103 struct qbman_fle *ip_fle, *op_fle, *sge, *fle;
1104 int data_len, data_offset;
1105 struct sec_flow_context *flc;
1106 struct ctxt_priv *priv = sess->ctxt;
1107 struct rte_mbuf *mbuf;
1108 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1111 data_len = sym_op->cipher.data.length;
1112 data_offset = sym_op->cipher.data.offset;
1114 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1115 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1116 if ((data_len & 7) || (data_offset & 7)) {
1117 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1121 data_len = data_len >> 3;
1122 data_offset = data_offset >> 3;
1126 mbuf = sym_op->m_dst;
1128 mbuf = sym_op->m_src;
1130 /* first FLE entry used to store mbuf and session ctxt */
1131 fle = (struct qbman_fle *)rte_malloc(NULL,
1132 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs),
1133 RTE_CACHE_LINE_SIZE);
1135 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE");
1138 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs));
1139 /* first FLE entry used to store mbuf and session ctxt */
1140 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1141 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1147 flc = &priv->flc_desc[0].flc;
1150 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d"
1151 " data_off: 0x%x\n",
1155 sym_op->m_src->data_off);
1158 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge));
1159 op_fle->length = data_len;
1160 DPAA2_SET_FLE_SG_EXT(op_fle);
1163 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1164 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1165 sge->length = mbuf->data_len - data_offset;
1171 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1172 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1173 sge->length = mbuf->data_len;
1176 DPAA2_SET_FLE_FIN(sge);
1179 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n",
1180 flc, fle, fle->addr_hi, fle->addr_lo,
1184 mbuf = sym_op->m_src;
1186 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge));
1187 ip_fle->length = sess->iv.length + data_len;
1188 DPAA2_SET_FLE_SG_EXT(ip_fle);
1191 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1192 DPAA2_SET_FLE_OFFSET(sge, 0);
1193 sge->length = sess->iv.length;
1198 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1199 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off);
1200 sge->length = mbuf->data_len - data_offset;
1206 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
1207 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off);
1208 sge->length = mbuf->data_len;
1211 DPAA2_SET_FLE_FIN(sge);
1212 DPAA2_SET_FLE_FIN(ip_fle);
1215 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle));
1216 DPAA2_SET_FD_LEN(fd, ip_fle->length);
1217 DPAA2_SET_FD_COMPOUND_FMT(fd);
1218 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1221 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1222 " off =%d, len =%d\n",
1223 DPAA2_GET_FD_ADDR(fd),
1224 DPAA2_GET_FD_BPID(fd),
1225 rte_dpaa2_bpid_info[bpid].meta_data_size,
1226 DPAA2_GET_FD_OFFSET(fd),
1227 DPAA2_GET_FD_LEN(fd));
1232 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op,
1233 struct qbman_fd *fd, uint16_t bpid)
1235 struct rte_crypto_sym_op *sym_op = op->sym;
1236 struct qbman_fle *fle, *sge;
1237 int retval, data_len, data_offset;
1238 struct sec_flow_context *flc;
1239 struct ctxt_priv *priv = sess->ctxt;
1240 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
1242 struct rte_mbuf *dst;
1244 data_len = sym_op->cipher.data.length;
1245 data_offset = sym_op->cipher.data.offset;
1247 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
1248 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
1249 if ((data_len & 7) || (data_offset & 7)) {
1250 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes");
1254 data_len = data_len >> 3;
1255 data_offset = data_offset >> 3;
1259 dst = sym_op->m_dst;
1261 dst = sym_op->m_src;
1263 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle));
1265 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE");
1268 memset(fle, 0, FLE_POOL_BUF_SIZE);
1269 /* TODO we are using the first FLE entry to store Mbuf.
1270 * Currently we donot know which FLE has the mbuf stored.
1271 * So while retreiving we can go back 1 FLE from the FD -ADDR
1272 * to get the MBUF Addr from the previous FLE.
1273 * We can have a better approach to use the inline Mbuf
1275 DPAA2_SET_FLE_ADDR(fle, (size_t)op);
1276 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv);
1280 if (likely(bpid < MAX_BPID)) {
1281 DPAA2_SET_FD_BPID(fd, bpid);
1282 DPAA2_SET_FLE_BPID(fle, bpid);
1283 DPAA2_SET_FLE_BPID(fle + 1, bpid);
1284 DPAA2_SET_FLE_BPID(sge, bpid);
1285 DPAA2_SET_FLE_BPID(sge + 1, bpid);
1287 DPAA2_SET_FD_IVP(fd);
1288 DPAA2_SET_FLE_IVP(fle);
1289 DPAA2_SET_FLE_IVP((fle + 1));
1290 DPAA2_SET_FLE_IVP(sge);
1291 DPAA2_SET_FLE_IVP((sge + 1));
1294 flc = &priv->flc_desc[0].flc;
1295 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
1296 DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length);
1297 DPAA2_SET_FD_COMPOUND_FMT(fd);
1298 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc));
1301 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d,"
1302 " data_off: 0x%x\n",
1306 sym_op->m_src->data_off);
1308 DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst));
1309 DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off);
1311 fle->length = data_len + sess->iv.length;
1314 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n",
1315 flc, fle, fle->addr_hi, fle->addr_lo,
1320 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge));
1321 fle->length = data_len + sess->iv.length;
1323 DPAA2_SET_FLE_SG_EXT(fle);
1325 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr));
1326 sge->length = sess->iv.length;
1329 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src));
1330 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off);
1332 sge->length = data_len;
1333 DPAA2_SET_FLE_FIN(sge);
1334 DPAA2_SET_FLE_FIN(fle);
1337 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d"
1338 " off =%d, len =%d\n",
1339 DPAA2_GET_FD_ADDR(fd),
1340 DPAA2_GET_FD_BPID(fd),
1341 rte_dpaa2_bpid_info[bpid].meta_data_size,
1342 DPAA2_GET_FD_OFFSET(fd),
1343 DPAA2_GET_FD_LEN(fd));
1349 build_sec_fd(struct rte_crypto_op *op,
1350 struct qbman_fd *fd, uint16_t bpid)
1353 dpaa2_sec_session *sess;
1355 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1356 sess = (dpaa2_sec_session *)get_sym_session_private_data(
1357 op->sym->session, cryptodev_driver_id);
1358 #ifdef RTE_LIBRTE_SECURITY
1359 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1360 sess = (dpaa2_sec_session *)get_sec_session_private_data(
1361 op->sym->sec_session);
1369 /* Any of the buffer is segmented*/
1370 if (!rte_pktmbuf_is_contiguous(op->sym->m_src) ||
1371 ((op->sym->m_dst != NULL) &&
1372 !rte_pktmbuf_is_contiguous(op->sym->m_dst))) {
1373 switch (sess->ctxt_type) {
1374 case DPAA2_SEC_CIPHER:
1375 ret = build_cipher_sg_fd(sess, op, fd, bpid);
1377 case DPAA2_SEC_AUTH:
1378 ret = build_auth_sg_fd(sess, op, fd, bpid);
1380 case DPAA2_SEC_AEAD:
1381 ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid);
1383 case DPAA2_SEC_CIPHER_HASH:
1384 ret = build_authenc_sg_fd(sess, op, fd, bpid);
1386 #ifdef RTE_LIBRTE_SECURITY
1387 case DPAA2_SEC_IPSEC:
1388 case DPAA2_SEC_PDCP:
1389 ret = build_proto_compound_sg_fd(sess, op, fd, bpid);
1392 case DPAA2_SEC_HASH_CIPHER:
1394 DPAA2_SEC_ERR("error: Unsupported session");
1397 switch (sess->ctxt_type) {
1398 case DPAA2_SEC_CIPHER:
1399 ret = build_cipher_fd(sess, op, fd, bpid);
1401 case DPAA2_SEC_AUTH:
1402 ret = build_auth_fd(sess, op, fd, bpid);
1404 case DPAA2_SEC_AEAD:
1405 ret = build_authenc_gcm_fd(sess, op, fd, bpid);
1407 case DPAA2_SEC_CIPHER_HASH:
1408 ret = build_authenc_fd(sess, op, fd, bpid);
1410 #ifdef RTE_LIBRTE_SECURITY
1411 case DPAA2_SEC_IPSEC:
1412 ret = build_proto_fd(sess, op, fd, bpid);
1414 case DPAA2_SEC_PDCP:
1415 ret = build_proto_compound_fd(sess, op, fd, bpid);
1418 case DPAA2_SEC_HASH_CIPHER:
1420 DPAA2_SEC_ERR("error: Unsupported session");
1428 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops,
1431 /* Function to transmit the frames to given device and VQ*/
1434 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1435 uint32_t frames_to_send, retry_count;
1436 struct qbman_eq_desc eqdesc;
1437 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1438 struct qbman_swp *swp;
1439 uint16_t num_tx = 0;
1440 uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1441 /*todo - need to support multiple buffer pools */
1443 struct rte_mempool *mb_pool;
1445 if (unlikely(nb_ops == 0))
1448 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1449 DPAA2_SEC_ERR("sessionless crypto op not supported");
1452 /*Prepare enqueue descriptor*/
1453 qbman_eq_desc_clear(&eqdesc);
1454 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1455 qbman_eq_desc_set_response(&eqdesc, 0, 0);
1456 qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid);
1458 if (!DPAA2_PER_LCORE_DPIO) {
1459 ret = dpaa2_affine_qbman_swp();
1462 "Failed to allocate IO portal, tid: %d\n",
1467 swp = DPAA2_PER_LCORE_PORTAL;
1470 frames_to_send = (nb_ops > dpaa2_eqcr_size) ?
1471 dpaa2_eqcr_size : nb_ops;
1473 for (loop = 0; loop < frames_to_send; loop++) {
1474 if ((*ops)->sym->m_src->seqn) {
1475 uint8_t dqrr_index = (*ops)->sym->m_src->seqn - 1;
1477 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index;
1478 DPAA2_PER_LCORE_DQRR_SIZE--;
1479 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1480 (*ops)->sym->m_src->seqn = DPAA2_INVALID_MBUF_SEQN;
1483 /*Clear the unused FD fields before sending*/
1484 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd));
1485 mb_pool = (*ops)->sym->m_src->pool;
1486 bpid = mempool_to_bpid(mb_pool);
1487 ret = build_sec_fd(*ops, &fd_arr[loop], bpid);
1489 DPAA2_SEC_ERR("error: Improper packet contents"
1490 " for crypto operation");
1498 while (loop < frames_to_send) {
1499 ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1502 frames_to_send - loop);
1503 if (unlikely(ret < 0)) {
1505 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1520 dpaa2_qp->tx_vq.tx_pkts += num_tx;
1521 dpaa2_qp->tx_vq.err_pkts += nb_ops;
1525 #ifdef RTE_LIBRTE_SECURITY
1526 static inline struct rte_crypto_op *
1527 sec_simple_fd_to_mbuf(const struct qbman_fd *fd)
1529 struct rte_crypto_op *op;
1530 uint16_t len = DPAA2_GET_FD_LEN(fd);
1532 dpaa2_sec_session *sess_priv __rte_unused;
1534 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
1535 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
1536 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
1538 diff = len - mbuf->pkt_len;
1539 mbuf->pkt_len += diff;
1540 mbuf->data_len += diff;
1541 op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova;
1542 mbuf->buf_iova = op->sym->aead.digest.phys_addr;
1543 op->sym->aead.digest.phys_addr = 0L;
1545 sess_priv = (dpaa2_sec_session *)get_sec_session_private_data(
1546 op->sym->sec_session);
1547 if (sess_priv->dir == DIR_ENC)
1548 mbuf->data_off += SEC_FLC_DHR_OUTBOUND;
1550 mbuf->data_off += SEC_FLC_DHR_INBOUND;
1556 static inline struct rte_crypto_op *
1557 sec_fd_to_mbuf(const struct qbman_fd *fd)
1559 struct qbman_fle *fle;
1560 struct rte_crypto_op *op;
1561 struct ctxt_priv *priv;
1562 struct rte_mbuf *dst, *src;
1564 #ifdef RTE_LIBRTE_SECURITY
1565 if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single)
1566 return sec_simple_fd_to_mbuf(fd);
1568 fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
1570 DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n",
1571 fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset);
1573 /* we are using the first FLE entry to store Mbuf.
1574 * Currently we donot know which FLE has the mbuf stored.
1575 * So while retreiving we can go back 1 FLE from the FD -ADDR
1576 * to get the MBUF Addr from the previous FLE.
1577 * We can have a better approach to use the inline Mbuf
1580 if (unlikely(DPAA2_GET_FD_IVP(fd))) {
1581 /* TODO complete it. */
1582 DPAA2_SEC_ERR("error: non inline buffer");
1585 op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1));
1588 src = op->sym->m_src;
1591 if (op->sym->m_dst) {
1592 dst = op->sym->m_dst;
1597 #ifdef RTE_LIBRTE_SECURITY
1598 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
1599 uint16_t len = DPAA2_GET_FD_LEN(fd);
1601 while (dst->next != NULL) {
1602 len -= dst->data_len;
1605 dst->data_len = len;
1608 DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p,"
1609 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n",
1612 DPAA2_GET_FD_ADDR(fd),
1613 DPAA2_GET_FD_BPID(fd),
1614 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
1615 DPAA2_GET_FD_OFFSET(fd),
1616 DPAA2_GET_FD_LEN(fd));
1618 /* free the fle memory */
1619 if (likely(rte_pktmbuf_is_contiguous(src))) {
1620 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1);
1621 rte_mempool_put(priv->fle_pool, (void *)(fle-1));
1623 rte_free((void *)(fle-1));
1629 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops,
1632 /* Function is responsible to receive frames for a given device and VQ*/
1633 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp;
1634 struct qbman_result *dq_storage;
1635 uint32_t fqid = dpaa2_qp->rx_vq.fqid;
1636 int ret, num_rx = 0;
1637 uint8_t is_last = 0, status;
1638 struct qbman_swp *swp;
1639 const struct qbman_fd *fd;
1640 struct qbman_pull_desc pulldesc;
1642 if (!DPAA2_PER_LCORE_DPIO) {
1643 ret = dpaa2_affine_qbman_swp();
1646 "Failed to allocate IO portal, tid: %d\n",
1651 swp = DPAA2_PER_LCORE_PORTAL;
1652 dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0];
1654 qbman_pull_desc_clear(&pulldesc);
1655 qbman_pull_desc_set_numframes(&pulldesc,
1656 (nb_ops > dpaa2_dqrr_size) ?
1657 dpaa2_dqrr_size : nb_ops);
1658 qbman_pull_desc_set_fq(&pulldesc, fqid);
1659 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1660 (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage),
1663 /*Issue a volatile dequeue command. */
1665 if (qbman_swp_pull(swp, &pulldesc)) {
1667 "SEC VDQ command is not issued : QBMAN busy");
1668 /* Portal was busy, try again */
1674 /* Receive the packets till Last Dequeue entry is found with
1675 * respect to the above issues PULL command.
1678 /* Check if the previous issued command is completed.
1679 * Also seems like the SWP is shared between the Ethernet Driver
1680 * and the SEC driver.
1682 while (!qbman_check_command_complete(dq_storage))
1685 /* Loop until the dq_storage is updated with
1686 * new token by QBMAN
1688 while (!qbman_check_new_result(dq_storage))
1690 /* Check whether Last Pull command is Expired and
1691 * setting Condition for Loop termination
1693 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1695 /* Check for valid frame. */
1696 status = (uint8_t)qbman_result_DQ_flags(dq_storage);
1698 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) {
1699 DPAA2_SEC_DP_DEBUG("No frame is delivered\n");
1704 fd = qbman_result_DQ_fd(dq_storage);
1705 ops[num_rx] = sec_fd_to_mbuf(fd);
1707 if (unlikely(fd->simple.frc)) {
1708 /* TODO Parse SEC errors */
1709 DPAA2_SEC_ERR("SEC returned Error - %x",
1711 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR;
1713 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1718 } /* End of Packet Rx loop */
1720 dpaa2_qp->rx_vq.rx_pkts += num_rx;
1722 DPAA2_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
1723 /*Return the total number of packets received to DPAA2 app*/
1727 /** Release queue pair */
1729 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
1731 struct dpaa2_sec_qp *qp =
1732 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id];
1734 PMD_INIT_FUNC_TRACE();
1736 if (qp->rx_vq.q_storage) {
1737 dpaa2_free_dq_storage(qp->rx_vq.q_storage);
1738 rte_free(qp->rx_vq.q_storage);
1742 dev->data->queue_pairs[queue_pair_id] = NULL;
1747 /** Setup a queue pair */
1749 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id,
1750 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf,
1751 __rte_unused int socket_id)
1753 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
1754 struct dpaa2_sec_qp *qp;
1755 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
1756 struct dpseci_rx_queue_cfg cfg;
1759 PMD_INIT_FUNC_TRACE();
1761 /* If qp is already in use free ring memory and qp metadata. */
1762 if (dev->data->queue_pairs[qp_id] != NULL) {
1763 DPAA2_SEC_INFO("QP already setup");
1767 DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p",
1768 dev, qp_id, qp_conf);
1770 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
1772 qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp),
1773 RTE_CACHE_LINE_SIZE);
1775 DPAA2_SEC_ERR("malloc failed for rx/tx queues");
1779 qp->rx_vq.crypto_data = dev->data;
1780 qp->tx_vq.crypto_data = dev->data;
1781 qp->rx_vq.q_storage = rte_malloc("sec dq storage",
1782 sizeof(struct queue_storage_info_t),
1783 RTE_CACHE_LINE_SIZE);
1784 if (!qp->rx_vq.q_storage) {
1785 DPAA2_SEC_ERR("malloc failed for q_storage");
1788 memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t));
1790 if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) {
1791 DPAA2_SEC_ERR("Unable to allocate dequeue storage");
1795 dev->data->queue_pairs[qp_id] = qp;
1797 cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX;
1798 cfg.user_ctx = (size_t)(&qp->rx_vq);
1799 retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
1804 /** Returns the size of the aesni gcm session structure */
1806 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused)
1808 PMD_INIT_FUNC_TRACE();
1810 return sizeof(dpaa2_sec_session);
1814 dpaa2_sec_cipher_init(struct rte_cryptodev *dev,
1815 struct rte_crypto_sym_xform *xform,
1816 dpaa2_sec_session *session)
1818 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1819 struct alginfo cipherdata;
1820 int bufsize, ret = 0;
1821 struct ctxt_priv *priv;
1822 struct sec_flow_context *flc;
1824 PMD_INIT_FUNC_TRACE();
1826 /* For SEC CIPHER only one descriptor is required. */
1827 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1828 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
1829 RTE_CACHE_LINE_SIZE);
1831 DPAA2_SEC_ERR("No Memory for priv CTXT");
1835 priv->fle_pool = dev_priv->fle_pool;
1837 flc = &priv->flc_desc[0].flc;
1839 session->ctxt_type = DPAA2_SEC_CIPHER;
1840 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length,
1841 RTE_CACHE_LINE_SIZE);
1842 if (session->cipher_key.data == NULL) {
1843 DPAA2_SEC_ERR("No Memory for cipher key");
1847 session->cipher_key.length = xform->cipher.key.length;
1849 memcpy(session->cipher_key.data, xform->cipher.key.data,
1850 xform->cipher.key.length);
1851 cipherdata.key = (size_t)session->cipher_key.data;
1852 cipherdata.keylen = session->cipher_key.length;
1853 cipherdata.key_enc_flags = 0;
1854 cipherdata.key_type = RTA_DATA_IMM;
1856 /* Set IV parameters */
1857 session->iv.offset = xform->cipher.iv.offset;
1858 session->iv.length = xform->cipher.iv.length;
1859 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
1862 switch (xform->cipher.algo) {
1863 case RTE_CRYPTO_CIPHER_AES_CBC:
1864 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1865 cipherdata.algmode = OP_ALG_AAI_CBC;
1866 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
1867 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1868 SHR_NEVER, &cipherdata,
1872 case RTE_CRYPTO_CIPHER_3DES_CBC:
1873 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
1874 cipherdata.algmode = OP_ALG_AAI_CBC;
1875 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
1876 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1877 SHR_NEVER, &cipherdata,
1881 case RTE_CRYPTO_CIPHER_DES_CBC:
1882 cipherdata.algtype = OP_ALG_ALGSEL_DES;
1883 cipherdata.algmode = OP_ALG_AAI_CBC;
1884 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
1885 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1886 SHR_NEVER, &cipherdata,
1890 case RTE_CRYPTO_CIPHER_AES_CTR:
1891 cipherdata.algtype = OP_ALG_ALGSEL_AES;
1892 cipherdata.algmode = OP_ALG_AAI_CTR;
1893 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
1894 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0,
1895 SHR_NEVER, &cipherdata,
1899 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
1900 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8;
1901 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2;
1902 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0,
1906 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
1907 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE;
1908 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3;
1909 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0,
1913 case RTE_CRYPTO_CIPHER_KASUMI_F8:
1914 case RTE_CRYPTO_CIPHER_AES_F8:
1915 case RTE_CRYPTO_CIPHER_AES_ECB:
1916 case RTE_CRYPTO_CIPHER_3DES_ECB:
1917 case RTE_CRYPTO_CIPHER_3DES_CTR:
1918 case RTE_CRYPTO_CIPHER_AES_XTS:
1919 case RTE_CRYPTO_CIPHER_ARC4:
1920 case RTE_CRYPTO_CIPHER_NULL:
1921 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
1922 xform->cipher.algo);
1926 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
1927 xform->cipher.algo);
1933 DPAA2_SEC_ERR("Crypto: Descriptor build failed");
1938 flc->word1_sdl = (uint8_t)bufsize;
1939 session->ctxt = priv;
1941 #ifdef CAAM_DESC_DEBUG
1943 for (i = 0; i < bufsize; i++)
1944 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]);
1949 rte_free(session->cipher_key.data);
1955 dpaa2_sec_auth_init(struct rte_cryptodev *dev,
1956 struct rte_crypto_sym_xform *xform,
1957 dpaa2_sec_session *session)
1959 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
1960 struct alginfo authdata;
1961 int bufsize, ret = 0;
1962 struct ctxt_priv *priv;
1963 struct sec_flow_context *flc;
1965 PMD_INIT_FUNC_TRACE();
1967 /* For SEC AUTH three descriptors are required for various stages */
1968 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
1969 sizeof(struct ctxt_priv) + 3 *
1970 sizeof(struct sec_flc_desc),
1971 RTE_CACHE_LINE_SIZE);
1973 DPAA2_SEC_ERR("No Memory for priv CTXT");
1977 priv->fle_pool = dev_priv->fle_pool;
1978 flc = &priv->flc_desc[DESC_INITFINAL].flc;
1980 session->ctxt_type = DPAA2_SEC_AUTH;
1981 session->auth_key.length = xform->auth.key.length;
1982 if (xform->auth.key.length) {
1983 session->auth_key.data = rte_zmalloc(NULL,
1984 xform->auth.key.length,
1985 RTE_CACHE_LINE_SIZE);
1986 if (session->auth_key.data == NULL) {
1987 DPAA2_SEC_ERR("Unable to allocate memory for auth key");
1991 memcpy(session->auth_key.data, xform->auth.key.data,
1992 xform->auth.key.length);
1993 authdata.key = (size_t)session->auth_key.data;
1994 authdata.key_enc_flags = 0;
1995 authdata.key_type = RTA_DATA_IMM;
1997 authdata.keylen = session->auth_key.length;
1999 session->digest_length = xform->auth.digest_length;
2000 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ?
2003 switch (xform->auth.algo) {
2004 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2005 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2006 authdata.algmode = OP_ALG_AAI_HMAC;
2007 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2008 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2009 1, 0, SHR_NEVER, &authdata,
2011 session->digest_length);
2013 case RTE_CRYPTO_AUTH_MD5_HMAC:
2014 authdata.algtype = OP_ALG_ALGSEL_MD5;
2015 authdata.algmode = OP_ALG_AAI_HMAC;
2016 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2017 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2018 1, 0, SHR_NEVER, &authdata,
2020 session->digest_length);
2022 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2023 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2024 authdata.algmode = OP_ALG_AAI_HMAC;
2025 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2026 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2027 1, 0, SHR_NEVER, &authdata,
2029 session->digest_length);
2031 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2032 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2033 authdata.algmode = OP_ALG_AAI_HMAC;
2034 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2035 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2036 1, 0, SHR_NEVER, &authdata,
2038 session->digest_length);
2040 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2041 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2042 authdata.algmode = OP_ALG_AAI_HMAC;
2043 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2044 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2045 1, 0, SHR_NEVER, &authdata,
2047 session->digest_length);
2049 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2050 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2051 authdata.algmode = OP_ALG_AAI_HMAC;
2052 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2053 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc,
2054 1, 0, SHR_NEVER, &authdata,
2056 session->digest_length);
2058 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2059 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9;
2060 authdata.algmode = OP_ALG_AAI_F9;
2061 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2;
2062 session->iv.offset = xform->auth.iv.offset;
2063 session->iv.length = xform->auth.iv.length;
2064 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc,
2067 session->digest_length);
2069 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2070 authdata.algtype = OP_ALG_ALGSEL_ZUCA;
2071 authdata.algmode = OP_ALG_AAI_F9;
2072 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3;
2073 session->iv.offset = xform->auth.iv.offset;
2074 session->iv.length = xform->auth.iv.length;
2075 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc,
2078 session->digest_length);
2080 case RTE_CRYPTO_AUTH_SHA1:
2081 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2082 authdata.algmode = OP_ALG_AAI_HASH;
2083 session->auth_alg = RTE_CRYPTO_AUTH_SHA1;
2084 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2085 1, 0, SHR_NEVER, &authdata,
2087 session->digest_length);
2089 case RTE_CRYPTO_AUTH_MD5:
2090 authdata.algtype = OP_ALG_ALGSEL_MD5;
2091 authdata.algmode = OP_ALG_AAI_HASH;
2092 session->auth_alg = RTE_CRYPTO_AUTH_MD5;
2093 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2094 1, 0, SHR_NEVER, &authdata,
2096 session->digest_length);
2098 case RTE_CRYPTO_AUTH_SHA256:
2099 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2100 authdata.algmode = OP_ALG_AAI_HASH;
2101 session->auth_alg = RTE_CRYPTO_AUTH_SHA256;
2102 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2103 1, 0, SHR_NEVER, &authdata,
2105 session->digest_length);
2107 case RTE_CRYPTO_AUTH_SHA384:
2108 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2109 authdata.algmode = OP_ALG_AAI_HASH;
2110 session->auth_alg = RTE_CRYPTO_AUTH_SHA384;
2111 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2112 1, 0, SHR_NEVER, &authdata,
2114 session->digest_length);
2116 case RTE_CRYPTO_AUTH_SHA512:
2117 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2118 authdata.algmode = OP_ALG_AAI_HASH;
2119 session->auth_alg = RTE_CRYPTO_AUTH_SHA512;
2120 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2121 1, 0, SHR_NEVER, &authdata,
2123 session->digest_length);
2125 case RTE_CRYPTO_AUTH_SHA224:
2126 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2127 authdata.algmode = OP_ALG_AAI_HASH;
2128 session->auth_alg = RTE_CRYPTO_AUTH_SHA224;
2129 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc,
2130 1, 0, SHR_NEVER, &authdata,
2132 session->digest_length);
2134 case RTE_CRYPTO_AUTH_AES_GMAC:
2135 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2136 case RTE_CRYPTO_AUTH_AES_CMAC:
2137 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2138 case RTE_CRYPTO_AUTH_KASUMI_F9:
2139 case RTE_CRYPTO_AUTH_NULL:
2140 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un",
2145 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2152 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2157 flc->word1_sdl = (uint8_t)bufsize;
2158 session->ctxt = priv;
2159 #ifdef CAAM_DESC_DEBUG
2161 for (i = 0; i < bufsize; i++)
2162 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2163 i, priv->flc_desc[DESC_INITFINAL].desc[i]);
2169 rte_free(session->auth_key.data);
2175 dpaa2_sec_aead_init(struct rte_cryptodev *dev,
2176 struct rte_crypto_sym_xform *xform,
2177 dpaa2_sec_session *session)
2179 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt;
2180 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2181 struct alginfo aeaddata;
2183 struct ctxt_priv *priv;
2184 struct sec_flow_context *flc;
2185 struct rte_crypto_aead_xform *aead_xform = &xform->aead;
2188 PMD_INIT_FUNC_TRACE();
2190 /* Set IV parameters */
2191 session->iv.offset = aead_xform->iv.offset;
2192 session->iv.length = aead_xform->iv.length;
2193 session->ctxt_type = DPAA2_SEC_AEAD;
2195 /* For SEC AEAD only one descriptor is required */
2196 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2197 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2198 RTE_CACHE_LINE_SIZE);
2200 DPAA2_SEC_ERR("No Memory for priv CTXT");
2204 priv->fle_pool = dev_priv->fle_pool;
2205 flc = &priv->flc_desc[0].flc;
2207 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2208 RTE_CACHE_LINE_SIZE);
2209 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2210 DPAA2_SEC_ERR("No Memory for aead key");
2214 memcpy(session->aead_key.data, aead_xform->key.data,
2215 aead_xform->key.length);
2217 session->digest_length = aead_xform->digest_length;
2218 session->aead_key.length = aead_xform->key.length;
2219 ctxt->auth_only_len = aead_xform->aad_length;
2221 aeaddata.key = (size_t)session->aead_key.data;
2222 aeaddata.keylen = session->aead_key.length;
2223 aeaddata.key_enc_flags = 0;
2224 aeaddata.key_type = RTA_DATA_IMM;
2226 switch (aead_xform->algo) {
2227 case RTE_CRYPTO_AEAD_AES_GCM:
2228 aeaddata.algtype = OP_ALG_ALGSEL_AES;
2229 aeaddata.algmode = OP_ALG_AAI_GCM;
2230 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2232 case RTE_CRYPTO_AEAD_AES_CCM:
2233 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u",
2238 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2243 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2246 priv->flc_desc[0].desc[0] = aeaddata.keylen;
2247 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2249 (unsigned int *)priv->flc_desc[0].desc,
2250 &priv->flc_desc[0].desc[1], 1);
2253 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2257 if (priv->flc_desc[0].desc[1] & 1) {
2258 aeaddata.key_type = RTA_DATA_IMM;
2260 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key);
2261 aeaddata.key_type = RTA_DATA_PTR;
2263 priv->flc_desc[0].desc[0] = 0;
2264 priv->flc_desc[0].desc[1] = 0;
2266 if (session->dir == DIR_ENC)
2267 bufsize = cnstr_shdsc_gcm_encap(
2268 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2269 &aeaddata, session->iv.length,
2270 session->digest_length);
2272 bufsize = cnstr_shdsc_gcm_decap(
2273 priv->flc_desc[0].desc, 1, 0, SHR_NEVER,
2274 &aeaddata, session->iv.length,
2275 session->digest_length);
2277 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2282 flc->word1_sdl = (uint8_t)bufsize;
2283 session->ctxt = priv;
2284 #ifdef CAAM_DESC_DEBUG
2286 for (i = 0; i < bufsize; i++)
2287 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n",
2288 i, priv->flc_desc[0].desc[i]);
2293 rte_free(session->aead_key.data);
2300 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev,
2301 struct rte_crypto_sym_xform *xform,
2302 dpaa2_sec_session *session)
2304 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2305 struct alginfo authdata, cipherdata;
2307 struct ctxt_priv *priv;
2308 struct sec_flow_context *flc;
2309 struct rte_crypto_cipher_xform *cipher_xform;
2310 struct rte_crypto_auth_xform *auth_xform;
2313 PMD_INIT_FUNC_TRACE();
2315 if (session->ext_params.aead_ctxt.auth_cipher_text) {
2316 cipher_xform = &xform->cipher;
2317 auth_xform = &xform->next->auth;
2318 session->ctxt_type =
2319 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2320 DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER;
2322 cipher_xform = &xform->next->cipher;
2323 auth_xform = &xform->auth;
2324 session->ctxt_type =
2325 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2326 DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH;
2329 /* Set IV parameters */
2330 session->iv.offset = cipher_xform->iv.offset;
2331 session->iv.length = cipher_xform->iv.length;
2333 /* For SEC AEAD only one descriptor is required */
2334 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2335 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc),
2336 RTE_CACHE_LINE_SIZE);
2338 DPAA2_SEC_ERR("No Memory for priv CTXT");
2342 priv->fle_pool = dev_priv->fle_pool;
2343 flc = &priv->flc_desc[0].flc;
2345 session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length,
2346 RTE_CACHE_LINE_SIZE);
2347 if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) {
2348 DPAA2_SEC_ERR("No Memory for cipher key");
2352 session->cipher_key.length = cipher_xform->key.length;
2353 session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length,
2354 RTE_CACHE_LINE_SIZE);
2355 if (session->auth_key.data == NULL && auth_xform->key.length > 0) {
2356 DPAA2_SEC_ERR("No Memory for auth key");
2357 rte_free(session->cipher_key.data);
2361 session->auth_key.length = auth_xform->key.length;
2362 memcpy(session->cipher_key.data, cipher_xform->key.data,
2363 cipher_xform->key.length);
2364 memcpy(session->auth_key.data, auth_xform->key.data,
2365 auth_xform->key.length);
2367 authdata.key = (size_t)session->auth_key.data;
2368 authdata.keylen = session->auth_key.length;
2369 authdata.key_enc_flags = 0;
2370 authdata.key_type = RTA_DATA_IMM;
2372 session->digest_length = auth_xform->digest_length;
2374 switch (auth_xform->algo) {
2375 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2376 authdata.algtype = OP_ALG_ALGSEL_SHA1;
2377 authdata.algmode = OP_ALG_AAI_HMAC;
2378 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
2380 case RTE_CRYPTO_AUTH_MD5_HMAC:
2381 authdata.algtype = OP_ALG_ALGSEL_MD5;
2382 authdata.algmode = OP_ALG_AAI_HMAC;
2383 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC;
2385 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2386 authdata.algtype = OP_ALG_ALGSEL_SHA224;
2387 authdata.algmode = OP_ALG_AAI_HMAC;
2388 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC;
2390 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2391 authdata.algtype = OP_ALG_ALGSEL_SHA256;
2392 authdata.algmode = OP_ALG_AAI_HMAC;
2393 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
2395 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2396 authdata.algtype = OP_ALG_ALGSEL_SHA384;
2397 authdata.algmode = OP_ALG_AAI_HMAC;
2398 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
2400 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2401 authdata.algtype = OP_ALG_ALGSEL_SHA512;
2402 authdata.algmode = OP_ALG_AAI_HMAC;
2403 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
2405 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2406 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2407 case RTE_CRYPTO_AUTH_NULL:
2408 case RTE_CRYPTO_AUTH_SHA1:
2409 case RTE_CRYPTO_AUTH_SHA256:
2410 case RTE_CRYPTO_AUTH_SHA512:
2411 case RTE_CRYPTO_AUTH_SHA224:
2412 case RTE_CRYPTO_AUTH_SHA384:
2413 case RTE_CRYPTO_AUTH_MD5:
2414 case RTE_CRYPTO_AUTH_AES_GMAC:
2415 case RTE_CRYPTO_AUTH_KASUMI_F9:
2416 case RTE_CRYPTO_AUTH_AES_CMAC:
2417 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2418 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2419 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2424 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2429 cipherdata.key = (size_t)session->cipher_key.data;
2430 cipherdata.keylen = session->cipher_key.length;
2431 cipherdata.key_enc_flags = 0;
2432 cipherdata.key_type = RTA_DATA_IMM;
2434 switch (cipher_xform->algo) {
2435 case RTE_CRYPTO_CIPHER_AES_CBC:
2436 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2437 cipherdata.algmode = OP_ALG_AAI_CBC;
2438 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC;
2440 case RTE_CRYPTO_CIPHER_3DES_CBC:
2441 cipherdata.algtype = OP_ALG_ALGSEL_3DES;
2442 cipherdata.algmode = OP_ALG_AAI_CBC;
2443 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC;
2445 case RTE_CRYPTO_CIPHER_DES_CBC:
2446 cipherdata.algtype = OP_ALG_ALGSEL_DES;
2447 cipherdata.algmode = OP_ALG_AAI_CBC;
2448 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC;
2450 case RTE_CRYPTO_CIPHER_AES_CTR:
2451 cipherdata.algtype = OP_ALG_ALGSEL_AES;
2452 cipherdata.algmode = OP_ALG_AAI_CTR;
2453 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR;
2455 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2456 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2457 case RTE_CRYPTO_CIPHER_NULL:
2458 case RTE_CRYPTO_CIPHER_3DES_ECB:
2459 case RTE_CRYPTO_CIPHER_3DES_CTR:
2460 case RTE_CRYPTO_CIPHER_AES_ECB:
2461 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2462 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2463 cipher_xform->algo);
2467 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2468 cipher_xform->algo);
2472 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
2475 priv->flc_desc[0].desc[0] = cipherdata.keylen;
2476 priv->flc_desc[0].desc[1] = authdata.keylen;
2477 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN,
2479 (unsigned int *)priv->flc_desc[0].desc,
2480 &priv->flc_desc[0].desc[2], 2);
2483 DPAA2_SEC_ERR("Crypto: Incorrect key lengths");
2487 if (priv->flc_desc[0].desc[2] & 1) {
2488 cipherdata.key_type = RTA_DATA_IMM;
2490 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
2491 cipherdata.key_type = RTA_DATA_PTR;
2493 if (priv->flc_desc[0].desc[2] & (1 << 1)) {
2494 authdata.key_type = RTA_DATA_IMM;
2496 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key);
2497 authdata.key_type = RTA_DATA_PTR;
2499 priv->flc_desc[0].desc[0] = 0;
2500 priv->flc_desc[0].desc[1] = 0;
2501 priv->flc_desc[0].desc[2] = 0;
2503 if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) {
2504 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1,
2506 &cipherdata, &authdata,
2508 session->digest_length,
2511 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
2516 DPAA2_SEC_ERR("Hash before cipher not supported");
2521 flc->word1_sdl = (uint8_t)bufsize;
2522 session->ctxt = priv;
2523 #ifdef CAAM_DESC_DEBUG
2525 for (i = 0; i < bufsize; i++)
2526 DPAA2_SEC_DEBUG("DESC[%d]:0x%x",
2527 i, priv->flc_desc[0].desc[i]);
2533 rte_free(session->cipher_key.data);
2534 rte_free(session->auth_key.data);
2540 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev,
2541 struct rte_crypto_sym_xform *xform, void *sess)
2543 dpaa2_sec_session *session = sess;
2546 PMD_INIT_FUNC_TRACE();
2548 if (unlikely(sess == NULL)) {
2549 DPAA2_SEC_ERR("Invalid session struct");
2553 memset(session, 0, sizeof(dpaa2_sec_session));
2554 /* Default IV length = 0 */
2555 session->iv.length = 0;
2558 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
2559 ret = dpaa2_sec_cipher_init(dev, xform, session);
2561 /* Authentication Only */
2562 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2563 xform->next == NULL) {
2564 ret = dpaa2_sec_auth_init(dev, xform, session);
2566 /* Cipher then Authenticate */
2567 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2568 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2569 session->ext_params.aead_ctxt.auth_cipher_text = true;
2570 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2571 ret = dpaa2_sec_auth_init(dev, xform, session);
2572 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL)
2573 ret = dpaa2_sec_cipher_init(dev, xform, session);
2575 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2576 /* Authenticate then Cipher */
2577 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
2578 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2579 session->ext_params.aead_ctxt.auth_cipher_text = false;
2580 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL)
2581 ret = dpaa2_sec_cipher_init(dev, xform, session);
2582 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL)
2583 ret = dpaa2_sec_auth_init(dev, xform, session);
2585 ret = dpaa2_sec_aead_chain_init(dev, xform, session);
2586 /* AEAD operation for AES-GCM kind of Algorithms */
2587 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD &&
2588 xform->next == NULL) {
2589 ret = dpaa2_sec_aead_init(dev, xform, session);
2592 DPAA2_SEC_ERR("Invalid crypto type");
2599 #ifdef RTE_LIBRTE_SECURITY
2601 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform,
2602 dpaa2_sec_session *session,
2603 struct alginfo *aeaddata)
2605 PMD_INIT_FUNC_TRACE();
2607 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length,
2608 RTE_CACHE_LINE_SIZE);
2609 if (session->aead_key.data == NULL && aead_xform->key.length > 0) {
2610 DPAA2_SEC_ERR("No Memory for aead key");
2613 memcpy(session->aead_key.data, aead_xform->key.data,
2614 aead_xform->key.length);
2616 session->digest_length = aead_xform->digest_length;
2617 session->aead_key.length = aead_xform->key.length;
2619 aeaddata->key = (size_t)session->aead_key.data;
2620 aeaddata->keylen = session->aead_key.length;
2621 aeaddata->key_enc_flags = 0;
2622 aeaddata->key_type = RTA_DATA_IMM;
2624 switch (aead_xform->algo) {
2625 case RTE_CRYPTO_AEAD_AES_GCM:
2626 switch (session->digest_length) {
2628 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8;
2631 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12;
2634 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16;
2637 DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d",
2638 session->digest_length);
2641 aeaddata->algmode = OP_ALG_AAI_GCM;
2642 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM;
2644 case RTE_CRYPTO_AEAD_AES_CCM:
2645 switch (session->digest_length) {
2647 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8;
2650 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12;
2653 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16;
2656 DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d",
2657 session->digest_length);
2660 aeaddata->algmode = OP_ALG_AAI_CCM;
2661 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM;
2664 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u",
2668 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ?
2675 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform,
2676 struct rte_crypto_auth_xform *auth_xform,
2677 dpaa2_sec_session *session,
2678 struct alginfo *cipherdata,
2679 struct alginfo *authdata)
2682 session->cipher_key.data = rte_zmalloc(NULL,
2683 cipher_xform->key.length,
2684 RTE_CACHE_LINE_SIZE);
2685 if (session->cipher_key.data == NULL &&
2686 cipher_xform->key.length > 0) {
2687 DPAA2_SEC_ERR("No Memory for cipher key");
2691 session->cipher_key.length = cipher_xform->key.length;
2692 memcpy(session->cipher_key.data, cipher_xform->key.data,
2693 cipher_xform->key.length);
2694 session->cipher_alg = cipher_xform->algo;
2696 session->cipher_key.data = NULL;
2697 session->cipher_key.length = 0;
2698 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
2702 session->auth_key.data = rte_zmalloc(NULL,
2703 auth_xform->key.length,
2704 RTE_CACHE_LINE_SIZE);
2705 if (session->auth_key.data == NULL &&
2706 auth_xform->key.length > 0) {
2707 DPAA2_SEC_ERR("No Memory for auth key");
2710 session->auth_key.length = auth_xform->key.length;
2711 memcpy(session->auth_key.data, auth_xform->key.data,
2712 auth_xform->key.length);
2713 session->auth_alg = auth_xform->algo;
2714 session->digest_length = auth_xform->digest_length;
2716 session->auth_key.data = NULL;
2717 session->auth_key.length = 0;
2718 session->auth_alg = RTE_CRYPTO_AUTH_NULL;
2721 authdata->key = (size_t)session->auth_key.data;
2722 authdata->keylen = session->auth_key.length;
2723 authdata->key_enc_flags = 0;
2724 authdata->key_type = RTA_DATA_IMM;
2725 switch (session->auth_alg) {
2726 case RTE_CRYPTO_AUTH_SHA1_HMAC:
2727 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96;
2728 authdata->algmode = OP_ALG_AAI_HMAC;
2730 case RTE_CRYPTO_AUTH_MD5_HMAC:
2731 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96;
2732 authdata->algmode = OP_ALG_AAI_HMAC;
2734 case RTE_CRYPTO_AUTH_SHA256_HMAC:
2735 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128;
2736 authdata->algmode = OP_ALG_AAI_HMAC;
2737 if (session->digest_length != 16)
2739 "+++Using sha256-hmac truncated len is non-standard,"
2740 "it will not work with lookaside proto");
2742 case RTE_CRYPTO_AUTH_SHA384_HMAC:
2743 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192;
2744 authdata->algmode = OP_ALG_AAI_HMAC;
2746 case RTE_CRYPTO_AUTH_SHA512_HMAC:
2747 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256;
2748 authdata->algmode = OP_ALG_AAI_HMAC;
2750 case RTE_CRYPTO_AUTH_AES_CMAC:
2751 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96;
2753 case RTE_CRYPTO_AUTH_NULL:
2754 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL;
2756 case RTE_CRYPTO_AUTH_SHA224_HMAC:
2757 case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2758 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2759 case RTE_CRYPTO_AUTH_SHA1:
2760 case RTE_CRYPTO_AUTH_SHA256:
2761 case RTE_CRYPTO_AUTH_SHA512:
2762 case RTE_CRYPTO_AUTH_SHA224:
2763 case RTE_CRYPTO_AUTH_SHA384:
2764 case RTE_CRYPTO_AUTH_MD5:
2765 case RTE_CRYPTO_AUTH_AES_GMAC:
2766 case RTE_CRYPTO_AUTH_KASUMI_F9:
2767 case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2768 case RTE_CRYPTO_AUTH_ZUC_EIA3:
2769 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
2773 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u",
2777 cipherdata->key = (size_t)session->cipher_key.data;
2778 cipherdata->keylen = session->cipher_key.length;
2779 cipherdata->key_enc_flags = 0;
2780 cipherdata->key_type = RTA_DATA_IMM;
2782 switch (session->cipher_alg) {
2783 case RTE_CRYPTO_CIPHER_AES_CBC:
2784 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC;
2785 cipherdata->algmode = OP_ALG_AAI_CBC;
2787 case RTE_CRYPTO_CIPHER_3DES_CBC:
2788 cipherdata->algtype = OP_PCL_IPSEC_3DES;
2789 cipherdata->algmode = OP_ALG_AAI_CBC;
2791 case RTE_CRYPTO_CIPHER_DES_CBC:
2792 cipherdata->algtype = OP_PCL_IPSEC_DES;
2793 cipherdata->algmode = OP_ALG_AAI_CBC;
2795 case RTE_CRYPTO_CIPHER_AES_CTR:
2796 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR;
2797 cipherdata->algmode = OP_ALG_AAI_CTR;
2799 case RTE_CRYPTO_CIPHER_NULL:
2800 cipherdata->algtype = OP_PCL_IPSEC_NULL;
2802 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2803 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2804 case RTE_CRYPTO_CIPHER_3DES_ECB:
2805 case RTE_CRYPTO_CIPHER_3DES_CTR:
2806 case RTE_CRYPTO_CIPHER_AES_ECB:
2807 case RTE_CRYPTO_CIPHER_KASUMI_F8:
2808 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u",
2809 session->cipher_alg);
2812 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
2813 session->cipher_alg);
2820 #ifdef RTE_LIBRTE_SECURITY_TEST
2821 static uint8_t aes_cbc_iv[] = {
2822 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
2823 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
2827 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev,
2828 struct rte_security_session_conf *conf,
2831 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec;
2832 struct rte_crypto_cipher_xform *cipher_xform = NULL;
2833 struct rte_crypto_auth_xform *auth_xform = NULL;
2834 struct rte_crypto_aead_xform *aead_xform = NULL;
2835 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
2836 struct ctxt_priv *priv;
2837 struct alginfo authdata, cipherdata;
2839 struct sec_flow_context *flc;
2840 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
2843 PMD_INIT_FUNC_TRACE();
2845 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
2846 sizeof(struct ctxt_priv) +
2847 sizeof(struct sec_flc_desc),
2848 RTE_CACHE_LINE_SIZE);
2851 DPAA2_SEC_ERR("No memory for priv CTXT");
2855 priv->fle_pool = dev_priv->fle_pool;
2856 flc = &priv->flc_desc[0].flc;
2858 memset(session, 0, sizeof(dpaa2_sec_session));
2860 if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
2861 cipher_xform = &conf->crypto_xform->cipher;
2862 if (conf->crypto_xform->next)
2863 auth_xform = &conf->crypto_xform->next->auth;
2864 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2865 session, &cipherdata, &authdata);
2866 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2867 auth_xform = &conf->crypto_xform->auth;
2868 if (conf->crypto_xform->next)
2869 cipher_xform = &conf->crypto_xform->next->cipher;
2870 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform,
2871 session, &cipherdata, &authdata);
2872 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
2873 aead_xform = &conf->crypto_xform->aead;
2874 ret = dpaa2_sec_ipsec_aead_init(aead_xform,
2875 session, &cipherdata);
2876 authdata.keylen = 0;
2877 authdata.algtype = 0;
2879 DPAA2_SEC_ERR("XFORM not specified");
2884 DPAA2_SEC_ERR("Failed to process xform");
2888 session->ctxt_type = DPAA2_SEC_IPSEC;
2889 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
2890 uint8_t *hdr = NULL;
2892 struct rte_ipv6_hdr ip6_hdr;
2893 struct ipsec_encap_pdb encap_pdb;
2895 flc->dhr = SEC_FLC_DHR_OUTBOUND;
2896 /* For Sec Proto only one descriptor is required. */
2897 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb));
2899 /* copy algo specific data to PDB */
2900 switch (cipherdata.algtype) {
2901 case OP_PCL_IPSEC_AES_CTR:
2902 encap_pdb.ctr.ctr_initial = 0x00000001;
2903 encap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2905 case OP_PCL_IPSEC_AES_GCM8:
2906 case OP_PCL_IPSEC_AES_GCM12:
2907 case OP_PCL_IPSEC_AES_GCM16:
2908 memcpy(encap_pdb.gcm.salt,
2909 (uint8_t *)&(ipsec_xform->salt), 4);
2913 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) |
2914 PDBOPTS_ESP_OIHI_PDB_INL |
2916 PDBHMO_ESP_ENCAP_DTTL |
2918 if (ipsec_xform->options.esn)
2919 encap_pdb.options |= PDBOPTS_ESP_ESN;
2920 encap_pdb.spi = ipsec_xform->spi;
2921 session->dir = DIR_ENC;
2922 if (ipsec_xform->tunnel.type ==
2923 RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
2924 encap_pdb.ip_hdr_len = sizeof(struct ip);
2925 ip4_hdr.ip_v = IPVERSION;
2927 ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr));
2928 ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp;
2931 ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl;
2932 ip4_hdr.ip_p = IPPROTO_ESP;
2934 ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip;
2935 ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip;
2936 ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *)
2937 &ip4_hdr, sizeof(struct ip));
2938 hdr = (uint8_t *)&ip4_hdr;
2939 } else if (ipsec_xform->tunnel.type ==
2940 RTE_SECURITY_IPSEC_TUNNEL_IPV6) {
2941 ip6_hdr.vtc_flow = rte_cpu_to_be_32(
2942 DPAA2_IPv6_DEFAULT_VTC_FLOW |
2943 ((ipsec_xform->tunnel.ipv6.dscp <<
2944 RTE_IPV6_HDR_TC_SHIFT) &
2945 RTE_IPV6_HDR_TC_MASK) |
2946 ((ipsec_xform->tunnel.ipv6.flabel <<
2947 RTE_IPV6_HDR_FL_SHIFT) &
2948 RTE_IPV6_HDR_FL_MASK));
2949 /* Payload length will be updated by HW */
2950 ip6_hdr.payload_len = 0;
2951 ip6_hdr.hop_limits =
2952 ipsec_xform->tunnel.ipv6.hlimit;
2953 ip6_hdr.proto = (ipsec_xform->proto ==
2954 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ?
2955 IPPROTO_ESP : IPPROTO_AH;
2956 memcpy(&ip6_hdr.src_addr,
2957 &ipsec_xform->tunnel.ipv6.src_addr, 16);
2958 memcpy(&ip6_hdr.dst_addr,
2959 &ipsec_xform->tunnel.ipv6.dst_addr, 16);
2960 encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr);
2961 hdr = (uint8_t *)&ip6_hdr;
2964 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc,
2965 1, 0, SHR_SERIAL, &encap_pdb,
2966 hdr, &cipherdata, &authdata);
2967 } else if (ipsec_xform->direction ==
2968 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
2969 struct ipsec_decap_pdb decap_pdb;
2971 flc->dhr = SEC_FLC_DHR_INBOUND;
2972 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb));
2973 /* copy algo specific data to PDB */
2974 switch (cipherdata.algtype) {
2975 case OP_PCL_IPSEC_AES_CTR:
2976 decap_pdb.ctr.ctr_initial = 0x00000001;
2977 decap_pdb.ctr.ctr_nonce = ipsec_xform->salt;
2979 case OP_PCL_IPSEC_AES_GCM8:
2980 case OP_PCL_IPSEC_AES_GCM12:
2981 case OP_PCL_IPSEC_AES_GCM16:
2982 memcpy(decap_pdb.gcm.salt,
2983 (uint8_t *)&(ipsec_xform->salt), 4);
2987 decap_pdb.options = (ipsec_xform->tunnel.type ==
2988 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ?
2989 sizeof(struct ip) << 16 :
2990 sizeof(struct rte_ipv6_hdr) << 16;
2991 if (ipsec_xform->options.esn)
2992 decap_pdb.options |= PDBOPTS_ESP_ESN;
2994 if (ipsec_xform->replay_win_sz) {
2996 win_sz = rte_align32pow2(ipsec_xform->replay_win_sz);
3005 decap_pdb.options |= PDBOPTS_ESP_ARS32;
3008 decap_pdb.options |= PDBOPTS_ESP_ARS64;
3011 decap_pdb.options |= PDBOPTS_ESP_ARS128;
3014 session->dir = DIR_DEC;
3015 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc,
3017 &decap_pdb, &cipherdata, &authdata);
3022 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3026 flc->word1_sdl = (uint8_t)bufsize;
3028 /* Enable the stashing control bit */
3029 DPAA2_SET_FLC_RSC(flc);
3030 flc->word2_rflc_31_0 = lower_32_bits(
3031 (size_t)&(((struct dpaa2_sec_qp *)
3032 dev->data->queue_pairs[0])->rx_vq) | 0x14);
3033 flc->word3_rflc_63_32 = upper_32_bits(
3034 (size_t)&(((struct dpaa2_sec_qp *)
3035 dev->data->queue_pairs[0])->rx_vq));
3037 /* Set EWS bit i.e. enable write-safe */
3038 DPAA2_SET_FLC_EWS(flc);
3039 /* Set BS = 1 i.e reuse input buffers as output buffers */
3040 DPAA2_SET_FLC_REUSE_BS(flc);
3041 /* Set FF = 10; reuse input buffers if they provide sufficient space */
3042 DPAA2_SET_FLC_REUSE_FF(flc);
3044 session->ctxt = priv;
3048 rte_free(session->auth_key.data);
3049 rte_free(session->cipher_key.data);
3055 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev,
3056 struct rte_security_session_conf *conf,
3059 struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp;
3060 struct rte_crypto_sym_xform *xform = conf->crypto_xform;
3061 struct rte_crypto_auth_xform *auth_xform = NULL;
3062 struct rte_crypto_cipher_xform *cipher_xform;
3063 dpaa2_sec_session *session = (dpaa2_sec_session *)sess;
3064 struct ctxt_priv *priv;
3065 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private;
3066 struct alginfo authdata, cipherdata;
3067 struct alginfo *p_authdata = NULL;
3069 struct sec_flow_context *flc;
3070 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
3076 PMD_INIT_FUNC_TRACE();
3078 memset(session, 0, sizeof(dpaa2_sec_session));
3080 priv = (struct ctxt_priv *)rte_zmalloc(NULL,
3081 sizeof(struct ctxt_priv) +
3082 sizeof(struct sec_flc_desc),
3083 RTE_CACHE_LINE_SIZE);
3086 DPAA2_SEC_ERR("No memory for priv CTXT");
3090 priv->fle_pool = dev_priv->fle_pool;
3091 flc = &priv->flc_desc[0].flc;
3093 /* find xfrm types */
3094 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) {
3095 cipher_xform = &xform->cipher;
3096 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
3097 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
3098 session->ext_params.aead_ctxt.auth_cipher_text = true;
3099 cipher_xform = &xform->cipher;
3100 auth_xform = &xform->next->auth;
3101 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
3102 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
3103 session->ext_params.aead_ctxt.auth_cipher_text = false;
3104 cipher_xform = &xform->next->cipher;
3105 auth_xform = &xform->auth;
3107 DPAA2_SEC_ERR("Invalid crypto type");
3111 session->ctxt_type = DPAA2_SEC_PDCP;
3113 session->cipher_key.data = rte_zmalloc(NULL,
3114 cipher_xform->key.length,
3115 RTE_CACHE_LINE_SIZE);
3116 if (session->cipher_key.data == NULL &&
3117 cipher_xform->key.length > 0) {
3118 DPAA2_SEC_ERR("No Memory for cipher key");
3122 session->cipher_key.length = cipher_xform->key.length;
3123 memcpy(session->cipher_key.data, cipher_xform->key.data,
3124 cipher_xform->key.length);
3126 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ?
3128 session->cipher_alg = cipher_xform->algo;
3130 session->cipher_key.data = NULL;
3131 session->cipher_key.length = 0;
3132 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL;
3133 session->dir = DIR_ENC;
3136 session->pdcp.domain = pdcp_xform->domain;
3137 session->pdcp.bearer = pdcp_xform->bearer;
3138 session->pdcp.pkt_dir = pdcp_xform->pkt_dir;
3139 session->pdcp.sn_size = pdcp_xform->sn_size;
3140 session->pdcp.hfn = pdcp_xform->hfn;
3141 session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold;
3142 session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd;
3143 /* hfv ovd offset location is stored in iv.offset value*/
3144 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset;
3146 cipherdata.key = (size_t)session->cipher_key.data;
3147 cipherdata.keylen = session->cipher_key.length;
3148 cipherdata.key_enc_flags = 0;
3149 cipherdata.key_type = RTA_DATA_IMM;
3151 switch (session->cipher_alg) {
3152 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
3153 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW;
3155 case RTE_CRYPTO_CIPHER_ZUC_EEA3:
3156 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC;
3158 case RTE_CRYPTO_CIPHER_AES_CTR:
3159 cipherdata.algtype = PDCP_CIPHER_TYPE_AES;
3161 case RTE_CRYPTO_CIPHER_NULL:
3162 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL;
3165 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u",
3166 session->cipher_alg);
3171 session->auth_key.data = rte_zmalloc(NULL,
3172 auth_xform->key.length,
3173 RTE_CACHE_LINE_SIZE);
3174 if (!session->auth_key.data &&
3175 auth_xform->key.length > 0) {
3176 DPAA2_SEC_ERR("No Memory for auth key");
3177 rte_free(session->cipher_key.data);
3181 session->auth_key.length = auth_xform->key.length;
3182 memcpy(session->auth_key.data, auth_xform->key.data,
3183 auth_xform->key.length);
3184 session->auth_alg = auth_xform->algo;
3186 session->auth_key.data = NULL;
3187 session->auth_key.length = 0;
3188 session->auth_alg = 0;
3190 authdata.key = (size_t)session->auth_key.data;
3191 authdata.keylen = session->auth_key.length;
3192 authdata.key_enc_flags = 0;
3193 authdata.key_type = RTA_DATA_IMM;
3195 if (session->auth_alg) {
3196 switch (session->auth_alg) {
3197 case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
3198 authdata.algtype = PDCP_AUTH_TYPE_SNOW;
3200 case RTE_CRYPTO_AUTH_ZUC_EIA3:
3201 authdata.algtype = PDCP_AUTH_TYPE_ZUC;
3203 case RTE_CRYPTO_AUTH_AES_CMAC:
3204 authdata.algtype = PDCP_AUTH_TYPE_AES;
3206 case RTE_CRYPTO_AUTH_NULL:
3207 authdata.algtype = PDCP_AUTH_TYPE_NULL;
3210 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u",
3215 p_authdata = &authdata;
3216 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3217 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane");
3221 if (rta_inline_pdcp_query(authdata.algtype,
3223 session->pdcp.sn_size,
3224 session->pdcp.hfn_ovd)) {
3225 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key);
3226 cipherdata.key_type = RTA_DATA_PTR;
3229 if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) {
3230 if (session->dir == DIR_ENC)
3231 bufsize = cnstr_shdsc_pdcp_c_plane_encap(
3232 priv->flc_desc[0].desc, 1, swap,
3234 session->pdcp.sn_size,
3236 pdcp_xform->pkt_dir,
3237 pdcp_xform->hfn_threshold,
3238 &cipherdata, &authdata,
3240 else if (session->dir == DIR_DEC)
3241 bufsize = cnstr_shdsc_pdcp_c_plane_decap(
3242 priv->flc_desc[0].desc, 1, swap,
3244 session->pdcp.sn_size,
3246 pdcp_xform->pkt_dir,
3247 pdcp_xform->hfn_threshold,
3248 &cipherdata, &authdata,
3251 if (session->dir == DIR_ENC)
3252 bufsize = cnstr_shdsc_pdcp_u_plane_encap(
3253 priv->flc_desc[0].desc, 1, swap,
3254 session->pdcp.sn_size,
3257 pdcp_xform->pkt_dir,
3258 pdcp_xform->hfn_threshold,
3259 &cipherdata, p_authdata, 0);
3260 else if (session->dir == DIR_DEC)
3261 bufsize = cnstr_shdsc_pdcp_u_plane_decap(
3262 priv->flc_desc[0].desc, 1, swap,
3263 session->pdcp.sn_size,
3266 pdcp_xform->pkt_dir,
3267 pdcp_xform->hfn_threshold,
3268 &cipherdata, p_authdata, 0);
3272 DPAA2_SEC_ERR("Crypto: Invalid buffer length");
3276 /* Enable the stashing control bit */
3277 DPAA2_SET_FLC_RSC(flc);
3278 flc->word2_rflc_31_0 = lower_32_bits(
3279 (size_t)&(((struct dpaa2_sec_qp *)
3280 dev->data->queue_pairs[0])->rx_vq) | 0x14);
3281 flc->word3_rflc_63_32 = upper_32_bits(
3282 (size_t)&(((struct dpaa2_sec_qp *)
3283 dev->data->queue_pairs[0])->rx_vq));
3285 flc->word1_sdl = (uint8_t)bufsize;
3287 /* TODO - check the perf impact or
3288 * align as per descriptor type
3289 * Set EWS bit i.e. enable write-safe
3290 * DPAA2_SET_FLC_EWS(flc);
3293 /* Set BS = 1 i.e reuse input buffers as output buffers */
3294 DPAA2_SET_FLC_REUSE_BS(flc);
3295 /* Set FF = 10; reuse input buffers if they provide sufficient space */
3296 DPAA2_SET_FLC_REUSE_FF(flc);
3298 session->ctxt = priv;
3302 rte_free(session->auth_key.data);
3303 rte_free(session->cipher_key.data);
3309 dpaa2_sec_security_session_create(void *dev,
3310 struct rte_security_session_conf *conf,
3311 struct rte_security_session *sess,
3312 struct rte_mempool *mempool)
3314 void *sess_private_data;
3315 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
3318 if (rte_mempool_get(mempool, &sess_private_data)) {
3319 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3323 switch (conf->protocol) {
3324 case RTE_SECURITY_PROTOCOL_IPSEC:
3325 ret = dpaa2_sec_set_ipsec_session(cdev, conf,
3328 case RTE_SECURITY_PROTOCOL_MACSEC:
3330 case RTE_SECURITY_PROTOCOL_PDCP:
3331 ret = dpaa2_sec_set_pdcp_session(cdev, conf,
3338 DPAA2_SEC_ERR("Failed to configure session parameters");
3339 /* Return session to mempool */
3340 rte_mempool_put(mempool, sess_private_data);
3344 set_sec_session_private_data(sess, sess_private_data);
3349 /** Clear the memory of session so it doesn't leave key material behind */
3351 dpaa2_sec_security_session_destroy(void *dev __rte_unused,
3352 struct rte_security_session *sess)
3354 PMD_INIT_FUNC_TRACE();
3355 void *sess_priv = get_sec_session_private_data(sess);
3357 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3360 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3363 rte_free(s->cipher_key.data);
3364 rte_free(s->auth_key.data);
3365 memset(s, 0, sizeof(dpaa2_sec_session));
3366 set_sec_session_private_data(sess, NULL);
3367 rte_mempool_put(sess_mp, sess_priv);
3373 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev,
3374 struct rte_crypto_sym_xform *xform,
3375 struct rte_cryptodev_sym_session *sess,
3376 struct rte_mempool *mempool)
3378 void *sess_private_data;
3381 if (rte_mempool_get(mempool, &sess_private_data)) {
3382 DPAA2_SEC_ERR("Couldn't get object from session mempool");
3386 ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data);
3388 DPAA2_SEC_ERR("Failed to configure session parameters");
3389 /* Return session to mempool */
3390 rte_mempool_put(mempool, sess_private_data);
3394 set_sym_session_private_data(sess, dev->driver_id,
3400 /** Clear the memory of session so it doesn't leave key material behind */
3402 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev,
3403 struct rte_cryptodev_sym_session *sess)
3405 PMD_INIT_FUNC_TRACE();
3406 uint8_t index = dev->driver_id;
3407 void *sess_priv = get_sym_session_private_data(sess, index);
3408 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv;
3412 rte_free(s->cipher_key.data);
3413 rte_free(s->auth_key.data);
3414 memset(s, 0, sizeof(dpaa2_sec_session));
3415 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
3416 set_sym_session_private_data(sess, index, NULL);
3417 rte_mempool_put(sess_mp, sess_priv);
3422 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused,
3423 struct rte_cryptodev_config *config __rte_unused)
3425 PMD_INIT_FUNC_TRACE();
3431 dpaa2_sec_dev_start(struct rte_cryptodev *dev)
3433 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3434 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3435 struct dpseci_attr attr;
3436 struct dpaa2_queue *dpaa2_q;
3437 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3438 dev->data->queue_pairs;
3439 struct dpseci_rx_queue_attr rx_attr;
3440 struct dpseci_tx_queue_attr tx_attr;
3443 PMD_INIT_FUNC_TRACE();
3445 memset(&attr, 0, sizeof(struct dpseci_attr));
3447 ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token);
3449 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED",
3451 goto get_attr_failure;
3453 ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr);
3455 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC");
3456 goto get_attr_failure;
3458 for (i = 0; i < attr.num_rx_queues && qp[i]; i++) {
3459 dpaa2_q = &qp[i]->rx_vq;
3460 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3462 dpaa2_q->fqid = rx_attr.fqid;
3463 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid);
3465 for (i = 0; i < attr.num_tx_queues && qp[i]; i++) {
3466 dpaa2_q = &qp[i]->tx_vq;
3467 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i,
3469 dpaa2_q->fqid = tx_attr.fqid;
3470 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid);
3475 dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3480 dpaa2_sec_dev_stop(struct rte_cryptodev *dev)
3482 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3483 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3486 PMD_INIT_FUNC_TRACE();
3488 ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token);
3490 DPAA2_SEC_ERR("Failure in disabling dpseci %d device",
3495 ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token);
3497 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret);
3503 dpaa2_sec_dev_close(struct rte_cryptodev *dev)
3505 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3506 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3509 PMD_INIT_FUNC_TRACE();
3511 /* Function is reverse of dpaa2_sec_dev_init.
3512 * It does the following:
3513 * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id
3514 * 2. Close the DPSECI device
3515 * 3. Free the allocated resources.
3518 /*Close the device at underlying layer*/
3519 ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token);
3521 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret);
3525 /*Free the allocated memory for ethernet private data and dpseci*/
3533 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev,
3534 struct rte_cryptodev_info *info)
3536 struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3538 PMD_INIT_FUNC_TRACE();
3540 info->max_nb_queue_pairs = internals->max_nb_queue_pairs;
3541 info->feature_flags = dev->feature_flags;
3542 info->capabilities = dpaa2_sec_capabilities;
3543 /* No limit of number of sessions */
3544 info->sym.max_nb_sessions = 0;
3545 info->driver_id = cryptodev_driver_id;
3550 void dpaa2_sec_stats_get(struct rte_cryptodev *dev,
3551 struct rte_cryptodev_stats *stats)
3553 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3554 struct fsl_mc_io dpseci;
3555 struct dpseci_sec_counters counters = {0};
3556 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3557 dev->data->queue_pairs;
3560 PMD_INIT_FUNC_TRACE();
3561 if (stats == NULL) {
3562 DPAA2_SEC_ERR("Invalid stats ptr NULL");
3565 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3566 if (qp == NULL || qp[i] == NULL) {
3567 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3571 stats->enqueued_count += qp[i]->tx_vq.tx_pkts;
3572 stats->dequeued_count += qp[i]->rx_vq.rx_pkts;
3573 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts;
3574 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts;
3577 /* In case as secondary process access stats, MCP portal in priv-hw
3578 * may have primary process address. Need the secondary process
3579 * based MCP portal address for this object.
3581 dpseci.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3582 ret = dpseci_get_sec_counters(&dpseci, CMD_PRI_LOW, priv->token,
3585 DPAA2_SEC_ERR("SEC counters failed");
3587 DPAA2_SEC_INFO("dpseci hardware stats:"
3588 "\n\tNum of Requests Dequeued = %" PRIu64
3589 "\n\tNum of Outbound Encrypt Requests = %" PRIu64
3590 "\n\tNum of Inbound Decrypt Requests = %" PRIu64
3591 "\n\tNum of Outbound Bytes Encrypted = %" PRIu64
3592 "\n\tNum of Outbound Bytes Protected = %" PRIu64
3593 "\n\tNum of Inbound Bytes Decrypted = %" PRIu64
3594 "\n\tNum of Inbound Bytes Validated = %" PRIu64,
3595 counters.dequeued_requests,
3596 counters.ob_enc_requests,
3597 counters.ib_dec_requests,
3598 counters.ob_enc_bytes,
3599 counters.ob_prot_bytes,
3600 counters.ib_dec_bytes,
3601 counters.ib_valid_bytes);
3606 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev)
3609 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **)
3610 (dev->data->queue_pairs);
3612 PMD_INIT_FUNC_TRACE();
3614 for (i = 0; i < dev->data->nb_queue_pairs; i++) {
3615 if (qp[i] == NULL) {
3616 DPAA2_SEC_DEBUG("Uninitialised queue pair");
3619 qp[i]->tx_vq.rx_pkts = 0;
3620 qp[i]->tx_vq.tx_pkts = 0;
3621 qp[i]->tx_vq.err_pkts = 0;
3622 qp[i]->rx_vq.rx_pkts = 0;
3623 qp[i]->rx_vq.tx_pkts = 0;
3624 qp[i]->rx_vq.err_pkts = 0;
3628 static void __rte_hot
3629 dpaa2_sec_process_parallel_event(struct qbman_swp *swp,
3630 const struct qbman_fd *fd,
3631 const struct qbman_result *dq,
3632 struct dpaa2_queue *rxq,
3633 struct rte_event *ev)
3635 /* Prefetching mbuf */
3636 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3637 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3639 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3640 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3642 ev->flow_id = rxq->ev.flow_id;
3643 ev->sub_event_type = rxq->ev.sub_event_type;
3644 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3645 ev->op = RTE_EVENT_OP_NEW;
3646 ev->sched_type = rxq->ev.sched_type;
3647 ev->queue_id = rxq->ev.queue_id;
3648 ev->priority = rxq->ev.priority;
3649 ev->event_ptr = sec_fd_to_mbuf(fd);
3651 qbman_swp_dqrr_consume(swp, dq);
3654 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused,
3655 const struct qbman_fd *fd,
3656 const struct qbman_result *dq,
3657 struct dpaa2_queue *rxq,
3658 struct rte_event *ev)
3661 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr;
3662 /* Prefetching mbuf */
3663 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-
3664 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size));
3666 /* Prefetching ipsec crypto_op stored in priv data of mbuf */
3667 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64));
3669 ev->flow_id = rxq->ev.flow_id;
3670 ev->sub_event_type = rxq->ev.sub_event_type;
3671 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
3672 ev->op = RTE_EVENT_OP_NEW;
3673 ev->sched_type = rxq->ev.sched_type;
3674 ev->queue_id = rxq->ev.queue_id;
3675 ev->priority = rxq->ev.priority;
3677 ev->event_ptr = sec_fd_to_mbuf(fd);
3678 dqrr_index = qbman_get_dqrr_idx(dq);
3679 crypto_op->sym->m_src->seqn = dqrr_index + 1;
3680 DPAA2_PER_LCORE_DQRR_SIZE++;
3681 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
3682 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src;
3686 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev,
3688 struct dpaa2_dpcon_dev *dpcon,
3689 const struct rte_event *event)
3691 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3692 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3693 struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id];
3694 struct dpseci_rx_queue_cfg cfg;
3698 if (event->sched_type == RTE_SCHED_TYPE_PARALLEL)
3699 qp->rx_vq.cb = dpaa2_sec_process_parallel_event;
3700 else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC)
3701 qp->rx_vq.cb = dpaa2_sec_process_atomic_event;
3705 priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) *
3706 (dpcon->num_priorities - 1);
3708 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3709 cfg.options = DPSECI_QUEUE_OPT_DEST;
3710 cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON;
3711 cfg.dest_cfg.dest_id = dpcon->dpcon_id;
3712 cfg.dest_cfg.priority = priority;
3714 cfg.options |= DPSECI_QUEUE_OPT_USER_CTX;
3715 cfg.user_ctx = (size_t)(qp);
3716 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) {
3717 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION;
3718 cfg.order_preservation_en = 1;
3720 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3723 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3727 memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event));
3733 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev,
3736 struct dpaa2_sec_dev_private *priv = dev->data->dev_private;
3737 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw;
3738 struct dpseci_rx_queue_cfg cfg;
3741 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg));
3742 cfg.options = DPSECI_QUEUE_OPT_DEST;
3743 cfg.dest_cfg.dest_type = DPSECI_DEST_NONE;
3745 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token,
3748 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret);
3753 static struct rte_cryptodev_ops crypto_ops = {
3754 .dev_configure = dpaa2_sec_dev_configure,
3755 .dev_start = dpaa2_sec_dev_start,
3756 .dev_stop = dpaa2_sec_dev_stop,
3757 .dev_close = dpaa2_sec_dev_close,
3758 .dev_infos_get = dpaa2_sec_dev_infos_get,
3759 .stats_get = dpaa2_sec_stats_get,
3760 .stats_reset = dpaa2_sec_stats_reset,
3761 .queue_pair_setup = dpaa2_sec_queue_pair_setup,
3762 .queue_pair_release = dpaa2_sec_queue_pair_release,
3763 .sym_session_get_size = dpaa2_sec_sym_session_get_size,
3764 .sym_session_configure = dpaa2_sec_sym_session_configure,
3765 .sym_session_clear = dpaa2_sec_sym_session_clear,
3768 #ifdef RTE_LIBRTE_SECURITY
3769 static const struct rte_security_capability *
3770 dpaa2_sec_capabilities_get(void *device __rte_unused)
3772 return dpaa2_sec_security_cap;
3775 static const struct rte_security_ops dpaa2_sec_security_ops = {
3776 .session_create = dpaa2_sec_security_session_create,
3777 .session_update = NULL,
3778 .session_stats_get = NULL,
3779 .session_destroy = dpaa2_sec_security_session_destroy,
3780 .set_pkt_metadata = NULL,
3781 .capabilities_get = dpaa2_sec_capabilities_get
3786 dpaa2_sec_uninit(const struct rte_cryptodev *dev)
3788 struct dpaa2_sec_dev_private *internals = dev->data->dev_private;
3790 rte_free(dev->security_ctx);
3792 rte_mempool_free(internals->fle_pool);
3794 DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u",
3795 dev->data->name, rte_socket_id());
3801 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev)
3803 struct dpaa2_sec_dev_private *internals;
3804 struct rte_device *dev = cryptodev->device;
3805 struct rte_dpaa2_device *dpaa2_dev;
3806 #ifdef RTE_LIBRTE_SECURITY
3807 struct rte_security_ctx *security_instance;
3809 struct fsl_mc_io *dpseci;
3811 struct dpseci_attr attr;
3815 PMD_INIT_FUNC_TRACE();
3816 dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
3817 hw_id = dpaa2_dev->object_id;
3819 cryptodev->driver_id = cryptodev_driver_id;
3820 cryptodev->dev_ops = &crypto_ops;
3822 cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst;
3823 cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst;
3824 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
3825 RTE_CRYPTODEV_FF_HW_ACCELERATED |
3826 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
3827 RTE_CRYPTODEV_FF_SECURITY |
3828 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
3829 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
3830 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
3831 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT |
3832 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
3834 internals = cryptodev->data->dev_private;
3837 * For secondary processes, we don't initialise any further as primary
3838 * has already done this work. Only check we don't need a different
3841 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
3842 DPAA2_SEC_DEBUG("Device already init by primary process");
3845 #ifdef RTE_LIBRTE_SECURITY
3846 /* Initialize security_ctx only for primary process*/
3847 security_instance = rte_malloc("rte_security_instances_ops",
3848 sizeof(struct rte_security_ctx), 0);
3849 if (security_instance == NULL)
3851 security_instance->device = (void *)cryptodev;
3852 security_instance->ops = &dpaa2_sec_security_ops;
3853 security_instance->sess_cnt = 0;
3854 cryptodev->security_ctx = security_instance;
3856 /*Open the rte device via MC and save the handle for further use*/
3857 dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1,
3858 sizeof(struct fsl_mc_io), 0);
3861 "Error in allocating the memory for dpsec object");
3864 dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
3866 retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token);
3868 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x",
3872 retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr);
3875 "Cannot get dpsec device attributed: Error = %x",
3879 snprintf(cryptodev->data->name, sizeof(cryptodev->data->name),
3882 internals->max_nb_queue_pairs = attr.num_tx_queues;
3883 cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs;
3884 internals->hw = dpseci;
3885 internals->token = token;
3887 snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d",
3888 getpid(), cryptodev->data->dev_id);
3889 internals->fle_pool = rte_mempool_create((const char *)str,
3892 FLE_POOL_CACHE_SIZE, 0,
3893 NULL, NULL, NULL, NULL,
3895 if (!internals->fle_pool) {
3896 DPAA2_SEC_ERR("Mempool (%s) creation failed", str);
3900 DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name);
3904 DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name);
3906 /* dpaa2_sec_uninit(crypto_dev_name); */
3911 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused,
3912 struct rte_dpaa2_device *dpaa2_dev)
3914 struct rte_cryptodev *cryptodev;
3915 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
3919 snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d",
3920 dpaa2_dev->object_id);
3922 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id());
3923 if (cryptodev == NULL)
3926 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3927 cryptodev->data->dev_private = rte_zmalloc_socket(
3928 "cryptodev private structure",
3929 sizeof(struct dpaa2_sec_dev_private),
3930 RTE_CACHE_LINE_SIZE,
3933 if (cryptodev->data->dev_private == NULL)
3934 rte_panic("Cannot allocate memzone for private "
3938 dpaa2_dev->cryptodev = cryptodev;
3939 cryptodev->device = &dpaa2_dev->device;
3941 /* init user callbacks */
3942 TAILQ_INIT(&(cryptodev->link_intr_cbs));
3944 if (dpaa2_svr_family == SVR_LX2160A)
3945 rta_set_sec_era(RTA_SEC_ERA_10);
3947 rta_set_sec_era(RTA_SEC_ERA_8);
3949 DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era());
3951 /* Invoke PMD device initialization function */
3952 retval = dpaa2_sec_dev_init(cryptodev);
3956 if (rte_eal_process_type() == RTE_PROC_PRIMARY)
3957 rte_free(cryptodev->data->dev_private);
3959 cryptodev->attached = RTE_CRYPTODEV_DETACHED;
3965 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev)
3967 struct rte_cryptodev *cryptodev;
3970 cryptodev = dpaa2_dev->cryptodev;
3971 if (cryptodev == NULL)
3974 ret = dpaa2_sec_uninit(cryptodev);
3978 return rte_cryptodev_pmd_destroy(cryptodev);
3981 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = {
3982 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
3983 .drv_type = DPAA2_CRYPTO,
3985 .name = "DPAA2 SEC PMD"
3987 .probe = cryptodev_dpaa2_sec_probe,
3988 .remove = cryptodev_dpaa2_sec_remove,
3991 static struct cryptodev_driver dpaa2_sec_crypto_drv;
3993 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver);
3994 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv,
3995 rte_dpaa2_sec_driver.driver, cryptodev_driver_id);
3996 RTE_LOG_REGISTER(dpaa2_logtype_sec, pmd.crypto.dpaa2, NOTICE);