From 3394ed4715703729da89c45afce823559cb2b2b5 Mon Sep 17 00:00:00 2001 From: Vakul Garg Date: Mon, 14 Oct 2019 12:23:25 +0530 Subject: [PATCH] crypto/dpaax_sec: support auth trailer in cipher-auth For cases like ESN where authentication data can be after cipher data, support for authentication trailer is required. This patch adds support of auth-only data trailing after cipher data. Signed-off-by: Vakul Garg Acked-by: Akhil Goyal --- drivers/crypto/caam_jr/caam_jr.c | 24 +-- drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 17 +- drivers/crypto/dpaa2_sec/hw/desc/ipsec.h | 167 ++++++++------------ drivers/crypto/dpaa_sec/dpaa_sec.c | 35 +++- 4 files changed, 121 insertions(+), 122 deletions(-) diff --git a/drivers/crypto/caam_jr/caam_jr.c b/drivers/crypto/caam_jr/caam_jr.c index 57101d9a64..6ceba18f1a 100644 --- a/drivers/crypto/caam_jr/caam_jr.c +++ b/drivers/crypto/caam_jr/caam_jr.c @@ -450,13 +450,11 @@ caam_jr_prep_cdb(struct caam_jr_session *ses) &alginfo_c, &alginfo_a); } } else { - /* Auth_only_len is set as 0 here and it will be - * overwritten in fd for each packet. - */ + /* Auth_only_len is overwritten in fd for each job */ shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a, - ses->iv.length, 0, + ses->iv.length, ses->digest_length, ses->dir); } } @@ -1066,10 +1064,11 @@ build_cipher_auth_sg(struct rte_crypto_op *op, struct caam_jr_session *ses) uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, ses->iv.offset); struct sec_job_descriptor_t *jobdescr; - uint32_t auth_only_len; - - auth_only_len = op->sym->auth.data.length - - op->sym->cipher.data.length; + uint16_t auth_hdr_len = sym->cipher.data.offset - + sym->auth.data.offset; + uint16_t auth_tail_len = sym->auth.data.length - + sym->cipher.data.length - auth_hdr_len; + uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; if (sym->m_dst) { mbuf = sym->m_dst; @@ -1208,10 +1207,11 @@ build_cipher_auth(struct rte_crypto_op *op, struct caam_jr_session *ses) uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, ses->iv.offset); struct sec_job_descriptor_t *jobdescr; - uint32_t auth_only_len; - - auth_only_len = op->sym->auth.data.length - - op->sym->cipher.data.length; + uint16_t auth_hdr_len = sym->cipher.data.offset - + sym->auth.data.offset; + uint16_t auth_tail_len = sym->auth.data.length - + sym->cipher.data.length - auth_hdr_len; + uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; src_start_addr = rte_pktmbuf_iova(sym->m_src); if (sym->m_dst) diff --git a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c index 7efa08bcdf..1008164109 100644 --- a/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c +++ b/drivers/crypto/dpaa2_sec/dpaa2_sec_dpseci.c @@ -583,8 +583,11 @@ build_authenc_sg_fd(dpaa2_sec_session *sess, struct ctxt_priv *priv = sess->ctxt; struct qbman_fle *fle, *sge, *ip_fle, *op_fle; struct sec_flow_context *flc; - uint32_t auth_only_len = sym_op->auth.data.length - - sym_op->cipher.data.length; + uint16_t auth_hdr_len = sym_op->cipher.data.offset - + sym_op->auth.data.offset; + uint16_t auth_tail_len = sym_op->auth.data.length - + sym_op->cipher.data.length - auth_hdr_len; + uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; int icv_len = sess->digest_length; uint8_t *old_icv; struct rte_mbuf *mbuf; @@ -727,8 +730,12 @@ build_authenc_fd(dpaa2_sec_session *sess, struct ctxt_priv *priv = sess->ctxt; struct qbman_fle *fle, *sge; struct sec_flow_context *flc; - uint32_t auth_only_len = sym_op->auth.data.length - - sym_op->cipher.data.length; + uint16_t auth_hdr_len = sym_op->cipher.data.offset - + sym_op->auth.data.offset; + uint16_t auth_tail_len = sym_op->auth.data.length - + sym_op->cipher.data.length - auth_hdr_len; + uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; + int icv_len = sess->digest_length, retval; uint8_t *old_icv; uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, @@ -2217,7 +2224,6 @@ dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, dpaa2_sec_session *session) { - struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; struct alginfo authdata, cipherdata; int bufsize; @@ -2411,7 +2417,6 @@ dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, 0, SHR_SERIAL, &cipherdata, &authdata, session->iv.length, - ctxt->auth_only_len, session->digest_length, session->dir); if (bufsize < 0) { diff --git a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h index d071f46fda..d1ffd7fd23 100644 --- a/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h +++ b/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h @@ -1412,9 +1412,6 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps, * * @ivlen: length of the IV to be read from the input frame, before any data * to be processed - * @auth_only_len: length of the data to be authenticated-only (commonly IP - * header, IV, Sequence number and SPI) - * Note: Extended Sequence Number processing is NOT supported * * @trunc_len: the length of the ICV to be written to the output frame. If 0, * then the corresponding length of the digest, according to the @@ -1425,30 +1422,30 @@ cnstr_shdsc_ipsec_new_decap(uint32_t *descbuf, bool ps, * will be done correctly: * For encapsulation: * Input: - * +----+----------------+---------------------------------------------+ - * | IV | Auth-only data | Padded data to be authenticated & Encrypted | - * +----+----------------+---------------------------------------------+ + * +----+----------------+-----------------------------------------------+ + * | IV | Auth-only head | Padded data to be auth & Enc | Auth-only tail | + * +----+----------------+-----------------------------------------------+ * Output: * +--------------------------------------+ * | Authenticated & Encrypted data | ICV | * +--------------------------------+-----+ - + * * For decapsulation: * Input: - * +----+----------------+--------------------------------+-----+ - * | IV | Auth-only data | Authenticated & Encrypted data | ICV | - * +----+----------------+--------------------------------+-----+ + * +----+----------------+-----------------+----------------------+ + * | IV | Auth-only head | Auth & Enc data | Auth-only tail | ICV | + * +----+----------------+-----------------+----------------------+ * Output: - * +----+--------------------------+ + * +----+---------------------------+ * | Decrypted & authenticated data | - * +----+--------------------------+ + * +----+---------------------------+ * * Note: This descriptor can use per-packet commands, encoded as below in the * DPOVRD register: - * 32 24 16 0 - * +------+---------------------+ - * | 0x80 | 0x00| auth_only_len | - * +------+---------------------+ + * 32 28 16 1 + * +------+------------------------------+ + * | 0x8 | auth_tail_len | auth_hdr_len | + * +------+------------------------------+ * * This mechanism is available only for SoCs having SEC ERA >= 3. In other * words, this will not work for P4080TO2 @@ -1465,7 +1462,7 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, enum rta_share_type share, struct alginfo *cipherdata, struct alginfo *authdata, - uint16_t ivlen, uint16_t auth_only_len, + uint16_t ivlen, uint8_t trunc_len, uint8_t dir) { struct program prg; @@ -1473,16 +1470,16 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, const bool need_dk = (dir == DIR_DEC) && (cipherdata->algtype == OP_ALG_ALGSEL_AES) && (cipherdata->algmode == OP_ALG_AAI_CBC); + int data_type; - LABEL(skip_patch_len); LABEL(keyjmp); LABEL(skipkeys); - LABEL(aonly_len_offset); - REFERENCE(pskip_patch_len); + LABEL(proc_icv); + LABEL(no_auth_tail); REFERENCE(pkeyjmp); REFERENCE(pskipkeys); - REFERENCE(read_len); - REFERENCE(write_len); + REFERENCE(p_proc_icv); + REFERENCE(p_no_auth_tail); PROGRAM_CNTXT_INIT(p, descbuf, 0); @@ -1500,48 +1497,15 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, SHR_HDR(p, share, 1, SC); - /* - * M0 will contain the value provided by the user when creating - * the shared descriptor. If the user provided an override in - * DPOVRD, then M0 will contain that value - */ - MATHB(p, MATH0, ADD, auth_only_len, MATH0, 4, IMMED2); - - if (rta_sec_era >= RTA_SEC_ERA_3) { - /* - * Check if the user wants to override the auth-only len - */ - MATHB(p, DPOVRD, ADD, 0x80000000, MATH2, 4, IMMED2); - - /* - * No need to patch the length of the auth-only data read if - * the user did not override it - */ - pskip_patch_len = JUMP(p, skip_patch_len, LOCAL_JUMP, ALL_TRUE, - MATH_N); - - /* Get auth-only len in M0 */ - MATHB(p, MATH2, AND, 0xFFFF, MATH0, 4, IMMED2); - - /* - * Since M0 is used in calculations, don't mangle it, copy - * its content to M1 and use this for patching. - */ - MATHB(p, MATH0, ADD, MATH1, MATH1, 4, 0); - - read_len = MOVE(p, DESCBUF, 0, MATH1, 0, 6, WAITCOMP | IMMED); - write_len = MOVE(p, MATH1, 0, DESCBUF, 0, 8, WAITCOMP | IMMED); - - SET_LABEL(p, skip_patch_len); - } - /* - * MATH0 contains the value in DPOVRD w/o the MSB, or the initial - * value, as provided by the user at descriptor creation time - */ - if (dir == DIR_ENC) - MATHB(p, MATH0, ADD, ivlen, MATH0, 4, IMMED2); - else - MATHB(p, MATH0, ADD, ivlen + trunc_len, MATH0, 4, IMMED2); + /* Collect the (auth_tail || auth_hdr) len from DPOVRD */ + MATHB(p, DPOVRD, ADD, 0x80000000, MATH2, 4, IMMED2); + + /* Get auth_hdr len in MATH0 */ + MATHB(p, MATH2, AND, 0xFFFF, MATH0, 4, IMMED2); + + /* Get auth_tail len in MATH2 */ + MATHB(p, MATH2, AND, 0xFFF0000, MATH2, 4, IMMED2); + MATHI(p, MATH2, RSHIFT, 16, MATH2, 4, IMMED2); pkeyjmp = JUMP(p, keyjmp, LOCAL_JUMP, ALL_TRUE, SHRD); @@ -1581,61 +1545,70 @@ cnstr_shdsc_authenc(uint32_t *descbuf, bool ps, bool swap, OP_ALG_AS_INITFINAL, ICV_CHECK_DISABLE, dir); } + /* Read IV */ + if (cipherdata->algmode == OP_ALG_AAI_CTR) + SEQLOAD(p, CONTEXT1, 16, ivlen, 0); + else + SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + + /* + * authenticate auth_hdr data + */ + MATHB(p, MATH0, ADD, ZERO, VSEQINSZ, 4, 0); + SEQFIFOLOAD(p, MSG2, 0, VLF); + /* * Prepare the length of the data to be both encrypted/decrypted * and authenticated/checked */ - MATHB(p, SEQINSZ, SUB, MATH0, VSEQINSZ, 4, 0); + MATHB(p, SEQINSZ, SUB, MATH2, VSEQINSZ, 4, 0); + if (dir == DIR_DEC) { + MATHB(p, VSEQINSZ, SUB, trunc_len, VSEQINSZ, 4, IMMED2); + data_type = MSGINSNOOP; + } else { + data_type = MSGOUTSNOOP; + } - MATHB(p, VSEQINSZ, SUB, MATH3, VSEQOUTSZ, 4, 0); + MATHB(p, VSEQINSZ, ADD, ZERO, VSEQOUTSZ, 4, 0); /* Prepare for writing the output frame */ SEQFIFOSTORE(p, MSG, 0, 0, VLF); - SET_LABEL(p, aonly_len_offset); - /* Read IV */ - if (cipherdata->algmode == OP_ALG_AAI_CTR) - SEQLOAD(p, CONTEXT1, 16, ivlen, 0); - else - SEQLOAD(p, CONTEXT1, 0, ivlen, 0); + /* Check if there is no auth-tail */ + MATHB(p, MATH2, ADD, ZERO, MATH2, 4, 0); + p_no_auth_tail = JUMP(p, no_auth_tail, LOCAL_JUMP, ALL_TRUE, MATH_Z); /* - * Read data needed only for authentication. This is overwritten above - * if the user requested it. + * Read input plain/cipher text, encrypt/decrypt & auth & write + * to output */ - SEQFIFOLOAD(p, MSG2, auth_only_len, 0); + SEQFIFOLOAD(p, data_type, 0, VLF | LAST1 | FLUSH1); + + /* Authenticate auth tail */ + MATHB(p, MATH2, ADD, ZERO, VSEQINSZ, 4, 0); + SEQFIFOLOAD(p, MSG2, 0, VLF | LAST2); + + /* Jump to process icv */ + p_proc_icv = JUMP(p, proc_icv, LOCAL_JUMP, ALL_FALSE, MATH_Z); + + SET_LABEL(p, no_auth_tail); - if (dir == DIR_ENC) { - /* - * Read input plaintext, encrypt and authenticate & write to - * output - */ - SEQFIFOLOAD(p, MSGOUTSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1); + SEQFIFOLOAD(p, data_type, 0, VLF | LAST1 | LAST2 | FLUSH1); + SET_LABEL(p, proc_icv); + + if (dir == DIR_ENC) /* Finally, write the ICV */ SEQSTORE(p, CONTEXT2, 0, trunc_len, 0); - } else { - /* - * Read input ciphertext, decrypt and authenticate & write to - * output - */ - SEQFIFOLOAD(p, MSGINSNOOP, 0, VLF | LAST1 | LAST2 | FLUSH1); - + else /* Read the ICV to check */ SEQFIFOLOAD(p, ICV2, trunc_len, LAST2); - } PATCH_JUMP(p, pkeyjmp, keyjmp); PATCH_JUMP(p, pskipkeys, skipkeys); - PATCH_JUMP(p, pskipkeys, skipkeys); - - if (rta_sec_era >= RTA_SEC_ERA_3) { - PATCH_JUMP(p, pskip_patch_len, skip_patch_len); - PATCH_MOVE(p, read_len, aonly_len_offset); - PATCH_MOVE(p, write_len, aonly_len_offset); - } - + PATCH_JUMP(p, p_no_auth_tail, no_auth_tail); + PATCH_JUMP(p, p_proc_icv, proc_icv); return PROGRAM_FINALIZE(p); } diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c index 3c7c4fad13..3edaa887a9 100644 --- a/drivers/crypto/dpaa_sec/dpaa_sec.c +++ b/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -742,7 +742,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) */ shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a, - ses->iv.length, 0, + ses->iv.length, ses->digest_length, ses->dir); } @@ -1753,7 +1753,8 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, struct rte_crypto_op *op; struct dpaa_sec_job *cf; dpaa_sec_session *ses; - uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0}; + uint16_t auth_hdr_len, auth_tail_len; + uint32_t index, flags[DPAA_SEC_BURST] = {0}; struct qman_fq *inq[DPAA_SEC_BURST]; while (nb_ops) { @@ -1809,8 +1810,10 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, goto send_pkts; } - auth_only_len = op->sym->auth.data.length - + auth_hdr_len = op->sym->auth.data.length - op->sym->cipher.data.length; + auth_tail_len = 0; + if (rte_pktmbuf_is_contiguous(op->sym->m_src) && ((op->sym->m_dst == NULL) || rte_pktmbuf_is_contiguous(op->sym->m_dst))) { @@ -1824,8 +1827,15 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, cf = build_cipher_only(op, ses); } else if (is_aead(ses)) { cf = build_cipher_auth_gcm(op, ses); - auth_only_len = ses->auth_only_len; + auth_hdr_len = ses->auth_only_len; } else if (is_auth_cipher(ses)) { + auth_hdr_len = + op->sym->cipher.data.offset + - op->sym->auth.data.offset; + auth_tail_len = + op->sym->auth.data.length + - op->sym->cipher.data.length + - auth_hdr_len; cf = build_cipher_auth(op, ses); } else { DPAA_SEC_DP_ERR("not supported ops"); @@ -1842,8 +1852,15 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, cf = build_cipher_only_sg(op, ses); } else if (is_aead(ses)) { cf = build_cipher_auth_gcm_sg(op, ses); - auth_only_len = ses->auth_only_len; + auth_hdr_len = ses->auth_only_len; } else if (is_auth_cipher(ses)) { + auth_hdr_len = + op->sym->cipher.data.offset + - op->sym->auth.data.offset; + auth_tail_len = + op->sym->auth.data.length + - op->sym->cipher.data.length + - auth_hdr_len; cf = build_cipher_auth_sg(op, ses); } else { DPAA_SEC_DP_ERR("not supported ops"); @@ -1865,12 +1882,16 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg)); fd->_format1 = qm_fd_compound; fd->length29 = 2 * sizeof(struct qm_sg_entry); + /* Auth_only_len is set as 0 in descriptor and it is * overwritten here in the fd.cmd which will update * the DPOVRD reg. */ - if (auth_only_len) - fd->cmd = 0x80000000 | auth_only_len; + if (auth_hdr_len || auth_tail_len) { + fd->cmd = 0x80000000; + fd->cmd |= + ((auth_tail_len << 16) | auth_hdr_len); + } /* In case of PDCP, per packet HFN is stored in * mbuf priv after sym_op. -- 2.20.1