X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fdpaa_sec%2Fdpaa_sec.c;h=454b9c47857fcf5da349c10901381054b41ac714;hb=25d703151d3c1183c29623dad24b54b48ddcfba0;hp=dc528eed30d551175ae828d4ad43e40100ba81d5;hpb=5906b0ad0f7550a14d6c4c7dbcb6919a5352007f;p=dpdk.git diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c index dc528eed30..454b9c4785 100644 --- a/drivers/crypto/dpaa_sec/dpaa_sec.c +++ b/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2017-2019 NXP + * Copyright 2017-2021 NXP * */ @@ -12,14 +12,15 @@ #include #include -#include +#include #include #include -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY #include #endif #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include #include +#include #include #include @@ -43,17 +45,7 @@ #include #include -enum rta_sec_era rta_sec_era; - -int dpaa_logtype_sec; - -static uint8_t cryptodev_driver_id; - -static __thread struct rte_crypto_op **dpaa_sec_ops; -static __thread int dpaa_sec_op_nb; - -static int -dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess); +uint8_t dpaa_cryptodev_driver_id; static inline void dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) @@ -94,31 +86,6 @@ dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count) return ctx; } -static inline rte_iova_t -dpaa_mem_vtop(void *vaddr) -{ - const struct rte_memseg *ms; - - ms = rte_mem_virt2memseg(vaddr, NULL); - if (ms) { - dpaax_iova_table_update(ms->iova, ms->addr, ms->len); - return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); - } - return (size_t)NULL; -} - -static inline void * -dpaa_mem_ptov(rte_iova_t paddr) -{ - void *va; - - va = (void *)dpaax_iova_table_get_va(paddr); - if (likely(va)) - return va; - - return rte_mem_iova2virt(paddr); -} - static void ern_sec_fq_handler(struct qman_portal *qm __rte_unused, struct qman_fq *fq, @@ -148,7 +115,7 @@ dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); fq_opts.fqd.context_b = fqid_out; - fq_opts.fqd.dest.channel = qm_channel_caam; + fq_opts.fqd.dest.channel = dpaa_get_qm_channel_caam(); fq_opts.fqd.dest.wq = 0; fq_in->cb.ern = ern_sec_fq_handler; @@ -172,7 +139,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, struct dpaa_sec_job *job; struct dpaa_sec_op_ctx *ctx; - if (dpaa_sec_op_nb >= DPAA_SEC_BURST) + if (DPAA_PER_LCORE_DPAA_SEC_OP_NB >= DPAA_SEC_BURST) return qman_cb_dqrr_defer; if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) @@ -183,7 +150,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -203,7 +170,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, } mbuf->data_len = len; } - dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; + DPAA_PER_LCORE_RTE_CRYPTO_OP[DPAA_PER_LCORE_DPAA_SEC_OP_NB++] = ctx->op; dpaa_sec_op_ending(ctx); return qman_cb_dqrr_consume; @@ -244,6 +211,13 @@ dpaa_sec_init_tx(struct qman_fq *fq) return ret; } +static inline int is_aead(dpaa_sec_session *ses) +{ + return ((ses->cipher_alg == 0) && + (ses->auth_alg == 0) && + (ses->aead_alg != 0)); +} + static inline int is_encode(dpaa_sec_session *ses) { return ses->dir == DIR_ENC; @@ -254,7 +228,7 @@ static inline int is_decode(dpaa_sec_session *ses) return ses->dir == DIR_DEC; } -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY static int dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) { @@ -262,7 +236,6 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) struct sec_cdb *cdb = &ses->cdb; struct alginfo *p_authdata = NULL; int32_t shared_desc_len = 0; - int err; #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN int swap = false; #else @@ -276,10 +249,6 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) cipherdata.algtype = ses->cipher_key.alg; cipherdata.algmode = ses->cipher_key.algmode; - cdb->sh_desc[0] = cipherdata.keylen; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; - if (ses->auth_alg) { authdata.key = (size_t)ses->auth_key.data; authdata.keylen = ses->auth_key.length; @@ -289,34 +258,35 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) authdata.algmode = ses->auth_key.algmode; p_authdata = &authdata; - - cdb->sh_desc[1] = authdata.keylen; - } - - err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, - (unsigned int *)cdb->sh_desc, - &cdb->sh_desc[2], 2); - if (err < 0) { - DPAA_SEC_ERR("Crypto: Incorrect key lengths"); - return err; } - if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) { - cipherdata.key = - (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key); - cipherdata.key_type = RTA_DATA_PTR; - } - if (!(cdb->sh_desc[2] & (1 << 1)) && authdata.keylen) { - authdata.key = - (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key); - authdata.key_type = RTA_DATA_PTR; + if (ses->pdcp.sdap_enabled) { + int nb_keys_to_inline = + rta_inline_pdcp_sdap_query(authdata.algtype, + cipherdata.algtype, + ses->pdcp.sn_size, + ses->pdcp.hfn_ovd); + if (nb_keys_to_inline >= 1) { + cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } + if (nb_keys_to_inline >= 2) { + authdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)authdata.key); + authdata.key_type = RTA_DATA_PTR; + } + } else { + if (rta_inline_pdcp_query(authdata.algtype, + cipherdata.algtype, + ses->pdcp.sn_size, + ses->pdcp.hfn_ovd)) { + cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } } - cdb->sh_desc[0] = 0; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; - if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) { if (ses->dir == DIR_ENC) shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap( @@ -338,25 +308,53 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) ses->pdcp.hfn_threshold, &cipherdata, &authdata, 0); + } else if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) { + shared_desc_len = cnstr_shdsc_pdcp_short_mac(cdb->sh_desc, + 1, swap, &authdata); } else { - if (ses->dir == DIR_ENC) - shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap( - cdb->sh_desc, 1, swap, - ses->pdcp.sn_size, - ses->pdcp.hfn, - ses->pdcp.bearer, - ses->pdcp.pkt_dir, - ses->pdcp.hfn_threshold, - &cipherdata, p_authdata, 0); - else if (ses->dir == DIR_DEC) - shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap( - cdb->sh_desc, 1, swap, - ses->pdcp.sn_size, - ses->pdcp.hfn, - ses->pdcp.bearer, - ses->pdcp.pkt_dir, - ses->pdcp.hfn_threshold, - &cipherdata, p_authdata, 0); + if (ses->dir == DIR_ENC) { + if (ses->pdcp.sdap_enabled) + shared_desc_len = + cnstr_shdsc_pdcp_sdap_u_plane_encap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata, 0); + else + shared_desc_len = + cnstr_shdsc_pdcp_u_plane_encap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata, 0); + } else if (ses->dir == DIR_DEC) { + if (ses->pdcp.sdap_enabled) + shared_desc_len = + cnstr_shdsc_pdcp_sdap_u_plane_decap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata, 0); + else + shared_desc_len = + cnstr_shdsc_pdcp_u_plane_decap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata, 0); + } } return shared_desc_len; } @@ -394,7 +392,7 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) cdb->sh_desc[0] = cipherdata.keylen; cdb->sh_desc[1] = authdata.keylen; err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, + DESC_JOB_IO_LEN, (unsigned int *)cdb->sh_desc, &cdb->sh_desc[2], 2); @@ -405,14 +403,14 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) if (cdb->sh_desc[2] & 1) cipherdata.key_type = RTA_DATA_IMM; else { - cipherdata.key = (size_t)dpaa_mem_vtop( + cipherdata.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)cipherdata.key); cipherdata.key_type = RTA_DATA_PTR; } if (cdb->sh_desc[2] & (1<<1)) authdata.key_type = RTA_DATA_IMM; else { - authdata.key = (size_t)dpaa_mem_vtop( + authdata.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)authdata.key); authdata.key_type = RTA_DATA_PTR; } @@ -454,7 +452,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) memset(cdb, 0, sizeof(struct sec_cdb)); switch (ses->ctxt) { -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY case DPAA_SEC_IPSEC: shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses); break; @@ -473,6 +471,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) switch (ses->cipher_alg) { case RTE_CRYPTO_CIPHER_AES_CBC: case RTE_CRYPTO_CIPHER_3DES_CBC: + case RTE_CRYPTO_CIPHER_DES_CBC: case RTE_CRYPTO_CIPHER_AES_CTR: case RTE_CRYPTO_CIPHER_3DES_CTR: shared_desc_len = cnstr_shdsc_blkcipher( @@ -507,6 +506,18 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) alginfo_a.algtype = ses->auth_key.alg; alginfo_a.algmode = ses->auth_key.algmode; switch (ses->auth_alg) { + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_SHA512: + shared_desc_len = cnstr_shdsc_hash( + cdb->sh_desc, true, + swap, SHR_NEVER, &alginfo_a, + !ses->dir, + ses->digest_length); + break; case RTE_CRYPTO_AUTH_MD5_HMAC: case RTE_CRYPTO_AUTH_SHA1_HMAC: case RTE_CRYPTO_AUTH_SHA224_HMAC: @@ -533,6 +544,15 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) !ses->dir, ses->digest_length); break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_AES_CMAC: + shared_desc_len = cnstr_shdsc_aes_mac( + cdb->sh_desc, + true, swap, SHR_NEVER, + &alginfo_a, + !ses->dir, + ses->digest_length); + break; default: DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg); } @@ -580,7 +600,7 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) cdb->sh_desc[0] = alginfo_c.keylen; cdb->sh_desc[1] = alginfo_a.keylen; err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, + DESC_JOB_IO_LEN, (unsigned int *)cdb->sh_desc, &cdb->sh_desc[2], 2); @@ -591,14 +611,14 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) if (cdb->sh_desc[2] & 1) alginfo_c.key_type = RTA_DATA_IMM; else { - alginfo_c.key = (size_t)dpaa_mem_vtop( + alginfo_c.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)alginfo_c.key); alginfo_c.key_type = RTA_DATA_PTR; } if (cdb->sh_desc[2] & (1<<1)) alginfo_a.key_type = RTA_DATA_IMM; else { - alginfo_a.key = (size_t)dpaa_mem_vtop( + alginfo_a.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)alginfo_a.key); alginfo_a.key_type = RTA_DATA_PTR; } @@ -674,7 +694,7 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -768,7 +788,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) in_sg->extension = 1; in_sg->final = 1; in_sg->length = data_len; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2])); /* 1st seg */ sg = in_sg + 1; @@ -788,13 +808,13 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) } else { sg->length = ses->iv.length; } - qm_sg_entry_set64(sg, dpaa_mem_vtop(iv_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr)); in_sg->length += sg->length; cpu_to_hw_sg(sg); sg++; } - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = data_offset; if (data_len <= (mbuf->data_len - data_offset)) { @@ -807,7 +827,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) (mbuf = mbuf->next)) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); if (data_len > mbuf->data_len) sg->length = mbuf->data_len; else @@ -821,7 +841,7 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; rte_memcpy(old_digest, sym->auth.digest.data, ses->digest_length); - start_addr = dpaa_mem_vtop(old_digest); + start_addr = rte_dpaa_mem_vtop(old_digest); qm_sg_entry_set64(sg, start_addr); sg->length = ses->digest_length; in_sg->length += ses->digest_length; @@ -888,7 +908,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) in_sg->extension = 1; in_sg->final = 1; in_sg->length = data_len; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2])); sg = &cf->sg[2]; if (ses->iv.length) { @@ -906,13 +926,13 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) } else { sg->length = ses->iv.length; } - qm_sg_entry_set64(sg, dpaa_mem_vtop(iv_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr)); in_sg->length += sg->length; cpu_to_hw_sg(sg); sg++; } - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = data_offset; sg->length = data_len; @@ -923,7 +943,7 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) rte_memcpy(old_digest, sym->auth.digest.data, ses->digest_length); /* let's check digest by hw */ - start_addr = dpaa_mem_vtop(old_digest); + start_addr = rte_dpaa_mem_vtop(old_digest); sg++; qm_sg_entry_set64(sg, start_addr); sg->length = ses->digest_length; @@ -987,12 +1007,12 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) out_sg = &cf->sg[0]; out_sg->extension = 1; out_sg->length = data_len; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); cpu_to_hw_sg(out_sg); /* 1st seg */ sg = &cf->sg[2]; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - data_offset; sg->offset = data_offset; @@ -1001,7 +1021,7 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1016,17 +1036,17 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) in_sg->length = data_len + ses->iv.length; sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 1st seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - data_offset; sg->offset = data_offset; @@ -1035,7 +1055,7 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1098,11 +1118,11 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) sg->extension = 1; sg->final = 1; sg->length = data_len + ses->iv.length; - qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(&cf->sg[2])); cpu_to_hw_sg(sg); sg = &cf->sg[2]; - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); @@ -1163,11 +1183,11 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output sg entries */ sg = &cf->sg[2]; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(out_sg); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->aead.data.offset; sg->offset = sym->aead.data.offset; @@ -1176,7 +1196,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1206,25 +1226,25 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input sg entries */ sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* 1st seg IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 2nd seg auth only */ if (ses->auth_only_len) { sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; cpu_to_hw_sg(sg); } /* 3rd seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->aead.data.offset; sg->offset = sym->aead.data.offset; @@ -1233,7 +1253,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1243,7 +1263,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; memcpy(ctx->digest, sym->aead.digest.data, ses->digest_length); - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; } sg->final = 1; @@ -1281,9 +1301,9 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input */ rte_prefetch0(cf->sg); sg = &cf->sg[2]; - qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg)); if (is_encode(ses)) { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1291,7 +1311,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; if (ses->auth_only_len) { qm_sg_entry_set64(sg, - dpaa_mem_vtop(sym->aead.aad.data)); + rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; length += sg->length; cpu_to_hw_sg(sg); @@ -1303,7 +1323,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg->final = 1; cpu_to_hw_sg(sg); } else { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1311,7 +1331,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; if (ses->auth_only_len) { qm_sg_entry_set64(sg, - dpaa_mem_vtop(sym->aead.aad.data)); + rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; length += sg->length; cpu_to_hw_sg(sg); @@ -1326,7 +1346,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) ses->digest_length); sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; length += sg->length; sg->final = 1; @@ -1340,7 +1360,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ sg++; - qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg)); qm_sg_entry_set64(sg, dst_start_addr + sym->aead.data.offset); sg->length = sym->aead.data.length; @@ -1409,11 +1429,11 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output sg entries */ sg = &cf->sg[2]; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(out_sg); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->auth.data.offset; sg->offset = sym->auth.data.offset; @@ -1422,7 +1442,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1451,17 +1471,17 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input sg entries */ sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* 1st seg IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 2nd seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->auth.data.offset; sg->offset = sym->auth.data.offset; @@ -1470,7 +1490,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1481,7 +1501,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; memcpy(ctx->digest, sym->auth.digest.data, ses->digest_length); - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; } sg->final = 1; @@ -1518,9 +1538,9 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input */ rte_prefetch0(cf->sg); sg = &cf->sg[2]; - qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg)); if (is_encode(ses)) { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1532,7 +1552,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) sg->final = 1; cpu_to_hw_sg(sg); } else { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1548,7 +1568,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) ses->digest_length); sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; length += sg->length; sg->final = 1; @@ -1562,7 +1582,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ sg++; - qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg)); qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); sg->length = sym->cipher.data.length; length = sg->length; @@ -1585,7 +1605,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) return cf; } -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY static inline struct dpaa_sec_job * build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) { @@ -1601,10 +1621,10 @@ build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) cf = &ctx->job; ctx->op = op; - src_start_addr = rte_pktmbuf_mtophys(sym->m_src); + src_start_addr = rte_pktmbuf_iova(sym->m_src); if (sym->m_dst) - dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst); + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); else dst_start_addr = src_start_addr; @@ -1656,11 +1676,11 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ out_sg = &cf->sg[0]; out_sg->extension = 1; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); /* 1st seg */ sg = &cf->sg[2]; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = 0; /* Successive segs */ @@ -1670,7 +1690,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) mbuf = mbuf->next; cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = 0; } sg->length = mbuf->buf_len - mbuf->data_off; @@ -1689,10 +1709,10 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) in_len = mbuf->data_len; sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; sg->offset = 0; @@ -1701,7 +1721,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; sg->offset = 0; in_len += sg->length; @@ -1736,13 +1756,20 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, uint32_t index, flags[DPAA_SEC_BURST] = {0}; struct qman_fq *inq[DPAA_SEC_BURST]; + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { + if (rte_dpaa_portal_init((void *)0)) { + DPAA_SEC_ERR("Failure in affining portal"); + return 0; + } + } + while (nb_ops) { frames_to_send = (nb_ops > DPAA_SEC_BURST) ? DPAA_SEC_BURST : nb_ops; for (loop = 0; loop < frames_to_send; loop++) { op = *(ops++); - if (op->sym->m_src->seqn != 0) { - index = op->sym->m_src->seqn - 1; + if (*dpaa_seqn(op->sym->m_src) != 0) { + index = *dpaa_seqn(op->sym->m_src) - 1; if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { /* QM_EQCR_DCA_IDXMASK = 0x0f */ flags[loop] = ((index & 0x0f) << 8); @@ -1757,10 +1784,10 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, case RTE_CRYPTO_OP_WITH_SESSION: ses = (dpaa_sec_session *) get_sym_session_private_data( - op->sym->session, - cryptodev_driver_id); + op->sym->session, + dpaa_cryptodev_driver_id); break; -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY case RTE_CRYPTO_OP_SECURITY_SESSION: ses = (dpaa_sec_session *) get_sec_session_private_data( @@ -1807,7 +1834,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, ((op->sym->m_dst == NULL) || rte_pktmbuf_is_contiguous(op->sym->m_dst))) { switch (ses->ctxt) { -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY case DPAA_SEC_PDCP: case DPAA_SEC_IPSEC: cf = build_proto(op, ses); @@ -1841,7 +1868,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, } } else { switch (ses->ctxt) { -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY case DPAA_SEC_PDCP: case DPAA_SEC_IPSEC: cf = build_proto_sg(op, ses); @@ -1884,7 +1911,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES]; fd->opaque_addr = 0; fd->cmd = 0; - qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg)); + qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg)); fd->_format1 = qm_fd_compound; fd->length29 = 2 * sizeof(struct qm_sg_entry); @@ -1898,7 +1925,7 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, ((auth_tail_len << 16) | auth_hdr_len); } -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY /* In case of PDCP, per packet HFN is stored in * mbuf priv after sym_op. */ @@ -1936,6 +1963,13 @@ dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, uint16_t num_rx; struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { + if (rte_dpaa_portal_init((void *)0)) { + DPAA_SEC_ERR("Failure in affining portal"); + return 0; + } + } + num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); dpaa_qp->rx_pkts += num_rx; @@ -2015,15 +2049,6 @@ dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, return 0; } -/** Return the number of allocated queue pairs */ -static uint32_t -dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) -{ - PMD_INIT_FUNC_TRACE(); - - return dev->data->nb_queue_pairs; -} - /** Returns the size of session structure */ static unsigned int dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) @@ -2057,6 +2082,10 @@ dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, session->cipher_key.alg = OP_ALG_ALGSEL_AES; session->cipher_key.algmode = OP_ALG_AAI_CBC; break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; case RTE_CRYPTO_CIPHER_3DES_CBC: session->cipher_key.alg = OP_ALG_ALGSEL_3DES; session->cipher_key.algmode = OP_ALG_AAI_CBC; @@ -2074,8 +2103,7 @@ dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, default: DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", xform->cipher.algo); - rte_free(session->cipher_key.data); - return -1; + return -ENOTSUP; } session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? DIR_ENC : DIR_DEC; @@ -2090,43 +2118,70 @@ dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, { session->ctxt = DPAA_SEC_AUTH; session->auth_alg = xform->auth.algo; - session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + session->auth_key.length = xform->auth.key.length; + if (xform->auth.key.length) { + session->auth_key.data = + rte_zmalloc(NULL, xform->auth.key.length, RTE_CACHE_LINE_SIZE); - if (session->auth_key.data == NULL && xform->auth.key.length > 0) { - DPAA_SEC_ERR("No Memory for auth key"); - return -ENOMEM; + if (session->auth_key.data == NULL) { + DPAA_SEC_ERR("No Memory for auth key"); + return -ENOMEM; + } + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + } - session->auth_key.length = xform->auth.key.length; session->digest_length = xform->auth.digest_length; if (session->cipher_alg == RTE_CRYPTO_CIPHER_NULL) { session->iv.offset = xform->auth.iv.offset; session->iv.length = xform->auth.iv.length; } - memcpy(session->auth_key.data, xform->auth.key.data, - xform->auth.key.length); - switch (xform->auth.algo) { + case RTE_CRYPTO_AUTH_SHA1: + session->auth_key.alg = OP_ALG_ALGSEL_SHA1; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_SHA1_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_SHA1; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_MD5: + session->auth_key.alg = OP_ALG_ALGSEL_MD5; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_MD5_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_MD5; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_SHA224: + session->auth_key.alg = OP_ALG_ALGSEL_SHA224; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_SHA224_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_SHA224; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_SHA256: + session->auth_key.alg = OP_ALG_ALGSEL_SHA256; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_SHA256_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_SHA256; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_SHA384: + session->auth_key.alg = OP_ALG_ALGSEL_SHA384; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_SHA384_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_SHA384; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_SHA512: + session->auth_key.alg = OP_ALG_ALGSEL_SHA512; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; case RTE_CRYPTO_AUTH_SHA512_HMAC: session->auth_key.alg = OP_ALG_ALGSEL_SHA512; session->auth_key.algmode = OP_ALG_AAI_HMAC; @@ -2139,11 +2194,18 @@ dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, session->auth_key.alg = OP_ALG_ALGSEL_ZUCA; session->auth_key.algmode = OP_ALG_AAI_F9; break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_CMAC; + break; default: DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u", xform->auth.algo); - rte_free(session->auth_key.data); - return -1; + return -ENOTSUP; } session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? @@ -2178,14 +2240,13 @@ dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused, RTE_CACHE_LINE_SIZE); if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { DPAA_SEC_ERR("No Memory for cipher key"); - return -1; + return -ENOMEM; } session->cipher_key.length = cipher_xform->key.length; session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL && auth_xform->key.length > 0) { DPAA_SEC_ERR("No Memory for auth key"); - rte_free(session->cipher_key.data); return -ENOMEM; } session->auth_key.length = auth_xform->key.length; @@ -2222,10 +2283,18 @@ dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused, session->auth_key.alg = OP_ALG_ALGSEL_SHA512; session->auth_key.algmode = OP_ALG_AAI_HMAC; break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_CMAC; + break; default: DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u", auth_xform->algo); - goto error_out; + return -ENOTSUP; } session->cipher_alg = cipher_xform->algo; @@ -2235,6 +2304,10 @@ dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused, session->cipher_key.alg = OP_ALG_ALGSEL_AES; session->cipher_key.algmode = OP_ALG_AAI_CBC; break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; case RTE_CRYPTO_CIPHER_3DES_CBC: session->cipher_key.alg = OP_ALG_ALGSEL_3DES; session->cipher_key.algmode = OP_ALG_AAI_CBC; @@ -2246,16 +2319,11 @@ dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused, default: DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", cipher_xform->algo); - goto error_out; + return -ENOTSUP; } session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? DIR_ENC : DIR_DEC; return 0; - -error_out: - rte_free(session->cipher_key.data); - rte_free(session->auth_key.data); - return -1; } static int @@ -2287,8 +2355,7 @@ dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, break; default: DPAA_SEC_ERR("unsupported AEAD alg %d", session->aead_alg); - rte_free(session->aead_key.data); - return -ENOMEM; + return -ENOTSUP; } session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? @@ -2321,7 +2388,7 @@ dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) { if (&qi->inq[i] == fq) { if (qman_retire_fq(fq, NULL) != 0) - DPAA_SEC_WARN("Queue is not retired\n"); + DPAA_SEC_DEBUG("Queue is not retired\n"); qman_oos_fq(fq); qi->inq_attach[i] = 0; return 0; @@ -2330,7 +2397,7 @@ dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) return -1; } -static int +int dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) { int ret; @@ -2339,9 +2406,9 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) ret = dpaa_sec_prep_cdb(sess); if (ret) { DPAA_SEC_ERR("Unable to prepare sec cdb"); - return -1; + return ret; } - if (unlikely(!RTE_PER_LCORE(dpaa_io))) { + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { ret = rte_dpaa_portal_init((void *)0); if (ret) { DPAA_SEC_ERR("Failure in affining portal"); @@ -2349,7 +2416,7 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) } } ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES], - dpaa_mem_vtop(&sess->cdb), + rte_dpaa_mem_vtop(&sess->cdb), qman_fq_fqid(&qp->outq)); if (ret) DPAA_SEC_ERR("Unable to init sec queue"); @@ -2357,6 +2424,18 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) return ret; } +static inline void +free_session_data(dpaa_sec_session *s) +{ + if (is_aead(s)) + rte_free(s->aead_key.data); + else { + rte_free(s->auth_key.data); + rte_free(s->cipher_key.data); + } + memset(s, 0, sizeof(dpaa_sec_session)); +} + static int dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) @@ -2402,7 +2481,7 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, ret = dpaa_sec_chain_init(dev, xform, session); } else { DPAA_SEC_ERR("Not supported: Auth then Cipher"); - return -EINVAL; + return -ENOTSUP; } /* Authenticate then Cipher */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && @@ -2418,7 +2497,7 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, ret = dpaa_sec_chain_init(dev, xform, session); } else { DPAA_SEC_ERR("Not supported: Auth then Cipher"); - return -EINVAL; + return -ENOTSUP; } /* AEAD operation for AES-GCM kind of Algorithms */ @@ -2441,6 +2520,7 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, if (session->inq[i] == NULL) { DPAA_SEC_ERR("unable to attach sec queue"); rte_spinlock_unlock(&internals->lock); + ret = -EBUSY; goto err1; } } @@ -2449,11 +2529,8 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, return 0; err1: - rte_free(session->cipher_key.data); - rte_free(session->auth_key.data); - memset(session, 0, sizeof(dpaa_sec_session)); - - return -EINVAL; + free_session_data(session); + return ret; } static int @@ -2501,9 +2578,7 @@ free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s) s->inq[i] = NULL; s->qp[i] = NULL; } - rte_free(s->cipher_key.data); - rte_free(s->auth_key.data); - memset(s, 0, sizeof(dpaa_sec_session)); + free_session_data(s); rte_mempool_put(sess_mp, (void *)s); } @@ -2523,7 +2598,7 @@ dpaa_sec_sym_session_clear(struct rte_cryptodev *dev, } } -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY static int dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, struct rte_security_ipsec_xform *ipsec_xform, @@ -2535,7 +2610,7 @@ dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, RTE_CACHE_LINE_SIZE); if (session->aead_key.data == NULL && aead_xform->key.length > 0) { DPAA_SEC_ERR("No Memory for aead key"); - return -1; + return -ENOMEM; } memcpy(session->aead_key.data, aead_xform->key.data, aead_xform->key.length); @@ -2558,7 +2633,7 @@ dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, default: DPAA_SEC_ERR("Crypto: Undefined GCM digest %d", session->digest_length); - return -1; + return -EINVAL; } if (session->dir == DIR_ENC) { memcpy(session->encap_pdb.gcm.salt, @@ -2573,7 +2648,7 @@ dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, default: DPAA_SEC_ERR("Crypto: Undefined AEAD specified %u", aead_xform->algo); - return -1; + return -ENOTSUP; } return 0; } @@ -2581,6 +2656,7 @@ dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, static int dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, struct rte_crypto_auth_xform *auth_xform, + struct rte_security_ipsec_xform *ipsec_xform, dpaa_sec_session *session) { if (cipher_xform) { @@ -2650,12 +2726,16 @@ dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, break; case RTE_CRYPTO_AUTH_AES_CMAC: session->auth_key.alg = OP_PCL_IPSEC_AES_CMAC_96; + session->auth_key.algmode = OP_ALG_AAI_CMAC; break; case RTE_CRYPTO_AUTH_NULL: session->auth_key.alg = OP_PCL_IPSEC_HMAC_NULL; break; - case RTE_CRYPTO_AUTH_SHA224_HMAC: case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_PCL_IPSEC_AES_XCBC_MAC_96; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: case RTE_CRYPTO_AUTH_SNOW3G_UIA2: case RTE_CRYPTO_AUTH_SHA1: case RTE_CRYPTO_AUTH_SHA256: @@ -2669,11 +2749,11 @@ dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, case RTE_CRYPTO_AUTH_ZUC_EIA3: DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", session->auth_alg); - return -1; + return -ENOTSUP; default: DPAA_SEC_ERR("Crypto: Undefined Auth specified %u", session->auth_alg); - return -1; + return -ENOTSUP; } switch (session->cipher_alg) { @@ -2681,6 +2761,10 @@ dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, session->cipher_key.alg = OP_PCL_IPSEC_AES_CBC; session->cipher_key.algmode = OP_ALG_AAI_CBC; break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_PCL_IPSEC_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; case RTE_CRYPTO_CIPHER_3DES_CBC: session->cipher_key.alg = OP_PCL_IPSEC_3DES; session->cipher_key.algmode = OP_ALG_AAI_CBC; @@ -2688,6 +2772,13 @@ dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, case RTE_CRYPTO_CIPHER_AES_CTR: session->cipher_key.alg = OP_PCL_IPSEC_AES_CTR; session->cipher_key.algmode = OP_ALG_AAI_CTR; + if (session->dir == DIR_ENC) { + session->encap_pdb.ctr.ctr_initial = 0x00000001; + session->encap_pdb.ctr.ctr_nonce = ipsec_xform->salt; + } else { + session->decap_pdb.ctr.ctr_initial = 0x00000001; + session->decap_pdb.ctr.ctr_nonce = ipsec_xform->salt; + } break; case RTE_CRYPTO_CIPHER_NULL: session->cipher_key.alg = OP_PCL_IPSEC_NULL; @@ -2699,11 +2790,11 @@ dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, case RTE_CRYPTO_CIPHER_KASUMI_F8: DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u", session->cipher_alg); - return -1; + return -ENOTSUP; default: DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", session->cipher_alg); - return -1; + return -ENOTSUP; } return 0; @@ -2739,13 +2830,13 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, if (conf->crypto_xform->next) auth_xform = &conf->crypto_xform->next->auth; ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform, - session); + ipsec_xform, session); } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = &conf->crypto_xform->auth; if (conf->crypto_xform->next) cipher_xform = &conf->crypto_xform->next->cipher; ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform, - session); + ipsec_xform, session); } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { aead_xform = &conf->crypto_xform->aead; ret = dpaa_sec_ipsec_aead_init(aead_xform, @@ -2826,6 +2917,27 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, sizeof(struct rte_ipv6_hdr) << 16; if (ipsec_xform->options.esn) session->decap_pdb.options |= PDBOPTS_ESP_ESN; + if (ipsec_xform->replay_win_sz) { + uint32_t win_sz; + win_sz = rte_align32pow2(ipsec_xform->replay_win_sz); + + switch (win_sz) { + case 1: + case 2: + case 4: + case 8: + case 16: + case 32: + session->decap_pdb.options |= PDBOPTS_ESP_ARS32; + break; + case 64: + session->decap_pdb.options |= PDBOPTS_ESP_ARS64; + break; + default: + session->decap_pdb.options |= + PDBOPTS_ESP_ARS128; + } + } } else goto out; rte_spinlock_lock(&internals->lock); @@ -2841,9 +2953,7 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, return 0; out: - rte_free(session->auth_key.data); - rte_free(session->cipher_key.data); - memset(session, 0, sizeof(dpaa_sec_session)); + free_session_data(session); return -1; } @@ -2859,6 +2969,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, dpaa_sec_session *session = (dpaa_sec_session *)sess; struct dpaa_sec_dev_private *dev_priv = dev->data->dev_private; uint32_t i; + int ret; PMD_INIT_FUNC_TRACE(); @@ -2898,7 +3009,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, default: DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", session->cipher_alg); - return -1; + return -EINVAL; } session->cipher_key.data = rte_zmalloc(NULL, @@ -2927,6 +3038,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) { DPAA_SEC_ERR( "PDCP Seq Num size should be 5/12 bits for cmode"); + ret = -EINVAL; goto out; } } @@ -2949,7 +3061,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", session->auth_alg); rte_free(session->cipher_key.data); - return -1; + return -EINVAL; } session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, @@ -2976,7 +3088,9 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, session->pdcp.hfn = pdcp_xform->hfn; session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold; session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd; - session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; + session->pdcp.sdap_enabled = pdcp_xform->sdap_enabled; + if (cipher_xform) + session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; rte_spinlock_lock(&dev_priv->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { @@ -2984,6 +3098,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, if (session->inq[i] == NULL) { DPAA_SEC_ERR("unable to attach sec queue"); rte_spinlock_unlock(&dev_priv->lock); + ret = -EBUSY; goto out; } } @@ -2993,7 +3108,7 @@ out: rte_free(session->auth_key.data); rte_free(session->cipher_key.data); memset(session, 0, sizeof(dpaa_sec_session)); - return -1; + return ret; } static int @@ -3098,7 +3213,7 @@ dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = dpaa_sec_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->driver_id = cryptodev_driver_id; + info->driver_id = dpaa_cryptodev_driver_id; } } @@ -3120,7 +3235,7 @@ dpaa_sec_process_parallel_event(void *event, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -3175,7 +3290,7 @@ dpaa_sec_process_atomic_event(void *event, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -3210,7 +3325,7 @@ dpaa_sec_process_atomic_event(void *event, DPAA_PER_LCORE_DQRR_HELD |= 1 << index; DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src; ev->impl_opaque = index + 1; - ctx->op->sym->m_src->seqn = (uint32_t)index + 1; + *dpaa_seqn(ctx->op->sym->m_src) = (uint32_t)index + 1; *bufs = (void *)ctx->op; rte_mempool_put(ctx->ctx_pool, (void *)ctx); @@ -3244,7 +3359,7 @@ dpaa_sec_eventq_attach(const struct rte_cryptodev *dev, break; case RTE_SCHED_TYPE_ORDERED: DPAA_SEC_ERR("Ordered queue schedule type is not supported\n"); - return -1; + return -ENOTSUP; default: opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK; qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event; @@ -3292,13 +3407,15 @@ static struct rte_cryptodev_ops crypto_ops = { .dev_infos_get = dpaa_sec_dev_infos_get, .queue_pair_setup = dpaa_sec_queue_pair_setup, .queue_pair_release = dpaa_sec_queue_pair_release, - .queue_pair_count = dpaa_sec_queue_pair_count, .sym_session_get_size = dpaa_sec_sym_session_get_size, .sym_session_configure = dpaa_sec_sym_session_configure, - .sym_session_clear = dpaa_sec_sym_session_clear + .sym_session_clear = dpaa_sec_sym_session_clear, + /* Raw data-path API related operations */ + .sym_get_raw_dp_ctx_size = dpaa_sec_get_dp_ctx_size, + .sym_configure_raw_dp_ctx = dpaa_sec_configure_raw_dp_ctx, }; -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY static const struct rte_security_capability * dpaa_sec_capabilities_get(void *device __rte_unused) { @@ -3337,7 +3454,7 @@ static int dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) { struct dpaa_sec_dev_private *internals; -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY struct rte_security_ctx *security_instance; #endif struct dpaa_sec_qp *qp; @@ -3346,7 +3463,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) PMD_INIT_FUNC_TRACE(); - cryptodev->driver_id = cryptodev_driver_id; + cryptodev->driver_id = dpaa_cryptodev_driver_id; cryptodev->dev_ops = &crypto_ops; cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; @@ -3355,6 +3472,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) RTE_CRYPTODEV_FF_HW_ACCELERATED | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_SECURITY | + RTE_CRYPTODEV_FF_SYM_RAW_DP | RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | @@ -3374,7 +3492,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) DPAA_SEC_WARN("Device already init by primary process"); return 0; } -#ifdef RTE_LIBRTE_SECURITY +#ifdef RTE_LIB_SECURITY /* Initialize security_ctx only for primary process*/ security_instance = rte_malloc("rte_security_instances_ops", sizeof(struct rte_security_ctx), 0); @@ -3413,7 +3531,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) init_error: DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name); - dpaa_sec_uninit(cryptodev); + rte_free(cryptodev->security_ctx); return -EFAULT; } @@ -3466,26 +3584,30 @@ cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused, } } - if (unlikely(!RTE_PER_LCORE(dpaa_io))) { + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { retval = rte_dpaa_portal_init((void *)1); if (retval) { DPAA_SEC_ERR("Unable to initialize portal"); - return retval; + goto out; } } /* Invoke PMD device initialization function */ retval = dpaa_sec_dev_init(cryptodev); - if (retval == 0) + if (retval == 0) { + rte_cryptodev_pmd_probing_finish(cryptodev); return 0; + } + retval = -ENXIO; +out: /* In case of error, cleanup is done */ if (rte_eal_process_type() == RTE_PROC_PRIMARY) rte_free(cryptodev->data->dev_private); rte_cryptodev_pmd_release_device(cryptodev); - return -ENXIO; + return retval; } static int @@ -3518,11 +3640,5 @@ static struct cryptodev_driver dpaa_sec_crypto_drv; RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver, - cryptodev_driver_id); - -RTE_INIT(dpaa_sec_init_log) -{ - dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa"); - if (dpaa_logtype_sec >= 0) - rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE); -} + dpaa_cryptodev_driver_id); +RTE_LOG_REGISTER(dpaa_logtype_sec, pmd.crypto.dpaa, NOTICE);