X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fdpaa_sec%2Fdpaa_sec.c;h=23a94d7e41ac82b475824a3706dbc69ae8a459d3;hb=8a3167dbe3ad16ac74e9d87735fa2d9d42d2644a;hp=38cfdd378e067f2bd3503a29129b01d3c5c7d0a4;hpb=fe3688ba795086ea00f57c5a5dd0bf0f0824e0df;p=dpdk.git diff --git a/drivers/crypto/dpaa_sec/dpaa_sec.c b/drivers/crypto/dpaa_sec/dpaa_sec.c index 38cfdd378e..23a94d7e41 100644 --- a/drivers/crypto/dpaa_sec/dpaa_sec.c +++ b/drivers/crypto/dpaa_sec/dpaa_sec.c @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. - * Copyright 2017-2019 NXP + * Copyright 2017-2022 NXP * */ @@ -12,28 +12,34 @@ #include #include -#include +#include #include #include +#ifdef RTE_LIB_SECURITY #include +#endif #include #include +#include +#include #include #include #include #include #include #include +#include #include #include -#include +#include /* RTA header files */ -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include @@ -41,17 +47,18 @@ #include #include -enum rta_sec_era rta_sec_era; - -int dpaa_logtype_sec; +#define DRIVER_DUMP_MODE "drv_dump_mode" -static uint8_t cryptodev_driver_id; +/* DPAA_SEC_DP_DUMP levels */ +enum dpaa_sec_dump_levels { + DPAA_SEC_DP_NO_DUMP, + DPAA_SEC_DP_ERR_DUMP, + DPAA_SEC_DP_FULL_DUMP +}; -static __thread struct rte_crypto_op **dpaa_sec_ops; -static __thread int dpaa_sec_op_nb; +uint8_t dpaa_sec_dp_dump = DPAA_SEC_DP_ERR_DUMP; -static int -dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess); +uint8_t dpaa_cryptodev_driver_id; static inline void dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) @@ -92,31 +99,6 @@ dpaa_sec_alloc_ctx(dpaa_sec_session *ses, int sg_count) return ctx; } -static inline rte_iova_t -dpaa_mem_vtop(void *vaddr) -{ - const struct rte_memseg *ms; - - ms = rte_mem_virt2memseg(vaddr, NULL); - if (ms) { - dpaax_iova_table_update(ms->iova, ms->addr, ms->len); - return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); - } - return (size_t)NULL; -} - -static inline void * -dpaa_mem_ptov(rte_iova_t paddr) -{ - void *va; - - va = (void *)dpaax_iova_table_get_va(paddr); - if (likely(va)) - return va; - - return rte_mem_iova2virt(paddr); -} - static void ern_sec_fq_handler(struct qman_portal *qm __rte_unused, struct qman_fq *fq, @@ -146,7 +128,7 @@ dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); fq_opts.fqd.context_b = fqid_out; - fq_opts.fqd.dest.channel = qm_channel_caam; + fq_opts.fqd.dest.channel = dpaa_get_qm_channel_caam(); fq_opts.fqd.dest.wq = 0; fq_in->cb.ern = ern_sec_fq_handler; @@ -170,7 +152,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, struct dpaa_sec_job *job; struct dpaa_sec_op_ctx *ctx; - if (dpaa_sec_op_nb >= DPAA_SEC_BURST) + if (DPAA_PER_LCORE_DPAA_SEC_OP_NB >= DPAA_SEC_BURST) return qman_cb_dqrr_defer; if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) @@ -181,7 +163,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -201,7 +183,7 @@ dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, } mbuf->data_len = len; } - dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; + DPAA_PER_LCORE_RTE_CRYPTO_OP[DPAA_PER_LCORE_DPAA_SEC_OP_NB++] = ctx->op; dpaa_sec_op_ending(ctx); return qman_cb_dqrr_consume; @@ -242,18 +224,6 @@ dpaa_sec_init_tx(struct qman_fq *fq) return ret; } -static inline int is_cipher_only(dpaa_sec_session *ses) -{ - return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && - (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); -} - -static inline int is_auth_only(dpaa_sec_session *ses) -{ - return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && - (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); -} - static inline int is_aead(dpaa_sec_session *ses) { return ((ses->cipher_alg == 0) && @@ -261,24 +231,6 @@ static inline int is_aead(dpaa_sec_session *ses) (ses->aead_alg != 0)); } -static inline int is_auth_cipher(dpaa_sec_session *ses) -{ - return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && - (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && - (ses->proto_alg != RTE_SECURITY_PROTOCOL_PDCP) && - (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); -} - -static inline int is_proto_ipsec(dpaa_sec_session *ses) -{ - return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); -} - -static inline int is_proto_pdcp(dpaa_sec_session *ses) -{ - return (ses->proto_alg == RTE_SECURITY_PROTOCOL_PDCP); -} - static inline int is_encode(dpaa_sec_session *ses) { return ses->dir == DIR_ENC; @@ -289,102 +241,7 @@ static inline int is_decode(dpaa_sec_session *ses) return ses->dir == DIR_DEC; } -static inline void -caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) -{ - switch (ses->auth_alg) { - case RTE_CRYPTO_AUTH_NULL: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_NULL : 0; - ses->digest_length = 0; - break; - case RTE_CRYPTO_AUTH_MD5_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - case RTE_CRYPTO_AUTH_SHA1_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - case RTE_CRYPTO_AUTH_SHA224_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - case RTE_CRYPTO_AUTH_SHA256_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - case RTE_CRYPTO_AUTH_SHA384_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - case RTE_CRYPTO_AUTH_SHA512_HMAC: - alginfo_a->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; - alginfo_a->algmode = OP_ALG_AAI_HMAC; - break; - default: - DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg); - } -} - -static inline void -caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) -{ - switch (ses->cipher_alg) { - case RTE_CRYPTO_CIPHER_NULL: - alginfo_c->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_NULL : 0; - break; - case RTE_CRYPTO_CIPHER_AES_CBC: - alginfo_c->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; - alginfo_c->algmode = OP_ALG_AAI_CBC; - break; - case RTE_CRYPTO_CIPHER_3DES_CBC: - alginfo_c->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; - alginfo_c->algmode = OP_ALG_AAI_CBC; - break; - case RTE_CRYPTO_CIPHER_AES_CTR: - alginfo_c->algtype = - (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? - OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; - alginfo_c->algmode = OP_ALG_AAI_CTR; - break; - default: - DPAA_SEC_ERR("unsupported cipher alg %d", ses->cipher_alg); - } -} - -static inline void -caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) -{ - switch (ses->aead_alg) { - case RTE_CRYPTO_AEAD_AES_GCM: - alginfo->algtype = OP_ALG_ALGSEL_AES; - alginfo->algmode = OP_ALG_AAI_GCM; - break; - default: - DPAA_SEC_ERR("unsupported AEAD alg %d", ses->aead_alg); - } -} - +#ifdef RTE_LIB_SECURITY static int dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) { @@ -392,95 +249,57 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) struct sec_cdb *cdb = &ses->cdb; struct alginfo *p_authdata = NULL; int32_t shared_desc_len = 0; - int err; #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN int swap = false; #else int swap = true; #endif - switch (ses->cipher_alg) { - case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: - cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW; - break; - case RTE_CRYPTO_CIPHER_ZUC_EEA3: - cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC; - break; - case RTE_CRYPTO_CIPHER_AES_CTR: - cipherdata.algtype = PDCP_CIPHER_TYPE_AES; - break; - case RTE_CRYPTO_CIPHER_NULL: - cipherdata.algtype = PDCP_CIPHER_TYPE_NULL; - break; - default: - DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", - ses->cipher_alg); - return -1; - } - cipherdata.key = (size_t)ses->cipher_key.data; cipherdata.keylen = ses->cipher_key.length; cipherdata.key_enc_flags = 0; cipherdata.key_type = RTA_DATA_IMM; - - cdb->sh_desc[0] = cipherdata.keylen; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; + cipherdata.algtype = ses->cipher_key.alg; + cipherdata.algmode = ses->cipher_key.algmode; if (ses->auth_alg) { - switch (ses->auth_alg) { - case RTE_CRYPTO_AUTH_SNOW3G_UIA2: - authdata.algtype = PDCP_AUTH_TYPE_SNOW; - break; - case RTE_CRYPTO_AUTH_ZUC_EIA3: - authdata.algtype = PDCP_AUTH_TYPE_ZUC; - break; - case RTE_CRYPTO_AUTH_AES_CMAC: - authdata.algtype = PDCP_AUTH_TYPE_AES; - break; - case RTE_CRYPTO_AUTH_NULL: - authdata.algtype = PDCP_AUTH_TYPE_NULL; - break; - default: - DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", - ses->auth_alg); - return -1; - } - authdata.key = (size_t)ses->auth_key.data; authdata.keylen = ses->auth_key.length; authdata.key_enc_flags = 0; authdata.key_type = RTA_DATA_IMM; + authdata.algtype = ses->auth_key.alg; + authdata.algmode = ses->auth_key.algmode; p_authdata = &authdata; - - cdb->sh_desc[1] = authdata.keylen; } - err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, - (unsigned int *)cdb->sh_desc, - &cdb->sh_desc[2], 2); - if (err < 0) { - DPAA_SEC_ERR("Crypto: Incorrect key lengths"); - return err; - } - - if (!(cdb->sh_desc[2] & 1) && cipherdata.keylen) { - cipherdata.key = - (size_t)dpaa_mem_vtop((void *)(size_t)cipherdata.key); - cipherdata.key_type = RTA_DATA_PTR; - } - if (!(cdb->sh_desc[2] & (1 << 1)) && authdata.keylen) { - authdata.key = - (size_t)dpaa_mem_vtop((void *)(size_t)authdata.key); - authdata.key_type = RTA_DATA_PTR; + if (ses->pdcp.sdap_enabled) { + int nb_keys_to_inline = + rta_inline_pdcp_sdap_query(authdata.algtype, + cipherdata.algtype, + ses->pdcp.sn_size, + ses->pdcp.hfn_ovd); + if (nb_keys_to_inline >= 1) { + cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } + if (nb_keys_to_inline >= 2) { + authdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)authdata.key); + authdata.key_type = RTA_DATA_PTR; + } + } else { + if (rta_inline_pdcp_query(authdata.algtype, + cipherdata.algtype, + ses->pdcp.sn_size, + ses->pdcp.hfn_ovd)) { + cipherdata.key = (size_t)rte_dpaa_mem_vtop((void *) + (size_t)cipherdata.key); + cipherdata.key_type = RTA_DATA_PTR; + } } - cdb->sh_desc[0] = 0; - cdb->sh_desc[1] = 0; - cdb->sh_desc[2] = 0; - if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_CONTROL) { if (ses->dir == DIR_ENC) shared_desc_len = cnstr_shdsc_pdcp_c_plane_encap( @@ -490,8 +309,7 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, - &cipherdata, &authdata, - 0); + &cipherdata, &authdata); else if (ses->dir == DIR_DEC) shared_desc_len = cnstr_shdsc_pdcp_c_plane_decap( cdb->sh_desc, 1, swap, @@ -500,29 +318,55 @@ dpaa_sec_prep_pdcp_cdb(dpaa_sec_session *ses) ses->pdcp.bearer, ses->pdcp.pkt_dir, ses->pdcp.hfn_threshold, - &cipherdata, &authdata, - 0); + &cipherdata, &authdata); + } else if (ses->pdcp.domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) { + shared_desc_len = cnstr_shdsc_pdcp_short_mac(cdb->sh_desc, + 1, swap, &authdata); } else { - if (ses->dir == DIR_ENC) - shared_desc_len = cnstr_shdsc_pdcp_u_plane_encap( - cdb->sh_desc, 1, swap, - ses->pdcp.sn_size, - ses->pdcp.hfn, - ses->pdcp.bearer, - ses->pdcp.pkt_dir, - ses->pdcp.hfn_threshold, - &cipherdata, p_authdata, 0); - else if (ses->dir == DIR_DEC) - shared_desc_len = cnstr_shdsc_pdcp_u_plane_decap( - cdb->sh_desc, 1, swap, - ses->pdcp.sn_size, - ses->pdcp.hfn, - ses->pdcp.bearer, - ses->pdcp.pkt_dir, - ses->pdcp.hfn_threshold, - &cipherdata, p_authdata, 0); + if (ses->dir == DIR_ENC) { + if (ses->pdcp.sdap_enabled) + shared_desc_len = + cnstr_shdsc_pdcp_sdap_u_plane_encap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata); + else + shared_desc_len = + cnstr_shdsc_pdcp_u_plane_encap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata); + } else if (ses->dir == DIR_DEC) { + if (ses->pdcp.sdap_enabled) + shared_desc_len = + cnstr_shdsc_pdcp_sdap_u_plane_decap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata); + else + shared_desc_len = + cnstr_shdsc_pdcp_u_plane_decap( + cdb->sh_desc, 1, swap, + ses->pdcp.sn_size, + ses->pdcp.hfn, + ses->pdcp.bearer, + ses->pdcp.pkt_dir, + ses->pdcp.hfn_threshold, + &cipherdata, p_authdata); + } } - return shared_desc_len; } @@ -540,32 +384,26 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) int swap = true; #endif - caam_cipher_alg(ses, &cipherdata); - if (cipherdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported cipher alg"); - return -ENOTSUP; - } - cipherdata.key = (size_t)ses->cipher_key.data; cipherdata.keylen = ses->cipher_key.length; cipherdata.key_enc_flags = 0; cipherdata.key_type = RTA_DATA_IMM; + cipherdata.algtype = ses->cipher_key.alg; + cipherdata.algmode = ses->cipher_key.algmode; - caam_auth_alg(ses, &authdata); - if (authdata.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported auth alg"); - return -ENOTSUP; + if (ses->auth_key.length) { + authdata.key = (size_t)ses->auth_key.data; + authdata.keylen = ses->auth_key.length; + authdata.key_enc_flags = 0; + authdata.key_type = RTA_DATA_IMM; + authdata.algtype = ses->auth_key.alg; + authdata.algmode = ses->auth_key.algmode; } - authdata.key = (size_t)ses->auth_key.data; - authdata.keylen = ses->auth_key.length; - authdata.key_enc_flags = 0; - authdata.key_type = RTA_DATA_IMM; - cdb->sh_desc[0] = cipherdata.keylen; cdb->sh_desc[1] = authdata.keylen; err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, + DESC_JOB_IO_LEN, (unsigned int *)cdb->sh_desc, &cdb->sh_desc[2], 2); @@ -576,14 +414,14 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) if (cdb->sh_desc[2] & 1) cipherdata.key_type = RTA_DATA_IMM; else { - cipherdata.key = (size_t)dpaa_mem_vtop( + cipherdata.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)cipherdata.key); cipherdata.key_type = RTA_DATA_PTR; } if (cdb->sh_desc[2] & (1<<1)) authdata.key_type = RTA_DATA_IMM; else { - authdata.key = (size_t)dpaa_mem_vtop( + authdata.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)authdata.key); authdata.key_type = RTA_DATA_PTR; } @@ -607,7 +445,7 @@ dpaa_sec_prep_ipsec_cdb(dpaa_sec_session *ses) } return shared_desc_len; } - +#endif /* prepare command block of the session */ static int dpaa_sec_prep_cdb(dpaa_sec_session *ses) @@ -624,46 +462,113 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) memset(cdb, 0, sizeof(struct sec_cdb)); - if (is_proto_ipsec(ses)) { + switch (ses->ctxt) { +#ifdef RTE_LIB_SECURITY + case DPAA_SEC_IPSEC: shared_desc_len = dpaa_sec_prep_ipsec_cdb(ses); - } else if (is_proto_pdcp(ses)) { + break; + case DPAA_SEC_PDCP: shared_desc_len = dpaa_sec_prep_pdcp_cdb(ses); - } else if (is_cipher_only(ses)) { - caam_cipher_alg(ses, &alginfo_c); - if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported cipher alg"); - return -ENOTSUP; - } - + break; +#endif + case DPAA_SEC_CIPHER: alginfo_c.key = (size_t)ses->cipher_key.data; alginfo_c.keylen = ses->cipher_key.length; alginfo_c.key_enc_flags = 0; alginfo_c.key_type = RTA_DATA_IMM; + alginfo_c.algtype = ses->cipher_key.alg; + alginfo_c.algmode = ses->cipher_key.algmode; - shared_desc_len = cnstr_shdsc_blkcipher( - cdb->sh_desc, true, - swap, SHR_NEVER, &alginfo_c, - NULL, - ses->iv.length, - ses->dir); - } else if (is_auth_only(ses)) { - caam_auth_alg(ses, &alginfo_a); - if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported auth alg"); + switch (ses->cipher_alg) { + case RTE_CRYPTO_CIPHER_AES_CBC: + case RTE_CRYPTO_CIPHER_3DES_CBC: + case RTE_CRYPTO_CIPHER_DES_CBC: + case RTE_CRYPTO_CIPHER_AES_CTR: + case RTE_CRYPTO_CIPHER_3DES_CTR: + shared_desc_len = cnstr_shdsc_blkcipher( + cdb->sh_desc, true, + swap, SHR_NEVER, &alginfo_c, + ses->iv.length, + ses->dir); + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + shared_desc_len = cnstr_shdsc_snow_f8( + cdb->sh_desc, true, swap, + &alginfo_c, + ses->dir); + break; + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + shared_desc_len = cnstr_shdsc_zuce( + cdb->sh_desc, true, swap, + &alginfo_c, + ses->dir); + break; + default: + DPAA_SEC_ERR("unsupported cipher alg %d", + ses->cipher_alg); return -ENOTSUP; } - + break; + case DPAA_SEC_AUTH: alginfo_a.key = (size_t)ses->auth_key.data; alginfo_a.keylen = ses->auth_key.length; alginfo_a.key_enc_flags = 0; alginfo_a.key_type = RTA_DATA_IMM; - - shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, - swap, SHR_NEVER, &alginfo_a, - !ses->dir, - ses->digest_length); - } else if (is_aead(ses)) { - caam_aead_alg(ses, &alginfo); + alginfo_a.algtype = ses->auth_key.alg; + alginfo_a.algmode = ses->auth_key.algmode; + switch (ses->auth_alg) { + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_SHA512: + shared_desc_len = cnstr_shdsc_hash( + cdb->sh_desc, true, + swap, SHR_NEVER, &alginfo_a, + !ses->dir, + ses->digest_length); + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + case RTE_CRYPTO_AUTH_SHA1_HMAC: + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_SHA256_HMAC: + case RTE_CRYPTO_AUTH_SHA384_HMAC: + case RTE_CRYPTO_AUTH_SHA512_HMAC: + shared_desc_len = cnstr_shdsc_hmac( + cdb->sh_desc, true, + swap, SHR_NEVER, &alginfo_a, + !ses->dir, + ses->digest_length); + break; + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + shared_desc_len = cnstr_shdsc_snow_f9( + cdb->sh_desc, true, swap, + &alginfo_a, + !ses->dir, + ses->digest_length); + break; + case RTE_CRYPTO_AUTH_ZUC_EIA3: + shared_desc_len = cnstr_shdsc_zuca( + cdb->sh_desc, true, swap, + &alginfo_a, + !ses->dir, + ses->digest_length); + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + case RTE_CRYPTO_AUTH_AES_CMAC: + shared_desc_len = cnstr_shdsc_aes_mac( + cdb->sh_desc, + true, swap, SHR_NEVER, + &alginfo_a, + !ses->dir, + ses->digest_length); + break; + default: + DPAA_SEC_ERR("unsupported auth alg %u", ses->auth_alg); + } + break; + case DPAA_SEC_AEAD: if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { DPAA_SEC_ERR("not supported aead alg"); return -ENOTSUP; @@ -672,6 +577,8 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) alginfo.keylen = ses->aead_key.length; alginfo.key_enc_flags = 0; alginfo.key_type = RTA_DATA_IMM; + alginfo.algtype = ses->aead_key.alg; + alginfo.algmode = ses->aead_key.algmode; if (ses->dir == DIR_ENC) shared_desc_len = cnstr_shdsc_gcm_encap( @@ -685,33 +592,26 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) &alginfo, ses->iv.length, ses->digest_length); - } else { - caam_cipher_alg(ses, &alginfo_c); - if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported cipher alg"); - return -ENOTSUP; - } - + break; + case DPAA_SEC_CIPHER_HASH: alginfo_c.key = (size_t)ses->cipher_key.data; alginfo_c.keylen = ses->cipher_key.length; alginfo_c.key_enc_flags = 0; alginfo_c.key_type = RTA_DATA_IMM; - - caam_auth_alg(ses, &alginfo_a); - if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { - DPAA_SEC_ERR("not supported auth alg"); - return -ENOTSUP; - } + alginfo_c.algtype = ses->cipher_key.alg; + alginfo_c.algmode = ses->cipher_key.algmode; alginfo_a.key = (size_t)ses->auth_key.data; alginfo_a.keylen = ses->auth_key.length; alginfo_a.key_enc_flags = 0; alginfo_a.key_type = RTA_DATA_IMM; + alginfo_a.algtype = ses->auth_key.alg; + alginfo_a.algmode = ses->auth_key.algmode; cdb->sh_desc[0] = alginfo_c.keylen; cdb->sh_desc[1] = alginfo_a.keylen; err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, - MIN_JOB_DESC_SIZE, + DESC_JOB_IO_LEN, (unsigned int *)cdb->sh_desc, &cdb->sh_desc[2], 2); @@ -722,14 +622,14 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) if (cdb->sh_desc[2] & 1) alginfo_c.key_type = RTA_DATA_IMM; else { - alginfo_c.key = (size_t)dpaa_mem_vtop( + alginfo_c.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)alginfo_c.key); alginfo_c.key_type = RTA_DATA_PTR; } if (cdb->sh_desc[2] & (1<<1)) alginfo_a.key_type = RTA_DATA_IMM; else { - alginfo_a.key = (size_t)dpaa_mem_vtop( + alginfo_a.key = (size_t)rte_dpaa_mem_vtop( (void *)(size_t)alginfo_a.key); alginfo_a.key_type = RTA_DATA_PTR; } @@ -741,8 +641,13 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) */ shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, true, swap, SHR_SERIAL, &alginfo_c, &alginfo_a, - ses->iv.length, 0, + ses->iv.length, ses->digest_length, ses->dir); + break; + case DPAA_SEC_HASH_CIPHER: + default: + DPAA_SEC_ERR("error: Unsupported session"); + return -ENOTSUP; } if (shared_desc_len < 0) { @@ -757,6 +662,139 @@ dpaa_sec_prep_cdb(dpaa_sec_session *ses) return 0; } +static void +dpaa_sec_dump(struct dpaa_sec_op_ctx *ctx, struct dpaa_sec_qp *qp) +{ + struct dpaa_sec_job *job = &ctx->job; + struct rte_crypto_op *op = ctx->op; + dpaa_sec_session *sess = NULL; + struct sec_cdb c_cdb, *cdb; + uint8_t bufsize; + struct rte_crypto_sym_op *sym_op; + struct qm_sg_entry sg[2]; + + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) + sess = (dpaa_sec_session *) + get_sym_session_private_data( + op->sym->session, + dpaa_cryptodev_driver_id); +#ifdef RTE_LIBRTE_SECURITY + else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + sess = (dpaa_sec_session *) + get_sec_session_private_data( + op->sym->sec_session); +#endif + if (sess == NULL) { + printf("session is NULL\n"); + goto mbuf_dump; + } + + cdb = &sess->cdb; + rte_memcpy(&c_cdb, cdb, sizeof(struct sec_cdb)); +#ifdef RTE_LIBRTE_SECURITY + printf("\nsession protocol type = %d\n", sess->proto_alg); +#endif + printf("\n****************************************\n" + "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n" + "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n" + "\tCipher key len:\t%"PRIu64"\n\tCipher alg:\t%d\n" + "\tCipher algmode:\t%d\n", sess->ctxt, + (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC", + sess->cipher_alg, sess->auth_alg, sess->aead_alg, + (uint64_t)sess->cipher_key.length, sess->cipher_key.alg, + sess->cipher_key.algmode); + rte_hexdump(stdout, "cipher key", sess->cipher_key.data, + sess->cipher_key.length); + rte_hexdump(stdout, "auth key", sess->auth_key.data, + sess->auth_key.length); + printf("\tAuth key len:\t%"PRIu64"\n\tAuth alg:\t%d\n" + "\tAuth algmode:\t%d\n\tIV len:\t\t%d\n\tIV offset:\t%d\n" + "\tdigest length:\t%d\n\tauth only len:\t\t%d\n" + "\taead cipher text:\t%d\n", + (uint64_t)sess->auth_key.length, sess->auth_key.alg, + sess->auth_key.algmode, + sess->iv.length, sess->iv.offset, + sess->digest_length, sess->auth_only_len, + sess->auth_cipher_text); +#ifdef RTE_LIBRTE_SECURITY + printf("PDCP session params:\n" + "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:" + "\t%d\n\tsn_size:\t%d\n\tsdap_enabled:\t%d\n\thfn_ovd_offset:" + "\t%d\n\thfn:\t\t%d\n" + "\thfn_threshold:\t0x%x\n", sess->pdcp.domain, + sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd, + sess->pdcp.sn_size, sess->pdcp.sdap_enabled, + sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn, + sess->pdcp.hfn_threshold); +#endif + c_cdb.sh_hdr.hi.word = rte_be_to_cpu_32(c_cdb.sh_hdr.hi.word); + c_cdb.sh_hdr.lo.word = rte_be_to_cpu_32(c_cdb.sh_hdr.lo.word); + bufsize = c_cdb.sh_hdr.hi.field.idlen; + + printf("cdb = %p\n\n", cdb); + printf("Descriptor size = %d\n", bufsize); + int m; + for (m = 0; m < bufsize; m++) + printf("0x%x\n", rte_be_to_cpu_32(c_cdb.sh_desc[m])); + + printf("\n"); +mbuf_dump: + sym_op = op->sym; + if (sym_op->m_src) { + printf("Source mbuf:\n"); + rte_pktmbuf_dump(stdout, sym_op->m_src, + sym_op->m_src->data_len); + } + if (sym_op->m_dst) { + printf("Destination mbuf:\n"); + rte_pktmbuf_dump(stdout, sym_op->m_dst, + sym_op->m_dst->data_len); + } + + printf("Session address = %p\ncipher offset: %d, length: %d\n" + "auth offset: %d, length: %d\n aead offset: %d, length: %d\n", + sym_op->session, sym_op->cipher.data.offset, + sym_op->cipher.data.length, + sym_op->auth.data.offset, sym_op->auth.data.length, + sym_op->aead.data.offset, sym_op->aead.data.length); + printf("\n"); + + printf("******************************************************\n"); + printf("ctx info:\n"); + printf("job->sg[0] output info:\n"); + memcpy(&sg[0], &job->sg[0], sizeof(sg[0])); + printf("\taddr = %"PRIx64",\n\tlen = %d,\n\tfinal = %d,\n\textention = %d" + "\n\tbpid = %d\n\toffset = %d\n", + (uint64_t)sg[0].addr, sg[0].length, sg[0].final, + sg[0].extension, sg[0].bpid, sg[0].offset); + printf("\njob->sg[1] input info:\n"); + memcpy(&sg[1], &job->sg[1], sizeof(sg[1])); + hw_sg_to_cpu(&sg[1]); + printf("\taddr = %"PRIx64",\n\tlen = %d,\n\tfinal = %d,\n\textention = %d" + "\n\tbpid = %d\n\toffset = %d\n", + (uint64_t)sg[1].addr, sg[1].length, sg[1].final, + sg[1].extension, sg[1].bpid, sg[1].offset); + + printf("\nctx pool addr = %p\n", ctx->ctx_pool); + if (ctx->ctx_pool) + printf("ctx pool available counts = %d\n", + rte_mempool_avail_count(ctx->ctx_pool)); + + printf("\nop pool addr = %p\n", op->mempool); + if (op->mempool) + printf("op pool available counts = %d\n", + rte_mempool_avail_count(op->mempool)); + + printf("********************************************************\n"); + printf("Queue data:\n"); + printf("\tFQID = 0x%x\n\tstate = %d\n\tnb_desc = %d\n" + "\tctx_pool = %p\n\trx_pkts = %d\n\ttx_pkts" + "= %d\n\trx_errs = %d\n\ttx_errs = %d\n\n", + qp->outq.fqid, qp->outq.state, qp->outq.nb_desc, + qp->ctx_pool, qp->rx_pkts, qp->tx_pkts, + qp->rx_errs, qp->tx_errs); +} + /* qp is lockless, should be accessed by only one thread */ static int dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) @@ -800,7 +838,7 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -824,12 +862,17 @@ dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) if (!ctx->fd_status) { op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; } else { - DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status); + if (dpaa_sec_dp_dump > DPAA_SEC_DP_NO_DUMP) { + DPAA_SEC_DP_WARN("SEC return err:0x%x\n", + ctx->fd_status); + if (dpaa_sec_dp_dump > DPAA_SEC_DP_ERR_DUMP) + dpaa_sec_dump(ctx, qp); + } op->status = RTE_CRYPTO_OP_STATUS_ERROR; } ops[pkts++] = op; - /* report op status to sym->op and then free the ctx memeory */ + /* report op status to sym->op and then free the ctx memory */ rte_mempool_put(ctx->ctx_pool, (void *)ctx); qman_dqrr_consume(fq, dq); @@ -848,6 +891,21 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) struct qm_sg_entry *sg, *out_sg, *in_sg; phys_addr_t start_addr; uint8_t *old_digest, extra_segs; + int data_len, data_offset; + + data_len = sym->auth.data.length; + data_offset = sym->auth.data.offset; + + if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || + ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { + if ((data_len & 7) || (data_offset & 7)) { + DPAA_SEC_ERR("AUTH: len/offset must be full bytes"); + return NULL; + } + + data_len = data_len >> 3; + data_offset = data_offset >> 3; + } if (is_decode(ses)) extra_segs = 3; @@ -878,23 +936,52 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* need to extend the input to a compound frame */ in_sg->extension = 1; in_sg->final = 1; - in_sg->length = sym->auth.data.length; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(&cf->sg[2])); + in_sg->length = data_len; + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2])); /* 1st seg */ sg = in_sg + 1; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); - sg->length = mbuf->data_len - sym->auth.data.offset; - sg->offset = sym->auth.data.offset; - /* Successive segs */ - mbuf = mbuf->next; - while (mbuf) { + if (ses->iv.length) { + uint8_t *iv_ptr; + + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { + iv_ptr = conv_to_snow_f9_iv(iv_ptr); + sg->length = 12; + } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { + iv_ptr = conv_to_zuc_eia_iv(iv_ptr); + sg->length = 8; + } else { + sg->length = ses->iv.length; + } + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr)); + in_sg->length += sg->length; cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); - sg->length = mbuf->data_len; - mbuf = mbuf->next; + } + + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + sg->offset = data_offset; + + if (data_len <= (mbuf->data_len - data_offset)) { + sg->length = data_len; + } else { + sg->length = mbuf->data_len - data_offset; + + /* remaining i/p segs */ + while ((data_len = data_len - sg->length) && + (mbuf = mbuf->next)) { + cpu_to_hw_sg(sg); + sg++; + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + if (data_len > mbuf->data_len) + sg->length = mbuf->data_len; + else + sg->length = data_len; + } } if (is_decode(ses)) { @@ -903,13 +990,10 @@ build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; rte_memcpy(old_digest, sym->auth.digest.data, ses->digest_length); - start_addr = dpaa_mem_vtop(old_digest); + start_addr = rte_dpaa_mem_vtop(old_digest); qm_sg_entry_set64(sg, start_addr); sg->length = ses->digest_length; in_sg->length += ses->digest_length; - } else { - /* Digest calculation case */ - sg->length -= ses->digest_length; } sg->final = 1; cpu_to_hw_sg(sg); @@ -933,9 +1017,24 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) struct rte_mbuf *mbuf = sym->m_src; struct dpaa_sec_job *cf; struct dpaa_sec_op_ctx *ctx; - struct qm_sg_entry *sg; + struct qm_sg_entry *sg, *in_sg; rte_iova_t start_addr; uint8_t *old_digest; + int data_len, data_offset; + + data_len = sym->auth.data.length; + data_offset = sym->auth.data.offset; + + if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || + ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { + if ((data_len & 7) || (data_offset & 7)) { + DPAA_SEC_ERR("AUTH: len/offset must be full bytes"); + return NULL; + } + + data_len = data_len >> 3; + data_offset = data_offset >> 3; + } ctx = dpaa_sec_alloc_ctx(ses, 4); if (!ctx) @@ -953,36 +1052,55 @@ build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) cpu_to_hw_sg(sg); /* input */ - sg = &cf->sg[1]; - if (is_decode(ses)) { - /* need to extend the input to a compound frame */ - sg->extension = 1; - qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); - sg->length = sym->auth.data.length + ses->digest_length; - sg->final = 1; + in_sg = &cf->sg[1]; + /* need to extend the input to a compound frame */ + in_sg->extension = 1; + in_sg->final = 1; + in_sg->length = data_len; + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2])); + sg = &cf->sg[2]; + + if (ses->iv.length) { + uint8_t *iv_ptr; + + iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, + ses->iv.offset); + + if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { + iv_ptr = conv_to_snow_f9_iv(iv_ptr); + sg->length = 12; + } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { + iv_ptr = conv_to_zuc_eia_iv(iv_ptr); + sg->length = 8; + } else { + sg->length = ses->iv.length; + } + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr)); + in_sg->length += sg->length; cpu_to_hw_sg(sg); + sg++; + } - sg = &cf->sg[2]; + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + sg->offset = data_offset; + sg->length = data_len; + + if (is_decode(ses)) { + /* Digest verification case */ + cpu_to_hw_sg(sg); /* hash result or digest, save digest first */ rte_memcpy(old_digest, sym->auth.digest.data, - ses->digest_length); - qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); - sg->length = sym->auth.data.length; - cpu_to_hw_sg(sg); - + ses->digest_length); /* let's check digest by hw */ - start_addr = dpaa_mem_vtop(old_digest); + start_addr = rte_dpaa_mem_vtop(old_digest); sg++; qm_sg_entry_set64(sg, start_addr); sg->length = ses->digest_length; - sg->final = 1; - cpu_to_hw_sg(sg); - } else { - qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); - sg->length = sym->auth.data.length; - sg->final = 1; - cpu_to_hw_sg(sg); + in_sg->length += ses->digest_length; } + sg->final = 1; + cpu_to_hw_sg(sg); + cpu_to_hw_sg(in_sg); return cf; } @@ -998,6 +1116,21 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) uint8_t req_segs; uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, ses->iv.offset); + int data_len, data_offset; + + data_len = sym->cipher.data.length; + data_offset = sym->cipher.data.offset; + + if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || + ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) { + if ((data_len & 7) || (data_offset & 7)) { + DPAA_SEC_ERR("CIPHER: len/offset must be full bytes"); + return NULL; + } + + data_len = data_len >> 3; + data_offset = data_offset >> 3; + } if (sym->m_dst) { mbuf = sym->m_dst; @@ -1006,7 +1139,6 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) mbuf = sym->m_src; req_segs = mbuf->nb_segs * 2 + 3; } - if (mbuf->nb_segs > MAX_SG_ENTRIES) { DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d", MAX_SG_ENTRIES); @@ -1023,22 +1155,22 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ out_sg = &cf->sg[0]; out_sg->extension = 1; - out_sg->length = sym->cipher.data.length; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); + out_sg->length = data_len; + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); cpu_to_hw_sg(out_sg); /* 1st seg */ sg = &cf->sg[2]; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); - sg->length = mbuf->data_len - sym->cipher.data.offset; - sg->offset = sym->cipher.data.offset; + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + sg->length = mbuf->data_len - data_offset; + sg->offset = data_offset; /* Successive segs */ mbuf = mbuf->next; while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1050,29 +1182,29 @@ build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) in_sg = &cf->sg[1]; in_sg->extension = 1; in_sg->final = 1; - in_sg->length = sym->cipher.data.length + ses->iv.length; + in_sg->length = data_len + ses->iv.length; sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 1st seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); - sg->length = mbuf->data_len - sym->cipher.data.offset; - sg->offset = sym->cipher.data.offset; + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + sg->length = mbuf->data_len - data_offset; + sg->offset = data_offset; /* Successive segs */ mbuf = mbuf->next; while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1092,6 +1224,21 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) rte_iova_t src_start_addr, dst_start_addr; uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, ses->iv.offset); + int data_len, data_offset; + + data_len = sym->cipher.data.length; + data_offset = sym->cipher.data.offset; + + if (ses->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || + ses->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) { + if ((data_len & 7) || (data_offset & 7)) { + DPAA_SEC_ERR("CIPHER: len/offset must be full bytes"); + return NULL; + } + + data_len = data_len >> 3; + data_offset = data_offset >> 3; + } ctx = dpaa_sec_alloc_ctx(ses, 4); if (!ctx) @@ -1109,8 +1256,8 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ sg = &cf->sg[0]; - qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); - sg->length = sym->cipher.data.length + ses->iv.length; + qm_sg_entry_set64(sg, dst_start_addr + data_offset); + sg->length = data_len + ses->iv.length; cpu_to_hw_sg(sg); /* input */ @@ -1119,18 +1266,18 @@ build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) /* need to extend the input to a compound frame */ sg->extension = 1; sg->final = 1; - sg->length = sym->cipher.data.length + ses->iv.length; - qm_sg_entry_set64(sg, dpaa_mem_vtop(&cf->sg[2])); + sg->length = data_len + ses->iv.length; + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(&cf->sg[2])); cpu_to_hw_sg(sg); sg = &cf->sg[2]; - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); - sg->length = sym->cipher.data.length; + qm_sg_entry_set64(sg, src_start_addr + data_offset); + sg->length = data_len; sg->final = 1; cpu_to_hw_sg(sg); @@ -1179,28 +1326,26 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) out_sg = &cf->sg[0]; out_sg->extension = 1; if (is_encode(ses)) - out_sg->length = sym->aead.data.length + ses->auth_only_len - + ses->digest_length; + out_sg->length = sym->aead.data.length + ses->digest_length; else - out_sg->length = sym->aead.data.length + ses->auth_only_len; + out_sg->length = sym->aead.data.length; /* output sg entries */ sg = &cf->sg[2]; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(out_sg); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); - sg->length = mbuf->data_len - sym->aead.data.offset + - ses->auth_only_len; - sg->offset = sym->aead.data.offset - ses->auth_only_len; + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); + sg->length = mbuf->data_len - sym->aead.data.offset; + sg->offset = sym->aead.data.offset; /* Successive segs */ mbuf = mbuf->next; while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1230,25 +1375,25 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input sg entries */ sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* 1st seg IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 2nd seg auth only */ if (ses->auth_only_len) { sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; cpu_to_hw_sg(sg); } /* 3rd seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->aead.data.offset; sg->offset = sym->aead.data.offset; @@ -1257,7 +1402,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1267,7 +1412,7 @@ build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; memcpy(ctx->digest, sym->aead.digest.data, ses->digest_length); - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; } sg->final = 1; @@ -1305,9 +1450,9 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input */ rte_prefetch0(cf->sg); sg = &cf->sg[2]; - qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg)); if (is_encode(ses)) { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1315,7 +1460,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; if (ses->auth_only_len) { qm_sg_entry_set64(sg, - dpaa_mem_vtop(sym->aead.aad.data)); + rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; length += sg->length; cpu_to_hw_sg(sg); @@ -1327,7 +1472,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg->final = 1; cpu_to_hw_sg(sg); } else { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1335,7 +1480,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; if (ses->auth_only_len) { qm_sg_entry_set64(sg, - dpaa_mem_vtop(sym->aead.aad.data)); + rte_dpaa_mem_vtop(sym->aead.aad.data)); sg->length = ses->auth_only_len; length += sg->length; cpu_to_hw_sg(sg); @@ -1350,7 +1495,7 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) ses->digest_length); sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; length += sg->length; sg->final = 1; @@ -1364,10 +1509,10 @@ build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ sg++; - qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg)); qm_sg_entry_set64(sg, - dst_start_addr + sym->aead.data.offset - ses->auth_only_len); - sg->length = sym->aead.data.length + ses->auth_only_len; + dst_start_addr + sym->aead.data.offset); + sg->length = sym->aead.data.length; length = sg->length; if (is_encode(ses)) { cpu_to_hw_sg(sg); @@ -1433,11 +1578,11 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output sg entries */ sg = &cf->sg[2]; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(out_sg); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->auth.data.offset; sg->offset = sym->auth.data.offset; @@ -1446,7 +1591,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1475,17 +1620,17 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input sg entries */ sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); cpu_to_hw_sg(in_sg); /* 1st seg IV */ - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; cpu_to_hw_sg(sg); /* 2nd seg */ sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len - sym->auth.data.offset; sg->offset = sym->auth.data.offset; @@ -1494,7 +1639,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; mbuf = mbuf->next; } @@ -1505,7 +1650,7 @@ build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) sg++; memcpy(ctx->digest, sym->auth.digest.data, ses->digest_length); - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; } sg->final = 1; @@ -1542,9 +1687,9 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) /* input */ rte_prefetch0(cf->sg); sg = &cf->sg[2]; - qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[1], rte_dpaa_mem_vtop(sg)); if (is_encode(ses)) { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1556,7 +1701,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) sg->final = 1; cpu_to_hw_sg(sg); } else { - qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr)); sg->length = ses->iv.length; length += sg->length; cpu_to_hw_sg(sg); @@ -1572,7 +1717,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) ses->digest_length); sg++; - qm_sg_entry_set64(sg, dpaa_mem_vtop(ctx->digest)); + qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest)); sg->length = ses->digest_length; length += sg->length; sg->final = 1; @@ -1586,7 +1731,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ sg++; - qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop(sg)); + qm_sg_entry_set64(&cf->sg[0], rte_dpaa_mem_vtop(sg)); qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); sg->length = sym->cipher.data.length; length = sg->length; @@ -1609,6 +1754,7 @@ build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) return cf; } +#ifdef RTE_LIB_SECURITY static inline struct dpaa_sec_job * build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) { @@ -1624,10 +1770,10 @@ build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) cf = &ctx->job; ctx->op = op; - src_start_addr = rte_pktmbuf_mtophys(sym->m_src); + src_start_addr = rte_pktmbuf_iova(sym->m_src); if (sym->m_dst) - dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst); + dst_start_addr = rte_pktmbuf_iova(sym->m_dst); else dst_start_addr = src_start_addr; @@ -1679,11 +1825,11 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) /* output */ out_sg = &cf->sg[0]; out_sg->extension = 1; - qm_sg_entry_set64(out_sg, dpaa_mem_vtop(&cf->sg[2])); + qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2])); /* 1st seg */ sg = &cf->sg[2]; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = 0; /* Successive segs */ @@ -1693,7 +1839,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) mbuf = mbuf->next; cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->offset = 0; } sg->length = mbuf->buf_len - mbuf->data_off; @@ -1712,10 +1858,10 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) in_len = mbuf->data_len; sg++; - qm_sg_entry_set64(in_sg, dpaa_mem_vtop(sg)); + qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg)); /* 1st seg */ - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; sg->offset = 0; @@ -1724,7 +1870,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) while (mbuf) { cpu_to_hw_sg(sg); sg++; - qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); + qm_sg_entry_set64(sg, rte_pktmbuf_iova(mbuf)); sg->length = mbuf->data_len; sg->offset = 0; in_len += sg->length; @@ -1740,6 +1886,7 @@ build_proto_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) return cf; } +#endif static uint16_t dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, @@ -1754,16 +1901,24 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, struct rte_crypto_op *op; struct dpaa_sec_job *cf; dpaa_sec_session *ses; - uint32_t auth_only_len, index, flags[DPAA_SEC_BURST] = {0}; + uint16_t auth_hdr_len, auth_tail_len; + uint32_t index, flags[DPAA_SEC_BURST] = {0}; struct qman_fq *inq[DPAA_SEC_BURST]; + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { + if (rte_dpaa_portal_init((void *)0)) { + DPAA_SEC_ERR("Failure in affining portal"); + return 0; + } + } + while (nb_ops) { frames_to_send = (nb_ops > DPAA_SEC_BURST) ? DPAA_SEC_BURST : nb_ops; for (loop = 0; loop < frames_to_send; loop++) { op = *(ops++); - if (op->sym->m_src->seqn != 0) { - index = op->sym->m_src->seqn - 1; + if (*dpaa_seqn(op->sym->m_src) != 0) { + index = *dpaa_seqn(op->sym->m_src) - 1; if (DPAA_PER_LCORE_DQRR_HELD & (1 << index)) { /* QM_EQCR_DCA_IDXMASK = 0x0f */ flags[loop] = ((index & 0x0f) << 8); @@ -1778,14 +1933,16 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, case RTE_CRYPTO_OP_WITH_SESSION: ses = (dpaa_sec_session *) get_sym_session_private_data( - op->sym->session, - cryptodev_driver_id); + op->sym->session, + dpaa_cryptodev_driver_id); break; +#ifdef RTE_LIB_SECURITY case RTE_CRYPTO_OP_SECURITY_SESSION: ses = (dpaa_sec_session *) get_sec_session_private_data( op->sym->sec_session); break; +#endif default: DPAA_SEC_DP_ERR( "sessionless crypto op not supported"); @@ -1793,6 +1950,14 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, nb_ops = loop; goto send_pkts; } + + if (!ses) { + DPAA_SEC_DP_ERR("session not available"); + frames_to_send = loop; + nb_ops = loop; + goto send_pkts; + } + if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) { if (dpaa_sec_attach_sess_q(qp, ses)) { frames_to_send = loop; @@ -1810,43 +1975,75 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, goto send_pkts; } - auth_only_len = op->sym->auth.data.length - + auth_hdr_len = op->sym->auth.data.length - op->sym->cipher.data.length; + auth_tail_len = 0; + if (rte_pktmbuf_is_contiguous(op->sym->m_src) && ((op->sym->m_dst == NULL) || rte_pktmbuf_is_contiguous(op->sym->m_dst))) { - if (is_proto_ipsec(ses)) { - cf = build_proto(op, ses); - } else if (is_proto_pdcp(ses)) { + switch (ses->ctxt) { +#ifdef RTE_LIB_SECURITY + case DPAA_SEC_PDCP: + case DPAA_SEC_IPSEC: cf = build_proto(op, ses); - } else if (is_auth_only(ses)) { + break; +#endif + case DPAA_SEC_AUTH: cf = build_auth_only(op, ses); - } else if (is_cipher_only(ses)) { + break; + case DPAA_SEC_CIPHER: cf = build_cipher_only(op, ses); - } else if (is_aead(ses)) { + break; + case DPAA_SEC_AEAD: cf = build_cipher_auth_gcm(op, ses); - auth_only_len = ses->auth_only_len; - } else if (is_auth_cipher(ses)) { + auth_hdr_len = ses->auth_only_len; + break; + case DPAA_SEC_CIPHER_HASH: + auth_hdr_len = + op->sym->cipher.data.offset + - op->sym->auth.data.offset; + auth_tail_len = + op->sym->auth.data.length + - op->sym->cipher.data.length + - auth_hdr_len; cf = build_cipher_auth(op, ses); - } else { + break; + default: DPAA_SEC_DP_ERR("not supported ops"); frames_to_send = loop; nb_ops = loop; goto send_pkts; } } else { - if (is_proto_pdcp(ses) || is_proto_ipsec(ses)) { + switch (ses->ctxt) { +#ifdef RTE_LIB_SECURITY + case DPAA_SEC_PDCP: + case DPAA_SEC_IPSEC: cf = build_proto_sg(op, ses); - } else if (is_auth_only(ses)) { + break; +#endif + case DPAA_SEC_AUTH: cf = build_auth_only_sg(op, ses); - } else if (is_cipher_only(ses)) { + break; + case DPAA_SEC_CIPHER: cf = build_cipher_only_sg(op, ses); - } else if (is_aead(ses)) { + break; + case DPAA_SEC_AEAD: cf = build_cipher_auth_gcm_sg(op, ses); - auth_only_len = ses->auth_only_len; - } else if (is_auth_cipher(ses)) { + auth_hdr_len = ses->auth_only_len; + break; + case DPAA_SEC_CIPHER_HASH: + auth_hdr_len = + op->sym->cipher.data.offset + - op->sym->auth.data.offset; + auth_tail_len = + op->sym->auth.data.length + - op->sym->cipher.data.length + - auth_hdr_len; cf = build_cipher_auth_sg(op, ses); - } else { + break; + default: DPAA_SEC_DP_ERR("not supported ops"); frames_to_send = loop; nb_ops = loop; @@ -1863,30 +2060,34 @@ dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES]; fd->opaque_addr = 0; fd->cmd = 0; - qm_fd_addr_set64(fd, dpaa_mem_vtop(cf->sg)); + qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg)); fd->_format1 = qm_fd_compound; fd->length29 = 2 * sizeof(struct qm_sg_entry); + /* Auth_only_len is set as 0 in descriptor and it is * overwritten here in the fd.cmd which will update * the DPOVRD reg. */ - if (auth_only_len) - fd->cmd = 0x80000000 | auth_only_len; + if (auth_hdr_len || auth_tail_len) { + fd->cmd = 0x80000000; + fd->cmd |= + ((auth_tail_len << 16) | auth_hdr_len); + } +#ifdef RTE_LIB_SECURITY /* In case of PDCP, per packet HFN is stored in * mbuf priv after sym_op. */ - if (is_proto_pdcp(ses) && ses->pdcp.hfn_ovd) { + if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) { fd->cmd = 0x80000000 | *((uint32_t *)((uint8_t *)op + ses->pdcp.hfn_ovd_offset)); - DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u,%u\n", + DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u\n", *((uint32_t *)((uint8_t *)op + ses->pdcp.hfn_ovd_offset)), - ses->pdcp.hfn_ovd, - is_proto_pdcp(ses)); + ses->pdcp.hfn_ovd); } - +#endif } send_pkts: loop = 0; @@ -1911,6 +2112,13 @@ dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, uint16_t num_rx; struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { + if (rte_dpaa_portal_init((void *)0)) { + DPAA_SEC_ERR("Failure in affining portal"); + return 0; + } + } + num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); dpaa_qp->rx_pkts += num_rx; @@ -1990,15 +2198,6 @@ dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, return 0; } -/** Return the number of allocated queue pairs */ -static uint32_t -dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) -{ - PMD_INIT_FUNC_TRACE(); - - return dev->data->nb_queue_pairs; -} - /** Returns the size of session structure */ static unsigned int dpaa_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) @@ -2013,6 +2212,7 @@ dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, struct rte_crypto_sym_xform *xform, dpaa_sec_session *session) { + session->ctxt = DPAA_SEC_CIPHER; session->cipher_alg = xform->cipher.algo; session->iv.length = xform->cipher.iv.length; session->iv.offset = xform->cipher.iv.offset; @@ -2026,6 +2226,34 @@ dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, memcpy(session->cipher_key.data, xform->cipher.key.data, xform->cipher.key.length); + switch (xform->cipher.algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_AES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_3DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + session->cipher_key.alg = OP_ALG_ALGSEL_AES; + session->cipher_key.algmode = OP_ALG_AAI_CTR; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + session->cipher_key.alg = OP_ALG_ALGSEL_SNOW_F8; + break; + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + session->cipher_key.alg = OP_ALG_ALGSEL_ZUCE; + break; + default: + DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", + xform->cipher.algo); + return -ENOTSUP; + } session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? DIR_ENC : DIR_DEC; @@ -2037,30 +2265,223 @@ dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, struct rte_crypto_sym_xform *xform, dpaa_sec_session *session) { + session->ctxt = DPAA_SEC_AUTH; session->auth_alg = xform->auth.algo; - session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, + session->auth_key.length = xform->auth.key.length; + if (xform->auth.key.length) { + session->auth_key.data = + rte_zmalloc(NULL, xform->auth.key.length, RTE_CACHE_LINE_SIZE); - if (session->auth_key.data == NULL && xform->auth.key.length > 0) { - DPAA_SEC_ERR("No Memory for auth key"); - return -ENOMEM; + if (session->auth_key.data == NULL) { + DPAA_SEC_ERR("No Memory for auth key"); + return -ENOMEM; + } + memcpy(session->auth_key.data, xform->auth.key.data, + xform->auth.key.length); + } - session->auth_key.length = xform->auth.key.length; session->digest_length = xform->auth.digest_length; + if (session->cipher_alg == RTE_CRYPTO_CIPHER_NULL) { + session->iv.offset = xform->auth.iv.offset; + session->iv.length = xform->auth.iv.length; + } + + switch (xform->auth.algo) { + case RTE_CRYPTO_AUTH_SHA1: + session->auth_key.alg = OP_ALG_ALGSEL_SHA1; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_SHA1_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA1; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5: + session->auth_key.alg = OP_ALG_ALGSEL_MD5; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_MD5; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224: + session->auth_key.alg = OP_ALG_ALGSEL_SHA224; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA224; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256: + session->auth_key.alg = OP_ALG_ALGSEL_SHA256; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA256; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384: + session->auth_key.alg = OP_ALG_ALGSEL_SHA384; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA384; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512: + session->auth_key.alg = OP_ALG_ALGSEL_SHA512; + session->auth_key.algmode = OP_ALG_AAI_HASH; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA512; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + session->auth_key.alg = OP_ALG_ALGSEL_SNOW_F9; + session->auth_key.algmode = OP_ALG_AAI_F9; + break; + case RTE_CRYPTO_AUTH_ZUC_EIA3: + session->auth_key.alg = OP_ALG_ALGSEL_ZUCA; + session->auth_key.algmode = OP_ALG_AAI_F9; + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_CMAC; + break; + default: + DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u", + xform->auth.algo); + return -ENOTSUP; + } - memcpy(session->auth_key.data, xform->auth.key.data, - xform->auth.key.length); session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? DIR_ENC : DIR_DEC; return 0; } +static int +dpaa_sec_chain_init(struct rte_cryptodev *dev __rte_unused, + struct rte_crypto_sym_xform *xform, + dpaa_sec_session *session) +{ + + struct rte_crypto_cipher_xform *cipher_xform; + struct rte_crypto_auth_xform *auth_xform; + + session->ctxt = DPAA_SEC_CIPHER_HASH; + if (session->auth_cipher_text) { + cipher_xform = &xform->cipher; + auth_xform = &xform->next->auth; + } else { + cipher_xform = &xform->next->cipher; + auth_xform = &xform->auth; + } + + /* Set IV parameters */ + session->iv.offset = cipher_xform->iv.offset; + session->iv.length = cipher_xform->iv.length; + + session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { + DPAA_SEC_ERR("No Memory for cipher key"); + return -ENOMEM; + } + session->cipher_key.length = cipher_xform->key.length; + session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->auth_key.data == NULL && auth_xform->key.length > 0) { + DPAA_SEC_ERR("No Memory for auth key"); + return -ENOMEM; + } + session->auth_key.length = auth_xform->key.length; + memcpy(session->cipher_key.data, cipher_xform->key.data, + cipher_xform->key.length); + memcpy(session->auth_key.data, auth_xform->key.data, + auth_xform->key.length); + + session->digest_length = auth_xform->digest_length; + session->auth_alg = auth_xform->algo; + + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA1; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_MD5; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA224; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA256; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA384; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + session->auth_key.alg = OP_ALG_ALGSEL_SHA512; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = OP_ALG_ALGSEL_AES; + session->auth_key.algmode = OP_ALG_AAI_CMAC; + break; + default: + DPAA_SEC_ERR("Crypto: Unsupported Auth specified %u", + auth_xform->algo); + return -ENOTSUP; + } + + session->cipher_alg = cipher_xform->algo; + + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_AES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_AES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + session->cipher_key.alg = OP_ALG_ALGSEL_3DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + session->cipher_key.alg = OP_ALG_ALGSEL_AES; + session->cipher_key.algmode = OP_ALG_AAI_CTR; + break; + default: + DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", + cipher_xform->algo); + return -ENOTSUP; + } + session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? + DIR_ENC : DIR_DEC; + return 0; +} + static int dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, struct rte_crypto_sym_xform *xform, dpaa_sec_session *session) { session->aead_alg = xform->aead.algo; + session->ctxt = DPAA_SEC_AEAD; session->iv.length = xform->aead.iv.length; session->iv.offset = xform->aead.iv.offset; session->auth_only_len = xform->aead.aad_length; @@ -2075,6 +2496,17 @@ dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, memcpy(session->aead_key.data, xform->aead.key.data, xform->aead.key.length); + + switch (session->aead_alg) { + case RTE_CRYPTO_AEAD_AES_GCM: + session->aead_key.alg = OP_ALG_ALGSEL_AES; + session->aead_key.algmode = OP_ALG_AAI_GCM; + break; + default: + DPAA_SEC_ERR("unsupported AEAD alg %d", session->aead_alg); + return -ENOTSUP; + } + session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? DIR_ENC : DIR_DEC; @@ -2086,7 +2518,7 @@ dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi) { unsigned int i; - for (i = 0; i < qi->max_nb_sessions * MAX_DPAA_CORES; i++) { + for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) { if (qi->inq_attach[i] == 0) { qi->inq_attach[i] = 1; return &qi->inq[i]; @@ -2102,9 +2534,10 @@ dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) { unsigned int i; - for (i = 0; i < qi->max_nb_sessions; i++) { + for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) { if (&qi->inq[i] == fq) { - qman_retire_fq(fq, NULL); + if (qman_retire_fq(fq, NULL) != 0) + DPAA_SEC_DEBUG("Queue is not retired\n"); qman_oos_fq(fq); qi->inq_attach[i] = 0; return 0; @@ -2113,7 +2546,7 @@ dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) return -1; } -static int +int dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) { int ret; @@ -2122,9 +2555,9 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) ret = dpaa_sec_prep_cdb(sess); if (ret) { DPAA_SEC_ERR("Unable to prepare sec cdb"); - return -1; + return ret; } - if (unlikely(!RTE_PER_LCORE(dpaa_io))) { + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { ret = rte_dpaa_portal_init((void *)0); if (ret) { DPAA_SEC_ERR("Failure in affining portal"); @@ -2132,7 +2565,7 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) } } ret = dpaa_sec_init_rx(sess->inq[rte_lcore_id() % MAX_DPAA_CORES], - dpaa_mem_vtop(&sess->cdb), + rte_dpaa_mem_vtop(&sess->cdb), qman_fq_fqid(&qp->outq)); if (ret) DPAA_SEC_ERR("Unable to init sec queue"); @@ -2140,6 +2573,18 @@ dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) return ret; } +static inline void +free_session_data(dpaa_sec_session *s) +{ + if (is_aead(s)) + rte_free(s->aead_key.data); + else { + rte_free(s->auth_key.data); + rte_free(s->cipher_key.data); + } + memset(s, 0, sizeof(dpaa_sec_session)); +} + static int dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *sess) @@ -2147,6 +2592,7 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, struct dpaa_sec_dev_private *internals = dev->data->dev_private; dpaa_sec_session *session = sess; uint32_t i; + int ret; PMD_INIT_FUNC_TRACE(); @@ -2162,51 +2608,68 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, /* Cipher Only */ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { session->auth_alg = RTE_CRYPTO_AUTH_NULL; - dpaa_sec_cipher_init(dev, xform, session); + ret = dpaa_sec_cipher_init(dev, xform, session); /* Authentication Only */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL) { session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; - dpaa_sec_auth_init(dev, xform, session); + session->ctxt = DPAA_SEC_AUTH; + ret = dpaa_sec_auth_init(dev, xform, session); /* Cipher then Authenticate */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { - dpaa_sec_cipher_init(dev, xform, session); - dpaa_sec_auth_init(dev, xform->next, session); + session->auth_cipher_text = 1; + if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL) + ret = dpaa_sec_auth_init(dev, xform, session); + else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL) + ret = dpaa_sec_cipher_init(dev, xform, session); + else + ret = dpaa_sec_chain_init(dev, xform, session); } else { DPAA_SEC_ERR("Not supported: Auth then Cipher"); - return -EINVAL; + return -ENOTSUP; } - /* Authenticate then Cipher */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { - dpaa_sec_auth_init(dev, xform, session); - dpaa_sec_cipher_init(dev, xform->next, session); + session->auth_cipher_text = 0; + if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL) + ret = dpaa_sec_cipher_init(dev, xform, session); + else if (xform->next->cipher.algo + == RTE_CRYPTO_CIPHER_NULL) + ret = dpaa_sec_auth_init(dev, xform, session); + else + ret = dpaa_sec_chain_init(dev, xform, session); } else { DPAA_SEC_ERR("Not supported: Auth then Cipher"); - return -EINVAL; + return -ENOTSUP; } /* AEAD operation for AES-GCM kind of Algorithms */ } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && xform->next == NULL) { - dpaa_sec_aead_init(dev, xform, session); + ret = dpaa_sec_aead_init(dev, xform, session); } else { DPAA_SEC_ERR("Invalid crypto type"); return -EINVAL; } + if (ret) { + DPAA_SEC_ERR("unable to init session"); + goto err1; + } + rte_spinlock_lock(&internals->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { session->inq[i] = dpaa_sec_attach_rxq(internals); if (session->inq[i] == NULL) { DPAA_SEC_ERR("unable to attach sec queue"); rte_spinlock_unlock(&internals->lock); + ret = -EBUSY; goto err1; } } @@ -2215,11 +2678,8 @@ dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, return 0; err1: - rte_free(session->cipher_key.data); - rte_free(session->auth_key.data); - memset(session, 0, sizeof(dpaa_sec_session)); - - return -EINVAL; + free_session_data(session); + return ret; } static int @@ -2267,9 +2727,7 @@ free_session_memory(struct rte_cryptodev *dev, dpaa_sec_session *s) s->inq[i] = NULL; s->qp[i] = NULL; } - rte_free(s->cipher_key.data); - rte_free(s->auth_key.data); - memset(s, 0, sizeof(dpaa_sec_session)); + free_session_data(s); rte_mempool_put(sess_mp, (void *)s); } @@ -2289,33 +2747,68 @@ dpaa_sec_sym_session_clear(struct rte_cryptodev *dev, } } +#ifdef RTE_LIB_SECURITY static int -dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, - struct rte_security_session_conf *conf, - void *sess) +dpaa_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, + struct rte_security_ipsec_xform *ipsec_xform, + dpaa_sec_session *session) { - struct dpaa_sec_dev_private *internals = dev->data->dev_private; - struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; - struct rte_crypto_auth_xform *auth_xform = NULL; - struct rte_crypto_cipher_xform *cipher_xform = NULL; - dpaa_sec_session *session = (dpaa_sec_session *)sess; - uint32_t i; - PMD_INIT_FUNC_TRACE(); - memset(session, 0, sizeof(dpaa_sec_session)); - if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { - cipher_xform = &conf->crypto_xform->cipher; - if (conf->crypto_xform->next) - auth_xform = &conf->crypto_xform->next->auth; - } else { - auth_xform = &conf->crypto_xform->auth; - if (conf->crypto_xform->next) - cipher_xform = &conf->crypto_xform->next->cipher; + session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, + RTE_CACHE_LINE_SIZE); + if (session->aead_key.data == NULL && aead_xform->key.length > 0) { + DPAA_SEC_ERR("No Memory for aead key"); + return -ENOMEM; } - session->proto_alg = conf->protocol; + memcpy(session->aead_key.data, aead_xform->key.data, + aead_xform->key.length); + + session->digest_length = aead_xform->digest_length; + session->aead_key.length = aead_xform->key.length; + + switch (aead_xform->algo) { + case RTE_CRYPTO_AEAD_AES_GCM: + switch (session->digest_length) { + case 8: + session->aead_key.alg = OP_PCL_IPSEC_AES_GCM8; + break; + case 12: + session->aead_key.alg = OP_PCL_IPSEC_AES_GCM12; + break; + case 16: + session->aead_key.alg = OP_PCL_IPSEC_AES_GCM16; + break; + default: + DPAA_SEC_ERR("Crypto: Undefined GCM digest %d", + session->digest_length); + return -EINVAL; + } + if (session->dir == DIR_ENC) { + memcpy(session->encap_pdb.gcm.salt, + (uint8_t *)&(ipsec_xform->salt), 4); + } else { + memcpy(session->decap_pdb.gcm.salt, + (uint8_t *)&(ipsec_xform->salt), 4); + } + session->aead_key.algmode = OP_ALG_AAI_GCM; + session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; + break; + default: + DPAA_SEC_ERR("Crypto: Undefined AEAD specified %u", + aead_xform->algo); + return -ENOTSUP; + } + return 0; +} - if (cipher_xform && cipher_xform->algo != RTE_CRYPTO_CIPHER_NULL) { +static int +dpaa_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, + struct rte_crypto_auth_xform *auth_xform, + struct rte_security_ipsec_xform *ipsec_xform, + dpaa_sec_session *session) +{ + if (cipher_xform) { session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, RTE_CACHE_LINE_SIZE); @@ -2324,20 +2817,10 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, DPAA_SEC_ERR("No Memory for cipher key"); return -ENOMEM; } + + session->cipher_key.length = cipher_xform->key.length; memcpy(session->cipher_key.data, cipher_xform->key.data, cipher_xform->key.length); - session->cipher_key.length = cipher_xform->key.length; - - switch (cipher_xform->algo) { - case RTE_CRYPTO_CIPHER_AES_CBC: - case RTE_CRYPTO_CIPHER_3DES_CBC: - case RTE_CRYPTO_CIPHER_AES_CTR: - break; - default: - DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u", - cipher_xform->algo); - goto out; - } session->cipher_alg = cipher_xform->algo; } else { session->cipher_key.data = NULL; @@ -2345,46 +2828,187 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; } - if (auth_xform && auth_xform->algo != RTE_CRYPTO_AUTH_NULL) { + if (auth_xform) { session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, RTE_CACHE_LINE_SIZE); if (session->auth_key.data == NULL && auth_xform->key.length > 0) { DPAA_SEC_ERR("No Memory for auth key"); - rte_free(session->cipher_key.data); return -ENOMEM; } + session->auth_key.length = auth_xform->key.length; memcpy(session->auth_key.data, auth_xform->key.data, auth_xform->key.length); - session->auth_key.length = auth_xform->key.length; - - switch (auth_xform->algo) { - case RTE_CRYPTO_AUTH_SHA1_HMAC: - case RTE_CRYPTO_AUTH_MD5_HMAC: - case RTE_CRYPTO_AUTH_SHA256_HMAC: - case RTE_CRYPTO_AUTH_SHA384_HMAC: - case RTE_CRYPTO_AUTH_SHA512_HMAC: - case RTE_CRYPTO_AUTH_AES_CMAC: - break; - default: - DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", - auth_xform->algo); - goto out; - } session->auth_alg = auth_xform->algo; + session->digest_length = auth_xform->digest_length; } else { session->auth_key.data = NULL; session->auth_key.length = 0; session->auth_alg = RTE_CRYPTO_AUTH_NULL; } + switch (session->auth_alg) { + case RTE_CRYPTO_AUTH_SHA1_HMAC: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA1_96; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_MD5_HMAC: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_MD5_96; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA256_HMAC: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_256_128; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + if (session->digest_length != 16) + DPAA_SEC_WARN( + "+++Using sha256-hmac truncated len is non-standard," + "it will not work with lookaside proto"); + break; + case RTE_CRYPTO_AUTH_SHA384_HMAC: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_384_192; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_SHA512_HMAC: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_SHA2_512_256; + session->auth_key.algmode = OP_ALG_AAI_HMAC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = OP_PCL_IPSEC_AES_CMAC_96; + session->auth_key.algmode = OP_ALG_AAI_CMAC; + break; + case RTE_CRYPTO_AUTH_NULL: + session->auth_key.alg = OP_PCL_IPSEC_HMAC_NULL; + break; + case RTE_CRYPTO_AUTH_AES_XCBC_MAC: + session->auth_key.alg = OP_PCL_IPSEC_AES_XCBC_MAC_96; + session->auth_key.algmode = OP_ALG_AAI_XCBC_MAC; + break; + case RTE_CRYPTO_AUTH_SHA224_HMAC: + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + case RTE_CRYPTO_AUTH_SHA1: + case RTE_CRYPTO_AUTH_SHA256: + case RTE_CRYPTO_AUTH_SHA512: + case RTE_CRYPTO_AUTH_SHA224: + case RTE_CRYPTO_AUTH_SHA384: + case RTE_CRYPTO_AUTH_MD5: + case RTE_CRYPTO_AUTH_AES_GMAC: + case RTE_CRYPTO_AUTH_KASUMI_F9: + case RTE_CRYPTO_AUTH_AES_CBC_MAC: + case RTE_CRYPTO_AUTH_ZUC_EIA3: + DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", + session->auth_alg); + return -ENOTSUP; + default: + DPAA_SEC_ERR("Crypto: Undefined Auth specified %u", + session->auth_alg); + return -ENOTSUP; + } + + switch (session->cipher_alg) { + case RTE_CRYPTO_CIPHER_AES_CBC: + session->cipher_key.alg = OP_PCL_IPSEC_AES_CBC; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_DES_CBC: + session->cipher_key.alg = OP_PCL_IPSEC_DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_3DES_CBC: + session->cipher_key.alg = OP_PCL_IPSEC_3DES; + session->cipher_key.algmode = OP_ALG_AAI_CBC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + session->cipher_key.alg = OP_PCL_IPSEC_AES_CTR; + session->cipher_key.algmode = OP_ALG_AAI_CTR; + if (session->dir == DIR_ENC) { + session->encap_pdb.ctr.ctr_initial = 0x00000001; + session->encap_pdb.ctr.ctr_nonce = ipsec_xform->salt; + } else { + session->decap_pdb.ctr.ctr_initial = 0x00000001; + session->decap_pdb.ctr.ctr_nonce = ipsec_xform->salt; + } + break; + case RTE_CRYPTO_CIPHER_NULL: + session->cipher_key.alg = OP_PCL_IPSEC_NULL; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + case RTE_CRYPTO_CIPHER_3DES_ECB: + case RTE_CRYPTO_CIPHER_AES_ECB: + case RTE_CRYPTO_CIPHER_KASUMI_F8: + DPAA_SEC_ERR("Crypto: Unsupported Cipher alg %u", + session->cipher_alg); + return -ENOTSUP; + default: + DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", + session->cipher_alg); + return -ENOTSUP; + } + + return 0; +} + +static int +dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, + void *sess) +{ + struct dpaa_sec_dev_private *internals = dev->data->dev_private; + struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; + struct rte_crypto_auth_xform *auth_xform = NULL; + struct rte_crypto_cipher_xform *cipher_xform = NULL; + struct rte_crypto_aead_xform *aead_xform = NULL; + dpaa_sec_session *session = (dpaa_sec_session *)sess; + uint32_t i; + int ret; + + PMD_INIT_FUNC_TRACE(); + + memset(session, 0, sizeof(dpaa_sec_session)); + session->proto_alg = conf->protocol; + session->ctxt = DPAA_SEC_IPSEC; + + if (ipsec_xform->life.bytes_hard_limit != 0 || + ipsec_xform->life.bytes_soft_limit != 0 || + ipsec_xform->life.packets_hard_limit != 0 || + ipsec_xform->life.packets_soft_limit != 0) + return -ENOTSUP; + + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) + session->dir = DIR_ENC; + else + session->dir = DIR_DEC; + + if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { + cipher_xform = &conf->crypto_xform->cipher; + if (conf->crypto_xform->next) + auth_xform = &conf->crypto_xform->next->auth; + ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform, + ipsec_xform, session); + } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + auth_xform = &conf->crypto_xform->auth; + if (conf->crypto_xform->next) + cipher_xform = &conf->crypto_xform->next->cipher; + ret = dpaa_sec_ipsec_proto_init(cipher_xform, auth_xform, + ipsec_xform, session); + } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { + aead_xform = &conf->crypto_xform->aead; + ret = dpaa_sec_ipsec_aead_init(aead_xform, + ipsec_xform, session); + } else { + DPAA_SEC_ERR("XFORM not specified"); + ret = -EINVAL; + goto out; + } + if (ret) { + DPAA_SEC_ERR("Failed to process xform"); + goto out; + } + if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) { - memset(&session->encap_pdb, 0, - sizeof(struct ipsec_encap_pdb) + - sizeof(session->ip4_hdr)); session->ip4_hdr.ip_v = IPVERSION; session->ip4_hdr.ip_hl = 5; session->ip4_hdr.ip_len = rte_cpu_to_be_16( @@ -2407,9 +3031,6 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, session->encap_pdb.ip_hdr_len = sizeof(struct ip); } else if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV6) { - memset(&session->encap_pdb, 0, - sizeof(struct ipsec_encap_pdb) + - sizeof(session->ip6_hdr)); session->ip6_hdr.vtc_flow = rte_cpu_to_be_32( DPAA_IPv6_DEFAULT_VTC_FLOW | ((ipsec_xform->tunnel.ipv6.dscp << @@ -2432,19 +3053,20 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, session->encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr); } + session->encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | PDBOPTS_ESP_OIHI_PDB_INL | PDBOPTS_ESP_IVSRC | - PDBHMO_ESP_ENCAP_DTTL | PDBHMO_ESP_SNR; + if (ipsec_xform->options.dec_ttl) + session->encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL; if (ipsec_xform->options.esn) session->encap_pdb.options |= PDBOPTS_ESP_ESN; session->encap_pdb.spi = ipsec_xform->spi; - session->dir = DIR_ENC; + } else if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { - memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); if (ipsec_xform->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) session->decap_pdb.options = sizeof(struct ip) << 16; else @@ -2452,7 +3074,27 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, sizeof(struct rte_ipv6_hdr) << 16; if (ipsec_xform->options.esn) session->decap_pdb.options |= PDBOPTS_ESP_ESN; - session->dir = DIR_DEC; + if (ipsec_xform->replay_win_sz) { + uint32_t win_sz; + win_sz = rte_align32pow2(ipsec_xform->replay_win_sz); + + switch (win_sz) { + case 1: + case 2: + case 4: + case 8: + case 16: + case 32: + session->decap_pdb.options |= PDBOPTS_ESP_ARS32; + break; + case 64: + session->decap_pdb.options |= PDBOPTS_ESP_ARS64; + break; + default: + session->decap_pdb.options |= + PDBOPTS_ESP_ARS128; + } + } } else goto out; rte_spinlock_lock(&internals->lock); @@ -2468,9 +3110,7 @@ dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, return 0; out: - rte_free(session->auth_key.data); - rte_free(session->cipher_key.data); - memset(session, 0, sizeof(dpaa_sec_session)); + free_session_data(session); return -1; } @@ -2486,6 +3126,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, dpaa_sec_session *session = (dpaa_sec_session *)sess; struct dpaa_sec_dev_private *dev_priv = dev->data->dev_private; uint32_t i; + int ret; PMD_INIT_FUNC_TRACE(); @@ -2494,11 +3135,13 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, /* find xfrm types */ if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { cipher_xform = &xform->cipher; - if (xform->next != NULL) + if (xform->next != NULL && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) auth_xform = &xform->next->auth; } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { auth_xform = &xform->auth; - if (xform->next != NULL) + if (xform->next != NULL && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) cipher_xform = &xform->next->cipher; } else { DPAA_SEC_ERR("Invalid crypto type"); @@ -2506,7 +3149,28 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, } session->proto_alg = conf->protocol; + session->ctxt = DPAA_SEC_PDCP; + if (cipher_xform) { + switch (cipher_xform->algo) { + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + session->cipher_key.alg = PDCP_CIPHER_TYPE_SNOW; + break; + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + session->cipher_key.alg = PDCP_CIPHER_TYPE_ZUC; + break; + case RTE_CRYPTO_CIPHER_AES_CTR: + session->cipher_key.alg = PDCP_CIPHER_TYPE_AES; + break; + case RTE_CRYPTO_CIPHER_NULL: + session->cipher_key.alg = PDCP_CIPHER_TYPE_NULL; + break; + default: + DPAA_SEC_ERR("Crypto: Undefined Cipher specified %u", + session->cipher_alg); + return -EINVAL; + } + session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, RTE_CACHE_LINE_SIZE); @@ -2533,11 +3197,31 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, pdcp_xform->sn_size != RTE_SECURITY_PDCP_SN_SIZE_12) { DPAA_SEC_ERR( "PDCP Seq Num size should be 5/12 bits for cmode"); + ret = -EINVAL; goto out; } } if (auth_xform) { + switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SNOW3G_UIA2: + session->auth_key.alg = PDCP_AUTH_TYPE_SNOW; + break; + case RTE_CRYPTO_AUTH_ZUC_EIA3: + session->auth_key.alg = PDCP_AUTH_TYPE_ZUC; + break; + case RTE_CRYPTO_AUTH_AES_CMAC: + session->auth_key.alg = PDCP_AUTH_TYPE_AES; + break; + case RTE_CRYPTO_AUTH_NULL: + session->auth_key.alg = PDCP_AUTH_TYPE_NULL; + break; + default: + DPAA_SEC_ERR("Crypto: Unsupported auth alg %u", + session->auth_alg); + rte_free(session->cipher_key.data); + return -EINVAL; + } session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, RTE_CACHE_LINE_SIZE); @@ -2563,7 +3247,9 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, session->pdcp.hfn = pdcp_xform->hfn; session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold; session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd; - session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; + session->pdcp.sdap_enabled = pdcp_xform->sdap_enabled; + if (cipher_xform) + session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; rte_spinlock_lock(&dev_priv->lock); for (i = 0; i < MAX_DPAA_CORES; i++) { @@ -2571,6 +3257,7 @@ dpaa_sec_set_pdcp_session(struct rte_cryptodev *dev, if (session->inq[i] == NULL) { DPAA_SEC_ERR("unable to attach sec queue"); rte_spinlock_unlock(&dev_priv->lock); + ret = -EBUSY; goto out; } } @@ -2580,7 +3267,7 @@ out: rte_free(session->auth_key.data); rte_free(session->cipher_key.data); memset(session, 0, sizeof(dpaa_sec_session)); - return -1; + return ret; } static int @@ -2639,7 +3326,7 @@ dpaa_sec_security_session_destroy(void *dev __rte_unused, } return 0; } - +#endif static int dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, struct rte_cryptodev_config *config __rte_unused) @@ -2685,7 +3372,7 @@ dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, info->feature_flags = dev->feature_flags; info->capabilities = dpaa_sec_capabilities; info->sym.max_nb_sessions = internals->max_nb_sessions; - info->driver_id = cryptodev_driver_id; + info->driver_id = dpaa_cryptodev_driver_id; } } @@ -2707,7 +3394,7 @@ dpaa_sec_process_parallel_event(void *event, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -2762,7 +3449,7 @@ dpaa_sec_process_atomic_event(void *event, * sg[0] is for output * sg[1] for input */ - job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); + job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd)); ctx = container_of(job, struct dpaa_sec_op_ctx, job); ctx->fd_status = fd->status; @@ -2797,7 +3484,7 @@ dpaa_sec_process_atomic_event(void *event, DPAA_PER_LCORE_DQRR_HELD |= 1 << index; DPAA_PER_LCORE_DQRR_MBUF(index) = ctx->op->sym->m_src; ev->impl_opaque = index + 1; - ctx->op->sym->m_src->seqn = (uint32_t)index + 1; + *dpaa_seqn(ctx->op->sym->m_src) = (uint32_t)index + 1; *bufs = (void *)ctx->op; rte_mempool_put(ctx->ctx_pool, (void *)ctx); @@ -2831,7 +3518,7 @@ dpaa_sec_eventq_attach(const struct rte_cryptodev *dev, break; case RTE_SCHED_TYPE_ORDERED: DPAA_SEC_ERR("Ordered queue schedule type is not supported\n"); - return -1; + return -ENOTSUP; default: opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK; qp->outq.cb.dqrr_dpdk_cb = dpaa_sec_process_parallel_event; @@ -2879,12 +3566,15 @@ static struct rte_cryptodev_ops crypto_ops = { .dev_infos_get = dpaa_sec_dev_infos_get, .queue_pair_setup = dpaa_sec_queue_pair_setup, .queue_pair_release = dpaa_sec_queue_pair_release, - .queue_pair_count = dpaa_sec_queue_pair_count, .sym_session_get_size = dpaa_sec_sym_session_get_size, .sym_session_configure = dpaa_sec_sym_session_configure, - .sym_session_clear = dpaa_sec_sym_session_clear + .sym_session_clear = dpaa_sec_sym_session_clear, + /* Raw data-path API related operations */ + .sym_get_raw_dp_ctx_size = dpaa_sec_get_dp_ctx_size, + .sym_configure_raw_dp_ctx = dpaa_sec_configure_raw_dp_ctx, }; +#ifdef RTE_LIB_SECURITY static const struct rte_security_capability * dpaa_sec_capabilities_get(void *device __rte_unused) { @@ -2899,7 +3589,7 @@ static const struct rte_security_ops dpaa_sec_security_ops = { .set_pkt_metadata = NULL, .capabilities_get = dpaa_sec_capabilities_get }; - +#endif static int dpaa_sec_uninit(struct rte_cryptodev *dev) { @@ -2919,18 +3609,82 @@ dpaa_sec_uninit(struct rte_cryptodev *dev) return 0; } +static int +check_devargs_handler(__rte_unused const char *key, const char *value, + __rte_unused void *opaque) +{ + dpaa_sec_dp_dump = atoi(value); + if (dpaa_sec_dp_dump > DPAA_SEC_DP_FULL_DUMP) { + DPAA_SEC_WARN("WARN: DPAA_SEC_DP_DUMP_LEVEL is not " + "supported, changing to FULL error prints\n"); + dpaa_sec_dp_dump = DPAA_SEC_DP_FULL_DUMP; + } + + return 0; +} + +static void +dpaa_sec_get_devargs(struct rte_devargs *devargs, const char *key) +{ + struct rte_kvargs *kvlist; + + if (!devargs) + return; + + kvlist = rte_kvargs_parse(devargs->args, NULL); + if (!kvlist) + return; + + if (!rte_kvargs_count(kvlist, key)) { + rte_kvargs_free(kvlist); + return; + } + + rte_kvargs_process(kvlist, key, + check_devargs_handler, NULL); + rte_kvargs_free(kvlist); +} + static int dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) { struct dpaa_sec_dev_private *internals; +#ifdef RTE_LIB_SECURITY struct rte_security_ctx *security_instance; +#endif struct dpaa_sec_qp *qp; uint32_t i, flags; int ret; + void *cmd_map; + int map_fd = -1; PMD_INIT_FUNC_TRACE(); - cryptodev->driver_id = cryptodev_driver_id; + internals = cryptodev->data->dev_private; + map_fd = open("/dev/mem", O_RDWR); + if (unlikely(map_fd < 0)) { + DPAA_SEC_ERR("Unable to open (/dev/mem)"); + return map_fd; + } + internals->sec_hw = mmap(NULL, MAP_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED, map_fd, SEC_BASE_ADDR); + if (internals->sec_hw == MAP_FAILED) { + DPAA_SEC_ERR("Memory map failed"); + close(map_fd); + return -EINVAL; + } + cmd_map = (uint8_t *)internals->sec_hw + + (BLOCK_OFFSET * QI_BLOCK_NUMBER) + CMD_REG; + if (!(be32_to_cpu(rte_read32(cmd_map)) & QICTL_DQEN)) + /* enable QI interface */ + rte_write32(cpu_to_be32(QICTL_DQEN), cmd_map); + + ret = munmap(internals->sec_hw, MAP_SIZE); + if (ret) + DPAA_SEC_WARN("munmap failed\n"); + + close(map_fd); + cryptodev->driver_id = dpaa_cryptodev_driver_id; cryptodev->dev_ops = &crypto_ops; cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; @@ -2939,13 +3693,13 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) RTE_CRYPTODEV_FF_HW_ACCELERATED | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_SECURITY | + RTE_CRYPTODEV_FF_SYM_RAW_DP | RTE_CRYPTODEV_FF_IN_PLACE_SGL | RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; - internals = cryptodev->data->dev_private; internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS; internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; @@ -2958,7 +3712,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) DPAA_SEC_WARN("Device already init by primary process"); return 0; } - +#ifdef RTE_LIB_SECURITY /* Initialize security_ctx only for primary process*/ security_instance = rte_malloc("rte_security_instances_ops", sizeof(struct rte_security_ctx), 0); @@ -2968,7 +3722,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) security_instance->ops = &dpaa_sec_security_ops; security_instance->sess_cnt = 0; cryptodev->security_ctx = security_instance; - +#endif rte_spinlock_init(&internals->lock); for (i = 0; i < internals->max_nb_queue_pairs; i++) { /* init qman fq for queue pair */ @@ -2982,7 +3736,7 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | QMAN_FQ_FLAG_TO_DCPORTAL; - for (i = 0; i < MAX_DPAA_CORES * internals->max_nb_sessions; i++) { + for (i = 0; i < RTE_DPAA_MAX_RX_QUEUE; i++) { /* create rx qman fq for sessions*/ ret = qman_create_fq(0, flags, &internals->inq[i]); if (unlikely(ret != 0)) { @@ -2991,13 +3745,15 @@ dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) } } + dpaa_sec_get_devargs(cryptodev->device->devargs, DRIVER_DUMP_MODE); + RTE_LOG(INFO, PMD, "%s cryptodev init\n", cryptodev->data->name); return 0; init_error: DPAA_SEC_ERR("driver %s: create failed\n", cryptodev->data->name); - dpaa_sec_uninit(cryptodev); + rte_free(cryptodev->security_ctx); return -EFAULT; } @@ -3010,8 +3766,7 @@ cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused, int retval; - snprintf(cryptodev_name, sizeof(cryptodev_name), "dpaa_sec-%d", - dpaa_dev->id.dev_id); + snprintf(cryptodev_name, sizeof(cryptodev_name), "%s", dpaa_dev->name); cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); if (cryptodev == NULL) @@ -3051,18 +3806,30 @@ cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv __rte_unused, } } + if (unlikely(!DPAA_PER_LCORE_PORTAL)) { + retval = rte_dpaa_portal_init((void *)1); + if (retval) { + DPAA_SEC_ERR("Unable to initialize portal"); + goto out; + } + } + /* Invoke PMD device initialization function */ retval = dpaa_sec_dev_init(cryptodev); - if (retval == 0) + if (retval == 0) { + rte_cryptodev_pmd_probing_finish(cryptodev); return 0; + } + retval = -ENXIO; +out: /* In case of error, cleanup is done */ if (rte_eal_process_type() == RTE_PROC_PRIMARY) rte_free(cryptodev->data->dev_private); rte_cryptodev_pmd_release_device(cryptodev); - return -ENXIO; + return retval; } static int @@ -3095,11 +3862,7 @@ static struct cryptodev_driver dpaa_sec_crypto_drv; RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver, - cryptodev_driver_id); - -RTE_INIT(dpaa_sec_init_log) -{ - dpaa_logtype_sec = rte_log_register("pmd.crypto.dpaa"); - if (dpaa_logtype_sec >= 0) - rte_log_set_level(dpaa_logtype_sec, RTE_LOG_NOTICE); -} + dpaa_cryptodev_driver_id); +RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA_SEC_PMD, + DRIVER_DUMP_MODE "="); +RTE_LOG_REGISTER(dpaa_logtype_sec, pmd.crypto.dpaa, NOTICE);