X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Faesni_mb%2Frte_aesni_mb_pmd.c;h=a01c826a3c306a3a1e30677d8226b7eb4979c69b;hb=328d690d2f6013b226004646db2ca53a5e16624e;hp=e49428358652de4a7bc79d254c81dcd09c49194e;hpb=c94c520b4163cf21759cedfe0fe744747ebb4425;p=dpdk.git diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index e494283586..a01c826a3c 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -125,6 +125,18 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) return AESNI_MB_OP_NOT_SUPPORTED; } +static inline int +is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode) +{ +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM || + (hash_alg == AES_GMAC && cipher_mode == GCM)); +#else + return ((hash_alg == AES_GMAC && cipher_mode == GCM) || + hash_alg == AES_CCM); +#endif +} + /** Set session authentication parameters */ static int aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, @@ -146,6 +158,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, return -1; } + /* Set IV parameters */ + sess->auth_iv.offset = xform->auth.iv.offset; + sess->auth_iv.length = xform->auth.iv.length; + /* Set the request digest size */ sess->auth.req_digest_len = xform->auth.digest_length; @@ -209,19 +225,11 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, sess->cipher.direction = DECRYPT; sess->auth.algo = AES_GMAC; - /* - * Multi-buffer lib supports 8, 12 and 16 bytes of digest. - * If size requested is different, generate the full digest - * (16 bytes) in a temporary location and then memcpy - * the requested number of bytes. - */ - if (sess->auth.req_digest_len != 16 && - sess->auth.req_digest_len != 12 && - sess->auth.req_digest_len != 8) { - sess->auth.gen_digest_len = 16; - } else { - sess->auth.gen_digest_len = sess->auth.req_digest_len; + if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; } + sess->auth.gen_digest_len = sess->auth.req_digest_len; sess->iv.length = xform->auth.iv.length; sess->iv.offset = xform->auth.iv.offset; @@ -249,6 +257,48 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, return 0; } +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { + sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN; + uint16_t zuc_eia3_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN); + if (sess->auth.req_digest_len != zuc_eia3_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16); + return 0; + } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { + sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN; + uint16_t snow3g_uia2_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN); + if (sess->auth.req_digest_len != snow3g_uia2_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data, + &sess->auth.pKeySched_snow3g_auth); + return 0; + } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) { + sess->auth.algo = IMB_AUTH_KASUMI_UIA1; + uint16_t kasumi_f9_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1); + if (sess->auth.req_digest_len != kasumi_f9_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data, + &sess->auth.pKeySched_kasumi_auth); + return 0; + } +#endif + switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_MD5_HMAC: sess->auth.algo = MD5; @@ -381,6 +431,11 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, uint8_t is_aes = 0; uint8_t is_3DES = 0; uint8_t is_docsis = 0; +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + uint8_t is_zuc = 0; + uint8_t is_snow3g = 0; + uint8_t is_kasumi = 0; +#endif if (xform == NULL) { sess->cipher.mode = NULL_CIPHER; @@ -434,6 +489,20 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, sess->cipher.mode = ECB; is_aes = 1; break; +#endif +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + sess->cipher.mode = IMB_CIPHER_ZUC_EEA3; + is_zuc = 1; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN; + is_snow3g = 1; + break; + case RTE_CRYPTO_CIPHER_KASUMI_F8: + sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN; + is_kasumi = 1; + break; #endif default: AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter"); @@ -477,7 +546,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, sess->cipher.expanded_aes_keys.encode, sess->cipher.expanded_aes_keys.decode); break; -#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3) +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM case AES_256_BYTES: sess->cipher.key_length_in_bytes = AES_256_BYTES; IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data, @@ -533,6 +602,32 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, } sess->cipher.key_length_in_bytes = 24; +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + } else if (is_zuc) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data, + 16); + } else if (is_snow3g) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data, + &sess->cipher.pKeySched_snow3g_cipher); + } else if (is_kasumi) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data, + &sess->cipher.pKeySched_kasumi_cipher); +#endif } else { if (xform->cipher.key.length != 8) { AESNI_MB_LOG(ERR, "Invalid cipher key length"); @@ -570,6 +665,14 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, return -EINVAL; } + /* Set IV parameters */ + sess->iv.offset = xform->aead.iv.offset; + sess->iv.length = xform->aead.iv.length; + + /* Set digest sizes */ + sess->auth.req_digest_len = xform->aead.digest_length; + sess->auth.gen_digest_len = sess->auth.req_digest_len; + switch (xform->aead.algo) { case RTE_CRYPTO_AEAD_AES_CCM: sess->cipher.mode = CCM; @@ -583,11 +686,24 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, sess->cipher.expanded_aes_keys.encode, sess->cipher.expanded_aes_keys.decode); break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; default: AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } + /* CCM digests must be between 4 and 16 and an even number */ + if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN || + sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN || + (sess->auth.req_digest_len & 1) == 1) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } break; case RTE_CRYPTO_AEAD_AES_GCM: @@ -615,27 +731,37 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, return -EINVAL; } + /* GCM digest size must be between 1 and 16 */ + if (sess->auth.req_digest_len == 0 || + sess->auth.req_digest_len > 16) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: + sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305; + sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305; + + if (xform->aead.key.length != 32) { + AESNI_MB_LOG(ERR, "Invalid key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 32; + memcpy(sess->cipher.expanded_aes_keys.encode, + xform->aead.key.data, 32); + if (sess->auth.req_digest_len != 16) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + break; +#endif default: AESNI_MB_LOG(ERR, "Unsupported aead mode parameter"); return -ENOTSUP; } - /* Set IV parameters */ - sess->iv.offset = xform->aead.iv.offset; - sess->iv.length = xform->aead.iv.length; - - sess->auth.req_digest_len = xform->aead.digest_length; - /* CCM digests must be between 4 and 16 and an even number */ - if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN || - sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN || - (sess->auth.req_digest_len & 1) == 1) { - AESNI_MB_LOG(ERR, "Invalid digest size\n"); - return -EINVAL; - } - sess->auth.gen_digest_len = sess->auth.req_digest_len; - return 0; } @@ -699,6 +825,7 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, /* Default IV length = 0 */ sess->iv.length = 0; + sess->auth_iv.length = 0; ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform); if (ret != 0) { @@ -981,7 +1108,8 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session, static inline void set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, union rte_crypto_sym_ofs sofs, void *buf, uint32_t len, - void *iv, void *aad, void *digest, void *udata) + struct rte_crypto_va_iova_ptr *iv, + struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata) { /* Set crypto operation */ job->chain_order = session->chain_order; @@ -994,7 +1122,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, /* Set authentication parameters */ job->hash_alg = session->auth.algo; - job->iv = iv; + job->iv = iv->va; switch (job->hash_alg) { case AES_XCBC: @@ -1009,7 +1137,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, break; case AES_CCM: - job->u.CCM.aad = (uint8_t *)aad + 18; + job->u.CCM.aad = (uint8_t *)aad->va + 18; job->u.CCM.aad_len_in_bytes = session->aead.aad_len; job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; @@ -1030,7 +1158,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, case AES_GMAC: if (session->cipher.mode == GCM) { - job->u.GCM.aad = aad; + job->u.GCM.aad = aad->va; job->u.GCM.aad_len_in_bytes = session->aead.aad_len; } else { /* For GMAC */ @@ -1042,6 +1170,14 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, job->aes_dec_key_expanded = &session->cipher.gcm_key; break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->u.CHACHA20_POLY1305.aad = aad->va; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode; + break; +#endif default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; @@ -1129,6 +1265,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, /* Set authentication parameters */ job->hash_alg = session->auth.algo; + const int aead = is_aead_algo(job->hash_alg, job->cipher_mode); + switch (job->hash_alg) { case AES_XCBC: job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded; @@ -1174,7 +1312,29 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->aes_enc_key_expanded = &session->cipher.gcm_key; job->aes_dec_key_expanded = &session->cipher.gcm_key; break; - +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + case IMB_AUTH_ZUC_EIA3_BITLEN: + job->u.ZUC_EIA3._key = session->auth.zuc_auth_key; + job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->auth_iv.offset); + break; + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth; + job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->auth_iv.offset); + break; + case IMB_AUTH_KASUMI_UIA1: + job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth; + break; +#endif +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode; + break; +#endif default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer; @@ -1192,6 +1352,24 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, } } + if (aead) + m_offset = op->sym->aead.data.offset; + else + m_offset = op->sym->cipher.data.offset; + +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) { + job->aes_enc_key_expanded = session->cipher.zuc_cipher_key; + job->aes_dec_key_expanded = session->cipher.zuc_cipher_key; + } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) { + job->enc_keys = &session->cipher.pKeySched_snow3g_cipher; + m_offset = 0; + } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) { + job->enc_keys = &session->cipher.pKeySched_kasumi_cipher; + m_offset = 0; + } +#endif + if (!op->sym->m_dst) { /* in-place operation */ m_dst = m_src; @@ -1206,20 +1384,13 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, oop = 1; } - if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && - session->cipher.mode == GCM)) - m_offset = op->sym->aead.data.offset; - else - m_offset = op->sym->cipher.data.offset; - /* Set digest output location */ if (job->hash_alg != NULL_HASH && session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { job->auth_tag_output = qp->temp_digests[*digest_idx]; *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { - if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && - session->cipher.mode == GCM)) + if (aead) job->auth_tag_output = op->sym->aead.digest.data; else job->auth_tag_output = op->sym->auth.digest.data; @@ -1279,7 +1450,21 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, session->iv.offset); break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->msg_len_to_cipher_in_bytes = + op->sym->aead.data.length; + job->msg_len_to_hash_in_bytes = + op->sym->aead.data.length; + + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + break; +#endif default: + /* For SNOW3G, length and offsets are already in bits */ job->cipher_start_src_offset_in_bytes = op->sym->cipher.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length; @@ -1292,6 +1477,13 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, session->iv.offset); } +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) + job->msg_len_to_cipher_in_bytes >>= 3; + else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1) + job->msg_len_to_hash_in_bytes >>= 3; +#endif + /* Set user data to be crypto operation data struct */ job->user_data = op; @@ -1469,9 +1661,7 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) break; if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { - if (job->hash_alg == AES_CCM || - (job->hash_alg == AES_GMAC && - sess->cipher.mode == GCM)) + if (is_aead_algo(job->hash_alg, sess->cipher.mode)) verify_digest(job, op->sym->aead.digest.data, sess->auth.req_digest_len, @@ -1750,7 +1940,7 @@ generate_sync_dgst(struct rte_crypto_sym_vec *vec, for (i = 0, k = 0; i != vec->num; i++) { if (vec->status[i] == 0) { - memcpy(vec->digest[i], dgst[i], len); + memcpy(vec->digest[i].va, dgst[i], len); k++; } } @@ -1766,7 +1956,7 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec, for (i = 0, k = 0; i != vec->num; i++) { if (vec->status[i] == 0) { - if (memcmp(vec->digest[i], dgst[i], len) != 0) + if (memcmp(vec->digest[i].va, dgst[i], len) != 0) vec->status[i] = EBADMSG; else k++; @@ -1829,9 +2019,8 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev, } /* Submit job for processing */ - set_cpu_mb_job_params(job, s, sofs, buf, len, - vec->iv[i], vec->aad[i], tmp_dgst[i], - &vec->status[i]); + set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i], + &vec->aad[i], tmp_dgst[i], &vec->status[i]); job = submit_sync_job(mb_mgr); j++; @@ -1915,6 +2104,7 @@ cryptodev_aesni_mb_create(const char *name, RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO | + RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA | RTE_CRYPTODEV_FF_SYM_SESSIONLESS; #ifdef AESNI_MB_DOCSIS_SEC_ENABLED @@ -2039,4 +2229,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv, cryptodev_aesni_mb_pmd_drv.driver, cryptodev_driver_id); -RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE); +RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);