X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Faesni_mb%2Frte_aesni_mb_pmd.c;h=03186485f993253827679e367ad4df1860710214;hb=53e6f86cf550a75fb1d55a76f411e214e783d14d;hp=4bfc752dcab743e8fb55eee068d7e3952bcfe511;hpb=20731cd8cb0799aff1cc2230d345c9e1f3420475;p=dpdk.git diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 4bfc752dca..03186485f9 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -12,11 +12,10 @@ #include #include #include +#include #include "aesni_mb_pmd_private.h" -int aesni_mb_logtype_driver; - #define AES_CCM_DIGEST_MIN_LEN 4 #define AES_CCM_DIGEST_MAX_LEN 16 #define HMAC_MAX_BLOCK_SIZE 128 @@ -126,6 +125,18 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform) return AESNI_MB_OP_NOT_SUPPORTED; } +static inline int +is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode) +{ +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM || + (hash_alg == AES_GMAC && cipher_mode == GCM)); +#else + return ((hash_alg == AES_GMAC && cipher_mode == GCM) || + hash_alg == AES_CCM); +#endif +} + /** Set session authentication parameters */ static int aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, @@ -147,6 +158,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, return -1; } + /* Set IV parameters */ + sess->auth_iv.offset = xform->auth.iv.offset; + sess->auth_iv.length = xform->auth.iv.length; + /* Set the request digest size */ sess->auth.req_digest_len = xform->auth.digest_length; @@ -210,19 +225,11 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, sess->cipher.direction = DECRYPT; sess->auth.algo = AES_GMAC; - /* - * Multi-buffer lib supports 8, 12 and 16 bytes of digest. - * If size requested is different, generate the full digest - * (16 bytes) in a temporary location and then memcpy - * the requested number of bytes. - */ - if (sess->auth.req_digest_len != 16 && - sess->auth.req_digest_len != 12 && - sess->auth.req_digest_len != 8) { - sess->auth.gen_digest_len = 16; - } else { - sess->auth.gen_digest_len = sess->auth.req_digest_len; + if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; } + sess->auth.gen_digest_len = sess->auth.req_digest_len; sess->iv.length = xform->auth.iv.length; sess->iv.offset = xform->auth.iv.offset; @@ -250,6 +257,48 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, return 0; } +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) { + sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN; + uint16_t zuc_eia3_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN); + if (sess->auth.req_digest_len != zuc_eia3_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16); + return 0; + } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { + sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN; + uint16_t snow3g_uia2_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN); + if (sess->auth.req_digest_len != snow3g_uia2_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data, + &sess->auth.pKeySched_snow3g_auth); + return 0; + } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) { + sess->auth.algo = IMB_AUTH_KASUMI_UIA1; + uint16_t kasumi_f9_digest_len = + get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1); + if (sess->auth.req_digest_len != kasumi_f9_digest_len) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + sess->auth.gen_digest_len = sess->auth.req_digest_len; + + IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data, + &sess->auth.pKeySched_kasumi_auth); + return 0; + } +#endif + switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_MD5_HMAC: sess->auth.algo = MD5; @@ -381,6 +430,12 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, { uint8_t is_aes = 0; uint8_t is_3DES = 0; + uint8_t is_docsis = 0; +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + uint8_t is_zuc = 0; + uint8_t is_snow3g = 0; + uint8_t is_kasumi = 0; +#endif if (xform == NULL) { sess->cipher.mode = NULL_CIPHER; @@ -417,7 +472,7 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, break; case RTE_CRYPTO_CIPHER_AES_DOCSISBPI: sess->cipher.mode = DOCSIS_SEC_BPI; - is_aes = 1; + is_docsis = 1; break; case RTE_CRYPTO_CIPHER_DES_CBC: sess->cipher.mode = DES; @@ -429,6 +484,26 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, sess->cipher.mode = DES3; is_3DES = 1; break; +#if IMB_VERSION(0, 53, 0) <= IMB_VERSION_NUM + case RTE_CRYPTO_CIPHER_AES_ECB: + sess->cipher.mode = ECB; + is_aes = 1; + break; +#endif +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + case RTE_CRYPTO_CIPHER_ZUC_EEA3: + sess->cipher.mode = IMB_CIPHER_ZUC_EEA3; + is_zuc = 1; + break; + case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: + sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN; + is_snow3g = 1; + break; + case RTE_CRYPTO_CIPHER_KASUMI_F8: + sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN; + is_kasumi = 1; + break; +#endif default: AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter"); return -ENOTSUP; @@ -463,6 +538,26 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } + } else if (is_docsis) { + switch (xform->cipher.key.length) { + case AES_128_BYTES: + sess->cipher.key_length_in_bytes = AES_128_BYTES; + IMB_AES_KEYEXP_128(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; +#endif + default: + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } } else if (is_3DES) { uint64_t *keys[3] = {sess->cipher.exp_3des_keys.key[0], sess->cipher.exp_3des_keys.key[1], @@ -507,6 +602,32 @@ aesni_mb_set_session_cipher_parameters(const MB_MGR *mb_mgr, } sess->cipher.key_length_in_bytes = 24; +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + } else if (is_zuc) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data, + 16); + } else if (is_snow3g) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data, + &sess->cipher.pKeySched_snow3g_cipher); + } else if (is_kasumi) { + if (xform->cipher.key.length != 16) { + AESNI_MB_LOG(ERR, "Invalid cipher key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 16; + IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data, + &sess->cipher.pKeySched_kasumi_cipher); +#endif } else { if (xform->cipher.key.length != 8) { AESNI_MB_LOG(ERR, "Invalid cipher key length"); @@ -544,6 +665,14 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, return -EINVAL; } + /* Set IV parameters */ + sess->iv.offset = xform->aead.iv.offset; + sess->iv.length = xform->aead.iv.length; + + /* Set digest sizes */ + sess->auth.req_digest_len = xform->aead.digest_length; + sess->auth.gen_digest_len = sess->auth.req_digest_len; + switch (xform->aead.algo) { case RTE_CRYPTO_AEAD_AES_CCM: sess->cipher.mode = CCM; @@ -557,11 +686,24 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, sess->cipher.expanded_aes_keys.encode, sess->cipher.expanded_aes_keys.decode); break; + case AES_256_BYTES: + sess->cipher.key_length_in_bytes = AES_256_BYTES; + IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data, + sess->cipher.expanded_aes_keys.encode, + sess->cipher.expanded_aes_keys.decode); + break; default: AESNI_MB_LOG(ERR, "Invalid cipher key length"); return -EINVAL; } + /* CCM digests must be between 4 and 16 and an even number */ + if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN || + sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN || + (sess->auth.req_digest_len & 1) == 1) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } break; case RTE_CRYPTO_AEAD_AES_GCM: @@ -589,27 +731,37 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr, return -EINVAL; } + /* GCM digest size must be between 1 and 16 */ + if (sess->auth.req_digest_len == 0 || + sess->auth.req_digest_len > 16) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: + sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305; + sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305; + + if (xform->aead.key.length != 32) { + AESNI_MB_LOG(ERR, "Invalid key length"); + return -EINVAL; + } + sess->cipher.key_length_in_bytes = 32; + memcpy(sess->cipher.expanded_aes_keys.encode, + xform->aead.key.data, 32); + if (sess->auth.req_digest_len != 16) { + AESNI_MB_LOG(ERR, "Invalid digest size\n"); + return -EINVAL; + } + break; +#endif default: AESNI_MB_LOG(ERR, "Unsupported aead mode parameter"); return -ENOTSUP; } - /* Set IV parameters */ - sess->iv.offset = xform->aead.iv.offset; - sess->iv.length = xform->aead.iv.length; - - sess->auth.req_digest_len = xform->aead.digest_length; - /* CCM digests must be between 4 and 16 and an even number */ - if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN || - sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN || - (sess->auth.req_digest_len & 1) == 1) { - AESNI_MB_LOG(ERR, "Invalid digest size\n"); - return -EINVAL; - } - sess->auth.gen_digest_len = sess->auth.req_digest_len; - return 0; } @@ -673,6 +825,7 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, /* Default IV length = 0 */ sess->iv.length = 0; + sess->auth_iv.length = 0; ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform); if (ret != 0) { @@ -699,6 +852,134 @@ aesni_mb_set_session_parameters(const MB_MGR *mb_mgr, return 0; } +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED +/** Check DOCSIS security session configuration is valid */ +static int +check_docsis_sec_session(struct rte_security_session_conf *conf) +{ + struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform; + struct rte_security_docsis_xform *docsis = &conf->docsis; + + /* Downlink: CRC generate -> Cipher encrypt */ + if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) { + + if (crypto_sym != NULL && + crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES || + crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) && + crypto_sym->cipher.iv.length == AES_BLOCK_SIZE && + crypto_sym->next == NULL) { + return 0; + } + /* Uplink: Cipher decrypt -> CRC verify */ + } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) { + + if (crypto_sym != NULL && + crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + (crypto_sym->cipher.key.length == IMB_KEY_AES_128_BYTES || + crypto_sym->cipher.key.length == IMB_KEY_AES_256_BYTES) && + crypto_sym->cipher.iv.length == AES_BLOCK_SIZE && + crypto_sym->next == NULL) { + return 0; + } + } + + return -EINVAL; +} + +/** Set DOCSIS security session auth (CRC) parameters */ +static int +aesni_mb_set_docsis_sec_session_auth_parameters(struct aesni_mb_session *sess, + struct rte_security_docsis_xform *xform) +{ + if (xform == NULL) { + AESNI_MB_LOG(ERR, "Invalid DOCSIS xform"); + return -EINVAL; + } + + /* Select CRC generate/verify */ + if (xform->direction == RTE_SECURITY_DOCSIS_UPLINK) { + sess->auth.algo = IMB_AUTH_DOCSIS_CRC32; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; + } else if (xform->direction == RTE_SECURITY_DOCSIS_DOWNLINK) { + sess->auth.algo = IMB_AUTH_DOCSIS_CRC32; + sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; + } else { + AESNI_MB_LOG(ERR, "Unsupported DOCSIS direction"); + return -ENOTSUP; + } + + sess->auth.req_digest_len = RTE_ETHER_CRC_LEN; + sess->auth.gen_digest_len = RTE_ETHER_CRC_LEN; + + return 0; +} + +/** + * Parse DOCSIS security session configuration and set private session + * parameters + */ +int +aesni_mb_set_docsis_sec_session_parameters( + __rte_unused struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, + void *sess) +{ + struct rte_security_docsis_xform *docsis_xform; + struct rte_crypto_sym_xform *cipher_xform; + struct aesni_mb_session *aesni_sess = sess; + struct aesni_mb_private *internals = dev->data->dev_private; + int ret; + + ret = check_docsis_sec_session(conf); + if (ret) { + AESNI_MB_LOG(ERR, "Unsupported DOCSIS security configuration"); + return ret; + } + + switch (conf->docsis.direction) { + case RTE_SECURITY_DOCSIS_UPLINK: + aesni_sess->chain_order = IMB_ORDER_CIPHER_HASH; + docsis_xform = &conf->docsis; + cipher_xform = conf->crypto_xform; + break; + case RTE_SECURITY_DOCSIS_DOWNLINK: + aesni_sess->chain_order = IMB_ORDER_HASH_CIPHER; + cipher_xform = conf->crypto_xform; + docsis_xform = &conf->docsis; + break; + default: + return -EINVAL; + } + + /* Default IV length = 0 */ + aesni_sess->iv.length = 0; + + ret = aesni_mb_set_docsis_sec_session_auth_parameters(aesni_sess, + docsis_xform); + if (ret != 0) { + AESNI_MB_LOG(ERR, "Invalid/unsupported DOCSIS parameters"); + return -EINVAL; + } + + ret = aesni_mb_set_session_cipher_parameters(internals->mb_mgr, + aesni_sess, cipher_xform); + + if (ret != 0) { + AESNI_MB_LOG(ERR, "Invalid/unsupported cipher parameters"); + return -EINVAL; + } + + return 0; +} +#endif + /** * burst enqueue, place crypto operations on ingress queue for processing. * @@ -737,6 +1018,13 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) get_sym_session_private_data( op->sym->session, cryptodev_driver_id); +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + if (likely(op->sym->sec_session != NULL)) + sess = (struct aesni_mb_session *) + get_sec_session_private_data( + op->sym->sec_session); +#endif } else { void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp); void *_sess_private_data = NULL; @@ -820,7 +1108,8 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session, static inline void set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, union rte_crypto_sym_ofs sofs, void *buf, uint32_t len, - void *iv, void *aad, void *digest, void *udata) + struct rte_crypto_va_iova_ptr *iv, + struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata) { /* Set crypto operation */ job->chain_order = session->chain_order; @@ -833,7 +1122,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, /* Set authentication parameters */ job->hash_alg = session->auth.algo; - job->iv = iv; + job->iv = iv->va; switch (job->hash_alg) { case AES_XCBC: @@ -848,7 +1137,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, break; case AES_CCM: - job->u.CCM.aad = (uint8_t *)aad + 18; + job->u.CCM.aad = (uint8_t *)aad->va + 18; job->u.CCM.aad_len_in_bytes = session->aead.aad_len; job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; @@ -869,7 +1158,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, case AES_GMAC: if (session->cipher.mode == GCM) { - job->u.GCM.aad = aad; + job->u.GCM.aad = aad->va; job->u.GCM.aad_len_in_bytes = session->aead.aad_len; } else { /* For GMAC */ @@ -881,6 +1170,14 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session, job->aes_dec_key_expanded = &session->cipher.gcm_key; break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->u.CHACHA20_POLY1305.aad = aad->va; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode; + break; +#endif default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; @@ -968,6 +1265,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, /* Set authentication parameters */ job->hash_alg = session->auth.algo; + const int aead = is_aead_algo(job->hash_alg, job->cipher_mode); + switch (job->hash_alg) { case AES_XCBC: job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded; @@ -1013,7 +1312,29 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, job->aes_enc_key_expanded = &session->cipher.gcm_key; job->aes_dec_key_expanded = &session->cipher.gcm_key; break; - +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + case IMB_AUTH_ZUC_EIA3_BITLEN: + job->u.ZUC_EIA3._key = session->auth.zuc_auth_key; + job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->auth_iv.offset); + break; + case IMB_AUTH_SNOW3G_UIA2_BITLEN: + job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth; + job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->auth_iv.offset); + break; + case IMB_AUTH_KASUMI_UIA1: + job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth; + break; +#endif +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data; + job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len; + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode; + break; +#endif default: job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner; job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer; @@ -1031,6 +1352,24 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, } } + if (aead) + m_offset = op->sym->aead.data.offset; + else + m_offset = op->sym->cipher.data.offset; + +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) { + job->aes_enc_key_expanded = session->cipher.zuc_cipher_key; + job->aes_dec_key_expanded = session->cipher.zuc_cipher_key; + } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) { + job->enc_keys = &session->cipher.pKeySched_snow3g_cipher; + m_offset = 0; + } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) { + job->enc_keys = &session->cipher.pKeySched_kasumi_cipher; + m_offset = 0; + } +#endif + if (!op->sym->m_dst) { /* in-place operation */ m_dst = m_src; @@ -1045,20 +1384,13 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, oop = 1; } - if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && - session->cipher.mode == GCM)) - m_offset = op->sym->aead.data.offset; - else - m_offset = op->sym->cipher.data.offset; - /* Set digest output location */ if (job->hash_alg != NULL_HASH && session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { job->auth_tag_output = qp->temp_digests[*digest_idx]; *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { - if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && - session->cipher.mode == GCM)) + if (aead) job->auth_tag_output = op->sym->aead.digest.data; else job->auth_tag_output = op->sym->auth.digest.data; @@ -1118,7 +1450,21 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, session->iv.offset); break; +#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM + case IMB_AUTH_CHACHA20_POLY1305: + job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset; + job->msg_len_to_cipher_in_bytes = + op->sym->aead.data.length; + job->msg_len_to_hash_in_bytes = + op->sym->aead.data.length; + + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, + session->iv.offset); + break; +#endif default: + /* For SNOW3G, length and offsets are already in bits */ job->cipher_start_src_offset_in_bytes = op->sym->cipher.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length; @@ -1131,12 +1477,119 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, session->iv.offset); } +#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM + if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) + job->msg_len_to_cipher_in_bytes >>= 3; + else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1) + job->msg_len_to_hash_in_bytes >>= 3; +#endif + /* Set user data to be crypto operation data struct */ job->user_data = op; return 0; } +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED +/** + * Process a crypto operation containing a security op and complete a + * JOB_AES_HMAC job structure for submission to the multi buffer library for + * processing. + */ +static inline int +set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, + struct rte_crypto_op *op, uint8_t *digest_idx) +{ + struct rte_mbuf *m_src, *m_dst; + struct rte_crypto_sym_op *sym; + struct aesni_mb_session *session; + + session = get_session(qp, op); + if (unlikely(session == NULL)) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -1; + } + + /* Only DOCSIS protocol operations supported now */ + if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI || + session->auth.algo != IMB_AUTH_DOCSIS_CRC32) { + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -1; + } + + sym = op->sym; + m_src = sym->m_src; + + if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) { + /* in-place operation */ + m_dst = m_src; + } else { + /* out-of-place operation not supported */ + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return -ENOTSUP; + } + + /* Set crypto operation */ + job->chain_order = session->chain_order; + + /* Set cipher parameters */ + job->cipher_direction = session->cipher.direction; + job->cipher_mode = session->cipher.mode; + + job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes; + job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode; + job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode; + + /* Set IV parameters */ + job->iv_len_in_bytes = session->iv.length; + job->iv = (uint8_t *)op + session->iv.offset; + + /* Set authentication parameters */ + job->hash_alg = session->auth.algo; + + /* Set digest output location */ + job->auth_tag_output = qp->temp_digests[*digest_idx]; + *digest_idx = (*digest_idx + 1) % MAX_JOBS; + + /* Set digest length */ + job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len; + + /* Set data parameters */ + job->src = rte_pktmbuf_mtod(m_src, uint8_t *); + job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, + sym->cipher.data.offset); + + job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset; + job->msg_len_to_cipher_in_bytes = sym->cipher.data.length; + + job->hash_start_src_offset_in_bytes = sym->auth.data.offset; + job->msg_len_to_hash_in_bytes = sym->auth.data.length; + + job->user_data = op; + + return 0; +} + +static inline void +verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status) +{ + uint16_t crc_offset; + uint8_t *crc; + + if (!job->msg_len_to_hash_in_bytes) + return; + + crc_offset = job->hash_start_src_offset_in_bytes + + job->msg_len_to_hash_in_bytes - + job->cipher_start_src_offset_in_bytes; + crc = job->dst + crc_offset; + + /* Verify CRC (at the end of the message) */ + if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0) + *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; +} +#endif + static inline void verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status) { @@ -1175,9 +1628,25 @@ static inline struct rte_crypto_op * post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) { struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data; - struct aesni_mb_session *sess = get_sym_session_private_data( - op->sym->session, - cryptodev_driver_id); + struct aesni_mb_session *sess = NULL; + +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + uint8_t is_docsis_sec = 0; + + if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { + /* + * Assuming at this point that if it's a security type op, that + * this is for DOCSIS + */ + is_docsis_sec = 1; + sess = get_sec_session_private_data(op->sym->sec_session); + } else +#endif + { + sess = get_sym_session_private_data(op->sym->session, + cryptodev_driver_id); + } + if (unlikely(sess == NULL)) { op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; return op; @@ -1192,13 +1661,16 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) break; if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { - if (job->hash_alg == AES_CCM || - (job->hash_alg == AES_GMAC && - sess->cipher.mode == GCM)) + if (is_aead_algo(job->hash_alg, sess->cipher.mode)) verify_digest(job, op->sym->aead.digest.data, sess->auth.req_digest_len, &op->status); +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + else if (is_docsis_sec) + verify_docsis_sec_crc(job, + &op->status); +#endif else verify_digest(job, op->sym->auth.digest.data, @@ -1361,7 +1833,14 @@ aesni_mb_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, if (retval < 0) break; - retval = set_mb_job_params(job, qp, op, &digest_idx); +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) + retval = set_sec_mb_job_params(job, qp, op, + &digest_idx); + else +#endif + retval = set_mb_job_params(job, qp, op, &digest_idx); + if (unlikely(retval != 0)) { qp->stats.dequeue_err_count++; set_job_null_op(job, op); @@ -1461,7 +1940,7 @@ generate_sync_dgst(struct rte_crypto_sym_vec *vec, for (i = 0, k = 0; i != vec->num; i++) { if (vec->status[i] == 0) { - memcpy(vec->digest[i], dgst[i], len); + memcpy(vec->digest[i].va, dgst[i], len); k++; } } @@ -1477,7 +1956,7 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec, for (i = 0, k = 0; i != vec->num; i++) { if (vec->status[i] == 0) { - if (memcmp(vec->digest[i], dgst[i], len) != 0) + if (memcmp(vec->digest[i].va, dgst[i], len) != 0) vec->status[i] = EBADMSG; else k++; @@ -1540,9 +2019,8 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev, } /* Submit job for processing */ - set_cpu_mb_job_params(job, s, sofs, buf, len, - vec->iv[i], vec->aad[i], tmp_dgst[i], - &vec->status[i]); + set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i], + &vec->aad[i], tmp_dgst[i], &vec->status[i]); job = submit_sync_job(mb_mgr); j++; @@ -1626,8 +2104,27 @@ cryptodev_aesni_mb_create(const char *name, RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO | + RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA | RTE_CRYPTODEV_FF_SYM_SESSIONLESS; +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + struct rte_security_ctx *security_instance; + security_instance = rte_malloc("aesni_mb_sec", + sizeof(struct rte_security_ctx), + RTE_CACHE_LINE_SIZE); + if (security_instance == NULL) { + AESNI_MB_LOG(ERR, "rte_security_ctx memory alloc failed"); + rte_cryptodev_pmd_destroy(dev); + return -ENOMEM; + } + + security_instance->device = (void *)dev; + security_instance->ops = rte_aesni_mb_pmd_sec_ops; + security_instance->sess_cnt = 0; + dev->security_ctx = security_instance; + dev->feature_flags |= RTE_CRYPTODEV_FF_SECURITY; +#endif + /* Check CPU for support for AES instruction set */ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI; @@ -1638,6 +2135,10 @@ cryptodev_aesni_mb_create(const char *name, mb_mgr = alloc_init_mb_mgr(vector_mode); if (mb_mgr == NULL) { +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + rte_free(dev->security_ctx); + dev->security_ctx = NULL; +#endif rte_cryptodev_pmd_destroy(dev); return -ENOMEM; } @@ -1705,6 +2206,11 @@ cryptodev_aesni_mb_remove(struct rte_vdev_device *vdev) RTE_PER_LCORE(sync_mb_mgr) = NULL; } +#ifdef AESNI_MB_DOCSIS_SEC_ENABLED + rte_free(cryptodev->security_ctx); + cryptodev->security_ctx = NULL; +#endif + return rte_cryptodev_pmd_destroy(cryptodev); } @@ -1723,8 +2229,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD, RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv, cryptodev_aesni_mb_pmd_drv.driver, cryptodev_driver_id); - -RTE_INIT(aesni_mb_init_log) -{ - aesni_mb_logtype_driver = rte_log_register("pmd.crypto.aesni_mb"); -} +RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);