X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Faesni_mb%2Frte_aesni_mb_pmd.c;h=b495a9679a0da1547cbc5c0afd3cdcee03f005c8;hb=317862a4e44f;hp=2c25b7b325590cf0ed7739950a08124a66d3da6d;hpb=c68d7aa354f6935e6f054fe7ced26c47d27cd420;p=dpdk.git diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c index 2c25b7b325..b495a9679a 100644 --- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c +++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c @@ -35,7 +35,7 @@ typedef void (*aes_keyexp_t)(const void *key, void *enc_exp_keys, void *dec_exp_ static void calculate_auth_precomputes(hash_one_block_t one_block_hash, uint8_t *ipad, uint8_t *opad, - uint8_t *hkey, uint16_t hkey_len, + const uint8_t *hkey, uint16_t hkey_len, uint16_t blocksize) { unsigned i, length; @@ -104,9 +104,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, struct aesni_mb_session *sess, const struct rte_crypto_sym_xform *xform) { - hash_one_block_t hash_oneblock_fn; + hash_one_block_t hash_oneblock_fn = NULL; unsigned int key_larger_block_size = 0; uint8_t hashed_key[HMAC_MAX_BLOCK_SIZE] = { 0 }; + uint32_t auth_precompute = 1; if (xform == NULL) { sess->auth.algo = NULL_HASH; @@ -173,6 +174,54 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, return 0; } + if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { + if (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) { + sess->cipher.direction = ENCRYPT; + sess->chain_order = CIPHER_HASH; + } else + sess->cipher.direction = DECRYPT; + + sess->auth.algo = AES_GMAC; + /* + * Multi-buffer lib supports 8, 12 and 16 bytes of digest. + * If size requested is different, generate the full digest + * (16 bytes) in a temporary location and then memcpy + * the requested number of bytes. + */ + if (sess->auth.req_digest_len != 16 && + sess->auth.req_digest_len != 12 && + sess->auth.req_digest_len != 8) { + sess->auth.gen_digest_len = 16; + } else { + sess->auth.gen_digest_len = sess->auth.req_digest_len; + } + sess->iv.length = xform->auth.iv.length; + sess->iv.offset = xform->auth.iv.offset; + + switch (xform->auth.key.length) { + case AES_128_BYTES: + IMB_AES128_GCM_PRE(mb_mgr, xform->auth.key.data, + &sess->cipher.gcm_key); + sess->cipher.key_length_in_bytes = AES_128_BYTES; + break; + case AES_192_BYTES: + IMB_AES192_GCM_PRE(mb_mgr, xform->auth.key.data, + &sess->cipher.gcm_key); + sess->cipher.key_length_in_bytes = AES_192_BYTES; + break; + case AES_256_BYTES: + IMB_AES256_GCM_PRE(mb_mgr, xform->auth.key.data, + &sess->cipher.gcm_key); + sess->cipher.key_length_in_bytes = AES_256_BYTES; + break; + default: + RTE_LOG(ERR, PMD, "failed to parse test type\n"); + return -EINVAL; + } + + return 0; + } + switch (xform->auth.algo) { case RTE_CRYPTO_AUTH_MD5_HMAC: sess->auth.algo = MD5; @@ -189,6 +238,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, key_larger_block_size = 1; } break; + case RTE_CRYPTO_AUTH_SHA1: + sess->auth.algo = PLAIN_SHA1; + auth_precompute = 0; + break; case RTE_CRYPTO_AUTH_SHA224_HMAC: sess->auth.algo = SHA_224; hash_oneblock_fn = mb_mgr->sha224_one_block; @@ -200,6 +253,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, key_larger_block_size = 1; } break; + case RTE_CRYPTO_AUTH_SHA224: + sess->auth.algo = PLAIN_SHA_224; + auth_precompute = 0; + break; case RTE_CRYPTO_AUTH_SHA256_HMAC: sess->auth.algo = SHA_256; hash_oneblock_fn = mb_mgr->sha256_one_block; @@ -211,6 +268,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, key_larger_block_size = 1; } break; + case RTE_CRYPTO_AUTH_SHA256: + sess->auth.algo = PLAIN_SHA_256; + auth_precompute = 0; + break; case RTE_CRYPTO_AUTH_SHA384_HMAC: sess->auth.algo = SHA_384; hash_oneblock_fn = mb_mgr->sha384_one_block; @@ -222,6 +283,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, key_larger_block_size = 1; } break; + case RTE_CRYPTO_AUTH_SHA384: + sess->auth.algo = PLAIN_SHA_384; + auth_precompute = 0; + break; case RTE_CRYPTO_AUTH_SHA512_HMAC: sess->auth.algo = SHA_512; hash_oneblock_fn = mb_mgr->sha512_one_block; @@ -233,6 +298,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, key_larger_block_size = 1; } break; + case RTE_CRYPTO_AUTH_SHA512: + sess->auth.algo = PLAIN_SHA_512; + auth_precompute = 0; + break; default: AESNI_MB_LOG(ERR, "Unsupported authentication algorithm selection"); return -ENOTSUP; @@ -254,6 +323,10 @@ aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr, else sess->auth.gen_digest_len = sess->auth.req_digest_len; + /* Plain SHA does not require precompute key */ + if (auth_precompute == 0) + return 0; + /* Calculate Authentication precomputes */ if (key_larger_block_size) { calculate_auth_precomputes(hash_oneblock_fn, @@ -643,7 +716,8 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) return NULL; - if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) + if (rte_mempool_get(qp->sess_mp_priv, + (void **)&_sess_private_data)) return NULL; sess = (struct aesni_mb_session *)_sess_private_data; @@ -651,7 +725,7 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr, sess, op->sym->xform) != 0)) { rte_mempool_put(qp->sess_mp, _sess); - rte_mempool_put(qp->sess_mp, _sess_private_data); + rte_mempool_put(qp->sess_mp_priv, _sess_private_data); sess = NULL; } op->sym->session = (struct rte_cryptodev_sym_session *)_sess; @@ -665,6 +739,56 @@ get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op) return sess; } +static inline uint64_t +auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session, + uint32_t oop) +{ + struct rte_mbuf *m_src, *m_dst; + uint8_t *p_src, *p_dst; + uintptr_t u_src, u_dst; + uint32_t cipher_end, auth_end; + + /* Only cipher then hash needs special calculation. */ + if (!oop || session->chain_order != CIPHER_HASH) + return op->sym->auth.data.offset; + + m_src = op->sym->m_src; + m_dst = op->sym->m_dst; + + p_src = rte_pktmbuf_mtod(m_src, uint8_t *); + p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); + u_src = (uintptr_t)p_src; + u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset; + + /** + * Copy the content between cipher offset and auth offset for generating + * correct digest. + */ + if (op->sym->cipher.data.offset > op->sym->auth.data.offset) + memcpy(p_dst + op->sym->auth.data.offset, + p_src + op->sym->auth.data.offset, + op->sym->cipher.data.offset - + op->sym->auth.data.offset); + + /** + * Copy the content between (cipher offset + length) and (auth offset + + * length) for generating correct digest + */ + cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length; + auth_end = op->sym->auth.data.offset + op->sym->auth.data.length; + if (cipher_end < auth_end) + memcpy(p_dst + cipher_end, p_src + cipher_end, + auth_end - cipher_end); + + /** + * Since intel-ipsec-mb only supports positive values, + * we need to deduct the correct offset between src and dst. + */ + + return u_src < u_dst ? (u_dst - u_src) : + (UINT64_MAX - u_src + u_dst + 1); +} + /** * Process a crypto operation and complete a JOB_AES_HMAC job structure for * submission to the multi buffer library for processing. @@ -683,7 +807,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, { struct rte_mbuf *m_src = op->sym->m_src, *m_dst; struct aesni_mb_session *session; - uint16_t m_offset = 0; + uint32_t m_offset, oop; session = get_session(qp, op); if (session == NULL) { @@ -735,8 +859,16 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, break; case AES_GMAC: - job->u.GCM.aad = op->sym->aead.aad.data; - job->u.GCM.aad_len_in_bytes = session->aead.aad_len; + if (session->cipher.mode == GCM) { + job->u.GCM.aad = op->sym->aead.aad.data; + job->u.GCM.aad_len_in_bytes = session->aead.aad_len; + } else { + /* For GMAC */ + job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src, + uint8_t *, op->sym->auth.data.offset); + job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length; + job->cipher_mode = GCM; + } job->aes_enc_key_expanded = &session->cipher.gcm_key; job->aes_dec_key_expanded = &session->cipher.gcm_key; break; @@ -758,37 +890,34 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, } } - /* Mutable crypto operation parameters */ - if (op->sym->m_dst) { - m_src = m_dst = op->sym->m_dst; - - /* append space for output data to mbuf */ - char *odata = rte_pktmbuf_append(m_dst, - rte_pktmbuf_data_len(op->sym->m_src)); - if (odata == NULL) { - AESNI_MB_LOG(ERR, "failed to allocate space in destination " - "mbuf for source data"); - op->status = RTE_CRYPTO_OP_STATUS_ERROR; - return -1; - } - - memcpy(odata, rte_pktmbuf_mtod(op->sym->m_src, void*), - rte_pktmbuf_data_len(op->sym->m_src)); - } else { + if (!op->sym->m_dst) { + /* in-place operation */ m_dst = m_src; - if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) - m_offset = op->sym->aead.data.offset; - else - m_offset = op->sym->cipher.data.offset; + oop = 0; + } else if (op->sym->m_dst == op->sym->m_src) { + /* in-place operation */ + m_dst = m_src; + oop = 0; + } else { + /* out-of-place operation */ + m_dst = op->sym->m_dst; + oop = 1; } + if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && + session->cipher.mode == GCM)) + m_offset = op->sym->aead.data.offset; + else + m_offset = op->sym->cipher.data.offset; + /* Set digest output location */ if (job->hash_alg != NULL_HASH && session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { job->auth_tag_output = qp->temp_digests[*digest_idx]; *digest_idx = (*digest_idx + 1) % MAX_JOBS; } else { - if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) + if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC && + session->cipher.mode == GCM)) job->auth_tag_output = op->sym->aead.digest.data; else job->auth_tag_output = op->sym->auth.digest.data; @@ -809,7 +938,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, /* Set IV parameters */ job->iv_len_in_bytes = session->iv.length; - /* Data Parameter */ + /* Data Parameters */ job->src = rte_pktmbuf_mtod(m_src, uint8_t *); job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset); @@ -826,11 +955,24 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, break; case AES_GMAC: - job->cipher_start_src_offset_in_bytes = - op->sym->aead.data.offset; - job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset; - job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length; - job->msg_len_to_hash_in_bytes = job->msg_len_to_cipher_in_bytes; + if (session->cipher.mode == GCM) { + job->cipher_start_src_offset_in_bytes = + op->sym->aead.data.offset; + job->hash_start_src_offset_in_bytes = + op->sym->aead.data.offset; + job->msg_len_to_cipher_in_bytes = + op->sym->aead.data.length; + job->msg_len_to_hash_in_bytes = + op->sym->aead.data.length; + } else { + job->cipher_start_src_offset_in_bytes = + op->sym->auth.data.offset; + job->hash_start_src_offset_in_bytes = + op->sym->auth.data.offset; + job->msg_len_to_cipher_in_bytes = 0; + job->msg_len_to_hash_in_bytes = 0; + } + job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset); break; @@ -840,7 +982,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, op->sym->cipher.data.offset; job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length; - job->hash_start_src_offset_in_bytes = op->sym->auth.data.offset; + job->hash_start_src_offset_in_bytes = auth_start_offset(op, + session, oop); job->msg_len_to_hash_in_bytes = op->sym->auth.data.length; job->iv = rte_crypto_op_ctod_offset(op, uint8_t *, @@ -854,26 +997,18 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp, } static inline void -verify_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op, - struct aesni_mb_session *sess) +verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status) { /* Verify digest if required */ - if (job->hash_alg == AES_CCM || job->hash_alg == AES_GMAC) { - if (memcmp(job->auth_tag_output, op->sym->aead.digest.data, - sess->auth.req_digest_len) != 0) - op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } else { - if (memcmp(job->auth_tag_output, op->sym->auth.digest.data, - sess->auth.req_digest_len) != 0) - op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; - } + if (memcmp(job->auth_tag_output, digest, len) != 0) + *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; } static inline void generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op, struct aesni_mb_session *sess) { - /* No extra copy neeed */ + /* No extra copy needed */ if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len)) return; @@ -908,13 +1043,24 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) case STS_COMPLETED: op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; - if (job->hash_alg != NULL_HASH) { - if (sess->auth.operation == - RTE_CRYPTO_AUTH_OP_VERIFY) - verify_digest(job, op, sess); + if (job->hash_alg == NULL_HASH) + break; + + if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { + if (job->hash_alg == AES_CCM || + (job->hash_alg == AES_GMAC && + sess->cipher.mode == GCM)) + verify_digest(job, + op->sym->aead.digest.data, + sess->auth.req_digest_len, + &op->status); else - generate_digest(job, op, sess); - } + verify_digest(job, + op->sym->auth.digest.data, + sess->auth.req_digest_len, + &op->status); + } else + generate_digest(job, op, sess); break; default: op->status = RTE_CRYPTO_OP_STATUS_ERROR; @@ -925,8 +1071,9 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job) if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { memset(sess, 0, sizeof(struct aesni_mb_session)); memset(op->sym->session, 0, - rte_cryptodev_sym_get_header_session_size()); - rte_mempool_put(qp->sess_mp, sess); + rte_cryptodev_sym_get_existing_header_session_size( + op->sym->session)); + rte_mempool_put(qp->sess_mp_priv, sess); rte_mempool_put(qp->sess_mp, op->sym->session); op->sym->session = NULL; } @@ -1116,7 +1263,9 @@ cryptodev_aesni_mb_create(const char *name, dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | - RTE_CRYPTODEV_FF_CPU_AESNI; + RTE_CRYPTODEV_FF_CPU_AESNI | + RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; + mb_mgr = alloc_mb_mgr(0); if (mb_mgr == NULL)