X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Faesni_gcm%2Faesni_gcm_pmd.c;h=d0368828e94692a796c105c5c44a6d4ac482ad7f;hb=671735bb4a254903cb3ee399b9cc16ce5247c3a8;hp=1d2a0ce00750bb6a014f0a8281f74055c272eba6;hpb=9c99878aa1b16de26fcce82c112b401766dd910e;p=dpdk.git diff --git a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c index 1d2a0ce007..d0368828e9 100644 --- a/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c +++ b/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -350,6 +350,76 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, &qp->gdata_ctx, tag, session->gen_digest_length); +#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM + } else if (session->op == AESNI_GMAC_OP_GENERATE) { + qp->ops[session->key].gmac_init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + session->iv.length); + + qp->ops[session->key].gmac_update(&session->gdata_key, + &qp->gdata_ctx, src, + (uint64_t)part_len); + total_len = data_length - part_len; + + while (total_len) { + m_src = m_src->next; + + RTE_ASSERT(m_src != NULL); + + src = rte_pktmbuf_mtod(m_src, uint8_t *); + part_len = (m_src->data_len < total_len) ? + m_src->data_len : total_len; + + qp->ops[session->key].gmac_update(&session->gdata_key, + &qp->gdata_ctx, src, + (uint64_t)part_len); + total_len -= part_len; + } + + if (session->req_digest_length != session->gen_digest_length) + tag = qp->temp_digest; + else + tag = sym_op->auth.digest.data; + + qp->ops[session->key].gmac_finalize(&session->gdata_key, + &qp->gdata_ctx, + tag, + session->gen_digest_length); + } else { /* AESNI_GMAC_OP_VERIFY */ + qp->ops[session->key].gmac_init(&session->gdata_key, + &qp->gdata_ctx, + iv_ptr, + session->iv.length); + + qp->ops[session->key].gmac_update(&session->gdata_key, + &qp->gdata_ctx, src, + (uint64_t)part_len); + total_len = data_length - part_len; + + while (total_len) { + m_src = m_src->next; + + RTE_ASSERT(m_src != NULL); + + src = rte_pktmbuf_mtod(m_src, uint8_t *); + part_len = (m_src->data_len < total_len) ? + m_src->data_len : total_len; + + qp->ops[session->key].gmac_update(&session->gdata_key, + &qp->gdata_ctx, src, + (uint64_t)part_len); + total_len -= part_len; + } + + tag = qp->temp_digest; + + qp->ops[session->key].gmac_finalize(&session->gdata_key, + &qp->gdata_ctx, + tag, + session->gen_digest_length); + } +#else } else if (session->op == AESNI_GMAC_OP_GENERATE) { qp->ops[session->key].init(&session->gdata_key, &qp->gdata_ctx, @@ -381,6 +451,7 @@ process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op, tag, session->gen_digest_length); } +#endif return 0; } @@ -464,9 +535,10 @@ aesni_gcm_sgl_encrypt(struct aesni_gcm_session *s, processed = 0; for (i = 0; i < vec->num; ++i) { aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, - &vec->sgl[i], vec->iv[i], vec->aad[i]); + &vec->src_sgl[i], vec->iv[i].va, + vec->aad[i].va); vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(s, - gdata_ctx, vec->digest[i]); + gdata_ctx, vec->digest[i].va); processed += (vec->status[i] == 0); } @@ -482,9 +554,10 @@ aesni_gcm_sgl_decrypt(struct aesni_gcm_session *s, processed = 0; for (i = 0; i < vec->num; ++i) { aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, - &vec->sgl[i], vec->iv[i], vec->aad[i]); + &vec->src_sgl[i], vec->iv[i].va, + vec->aad[i].va); vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(s, - gdata_ctx, vec->digest[i]); + gdata_ctx, vec->digest[i].va); processed += (vec->status[i] == 0); } @@ -499,15 +572,15 @@ aesni_gmac_sgl_generate(struct aesni_gcm_session *s, processed = 0; for (i = 0; i < vec->num; ++i) { - if (vec->sgl[i].num != 1) { + if (vec->src_sgl[i].num != 1) { vec->status[i] = ENOTSUP; continue; } aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, - &vec->sgl[i], vec->iv[i]); + &vec->src_sgl[i], vec->iv[i].va); vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(s, - gdata_ctx, vec->digest[i]); + gdata_ctx, vec->digest[i].va); processed += (vec->status[i] == 0); } @@ -522,15 +595,15 @@ aesni_gmac_sgl_verify(struct aesni_gcm_session *s, processed = 0; for (i = 0; i < vec->num; ++i) { - if (vec->sgl[i].num != 1) { + if (vec->src_sgl[i].num != 1) { vec->status[i] = ENOTSUP; continue; } aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, - &vec->sgl[i], vec->iv[i]); + &vec->src_sgl[i], vec->iv[i].va); vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(s, - gdata_ctx, vec->digest[i]); + gdata_ctx, vec->digest[i].va); processed += (vec->status[i] == 0); } @@ -769,8 +842,14 @@ aesni_gcm_create(const char *name, init_mb_mgr_avx2(mb_mgr); break; case RTE_AESNI_GCM_AVX512: - dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; - init_mb_mgr_avx512(mb_mgr); + if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_VAES)) { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX512; + init_mb_mgr_avx512(mb_mgr); + } else { + dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2; + init_mb_mgr_avx2(mb_mgr); + vector_mode = RTE_AESNI_GCM_AVX2; + } break; default: AESNI_GCM_LOG(ERR, "Unsupported vector mode %u\n", vector_mode); @@ -791,6 +870,11 @@ aesni_gcm_create(const char *name, internals->ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update; internals->ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize; internals->ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize; +#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM + internals->ops[GCM_KEY_128].gmac_init = mb_mgr->gmac128_init; + internals->ops[GCM_KEY_128].gmac_update = mb_mgr->gmac128_update; + internals->ops[GCM_KEY_128].gmac_finalize = mb_mgr->gmac128_finalize; +#endif internals->ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc; internals->ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec; @@ -800,6 +884,11 @@ aesni_gcm_create(const char *name, internals->ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update; internals->ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize; internals->ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize; +#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM + internals->ops[GCM_KEY_192].gmac_init = mb_mgr->gmac192_init; + internals->ops[GCM_KEY_192].gmac_update = mb_mgr->gmac192_update; + internals->ops[GCM_KEY_192].gmac_finalize = mb_mgr->gmac192_finalize; +#endif internals->ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc; internals->ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec; @@ -809,6 +898,11 @@ aesni_gcm_create(const char *name, internals->ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update; internals->ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize; internals->ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize; +#if IMB_VERSION(0, 54, 0) < IMB_VERSION_NUM + internals->ops[GCM_KEY_256].gmac_init = mb_mgr->gmac256_init; + internals->ops[GCM_KEY_256].gmac_update = mb_mgr->gmac256_update; + internals->ops[GCM_KEY_256].gmac_finalize = mb_mgr->gmac256_finalize; +#endif internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs; @@ -887,4 +981,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD, "socket_id="); RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv.driver, cryptodev_driver_id); -RTE_LOG_REGISTER(aesni_gcm_logtype_driver, pmd.crypto.aesni_gcm, NOTICE); +RTE_LOG_REGISTER_DEFAULT(aesni_gcm_logtype_driver, NOTICE);