crypto/qat: update raw data path

[dpdk.git] / drivers / crypto / aesni_mb / rte_aesni_mb_pmd.c

diff --git a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
index 7dbe40e..a01c826 100644 (file)
--- a/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
+++ b/drivers/crypto/aesni_mb/rte_aesni_mb_pmd.c
@@ -125,6 +125,18 @@ aesni_mb_get_chain_order(const struct rte_crypto_sym_xform *xform)
        return AESNI_MB_OP_NOT_SUPPORTED;
 }
 
+static inline int
+is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)
+{
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+       return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM ||
+               (hash_alg == AES_GMAC && cipher_mode == GCM));
+#else
+       return ((hash_alg == AES_GMAC && cipher_mode == GCM) ||
+               hash_alg == AES_CCM);
+#endif
+}
+
 /** Set session authentication parameters */
 static int
 aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
@@ -674,6 +686,12 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
                                        sess->cipher.expanded_aes_keys.encode,
                                        sess->cipher.expanded_aes_keys.decode);
                        break;
+               case AES_256_BYTES:
+                       sess->cipher.key_length_in_bytes = AES_256_BYTES;
+                       IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
+                                       sess->cipher.expanded_aes_keys.encode,
+                                       sess->cipher.expanded_aes_keys.decode);
+                       break;
                default:
                        AESNI_MB_LOG(ERR, "Invalid cipher key length");
                        return -EINVAL;
@@ -721,6 +739,24 @@ aesni_mb_set_session_aead_parameters(const MB_MGR *mb_mgr,
                }
                break;
 
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+       case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
+               sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
+               sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
+
+               if (xform->aead.key.length != 32) {
+                       AESNI_MB_LOG(ERR, "Invalid key length");
+                       return -EINVAL;
+               }
+               sess->cipher.key_length_in_bytes = 32;
+               memcpy(sess->cipher.expanded_aes_keys.encode,
+                       xform->aead.key.data, 32);
+               if (sess->auth.req_digest_len != 16) {
+                       AESNI_MB_LOG(ERR, "Invalid digest size\n");
+                       return -EINVAL;
+               }
+               break;
+#endif
        default:
                AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
                return -ENOTSUP;
@@ -1072,7 +1108,8 @@ auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
 static inline void
 set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
                union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
-               void *iv, void *aad, void *digest, void *udata)
+               struct rte_crypto_va_iova_ptr *iv,
+               struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
 {
        /* Set crypto operation */
        job->chain_order = session->chain_order;
@@ -1085,7 +1122,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
 
        /* Set authentication parameters */
        job->hash_alg = session->auth.algo;
-       job->iv = iv;
+       job->iv = iv->va;
 
        switch (job->hash_alg) {
        case AES_XCBC:
@@ -1100,7 +1137,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
                break;
 
        case AES_CCM:
-               job->u.CCM.aad = (uint8_t *)aad + 18;
+               job->u.CCM.aad = (uint8_t *)aad->va + 18;
                job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
                job->aes_enc_key_expanded =
                                session->cipher.expanded_aes_keys.encode;
@@ -1121,7 +1158,7 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
 
        case AES_GMAC:
                if (session->cipher.mode == GCM) {
-                       job->u.GCM.aad = aad;
+                       job->u.GCM.aad = aad->va;
                        job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
                } else {
                        /* For GMAC */
@@ -1133,6 +1170,14 @@ set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
                job->aes_dec_key_expanded = &session->cipher.gcm_key;
                break;
 
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+       case IMB_AUTH_CHACHA20_POLY1305:
+               job->u.CHACHA20_POLY1305.aad = aad->va;
+               job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+               job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+               job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+               break;
+#endif
        default:
                job->u.HMAC._hashed_auth_key_xor_ipad =
                                session->auth.pads.inner;
@@ -1220,6 +1265,8 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
        /* Set authentication parameters */
        job->hash_alg = session->auth.algo;
 
+       const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
+
        switch (job->hash_alg) {
        case AES_XCBC:
                job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
@@ -1279,6 +1326,14 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
        case IMB_AUTH_KASUMI_UIA1:
                job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
                break;
+#endif
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+       case IMB_AUTH_CHACHA20_POLY1305:
+               job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
+               job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+               job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+               job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+               break;
 #endif
        default:
                job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
@@ -1297,8 +1352,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
                }
        }
 
-       if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
-                       session->cipher.mode == GCM))
+       if (aead)
                m_offset = op->sym->aead.data.offset;
        else
                m_offset = op->sym->cipher.data.offset;
@@ -1336,8 +1390,7 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
                job->auth_tag_output = qp->temp_digests[*digest_idx];
                *digest_idx = (*digest_idx + 1) % MAX_JOBS;
        } else {
-               if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
-                               session->cipher.mode == GCM))
+               if (aead)
                        job->auth_tag_output = op->sym->aead.digest.data;
                else
                        job->auth_tag_output = op->sym->auth.digest.data;
@@ -1397,6 +1450,19 @@ set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
                                session->iv.offset);
                break;
 
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+       case IMB_AUTH_CHACHA20_POLY1305:
+               job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
+               job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+               job->msg_len_to_cipher_in_bytes =
+                               op->sym->aead.data.length;
+               job->msg_len_to_hash_in_bytes =
+                                       op->sym->aead.data.length;
+
+               job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+                               session->iv.offset);
+               break;
+#endif
        default:
                /* For SNOW3G, length and offsets are already in bits */
                job->cipher_start_src_offset_in_bytes =
@@ -1595,9 +1661,7 @@ post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
                                break;
 
                        if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
-                               if (job->hash_alg == AES_CCM ||
-                                       (job->hash_alg == AES_GMAC &&
-                                               sess->cipher.mode == GCM))
+                               if (is_aead_algo(job->hash_alg, sess->cipher.mode))
                                        verify_digest(job,
                                                op->sym->aead.digest.data,
                                                sess->auth.req_digest_len,
@@ -1876,7 +1940,7 @@ generate_sync_dgst(struct rte_crypto_sym_vec *vec,
 
        for (i = 0, k = 0; i != vec->num; i++) {
                if (vec->status[i] == 0) {
-                       memcpy(vec->digest[i], dgst[i], len);
+                       memcpy(vec->digest[i].va, dgst[i], len);
                        k++;
                }
        }
@@ -1892,7 +1956,7 @@ verify_sync_dgst(struct rte_crypto_sym_vec *vec,
 
        for (i = 0, k = 0; i != vec->num; i++) {
                if (vec->status[i] == 0) {
-                       if (memcmp(vec->digest[i], dgst[i], len) != 0)
+                       if (memcmp(vec->digest[i].va, dgst[i], len) != 0)
                                vec->status[i] = EBADMSG;
                        else
                                k++;
@@ -1955,9 +2019,8 @@ aesni_mb_cpu_crypto_process_bulk(struct rte_cryptodev *dev,
                }
 
                /* Submit job for processing */
-               set_cpu_mb_job_params(job, s, sofs, buf, len,
-                       vec->iv[i], vec->aad[i], tmp_dgst[i],
-                       &vec->status[i]);
+               set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],
+                       &vec->aad[i], tmp_dgst[i], &vec->status[i]);
                job = submit_sync_job(mb_mgr);
                j++;
 
@@ -2166,4 +2229,4 @@ RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_MB_PMD,
 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
                cryptodev_aesni_mb_pmd_drv.driver,
                cryptodev_driver_id);
-RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);
+RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);