net/octeontx: support fast mbuf free
[dpdk.git] / drivers / crypto / qat / qat_sym_session.c
index 1adef8f..3727d56 100644 (file)
 #include "qat_sym_session.h"
 #include "qat_sym_pmd.h"
 
+/* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha1InitialState[] = {
+       0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba,
+       0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0};
+
+/* SHA 224 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha224InitialState[] = {
+       0xc1, 0x05, 0x9e, 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd,
+       0x17, 0xf7, 0x0e, 0x59, 0x39, 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58,
+       0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, 0xfa, 0x4f, 0xa4};
+
+/* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha256InitialState[] = {
+       0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3,
+       0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05,
+       0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19};
+
+/* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha384InitialState[] = {
+       0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29,
+       0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70,
+       0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67,
+       0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87,
+       0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f,
+       0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4};
+
+/* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */
+static const uint8_t sha512InitialState[] = {
+       0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae,
+       0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94,
+       0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51,
+       0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c,
+       0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd,
+       0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79};
+
 /** Frees a context previously created
  *  Depends on openssl libcrypto
  */
@@ -35,7 +70,7 @@ bpi_cipher_ctx_free(void *bpi_ctx)
 static int
 bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
                enum rte_crypto_cipher_operation direction __rte_unused,
-               const uint8_t *key, void **ctx)
+               const uint8_t *key, uint16_t key_length, void **ctx)
 {
        const EVP_CIPHER *algo = NULL;
        int ret;
@@ -49,7 +84,10 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
        if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
                algo = EVP_des_ecb();
        else
-               algo = EVP_aes_128_ecb();
+               if (key_length == ICP_QAT_HW_AES_128_KEY_SZ)
+                       algo = EVP_aes_128_ecb();
+               else
+                       algo = EVP_aes_256_ecb();
 
        /* IV will be ECB encrypted whether direction is encrypt or decrypt*/
        if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
@@ -286,6 +324,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
                                        cipher_xform->algo,
                                        cipher_xform->op,
                                        cipher_xform->key.data,
+                                       cipher_xform->key.length,
                                        &session->bpi_ctx);
                if (ret != 0) {
                        QAT_LOG(ERR, "failed to create DES BPI ctx");
@@ -304,6 +343,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
                                        cipher_xform->algo,
                                        cipher_xform->op,
                                        cipher_xform->key.data,
+                                       cipher_xform->key.length,
                                        &session->bpi_ctx);
                if (ret != 0) {
                        QAT_LOG(ERR, "failed to create AES BPI ctx");
@@ -459,18 +499,23 @@ qat_sym_session_set_ext_hash_flags(struct qat_sym_session *session,
 }
 
 static void
-qat_sym_session_handle_mixed(struct qat_sym_session *session)
+qat_sym_session_handle_mixed(const struct rte_cryptodev *dev,
+               struct qat_sym_session *session)
 {
+       const struct qat_sym_dev_private *qat_private = dev->data->dev_private;
+       enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities &
+                       QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3;
+
        if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
                        session->qat_cipher_alg !=
                        ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
-               session->min_qat_dev_gen = QAT_GEN3;
+               session->min_qat_dev_gen = min_dev_gen;
                qat_sym_session_set_ext_hash_flags(session,
                        1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
        } else if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
                        session->qat_cipher_alg !=
                        ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
-               session->min_qat_dev_gen = QAT_GEN3;
+               session->min_qat_dev_gen = min_dev_gen;
                qat_sym_session_set_ext_hash_flags(session,
                        1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
        } else if ((session->aes_cmac ||
@@ -479,7 +524,7 @@ qat_sym_session_handle_mixed(struct qat_sym_session *session)
                        ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
                        session->qat_cipher_alg ==
                        ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
-               session->min_qat_dev_gen = QAT_GEN3;
+               session->min_qat_dev_gen = min_dev_gen;
                qat_sym_session_set_ext_hash_flags(session, 0);
        }
 }
@@ -532,7 +577,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
                        if (ret < 0)
                                return ret;
                        /* Special handling of mixed hash+cipher algorithms */
-                       qat_sym_session_handle_mixed(session);
+                       qat_sym_session_handle_mixed(dev, session);
                }
                break;
        case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
@@ -551,7 +596,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
                        if (ret < 0)
                                return ret;
                        /* Special handling of mixed hash+cipher algorithms */
-                       qat_sym_session_handle_mixed(session);
+                       qat_sym_session_handle_mixed(dev, session);
                }
                break;
        case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
@@ -576,68 +621,69 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
 }
 
 static int
-qat_sym_session_handle_single_pass(struct qat_sym_session *session,
+qat_sym_session_handle_single_pass(struct qat_sym_dev_private *internals,
+               struct qat_sym_session *session,
                struct rte_crypto_aead_xform *aead_xform)
 {
-       struct icp_qat_fw_la_cipher_req_params *cipher_param =
-                       (void *) &session->fw_req.serv_specif_rqpars;
+       enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
 
-       session->is_single_pass = 1;
-       session->min_qat_dev_gen = QAT_GEN3;
-       session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER;
-       if (aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+       if (qat_dev_gen == QAT_GEN3 &&
+                       aead_xform->iv.length == QAT_AES_GCM_SPC_IV_SIZE) {
+               /* Use faster Single-Pass GCM */
+               struct icp_qat_fw_la_cipher_req_params *cipher_param =
+                               (void *) &session->fw_req.serv_specif_rqpars;
+
+               session->is_single_pass = 1;
+               session->min_qat_dev_gen = QAT_GEN3;
+               session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER;
                session->qat_mode = ICP_QAT_HW_CIPHER_AEAD_MODE;
+               session->cipher_iv.offset = aead_xform->iv.offset;
+               session->cipher_iv.length = aead_xform->iv.length;
+               if (qat_sym_session_aead_create_cd_cipher(session,
+                               aead_xform->key.data, aead_xform->key.length))
+                       return -EINVAL;
+               session->aad_len = aead_xform->aad_length;
+               session->digest_length = aead_xform->digest_length;
+               if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+                       session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+                       session->auth_op = ICP_QAT_HW_AUTH_GENERATE;
+                       ICP_QAT_FW_LA_RET_AUTH_SET(
+                               session->fw_req.comn_hdr.serv_specif_flags,
+                               ICP_QAT_FW_LA_RET_AUTH_RES);
+               } else {
+                       session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+                       session->auth_op = ICP_QAT_HW_AUTH_VERIFY;
+                       ICP_QAT_FW_LA_CMP_AUTH_SET(
+                               session->fw_req.comn_hdr.serv_specif_flags,
+                               ICP_QAT_FW_LA_CMP_AUTH_RES);
+               }
+               ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+                               session->fw_req.comn_hdr.serv_specif_flags,
+                               ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
+               ICP_QAT_FW_LA_PROTO_SET(
+                               session->fw_req.comn_hdr.serv_specif_flags,
+                               ICP_QAT_FW_LA_NO_PROTO);
                ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
-                       session->fw_req.comn_hdr.serv_specif_flags,
-                       ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
-       } else {
-               /* Chacha-Poly is special case that use QAT CTR mode */
-               session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+                               session->fw_req.comn_hdr.serv_specif_flags,
+                               ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
+               session->fw_req.comn_hdr.service_cmd_id =
+                               ICP_QAT_FW_LA_CMD_CIPHER;
+               session->cd.cipher.cipher_config.val =
+                               ICP_QAT_HW_CIPHER_CONFIG_BUILD(
+                                       ICP_QAT_HW_CIPHER_AEAD_MODE,
+                                       session->qat_cipher_alg,
+                                       ICP_QAT_HW_CIPHER_NO_CONVERT,
+                                       session->qat_dir);
+               QAT_FIELD_SET(session->cd.cipher.cipher_config.val,
+                               aead_xform->digest_length,
+                               QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
+                               QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
+               session->cd.cipher.cipher_config.reserved =
+                               ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
+                                       aead_xform->aad_length);
+               cipher_param->spc_aad_sz = aead_xform->aad_length;
+               cipher_param->spc_auth_res_sz = aead_xform->digest_length;
        }
-       session->cipher_iv.offset = aead_xform->iv.offset;
-       session->cipher_iv.length = aead_xform->iv.length;
-       if (qat_sym_session_aead_create_cd_cipher(session,
-                       aead_xform->key.data, aead_xform->key.length))
-               return -EINVAL;
-       session->aad_len = aead_xform->aad_length;
-       session->digest_length = aead_xform->digest_length;
-       if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
-               session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
-               session->auth_op = ICP_QAT_HW_AUTH_GENERATE;
-               ICP_QAT_FW_LA_RET_AUTH_SET(
-                       session->fw_req.comn_hdr.serv_specif_flags,
-                       ICP_QAT_FW_LA_RET_AUTH_RES);
-       } else {
-               session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
-               session->auth_op = ICP_QAT_HW_AUTH_VERIFY;
-               ICP_QAT_FW_LA_CMP_AUTH_SET(
-                       session->fw_req.comn_hdr.serv_specif_flags,
-                       ICP_QAT_FW_LA_CMP_AUTH_RES);
-       }
-       ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
-                       session->fw_req.comn_hdr.serv_specif_flags,
-                       ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
-       ICP_QAT_FW_LA_PROTO_SET(
-                       session->fw_req.comn_hdr.serv_specif_flags,
-                       ICP_QAT_FW_LA_NO_PROTO);
-       session->fw_req.comn_hdr.service_cmd_id =
-                       ICP_QAT_FW_LA_CMD_CIPHER;
-       session->cd.cipher.cipher_config.val =
-                       ICP_QAT_HW_CIPHER_CONFIG_BUILD(
-                               ICP_QAT_HW_CIPHER_AEAD_MODE,
-                               session->qat_cipher_alg,
-                               ICP_QAT_HW_CIPHER_NO_CONVERT,
-                               session->qat_dir);
-       QAT_FIELD_SET(session->cd.cipher.cipher_config.val,
-                       aead_xform->digest_length,
-                       QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
-                       QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
-       session->cd.cipher.cipher_config.reserved =
-                       ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
-                               aead_xform->aad_length);
-       cipher_param->spc_aad_sz = aead_xform->aad_length;
-       cipher_param->spc_auth_res_sz = aead_xform->digest_length;
-
        return 0;
 }
 
@@ -652,7 +698,31 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
        uint8_t key_length = auth_xform->key.length;
        session->aes_cmac = 0;
 
+       session->auth_iv.offset = auth_xform->iv.offset;
+       session->auth_iv.length = auth_xform->iv.length;
+       session->auth_mode = ICP_QAT_HW_AUTH_MODE1;
+
        switch (auth_xform->algo) {
+       case RTE_CRYPTO_AUTH_SHA1:
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
+               session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
+               break;
+       case RTE_CRYPTO_AUTH_SHA224:
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
+               session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
+               break;
+       case RTE_CRYPTO_AUTH_SHA256:
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
+               session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
+               break;
+       case RTE_CRYPTO_AUTH_SHA384:
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
+               session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
+               break;
+       case RTE_CRYPTO_AUTH_SHA512:
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
+               session->auth_mode = ICP_QAT_HW_AUTH_MODE0;
+               break;
        case RTE_CRYPTO_AUTH_SHA1_HMAC:
                session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
                break;
@@ -683,6 +753,8 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
                }
                session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
                session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
+               if (session->auth_iv.length == 0)
+                       session->auth_iv.length = AES_GCM_J0_LEN;
 
                break;
        case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
@@ -706,11 +778,6 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
                }
                session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
                break;
-       case RTE_CRYPTO_AUTH_SHA1:
-       case RTE_CRYPTO_AUTH_SHA256:
-       case RTE_CRYPTO_AUTH_SHA512:
-       case RTE_CRYPTO_AUTH_SHA224:
-       case RTE_CRYPTO_AUTH_SHA384:
        case RTE_CRYPTO_AUTH_MD5:
        case RTE_CRYPTO_AUTH_AES_CBC_MAC:
                QAT_LOG(ERR, "Crypto: Unsupported hash alg %u",
@@ -722,9 +789,6 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
                return -EINVAL;
        }
 
-       session->auth_iv.offset = auth_xform->iv.offset;
-       session->auth_iv.length = auth_xform->iv.length;
-
        if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
                if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
                        session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
@@ -790,10 +854,6 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
 {
        struct rte_crypto_aead_xform *aead_xform = &xform->aead;
        enum rte_crypto_auth_operation crypto_operation;
-       struct qat_sym_dev_private *internals =
-                       dev->data->dev_private;
-       enum qat_device_gen qat_dev_gen =
-                       internals->qat_dev->qat_dev_gen;
 
        /*
         * Store AEAD IV parameters as cipher IV,
@@ -802,7 +862,8 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
        session->cipher_iv.offset = xform->aead.iv.offset;
        session->cipher_iv.length = xform->aead.iv.length;
 
-       session->is_single_pass = 0;
+       session->auth_mode = ICP_QAT_HW_AUTH_MODE1;
+
        switch (aead_xform->algo) {
        case RTE_CRYPTO_AEAD_AES_GCM:
                if (qat_sym_validate_aes_key(aead_xform->key.length,
@@ -811,13 +872,10 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
                        return -EINVAL;
                }
                session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
-               session->qat_hash_alg =
-                               ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
-               if (qat_dev_gen > QAT_GEN2 && aead_xform->iv.length ==
-                               QAT_AES_GCM_SPC_IV_SIZE) {
-                       return qat_sym_session_handle_single_pass(session,
-                                               aead_xform);
-               }
+               session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
+               if (session->cipher_iv.length == 0)
+                       session->cipher_iv.length = AES_GCM_J0_LEN;
+
                break;
        case RTE_CRYPTO_AEAD_AES_CCM:
                if (qat_sym_validate_aes_key(aead_xform->key.length,
@@ -828,19 +886,23 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
                session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
                session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
                break;
-       case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
-               if (aead_xform->key.length != ICP_QAT_HW_CHACHAPOLY_KEY_SZ)
-                       return -EINVAL;
-               session->qat_cipher_alg =
-                               ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305;
-               return qat_sym_session_handle_single_pass(session,
-                                               aead_xform);
        default:
                QAT_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
                                aead_xform->algo);
                return -EINVAL;
        }
 
+       session->is_single_pass = 0;
+       if (aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) {
+               /* Use faster Single-Pass GCM if possible */
+               int res = qat_sym_session_handle_single_pass(
+                               dev->data->dev_private, session, aead_xform);
+               if (res < 0)
+                       return res;
+               if (session->is_single_pass)
+                       return 0;
+       }
+
        if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
                        aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
                        (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
@@ -1652,10 +1714,11 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
        hash = (struct icp_qat_hw_auth_setup *)cdesc->cd_cur_ptr;
        hash->auth_config.reserved = 0;
        hash->auth_config.config =
-                       ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
+                       ICP_QAT_HW_AUTH_CONFIG_BUILD(cdesc->auth_mode,
                                cdesc->qat_hash_alg, digestsize);
 
-       if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2
+       if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0
+               || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2
                || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9
                || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3
                || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC
@@ -1678,6 +1741,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
         */
        switch (cdesc->qat_hash_alg) {
        case ICP_QAT_HW_AUTH_ALGO_SHA1:
+               if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) {
+                       /* Plain SHA-1 */
+                       rte_memcpy(cdesc->cd_cur_ptr, sha1InitialState,
+                                       sizeof(sha1InitialState));
+                       state1_size = qat_hash_get_state1_size(
+                                       cdesc->qat_hash_alg);
+                       break;
+               }
+               /* SHA-1 HMAC */
                if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1, authkey,
                        authkeylen, cdesc->cd_cur_ptr, &state1_size,
                        cdesc->aes_cmac)) {
@@ -1687,6 +1759,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
                state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA224:
+               if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) {
+                       /* Plain SHA-224 */
+                       rte_memcpy(cdesc->cd_cur_ptr, sha224InitialState,
+                                       sizeof(sha224InitialState));
+                       state1_size = qat_hash_get_state1_size(
+                                       cdesc->qat_hash_alg);
+                       break;
+               }
+               /* SHA-224 HMAC */
                if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224, authkey,
                        authkeylen, cdesc->cd_cur_ptr, &state1_size,
                        cdesc->aes_cmac)) {
@@ -1696,6 +1777,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
                state2_size = ICP_QAT_HW_SHA224_STATE2_SZ;
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA256:
+               if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) {
+                       /* Plain SHA-256 */
+                       rte_memcpy(cdesc->cd_cur_ptr, sha256InitialState,
+                                       sizeof(sha256InitialState));
+                       state1_size = qat_hash_get_state1_size(
+                                       cdesc->qat_hash_alg);
+                       break;
+               }
+               /* SHA-256 HMAC */
                if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256, authkey,
                        authkeylen, cdesc->cd_cur_ptr,  &state1_size,
                        cdesc->aes_cmac)) {
@@ -1705,6 +1795,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
                state2_size = ICP_QAT_HW_SHA256_STATE2_SZ;
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA384:
+               if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) {
+                       /* Plain SHA-384 */
+                       rte_memcpy(cdesc->cd_cur_ptr, sha384InitialState,
+                                       sizeof(sha384InitialState));
+                       state1_size = qat_hash_get_state1_size(
+                                       cdesc->qat_hash_alg);
+                       break;
+               }
+               /* SHA-384 HMAC */
                if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384, authkey,
                        authkeylen, cdesc->cd_cur_ptr, &state1_size,
                        cdesc->aes_cmac)) {
@@ -1714,6 +1813,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
                state2_size = ICP_QAT_HW_SHA384_STATE2_SZ;
                break;
        case ICP_QAT_HW_AUTH_ALGO_SHA512:
+               if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) {
+                       /* Plain SHA-512 */
+                       rte_memcpy(cdesc->cd_cur_ptr, sha512InitialState,
+                                       sizeof(sha512InitialState));
+                       state1_size = qat_hash_get_state1_size(
+                                       cdesc->qat_hash_alg);
+                       break;
+               }
+               /* SHA-512 HMAC */
                if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512, authkey,
                        authkeylen, cdesc->cd_cur_ptr,  &state1_size,
                        cdesc->aes_cmac)) {
@@ -1915,6 +2023,9 @@ int qat_sym_validate_aes_docsisbpi_key(int key_len,
        case ICP_QAT_HW_AES_128_KEY_SZ:
                *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
                break;
+       case ICP_QAT_HW_AES_256_KEY_SZ:
+               *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
+               break;
        default:
                return -EINVAL;
        }