X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fqat%2Fqat_sym_session.c;h=3f2f6736fca2576e73615fc607b2230abad99618;hb=5731efea6f5d22517ca7522ded896471a9a658a1;hp=e5167b3fae569aedefe94397f74d4a065dfacefc;hpb=186b14d6850654eb84a8ae9ea29b736f0ba5e093;p=dpdk.git diff --git a/drivers/crypto/qat/qat_sym_session.c b/drivers/crypto/qat/qat_sym_session.c index e5167b3fae..3f2f6736fc 100644 --- a/drivers/crypto/qat/qat_sym_session.c +++ b/drivers/crypto/qat/qat_sym_session.c @@ -14,11 +14,72 @@ #include #include #include +#ifdef RTE_LIB_SECURITY +#include +#endif #include "qat_logs.h" #include "qat_sym_session.h" #include "qat_sym_pmd.h" +/* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha1InitialState[] = { + 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 0x89, 0x98, 0xba, + 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0}; + +/* SHA 224 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha224InitialState[] = { + 0xc1, 0x05, 0x9e, 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, + 0x17, 0xf7, 0x0e, 0x59, 0x39, 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, + 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, 0xfa, 0x4f, 0xa4}; + +/* SHA 256 - 32 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha256InitialState[] = { + 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 0x85, 0x3c, 0x6e, 0xf3, + 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 0x9b, 0x05, + 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19}; + +/* SHA 384 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha384InitialState[] = { + 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, + 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, + 0xdd, 0x17, 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, + 0x33, 0x26, 0x67, 0xff, 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, + 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 0x2e, 0x0d, 0x64, 0xf9, 0x8f, + 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 0xa4}; + +/* SHA 512 - 64 bytes - Initialiser state can be found in FIPS stds 180-2 */ +static const uint8_t sha512InitialState[] = { + 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 0x67, 0xae, + 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, + 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, + 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, + 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, + 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79}; + +static int +qat_sym_cd_cipher_set(struct qat_sym_session *cd, + const uint8_t *enckey, + uint32_t enckeylen); + +static int +qat_sym_cd_auth_set(struct qat_sym_session *cdesc, + const uint8_t *authkey, + uint32_t authkeylen, + uint32_t aad_length, + uint32_t digestsize, + unsigned int operation); +static void +qat_sym_session_init_common_hdr(struct qat_sym_session *session); + +/* Req/cd init functions */ + +static void +qat_sym_session_finalize(struct qat_sym_session *session) +{ + qat_sym_session_init_common_hdr(session); +} + /** Frees a context previously created * Depends on openssl libcrypto */ @@ -35,7 +96,7 @@ bpi_cipher_ctx_free(void *bpi_ctx) static int bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, enum rte_crypto_cipher_operation direction __rte_unused, - const uint8_t *key, void **ctx) + const uint8_t *key, uint16_t key_length, void **ctx) { const EVP_CIPHER *algo = NULL; int ret; @@ -49,7 +110,10 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo, if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI) algo = EVP_des_ecb(); else - algo = EVP_aes_128_ecb(); + if (key_length == ICP_QAT_HW_AES_128_KEY_SZ) + algo = EVP_aes_128_ecb(); + else + algo = EVP_aes_256_ecb(); /* IV will be ECB encrypted whether direction is encrypt or decrypt*/ if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) { @@ -205,6 +269,8 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, { struct qat_sym_dev_private *internals = dev->data->dev_private; struct rte_crypto_cipher_xform *cipher_xform = NULL; + enum qat_device_gen qat_dev_gen = + internals->qat_dev->qat_dev_gen; int ret; /* Get cipher xform from crypto xform chain */ @@ -231,6 +297,8 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, goto error_out; } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + if (qat_dev_gen == QAT_GEN4) + session->is_ucs = 1; break; case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: if (qat_sym_validate_snow3g_key(cipher_xform->key.length, @@ -286,6 +354,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, cipher_xform->algo, cipher_xform->op, cipher_xform->key.data, + cipher_xform->key.length, &session->bpi_ctx); if (ret != 0) { QAT_LOG(ERR, "failed to create DES BPI ctx"); @@ -304,6 +373,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, cipher_xform->algo, cipher_xform->op, cipher_xform->key.data, + cipher_xform->key.length, &session->bpi_ctx); if (ret != 0) { QAT_LOG(ERR, "failed to create AES BPI ctx"); @@ -368,7 +438,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev, else session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; - if (qat_sym_session_aead_create_cd_cipher(session, + if (qat_sym_cd_cipher_set(session, cipher_xform->key.data, cipher_xform->key.length)) { ret = -EINVAL; @@ -416,19 +486,106 @@ qat_sym_session_configure(struct rte_cryptodev *dev, return 0; } +static void +qat_sym_session_set_ext_hash_flags(struct qat_sym_session *session, + uint8_t hash_flag) +{ + struct icp_qat_fw_comn_req_hdr *header = &session->fw_req.comn_hdr; + struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *cd_ctrl = + (struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *) + session->fw_req.cd_ctrl.content_desc_ctrl_lw; + + /* Set the Use Extended Protocol Flags bit in LW 1 */ + QAT_FIELD_SET(header->comn_req_flags, + QAT_COMN_EXT_FLAGS_USED, + QAT_COMN_EXT_FLAGS_BITPOS, + QAT_COMN_EXT_FLAGS_MASK); + + /* Set Hash Flags in LW 28 */ + cd_ctrl->hash_flags |= hash_flag; + + /* Set proto flags in LW 1 */ + switch (session->qat_cipher_alg) { + case ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_SNOW_3G_PROTO); + ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( + header->serv_specif_flags, 0); + break; + case ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( + header->serv_specif_flags, + ICP_QAT_FW_LA_ZUC_3G_PROTO); + break; + default: + ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET( + header->serv_specif_flags, 0); + break; + } +} + +static void +qat_sym_session_handle_mixed(const struct rte_cryptodev *dev, + struct qat_sym_session *session) +{ + const struct qat_sym_dev_private *qat_private = dev->data->dev_private; + enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities & + QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3; + + if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 && + session->qat_cipher_alg != + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { + session->min_qat_dev_gen = min_dev_gen; + qat_sym_session_set_ext_hash_flags(session, + 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS); + } else if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 && + session->qat_cipher_alg != + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { + session->min_qat_dev_gen = min_dev_gen; + qat_sym_session_set_ext_hash_flags(session, + 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS); + } else if ((session->aes_cmac || + session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) && + (session->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || + session->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) { + session->min_qat_dev_gen = min_dev_gen; + qat_sym_session_set_ext_hash_flags(session, 0); + } +} + int qat_sym_session_set_parameters(struct rte_cryptodev *dev, struct rte_crypto_sym_xform *xform, void *session_private) { struct qat_sym_session *session = session_private; + struct qat_sym_dev_private *internals = dev->data->dev_private; + enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen; int ret; int qat_cmd_id; + int handle_mixed = 0; + /* Verify the session physical address is known */ + rte_iova_t session_paddr = rte_mempool_virt2iova(session); + if (session_paddr == 0 || session_paddr == RTE_BAD_IOVA) { + QAT_LOG(ERR, + "Session physical address unknown. Bad memory pool."); + return -EINVAL; + } + + memset(session, 0, sizeof(*session)); /* Set context descriptor physical address */ - session->cd_paddr = rte_mempool_virt2iova(session) + + session->cd_paddr = session_paddr + offsetof(struct qat_sym_session, cd); session->min_qat_dev_gen = QAT_GEN1; + session->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE; + session->is_ucs = 0; /* Get requested QAT command id */ qat_cmd_id = qat_get_cmd_id(xform); @@ -447,10 +604,14 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, ret = qat_sym_session_configure_auth(dev, xform, session); if (ret < 0) return ret; + session->is_single_pass_gmac = + qat_dev_gen == QAT_GEN3 && + xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC && + xform->auth.iv.length == QAT_AES_GCM_SPC_IV_SIZE; break; case ICP_QAT_FW_LA_CMD_CIPHER_HASH: if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { - ret = qat_sym_session_configure_aead(xform, + ret = qat_sym_session_configure_aead(dev, xform, session); if (ret < 0) return ret; @@ -463,11 +624,12 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, xform, session); if (ret < 0) return ret; + handle_mixed = 1; } break; case ICP_QAT_FW_LA_CMD_HASH_CIPHER: if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { - ret = qat_sym_session_configure_aead(xform, + ret = qat_sym_session_configure_aead(dev, xform, session); if (ret < 0) return ret; @@ -480,6 +642,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, xform, session); if (ret < 0) return ret; + handle_mixed = 1; } break; case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM: @@ -499,6 +662,41 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev, session->qat_cmd); return -ENOTSUP; } + qat_sym_session_finalize(session); + if (handle_mixed) { + /* Special handling of mixed hash+cipher algorithms */ + qat_sym_session_handle_mixed(dev, session); + } + + return 0; +} + +static int +qat_sym_session_handle_single_pass(struct qat_sym_session *session, + const struct rte_crypto_aead_xform *aead_xform) +{ + session->is_single_pass = 1; + session->is_auth = 1; + session->min_qat_dev_gen = QAT_GEN3; + session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER; + /* Chacha-Poly is special case that use QAT CTR mode */ + if (aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) { + session->qat_mode = ICP_QAT_HW_CIPHER_AEAD_MODE; + } else { + session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; + } + session->cipher_iv.offset = aead_xform->iv.offset; + session->cipher_iv.length = aead_xform->iv.length; + session->aad_len = aead_xform->aad_length; + session->digest_length = aead_xform->digest_length; + + if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { + session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + session->auth_op = ICP_QAT_HW_AUTH_GENERATE; + } else { + session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + session->auth_op = ICP_QAT_HW_AUTH_VERIFY; + } return 0; } @@ -512,9 +710,38 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, struct qat_sym_dev_private *internals = dev->data->dev_private; const uint8_t *key_data = auth_xform->key.data; uint8_t key_length = auth_xform->key.length; + enum qat_device_gen qat_dev_gen = + internals->qat_dev->qat_dev_gen; + session->aes_cmac = 0; + session->auth_key_length = auth_xform->key.length; + session->auth_iv.offset = auth_xform->iv.offset; + session->auth_iv.length = auth_xform->iv.length; + session->auth_mode = ICP_QAT_HW_AUTH_MODE1; + session->is_auth = 1; + session->digest_length = auth_xform->digest_length; switch (auth_xform->algo) { + case RTE_CRYPTO_AUTH_SHA1: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1; + session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; + case RTE_CRYPTO_AUTH_SHA224: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224; + session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; + case RTE_CRYPTO_AUTH_SHA256: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256; + session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; + case RTE_CRYPTO_AUTH_SHA384: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384; + session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; + case RTE_CRYPTO_AUTH_SHA512: + session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512; + session->auth_mode = ICP_QAT_HW_AUTH_MODE0; + break; case RTE_CRYPTO_AUTH_SHA1_HMAC: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1; break; @@ -545,7 +772,14 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; - + if (session->auth_iv.length == 0) + session->auth_iv.length = AES_GCM_J0_LEN; + else + session->is_iv12B = 1; + if (qat_dev_gen == QAT_GEN4) { + session->is_cnt_zero = 1; + session->is_ucs = 1; + } break; case RTE_CRYPTO_AUTH_SNOW3G_UIA2: session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2; @@ -568,11 +802,6 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, } session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3; break; - case RTE_CRYPTO_AUTH_SHA1: - case RTE_CRYPTO_AUTH_SHA256: - case RTE_CRYPTO_AUTH_SHA512: - case RTE_CRYPTO_AUTH_SHA224: - case RTE_CRYPTO_AUTH_SHA384: case RTE_CRYPTO_AUTH_MD5: case RTE_CRYPTO_AUTH_AES_CBC_MAC: QAT_LOG(ERR, "Crypto: Unsupported hash alg %u", @@ -584,10 +813,8 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, return -EINVAL; } - session->auth_iv.offset = auth_xform->iv.offset; - session->auth_iv.length = auth_xform->iv.length; - if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) { + session->is_gmac = 1; if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) { session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH; session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; @@ -595,13 +822,12 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, * It needs to create cipher desc content first, * then authentication */ - - if (qat_sym_session_aead_create_cd_cipher(session, + if (qat_sym_cd_cipher_set(session, auth_xform->key.data, auth_xform->key.length)) return -EINVAL; - if (qat_sym_session_aead_create_cd_auth(session, + if (qat_sym_cd_auth_set(session, key_data, key_length, 0, @@ -616,7 +842,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, * then cipher */ - if (qat_sym_session_aead_create_cd_auth(session, + if (qat_sym_cd_auth_set(session, key_data, key_length, 0, @@ -624,15 +850,13 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, auth_xform->op)) return -EINVAL; - if (qat_sym_session_aead_create_cd_cipher(session, + if (qat_sym_cd_cipher_set(session, auth_xform->key.data, auth_xform->key.length)) return -EINVAL; } - /* Restore to authentication only only */ - session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH; } else { - if (qat_sym_session_aead_create_cd_auth(session, + if (qat_sym_cd_auth_set(session, key_data, key_length, 0, @@ -641,16 +865,20 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev, return -EINVAL; } - session->digest_length = auth_xform->digest_length; return 0; } int -qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, +qat_sym_session_configure_aead(struct rte_cryptodev *dev, + struct rte_crypto_sym_xform *xform, struct qat_sym_session *session) { struct rte_crypto_aead_xform *aead_xform = &xform->aead; enum rte_crypto_auth_operation crypto_operation; + struct qat_sym_dev_private *internals = + dev->data->dev_private; + enum qat_device_gen qat_dev_gen = + internals->qat_dev->qat_dev_gen; /* * Store AEAD IV parameters as cipher IV, @@ -659,6 +887,11 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, session->cipher_iv.offset = xform->aead.iv.offset; session->cipher_iv.length = xform->aead.iv.length; + session->auth_mode = ICP_QAT_HW_AUTH_MODE1; + session->is_auth = 1; + session->digest_length = aead_xform->digest_length; + + session->is_single_pass = 0; switch (aead_xform->algo) { case RTE_CRYPTO_AEAD_AES_GCM: if (qat_sym_validate_aes_key(aead_xform->key.length, @@ -668,6 +901,18 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128; + + if (qat_dev_gen == QAT_GEN4) + session->is_ucs = 1; + if (session->cipher_iv.length == 0) { + session->cipher_iv.length = AES_GCM_J0_LEN; + break; + } + session->is_iv12B = 1; + if (qat_dev_gen < QAT_GEN3) + break; + qat_sym_session_handle_single_pass(session, + aead_xform); break; case RTE_CRYPTO_AEAD_AES_CCM: if (qat_sym_validate_aes_key(aead_xform->key.length, @@ -677,6 +922,18 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, } session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE; session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC; + if (qat_dev_gen == QAT_GEN4) + session->is_ucs = 1; + break; + case RTE_CRYPTO_AEAD_CHACHA20_POLY1305: + if (aead_xform->key.length != ICP_QAT_HW_CHACHAPOLY_KEY_SZ) + return -EINVAL; + if (qat_dev_gen == QAT_GEN4) + session->is_ucs = 1; + session->qat_cipher_alg = + ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305; + qat_sym_session_handle_single_pass(session, + aead_xform); break; default: QAT_LOG(ERR, "Crypto: Undefined AEAD specified %u\n", @@ -684,7 +941,11 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, return -EINVAL; } - if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && + if (session->is_single_pass) { + if (qat_sym_cd_cipher_set(session, + aead_xform->key.data, aead_xform->key.length)) + return -EINVAL; + } else if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) || (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT && aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) { @@ -696,12 +957,12 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY; - if (qat_sym_session_aead_create_cd_cipher(session, + if (qat_sym_cd_cipher_set(session, aead_xform->key.data, aead_xform->key.length)) return -EINVAL; - if (qat_sym_session_aead_create_cd_auth(session, + if (qat_sym_cd_auth_set(session, aead_xform->key.data, aead_xform->key.length, aead_xform->aad_length, @@ -718,7 +979,7 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ? RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE; - if (qat_sym_session_aead_create_cd_auth(session, + if (qat_sym_cd_auth_set(session, aead_xform->key.data, aead_xform->key.length, aead_xform->aad_length, @@ -726,13 +987,12 @@ qat_sym_session_configure_aead(struct rte_crypto_sym_xform *xform, crypto_operation)) return -EINVAL; - if (qat_sym_session_aead_create_cd_cipher(session, + if (qat_sym_cd_cipher_set(session, aead_xform->key.data, aead_xform->key.length)) return -EINVAL; } - session->digest_length = aead_xform->digest_length; return 0; } @@ -955,6 +1215,9 @@ static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, uint64_t *hash_state_out_be64; int i; + /* Initialize to avoid gcc warning */ + memset(digest, 0, sizeof(digest)); + digest_size = qat_hash_get_digest_size(hash_alg); if (digest_size <= 0) return -EFAULT; @@ -1204,12 +1467,17 @@ static int qat_sym_do_precomputes(enum icp_qat_hw_auth_algo hash_alg, } static void -qat_sym_session_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, - enum qat_sym_proto_flag proto_flags) +qat_sym_session_init_common_hdr(struct qat_sym_session *session) { + struct icp_qat_fw_la_bulk_req *req_tmpl = &session->fw_req; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + enum qat_sym_proto_flag proto_flags = session->qat_proto_flag; + uint32_t slice_flags = session->slice_types; + header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; + header->service_cmd_id = session->qat_cmd; header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, QAT_COMN_PTR_TYPE_FLAT); @@ -1241,41 +1509,55 @@ qat_sym_session_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, break; } - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); -} + /* More than one of the following flags can be set at once */ + if (QAT_SESSION_IS_SLICE_SET(slice_flags, QAT_CRYPTO_SLICE_SPC)) { + ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET( + header->serv_specif_flags, + ICP_QAT_FW_LA_SINGLE_PASS_PROTO); + } + if (QAT_SESSION_IS_SLICE_SET(slice_flags, QAT_CRYPTO_SLICE_UCS)) { + ICP_QAT_FW_LA_SLICE_TYPE_SET( + header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + } -/* - * Snow3G and ZUC should never use this function - * and set its protocol flag in both cipher and auth part of content - * descriptor building function - */ -static enum qat_sym_proto_flag -qat_get_crypto_proto_flag(uint16_t flags) -{ - int proto = ICP_QAT_FW_LA_PROTO_GET(flags); - enum qat_sym_proto_flag qat_proto_flag = - QAT_CRYPTO_PROTO_FLAG_NONE; + if (session->is_auth) { + if (session->auth_op == ICP_QAT_HW_AUTH_VERIFY) { + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_CMP_AUTH_RES); + } else if (session->auth_op == ICP_QAT_HW_AUTH_GENERATE) { + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + } + } else { + ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_RET_AUTH_RES); + ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + } - switch (proto) { - case ICP_QAT_FW_LA_GCM_PROTO: - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; - break; - case ICP_QAT_FW_LA_CCM_PROTO: - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; - break; + if (session->is_iv12B) { + ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( + header->serv_specif_flags, + ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); } - return qat_proto_flag; + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); } -int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, +int qat_sym_cd_cipher_set(struct qat_sym_session *cdesc, const uint8_t *cipherkey, uint32_t cipherkeylen) { struct icp_qat_hw_cipher_algo_blk *cipher; + struct icp_qat_hw_cipher_algo_blk20 *cipher20; struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; @@ -1283,8 +1565,12 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; enum icp_qat_hw_cipher_convert key_convert; - enum qat_sym_proto_flag qat_proto_flag = - QAT_CRYPTO_PROTO_FLAG_NONE; + struct icp_qat_fw_la_cipher_20_req_params *req_ucs = + (struct icp_qat_fw_la_cipher_20_req_params *) + &cdesc->fw_req.serv_specif_rqpars; + struct icp_qat_fw_la_cipher_req_params *req_cipher = + (struct icp_qat_fw_la_cipher_req_params *) + &cdesc->fw_req.serv_specif_rqpars; uint32_t total_key_size; uint16_t cipher_offset, cd_size; uint32_t wordIndex = 0; @@ -1320,9 +1606,16 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) { /* * CTR Streaming ciphers are a special case. Decrypt = encrypt - * Overriding default values previously set + * Overriding default values previously set. + * Chacha20-Poly1305 is special case, CTR but single-pass + * so both direction need to be used. */ cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; + if (cdesc->qat_cipher_alg == + ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305 && + cdesc->auth_op == ICP_QAT_HW_AUTH_VERIFY) { + cdesc->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT; + } key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || cdesc->qat_cipher_alg == @@ -1330,6 +1623,8 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; + else if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE) + key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; else key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; @@ -1338,7 +1633,7 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) { total_key_size = ICP_QAT_HW_KASUMI_F8_KEY_SZ; @@ -1348,35 +1643,26 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) { total_key_size = ICP_QAT_HW_3DES_KEY_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_3DES_BLK_SZ >> 3; - qat_proto_flag = - qat_get_crypto_proto_flag(header->serv_specif_flags); } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_DES) { total_key_size = ICP_QAT_HW_DES_KEY_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_DES_BLK_SZ >> 3; - qat_proto_flag = - qat_get_crypto_proto_flag(header->serv_specif_flags); } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { total_key_size = ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; cdesc->min_qat_dev_gen = QAT_GEN2; } else { total_key_size = cipherkeylen; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; - qat_proto_flag = - qat_get_crypto_proto_flag(header->serv_specif_flags); } - cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3; cipher_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd); cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; - header->service_cmd_id = cdesc->qat_cmd; - qat_sym_session_init_common_hdr(header, qat_proto_flag); - cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr; + cipher20 = (struct icp_qat_hw_cipher_algo_blk20 *)cdesc->cd_cur_ptr; cipher->cipher_config.val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode, cdesc->qat_cipher_alg, key_convert, @@ -1396,12 +1682,38 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + cipherkeylen + cipherkeylen; + } else if (cdesc->is_ucs) { + const uint8_t *final_key = cipherkey; + + cdesc->slice_types |= QAT_CRYPTO_SLICE_UCS; + total_key_size = RTE_ALIGN_CEIL(cipherkeylen, + ICP_QAT_HW_AES_128_KEY_SZ); + cipher20->cipher_config.reserved[0] = 0; + cipher20->cipher_config.reserved[1] = 0; + cipher20->cipher_config.reserved[2] = 0; + + rte_memcpy(cipher20->key, final_key, cipherkeylen); + cdesc->cd_cur_ptr += + sizeof(struct icp_qat_hw_ucs_cipher_config) + + cipherkeylen; } else { memcpy(cipher->key, cipherkey, cipherkeylen); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + cipherkeylen; } + if (cdesc->is_single_pass) { + QAT_FIELD_SET(cipher->cipher_config.val, + cdesc->digest_length, + QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS, + QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK); + /* UCS and SPC 1.8/2.0 share configuration of 2nd config word */ + cdesc->cd.cipher.cipher_config.reserved = + ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER( + cdesc->aad_len); + cdesc->slice_types |= QAT_CRYPTO_SLICE_SPC; + } + if (total_key_size > cipherkeylen) { uint32_t padding_size = total_key_size-cipherkeylen; if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) @@ -1420,13 +1732,28 @@ int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc, cdesc->cd_cur_ptr += padding_size; } + if (cdesc->is_ucs) { + /* + * These values match in terms of position auth + * slice request fields + */ + req_ucs->spc_auth_res_sz = cdesc->digest_length; + if (!cdesc->is_gmac) { + req_ucs->spc_aad_sz = cdesc->aad_len; + req_ucs->spc_aad_offset = 0; + } + } else if (cdesc->is_single_pass) { + req_cipher->spc_aad_sz = cdesc->aad_len; + req_cipher->spc_auth_res_sz = cdesc->digest_length; + } cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3; + cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3; return 0; } -int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, +int qat_sym_cd_auth_set(struct qat_sym_session *cdesc, const uint8_t *authkey, uint32_t authkeylen, uint32_t aad_length, @@ -1437,21 +1764,18 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, struct icp_qat_hw_cipher_algo_blk *cipherconfig; struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; struct icp_qat_fw_la_auth_req_params *auth_param = (struct icp_qat_fw_la_auth_req_params *) ((char *)&req_tmpl->serv_specif_rqpars + - sizeof(struct icp_qat_fw_la_cipher_req_params)); - uint16_t state1_size = 0, state2_size = 0; + ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); + uint16_t state1_size = 0, state2_size = 0, cd_extra_size = 0; uint16_t hash_offset, cd_size; uint32_t *aad_len = NULL; uint32_t wordIndex = 0; uint32_t *pTempKey; - enum qat_sym_proto_flag qat_proto_flag = - QAT_CRYPTO_PROTO_FLAG_NONE; if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, @@ -1474,19 +1798,10 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, return -EFAULT; } - if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_CMP_AUTH_RES); + if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) cdesc->auth_op = ICP_QAT_HW_AUTH_VERIFY; - } else { - ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_RET_AUTH_RES); - ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_CMP_AUTH_RES); + else cdesc->auth_op = ICP_QAT_HW_AUTH_GENERATE; - } /* * Setup the inner hash config @@ -1495,15 +1810,17 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, hash = (struct icp_qat_hw_auth_setup *)cdesc->cd_cur_ptr; hash->auth_config.reserved = 0; hash->auth_config.config = - ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, + ICP_QAT_HW_AUTH_CONFIG_BUILD(cdesc->auth_mode, cdesc->qat_hash_alg, digestsize); - if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0 + || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL + || cdesc->is_cnt_zero ) hash->auth_counter.counter = 0; else { @@ -1521,6 +1838,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, */ switch (cdesc->qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) { + /* Plain SHA-1 */ + rte_memcpy(cdesc->cd_cur_ptr, sha1InitialState, + sizeof(sha1InitialState)); + state1_size = qat_hash_get_state1_size( + cdesc->qat_hash_alg); + break; + } + /* SHA-1 HMAC */ if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size, cdesc->aes_cmac)) { @@ -1530,6 +1856,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8); break; case ICP_QAT_HW_AUTH_ALGO_SHA224: + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) { + /* Plain SHA-224 */ + rte_memcpy(cdesc->cd_cur_ptr, sha224InitialState, + sizeof(sha224InitialState)); + state1_size = qat_hash_get_state1_size( + cdesc->qat_hash_alg); + break; + } + /* SHA-224 HMAC */ if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size, cdesc->aes_cmac)) { @@ -1539,6 +1874,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, state2_size = ICP_QAT_HW_SHA224_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA256: + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) { + /* Plain SHA-256 */ + rte_memcpy(cdesc->cd_cur_ptr, sha256InitialState, + sizeof(sha256InitialState)); + state1_size = qat_hash_get_state1_size( + cdesc->qat_hash_alg); + break; + } + /* SHA-256 HMAC */ if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size, cdesc->aes_cmac)) { @@ -1548,6 +1892,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, state2_size = ICP_QAT_HW_SHA256_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA384: + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) { + /* Plain SHA-384 */ + rte_memcpy(cdesc->cd_cur_ptr, sha384InitialState, + sizeof(sha384InitialState)); + state1_size = qat_hash_get_state1_size( + cdesc->qat_hash_alg); + break; + } + /* SHA-384 HMAC */ if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size, cdesc->aes_cmac)) { @@ -1557,6 +1910,15 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, state2_size = ICP_QAT_HW_SHA384_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA512: + if (cdesc->auth_mode == ICP_QAT_HW_AUTH_MODE0) { + /* Plain SHA-512 */ + rte_memcpy(cdesc->cd_cur_ptr, sha512InitialState, + sizeof(sha512InitialState)); + state1_size = qat_hash_get_state1_size( + cdesc->qat_hash_alg); + break; + } + /* SHA-512 HMAC */ if (qat_sym_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size, cdesc->aes_cmac)) { @@ -1582,7 +1944,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, break; case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ; if (qat_sym_do_precomputes(cdesc->qat_hash_alg, authkey, authkeylen, cdesc->cd_cur_ptr + state1_size, @@ -1605,7 +1967,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, cdesc->aad_len = aad_length; break; case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2); state2_size = ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ; @@ -1621,7 +1983,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, memcpy(cipherconfig->key, authkey, authkeylen); memset(cipherconfig->key + authkeylen, 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); - cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + + cd_extra_size += sizeof(struct icp_qat_hw_cipher_config) + authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; auth_param->hash_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; break; @@ -1629,7 +1991,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, hash->auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE0, cdesc->qat_hash_alg, digestsize); - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3); state2_size = ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ; @@ -1637,8 +1999,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); - cdesc->cd_cur_ptr += state1_size + state2_size - + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; + cd_extra_size += ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; cdesc->min_qat_dev_gen = QAT_GEN2; @@ -1658,7 +2019,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC: - qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; + cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC); state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ + @@ -1706,10 +2067,6 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, return -EFAULT; } - /* Request template setup */ - qat_sym_session_init_common_hdr(header, qat_proto_flag); - header->service_cmd_id = cdesc->qat_cmd; - /* Auth CD config setup */ hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3; hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; @@ -1724,7 +2081,7 @@ int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc, RTE_ALIGN_CEIL(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); - cdesc->cd_cur_ptr += state1_size + state2_size; + cdesc->cd_cur_ptr += state1_size + state2_size + cd_extra_size; cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; @@ -1758,6 +2115,9 @@ int qat_sym_validate_aes_docsisbpi_key(int key_len, case ICP_QAT_HW_AES_128_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; break; + case ICP_QAT_HW_AES_256_KEY_SZ: + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + break; default: return -EINVAL; } @@ -1825,3 +2185,152 @@ int qat_sym_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg) } return 0; } + +#ifdef RTE_LIB_SECURITY +static int +qat_sec_session_check_docsis(struct rte_security_session_conf *conf) +{ + struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform; + struct rte_security_docsis_xform *docsis = &conf->docsis; + + /* CRC generate -> Cipher encrypt */ + if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) { + + if (crypto_sym != NULL && + crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + (crypto_sym->cipher.key.length == + ICP_QAT_HW_AES_128_KEY_SZ || + crypto_sym->cipher.key.length == + ICP_QAT_HW_AES_256_KEY_SZ) && + crypto_sym->cipher.iv.length == ICP_QAT_HW_AES_BLK_SZ && + crypto_sym->next == NULL) { + return 0; + } + /* Cipher decrypt -> CRC verify */ + } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) { + + if (crypto_sym != NULL && + crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT && + crypto_sym->cipher.algo == + RTE_CRYPTO_CIPHER_AES_DOCSISBPI && + (crypto_sym->cipher.key.length == + ICP_QAT_HW_AES_128_KEY_SZ || + crypto_sym->cipher.key.length == + ICP_QAT_HW_AES_256_KEY_SZ) && + crypto_sym->cipher.iv.length == ICP_QAT_HW_AES_BLK_SZ && + crypto_sym->next == NULL) { + return 0; + } + } + + return -EINVAL; +} + +static int +qat_sec_session_set_docsis_parameters(struct rte_cryptodev *dev, + struct rte_security_session_conf *conf, void *session_private) +{ + int ret; + int qat_cmd_id; + struct rte_crypto_sym_xform *xform = NULL; + struct qat_sym_session *session = session_private; + + /* Clear the session */ + memset(session, 0, qat_sym_session_get_private_size(dev)); + + ret = qat_sec_session_check_docsis(conf); + if (ret) { + QAT_LOG(ERR, "Unsupported DOCSIS security configuration"); + return ret; + } + + xform = conf->crypto_xform; + + /* Verify the session physical address is known */ + rte_iova_t session_paddr = rte_mempool_virt2iova(session); + if (session_paddr == 0 || session_paddr == RTE_BAD_IOVA) { + QAT_LOG(ERR, + "Session physical address unknown. Bad memory pool."); + return -EINVAL; + } + + /* Set context descriptor physical address */ + session->cd_paddr = session_paddr + + offsetof(struct qat_sym_session, cd); + + session->min_qat_dev_gen = QAT_GEN1; + + /* Get requested QAT command id - should be cipher */ + qat_cmd_id = qat_get_cmd_id(xform); + if (qat_cmd_id != ICP_QAT_FW_LA_CMD_CIPHER) { + QAT_LOG(ERR, "Unsupported xform chain requested"); + return -ENOTSUP; + } + session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id; + + ret = qat_sym_session_configure_cipher(dev, xform, session); + if (ret < 0) + return ret; + qat_sym_session_finalize(session); + + return 0; +} + +int +qat_security_session_create(void *dev, + struct rte_security_session_conf *conf, + struct rte_security_session *sess, + struct rte_mempool *mempool) +{ + void *sess_private_data; + struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; + int ret; + + if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL || + conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) { + QAT_LOG(ERR, "Invalid security protocol"); + return -EINVAL; + } + + if (rte_mempool_get(mempool, &sess_private_data)) { + QAT_LOG(ERR, "Couldn't get object from session mempool"); + return -ENOMEM; + } + + ret = qat_sec_session_set_docsis_parameters(cdev, conf, + sess_private_data); + if (ret != 0) { + QAT_LOG(ERR, "Failed to configure session parameters"); + /* Return session to mempool */ + rte_mempool_put(mempool, sess_private_data); + return ret; + } + + set_sec_session_private_data(sess, sess_private_data); + + return ret; +} + +int +qat_security_session_destroy(void *dev __rte_unused, + struct rte_security_session *sess) +{ + void *sess_priv = get_sec_session_private_data(sess); + struct qat_sym_session *s = (struct qat_sym_session *)sess_priv; + + if (sess_priv) { + if (s->bpi_ctx) + bpi_cipher_ctx_free(s->bpi_ctx); + memset(s, 0, qat_sym_session_get_private_size(dev)); + struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); + + set_sec_session_private_data(sess, NULL); + rte_mempool_put(sess_mp, sess_priv); + } + return 0; +} +#endif