/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
- * Copyright(c) 2015-2019 Intel Corporation
+ * Copyright(c) 2015-2022 Intel Corporation
*/
+#define OPENSSL_API_COMPAT 0x10100000L
+
#include <openssl/sha.h> /* Needed to calculate pre-compute values */
#include <openssl/aes.h> /* Needed to calculate pre-compute values */
#include <openssl/md5.h> /* Needed to calculate pre-compute values */
#include <rte_log.h>
#include <rte_malloc.h>
#include <rte_crypto_sym.h>
+#ifdef RTE_LIB_SECURITY
+#include <rte_security.h>
+#endif
#include "qat_logs.h"
#include "qat_sym_session.h"
-#include "qat_sym_pmd.h"
+#include "qat_sym.h"
/* SHA1 - 20 bytes - Initialiser state can be found in FIPS stds 180-2 */
static const uint8_t sha1InitialState[] = {
0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd,
0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 0x7e, 0x21, 0x79};
+static int
+qat_sym_cd_cipher_set(struct qat_sym_session *cd,
+ const uint8_t *enckey,
+ uint32_t enckeylen);
+
+static int
+qat_sym_cd_auth_set(struct qat_sym_session *cdesc,
+ const uint8_t *authkey,
+ uint32_t authkeylen,
+ uint32_t aad_length,
+ uint32_t digestsize,
+ unsigned int operation);
+static void
+qat_sym_session_init_common_hdr(struct qat_sym_session *session);
+
+/* Req/cd init functions */
+
+static void
+qat_sym_session_finalize(struct qat_sym_session *session)
+{
+ qat_sym_session_init_common_hdr(session);
+}
+
/** Frees a context previously created
* Depends on openssl libcrypto
*/
static int
qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
- struct qat_sym_dev_private *internals)
+ struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
static int
qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
- struct qat_sym_dev_private *internals)
+ struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
struct rte_crypto_sym_xform *xform,
struct qat_sym_session *session)
{
- struct qat_sym_dev_private *internals = dev->data->dev_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
struct rte_crypto_cipher_xform *cipher_xform = NULL;
+ enum qat_device_gen qat_dev_gen =
+ internals->qat_dev->qat_dev_gen;
int ret;
/* Get cipher xform from crypto xform chain */
goto error_out;
}
session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
+ if (qat_dev_gen == QAT_GEN4)
+ session->is_ucs = 1;
break;
case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
if (qat_sym_validate_snow3g_key(cipher_xform->key.length,
else
session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
- if (qat_sym_session_aead_create_cd_cipher(session,
+ if (qat_sym_cd_cipher_set(session,
cipher_xform->key.data,
cipher_xform->key.length)) {
ret = -EINVAL;
return 0;
}
-static void
-qat_sym_session_set_ext_hash_flags(struct qat_sym_session *session,
- uint8_t hash_flag)
-{
- struct icp_qat_fw_comn_req_hdr *header = &session->fw_req.comn_hdr;
- struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *cd_ctrl =
- (struct icp_qat_fw_cipher_auth_cd_ctrl_hdr *)
- session->fw_req.cd_ctrl.content_desc_ctrl_lw;
-
- /* Set the Use Extended Protocol Flags bit in LW 1 */
- QAT_FIELD_SET(header->comn_req_flags,
- QAT_COMN_EXT_FLAGS_USED,
- QAT_COMN_EXT_FLAGS_BITPOS,
- QAT_COMN_EXT_FLAGS_MASK);
-
- /* Set Hash Flags in LW 28 */
- cd_ctrl->hash_flags |= hash_flag;
-
- /* Set proto flags in LW 1 */
- switch (session->qat_cipher_alg) {
- case ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2:
- ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_SNOW_3G_PROTO);
- ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
- header->serv_specif_flags, 0);
- break;
- case ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3:
- ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_PROTO);
- ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
- header->serv_specif_flags,
- ICP_QAT_FW_LA_ZUC_3G_PROTO);
- break;
- default:
- ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_PROTO);
- ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(
- header->serv_specif_flags, 0);
- break;
- }
-}
-
-static void
-qat_sym_session_handle_mixed(const struct rte_cryptodev *dev,
- struct qat_sym_session *session)
-{
- const struct qat_sym_dev_private *qat_private = dev->data->dev_private;
- enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities &
- QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3;
-
- if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
- session->qat_cipher_alg !=
- ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
- session->min_qat_dev_gen = min_dev_gen;
- qat_sym_session_set_ext_hash_flags(session,
- 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
- } else if (session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
- session->qat_cipher_alg !=
- ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
- session->min_qat_dev_gen = min_dev_gen;
- qat_sym_session_set_ext_hash_flags(session,
- 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
- } else if ((session->aes_cmac ||
- session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) &&
- (session->qat_cipher_alg ==
- ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
- session->qat_cipher_alg ==
- ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
- session->min_qat_dev_gen = min_dev_gen;
- qat_sym_session_set_ext_hash_flags(session, 0);
- }
-}
-
int
qat_sym_session_set_parameters(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform, void *session_private)
{
struct qat_sym_session *session = session_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
+ enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
int ret;
int qat_cmd_id;
return -EINVAL;
}
+ memset(session, 0, sizeof(*session));
/* Set context descriptor physical address */
session->cd_paddr = session_paddr +
offsetof(struct qat_sym_session, cd);
- session->min_qat_dev_gen = QAT_GEN1;
+ session->dev_id = internals->dev_id;
+ session->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE;
+ session->is_ucs = 0;
/* Get requested QAT command id */
qat_cmd_id = qat_get_cmd_id(xform);
ret = qat_sym_session_configure_auth(dev, xform, session);
if (ret < 0)
return ret;
+ session->is_single_pass_gmac =
+ qat_dev_gen == QAT_GEN3 &&
+ xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC &&
+ xform->auth.iv.length == QAT_AES_GCM_SPC_IV_SIZE;
break;
case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
xform, session);
if (ret < 0)
return ret;
- /* Special handling of mixed hash+cipher algorithms */
- qat_sym_session_handle_mixed(dev, session);
}
break;
case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
xform, session);
if (ret < 0)
return ret;
- /* Special handling of mixed hash+cipher algorithms */
- qat_sym_session_handle_mixed(dev, session);
}
break;
case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
session->qat_cmd);
return -ENOTSUP;
}
+ qat_sym_session_finalize(session);
- return 0;
+ return qat_sym_gen_dev_ops[qat_dev_gen].set_session((void *)dev,
+ (void *)session);
}
static int
-qat_sym_session_handle_single_pass(struct qat_sym_dev_private *internals,
- struct qat_sym_session *session,
- struct rte_crypto_aead_xform *aead_xform)
+qat_sym_session_handle_single_pass(struct qat_sym_session *session,
+ const struct rte_crypto_aead_xform *aead_xform)
{
- enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
+ session->is_single_pass = 1;
+ session->is_auth = 1;
+ session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER;
+ /* Chacha-Poly is special case that use QAT CTR mode */
+ if (aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM)
+ session->qat_mode = ICP_QAT_HW_CIPHER_AEAD_MODE;
+ else
+ session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
- if (qat_dev_gen == QAT_GEN3 &&
- aead_xform->iv.length == QAT_AES_GCM_SPC_IV_SIZE) {
- /* Use faster Single-Pass GCM */
- struct icp_qat_fw_la_cipher_req_params *cipher_param =
- (void *) &session->fw_req.serv_specif_rqpars;
+ session->cipher_iv.offset = aead_xform->iv.offset;
+ session->cipher_iv.length = aead_xform->iv.length;
+ session->aad_len = aead_xform->aad_length;
+ session->digest_length = aead_xform->digest_length;
- session->is_single_pass = 1;
- session->min_qat_dev_gen = QAT_GEN3;
- session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER;
- session->qat_mode = ICP_QAT_HW_CIPHER_AEAD_MODE;
- session->cipher_iv.offset = aead_xform->iv.offset;
- session->cipher_iv.length = aead_xform->iv.length;
- if (qat_sym_session_aead_create_cd_cipher(session,
- aead_xform->key.data, aead_xform->key.length))
- return -EINVAL;
- session->aad_len = aead_xform->aad_length;
- session->digest_length = aead_xform->digest_length;
- if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
- session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
- session->auth_op = ICP_QAT_HW_AUTH_GENERATE;
- ICP_QAT_FW_LA_RET_AUTH_SET(
- session->fw_req.comn_hdr.serv_specif_flags,
- ICP_QAT_FW_LA_RET_AUTH_RES);
- } else {
- session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
- session->auth_op = ICP_QAT_HW_AUTH_VERIFY;
- ICP_QAT_FW_LA_CMP_AUTH_SET(
- session->fw_req.comn_hdr.serv_specif_flags,
- ICP_QAT_FW_LA_CMP_AUTH_RES);
- }
- ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
- session->fw_req.comn_hdr.serv_specif_flags,
- ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
- ICP_QAT_FW_LA_PROTO_SET(
- session->fw_req.comn_hdr.serv_specif_flags,
- ICP_QAT_FW_LA_NO_PROTO);
- ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
- session->fw_req.comn_hdr.serv_specif_flags,
- ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
- session->fw_req.comn_hdr.service_cmd_id =
- ICP_QAT_FW_LA_CMD_CIPHER;
- session->cd.cipher.cipher_config.val =
- ICP_QAT_HW_CIPHER_CONFIG_BUILD(
- ICP_QAT_HW_CIPHER_AEAD_MODE,
- session->qat_cipher_alg,
- ICP_QAT_HW_CIPHER_NO_CONVERT,
- session->qat_dir);
- QAT_FIELD_SET(session->cd.cipher.cipher_config.val,
- aead_xform->digest_length,
- QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
- QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
- session->cd.cipher.cipher_config.reserved =
- ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
- aead_xform->aad_length);
- cipher_param->spc_aad_sz = aead_xform->aad_length;
- cipher_param->spc_auth_res_sz = aead_xform->digest_length;
+ if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
+ session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+ session->auth_op = ICP_QAT_HW_AUTH_GENERATE;
+ } else {
+ session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+ session->auth_op = ICP_QAT_HW_AUTH_VERIFY;
}
+
return 0;
}
struct qat_sym_session *session)
{
struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform);
- struct qat_sym_dev_private *internals = dev->data->dev_private;
+ struct qat_cryptodev_private *internals = dev->data->dev_private;
const uint8_t *key_data = auth_xform->key.data;
uint8_t key_length = auth_xform->key.length;
- session->aes_cmac = 0;
+ enum qat_device_gen qat_dev_gen =
+ internals->qat_dev->qat_dev_gen;
+ session->aes_cmac = 0;
+ session->auth_key_length = auth_xform->key.length;
session->auth_iv.offset = auth_xform->iv.offset;
session->auth_iv.length = auth_xform->iv.length;
session->auth_mode = ICP_QAT_HW_AUTH_MODE1;
+ session->is_auth = 1;
+ session->digest_length = auth_xform->digest_length;
switch (auth_xform->algo) {
case RTE_CRYPTO_AUTH_SHA1:
session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
if (session->auth_iv.length == 0)
session->auth_iv.length = AES_GCM_J0_LEN;
-
+ else
+ session->is_iv12B = 1;
+ if (qat_dev_gen == QAT_GEN4) {
+ session->is_cnt_zero = 1;
+ session->is_ucs = 1;
+ }
break;
case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
}
if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
+ session->is_gmac = 1;
if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
* It needs to create cipher desc content first,
* then authentication
*/
-
- if (qat_sym_session_aead_create_cd_cipher(session,
+ if (qat_sym_cd_cipher_set(session,
auth_xform->key.data,
auth_xform->key.length))
return -EINVAL;
- if (qat_sym_session_aead_create_cd_auth(session,
+ if (qat_sym_cd_auth_set(session,
key_data,
key_length,
0,
* then cipher
*/
- if (qat_sym_session_aead_create_cd_auth(session,
+ if (qat_sym_cd_auth_set(session,
key_data,
key_length,
0,
auth_xform->op))
return -EINVAL;
- if (qat_sym_session_aead_create_cd_cipher(session,
+ if (qat_sym_cd_cipher_set(session,
auth_xform->key.data,
auth_xform->key.length))
return -EINVAL;
}
- /* Restore to authentication only only */
- session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
} else {
- if (qat_sym_session_aead_create_cd_auth(session,
+ if (qat_sym_cd_auth_set(session,
key_data,
key_length,
0,
return -EINVAL;
}
- session->digest_length = auth_xform->digest_length;
return 0;
}
{
struct rte_crypto_aead_xform *aead_xform = &xform->aead;
enum rte_crypto_auth_operation crypto_operation;
+ struct qat_cryptodev_private *internals =
+ dev->data->dev_private;
+ enum qat_device_gen qat_dev_gen =
+ internals->qat_dev->qat_dev_gen;
/*
* Store AEAD IV parameters as cipher IV,
session->cipher_iv.length = xform->aead.iv.length;
session->auth_mode = ICP_QAT_HW_AUTH_MODE1;
+ session->is_auth = 1;
+ session->digest_length = aead_xform->digest_length;
+ session->is_single_pass = 0;
switch (aead_xform->algo) {
case RTE_CRYPTO_AEAD_AES_GCM:
if (qat_sym_validate_aes_key(aead_xform->key.length,
}
session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
- if (session->cipher_iv.length == 0)
- session->cipher_iv.length = AES_GCM_J0_LEN;
+ if (qat_dev_gen == QAT_GEN4)
+ session->is_ucs = 1;
+ if (session->cipher_iv.length == 0) {
+ session->cipher_iv.length = AES_GCM_J0_LEN;
+ break;
+ }
+ session->is_iv12B = 1;
+ if (qat_dev_gen < QAT_GEN3)
+ break;
+ qat_sym_session_handle_single_pass(session,
+ aead_xform);
break;
case RTE_CRYPTO_AEAD_AES_CCM:
if (qat_sym_validate_aes_key(aead_xform->key.length,
}
session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
+ if (qat_dev_gen == QAT_GEN4)
+ session->is_ucs = 1;
+ break;
+ case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
+ if (aead_xform->key.length != ICP_QAT_HW_CHACHAPOLY_KEY_SZ)
+ return -EINVAL;
+ if (qat_dev_gen == QAT_GEN4)
+ session->is_ucs = 1;
+ session->qat_cipher_alg =
+ ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305;
+ qat_sym_session_handle_single_pass(session,
+ aead_xform);
break;
default:
QAT_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
return -EINVAL;
}
- session->is_single_pass = 0;
- if (aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) {
- /* Use faster Single-Pass GCM if possible */
- int res = qat_sym_session_handle_single_pass(
- dev->data->dev_private, session, aead_xform);
- if (res < 0)
- return res;
- if (session->is_single_pass)
- return 0;
- }
-
- if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
+ if (session->is_single_pass) {
+ if (qat_sym_cd_cipher_set(session,
+ aead_xform->key.data, aead_xform->key.length))
+ return -EINVAL;
+ } else if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
(aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
- if (qat_sym_session_aead_create_cd_cipher(session,
+ if (qat_sym_cd_cipher_set(session,
aead_xform->key.data,
aead_xform->key.length))
return -EINVAL;
- if (qat_sym_session_aead_create_cd_auth(session,
+ if (qat_sym_cd_auth_set(session,
aead_xform->key.data,
aead_xform->key.length,
aead_xform->aad_length,
crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
- if (qat_sym_session_aead_create_cd_auth(session,
+ if (qat_sym_cd_auth_set(session,
aead_xform->key.data,
aead_xform->key.length,
aead_xform->aad_length,
crypto_operation))
return -EINVAL;
- if (qat_sym_session_aead_create_cd_cipher(session,
+ if (qat_sym_cd_cipher_set(session,
aead_xform->key.data,
aead_xform->key.length))
return -EINVAL;
}
- session->digest_length = aead_xform->digest_length;
return 0;
}
return 0;
}
-static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
- uint8_t *data_in,
- uint8_t *data_out)
+static int
+partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg,
+ uint8_t *data_in, uint8_t *data_out)
{
int digest_size;
uint8_t digest[qat_hash_get_digest_size(
uint64_t *hash_state_out_be64;
int i;
+ /* Initialize to avoid gcc warning */
+ memset(digest, 0, sizeof(digest));
+
digest_size = qat_hash_get_digest_size(hash_alg);
if (digest_size <= 0)
return -EFAULT;
}
static void
-qat_sym_session_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header,
- enum qat_sym_proto_flag proto_flags)
+qat_sym_session_init_common_hdr(struct qat_sym_session *session)
{
+ struct icp_qat_fw_la_bulk_req *req_tmpl = &session->fw_req;
+ struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
+ enum qat_sym_proto_flag proto_flags = session->qat_proto_flag;
+ uint32_t slice_flags = session->slice_types;
+
header->hdr_flags =
ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
+ header->service_cmd_id = session->qat_cmd;
header->comn_req_flags =
ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
QAT_COMN_PTR_TYPE_FLAT);
break;
}
- ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_UPDATE_STATE);
- ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
-}
+ /* More than one of the following flags can be set at once */
+ if (QAT_SESSION_IS_SLICE_SET(slice_flags, QAT_CRYPTO_SLICE_SPC)) {
+ ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
+ header->serv_specif_flags,
+ ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
+ }
+ if (QAT_SESSION_IS_SLICE_SET(slice_flags, QAT_CRYPTO_SLICE_UCS)) {
+ ICP_QAT_FW_LA_SLICE_TYPE_SET(
+ header->serv_specif_flags,
+ ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
+ }
-/*
- * Snow3G and ZUC should never use this function
- * and set its protocol flag in both cipher and auth part of content
- * descriptor building function
- */
-static enum qat_sym_proto_flag
-qat_get_crypto_proto_flag(uint16_t flags)
-{
- int proto = ICP_QAT_FW_LA_PROTO_GET(flags);
- enum qat_sym_proto_flag qat_proto_flag =
- QAT_CRYPTO_PROTO_FLAG_NONE;
+ if (session->is_auth) {
+ if (session->auth_op == ICP_QAT_HW_AUTH_VERIFY) {
+ ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_RET_AUTH_RES);
+ ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_CMP_AUTH_RES);
+ } else if (session->auth_op == ICP_QAT_HW_AUTH_GENERATE) {
+ ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_RET_AUTH_RES);
+ ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
+ }
+ } else {
+ ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_RET_AUTH_RES);
+ ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
+ }
- switch (proto) {
- case ICP_QAT_FW_LA_GCM_PROTO:
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
- break;
- case ICP_QAT_FW_LA_CCM_PROTO:
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM;
- break;
+ if (session->is_iv12B) {
+ ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
+ header->serv_specif_flags,
+ ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
}
- return qat_proto_flag;
+ ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_UPDATE_STATE);
+ ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
+ ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
}
-int qat_sym_session_aead_create_cd_cipher(struct qat_sym_session *cdesc,
+int qat_sym_cd_cipher_set(struct qat_sym_session *cdesc,
const uint8_t *cipherkey,
uint32_t cipherkeylen)
{
struct icp_qat_hw_cipher_algo_blk *cipher;
+ struct icp_qat_hw_cipher_algo_blk20 *cipher20;
struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req;
struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
enum icp_qat_hw_cipher_convert key_convert;
- enum qat_sym_proto_flag qat_proto_flag =
- QAT_CRYPTO_PROTO_FLAG_NONE;
+ struct icp_qat_fw_la_cipher_20_req_params *req_ucs =
+ (struct icp_qat_fw_la_cipher_20_req_params *)
+ &cdesc->fw_req.serv_specif_rqpars;
+ struct icp_qat_fw_la_cipher_req_params *req_cipher =
+ (struct icp_qat_fw_la_cipher_req_params *)
+ &cdesc->fw_req.serv_specif_rqpars;
uint32_t total_key_size;
uint16_t cipher_offset, cd_size;
uint32_t wordIndex = 0;
if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
/*
* CTR Streaming ciphers are a special case. Decrypt = encrypt
- * Overriding default values previously set
+ * Overriding default values previously set.
+ * Chacha20-Poly1305 is special case, CTR but single-pass
+ * so both direction need to be used.
*/
cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
+ if (cdesc->qat_cipher_alg ==
+ ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305 &&
+ cdesc->auth_op == ICP_QAT_HW_AUTH_VERIFY) {
+ cdesc->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
+ }
key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
} else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2
|| cdesc->qat_cipher_alg ==
key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT;
else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT)
key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
+ else if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE)
+ key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT;
else
key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT;
ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ;
cipher_cd_ctrl->cipher_state_sz =
ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3;
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
} else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) {
total_key_size = ICP_QAT_HW_KASUMI_F8_KEY_SZ;
} else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) {
total_key_size = ICP_QAT_HW_3DES_KEY_SZ;
cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_3DES_BLK_SZ >> 3;
- qat_proto_flag =
- qat_get_crypto_proto_flag(header->serv_specif_flags);
} else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_DES) {
total_key_size = ICP_QAT_HW_DES_KEY_SZ;
cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_DES_BLK_SZ >> 3;
- qat_proto_flag =
- qat_get_crypto_proto_flag(header->serv_specif_flags);
} else if (cdesc->qat_cipher_alg ==
ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
total_key_size = ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ +
ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
cipher_cd_ctrl->cipher_state_sz =
ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3;
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
- cdesc->min_qat_dev_gen = QAT_GEN2;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
} else {
total_key_size = cipherkeylen;
cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3;
- qat_proto_flag =
- qat_get_crypto_proto_flag(header->serv_specif_flags);
}
- cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3;
cipher_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd);
cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3;
- header->service_cmd_id = cdesc->qat_cmd;
- qat_sym_session_init_common_hdr(header, qat_proto_flag);
-
cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr;
+ cipher20 = (struct icp_qat_hw_cipher_algo_blk20 *)cdesc->cd_cur_ptr;
cipher->cipher_config.val =
ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode,
cdesc->qat_cipher_alg, key_convert,
cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) +
cipherkeylen + cipherkeylen;
+ } else if (cdesc->is_ucs) {
+ const uint8_t *final_key = cipherkey;
+
+ cdesc->slice_types |= QAT_CRYPTO_SLICE_UCS;
+ total_key_size = RTE_ALIGN_CEIL(cipherkeylen,
+ ICP_QAT_HW_AES_128_KEY_SZ);
+ cipher20->cipher_config.reserved[0] = 0;
+ cipher20->cipher_config.reserved[1] = 0;
+ cipher20->cipher_config.reserved[2] = 0;
+
+ rte_memcpy(cipher20->key, final_key, cipherkeylen);
+ cdesc->cd_cur_ptr +=
+ sizeof(struct icp_qat_hw_ucs_cipher_config) +
+ cipherkeylen;
} else {
memcpy(cipher->key, cipherkey, cipherkeylen);
cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) +
cipherkeylen;
}
+ if (cdesc->is_single_pass) {
+ QAT_FIELD_SET(cipher->cipher_config.val,
+ cdesc->digest_length,
+ QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
+ QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
+ /* UCS and SPC 1.8/2.0 share configuration of 2nd config word */
+ cdesc->cd.cipher.cipher_config.reserved =
+ ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(
+ cdesc->aad_len);
+ cdesc->slice_types |= QAT_CRYPTO_SLICE_SPC;
+ }
+
if (total_key_size > cipherkeylen) {
uint32_t padding_size = total_key_size-cipherkeylen;
if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES)
cdesc->cd_cur_ptr += padding_size;
}
+ if (cdesc->is_ucs) {
+ /*
+ * These values match in terms of position auth
+ * slice request fields
+ */
+ req_ucs->spc_auth_res_sz = cdesc->digest_length;
+ if (!cdesc->is_gmac) {
+ req_ucs->spc_aad_sz = cdesc->aad_len;
+ req_ucs->spc_aad_offset = 0;
+ }
+ } else if (cdesc->is_single_pass) {
+ req_cipher->spc_aad_sz = cdesc->aad_len;
+ req_cipher->spc_auth_res_sz = cdesc->digest_length;
+ }
cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd;
cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3;
+ cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3;
return 0;
}
-int qat_sym_session_aead_create_cd_auth(struct qat_sym_session *cdesc,
+int qat_sym_cd_auth_set(struct qat_sym_session *cdesc,
const uint8_t *authkey,
uint32_t authkeylen,
uint32_t aad_length,
struct icp_qat_hw_cipher_algo_blk *cipherconfig;
struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req;
struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
- struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
void *ptr = &req_tmpl->cd_ctrl;
struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
uint32_t *aad_len = NULL;
uint32_t wordIndex = 0;
uint32_t *pTempKey;
- enum qat_sym_proto_flag qat_proto_flag =
- QAT_CRYPTO_PROTO_FLAG_NONE;
if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl,
return -EFAULT;
}
- if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_RET_AUTH_RES);
- ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_CMP_AUTH_RES);
+ if (operation == RTE_CRYPTO_AUTH_OP_VERIFY)
cdesc->auth_op = ICP_QAT_HW_AUTH_VERIFY;
- } else {
- ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_RET_AUTH_RES);
- ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
- ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
+ else
cdesc->auth_op = ICP_QAT_HW_AUTH_GENERATE;
- }
/*
* Setup the inner hash config
|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC
|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC
|| cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL
+ || cdesc->is_cnt_zero
)
hash->auth_counter.counter = 0;
else {
break;
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM;
state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ;
if (qat_sym_do_precomputes(cdesc->qat_hash_alg, authkey,
authkeylen, cdesc->cd_cur_ptr + state1_size,
cdesc->aad_len = aad_length;
break;
case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G;
state1_size = qat_hash_get_state1_size(
ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2);
state2_size = ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ;
hash->auth_config.config =
ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE0,
cdesc->qat_hash_alg, digestsize);
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC;
state1_size = qat_hash_get_state1_size(
ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3);
state2_size = ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ;
memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen);
cd_extra_size += ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ;
auth_param->hash_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3;
- cdesc->min_qat_dev_gen = QAT_GEN2;
break;
case ICP_QAT_HW_AUTH_ALGO_MD5:
state2_size = ICP_QAT_HW_NULL_STATE2_SZ;
break;
case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
- qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM;
+ cdesc->qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM;
state1_size = qat_hash_get_state1_size(
ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC);
state2_size = ICP_QAT_HW_AES_CBC_MAC_KEY_SZ +
return -EFAULT;
}
- /* Request template setup */
- qat_sym_session_init_common_hdr(header, qat_proto_flag);
- header->service_cmd_id = cdesc->qat_cmd;
-
/* Auth CD config setup */
hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3;
hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
}
return 0;
}
+
+#ifdef RTE_LIB_SECURITY
+static int
+qat_sec_session_check_docsis(struct rte_security_session_conf *conf)
+{
+ struct rte_crypto_sym_xform *crypto_sym = conf->crypto_xform;
+ struct rte_security_docsis_xform *docsis = &conf->docsis;
+
+ /* CRC generate -> Cipher encrypt */
+ if (docsis->direction == RTE_SECURITY_DOCSIS_DOWNLINK) {
+
+ if (crypto_sym != NULL &&
+ crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+ crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT &&
+ crypto_sym->cipher.algo ==
+ RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+ (crypto_sym->cipher.key.length ==
+ ICP_QAT_HW_AES_128_KEY_SZ ||
+ crypto_sym->cipher.key.length ==
+ ICP_QAT_HW_AES_256_KEY_SZ) &&
+ crypto_sym->cipher.iv.length == ICP_QAT_HW_AES_BLK_SZ &&
+ crypto_sym->next == NULL) {
+ return 0;
+ }
+ /* Cipher decrypt -> CRC verify */
+ } else if (docsis->direction == RTE_SECURITY_DOCSIS_UPLINK) {
+
+ if (crypto_sym != NULL &&
+ crypto_sym->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+ crypto_sym->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
+ crypto_sym->cipher.algo ==
+ RTE_CRYPTO_CIPHER_AES_DOCSISBPI &&
+ (crypto_sym->cipher.key.length ==
+ ICP_QAT_HW_AES_128_KEY_SZ ||
+ crypto_sym->cipher.key.length ==
+ ICP_QAT_HW_AES_256_KEY_SZ) &&
+ crypto_sym->cipher.iv.length == ICP_QAT_HW_AES_BLK_SZ &&
+ crypto_sym->next == NULL) {
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+static int
+qat_sec_session_set_docsis_parameters(struct rte_cryptodev *dev,
+ struct rte_security_session_conf *conf, void *session_private)
+{
+ int ret;
+ int qat_cmd_id;
+ struct rte_crypto_sym_xform *xform = NULL;
+ struct qat_sym_session *session = session_private;
+
+ /* Clear the session */
+ memset(session, 0, qat_sym_session_get_private_size(dev));
+
+ ret = qat_sec_session_check_docsis(conf);
+ if (ret) {
+ QAT_LOG(ERR, "Unsupported DOCSIS security configuration");
+ return ret;
+ }
+
+ xform = conf->crypto_xform;
+
+ /* Verify the session physical address is known */
+ rte_iova_t session_paddr = rte_mempool_virt2iova(session);
+ if (session_paddr == 0 || session_paddr == RTE_BAD_IOVA) {
+ QAT_LOG(ERR,
+ "Session physical address unknown. Bad memory pool.");
+ return -EINVAL;
+ }
+
+ /* Set context descriptor physical address */
+ session->cd_paddr = session_paddr +
+ offsetof(struct qat_sym_session, cd);
+
+ /* Get requested QAT command id - should be cipher */
+ qat_cmd_id = qat_get_cmd_id(xform);
+ if (qat_cmd_id != ICP_QAT_FW_LA_CMD_CIPHER) {
+ QAT_LOG(ERR, "Unsupported xform chain requested");
+ return -ENOTSUP;
+ }
+ session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
+
+ ret = qat_sym_session_configure_cipher(dev, xform, session);
+ if (ret < 0)
+ return ret;
+ qat_sym_session_finalize(session);
+
+ return 0;
+}
+
+int
+qat_security_session_create(void *dev,
+ struct rte_security_session_conf *conf,
+ struct rte_security_session *sess,
+ struct rte_mempool *mempool)
+{
+ void *sess_private_data;
+ struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev;
+ struct qat_cryptodev_private *internals = cdev->data->dev_private;
+ enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
+ struct qat_sym_session *sym_session = NULL;
+ int ret;
+
+ if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL ||
+ conf->protocol != RTE_SECURITY_PROTOCOL_DOCSIS) {
+ QAT_LOG(ERR, "Invalid security protocol");
+ return -EINVAL;
+ }
+
+ if (rte_mempool_get(mempool, &sess_private_data)) {
+ QAT_LOG(ERR, "Couldn't get object from session mempool");
+ return -ENOMEM;
+ }
+
+ ret = qat_sec_session_set_docsis_parameters(cdev, conf,
+ sess_private_data);
+ if (ret != 0) {
+ QAT_LOG(ERR, "Failed to configure session parameters");
+ /* Return session to mempool */
+ rte_mempool_put(mempool, sess_private_data);
+ return ret;
+ }
+
+ set_sec_session_private_data(sess, sess_private_data);
+ sym_session = (struct qat_sym_session *)sess_private_data;
+ sym_session->dev_id = internals->dev_id;
+
+ return qat_sym_gen_dev_ops[qat_dev_gen].set_session((void *)cdev,
+ sess_private_data);
+}
+
+int
+qat_security_session_destroy(void *dev __rte_unused,
+ struct rte_security_session *sess)
+{
+ void *sess_priv = get_sec_session_private_data(sess);
+ struct qat_sym_session *s = (struct qat_sym_session *)sess_priv;
+
+ if (sess_priv) {
+ if (s->bpi_ctx)
+ bpi_cipher_ctx_free(s->bpi_ctx);
+ memset(s, 0, qat_sym_session_get_private_size(dev));
+ struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
+
+ set_sec_session_private_data(sess, NULL);
+ rte_mempool_put(sess_mp, sess_priv);
+ }
+ return 0;
+}
+#endif