return AESNI_MB_OP_NOT_SUPPORTED;
}
+static inline int
+is_aead_algo(JOB_HASH_ALG hash_alg, JOB_CIPHER_MODE cipher_mode)
+{
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ return (hash_alg == IMB_AUTH_CHACHA20_POLY1305 || hash_alg == AES_CCM ||
+ (hash_alg == AES_GMAC && cipher_mode == GCM));
+#else
+ return ((hash_alg == AES_GMAC && cipher_mode == GCM) ||
+ hash_alg == AES_CCM);
+#endif
+}
+
/** Set session authentication parameters */
static int
aesni_mb_set_session_auth_parameters(const MB_MGR *mb_mgr,
return -1;
}
+ /* Set IV parameters */
+ sess->auth_iv.offset = xform->auth.iv.offset;
+ sess->auth_iv.length = xform->auth.iv.length;
+
/* Set the request digest size */
sess->auth.req_digest_len = xform->auth.digest_length;
sess->cipher.direction = DECRYPT;
sess->auth.algo = AES_GMAC;
- /*
- * Multi-buffer lib supports 8, 12 and 16 bytes of digest.
- * If size requested is different, generate the full digest
- * (16 bytes) in a temporary location and then memcpy
- * the requested number of bytes.
- */
- if (sess->auth.req_digest_len != 16 &&
- sess->auth.req_digest_len != 12 &&
- sess->auth.req_digest_len != 8) {
- sess->auth.gen_digest_len = 16;
- } else {
- sess->auth.gen_digest_len = sess->auth.req_digest_len;
+ if (sess->auth.req_digest_len > get_digest_byte_length(AES_GMAC)) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
}
+ sess->auth.gen_digest_len = sess->auth.req_digest_len;
sess->iv.length = xform->auth.iv.length;
sess->iv.offset = xform->auth.iv.offset;
return 0;
}
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (xform->auth.algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
+ sess->auth.algo = IMB_AUTH_ZUC_EIA3_BITLEN;
+ uint16_t zuc_eia3_digest_len =
+ get_truncated_digest_byte_length(IMB_AUTH_ZUC_EIA3_BITLEN);
+ if (sess->auth.req_digest_len != zuc_eia3_digest_len) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
+ sess->auth.gen_digest_len = sess->auth.req_digest_len;
+
+ memcpy(sess->auth.zuc_auth_key, xform->auth.key.data, 16);
+ return 0;
+ } else if (xform->auth.algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
+ sess->auth.algo = IMB_AUTH_SNOW3G_UIA2_BITLEN;
+ uint16_t snow3g_uia2_digest_len =
+ get_truncated_digest_byte_length(IMB_AUTH_SNOW3G_UIA2_BITLEN);
+ if (sess->auth.req_digest_len != snow3g_uia2_digest_len) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
+ sess->auth.gen_digest_len = sess->auth.req_digest_len;
+
+ IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->auth.key.data,
+ &sess->auth.pKeySched_snow3g_auth);
+ return 0;
+ } else if (xform->auth.algo == RTE_CRYPTO_AUTH_KASUMI_F9) {
+ sess->auth.algo = IMB_AUTH_KASUMI_UIA1;
+ uint16_t kasumi_f9_digest_len =
+ get_truncated_digest_byte_length(IMB_AUTH_KASUMI_UIA1);
+ if (sess->auth.req_digest_len != kasumi_f9_digest_len) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
+ sess->auth.gen_digest_len = sess->auth.req_digest_len;
+
+ IMB_KASUMI_INIT_F9_KEY_SCHED(mb_mgr, xform->auth.key.data,
+ &sess->auth.pKeySched_kasumi_auth);
+ return 0;
+ }
+#endif
+
switch (xform->auth.algo) {
case RTE_CRYPTO_AUTH_MD5_HMAC:
sess->auth.algo = MD5;
uint8_t is_aes = 0;
uint8_t is_3DES = 0;
uint8_t is_docsis = 0;
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ uint8_t is_zuc = 0;
+ uint8_t is_snow3g = 0;
+ uint8_t is_kasumi = 0;
+#endif
if (xform == NULL) {
sess->cipher.mode = NULL_CIPHER;
sess->cipher.mode = ECB;
is_aes = 1;
break;
+#endif
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ case RTE_CRYPTO_CIPHER_ZUC_EEA3:
+ sess->cipher.mode = IMB_CIPHER_ZUC_EEA3;
+ is_zuc = 1;
+ break;
+ case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
+ sess->cipher.mode = IMB_CIPHER_SNOW3G_UEA2_BITLEN;
+ is_snow3g = 1;
+ break;
+ case RTE_CRYPTO_CIPHER_KASUMI_F8:
+ sess->cipher.mode = IMB_CIPHER_KASUMI_UEA1_BITLEN;
+ is_kasumi = 1;
+ break;
#endif
default:
AESNI_MB_LOG(ERR, "Unsupported cipher mode parameter");
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
-#if IMB_VERSION_NUM >= IMB_VERSION(0, 53, 3)
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
case AES_256_BYTES:
sess->cipher.key_length_in_bytes = AES_256_BYTES;
IMB_AES_KEYEXP_256(mb_mgr, xform->cipher.key.data,
}
sess->cipher.key_length_in_bytes = 24;
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ } else if (is_zuc) {
+ if (xform->cipher.key.length != 16) {
+ AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ return -EINVAL;
+ }
+ sess->cipher.key_length_in_bytes = 16;
+ memcpy(sess->cipher.zuc_cipher_key, xform->cipher.key.data,
+ 16);
+ } else if (is_snow3g) {
+ if (xform->cipher.key.length != 16) {
+ AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ return -EINVAL;
+ }
+ sess->cipher.key_length_in_bytes = 16;
+ IMB_SNOW3G_INIT_KEY_SCHED(mb_mgr, xform->cipher.key.data,
+ &sess->cipher.pKeySched_snow3g_cipher);
+ } else if (is_kasumi) {
+ if (xform->cipher.key.length != 16) {
+ AESNI_MB_LOG(ERR, "Invalid cipher key length");
+ return -EINVAL;
+ }
+ sess->cipher.key_length_in_bytes = 16;
+ IMB_KASUMI_INIT_F8_KEY_SCHED(mb_mgr, xform->cipher.key.data,
+ &sess->cipher.pKeySched_kasumi_cipher);
+#endif
} else {
if (xform->cipher.key.length != 8) {
AESNI_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
+ /* Set IV parameters */
+ sess->iv.offset = xform->aead.iv.offset;
+ sess->iv.length = xform->aead.iv.length;
+
+ /* Set digest sizes */
+ sess->auth.req_digest_len = xform->aead.digest_length;
+ sess->auth.gen_digest_len = sess->auth.req_digest_len;
+
switch (xform->aead.algo) {
case RTE_CRYPTO_AEAD_AES_CCM:
sess->cipher.mode = CCM;
sess->cipher.expanded_aes_keys.encode,
sess->cipher.expanded_aes_keys.decode);
break;
+ case AES_256_BYTES:
+ sess->cipher.key_length_in_bytes = AES_256_BYTES;
+ IMB_AES_KEYEXP_256(mb_mgr, xform->aead.key.data,
+ sess->cipher.expanded_aes_keys.encode,
+ sess->cipher.expanded_aes_keys.decode);
+ break;
default:
AESNI_MB_LOG(ERR, "Invalid cipher key length");
return -EINVAL;
}
+ /* CCM digests must be between 4 and 16 and an even number */
+ if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
+ sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
+ (sess->auth.req_digest_len & 1) == 1) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
break;
case RTE_CRYPTO_AEAD_AES_GCM:
return -EINVAL;
}
+ /* GCM digest size must be between 1 and 16 */
+ if (sess->auth.req_digest_len == 0 ||
+ sess->auth.req_digest_len > 16) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
break;
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
+ sess->cipher.mode = IMB_CIPHER_CHACHA20_POLY1305;
+ sess->auth.algo = IMB_AUTH_CHACHA20_POLY1305;
+
+ if (xform->aead.key.length != 32) {
+ AESNI_MB_LOG(ERR, "Invalid key length");
+ return -EINVAL;
+ }
+ sess->cipher.key_length_in_bytes = 32;
+ memcpy(sess->cipher.expanded_aes_keys.encode,
+ xform->aead.key.data, 32);
+ if (sess->auth.req_digest_len != 16) {
+ AESNI_MB_LOG(ERR, "Invalid digest size\n");
+ return -EINVAL;
+ }
+ break;
+#endif
default:
AESNI_MB_LOG(ERR, "Unsupported aead mode parameter");
return -ENOTSUP;
}
- /* Set IV parameters */
- sess->iv.offset = xform->aead.iv.offset;
- sess->iv.length = xform->aead.iv.length;
-
- sess->auth.req_digest_len = xform->aead.digest_length;
- /* CCM digests must be between 4 and 16 and an even number */
- if (sess->auth.req_digest_len < AES_CCM_DIGEST_MIN_LEN ||
- sess->auth.req_digest_len > AES_CCM_DIGEST_MAX_LEN ||
- (sess->auth.req_digest_len & 1) == 1) {
- AESNI_MB_LOG(ERR, "Invalid digest size\n");
- return -EINVAL;
- }
- sess->auth.gen_digest_len = sess->auth.req_digest_len;
-
return 0;
}
/* Default IV length = 0 */
sess->iv.length = 0;
+ sess->auth_iv.length = 0;
ret = aesni_mb_set_session_auth_parameters(mb_mgr, sess, auth_xform);
if (ret != 0) {
static inline void
set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
- void *iv, void *aad, void *digest, void *udata)
+ struct rte_crypto_va_iova_ptr *iv,
+ struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
{
/* Set crypto operation */
job->chain_order = session->chain_order;
/* Set authentication parameters */
job->hash_alg = session->auth.algo;
- job->iv = iv;
+ job->iv = iv->va;
switch (job->hash_alg) {
case AES_XCBC:
break;
case AES_CCM:
- job->u.CCM.aad = (uint8_t *)aad + 18;
+ job->u.CCM.aad = (uint8_t *)aad->va + 18;
job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
job->aes_enc_key_expanded =
session->cipher.expanded_aes_keys.encode;
case AES_GMAC:
if (session->cipher.mode == GCM) {
- job->u.GCM.aad = aad;
+ job->u.GCM.aad = aad->va;
job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
} else {
/* For GMAC */
job->aes_dec_key_expanded = &session->cipher.gcm_key;
break;
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->u.CHACHA20_POLY1305.aad = aad->va;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ break;
+#endif
default:
job->u.HMAC._hashed_auth_key_xor_ipad =
session->auth.pads.inner;
/* Set authentication parameters */
job->hash_alg = session->auth.algo;
+ const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
+
switch (job->hash_alg) {
case AES_XCBC:
job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
job->aes_enc_key_expanded = &session->cipher.gcm_key;
job->aes_dec_key_expanded = &session->cipher.gcm_key;
break;
-
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_ZUC_EIA3_BITLEN:
+ job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
+ job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_SNOW3G_UIA2_BITLEN:
+ job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
+ job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_KASUMI_UIA1:
+ job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
+ break;
+#endif
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ break;
+#endif
default:
job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
}
}
+ if (aead)
+ m_offset = op->sym->aead.data.offset;
+ else
+ m_offset = op->sym->cipher.data.offset;
+
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
+ job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
+ job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
+ } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
+ m_offset = 0;
+ } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
+ m_offset = 0;
+ }
+#endif
+
if (!op->sym->m_dst) {
/* in-place operation */
m_dst = m_src;
oop = 1;
}
- if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
- session->cipher.mode == GCM))
- m_offset = op->sym->aead.data.offset;
- else
- m_offset = op->sym->cipher.data.offset;
-
/* Set digest output location */
if (job->hash_alg != NULL_HASH &&
session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
job->auth_tag_output = qp->temp_digests[*digest_idx];
*digest_idx = (*digest_idx + 1) % MAX_JOBS;
} else {
- if (job->hash_alg == AES_CCM || (job->hash_alg == AES_GMAC &&
- session->cipher.mode == GCM))
+ if (aead)
job->auth_tag_output = op->sym->aead.digest.data;
else
job->auth_tag_output = op->sym->auth.digest.data;
session->iv.offset);
break;
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->msg_len_to_cipher_in_bytes =
+ op->sym->aead.data.length;
+ job->msg_len_to_hash_in_bytes =
+ op->sym->aead.data.length;
+
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset);
+ break;
+#endif
default:
+ /* For SNOW3G, length and offsets are already in bits */
job->cipher_start_src_offset_in_bytes =
op->sym->cipher.data.offset;
job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
session->iv.offset);
}
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
+ job->msg_len_to_cipher_in_bytes >>= 3;
+ else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
+ job->msg_len_to_hash_in_bytes >>= 3;
+#endif
+
/* Set user data to be crypto operation data struct */
job->user_data = op;
break;
if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
- if (job->hash_alg == AES_CCM ||
- (job->hash_alg == AES_GMAC &&
- sess->cipher.mode == GCM))
+ if (is_aead_algo(job->hash_alg, sess->cipher.mode))
verify_digest(job,
op->sym->aead.digest.data,
sess->auth.req_digest_len,
for (i = 0, k = 0; i != vec->num; i++) {
if (vec->status[i] == 0) {
- memcpy(vec->digest[i], dgst[i], len);
+ memcpy(vec->digest[i].va, dgst[i], len);
k++;
}
}
for (i = 0, k = 0; i != vec->num; i++) {
if (vec->status[i] == 0) {
- if (memcmp(vec->digest[i], dgst[i], len) != 0)
+ if (memcmp(vec->digest[i].va, dgst[i], len) != 0)
vec->status[i] = EBADMSG;
else
k++;
}
/* Submit job for processing */
- set_cpu_mb_job_params(job, s, sofs, buf, len,
- vec->iv[i], vec->aad[i], tmp_dgst[i],
- &vec->status[i]);
+ set_cpu_mb_job_params(job, s, sofs, buf, len, &vec->iv[i],
+ &vec->aad[i], tmp_dgst[i], &vec->status[i]);
job = submit_sync_job(mb_mgr);
j++;
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
+ RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_mb_crypto_drv,
cryptodev_aesni_mb_pmd_drv.driver,
cryptodev_driver_id);
-RTE_LOG_REGISTER(aesni_mb_logtype_driver, pmd.crypto.aesni_mb, NOTICE);
+RTE_LOG_REGISTER_DEFAULT(aesni_mb_logtype_driver, NOTICE);
git.droids-corp.org / dpdk.git / blobdiff