git.droids-corp.org
/
dpdk.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
common/mlx5: rename GENEVE TLV option object
[dpdk.git]
/
drivers
/
common
/
cpt
/
cpt_ucode.h
diff --git
a/drivers/common/cpt/cpt_ucode.h
b/drivers/common/cpt/cpt_ucode.h
index
fd56f4c
..
ee6d49a
100644
(file)
--- a/
drivers/common/cpt/cpt_ucode.h
+++ b/
drivers/common/cpt/cpt_ucode.h
@@
-77,11
+77,11
@@
cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
}
static __rte_always_inline void
}
static __rte_always_inline void
-cpt_fc_salt_update(
void *
ctx,
+cpt_fc_salt_update(
struct cpt_ctx *cpt_
ctx,
uint8_t *salt)
{
uint8_t *salt)
{
-
struct cpt_ctx *cpt_ctx =
ctx;
- memcpy(
&cpt_ctx->fctx.
enc.encr_iv, salt, 4);
+
mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.f
ctx;
+ memcpy(
fctx->
enc.encr_iv, salt, 4);
}
static __rte_always_inline int
}
static __rte_always_inline int
@@
-190,10
+190,12
@@
static __rte_always_inline void
cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
+ mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
uint32_t keyx[4];
uint32_t keyx[4];
+
cpt_ctx->snow3g = 1;
gen_key_snow3g(key, keyx);
cpt_ctx->snow3g = 1;
gen_key_snow3g(key, keyx);
- memcpy(
cpt_ctx->zs_ctx.
ci_key, keyx, key_len);
+ memcpy(
zs_ctx->
ci_key, keyx, key_len);
cpt_ctx->zsk_flags = 0;
}
cpt_ctx->zsk_flags = 0;
}
@@
-201,9
+203,11
@@
static __rte_always_inline void
cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
+ mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
+
cpt_ctx->snow3g = 0;
cpt_ctx->snow3g = 0;
- memcpy(
cpt_ctx->zs_ctx.
ci_key, key, key_len);
- memcpy(
cpt_ctx->zs_ctx.
zuc_const, zuc_d, 32);
+ memcpy(
zs_ctx->
ci_key, key, key_len);
+ memcpy(
zs_ctx->
zuc_const, zuc_d, 32);
cpt_ctx->zsk_flags = 0;
}
cpt_ctx->zsk_flags = 0;
}
@@
-211,8
+215,10
@@
static __rte_always_inline void
cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
+ mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
+
cpt_ctx->k_ecb = 1;
cpt_ctx->k_ecb = 1;
- memcpy(
cpt_ctx->k_ctx.
ci_key, key, key_len);
+ memcpy(
k_ctx->
ci_key, key, key_len);
cpt_ctx->zsk_flags = 0;
}
cpt_ctx->zsk_flags = 0;
}
@@
-220,16
+226,17
@@
static __rte_always_inline void
cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key,
uint16_t key_len)
{
- memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
+ mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
+
+ memcpy(k_ctx->ci_key, key, key_len);
cpt_ctx->zsk_flags = 0;
}
static __rte_always_inline int
cpt_ctx->zsk_flags = 0;
}
static __rte_always_inline int
-cpt_fc_ciph_set_key(
void *ctx, cipher_type_t type, const uint8_t *key
,
- uint16_t key_len, uint8_t *salt)
+cpt_fc_ciph_set_key(
struct cpt_ctx *cpt_ctx, cipher_type_t type
,
+
const uint8_t *key,
uint16_t key_len, uint8_t *salt)
{
{
- struct cpt_ctx *cpt_ctx = ctx;
- mc_fc_context_t *fctx = &cpt_ctx->fctx;
+ mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
int ret;
ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len);
int ret;
ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len);
@@
-480,10
+487,8
@@
cpt_digest_gen_prep(uint32_t flags,
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
vq_cmd_word0_t vq_cmd_w0;
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
void *c_vaddr, *m_vaddr;
uint64_t c_dma, m_dma;
void *c_vaddr, *m_vaddr;
uint64_t c_dma, m_dma;
- opcode_info_t opcode;
ctx = params->ctx_buf.vaddr;
meta_p = ¶ms->meta_buf;
ctx = params->ctx_buf.vaddr;
meta_p = ¶ms->meta_buf;
@@
-518,31
+523,27
@@
cpt_digest_gen_prep(uint32_t flags,
data_len = AUTH_DLEN(d_lens);
/*GP op header */
data_len = AUTH_DLEN(d_lens);
/*GP op header */
- vq_cmd_w0.
u64
= 0;
+ vq_cmd_w0.
s.opcode.minor
= 0;
vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
if (ctx->hmac) {
vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
if (ctx->hmac) {
-
opcode.s
.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
+
vq_cmd_w0.s.opcode
.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
vq_cmd_w0.s.param1 = key_len;
vq_cmd_w0.s.param1 = key_len;
- vq_cmd_w0.s.dlen = data_len + R
OUNDUP8(key_len
);
+ vq_cmd_w0.s.dlen = data_len + R
TE_ALIGN_CEIL(key_len, 8
);
} else {
} else {
-
opcode.s
.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
+
vq_cmd_w0.s.opcode
.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
vq_cmd_w0.s.param1 = 0;
vq_cmd_w0.s.dlen = data_len;
}
vq_cmd_w0.s.param1 = 0;
vq_cmd_w0.s.dlen = data_len;
}
- opcode.s.minor = 0;
-
/* Null auth only case enters the if */
if (unlikely(!hash_type && !ctx->enc_cipher)) {
/* Null auth only case enters the if */
if (unlikely(!hash_type && !ctx->enc_cipher)) {
-
opcode.s
.major = CPT_MAJOR_OP_MISC;
+
vq_cmd_w0.s.opcode
.major = CPT_MAJOR_OP_MISC;
/* Minor op is passthrough */
/* Minor op is passthrough */
-
opcode.s
.minor = 0x03;
+
vq_cmd_w0.s.opcode
.minor = 0x03;
/* Send out completion code only */
vq_cmd_w0.s.param2 = 0x1;
}
/* Send out completion code only */
vq_cmd_w0.s.param2 = 0x1;
}
- vq_cmd_w0.s.opcode = opcode.flags;
-
/* DPTR has SG list */
in_buffer = m_vaddr;
dptr_dma = m_dma;
/* DPTR has SG list */
in_buffer = m_vaddr;
dptr_dma = m_dma;
@@
-563,7
+564,8
@@
cpt_digest_gen_prep(uint32_t flags,
uint64_t k_dma = params->ctx_buf.dma_addr +
offsetof(struct cpt_ctx, auth_key);
/* Key */
uint64_t k_dma = params->ctx_buf.dma_addr +
offsetof(struct cpt_ctx, auth_key);
/* Key */
- i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len));
+ i = fill_sg_comp(gather_comp, i, k_dma,
+ RTE_ALIGN_CEIL(key_len, 8));
}
/* input data */
}
/* input data */
@@
-633,9
+635,6
@@
cpt_digest_gen_prep(uint32_t flags,
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-643,7
+642,6
@@
cpt_digest_gen_prep(uint32_t flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-671,12
+669,10
@@
cpt_enc_hmac_prep(uint32_t flags,
uint32_t encr_data_len, auth_data_len, aad_len = 0;
uint32_t passthrough_len = 0;
void *m_vaddr, *offset_vaddr;
uint32_t encr_data_len, auth_data_len, aad_len = 0;
uint32_t passthrough_len = 0;
void *m_vaddr, *offset_vaddr;
- uint64_t m_dma, offset_dma
, ctx_dma
;
+ uint64_t m_dma, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
void *c_vaddr;
uint64_t c_dma;
void *c_vaddr;
uint64_t c_dma;
- opcode_info_t opcode;
meta_p = &fc_params->meta_buf;
m_vaddr = meta_p->vaddr;
meta_p = &fc_params->meta_buf;
m_vaddr = meta_p->vaddr;
@@
-755,8
+751,10
@@
cpt_enc_hmac_prep(uint32_t flags,
}
/* Encryption */
}
/* Encryption */
- opcode.s.major = CPT_MAJOR_OP_FC;
- opcode.s.minor = 0;
+ vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
+ vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT;
+ vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc <<
+ CPT_HMAC_FIRST_BIT_POS);
if (hash_type == GMAC_TYPE) {
encr_offset = 0;
if (hash_type == GMAC_TYPE) {
encr_offset = 0;
@@
-767,10
+765,12
@@
cpt_enc_hmac_prep(uint32_t flags,
enc_dlen = encr_data_len + encr_offset;
if (unlikely(encr_data_len & 0xf)) {
if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
enc_dlen = encr_data_len + encr_offset;
if (unlikely(encr_data_len & 0xf)) {
if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
- enc_dlen = ROUNDUP8(encr_data_len) + encr_offset;
+ enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
+ encr_offset;
else if (likely((cipher_type == AES_CBC) ||
(cipher_type == AES_ECB)))
else if (likely((cipher_type == AES_CBC) ||
(cipher_type == AES_ECB)))
- enc_dlen = ROUNDUP16(encr_data_len) + encr_offset;
+ enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
+ encr_offset;
}
if (unlikely(auth_dlen > enc_dlen)) {
}
if (unlikely(auth_dlen > enc_dlen)) {
@@
-781,8
+781,10
@@
cpt_enc_hmac_prep(uint32_t flags,
outputlen = enc_dlen + mac_len;
}
outputlen = enc_dlen + mac_len;
}
+ if (cpt_ctx->auth_enc != 0)
+ outputlen = enc_dlen;
+
/* GP op header */
/* GP op header */
- vq_cmd_w0.u64 = 0;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
/*
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
/*
@@
-813,8
+815,6
@@
cpt_enc_hmac_prep(uint32_t flags,
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
- vq_cmd_w0.s.opcode = opcode.flags;
-
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
@@
-843,9
+843,7
@@
cpt_enc_hmac_prep(uint32_t flags,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
- opcode.s.major |= CPT_DMA_MODE;
-
- vq_cmd_w0.s.opcode = opcode.flags;
+ vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
@@
-1003,13
+1001,6
@@
cpt_enc_hmac_prep(uint32_t flags,
req->ist.ei2 = rptr_dma;
}
req->ist.ei2 = rptr_dma;
}
- ctx_dma = fc_params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, fctx);
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = ctx_dma;
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-1017,7
+1008,6
@@
cpt_enc_hmac_prep(uint32_t flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-1044,10
+1034,8
@@
cpt_dec_hmac_prep(uint32_t flags,
uint32_t encr_data_len, auth_data_len, aad_len = 0;
uint32_t passthrough_len = 0;
void *m_vaddr, *offset_vaddr;
uint32_t encr_data_len, auth_data_len, aad_len = 0;
uint32_t passthrough_len = 0;
void *m_vaddr, *offset_vaddr;
- uint64_t m_dma, offset_dma, ctx_dma;
- opcode_info_t opcode;
+ uint64_t m_dma, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
void *c_vaddr;
uint64_t c_dma;
void *c_vaddr;
uint64_t c_dma;
@@
-1128,8
+1116,10
@@
cpt_dec_hmac_prep(uint32_t flags,
m_dma += size;
/* Decryption */
m_dma += size;
/* Decryption */
- opcode.s.major = CPT_MAJOR_OP_FC;
- opcode.s.minor = 1;
+ vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
+ vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT;
+ vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth <<
+ CPT_HMAC_FIRST_BIT_POS);
if (hash_type == GMAC_TYPE) {
encr_offset = 0;
if (hash_type == GMAC_TYPE) {
encr_offset = 0;
@@
-1147,7
+1137,9
@@
cpt_dec_hmac_prep(uint32_t flags,
outputlen = enc_dlen;
}
outputlen = enc_dlen;
}
- vq_cmd_w0.u64 = 0;
+ if (cpt_ctx->dec_auth != 0)
+ outputlen = inputlen = enc_dlen;
+
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
@@
-1184,8
+1176,6
@@
cpt_dec_hmac_prep(uint32_t flags,
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
- vq_cmd_w0.s.opcode = opcode.flags;
-
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
OFF_CTRL_LEN);
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
OFF_CTRL_LEN);
@@
-1215,9
+1205,7
@@
cpt_dec_hmac_prep(uint32_t flags,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
- opcode.s.major |= CPT_DMA_MODE;
-
- vq_cmd_w0.s.opcode = opcode.flags;
+ vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
if (likely(iv_len)) {
uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
@@
-1388,13
+1376,6
@@
cpt_dec_hmac_prep(uint32_t flags,
req->ist.ei2 = rptr_dma;
}
req->ist.ei2 = rptr_dma;
}
- ctx_dma = fc_params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, fctx);
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = ctx_dma;
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-1402,7
+1383,6
@@
cpt_dec_hmac_prep(uint32_t flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-1433,8
+1413,6
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
uint64_t *offset_vaddr, offset_dma;
uint32_t *iv_s, iv[4];
vq_cmd_word0_t vq_cmd_w0;
uint64_t *offset_vaddr, offset_dma;
uint32_t *iv_s, iv[4];
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
- opcode_info_t opcode;
buf_p = ¶ms->meta_buf;
m_vaddr = buf_p->vaddr;
buf_p = ¶ms->meta_buf;
m_vaddr = buf_p->vaddr;
@@
-1468,11
+1446,11
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
-
opcode.s
.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+
vq_cmd_w0.s.opcode
.major = CPT_MAJOR_OP_ZUC_SNOW3G;
/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
opcode.s
.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
+
vq_cmd_w0.s.opcode
.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
(0 << 3) | (flags & 0x7));
if (flags == 0x1) {
(0 << 3) | (flags & 0x7));
if (flags == 0x1) {
@@
-1535,7
+1513,6
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
/*
* GP op header, lengths are expected in bits.
*/
/*
* GP op header, lengths are expected in bits.
*/
- vq_cmd_w0.u64 = 0;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
@@
-1568,8
+1545,6
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
- vq_cmd_w0.s.opcode = opcode.flags;
-
if (likely(iv_len)) {
uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
if (likely(iv_len)) {
uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
@@
-1592,9
+1567,7
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
m_dma += OFF_CTRL_LEN + iv_len;
m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
m_dma += OFF_CTRL_LEN + iv_len;
- opcode.s.major |= CPT_DMA_MODE;
-
- vq_cmd_w0.s.opcode = opcode.flags;
+ vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
/* DPTR has SG list */
in_buffer = m_vaddr;
/* DPTR has SG list */
in_buffer = m_vaddr;
@@
-1710,12
+1683,6
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
req->ist.ei2 = rptr_dma;
}
req->ist.ei2 = rptr_dma;
}
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, zs_ctx);
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-1723,7
+1690,6
@@
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-1753,8
+1719,6
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
uint64_t *offset_vaddr, offset_dma;
uint32_t *iv_s, iv[4], j;
vq_cmd_word0_t vq_cmd_w0;
uint64_t *offset_vaddr, offset_dma;
uint32_t *iv_s, iv[4], j;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
- opcode_info_t opcode;
buf_p = ¶ms->meta_buf;
m_vaddr = buf_p->vaddr;
buf_p = ¶ms->meta_buf;
m_vaddr = buf_p->vaddr;
@@
-1793,11
+1757,12
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
- opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G;
/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
/* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
-
opcode.s
.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
+
vq_cmd_w0.s.opcode
.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
(0 << 3) | (flags & 0x7));
/* consider iv len */
(0 << 3) | (flags & 0x7));
/* consider iv len */
@@
-1826,7
+1791,6
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
/*
* GP op header, lengths are expected in bits.
*/
/*
* GP op header, lengths are expected in bits.
*/
- vq_cmd_w0.u64 = 0;
vq_cmd_w0.s.param1 = encr_data_len;
/*
vq_cmd_w0.s.param1 = encr_data_len;
/*
@@
-1858,8
+1822,6
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
- vq_cmd_w0.s.opcode = opcode.flags;
-
if (likely(iv_len)) {
uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
if (likely(iv_len)) {
uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
+ OFF_CTRL_LEN);
@@
-1883,9
+1845,7
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
m_dma += OFF_CTRL_LEN + iv_len;
m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
m_dma += OFF_CTRL_LEN + iv_len;
- opcode.s.major |= CPT_DMA_MODE;
-
- vq_cmd_w0.s.opcode = opcode.flags;
+ vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
/* DPTR has SG list */
in_buffer = m_vaddr;
/* DPTR has SG list */
in_buffer = m_vaddr;
@@
-1974,12
+1934,6
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
req->ist.ei2 = rptr_dma;
}
req->ist.ei2 = rptr_dma;
}
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, zs_ctx);
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-1987,7
+1941,6
@@
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-2019,8
+1972,6
@@
cpt_kasumi_enc_prep(uint32_t req_flags,
uint64_t m_dma, c_dma;
uint64_t *offset_vaddr, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
uint64_t m_dma, c_dma;
uint64_t *offset_vaddr, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
- opcode_info_t opcode;
uint8_t *in_buffer;
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
uint8_t *in_buffer;
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
@@
-2070,19
+2021,17
@@
cpt_kasumi_enc_prep(uint32_t req_flags,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
-
opcode.s
.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+
vq_cmd_w0.s.opcode
.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
-
opcode.s
.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+
vq_cmd_w0.s.opcode
.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
(dir << 4) | (0 << 3) | (flags & 0x7));
/*
* GP op header, lengths are expected in bits.
*/
(dir << 4) | (0 << 3) | (flags & 0x7));
/*
* GP op header, lengths are expected in bits.
*/
- vq_cmd_w0.u64 = 0;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param2 = auth_data_len;
- vq_cmd_w0.s.opcode = opcode.flags;
/* consider iv len */
if (flags == 0x0) {
/* consider iv len */
if (flags == 0x0) {
@@
-2221,12
+2170,6
@@
cpt_kasumi_enc_prep(uint32_t req_flags,
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, k_ctx);
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-2234,7
+2177,6
@@
cpt_kasumi_enc_prep(uint32_t req_flags,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-2263,8
+2205,6
@@
cpt_kasumi_dec_prep(uint64_t d_offs,
uint64_t m_dma, c_dma;
uint64_t *offset_vaddr, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
uint64_t m_dma, c_dma;
uint64_t *offset_vaddr, offset_dma;
vq_cmd_word0_t vq_cmd_w0;
- vq_cmd_word3_t vq_cmd_w3;
- opcode_info_t opcode;
uint8_t *in_buffer;
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
uint8_t *in_buffer;
uint32_t g_size_bytes, s_size_bytes;
uint64_t dptr_dma, rptr_dma;
@@
-2303,18
+2243,17
@@
cpt_kasumi_dec_prep(uint64_t d_offs,
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
m_vaddr = (uint8_t *)m_vaddr + size;
m_dma += size;
- opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
/* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
-
opcode.s
.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
+
vq_cmd_w0.s.opcode
.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
(dir << 4) | (0 << 3) | (flags & 0x7));
/*
* GP op header, lengths are expected in bits.
*/
(dir << 4) | (0 << 3) | (flags & 0x7));
/*
* GP op header, lengths are expected in bits.
*/
- vq_cmd_w0.u64 = 0;
vq_cmd_w0.s.param1 = encr_data_len;
vq_cmd_w0.s.param1 = encr_data_len;
- vq_cmd_w0.s.opcode = opcode.flags;
/* consider iv len */
encr_offset += iv_len;
/* consider iv len */
encr_offset += iv_len;
@@
-2411,12
+2350,6
@@
cpt_kasumi_dec_prep(uint64_t d_offs,
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
req->ist.ei1 = dptr_dma;
req->ist.ei2 = rptr_dma;
- /* vq command w3 */
- vq_cmd_w3.u64 = 0;
- vq_cmd_w3.s.grp = 0;
- vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
- offsetof(struct cpt_ctx, k_ctx);
-
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
/* 16 byte aligned cpt res address */
req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
*req->completion_addr = COMPLETION_CODE_INIT;
@@
-2424,7
+2357,6
@@
cpt_kasumi_dec_prep(uint64_t d_offs,
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
/* Fill microcode part of instruction */
req->ist.ei0 = vq_cmd_w0.u64;
- req->ist.ei3 = vq_cmd_w3.u64;
req->op = op;
req->op = op;
@@
-2492,11
+2424,12
@@
cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
}
static __rte_always_inline int
}
static __rte_always_inline int
-cpt_fc_auth_set_key(
void *ctx, auth_type_t type, const uint8_t *key
,
- uint16_t key_len, uint16_t mac_len)
+cpt_fc_auth_set_key(
struct cpt_ctx *cpt_ctx, auth_type_t type
,
+
const uint8_t *key,
uint16_t key_len, uint16_t mac_len)
{
{
- struct cpt_ctx *cpt_ctx = ctx;
- mc_fc_context_t *fctx = &cpt_ctx->fctx;
+ mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
+ mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
+ mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
uint32_t keyx[4];
if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
uint32_t keyx[4];
@@
-2511,26
+2444,26
@@
cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key,
case SNOW3G_UIA2:
cpt_ctx->snow3g = 1;
gen_key_snow3g(key, keyx);
case SNOW3G_UIA2:
cpt_ctx->snow3g = 1;
gen_key_snow3g(key, keyx);
- memcpy(
cpt_ctx->zs_ctx.
ci_key, keyx, key_len);
+ memcpy(
zs_ctx->
ci_key, keyx, key_len);
cpt_ctx->fc_type = ZUC_SNOW3G;
cpt_ctx->zsk_flags = 0x1;
break;
case ZUC_EIA3:
cpt_ctx->snow3g = 0;
cpt_ctx->fc_type = ZUC_SNOW3G;
cpt_ctx->zsk_flags = 0x1;
break;
case ZUC_EIA3:
cpt_ctx->snow3g = 0;
- memcpy(
cpt_ctx->zs_ctx.
ci_key, key, key_len);
- memcpy(
cpt_ctx->zs_ctx.
zuc_const, zuc_d, 32);
+ memcpy(
zs_ctx->
ci_key, key, key_len);
+ memcpy(
zs_ctx->
zuc_const, zuc_d, 32);
cpt_ctx->fc_type = ZUC_SNOW3G;
cpt_ctx->zsk_flags = 0x1;
break;
case KASUMI_F9_ECB:
/* Kasumi ECB mode */
cpt_ctx->k_ecb = 1;
cpt_ctx->fc_type = ZUC_SNOW3G;
cpt_ctx->zsk_flags = 0x1;
break;
case KASUMI_F9_ECB:
/* Kasumi ECB mode */
cpt_ctx->k_ecb = 1;
- memcpy(
cpt_ctx->k_ctx.
ci_key, key, key_len);
+ memcpy(
k_ctx->
ci_key, key, key_len);
cpt_ctx->fc_type = KASUMI;
cpt_ctx->zsk_flags = 0x1;
break;
case KASUMI_F9_CBC:
cpt_ctx->fc_type = KASUMI;
cpt_ctx->zsk_flags = 0x1;
break;
case KASUMI_F9_CBC:
- memcpy(
cpt_ctx->k_ctx.
ci_key, key, key_len);
+ memcpy(
k_ctx->
ci_key, key, key_len);
cpt_ctx->fc_type = KASUMI;
cpt_ctx->zsk_flags = 0x1;
break;
cpt_ctx->fc_type = KASUMI;
cpt_ctx->zsk_flags = 0x1;
break;
@@
-2643,6
+2576,7
@@
fill_sess_cipher(struct rte_crypto_sym_xform *xform,
struct cpt_sess_misc *sess)
{
struct rte_crypto_cipher_xform *c_form;
struct cpt_sess_misc *sess)
{
struct rte_crypto_cipher_xform *c_form;
+ struct cpt_ctx *ctx = SESS_PRIV(sess);
cipher_type_t enc_type = 0; /* NULL Cipher type */
uint32_t cipher_key_len = 0;
uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
cipher_type_t enc_type = 0; /* NULL Cipher type */
uint32_t cipher_key_len = 0;
uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
@@
-2651,9
+2585,14
@@
fill_sess_cipher(struct rte_crypto_sym_xform *xform,
if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
- else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
+ else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
{
sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
- else {
+ if (xform->next != NULL &&
+ xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
+ /* Perform decryption followed by auth verify */
+ ctx->dec_auth = 1;
+ }
+ } else {
CPT_LOG_DP_ERR("Unknown cipher operation\n");
return -1;
}
CPT_LOG_DP_ERR("Unknown cipher operation\n");
return -1;
}
@@
-2744,10
+2683,18
@@
static __rte_always_inline int
fill_sess_auth(struct rte_crypto_sym_xform *xform,
struct cpt_sess_misc *sess)
{
fill_sess_auth(struct rte_crypto_sym_xform *xform,
struct cpt_sess_misc *sess)
{
+ struct cpt_ctx *ctx = SESS_PRIV(sess);
struct rte_crypto_auth_xform *a_form;
auth_type_t auth_type = 0; /* NULL Auth type */
uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
struct rte_crypto_auth_xform *a_form;
auth_type_t auth_type = 0; /* NULL Auth type */
uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
+ if (xform->next != NULL &&
+ xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
+ xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
+ /* Perform auth followed by encryption */
+ ctx->auth_enc = 1;
+ }
+
a_form = &xform->auth;
if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
a_form = &xform->auth;
if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
@@
-3070,6
+3017,7
@@
fill_fc_params(struct rte_crypto_op *cop,
{
uint32_t space = 0;
struct rte_crypto_sym_op *sym_op = cop->sym;
{
uint32_t space = 0;
struct rte_crypto_sym_op *sym_op = cop->sym;
+ struct cpt_ctx *ctx = SESS_PRIV(sess_misc);
void *mdata = NULL;
uintptr_t *op;
uint32_t mc_hash_off;
void *mdata = NULL;
uintptr_t *op;
uint32_t mc_hash_off;
@@
-3197,9
+3145,10
@@
fill_fc_params(struct rte_crypto_op *cop,
m = m_src;
/* hmac immediately following data is best case */
m = m_src;
/* hmac immediately following data is best case */
- if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
+ if (!ctx->dec_auth && !ctx->auth_enc &&
+ (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
mc_hash_off !=
mc_hash_off !=
- (uint8_t *)sym_op->auth.digest.data)) {
+ (uint8_t *)sym_op->auth.digest.data))
)
{
flags |= VALID_MAC_BUF;
fc_params.mac_buf.size =
sess_misc->mac_len;
flags |= VALID_MAC_BUF;
fc_params.mac_buf.size =
sess_misc->mac_len;
@@
-3214,7
+3163,9
@@
fill_fc_params(struct rte_crypto_op *cop,
fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
- if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE))
+ if (!ctx->dec_auth &&
+ unlikely(sess_misc->is_null ||
+ sess_misc->cpt_op == CPT_OP_DECODE))
inplace = 0;
if (likely(!m_dst && inplace)) {
inplace = 0;
if (likely(!m_dst && inplace)) {