X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcommon%2Fcpt%2Fcpt_ucode.h;h=ee6d49aae77beac3c911e63746b2487240da2922;hb=d38b62cd6ccffd1e269dbf9942f4f60022c38040;hp=fd56f4c8079548cc9e1cf40bf9b33582d3112e6c;hpb=7293bae19a30b0fb9b8d50ac5f1997c6da32e2c2;p=dpdk.git diff --git a/drivers/common/cpt/cpt_ucode.h b/drivers/common/cpt/cpt_ucode.h index fd56f4c807..ee6d49aae7 100644 --- a/drivers/common/cpt/cpt_ucode.h +++ b/drivers/common/cpt/cpt_ucode.h @@ -77,11 +77,11 @@ cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) } static __rte_always_inline void -cpt_fc_salt_update(void *ctx, +cpt_fc_salt_update(struct cpt_ctx *cpt_ctx, uint8_t *salt) { - struct cpt_ctx *cpt_ctx = ctx; - memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); + mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; + memcpy(fctx->enc.encr_iv, salt, 4); } static __rte_always_inline int @@ -190,10 +190,12 @@ static __rte_always_inline void cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, uint16_t key_len) { + mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; uint32_t keyx[4]; + cpt_ctx->snow3g = 1; gen_key_snow3g(key, keyx); - memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); + memcpy(zs_ctx->ci_key, keyx, key_len); cpt_ctx->zsk_flags = 0; } @@ -201,9 +203,11 @@ static __rte_always_inline void cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, uint16_t key_len) { + mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; + cpt_ctx->snow3g = 0; - memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); - memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); + memcpy(zs_ctx->ci_key, key, key_len); + memcpy(zs_ctx->zuc_const, zuc_d, 32); cpt_ctx->zsk_flags = 0; } @@ -211,8 +215,10 @@ static __rte_always_inline void cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, uint16_t key_len) { + mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; + cpt_ctx->k_ecb = 1; - memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); + memcpy(k_ctx->ci_key, key, key_len); cpt_ctx->zsk_flags = 0; } @@ -220,16 +226,17 @@ static __rte_always_inline void cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, uint16_t key_len) { - memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); + mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; + + memcpy(k_ctx->ci_key, key, key_len); cpt_ctx->zsk_flags = 0; } static __rte_always_inline int -cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, - uint16_t key_len, uint8_t *salt) +cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type, + const uint8_t *key, uint16_t key_len, uint8_t *salt) { - struct cpt_ctx *cpt_ctx = ctx; - mc_fc_context_t *fctx = &cpt_ctx->fctx; + mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; int ret; ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); @@ -480,10 +487,8 @@ cpt_digest_gen_prep(uint32_t flags, uint32_t g_size_bytes, s_size_bytes; uint64_t dptr_dma, rptr_dma; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; void *c_vaddr, *m_vaddr; uint64_t c_dma, m_dma; - opcode_info_t opcode; ctx = params->ctx_buf.vaddr; meta_p = ¶ms->meta_buf; @@ -518,31 +523,27 @@ cpt_digest_gen_prep(uint32_t flags, data_len = AUTH_DLEN(d_lens); /*GP op header */ - vq_cmd_w0.u64 = 0; + vq_cmd_w0.s.opcode.minor = 0; vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); if (ctx->hmac) { - opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; vq_cmd_w0.s.param1 = key_len; - vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); + vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8); } else { - opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; vq_cmd_w0.s.param1 = 0; vq_cmd_w0.s.dlen = data_len; } - opcode.s.minor = 0; - /* Null auth only case enters the if */ if (unlikely(!hash_type && !ctx->enc_cipher)) { - opcode.s.major = CPT_MAJOR_OP_MISC; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC; /* Minor op is passthrough */ - opcode.s.minor = 0x03; + vq_cmd_w0.s.opcode.minor = 0x03; /* Send out completion code only */ vq_cmd_w0.s.param2 = 0x1; } - vq_cmd_w0.s.opcode = opcode.flags; - /* DPTR has SG list */ in_buffer = m_vaddr; dptr_dma = m_dma; @@ -563,7 +564,8 @@ cpt_digest_gen_prep(uint32_t flags, uint64_t k_dma = params->ctx_buf.dma_addr + offsetof(struct cpt_ctx, auth_key); /* Key */ - i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); + i = fill_sg_comp(gather_comp, i, k_dma, + RTE_ALIGN_CEIL(key_len, 8)); } /* input data */ @@ -633,9 +635,6 @@ cpt_digest_gen_prep(uint32_t flags, req->ist.ei1 = dptr_dma; req->ist.ei2 = rptr_dma; - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -643,7 +642,6 @@ cpt_digest_gen_prep(uint32_t flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -671,12 +669,10 @@ cpt_enc_hmac_prep(uint32_t flags, uint32_t encr_data_len, auth_data_len, aad_len = 0; uint32_t passthrough_len = 0; void *m_vaddr, *offset_vaddr; - uint64_t m_dma, offset_dma, ctx_dma; + uint64_t m_dma, offset_dma; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; void *c_vaddr; uint64_t c_dma; - opcode_info_t opcode; meta_p = &fc_params->meta_buf; m_vaddr = meta_p->vaddr; @@ -755,8 +751,10 @@ cpt_enc_hmac_prep(uint32_t flags, } /* Encryption */ - opcode.s.major = CPT_MAJOR_OP_FC; - opcode.s.minor = 0; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; + vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT; + vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc << + CPT_HMAC_FIRST_BIT_POS); if (hash_type == GMAC_TYPE) { encr_offset = 0; @@ -767,10 +765,12 @@ cpt_enc_hmac_prep(uint32_t flags, enc_dlen = encr_data_len + encr_offset; if (unlikely(encr_data_len & 0xf)) { if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) - enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; + enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + + encr_offset; else if (likely((cipher_type == AES_CBC) || (cipher_type == AES_ECB))) - enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; + enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) + + encr_offset; } if (unlikely(auth_dlen > enc_dlen)) { @@ -781,8 +781,10 @@ cpt_enc_hmac_prep(uint32_t flags, outputlen = enc_dlen + mac_len; } + if (cpt_ctx->auth_enc != 0) + outputlen = enc_dlen; + /* GP op header */ - vq_cmd_w0.u64 = 0; vq_cmd_w0.s.param1 = encr_data_len; vq_cmd_w0.s.param2 = auth_data_len; /* @@ -813,8 +815,6 @@ cpt_enc_hmac_prep(uint32_t flags, vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; - vq_cmd_w0.s.opcode = opcode.flags; - if (likely(iv_len)) { uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); @@ -843,9 +843,7 @@ cpt_enc_hmac_prep(uint32_t flags, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major |= CPT_DMA_MODE; - - vq_cmd_w0.s.opcode = opcode.flags; + vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; if (likely(iv_len)) { uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr @@ -1003,13 +1001,6 @@ cpt_enc_hmac_prep(uint32_t flags, req->ist.ei2 = rptr_dma; } - ctx_dma = fc_params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, fctx); - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = ctx_dma; - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -1017,7 +1008,6 @@ cpt_enc_hmac_prep(uint32_t flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -1044,10 +1034,8 @@ cpt_dec_hmac_prep(uint32_t flags, uint32_t encr_data_len, auth_data_len, aad_len = 0; uint32_t passthrough_len = 0; void *m_vaddr, *offset_vaddr; - uint64_t m_dma, offset_dma, ctx_dma; - opcode_info_t opcode; + uint64_t m_dma, offset_dma; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; void *c_vaddr; uint64_t c_dma; @@ -1128,8 +1116,10 @@ cpt_dec_hmac_prep(uint32_t flags, m_dma += size; /* Decryption */ - opcode.s.major = CPT_MAJOR_OP_FC; - opcode.s.minor = 1; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC; + vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT; + vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth << + CPT_HMAC_FIRST_BIT_POS); if (hash_type == GMAC_TYPE) { encr_offset = 0; @@ -1147,7 +1137,9 @@ cpt_dec_hmac_prep(uint32_t flags, outputlen = enc_dlen; } - vq_cmd_w0.u64 = 0; + if (cpt_ctx->dec_auth != 0) + outputlen = inputlen = enc_dlen; + vq_cmd_w0.s.param1 = encr_data_len; vq_cmd_w0.s.param2 = auth_data_len; @@ -1184,8 +1176,6 @@ cpt_dec_hmac_prep(uint32_t flags, vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; - vq_cmd_w0.s.opcode = opcode.flags; - if (likely(iv_len)) { uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); @@ -1215,9 +1205,7 @@ cpt_dec_hmac_prep(uint32_t flags, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major |= CPT_DMA_MODE; - - vq_cmd_w0.s.opcode = opcode.flags; + vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; if (likely(iv_len)) { uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + @@ -1388,13 +1376,6 @@ cpt_dec_hmac_prep(uint32_t flags, req->ist.ei2 = rptr_dma; } - ctx_dma = fc_params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, fctx); - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = ctx_dma; - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -1402,7 +1383,6 @@ cpt_dec_hmac_prep(uint32_t flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -1433,8 +1413,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t *offset_vaddr, offset_dma; uint32_t *iv_s, iv[4]; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; - opcode_info_t opcode; buf_p = ¶ms->meta_buf; m_vaddr = buf_p->vaddr; @@ -1468,11 +1446,11 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ - opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | + vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | (0 << 3) | (flags & 0x7)); if (flags == 0x1) { @@ -1535,7 +1513,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, /* * GP op header, lengths are expected in bits. */ - vq_cmd_w0.u64 = 0; vq_cmd_w0.s.param1 = encr_data_len; vq_cmd_w0.s.param2 = auth_data_len; @@ -1568,8 +1545,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; - vq_cmd_w0.s.opcode = opcode.flags; - if (likely(iv_len)) { uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); @@ -1592,9 +1567,7 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; m_dma += OFF_CTRL_LEN + iv_len; - opcode.s.major |= CPT_DMA_MODE; - - vq_cmd_w0.s.opcode = opcode.flags; + vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; /* DPTR has SG list */ in_buffer = m_vaddr; @@ -1710,12 +1683,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, req->ist.ei2 = rptr_dma; } - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, zs_ctx); - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -1723,7 +1690,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -1753,8 +1719,6 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, uint64_t *offset_vaddr, offset_dma; uint32_t *iv_s, iv[4], j; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; - opcode_info_t opcode; buf_p = ¶ms->meta_buf; m_vaddr = buf_p->vaddr; @@ -1793,11 +1757,12 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; + vq_cmd_w0.u64 = 0; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G; /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ - opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | + vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | (0 << 3) | (flags & 0x7)); /* consider iv len */ @@ -1826,7 +1791,6 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, /* * GP op header, lengths are expected in bits. */ - vq_cmd_w0.u64 = 0; vq_cmd_w0.s.param1 = encr_data_len; /* @@ -1858,8 +1822,6 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; - vq_cmd_w0.s.opcode = opcode.flags; - if (likely(iv_len)) { uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); @@ -1883,9 +1845,7 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; m_dma += OFF_CTRL_LEN + iv_len; - opcode.s.major |= CPT_DMA_MODE; - - vq_cmd_w0.s.opcode = opcode.flags; + vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE; /* DPTR has SG list */ in_buffer = m_vaddr; @@ -1974,12 +1934,6 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, req->ist.ei2 = rptr_dma; } - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, zs_ctx); - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -1987,7 +1941,6 @@ cpt_zuc_snow3g_dec_prep(uint32_t req_flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -2019,8 +1972,6 @@ cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t m_dma, c_dma; uint64_t *offset_vaddr, offset_dma; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; - opcode_info_t opcode; uint8_t *in_buffer; uint32_t g_size_bytes, s_size_bytes; uint64_t dptr_dma, rptr_dma; @@ -2070,19 +2021,17 @@ cpt_kasumi_enc_prep(uint32_t req_flags, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ - opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | + vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) | (flags & 0x7)); /* * GP op header, lengths are expected in bits. */ - vq_cmd_w0.u64 = 0; vq_cmd_w0.s.param1 = encr_data_len; vq_cmd_w0.s.param2 = auth_data_len; - vq_cmd_w0.s.opcode = opcode.flags; /* consider iv len */ if (flags == 0x0) { @@ -2221,12 +2170,6 @@ cpt_kasumi_enc_prep(uint32_t req_flags, req->ist.ei1 = dptr_dma; req->ist.ei2 = rptr_dma; - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, k_ctx); - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -2234,7 +2177,6 @@ cpt_kasumi_enc_prep(uint32_t req_flags, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -2263,8 +2205,6 @@ cpt_kasumi_dec_prep(uint64_t d_offs, uint64_t m_dma, c_dma; uint64_t *offset_vaddr, offset_dma; vq_cmd_word0_t vq_cmd_w0; - vq_cmd_word3_t vq_cmd_w3; - opcode_info_t opcode; uint8_t *in_buffer; uint32_t g_size_bytes, s_size_bytes; uint64_t dptr_dma, rptr_dma; @@ -2303,18 +2243,17 @@ cpt_kasumi_dec_prep(uint64_t d_offs, m_vaddr = (uint8_t *)m_vaddr + size; m_dma += size; - opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; + vq_cmd_w0.u64 = 0; + vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ - opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | + vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | (dir << 4) | (0 << 3) | (flags & 0x7)); /* * GP op header, lengths are expected in bits. */ - vq_cmd_w0.u64 = 0; vq_cmd_w0.s.param1 = encr_data_len; - vq_cmd_w0.s.opcode = opcode.flags; /* consider iv len */ encr_offset += iv_len; @@ -2411,12 +2350,6 @@ cpt_kasumi_dec_prep(uint64_t d_offs, req->ist.ei1 = dptr_dma; req->ist.ei2 = rptr_dma; - /* vq command w3 */ - vq_cmd_w3.u64 = 0; - vq_cmd_w3.s.grp = 0; - vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + - offsetof(struct cpt_ctx, k_ctx); - /* 16 byte aligned cpt res address */ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); *req->completion_addr = COMPLETION_CODE_INIT; @@ -2424,7 +2357,6 @@ cpt_kasumi_dec_prep(uint64_t d_offs, /* Fill microcode part of instruction */ req->ist.ei0 = vq_cmd_w0.u64; - req->ist.ei3 = vq_cmd_w3.u64; req->op = op; @@ -2492,11 +2424,12 @@ cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, } static __rte_always_inline int -cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, - uint16_t key_len, uint16_t mac_len) +cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type, + const uint8_t *key, uint16_t key_len, uint16_t mac_len) { - struct cpt_ctx *cpt_ctx = ctx; - mc_fc_context_t *fctx = &cpt_ctx->fctx; + mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx; + mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx; + mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx; if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { uint32_t keyx[4]; @@ -2511,26 +2444,26 @@ cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, case SNOW3G_UIA2: cpt_ctx->snow3g = 1; gen_key_snow3g(key, keyx); - memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); + memcpy(zs_ctx->ci_key, keyx, key_len); cpt_ctx->fc_type = ZUC_SNOW3G; cpt_ctx->zsk_flags = 0x1; break; case ZUC_EIA3: cpt_ctx->snow3g = 0; - memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); - memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); + memcpy(zs_ctx->ci_key, key, key_len); + memcpy(zs_ctx->zuc_const, zuc_d, 32); cpt_ctx->fc_type = ZUC_SNOW3G; cpt_ctx->zsk_flags = 0x1; break; case KASUMI_F9_ECB: /* Kasumi ECB mode */ cpt_ctx->k_ecb = 1; - memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); + memcpy(k_ctx->ci_key, key, key_len); cpt_ctx->fc_type = KASUMI; cpt_ctx->zsk_flags = 0x1; break; case KASUMI_F9_CBC: - memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); + memcpy(k_ctx->ci_key, key, key_len); cpt_ctx->fc_type = KASUMI; cpt_ctx->zsk_flags = 0x1; break; @@ -2643,6 +2576,7 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cpt_sess_misc *sess) { struct rte_crypto_cipher_xform *c_form; + struct cpt_ctx *ctx = SESS_PRIV(sess); cipher_type_t enc_type = 0; /* NULL Cipher type */ uint32_t cipher_key_len = 0; uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; @@ -2651,9 +2585,14 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; - else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) + else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; - else { + if (xform->next != NULL && + xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { + /* Perform decryption followed by auth verify */ + ctx->dec_auth = 1; + } + } else { CPT_LOG_DP_ERR("Unknown cipher operation\n"); return -1; } @@ -2744,10 +2683,18 @@ static __rte_always_inline int fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cpt_sess_misc *sess) { + struct cpt_ctx *ctx = SESS_PRIV(sess); struct rte_crypto_auth_xform *a_form; auth_type_t auth_type = 0; /* NULL Auth type */ uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; + if (xform->next != NULL && + xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER && + xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { + /* Perform auth followed by encryption */ + ctx->auth_enc = 1; + } + a_form = &xform->auth; if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) @@ -3070,6 +3017,7 @@ fill_fc_params(struct rte_crypto_op *cop, { uint32_t space = 0; struct rte_crypto_sym_op *sym_op = cop->sym; + struct cpt_ctx *ctx = SESS_PRIV(sess_misc); void *mdata = NULL; uintptr_t *op; uint32_t mc_hash_off; @@ -3197,9 +3145,10 @@ fill_fc_params(struct rte_crypto_op *cop, m = m_src; /* hmac immediately following data is best case */ - if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + + if (!ctx->dec_auth && !ctx->auth_enc && + (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + mc_hash_off != - (uint8_t *)sym_op->auth.digest.data)) { + (uint8_t *)sym_op->auth.digest.data))) { flags |= VALID_MAC_BUF; fc_params.mac_buf.size = sess_misc->mac_len; @@ -3214,7 +3163,9 @@ fill_fc_params(struct rte_crypto_op *cop, fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; - if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) + if (!ctx->dec_auth && + unlikely(sess_misc->is_null || + sess_misc->cpt_op == CPT_OP_DECODE)) inplace = 0; if (likely(!m_dst && inplace)) {