X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fcnxk%2Fcnxk_se.h;h=7959c4c7af8d3aa3ef3c4280195148c0c4c262ec;hb=90a2ec4ae81f2ef52f7c14bfc9307e75a4127fa4;hp=3ed6b909fe8eaba002075a578320c7969f092f34;hpb=252947b9509ad701cb019dff8a5a81d329d7b251;p=dpdk.git diff --git a/drivers/crypto/cnxk/cnxk_se.h b/drivers/crypto/cnxk/cnxk_se.h index 3ed6b909fe..7959c4c7af 100644 --- a/drivers/crypto/cnxk/cnxk_se.h +++ b/drivers/crypto/cnxk/cnxk_se.h @@ -36,6 +36,29 @@ struct cnxk_se_sess { struct roc_se_ctx roc_se_ctx; } __rte_cache_aligned; +static inline void +pdcp_iv_copy(uint8_t *iv_d, uint8_t *iv_s, const uint8_t pdcp_alg_type) +{ + uint32_t *iv_s_temp, iv_temp[4]; + int j; + + if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) { + /* + * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 + * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 + */ + + iv_s_temp = (uint32_t *)iv_s; + + for (j = 0; j < 4; j++) + iv_temp[j] = iv_s_temp[3 - j]; + memcpy(iv_d, iv_temp, 16); + } else { + /* ZUC doesn't need a swap */ + memcpy(iv_d, iv_s, 16); + } +} + static __rte_always_inline int cpt_mac_len_verify(struct rte_crypto_auth_xform *auth) { @@ -947,21 +970,20 @@ cpt_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, } static __rte_always_inline int -cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, - struct roc_se_fc_params *params, - struct cpt_inst_s *inst) +cpt_zuc_snow3g_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, + struct roc_se_fc_params *params, struct cpt_inst_s *inst) { uint32_t size; int32_t inputlen, outputlen; struct roc_se_ctx *se_ctx; uint32_t mac_len = 0; - uint8_t pdcp_alg_type, j; - uint32_t encr_offset = 0, auth_offset = 0; - uint32_t encr_data_len = 0, auth_data_len = 0; - int flags, iv_len = 16; + uint8_t pdcp_alg_type; + uint32_t encr_offset, auth_offset; + uint32_t encr_data_len, auth_data_len; + int flags, iv_len; uint64_t offset_ctrl; uint64_t *offset_vaddr; - uint32_t *iv_s, iv[4]; + uint8_t *iv_s; union cpt_inst_w4 cpt_inst_w4; se_ctx = params->ctx_buf.vaddr; @@ -971,12 +993,12 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G; - /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ - - cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) | - (0 << 4) | (0 << 3) | (flags & 0x7)); + cpt_inst_w4.s.opcode_minor = se_ctx->template_w4.s.opcode_minor; if (flags == 0x1) { + iv_s = params->auth_iv_buf; + iv_len = params->auth_iv_len; + /* * Microcode expects offsets in bytes * TODO: Rounding off @@ -995,7 +1017,12 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); + encr_data_len = 0; + encr_offset = 0; } else { + iv_s = params->iv_buf; + iv_len = params->cipher_iv_len; + /* EEA3 or UEA2 */ /* * Microcode expects offsets in bytes @@ -1013,6 +1040,9 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, /* iv offset is 0 */ offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); + + auth_data_len = 0; + auth_offset = 0; } if (unlikely((encr_offset >> 16) || (auth_offset >> 8))) { @@ -1022,23 +1052,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, return -1; } - /* IV */ - iv_s = (flags == 0x1) ? params->auth_iv_buf : params->iv_buf; - - if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) { - /* - * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 - * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 - */ - - for (j = 0; j < 4; j++) - iv[j] = iv_s[3 - j]; - } else { - /* ZUC doesn't need a swap */ - for (j = 0; j < 4; j++) - iv[j] = iv_s[j]; - } - /* * GP op header, lengths are expected in bits. */ @@ -1067,11 +1080,8 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN; - if (likely(iv_len)) { - uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr + - ROC_SE_OFF_CTRL_LEN); - memcpy(iv_d, iv, 16); - } + uint8_t *iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN); + pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type); *offset_vaddr = offset_ctrl; } else { @@ -1080,7 +1090,7 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, struct roc_se_sglist_comp *gather_comp; struct roc_se_sglist_comp *scatter_comp; uint8_t *in_buffer; - uint32_t *iv_d; + uint8_t *iv_d; /* save space for iv */ offset_vaddr = m_vaddr; @@ -1112,9 +1122,8 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, /* iv offset is 0 */ *offset_vaddr = offset_ctrl; - iv_d = (uint32_t *)((uint8_t *)offset_vaddr + - ROC_SE_OFF_CTRL_LEN); - memcpy(iv_d, iv, 16); + iv_d = ((uint8_t *)offset_vaddr + ROC_SE_OFF_CTRL_LEN); + pdcp_iv_copy(iv_d, iv_s, pdcp_alg_type); /* input data */ size = inputlen - iv_len; @@ -1208,209 +1217,6 @@ cpt_zuc_snow3g_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, return 0; } -static __rte_always_inline int -cpt_zuc_snow3g_dec_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, - struct roc_se_fc_params *params, - struct cpt_inst_s *inst) -{ - uint32_t size; - int32_t inputlen = 0, outputlen; - struct roc_se_ctx *se_ctx; - uint8_t pdcp_alg_type, iv_len = 16; - uint32_t encr_offset; - uint32_t encr_data_len; - int flags; - uint64_t *offset_vaddr; - uint32_t *iv_s, iv[4], j; - union cpt_inst_w4 cpt_inst_w4; - - /* - * Microcode expects offsets in bytes - * TODO: Rounding off - */ - encr_offset = ROC_SE_ENCR_OFFSET(d_offs) / 8; - encr_data_len = ROC_SE_ENCR_DLEN(d_lens); - - se_ctx = params->ctx_buf.vaddr; - flags = se_ctx->zsk_flags; - pdcp_alg_type = se_ctx->pdcp_alg_type; - - cpt_inst_w4.u64 = 0; - cpt_inst_w4.s.opcode_major = ROC_SE_MAJOR_OP_ZUC_SNOW3G; - - /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ - - cpt_inst_w4.s.opcode_minor = ((1 << 7) | (pdcp_alg_type << 5) | - (0 << 4) | (0 << 3) | (flags & 0x7)); - - /* consider iv len */ - encr_offset += iv_len; - - inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); - outputlen = inputlen; - - /* IV */ - iv_s = params->iv_buf; - if (pdcp_alg_type == ROC_SE_PDCP_ALG_TYPE_SNOW3G) { - /* - * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 - * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 - */ - - for (j = 0; j < 4; j++) - iv[j] = iv_s[3 - j]; - } else { - /* ZUC doesn't need a swap */ - for (j = 0; j < 4; j++) - iv[j] = iv_s[j]; - } - - /* - * GP op header, lengths are expected in bits. - */ - cpt_inst_w4.s.param1 = encr_data_len; - - /* - * In cn9k, cn10k since we have a limitation of - * IV & Offset control word not part of instruction - * and need to be part of Data Buffer, we check if - * head room is there and then only do the Direct mode processing - */ - if (likely((req_flags & ROC_SE_SINGLE_BUF_INPLACE) && - (req_flags & ROC_SE_SINGLE_BUF_HEADROOM))) { - void *dm_vaddr = params->bufs[0].vaddr; - - /* Use Direct mode */ - - offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - - ROC_SE_OFF_CTRL_LEN - iv_len); - - /* DPTR */ - inst->dptr = (uint64_t)offset_vaddr; - - /* RPTR should just exclude offset control word */ - inst->rptr = (uint64_t)dm_vaddr - iv_len; - - cpt_inst_w4.s.dlen = inputlen + ROC_SE_OFF_CTRL_LEN; - - if (likely(iv_len)) { - uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr + - ROC_SE_OFF_CTRL_LEN); - memcpy(iv_d, iv, 16); - } - - /* iv offset is 0 */ - *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); - } else { - void *m_vaddr = params->meta_buf.vaddr; - uint32_t i, g_size_bytes, s_size_bytes; - struct roc_se_sglist_comp *gather_comp; - struct roc_se_sglist_comp *scatter_comp; - uint8_t *in_buffer; - uint32_t *iv_d; - - /* save space for offset and iv... */ - offset_vaddr = m_vaddr; - - m_vaddr = (uint8_t *)m_vaddr + ROC_SE_OFF_CTRL_LEN + iv_len; - - cpt_inst_w4.s.opcode_major |= (uint64_t)ROC_SE_DMA_MODE; - - /* DPTR has SG list */ - in_buffer = m_vaddr; - - ((uint16_t *)in_buffer)[0] = 0; - ((uint16_t *)in_buffer)[1] = 0; - - /* TODO Add error check if space will be sufficient */ - gather_comp = - (struct roc_se_sglist_comp *)((uint8_t *)m_vaddr + 8); - - /* - * Input Gather List - */ - i = 0; - - /* Offset control word */ - - /* iv offset is 0 */ - *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); - - i = fill_sg_comp(gather_comp, i, (uint64_t)offset_vaddr, - ROC_SE_OFF_CTRL_LEN + iv_len); - - iv_d = (uint32_t *)((uint8_t *)offset_vaddr + - ROC_SE_OFF_CTRL_LEN); - memcpy(iv_d, iv, 16); - - /* Add input data */ - size = inputlen - iv_len; - if (size) { - i = fill_sg_comp_from_iov(gather_comp, i, - params->src_iov, 0, &size, - NULL, 0); - if (unlikely(size)) { - plt_dp_err("Insufficient buffer space," - " size %d needed", - size); - return -1; - } - } - ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); - g_size_bytes = - ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); - - /* - * Output Scatter List - */ - - i = 0; - scatter_comp = - (struct roc_se_sglist_comp *)((uint8_t *)gather_comp + - g_size_bytes); - - /* IV */ - i = fill_sg_comp(scatter_comp, i, - (uint64_t)offset_vaddr + ROC_SE_OFF_CTRL_LEN, - iv_len); - - /* Add output data */ - size = outputlen - iv_len; - if (size) { - i = fill_sg_comp_from_iov(scatter_comp, i, - params->dst_iov, 0, &size, - NULL, 0); - - if (unlikely(size)) { - plt_dp_err("Insufficient buffer space," - " size %d needed", - size); - return -1; - } - } - ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); - s_size_bytes = - ((i + 3) / 4) * sizeof(struct roc_se_sglist_comp); - - size = g_size_bytes + s_size_bytes + ROC_SE_SG_LIST_HDR_SIZE; - - /* This is DPTR len in case of SG mode */ - cpt_inst_w4.s.dlen = size; - - inst->dptr = (uint64_t)in_buffer; - } - - if (unlikely((encr_offset >> 16))) { - plt_dp_err("Offset not supported"); - plt_dp_err("enc_offset: %d", encr_offset); - return -1; - } - - inst->w4.u64 = cpt_inst_w4.u64; - - return 0; -} - static __rte_always_inline int cpt_kasumi_enc_prep(uint32_t req_flags, uint64_t d_offs, uint64_t d_lens, struct roc_se_fc_params *params, struct cpt_inst_s *inst) @@ -1749,8 +1555,8 @@ cpt_fc_dec_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, if (likely(fc_type == ROC_SE_FC_GEN)) { ret = cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, inst); } else if (fc_type == ROC_SE_PDCP) { - ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, - inst); + ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params, + inst); } else if (fc_type == ROC_SE_KASUMI) { ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, inst); } @@ -1778,8 +1584,8 @@ cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, if (likely(fc_type == ROC_SE_FC_GEN)) { ret = cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, inst); } else if (fc_type == ROC_SE_PDCP) { - ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, - inst); + ret = cpt_zuc_snow3g_prep(flags, d_offs, d_lens, fc_params, + inst); } else if (fc_type == ROC_SE_KASUMI) { ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, inst); @@ -1915,7 +1721,7 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess) break; case RTE_CRYPTO_CIPHER_ZUC_EEA3: enc_type = ROC_SE_ZUC_EEA3; - cipher_key_len = 16; + cipher_key_len = c_form->key.length; zsk_flag = ROC_SE_ZS_EA; break; case RTE_CRYPTO_CIPHER_AES_XTS: @@ -1959,6 +1765,8 @@ fill_sess_cipher(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess) NULL))) return -1; + if ((enc_type >= ROC_SE_ZUC_EEA3) && (enc_type <= ROC_SE_AES_CTR_EEA2)) + roc_se_ctx_swap(&sess->roc_se_ctx); return 0; } @@ -2062,6 +1870,10 @@ fill_sess_auth(struct rte_crypto_sym_xform *xform, struct cnxk_se_sess *sess) a_form->digest_length))) return -1; + if ((auth_type >= ROC_SE_ZUC_EIA3) && + (auth_type <= ROC_SE_AES_CMAC_EIA2)) + roc_se_ctx_swap(&sess->roc_se_ctx); + return 0; } @@ -2259,6 +2071,9 @@ fill_fc_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess, uint32_t iv_buf[4]; int ret; + fc_params.cipher_iv_len = sess->iv_length; + fc_params.auth_iv_len = sess->auth_iv_length; + if (likely(sess->iv_length)) { flags |= ROC_SE_VALID_IV_BUF; fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, uint8_t *, @@ -2569,6 +2384,7 @@ fill_digest_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess, */ d_offs = auth_range_off; auth_range_off = 0; + params.auth_iv_len = sess->auth_iv_length; params.auth_iv_buf = rte_crypto_op_ctod_offset( cop, uint8_t *, sess->auth_iv_offset); if (sess->zsk_flag == ROC_SE_K_F9) {