#ifndef _CPT_UCODE_H_
#define _CPT_UCODE_H_
-
#include <stdbool.h>
#include "cpt_common.h"
return (uint32_t)i;
}
-static __rte_always_inline int
+static __rte_always_inline void
+cpt_digest_gen_prep(uint32_t flags,
+ uint64_t d_lens,
+ digest_params_t *params,
+ void *op,
+ void **prep_req)
+{
+ struct cpt_request_info *req;
+ uint32_t size, i;
+ int32_t m_size;
+ uint16_t data_len, mac_len, key_len;
+ auth_type_t hash_type;
+ buf_ptr_t *meta_p;
+ struct cpt_ctx *ctx;
+ sg_comp_t *gather_comp;
+ sg_comp_t *scatter_comp;
+ uint8_t *in_buffer;
+ uint32_t g_size_bytes, s_size_bytes;
+ uint64_t dptr_dma, rptr_dma;
+ vq_cmd_word0_t vq_cmd_w0;
+ vq_cmd_word3_t vq_cmd_w3;
+ void *c_vaddr, *m_vaddr;
+ uint64_t c_dma, m_dma;
+ opcode_info_t opcode;
+
+ ctx = params->ctx_buf.vaddr;
+ meta_p = ¶ms->meta_buf;
+
+ m_vaddr = meta_p->vaddr;
+ m_dma = meta_p->dma_addr;
+ m_size = meta_p->size;
+
+ /*
+ * Save initial space that followed app data for completion code &
+ * alternate completion code to fall in same cache line as app data
+ */
+ m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
+ m_dma += COMPLETION_CODE_SIZE;
+ size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
+ (uint8_t *)m_vaddr;
+ c_vaddr = (uint8_t *)m_vaddr + size;
+ c_dma = m_dma + size;
+ size += sizeof(cpt_res_s_t);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ req = m_vaddr;
+
+ size = sizeof(struct cpt_request_info);
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ hash_type = ctx->hash_type;
+ mac_len = ctx->mac_len;
+ key_len = ctx->auth_key_len;
+ data_len = AUTH_DLEN(d_lens);
+
+ /*GP op header */
+ vq_cmd_w0.u64 = 0;
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(((uint16_t)hash_type << 8));
+ if (ctx->hmac) {
+ opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
+ vq_cmd_w0.s.param1 = rte_cpu_to_be_16(key_len);
+ vq_cmd_w0.s.dlen =
+ rte_cpu_to_be_16((data_len + ROUNDUP8(key_len)));
+ } else {
+ opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
+ vq_cmd_w0.s.param1 = 0;
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(data_len);
+ }
+
+ opcode.s.minor = 0;
+
+ /* Null auth only case enters the if */
+ if (unlikely(!hash_type && !ctx->enc_cipher)) {
+ opcode.s.major = CPT_MAJOR_OP_MISC;
+ /* Minor op is passthrough */
+ opcode.s.minor = 0x03;
+ /* Send out completion code only */
+ vq_cmd_w0.s.param2 = rte_cpu_to_be_16(0x1);
+ }
+
+ vq_cmd_w0.s.opcode = rte_cpu_to_be_16(opcode.flags);
+
+ /* DPTR has SG list */
+ in_buffer = m_vaddr;
+ dptr_dma = m_dma;
+
+ ((uint16_t *)in_buffer)[0] = 0;
+ ((uint16_t *)in_buffer)[1] = 0;
+
+ /* TODO Add error check if space will be sufficient */
+ gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
+
+ /*
+ * Input gather list
+ */
+
+ i = 0;
+
+ if (ctx->hmac) {
+ uint64_t k_dma = params->ctx_buf.dma_addr +
+ offsetof(struct cpt_ctx, auth_key);
+ /* Key */
+ i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len));
+ }
+
+ /* input data */
+ size = data_len;
+ if (size) {
+ i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
+ 0, &size, NULL, 0);
+ if (unlikely(size)) {
+ CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
+ " by %dB", size);
+ return;
+ }
+ } else {
+ /*
+ * Looks like we need to support zero data
+ * gather ptr in case of hash & hmac
+ */
+ i++;
+ }
+ ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
+ g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ /*
+ * Output Gather list
+ */
+
+ i = 0;
+ scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
+
+ if (flags & VALID_MAC_BUF) {
+ if (unlikely(params->mac_buf.size < mac_len)) {
+ CPT_LOG_DP_ERR("Insufficient MAC size");
+ return;
+ }
+
+ size = mac_len;
+ i = fill_sg_comp_from_buf_min(scatter_comp, i,
+ ¶ms->mac_buf, &size);
+ } else {
+ size = mac_len;
+ i = fill_sg_comp_from_iov(scatter_comp, i,
+ params->src_iov, data_len,
+ &size, NULL, 0);
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient dst IOV size, short by"
+ " %dB", size);
+ return;
+ }
+ }
+
+ ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
+ s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
+
+ size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
+
+ /* This is DPTR len incase of SG mode */
+ vq_cmd_w0.s.dlen = rte_cpu_to_be_16(size);
+
+ m_vaddr = (uint8_t *)m_vaddr + size;
+ m_dma += size;
+ m_size -= size;
+
+ /* cpt alternate completion address saved earlier */
+ req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
+ *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
+ rptr_dma = c_dma - 8;
+
+ req->ist.ei1 = dptr_dma;
+ req->ist.ei2 = rptr_dma;
+ /* First 16-bit swap then 64-bit swap */
+ /* TODO: HACK: Reverse the vq_cmd and cpt_req bit field definitions
+ * to eliminate all the swapping
+ */
+ vq_cmd_w0.u64 = rte_cpu_to_be_64(vq_cmd_w0.u64);
+
+ /* vq command w3 */
+ vq_cmd_w3.u64 = 0;
+
+ /* 16 byte aligned cpt res address */
+ req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
+ *req->completion_addr = COMPLETION_CODE_INIT;
+ req->comp_baddr = c_dma;
+
+ /* Fill microcode part of instruction */
+ req->ist.ei0 = vq_cmd_w0.u64;
+ req->ist.ei3 = vq_cmd_w3.u64;
+
+ req->op = op;
+
+ *prep_req = req;
+ return;
+}
+
+static __rte_always_inline void
cpt_enc_hmac_prep(uint32_t flags,
uint64_t d_offs,
uint64_t d_lens,
if (unlikely(size)) {
CPT_LOG_DP_ERR("Insufficient buffer space,"
" size %d needed", size);
- return ERR_BAD_INPUT_ARG;
+ return;
}
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
aad_buf,
aad_offset);
}
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer"
+ " space, size %d needed",
+ size);
+ return;
+ }
}
/* mac_data */
if (mac_len) {
CPT_LOG_DP_ERR("Insufficient buffer"
" space, size %d needed",
size);
- return ERR_BAD_INPUT_ARG;
+ return;
}
}
}
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
-static __rte_always_inline int
+static __rte_always_inline void
cpt_dec_hmac_prep(uint32_t flags,
uint64_t d_offs,
uint64_t d_lens,
aad_buf,
aad_offset);
}
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer"
+ " space, size %d needed",
+ size);
+ return;
+ }
}
/* mac data */
uint32_t aad_offset = aad_len ?
passthrough_len : 0;
- if (!fc_params->src_iov)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(!fc_params->src_iov)) {
+ CPT_LOG_DP_ERR("Bad input args");
+ return;
+ }
i = fill_sg_comp_from_iov(
gather_comp, i,
aad_offset);
}
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer"
+ " space, size %d needed",
+ size);
+ return;
+ }
}
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
uint32_t aad_offset = aad_len ?
passthrough_len : 0;
- if (!fc_params->dst_iov)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(!fc_params->dst_iov)) {
+ CPT_LOG_DP_ERR("Bad input args");
+ return;
+ }
i = fill_sg_comp_from_iov(scatter_comp, i,
fc_params->dst_iov, 0,
aad_offset);
}
- if (unlikely(size))
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
-static __rte_always_inline int
+static __rte_always_inline void
cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
uint64_t d_offs,
uint64_t d_lens,
i = fill_sg_comp_from_iov(gather_comp, i,
params->src_iov,
0, &size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
/* mac data */
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
-static __rte_always_inline int
+static __rte_always_inline void
cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
uint64_t d_offs,
uint64_t d_lens,
i = fill_sg_comp_from_iov(gather_comp, i,
params->src_iov,
0, &size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
-static __rte_always_inline int
+static __rte_always_inline void
cpt_kasumi_enc_prep(uint32_t req_flags,
uint64_t d_offs,
uint64_t d_lens,
params->src_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
/* mac data */
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
-static __rte_always_inline int
+static __rte_always_inline void
cpt_kasumi_dec_prep(uint64_t d_offs,
uint64_t d_lens,
fc_params_t *params,
i = fill_sg_comp_from_iov(gather_comp, i,
params->src_iov,
0, &size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
i = fill_sg_comp_from_iov(scatter_comp, i,
params->dst_iov, 0,
&size, NULL, 0);
- if (size)
- return ERR_BAD_INPUT_ARG;
+ if (unlikely(size)) {
+ CPT_LOG_DP_ERR("Insufficient buffer space,"
+ " size %d needed", size);
+ return;
+ }
}
((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
req->op = op;
*prep_req = req;
- return 0;
+ return;
}
static __rte_always_inline void *
uint64_t d_offs,
uint64_t d_lens,
fc_params_t *fc_params,
- void *op, int *ret_val)
+ void *op)
{
struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
uint8_t fc_type;
void *prep_req = NULL;
- int ret;
fc_type = ctx->fc_type;
if (likely(fc_type == FC_GEN)) {
- ret = cpt_dec_hmac_prep(flags, d_offs, d_lens,
- fc_params, op, &prep_req);
+ cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op,
+ &prep_req);
} else if (fc_type == ZUC_SNOW3G) {
- ret = cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens,
- fc_params, op, &prep_req);
+ cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op,
+ &prep_req);
} else if (fc_type == KASUMI) {
- ret = cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op,
- &prep_req);
- } else {
- /*
- * For AUTH_ONLY case,
- * MC only supports digest generation and verification
- * should be done in software by memcmp()
- */
-
- ret = ERR_EIO;
+ cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req);
}
- if (unlikely(!prep_req))
- *ret_val = ret;
+ /*
+ * For AUTH_ONLY case,
+ * MC only supports digest generation and verification
+ * should be done in software by memcmp()
+ */
+
return prep_req;
}
static __rte_always_inline void *__hot
cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
- fc_params_t *fc_params, void *op, int *ret_val)
+ fc_params_t *fc_params, void *op)
{
struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
uint8_t fc_type;
void *prep_req = NULL;
- int ret;
fc_type = ctx->fc_type;
/* Common api for rest of the ops */
if (likely(fc_type == FC_GEN)) {
- ret = cpt_enc_hmac_prep(flags, d_offs, d_lens,
- fc_params, op, &prep_req);
+ cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op,
+ &prep_req);
} else if (fc_type == ZUC_SNOW3G) {
- ret = cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens,
- fc_params, op, &prep_req);
+ cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op,
+ &prep_req);
} else if (fc_type == KASUMI) {
- ret = cpt_kasumi_enc_prep(flags, d_offs, d_lens,
- fc_params, op, &prep_req);
- } else {
- ret = ERR_EIO;
+ cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op,
+ &prep_req);
+ } else if (fc_type == HASH_HMAC) {
+ cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req);
}
- if (unlikely(!prep_req))
- *ret_val = ret;
return prep_req;
}
return 0;
}
-static __rte_always_inline void *
+static __rte_always_inline int
fill_fc_params(struct rte_crypto_op *cop,
struct cpt_sess_misc *sess_misc,
+ struct cpt_qp_meta_info *m_info,
void **mdata_ptr,
- int *op_ret)
+ void **prep_req)
{
uint32_t space = 0;
struct rte_crypto_sym_op *sym_op = cop->sym;
- void *mdata;
+ void *mdata = NULL;
uintptr_t *op;
uint32_t mc_hash_off;
uint32_t flags = 0;
uint64_t d_offs, d_lens;
- void *prep_req = NULL;
struct rte_mbuf *m_src, *m_dst;
uint8_t cpt_op = sess_misc->cpt_op;
uint8_t zsk_flag = sess_misc->zsk_flag;
char src[SRC_IOV_SIZE];
char dst[SRC_IOV_SIZE];
uint32_t iv_buf[4];
- struct cptvf_meta_info *cpt_m_info =
- (struct cptvf_meta_info *)(*mdata_ptr);
+ int ret;
if (likely(sess_misc->iv_length)) {
flags |= VALID_IV_BUF;
&fc_params,
&flags))) {
CPT_LOG_DP_ERR("Prepare inplace src iov failed");
- *op_ret = -1;
- return NULL;
+ ret = -EINVAL;
+ goto err_exit;
}
} else {
/* Store SG I/O in the api for reuse */
if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
CPT_LOG_DP_ERR("Prepare src iov failed");
- *op_ret = -1;
- return NULL;
+ ret = -EINVAL;
+ goto err_exit;
}
if (unlikely(m_dst != NULL)) {
"m_dst %p, need %u"
" more",
m_dst, pkt_len);
- return NULL;
+ ret = -EINVAL;
+ goto err_exit;
}
}
if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
CPT_LOG_DP_ERR("Prepare dst iov failed for "
"m_dst %p", m_dst);
- return NULL;
+ ret = -EINVAL;
+ goto err_exit;
}
} else {
fc_params.dst_iov = (void *)src;
}
if (likely(flags & SINGLE_BUF_HEADTAILROOM))
- mdata = alloc_op_meta(m_src,
- &fc_params.meta_buf,
- cpt_m_info->cptvf_op_sb_mlen,
- cpt_m_info->cptvf_meta_pool);
+ mdata = alloc_op_meta(m_src, &fc_params.meta_buf,
+ m_info->lb_mlen, m_info->pool);
else
- mdata = alloc_op_meta(NULL,
- &fc_params.meta_buf,
- cpt_m_info->cptvf_op_mlen,
- cpt_m_info->cptvf_meta_pool);
+ mdata = alloc_op_meta(NULL, &fc_params.meta_buf,
+ m_info->sg_mlen, m_info->pool);
if (unlikely(mdata == NULL)) {
CPT_LOG_DP_ERR("Error allocating meta buffer for request");
- return NULL;
+ ret = -ENOMEM;
+ goto err_exit;
}
op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
/* Finally prepare the instruction */
if (cpt_op & CPT_OP_ENCODE)
- prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
- &fc_params, op, op_ret);
+ *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
+ &fc_params, op);
else
- prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
- &fc_params, op, op_ret);
+ *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
+ &fc_params, op);
+
+ if (unlikely(*prep_req == NULL)) {
+ CPT_LOG_DP_ERR("Preparing request failed due to bad input arg");
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
- if (unlikely(!prep_req))
- free_op_meta(mdata, cpt_m_info->cptvf_meta_pool);
*mdata_ptr = mdata;
- return prep_req;
+
+ return 0;
+
+free_mdata_and_exit:
+ free_op_meta(mdata, m_info->pool);
+err_exit:
+ return ret;
+}
+
+static __rte_always_inline void
+compl_auth_verify(struct rte_crypto_op *op,
+ uint8_t *gen_mac,
+ uint64_t mac_len)
+{
+ uint8_t *mac;
+ struct rte_crypto_sym_op *sym_op = op->sym;
+
+ if (sym_op->auth.digest.data)
+ mac = sym_op->auth.digest.data;
+ else
+ mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
+ uint8_t *,
+ sym_op->auth.data.length +
+ sym_op->auth.data.offset);
+ if (!mac) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return;
+ }
+
+ if (memcmp(mac, gen_mac, mac_len))
+ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ else
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
}
static __rte_always_inline int
return -1;
}
+static __rte_always_inline void
+find_kasumif9_direction_and_length(uint8_t *src,
+ uint32_t counter_num_bytes,
+ uint32_t *addr_length_in_bits,
+ uint8_t *addr_direction)
+{
+ uint8_t found = 0;
+ uint32_t pos;
+ uint8_t last_byte;
+ while (!found && counter_num_bytes > 0) {
+ counter_num_bytes--;
+ if (src[counter_num_bytes] == 0x00)
+ continue;
+ pos = rte_bsf32(src[counter_num_bytes]);
+ if (pos == 7) {
+ if (likely(counter_num_bytes > 0)) {
+ last_byte = src[counter_num_bytes - 1];
+ *addr_direction = last_byte & 0x1;
+ *addr_length_in_bits = counter_num_bytes * 8
+ - 1;
+ }
+ } else {
+ last_byte = src[counter_num_bytes];
+ *addr_direction = (last_byte >> (pos + 1)) & 0x1;
+ *addr_length_in_bits = counter_num_bytes * 8
+ + (8 - (pos + 2));
+ }
+ found = 1;
+ }
+}
+
+/*
+ * This handles all auth only except AES_GMAC
+ */
+static __rte_always_inline int
+fill_digest_params(struct rte_crypto_op *cop,
+ struct cpt_sess_misc *sess,
+ struct cpt_qp_meta_info *m_info,
+ void **mdata_ptr,
+ void **prep_req)
+{
+ uint32_t space = 0;
+ struct rte_crypto_sym_op *sym_op = cop->sym;
+ void *mdata;
+ phys_addr_t mphys;
+ uint64_t *op;
+ uint32_t auth_range_off;
+ uint32_t flags = 0;
+ uint64_t d_offs = 0, d_lens;
+ struct rte_mbuf *m_src, *m_dst;
+ uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
+ uint8_t zsk_flag = sess->zsk_flag;
+ uint16_t mac_len = sess->mac_len;
+ fc_params_t params;
+ char src[SRC_IOV_SIZE];
+ uint8_t iv_buf[16];
+ int ret;
+
+ memset(¶ms, 0, sizeof(fc_params_t));
+
+ m_src = sym_op->m_src;
+
+ /* For just digest lets force mempool alloc */
+ mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen,
+ m_info->pool);
+ if (mdata == NULL) {
+ ret = -ENOMEM;
+ goto err_exit;
+ }
+
+ mphys = params.meta_buf.dma_addr;
+
+ op = mdata;
+ op[0] = (uintptr_t)mdata;
+ op[1] = (uintptr_t)cop;
+ op[2] = op[3] = 0; /* Used to indicate auth verify */
+ space += 4 * sizeof(uint64_t);
+
+ auth_range_off = sym_op->auth.data.offset;
+
+ flags = VALID_MAC_BUF;
+ params.src_iov = (void *)src;
+ if (unlikely(zsk_flag)) {
+ /*
+ * Since for Zuc, Kasumi, Snow3g offsets are in bits
+ * we will send pass through even for auth only case,
+ * let MC handle it
+ */
+ d_offs = auth_range_off;
+ auth_range_off = 0;
+ params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
+ uint8_t *, sess->auth_iv_offset);
+ if (zsk_flag == K_F9) {
+ uint32_t length_in_bits, num_bytes;
+ uint8_t *src, direction = 0;
+ uint32_t counter_num_bytes;
+
+ memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
+ uint8_t *), 8);
+ /*
+ * This is kasumi f9, take direction from
+ * source buffer
+ */
+ length_in_bits = cop->sym->auth.data.length;
+ num_bytes = (length_in_bits >> 3);
+ counter_num_bytes = num_bytes;
+ src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
+ find_kasumif9_direction_and_length(src,
+ counter_num_bytes,
+ &length_in_bits,
+ &direction);
+ length_in_bits -= 64;
+ cop->sym->auth.data.offset += 64;
+ d_offs = cop->sym->auth.data.offset;
+ auth_range_off = d_offs / 8;
+ cop->sym->auth.data.length = length_in_bits;
+
+ /* Store it at end of auth iv */
+ iv_buf[8] = direction;
+ params.auth_iv_buf = iv_buf;
+ }
+ }
+
+ d_lens = sym_op->auth.data.length;
+
+ params.ctx_buf.vaddr = SESS_PRIV(sess);
+ params.ctx_buf.dma_addr = sess->ctx_dma_addr;
+
+ if (auth_op == CPT_OP_AUTH_GENERATE) {
+ if (sym_op->auth.digest.data) {
+ /*
+ * Digest to be generated
+ * in separate buffer
+ */
+ params.mac_buf.size =
+ sess->mac_len;
+ params.mac_buf.vaddr =
+ sym_op->auth.digest.data;
+ params.mac_buf.dma_addr =
+ sym_op->auth.digest.phys_addr;
+ } else {
+ uint32_t off = sym_op->auth.data.offset +
+ sym_op->auth.data.length;
+ int32_t dlen, space;
+
+ m_dst = sym_op->m_dst ?
+ sym_op->m_dst : sym_op->m_src;
+ dlen = rte_pktmbuf_pkt_len(m_dst);
+
+ space = off + mac_len - dlen;
+ if (space > 0)
+ if (!rte_pktmbuf_append(m_dst, space)) {
+ CPT_LOG_DP_ERR("Failed to extend "
+ "mbuf by %uB", space);
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
+
+ params.mac_buf.vaddr =
+ rte_pktmbuf_mtod_offset(m_dst, void *, off);
+ params.mac_buf.dma_addr =
+ rte_pktmbuf_mtophys_offset(m_dst, off);
+ params.mac_buf.size = mac_len;
+ }
+ } else {
+ /* Need space for storing generated mac */
+ params.mac_buf.vaddr = (uint8_t *)mdata + space;
+ params.mac_buf.dma_addr = mphys + space;
+ params.mac_buf.size = mac_len;
+ space += RTE_ALIGN_CEIL(mac_len, 8);
+ op[2] = (uintptr_t)params.mac_buf.vaddr;
+ op[3] = mac_len;
+ }
+
+ params.meta_buf.vaddr = (uint8_t *)mdata + space;
+ params.meta_buf.dma_addr = mphys + space;
+ params.meta_buf.size -= space;
+
+ /* Out of place processing */
+ params.src_iov = (void *)src;
+
+ /*Store SG I/O in the api for reuse */
+ if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
+ CPT_LOG_DP_ERR("Prepare src iov failed");
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
+
+ *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op);
+ if (unlikely(*prep_req == NULL)) {
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
+
+ *mdata_ptr = mdata;
+
+ return 0;
+
+free_mdata_and_exit:
+ free_op_meta(mdata, m_info->pool);
+err_exit:
+ return ret;
+}
+
#endif /*_CPT_UCODE_H_ */