+static __rte_always_inline void
+compl_auth_verify(struct rte_crypto_op *op, uint8_t *gen_mac, uint64_t mac_len)
+{
+ uint8_t *mac;
+ struct rte_crypto_sym_op *sym_op = op->sym;
+
+ if (sym_op->auth.digest.data)
+ mac = sym_op->auth.digest.data;
+ else
+ mac = rte_pktmbuf_mtod_offset(sym_op->m_src, uint8_t *,
+ sym_op->auth.data.length +
+ sym_op->auth.data.offset);
+ if (!mac) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return;
+ }
+
+ if (memcmp(mac, gen_mac, mac_len))
+ op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+ else
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+}
+
+static __rte_always_inline void
+find_kasumif9_direction_and_length(uint8_t *src, uint32_t counter_num_bytes,
+ uint32_t *addr_length_in_bits,
+ uint8_t *addr_direction)
+{
+ uint8_t found = 0;
+ uint32_t pos;
+ uint8_t last_byte;
+ while (!found && counter_num_bytes > 0) {
+ counter_num_bytes--;
+ if (src[counter_num_bytes] == 0x00)
+ continue;
+ pos = rte_bsf32(src[counter_num_bytes]);
+ if (pos == 7) {
+ if (likely(counter_num_bytes > 0)) {
+ last_byte = src[counter_num_bytes - 1];
+ *addr_direction = last_byte & 0x1;
+ *addr_length_in_bits =
+ counter_num_bytes * 8 - 1;
+ }
+ } else {
+ last_byte = src[counter_num_bytes];
+ *addr_direction = (last_byte >> (pos + 1)) & 0x1;
+ *addr_length_in_bits =
+ counter_num_bytes * 8 + (8 - (pos + 2));
+ }
+ found = 1;
+ }
+}
+
+/*
+ * This handles all auth only except AES_GMAC
+ */
+static __rte_always_inline int
+fill_digest_params(struct rte_crypto_op *cop, struct cnxk_se_sess *sess,
+ struct cpt_qp_meta_info *m_info,
+ struct cpt_inflight_req *infl_req, struct cpt_inst_s *inst)
+{
+ uint32_t space = 0;
+ struct rte_crypto_sym_op *sym_op = cop->sym;
+ void *mdata;
+ uint32_t auth_range_off;
+ uint32_t flags = 0;
+ uint64_t d_offs = 0, d_lens;
+ struct rte_mbuf *m_src, *m_dst;
+ uint16_t auth_op = sess->cpt_op & ROC_SE_OP_AUTH_MASK;
+ uint16_t mac_len = sess->mac_len;
+ struct roc_se_fc_params params;
+ char src[SRC_IOV_SIZE];
+ uint8_t iv_buf[16];
+ int ret;
+
+ memset(¶ms, 0, sizeof(struct roc_se_fc_params));
+
+ m_src = sym_op->m_src;
+
+ mdata = alloc_op_meta(¶ms.meta_buf, m_info->mlen, m_info->pool,
+ infl_req);
+ if (mdata == NULL) {
+ ret = -ENOMEM;
+ goto err_exit;
+ }
+
+ auth_range_off = sym_op->auth.data.offset;
+
+ flags = ROC_SE_VALID_MAC_BUF;
+ params.src_iov = (void *)src;
+ if (unlikely(sess->zsk_flag)) {
+ /*
+ * Since for Zuc, Kasumi, Snow3g offsets are in bits
+ * we will send pass through even for auth only case,
+ * let MC handle it
+ */
+ d_offs = auth_range_off;
+ auth_range_off = 0;
+ params.auth_iv_len = sess->auth_iv_length;
+ params.auth_iv_buf = rte_crypto_op_ctod_offset(
+ cop, uint8_t *, sess->auth_iv_offset);
+ if (sess->zsk_flag == ROC_SE_K_F9) {
+ uint32_t length_in_bits, num_bytes;
+ uint8_t *src, direction = 0;
+
+ memcpy(iv_buf,
+ rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *), 8);
+ /*
+ * This is kasumi f9, take direction from
+ * source buffer
+ */
+ length_in_bits = cop->sym->auth.data.length;
+ num_bytes = (length_in_bits >> 3);
+ src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
+ find_kasumif9_direction_and_length(
+ src, num_bytes, &length_in_bits, &direction);
+ length_in_bits -= 64;
+ cop->sym->auth.data.offset += 64;
+ d_offs = cop->sym->auth.data.offset;
+ auth_range_off = d_offs / 8;
+ cop->sym->auth.data.length = length_in_bits;
+
+ /* Store it at end of auth iv */
+ iv_buf[8] = direction;
+ params.auth_iv_buf = iv_buf;
+ }
+ }
+
+ d_lens = sym_op->auth.data.length;
+
+ params.ctx_buf.vaddr = &sess->roc_se_ctx;
+
+ if (auth_op == ROC_SE_OP_AUTH_GENERATE) {
+ if (sym_op->auth.digest.data) {
+ /*
+ * Digest to be generated
+ * in separate buffer
+ */
+ params.mac_buf.size = sess->mac_len;
+ params.mac_buf.vaddr = sym_op->auth.digest.data;
+ } else {
+ uint32_t off = sym_op->auth.data.offset +
+ sym_op->auth.data.length;
+ int32_t dlen, space;
+
+ m_dst = sym_op->m_dst ? sym_op->m_dst : sym_op->m_src;
+ dlen = rte_pktmbuf_pkt_len(m_dst);
+
+ space = off + mac_len - dlen;
+ if (space > 0)
+ if (!rte_pktmbuf_append(m_dst, space)) {
+ plt_dp_err("Failed to extend "
+ "mbuf by %uB",
+ space);
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
+
+ params.mac_buf.vaddr =
+ rte_pktmbuf_mtod_offset(m_dst, void *, off);
+ params.mac_buf.size = mac_len;
+ }
+ } else {
+ uint64_t *op = mdata;
+
+ /* Need space for storing generated mac */
+ space += 2 * sizeof(uint64_t);
+
+ params.mac_buf.vaddr = (uint8_t *)mdata + space;
+ params.mac_buf.size = mac_len;
+ space += RTE_ALIGN_CEIL(mac_len, 8);
+ op[0] = (uintptr_t)params.mac_buf.vaddr;
+ op[1] = mac_len;
+ infl_req->op_flags |= CPT_OP_FLAGS_AUTH_VERIFY;
+ }
+
+ params.meta_buf.vaddr = (uint8_t *)mdata + space;
+ params.meta_buf.size -= space;
+
+ /* Out of place processing */
+ params.src_iov = (void *)src;
+
+ /*Store SG I/O in the api for reuse */
+ if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
+ plt_dp_err("Prepare src iov failed");
+ ret = -EINVAL;
+ goto free_mdata_and_exit;
+ }
+
+ ret = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, inst);
+ if (ret)
+ goto free_mdata_and_exit;
+
+ return 0;
+
+free_mdata_and_exit:
+ if (infl_req->op_flags & CPT_OP_FLAGS_METABUF)
+ rte_mempool_put(m_info->pool, infl_req->mdata);
+err_exit:
+ return ret;
+}