+ return nb_enqueued;
+}
+
+/** Get multi buffer session */
+static inline struct aesni_mb_session *
+get_session(struct aesni_mb_qp *qp, struct rte_crypto_op *op)
+{
+ struct aesni_mb_session *sess = NULL;
+
+ if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
+ if (likely(op->sym->session != NULL))
+ sess = (struct aesni_mb_session *)
+ get_sym_session_private_data(
+ op->sym->session,
+ cryptodev_driver_id);
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ } else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ if (likely(op->sym->sec_session != NULL))
+ sess = (struct aesni_mb_session *)
+ get_sec_session_private_data(
+ op->sym->sec_session);
+#endif
+ } else {
+ void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp);
+ void *_sess_private_data = NULL;
+
+ if (_sess == NULL)
+ return NULL;
+
+ if (rte_mempool_get(qp->sess_mp_priv,
+ (void **)&_sess_private_data))
+ return NULL;
+
+ sess = (struct aesni_mb_session *)_sess_private_data;
+
+ if (unlikely(aesni_mb_set_session_parameters(qp->mb_mgr,
+ sess, op->sym->xform) != 0)) {
+ rte_mempool_put(qp->sess_mp, _sess);
+ rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
+ sess = NULL;
+ }
+ op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
+ set_sym_session_private_data(op->sym->session,
+ cryptodev_driver_id, _sess_private_data);
+ }
+
+ if (unlikely(sess == NULL))
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+
+ return sess;
+}
+
+static inline uint64_t
+auth_start_offset(struct rte_crypto_op *op, struct aesni_mb_session *session,
+ uint32_t oop)
+{
+ struct rte_mbuf *m_src, *m_dst;
+ uint8_t *p_src, *p_dst;
+ uintptr_t u_src, u_dst;
+ uint32_t cipher_end, auth_end;
+
+ /* Only cipher then hash needs special calculation. */
+ if (!oop || session->chain_order != CIPHER_HASH)
+ return op->sym->auth.data.offset;
+
+ m_src = op->sym->m_src;
+ m_dst = op->sym->m_dst;
+
+ p_src = rte_pktmbuf_mtod(m_src, uint8_t *);
+ p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *);
+ u_src = (uintptr_t)p_src;
+ u_dst = (uintptr_t)p_dst + op->sym->auth.data.offset;
+
+ /**
+ * Copy the content between cipher offset and auth offset for generating
+ * correct digest.
+ */
+ if (op->sym->cipher.data.offset > op->sym->auth.data.offset)
+ memcpy(p_dst + op->sym->auth.data.offset,
+ p_src + op->sym->auth.data.offset,
+ op->sym->cipher.data.offset -
+ op->sym->auth.data.offset);
+
+ /**
+ * Copy the content between (cipher offset + length) and (auth offset +
+ * length) for generating correct digest
+ */
+ cipher_end = op->sym->cipher.data.offset + op->sym->cipher.data.length;
+ auth_end = op->sym->auth.data.offset + op->sym->auth.data.length;
+ if (cipher_end < auth_end)
+ memcpy(p_dst + cipher_end, p_src + cipher_end,
+ auth_end - cipher_end);
+
+ /**
+ * Since intel-ipsec-mb only supports positive values,
+ * we need to deduct the correct offset between src and dst.
+ */
+
+ return u_src < u_dst ? (u_dst - u_src) :
+ (UINT64_MAX - u_src + u_dst + 1);
+}
+
+static inline void
+set_cpu_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_session *session,
+ union rte_crypto_sym_ofs sofs, void *buf, uint32_t len,
+ struct rte_crypto_va_iova_ptr *iv,
+ struct rte_crypto_va_iova_ptr *aad, void *digest, void *udata)
+{
+ /* Set crypto operation */
+ job->chain_order = session->chain_order;
+
+ /* Set cipher parameters */
+ job->cipher_direction = session->cipher.direction;
+ job->cipher_mode = session->cipher.mode;
+
+ job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+
+ /* Set authentication parameters */
+ job->hash_alg = session->auth.algo;
+ job->iv = iv->va;
+
+ switch (job->hash_alg) {
+ case AES_XCBC:
+ job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
+ job->u.XCBC._k2 = session->auth.xcbc.k2;
+ job->u.XCBC._k3 = session->auth.xcbc.k3;
+
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CCM:
+ job->u.CCM.aad = (uint8_t *)aad->va + 18;
+ job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ job->iv++;
+ break;
+
+ case AES_CMAC:
+ job->u.CMAC._key_expanded = session->auth.cmac.expkey;
+ job->u.CMAC._skey1 = session->auth.cmac.skey1;
+ job->u.CMAC._skey2 = session->auth.cmac.skey2;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_GMAC:
+ if (session->cipher.mode == GCM) {
+ job->u.GCM.aad = aad->va;
+ job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+ } else {
+ /* For GMAC */
+ job->u.GCM.aad = buf;
+ job->u.GCM.aad_len_in_bytes = len;
+ job->cipher_mode = GCM;
+ }
+ job->aes_enc_key_expanded = &session->cipher.gcm_key;
+ job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ break;
+
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->u.CHACHA20_POLY1305.aad = aad->va;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ break;
+#endif
+ default:
+ job->u.HMAC._hashed_auth_key_xor_ipad =
+ session->auth.pads.inner;
+ job->u.HMAC._hashed_auth_key_xor_opad =
+ session->auth.pads.outer;
+
+ if (job->cipher_mode == DES3) {
+ job->aes_enc_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ job->aes_dec_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ } else {
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ }
+ }
+
+ /*
+ * Multi-buffer library current only support returning a truncated
+ * digest length as specified in the relevant IPsec RFCs
+ */
+
+ /* Set digest location and length */
+ job->auth_tag_output = digest;
+ job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
+
+ /* Set IV parameters */
+ job->iv_len_in_bytes = session->iv.length;
+
+ /* Data Parameters */
+ job->src = buf;
+ job->dst = (uint8_t *)buf + sofs.ofs.cipher.head;
+ job->cipher_start_src_offset_in_bytes = sofs.ofs.cipher.head;
+ job->hash_start_src_offset_in_bytes = sofs.ofs.auth.head;
+ if (job->hash_alg == AES_GMAC && session->cipher.mode != GCM) {
+ job->msg_len_to_hash_in_bytes = 0;
+ job->msg_len_to_cipher_in_bytes = 0;
+ } else {
+ job->msg_len_to_hash_in_bytes = len - sofs.ofs.auth.head -
+ sofs.ofs.auth.tail;
+ job->msg_len_to_cipher_in_bytes = len - sofs.ofs.cipher.head -
+ sofs.ofs.cipher.tail;
+ }
+
+ job->user_data = udata;
+}
+
+/**
+ * Process a crypto operation and complete a JOB_AES_HMAC job structure for
+ * submission to the multi buffer library for processing.
+ *
+ * @param qp queue pair
+ * @param job JOB_AES_HMAC structure to fill
+ * @param m mbuf to process
+ *
+ * @return
+ * - Completed JOB_AES_HMAC structure pointer on success
+ * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
+ */
+static inline int
+set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+ struct rte_crypto_op *op, uint8_t *digest_idx)
+{
+ struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct aesni_mb_session *session;
+ uint32_t m_offset, oop;
+
+ session = get_session(qp, op);
+ if (session == NULL) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return -1;
+ }
+
+ /* Set crypto operation */
+ job->chain_order = session->chain_order;
+
+ /* Set cipher parameters */
+ job->cipher_direction = session->cipher.direction;
+ job->cipher_mode = session->cipher.mode;
+
+ job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+
+ /* Set authentication parameters */
+ job->hash_alg = session->auth.algo;
+
+ const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
+
+ switch (job->hash_alg) {
+ case AES_XCBC:
+ job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
+ job->u.XCBC._k2 = session->auth.xcbc.k2;
+ job->u.XCBC._k3 = session->auth.xcbc.k3;
+
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CCM:
+ job->u.CCM.aad = op->sym->aead.aad.data + 18;
+ job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CMAC:
+ job->u.CMAC._key_expanded = session->auth.cmac.expkey;
+ job->u.CMAC._skey1 = session->auth.cmac.skey1;
+ job->u.CMAC._skey2 = session->auth.cmac.skey2;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_GMAC:
+ if (session->cipher.mode == GCM) {
+ job->u.GCM.aad = op->sym->aead.aad.data;
+ job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+ } else {
+ /* For GMAC */
+ job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
+ uint8_t *, op->sym->auth.data.offset);
+ job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
+ job->cipher_mode = GCM;
+ }
+ job->aes_enc_key_expanded = &session->cipher.gcm_key;
+ job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ break;
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_ZUC_EIA3_BITLEN:
+ job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
+ job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_SNOW3G_UIA2_BITLEN:
+ job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
+ job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_KASUMI_UIA1:
+ job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
+ break;
+#endif
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ break;
+#endif
+ default:
+ job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
+ job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
+
+ if (job->cipher_mode == DES3) {
+ job->aes_enc_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ job->aes_dec_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ } else {
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ }
+ }
+
+ if (aead)
+ m_offset = op->sym->aead.data.offset;
+ else
+ m_offset = op->sym->cipher.data.offset;
+
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
+ job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
+ job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
+ } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
+ m_offset = 0;
+ } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
+ m_offset = 0;
+ }
+#endif
+
+ if (!op->sym->m_dst) {
+ /* in-place operation */
+ m_dst = m_src;
+ oop = 0;
+ } else if (op->sym->m_dst == op->sym->m_src) {
+ /* in-place operation */
+ m_dst = m_src;
+ oop = 0;
+ } else {
+ /* out-of-place operation */
+ m_dst = op->sym->m_dst;
+ oop = 1;
+ }
+
+ /* Set digest output location */
+ if (job->hash_alg != NULL_HASH &&
+ session->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
+ job->auth_tag_output = qp->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+ } else {
+ if (aead)
+ job->auth_tag_output = op->sym->aead.digest.data;
+ else
+ job->auth_tag_output = op->sym->auth.digest.data;
+
+ if (session->auth.req_digest_len != session->auth.gen_digest_len) {
+ job->auth_tag_output = qp->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+ }
+ }
+ /*
+ * Multi-buffer library current only support returning a truncated
+ * digest length as specified in the relevant IPsec RFCs
+ */
+
+ /* Set digest length */
+ job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
+
+ /* Set IV parameters */
+ job->iv_len_in_bytes = session->iv.length;
+
+ /* Data Parameters */
+ job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+ job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *, m_offset);
+
+ switch (job->hash_alg) {
+ case AES_CCM:
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->msg_len_to_cipher_in_bytes = op->sym->aead.data.length;
+ job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->msg_len_to_hash_in_bytes = op->sym->aead.data.length;
+
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset + 1);
+ break;
+
+ case AES_GMAC:
+ if (session->cipher.mode == GCM) {
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->aead.data.offset;
+ job->msg_len_to_cipher_in_bytes =
+ op->sym->aead.data.length;
+ job->msg_len_to_hash_in_bytes =
+ op->sym->aead.data.length;
+ } else {
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->auth.data.offset;
+ job->hash_start_src_offset_in_bytes =
+ op->sym->auth.data.offset;
+ job->msg_len_to_cipher_in_bytes = 0;
+ job->msg_len_to_hash_in_bytes = 0;
+ }
+
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset);
+ break;
+
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->cipher_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->hash_start_src_offset_in_bytes = op->sym->aead.data.offset;
+ job->msg_len_to_cipher_in_bytes =
+ op->sym->aead.data.length;
+ job->msg_len_to_hash_in_bytes =
+ op->sym->aead.data.length;
+
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset);
+ break;
+#endif
+ default:
+ /* For SNOW3G, length and offsets are already in bits */
+ job->cipher_start_src_offset_in_bytes =
+ op->sym->cipher.data.offset;
+ job->msg_len_to_cipher_in_bytes = op->sym->cipher.data.length;
+
+ job->hash_start_src_offset_in_bytes = auth_start_offset(op,
+ session, oop);
+ job->msg_len_to_hash_in_bytes = op->sym->auth.data.length;
+
+ job->iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->iv.offset);
+ }
+
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3)
+ job->msg_len_to_cipher_in_bytes >>= 3;
+ else if (job->hash_alg == IMB_AUTH_KASUMI_UIA1)
+ job->msg_len_to_hash_in_bytes >>= 3;
+#endif
+
+ /* Set user data to be crypto operation data struct */
+ job->user_data = op;
+
+ return 0;
+}
+
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+/**
+ * Process a crypto operation containing a security op and complete a
+ * JOB_AES_HMAC job structure for submission to the multi buffer library for
+ * processing.
+ */
+static inline int
+set_sec_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+ struct rte_crypto_op *op, uint8_t *digest_idx)
+{
+ struct rte_mbuf *m_src, *m_dst;
+ struct rte_crypto_sym_op *sym;
+ struct aesni_mb_session *session;
+
+ session = get_session(qp, op);
+ if (unlikely(session == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return -1;
+ }
+
+ /* Only DOCSIS protocol operations supported now */
+ if (session->cipher.mode != IMB_CIPHER_DOCSIS_SEC_BPI ||
+ session->auth.algo != IMB_AUTH_DOCSIS_CRC32) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return -1;
+ }
+
+ sym = op->sym;
+ m_src = sym->m_src;
+
+ if (likely(sym->m_dst == NULL || sym->m_dst == m_src)) {
+ /* in-place operation */
+ m_dst = m_src;
+ } else {
+ /* out-of-place operation not supported */
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return -ENOTSUP;
+ }
+
+ /* Set crypto operation */
+ job->chain_order = session->chain_order;
+
+ /* Set cipher parameters */
+ job->cipher_direction = session->cipher.direction;
+ job->cipher_mode = session->cipher.mode;
+
+ job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.decode;
+
+ /* Set IV parameters */
+ job->iv_len_in_bytes = session->iv.length;
+ job->iv = (uint8_t *)op + session->iv.offset;
+
+ /* Set authentication parameters */
+ job->hash_alg = session->auth.algo;
+
+ /* Set digest output location */
+ job->auth_tag_output = qp->temp_digests[*digest_idx];
+ *digest_idx = (*digest_idx + 1) % MAX_JOBS;
+
+ /* Set digest length */
+ job->auth_tag_output_len_in_bytes = session->auth.gen_digest_len;
+
+ /* Set data parameters */
+ job->src = rte_pktmbuf_mtod(m_src, uint8_t *);
+ job->dst = rte_pktmbuf_mtod_offset(m_dst, uint8_t *,
+ sym->cipher.data.offset);
+
+ job->cipher_start_src_offset_in_bytes = sym->cipher.data.offset;
+ job->msg_len_to_cipher_in_bytes = sym->cipher.data.length;
+
+ job->hash_start_src_offset_in_bytes = sym->auth.data.offset;
+ job->msg_len_to_hash_in_bytes = sym->auth.data.length;
+
+ job->user_data = op;
+
+ return 0;
+}
+
+static inline void
+verify_docsis_sec_crc(JOB_AES_HMAC *job, uint8_t *status)
+{
+ uint16_t crc_offset;
+ uint8_t *crc;
+
+ if (!job->msg_len_to_hash_in_bytes)
+ return;
+
+ crc_offset = job->hash_start_src_offset_in_bytes +
+ job->msg_len_to_hash_in_bytes -
+ job->cipher_start_src_offset_in_bytes;
+ crc = job->dst + crc_offset;
+
+ /* Verify CRC (at the end of the message) */
+ if (memcmp(job->auth_tag_output, crc, RTE_ETHER_CRC_LEN) != 0)
+ *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+}
+#endif
+
+static inline void
+verify_digest(JOB_AES_HMAC *job, void *digest, uint16_t len, uint8_t *status)
+{
+ /* Verify digest if required */
+ if (memcmp(job->auth_tag_output, digest, len) != 0)
+ *status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
+}
+
+static inline void
+generate_digest(JOB_AES_HMAC *job, struct rte_crypto_op *op,
+ struct aesni_mb_session *sess)
+{
+ /* No extra copy needed */
+ if (likely(sess->auth.req_digest_len == sess->auth.gen_digest_len))
+ return;
+
+ /*
+ * This can only happen for HMAC, so only digest
+ * for authentication algos is required
+ */
+ memcpy(op->sym->auth.digest.data, job->auth_tag_output,
+ sess->auth.req_digest_len);
+}
+
+/**
+ * Process a completed job and return rte_mbuf which job processed
+ *
+ * @param qp Queue Pair to process
+ * @param job JOB_AES_HMAC job to process
+ *
+ * @return
+ * - Returns processed crypto operation.
+ * - Returns NULL on invalid job
+ */
+static inline struct rte_crypto_op *
+post_process_mb_job(struct aesni_mb_qp *qp, JOB_AES_HMAC *job)
+{
+ struct rte_crypto_op *op = (struct rte_crypto_op *)job->user_data;
+ struct aesni_mb_session *sess = NULL;
+
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ uint8_t is_docsis_sec = 0;
+
+ if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) {
+ /*
+ * Assuming at this point that if it's a security type op, that
+ * this is for DOCSIS
+ */
+ is_docsis_sec = 1;
+ sess = get_sec_session_private_data(op->sym->sec_session);
+ } else
+#endif
+ {
+ sess = get_sym_session_private_data(op->sym->session,
+ cryptodev_driver_id);
+ }
+
+ if (unlikely(sess == NULL)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return op;
+ }
+
+ if (likely(op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)) {
+ switch (job->status) {
+ case STS_COMPLETED:
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+
+ if (job->hash_alg == NULL_HASH)
+ break;
+
+ if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
+ if (is_aead_algo(job->hash_alg, sess->cipher.mode))
+ verify_digest(job,
+ op->sym->aead.digest.data,
+ sess->auth.req_digest_len,
+ &op->status);
+#ifdef AESNI_MB_DOCSIS_SEC_ENABLED
+ else if (is_docsis_sec)
+ verify_docsis_sec_crc(job,
+ &op->status);
+#endif
+ else
+ verify_digest(job,
+ op->sym->auth.digest.data,
+ sess->auth.req_digest_len,
+ &op->status);
+ } else
+ generate_digest(job, op, sess);
+ break;
+ default:
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ }
+ }