+ job->user_data = udata;
+}
+
+/**
+ * Process a crypto operation and complete a JOB_AES_HMAC job structure for
+ * submission to the multi buffer library for processing.
+ *
+ * @param qp queue pair
+ * @param job JOB_AES_HMAC structure to fill
+ * @param m mbuf to process
+ *
+ * @return
+ * - Completed JOB_AES_HMAC structure pointer on success
+ * - NULL pointer if completion of JOB_AES_HMAC structure isn't possible
+ */
+static inline int
+set_mb_job_params(JOB_AES_HMAC *job, struct aesni_mb_qp *qp,
+ struct rte_crypto_op *op, uint8_t *digest_idx)
+{
+ struct rte_mbuf *m_src = op->sym->m_src, *m_dst;
+ struct aesni_mb_session *session;
+ uint32_t m_offset, oop;
+
+ session = get_session(qp, op);
+ if (session == NULL) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
+ return -1;
+ }
+
+ /* Set crypto operation */
+ job->chain_order = session->chain_order;
+
+ /* Set cipher parameters */
+ job->cipher_direction = session->cipher.direction;
+ job->cipher_mode = session->cipher.mode;
+
+ job->aes_key_len_in_bytes = session->cipher.key_length_in_bytes;
+
+ /* Set authentication parameters */
+ job->hash_alg = session->auth.algo;
+
+ const int aead = is_aead_algo(job->hash_alg, job->cipher_mode);
+
+ switch (job->hash_alg) {
+ case AES_XCBC:
+ job->u.XCBC._k1_expanded = session->auth.xcbc.k1_expanded;
+ job->u.XCBC._k2 = session->auth.xcbc.k2;
+ job->u.XCBC._k3 = session->auth.xcbc.k3;
+
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CCM:
+ job->u.CCM.aad = op->sym->aead.aad.data + 18;
+ job->u.CCM.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_CMAC:
+ job->u.CMAC._key_expanded = session->auth.cmac.expkey;
+ job->u.CMAC._skey1 = session->auth.cmac.skey1;
+ job->u.CMAC._skey2 = session->auth.cmac.skey2;
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ break;
+
+ case AES_GMAC:
+ if (session->cipher.mode == GCM) {
+ job->u.GCM.aad = op->sym->aead.aad.data;
+ job->u.GCM.aad_len_in_bytes = session->aead.aad_len;
+ } else {
+ /* For GMAC */
+ job->u.GCM.aad = rte_pktmbuf_mtod_offset(m_src,
+ uint8_t *, op->sym->auth.data.offset);
+ job->u.GCM.aad_len_in_bytes = op->sym->auth.data.length;
+ job->cipher_mode = GCM;
+ }
+ job->aes_enc_key_expanded = &session->cipher.gcm_key;
+ job->aes_dec_key_expanded = &session->cipher.gcm_key;
+ break;
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_ZUC_EIA3_BITLEN:
+ job->u.ZUC_EIA3._key = session->auth.zuc_auth_key;
+ job->u.ZUC_EIA3._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_SNOW3G_UIA2_BITLEN:
+ job->u.SNOW3G_UIA2._key = (void *) &session->auth.pKeySched_snow3g_auth;
+ job->u.SNOW3G_UIA2._iv = rte_crypto_op_ctod_offset(op, uint8_t *,
+ session->auth_iv.offset);
+ break;
+ case IMB_AUTH_KASUMI_UIA1:
+ job->u.KASUMI_UIA1._key = (void *) &session->auth.pKeySched_kasumi_auth;
+ break;
+#endif
+#if IMB_VERSION(0, 54, 3) <= IMB_VERSION_NUM
+ case IMB_AUTH_CHACHA20_POLY1305:
+ job->u.CHACHA20_POLY1305.aad = op->sym->aead.aad.data;
+ job->u.CHACHA20_POLY1305.aad_len_in_bytes = session->aead.aad_len;
+ job->aes_enc_key_expanded = session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded = session->cipher.expanded_aes_keys.encode;
+ break;
+#endif
+ default:
+ job->u.HMAC._hashed_auth_key_xor_ipad = session->auth.pads.inner;
+ job->u.HMAC._hashed_auth_key_xor_opad = session->auth.pads.outer;
+
+ if (job->cipher_mode == DES3) {
+ job->aes_enc_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ job->aes_dec_key_expanded =
+ session->cipher.exp_3des_keys.ks_ptr;
+ } else {
+ job->aes_enc_key_expanded =
+ session->cipher.expanded_aes_keys.encode;
+ job->aes_dec_key_expanded =
+ session->cipher.expanded_aes_keys.decode;
+ }
+ }
+
+ if (aead)
+ m_offset = op->sym->aead.data.offset;
+ else
+ m_offset = op->sym->cipher.data.offset;
+
+#if IMB_VERSION(0, 53, 3) <= IMB_VERSION_NUM
+ if (job->cipher_mode == IMB_CIPHER_ZUC_EEA3) {
+ job->aes_enc_key_expanded = session->cipher.zuc_cipher_key;
+ job->aes_dec_key_expanded = session->cipher.zuc_cipher_key;
+ } else if (job->cipher_mode == IMB_CIPHER_SNOW3G_UEA2_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_snow3g_cipher;
+ m_offset = 0;
+ } else if (job->cipher_mode == IMB_CIPHER_KASUMI_UEA1_BITLEN) {
+ job->enc_keys = &session->cipher.pKeySched_kasumi_cipher;
+ m_offset = 0;
+ }
+#endif
+
+ if (!op->sym->m_dst) {
+ /* in-place operation */
+ m_dst = m_src;
+ oop = 0;
+ } else if (op->sym->m_dst == op->sym->m_src) {
+ /* in-place operation */
+ m_dst = m_src;
+ oop = 0;
+ } else {
+ /* out-of-place operation */
+ m_dst = op->sym->m_dst;
+ oop = 1;
+ }
+
+ /* Set digest output location */