+ while (remain_len) {
+ nb_segs--;
+ mbuf = mbuf->next;
+ if (unlikely(mbuf == NULL || nb_segs == 0)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ return 0;
+ }
+ if (unlikely(mlx5_crypto_klm_set(qp, mbuf, ++klm, 0,
+ &remain_len) == UINT32_MAX)) {
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ return 0;
+ }
+ klm_n++;
+ }
+ return klm_n;
+}
+
+static __rte_always_inline int
+mlx5_crypto_wqe_set(struct mlx5_crypto_priv *priv,
+ struct mlx5_crypto_qp *qp,
+ struct rte_crypto_op *op,
+ struct mlx5_umr_wqe *umr)
+{
+ struct mlx5_crypto_session *sess = get_sym_session_private_data
+ (op->sym->session, mlx5_crypto_driver_id);
+ struct mlx5_wqe_cseg *cseg = &umr->ctr;
+ struct mlx5_wqe_mkey_cseg *mkc = &umr->mkc;
+ struct mlx5_wqe_dseg *klms = &umr->kseg[0];
+ struct mlx5_wqe_umr_bsf_seg *bsf = ((struct mlx5_wqe_umr_bsf_seg *)
+ RTE_PTR_ADD(umr, priv->umr_wqe_size)) - 1;
+ uint32_t ds;
+ bool ipl = op->sym->m_dst == NULL || op->sym->m_dst == op->sym->m_src;
+ /* Set UMR WQE. */
+ uint32_t klm_n = mlx5_crypto_klms_set(qp, op,
+ ipl ? op->sym->m_src : op->sym->m_dst, klms);
+
+ if (unlikely(klm_n == 0))
+ return 0;
+ bsf->bs_bpt_eo_es = sess->bs_bpt_eo_es;
+ if (unlikely(!sess->bsp_res)) {
+ bsf->bsp_res = mlx5_crypto_get_block_size(op);
+ if (unlikely(bsf->bsp_res == UINT32_MAX)) {
+ op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
+ return 0;
+ }
+ } else {
+ bsf->bsp_res = sess->bsp_res;
+ }
+ bsf->raw_data_size = rte_cpu_to_be_32(op->sym->cipher.data.length);
+ memcpy(bsf->xts_initial_tweak,
+ rte_crypto_op_ctod_offset(op, uint8_t *, sess->iv_offset), 16);
+ bsf->res_dp = sess->dek_id;
+ mkc->len = rte_cpu_to_be_64(op->sym->cipher.data.length);
+ cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) | MLX5_OPCODE_UMR);
+ qp->db_pi += priv->umr_wqe_stride;
+ /* Set RDMA_WRITE WQE. */
+ cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+ klms = RTE_PTR_ADD(cseg, sizeof(struct mlx5_rdma_write_wqe));
+ if (!ipl) {
+ klm_n = mlx5_crypto_klms_set(qp, op, op->sym->m_src, klms);
+ if (unlikely(klm_n == 0))
+ return 0;
+ } else {
+ memcpy(klms, &umr->kseg[0], sizeof(*klms) * klm_n);
+ }
+ ds = 2 + klm_n;
+ cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
+ cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+ MLX5_OPCODE_RDMA_WRITE);
+ ds = RTE_ALIGN(ds, 4);
+ qp->db_pi += ds >> 2;
+ /* Set NOP WQE if needed. */
+ if (priv->max_rdmar_ds > ds) {
+ cseg += ds;
+ ds = priv->max_rdmar_ds - ds;
+ cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) | ds);
+ cseg->opcode = rte_cpu_to_be_32((qp->db_pi << 8) |
+ MLX5_OPCODE_NOP);
+ qp->db_pi += ds >> 2; /* Here, DS is 4 aligned for sure. */
+ }
+ qp->wqe = (uint8_t *)cseg;
+ return 1;
+}
+
+static uint16_t
+mlx5_crypto_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
+{
+ struct mlx5_crypto_qp *qp = queue_pair;
+ struct mlx5_crypto_priv *priv = qp->priv;
+ struct mlx5_umr_wqe *umr;
+ struct rte_crypto_op *op;
+ uint16_t mask = qp->entries_n - 1;
+ uint16_t remain = qp->entries_n - (qp->pi - qp->ci);
+ uint32_t idx;
+
+ if (remain < nb_ops)
+ nb_ops = remain;
+ else
+ remain = nb_ops;
+ if (unlikely(remain == 0))
+ return 0;
+ do {
+ idx = qp->pi & mask;
+ op = *ops++;
+ umr = RTE_PTR_ADD(qp->qp_obj.umem_buf,
+ priv->wqe_set_size * idx);
+ if (unlikely(mlx5_crypto_wqe_set(priv, qp, op, umr) == 0)) {
+ qp->stats.enqueue_err_count++;
+ if (remain != nb_ops) {
+ qp->stats.enqueued_count -= remain;
+ break;
+ }
+ return 0;
+ }
+ qp->ops[idx] = op;
+ qp->pi++;
+ } while (--remain);
+ qp->stats.enqueued_count += nb_ops;
+ mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)qp->wqe,
+ qp->db_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR],
+ !priv->uar.dbnc);
+ return nb_ops;
+}
+
+static __rte_noinline void
+mlx5_crypto_cqe_err_handle(struct mlx5_crypto_qp *qp, struct rte_crypto_op *op)
+{
+ const uint32_t idx = qp->ci & (qp->entries_n - 1);
+ volatile struct mlx5_err_cqe *cqe = (volatile struct mlx5_err_cqe *)
+ &qp->cq_obj.cqes[idx];
+
+ op->status = RTE_CRYPTO_OP_STATUS_ERROR;
+ qp->stats.dequeue_err_count++;
+ DRV_LOG(ERR, "CQE ERR:%x.\n", rte_be_to_cpu_32(cqe->syndrome));
+}
+
+static uint16_t
+mlx5_crypto_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
+ uint16_t nb_ops)
+{
+ struct mlx5_crypto_qp *qp = queue_pair;
+ volatile struct mlx5_cqe *restrict cqe;
+ struct rte_crypto_op *restrict op;
+ const unsigned int cq_size = qp->entries_n;
+ const unsigned int mask = cq_size - 1;
+ uint32_t idx;
+ uint32_t next_idx = qp->ci & mask;
+ const uint16_t max = RTE_MIN((uint16_t)(qp->pi - qp->ci), nb_ops);
+ uint16_t i = 0;
+ int ret;
+
+ if (unlikely(max == 0))
+ return 0;
+ do {
+ idx = next_idx;
+ next_idx = (qp->ci + 1) & mask;
+ op = qp->ops[idx];
+ cqe = &qp->cq_obj.cqes[idx];
+ ret = check_cqe(cqe, cq_size, qp->ci);
+ rte_io_rmb();
+ if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
+ if (unlikely(ret != MLX5_CQE_STATUS_HW_OWN))
+ mlx5_crypto_cqe_err_handle(qp, op);
+ break;
+ }
+ op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
+ ops[i++] = op;
+ qp->ci++;
+ } while (i < max);
+ if (likely(i != 0)) {
+ rte_io_wmb();
+ qp->cq_obj.db_rec[0] = rte_cpu_to_be_32(qp->ci);
+ qp->stats.dequeued_count += i;
+ }
+ return i;
+}
+
+static void
+mlx5_crypto_qp_init(struct mlx5_crypto_priv *priv, struct mlx5_crypto_qp *qp)
+{
+ uint32_t i;
+
+ for (i = 0 ; i < qp->entries_n; i++) {
+ struct mlx5_wqe_cseg *cseg = RTE_PTR_ADD(qp->qp_obj.umem_buf,
+ i * priv->wqe_set_size);
+ struct mlx5_wqe_umr_cseg *ucseg = (struct mlx5_wqe_umr_cseg *)
+ (cseg + 1);
+ struct mlx5_wqe_umr_bsf_seg *bsf =
+ (struct mlx5_wqe_umr_bsf_seg *)(RTE_PTR_ADD(cseg,
+ priv->umr_wqe_size)) - 1;
+ struct mlx5_wqe_rseg *rseg;
+
+ /* Init UMR WQE. */
+ cseg->sq_ds = rte_cpu_to_be_32((qp->qp_obj.qp->id << 8) |
+ (priv->umr_wqe_size / MLX5_WSEG_SIZE));
+ cseg->flags = RTE_BE32(MLX5_COMP_ONLY_FIRST_ERR <<
+ MLX5_COMP_MODE_OFFSET);
+ cseg->misc = rte_cpu_to_be_32(qp->mkey[i]->id);
+ ucseg->if_cf_toe_cq_res = RTE_BE32(1u << MLX5_UMRC_IF_OFFSET);
+ ucseg->mkey_mask = RTE_BE64(1u << 0); /* Mkey length bit. */
+ ucseg->ko_to_bs = rte_cpu_to_be_32
+ ((MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size) <<
+ MLX5_UMRC_KO_OFFSET) | (4 << MLX5_UMRC_TO_BS_OFFSET));
+ bsf->keytag = priv->keytag;
+ /* Init RDMA WRITE WQE. */
+ cseg = RTE_PTR_ADD(cseg, priv->umr_wqe_size);
+ cseg->flags = RTE_BE32((MLX5_COMP_ALWAYS <<
+ MLX5_COMP_MODE_OFFSET) |
+ MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE);
+ rseg = (struct mlx5_wqe_rseg *)(cseg + 1);
+ rseg->rkey = rte_cpu_to_be_32(qp->mkey[i]->id);
+ }
+}
+
+static int
+mlx5_crypto_indirect_mkeys_prepare(struct mlx5_crypto_priv *priv,
+ struct mlx5_crypto_qp *qp)
+{
+ struct mlx5_umr_wqe *umr;
+ uint32_t i;
+ struct mlx5_devx_mkey_attr attr = {
+ .pd = priv->cdev->pdn,
+ .umr_en = 1,
+ .crypto_en = 1,
+ .set_remote_rw = 1,
+ .klm_num = MLX5_CRYPTO_KLM_SEGS_NUM(priv->umr_wqe_size),
+ };
+
+ for (umr = (struct mlx5_umr_wqe *)qp->qp_obj.umem_buf, i = 0;
+ i < qp->entries_n; i++, umr = RTE_PTR_ADD(umr, priv->wqe_set_size)) {
+ attr.klm_array = (struct mlx5_klm *)&umr->kseg[0];
+ qp->mkey[i] = mlx5_devx_cmd_mkey_create(priv->cdev->ctx, &attr);
+ if (!qp->mkey[i])
+ goto error;