seg->imm = imm;
}
-/**
- * Query LKey from a packet buffer for QP. If not found, add the mempool.
- *
- * @param priv
- * Pointer to the priv object.
- * @param mr_ctrl
- * Pointer to per-queue MR control structure.
- * @param mbuf
- * Pointer to source mbuf, to search in.
- *
- * @return
- * Searched LKey on success, UINT32_MAX on no match.
- */
-static inline uint32_t
-mlx5_regex_addr2mr(struct mlx5_regex_priv *priv, struct mlx5_mr_ctrl *mr_ctrl,
- struct rte_mbuf *mbuf)
-{
- uintptr_t addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
- uint32_t lkey;
-
- /* Check generation bit to see if there's any change on existing MRs. */
- if (unlikely(*mr_ctrl->dev_gen_ptr != mr_ctrl->cur_gen))
- mlx5_mr_flush_local_cache(mr_ctrl);
- /* Linear search on MR cache array. */
- lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru,
- MLX5_MR_CACHE_N, addr);
- if (likely(lkey != UINT32_MAX))
- return lkey;
- /* Take slower bottom-half on miss. */
- return mlx5_mr_addr2mr_bh(priv->pd, 0, &priv->mr_scache, mr_ctrl, addr,
- !!(mbuf->ol_flags & EXT_ATTACHED_MBUF));
-}
-
-
static inline void
__prep_one(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp_obj,
struct rte_regex_ops *op, struct mlx5_regex_job *job,
struct mlx5_klm klm;
klm.byte_count = rte_pktmbuf_data_len(op->mbuf);
- klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, op->mbuf);
+ klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, op->mbuf);
klm.address = rte_pktmbuf_mtod(op->mbuf, uintptr_t);
__prep_one(priv, qp_obj, op, job, qp_obj->pi, &klm);
qp_obj->db_pi = qp_obj->pi;
}
static inline void
-send_doorbell(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp_obj)
+send_doorbell(struct mlx5_regex_priv *priv, struct mlx5_regex_hw_qp *qp)
{
- struct mlx5dv_devx_uar *uar = priv->uar;
- size_t wqe_offset = (qp_obj->db_pi & (qp_size_get(qp_obj) - 1)) *
- (MLX5_SEND_WQE_BB << (priv->has_umr ? 2 : 0)) +
- (priv->has_umr ? MLX5_REGEX_UMR_WQE_SIZE : 0);
- uint8_t *wqe = (uint8_t *)(uintptr_t)qp_obj->qp_obj.wqes + wqe_offset;
+ size_t wqe_offset = (qp->db_pi & (qp_size_get(qp) - 1)) *
+ (MLX5_SEND_WQE_BB << (priv->has_umr ? 2 : 0)) +
+ (priv->has_umr ? MLX5_REGEX_UMR_WQE_SIZE : 0);
+ uint8_t *wqe = (uint8_t *)(uintptr_t)qp->qp_obj.wqes + wqe_offset;
+ uint32_t actual_pi = (priv->has_umr ? (qp->db_pi * 4 + 3) : qp->db_pi) &
+ MLX5_REGEX_MAX_WQE_INDEX;
+
/* Or the fm_ce_se instead of set, avoid the fence be cleared. */
((struct mlx5_wqe_ctrl_seg *)wqe)->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
- uint64_t *doorbell_addr =
- (uint64_t *)((uint8_t *)uar->base_addr + 0x800);
- rte_io_wmb();
- qp_obj->qp_obj.db_rec[MLX5_SND_DBR] = rte_cpu_to_be_32((priv->has_umr ?
- (qp_obj->db_pi * 4 + 3) : qp_obj->db_pi)
- & MLX5_REGEX_MAX_WQE_INDEX);
- rte_wmb();
- *doorbell_addr = *(volatile uint64_t *)wqe;
- rte_wmb();
+ mlx5_doorbell_ring(&priv->uar.bf_db, *(volatile uint64_t *)wqe,
+ actual_pi, &qp->qp_obj.db_rec[MLX5_SND_DBR],
+ !priv->uar.dbnc);
}
static inline int
uint32_t len = 0;
struct mlx5_klm *mkey_klm = NULL;
struct mlx5_klm klm;
+ uintptr_t addr;
while (left_ops--)
rte_prefetch0(op[left_ops]);
klm.mkey = rte_cpu_to_be_32
(qp->jobs[mkey_job_id].imkey->id);
while (mbuf) {
+ addr = rte_pktmbuf_mtod(mbuf, uintptr_t);
/* Build indirect mkey seg's KLM. */
- mkey_klm->mkey = mlx5_regex_addr2mr
- (priv, &qp->mr_ctrl, mbuf);
- mkey_klm->address = rte_cpu_to_be_64
- (rte_pktmbuf_mtod(mbuf, uintptr_t));
+ mkey_klm->mkey = mlx5_mr_mb2mr(&qp->mr_ctrl,
+ mbuf);
+ mkey_klm->address = rte_cpu_to_be_64(addr);
mkey_klm->byte_count = rte_cpu_to_be_32
(rte_pktmbuf_data_len(mbuf));
/*
klm.byte_count = scatter_size;
} else {
/* The single mubf case. Build the KLM directly. */
- klm.mkey = mlx5_regex_addr2mr(priv, &qp->mr_ctrl, mbuf);
+ klm.mkey = mlx5_mr_mb2mr(&qp->mr_ctrl, mbuf);
klm.address = rte_pktmbuf_mtod(mbuf, uintptr_t);
klm.byte_count = rte_pktmbuf_data_len(mbuf);
}
uint16_t wq_counter
= (rte_be_to_cpu_16(cqe->wqe_counter) + 1) &
MLX5_REGEX_MAX_WQE_INDEX;
- size_t hw_qpid = cqe->rsvd3[2];
+ size_t hw_qpid = cqe->user_index_bytes[2];
struct mlx5_regex_hw_qp *qp_obj = &queue->qps[hw_qpid];
/* UMR mode WQE counter move as WQE set(4 WQEBBS).*/
static int
setup_buffers(struct mlx5_regex_priv *priv, struct mlx5_regex_qp *qp)
{
- struct ibv_pd *pd = priv->pd;
+ struct ibv_pd *pd = priv->cdev->pd;
uint32_t i;
int err;
err = setup_buffers(priv, qp);
if (err) {
rte_free(qp->jobs);
+ qp->jobs = NULL;
return err;
}
if (priv->has_umr) {
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
- if (regex_get_pdn(priv->pd, &attr.pd)) {
- err = -rte_errno;
- DRV_LOG(ERR, "Failed to get pdn.");
- mlx5_regexdev_teardown_fastpath(priv, qp_id);
- return err;
- }
+ attr.pd = priv->cdev->pdn;
#endif
for (i = 0; i < qp->nb_desc; i++) {
attr.klm_num = MLX5_REGEX_MAX_KLM_NUM;
struct mlx5_regex_qp *qp = &priv->qps[qp_id];
uint32_t i;
- if (qp) {
+ if (qp->jobs) {
for (i = 0; i < qp->nb_desc; i++) {
if (qp->jobs[i].imkey)
claim_zero(mlx5_devx_cmd_destroy
(qp->jobs[i].imkey));
}
free_buffers(qp);
- if (qp->jobs)
- rte_free(qp->jobs);
+ rte_free(qp->jobs);
+ qp->jobs = NULL;
}
}