void
mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv)
{
- if (priv->uar) {
- mlx5_glue->devx_free_uar(priv->uar);
- priv->uar = NULL;
- }
+ mlx5_devx_uar_release(&priv->uar);
#ifdef HAVE_IBV_DEVX_EVENT
if (priv->eventc) {
mlx5_os_devx_destroy_event_channel(priv->eventc);
}
/* Prepare all the global resources for all the event objects.*/
-static int
+int
mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
{
- if (priv->eventc)
- return 0;
priv->eventc = mlx5_os_devx_create_event_channel(priv->cdev->ctx,
MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
if (!priv->eventc) {
rte_errno);
goto error;
}
- /*
- * This PMD always claims the write memory barrier on UAR
- * registers writings, it is safe to allocate UAR with any
- * memory mapping type.
- */
- priv->uar = mlx5_devx_alloc_uar(priv->cdev);
- if (!priv->uar) {
- rte_errno = errno;
+ if (mlx5_devx_uar_prepare(priv->cdev, &priv->uar) != 0) {
DRV_LOG(ERR, "Failed to allocate UAR.");
goto error;
}
uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci;
uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq_obj.cq->id;
uint64_t db_be = rte_cpu_to_be_64(doorbell);
- uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL);
-
- rte_io_wmb();
- cq->cq_obj.db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
- rte_wmb();
-#ifdef RTE_ARCH_64
- *(uint64_t *)addr = db_be;
-#else
- *(uint32_t *)addr = db_be;
- rte_io_wmb();
- *((uint32_t *)addr + 1) = db_be >> 32;
-#endif
+
+ mlx5_doorbell_ring(&priv->uar.cq_db, db_be, doorbell_hi,
+ &cq->cq_obj.db_rec[MLX5_CQ_ARM_DB], 0);
cq->arm_sn++;
cq->armed = 1;
}
{
struct mlx5_devx_cq_attr attr = {
.use_first_only = 1,
- .uar_page_id = priv->uar->page_id,
+ .uar_page_id = mlx5_os_get_devx_uar_page_id(priv->uar.obj),
};
uint16_t event_nums[1] = {0};
int ret;
};
uint32_t word;
} last_word;
- uint16_t next_wqe_counter = cq->cq_ci;
+ uint16_t next_wqe_counter = eqp->qp_pi;
uint16_t cur_wqe_counter;
uint16_t comp;
rte_io_wmb();
/* Ring CQ doorbell record. */
cq->cq_obj.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
+ eqp->qp_pi += comp;
rte_io_wmb();
/* Ring SW QP doorbell record. */
- eqp->sw_qp.db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cq_size);
+ eqp->sw_qp.db_rec[0] = rte_cpu_to_be_32(eqp->qp_pi + cq_size);
}
return comp;
}
return max;
}
+void
+mlx5_vdpa_drain_cq(struct mlx5_vdpa_priv *priv)
+{
+ unsigned int i;
+
+ for (i = 0; i < priv->caps.max_num_virtio_queues * 2; i++) {
+ struct mlx5_vdpa_cq *cq = &priv->virtqs[i].eqp.cq;
+
+ mlx5_vdpa_queue_complete(cq);
+ if (cq->cq_obj.cq) {
+ cq->cq_obj.cqes[0].wqe_counter =
+ rte_cpu_to_be_16(UINT16_MAX);
+ priv->virtqs[i].eqp.qp_pi = 0;
+ if (!cq->armed)
+ mlx5_vdpa_cq_arm(priv, cq);
+ }
+ }
+}
+
/* Wait on all CQs channel for completion event. */
static struct mlx5_vdpa_cq *
mlx5_vdpa_event_wait(struct mlx5_vdpa_priv *priv __rte_unused)
flags = fcntl(priv->err_chnl->fd, F_GETFL);
ret = fcntl(priv->err_chnl->fd, F_SETFL, flags | O_NONBLOCK);
if (ret) {
+ rte_errno = errno;
DRV_LOG(ERR, "Failed to change device event channel FD.");
goto error;
}
-
+ priv->err_intr_handle =
+ rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+ if (priv->err_intr_handle == NULL) {
+ DRV_LOG(ERR, "Fail to allocate intr_handle");
+ goto error;
+ }
if (rte_intr_fd_set(priv->err_intr_handle, priv->err_chnl->fd))
goto error;
if (rte_intr_type_set(priv->err_intr_handle, RTE_INTR_HANDLE_EXT))
goto error;
- if (rte_intr_callback_register(priv->err_intr_handle,
- mlx5_vdpa_err_interrupt_handler,
- priv)) {
+ ret = rte_intr_callback_register(priv->err_intr_handle,
+ mlx5_vdpa_err_interrupt_handler,
+ priv);
+ if (ret != 0) {
rte_intr_fd_set(priv->err_intr_handle, 0);
DRV_LOG(ERR, "Failed to register error interrupt for device %d.",
priv->vid);
+ rte_errno = -ret;
goto error;
} else {
DRV_LOG(DEBUG, "Registered error interrupt for device%d.",
mlx5_glue->devx_destroy_event_channel(priv->err_chnl);
priv->err_chnl = NULL;
}
+ rte_intr_instance_free(priv->err_intr_handle);
}
int
return 0;
}
+static int
+mlx5_vdpa_qps2rst2rts(struct mlx5_vdpa_event_qp *eqp)
+{
+ if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_QP_2RST,
+ eqp->sw_qp.qp->id)) {
+ DRV_LOG(ERR, "Failed to modify FW QP to RST state(%u).",
+ rte_errno);
+ return -1;
+ }
+ if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp.qp,
+ MLX5_CMD_OP_QP_2RST, eqp->fw_qp->id)) {
+ DRV_LOG(ERR, "Failed to modify SW QP to RST state(%u).",
+ rte_errno);
+ return -1;
+ }
+ return mlx5_vdpa_qps2rts(eqp);
+}
+
int
-mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
+mlx5_vdpa_event_qp_prepare(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
int callfd, struct mlx5_vdpa_event_qp *eqp)
{
struct mlx5_devx_qp_attr attr = {0};
uint16_t log_desc_n = rte_log2_u32(desc_n);
uint32_t ret;
- if (mlx5_vdpa_event_qp_global_prepare(priv))
- return -1;
+ if (eqp->cq.cq_obj.cq != NULL && log_desc_n == eqp->cq.log_desc_n) {
+ /* Reuse existing resources. */
+ eqp->cq.callfd = callfd;
+ /* FW will set event qp to error state in q destroy. */
+ if (!mlx5_vdpa_qps2rst2rts(eqp)) {
+ rte_write32(rte_cpu_to_be_32(RTE_BIT32(log_desc_n)),
+ &eqp->sw_qp.db_rec[0]);
+ return 0;
+ }
+ }
+ if (eqp->fw_qp)
+ mlx5_vdpa_event_qp_destroy(eqp);
if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
return -1;
attr.pd = priv->cdev->pdn;
DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);
goto error;
}
- attr.uar_index = priv->uar->page_id;
+ attr.uar_index = mlx5_os_get_devx_uar_page_id(priv->uar.obj);
attr.cqn = eqp->cq.cq_obj.cq->id;
- attr.rq_size = RTE_BIT32(log_desc_n);
+ attr.num_of_receive_wqes = RTE_BIT32(log_desc_n);
attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
- attr.sq_size = 0; /* No need SQ. */
+ attr.num_of_send_wqbbs = 0; /* No need SQ. */
attr.ts_format =
mlx5_ts_format_conv(priv->cdev->config.hca_attr.qp_ts_format);
- ret = mlx5_devx_qp_create(priv->cdev->ctx, &(eqp->sw_qp), log_desc_n,
- &attr, SOCKET_ID_ANY);
+ ret = mlx5_devx_qp_create(priv->cdev->ctx, &(eqp->sw_qp),
+ attr.num_of_receive_wqes *
+ MLX5_WSEG_SIZE, &attr, SOCKET_ID_ANY);
if (ret) {
DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno);
goto error;
}
if (mlx5_vdpa_qps2rts(eqp))
goto error;
+ eqp->qp_pi = 0;
/* First ringing. */
- rte_write32(rte_cpu_to_be_32(RTE_BIT32(log_desc_n)),
+ if (eqp->sw_qp.db_rec)
+ rte_write32(rte_cpu_to_be_32(RTE_BIT32(log_desc_n)),
&eqp->sw_qp.db_rec[0]);
return 0;
error:
git.droids-corp.org / dpdk.git / blobdiff