vdpa/mlx5: fix completion queue arming
[dpdk.git] / drivers / vdpa / mlx5 / mlx5_vdpa_event.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2019 Mellanox Technologies, Ltd
3  */
4 #include <unistd.h>
5 #include <stdint.h>
6 #include <fcntl.h>
7
8 #include <rte_malloc.h>
9 #include <rte_errno.h>
10 #include <rte_lcore.h>
11 #include <rte_atomic.h>
12 #include <rte_common.h>
13 #include <rte_io.h>
14
15 #include <mlx5_common.h>
16
17 #include "mlx5_vdpa_utils.h"
18 #include "mlx5_vdpa.h"
19
20
21 void
22 mlx5_vdpa_event_qp_global_release(struct mlx5_vdpa_priv *priv)
23 {
24         if (priv->uar) {
25                 mlx5_glue->devx_free_uar(priv->uar);
26                 priv->uar = NULL;
27         }
28         if (priv->eventc) {
29                 mlx5_glue->devx_destroy_event_channel(priv->eventc);
30                 priv->eventc = NULL;
31         }
32         priv->eqn = 0;
33 }
34
35 /* Prepare all the global resources for all the event objects.*/
36 static int
37 mlx5_vdpa_event_qp_global_prepare(struct mlx5_vdpa_priv *priv)
38 {
39         uint32_t lcore;
40
41         if (priv->eventc)
42                 return 0;
43         lcore = (uint32_t)rte_lcore_to_cpu_id(-1);
44         if (mlx5_glue->devx_query_eqn(priv->ctx, lcore, &priv->eqn)) {
45                 rte_errno = errno;
46                 DRV_LOG(ERR, "Failed to query EQ number %d.", rte_errno);
47                 return -1;
48         }
49         priv->eventc = mlx5_glue->devx_create_event_channel(priv->ctx,
50                            MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA);
51         if (!priv->eventc) {
52                 rte_errno = errno;
53                 DRV_LOG(ERR, "Failed to create event channel %d.",
54                         rte_errno);
55                 goto error;
56         }
57         priv->uar = mlx5_glue->devx_alloc_uar(priv->ctx, 0);
58         if (!priv->uar) {
59                 rte_errno = errno;
60                 DRV_LOG(ERR, "Failed to allocate UAR.");
61                 goto error;
62         }
63         return 0;
64 error:
65         mlx5_vdpa_event_qp_global_release(priv);
66         return -1;
67 }
68
69 static void
70 mlx5_vdpa_cq_destroy(struct mlx5_vdpa_cq *cq)
71 {
72         if (cq->cq)
73                 claim_zero(mlx5_devx_cmd_destroy(cq->cq));
74         if (cq->umem_obj)
75                 claim_zero(mlx5_glue->devx_umem_dereg(cq->umem_obj));
76         if (cq->umem_buf)
77                 rte_free((void *)(uintptr_t)cq->umem_buf);
78         memset(cq, 0, sizeof(*cq));
79 }
80
81 static inline void
82 mlx5_vdpa_cq_arm(struct mlx5_vdpa_priv *priv, struct mlx5_vdpa_cq *cq)
83 {
84         uint32_t arm_sn = cq->arm_sn << MLX5_CQ_SQN_OFFSET;
85         uint32_t cq_ci = cq->cq_ci & MLX5_CI_MASK;
86         uint32_t doorbell_hi = arm_sn | MLX5_CQ_DBR_CMD_ALL | cq_ci;
87         uint64_t doorbell = ((uint64_t)doorbell_hi << 32) | cq->cq->id;
88         uint64_t db_be = rte_cpu_to_be_64(doorbell);
89         uint32_t *addr = RTE_PTR_ADD(priv->uar->base_addr, MLX5_CQ_DOORBELL);
90
91         rte_io_wmb();
92         cq->db_rec[MLX5_CQ_ARM_DB] = rte_cpu_to_be_32(doorbell_hi);
93         rte_wmb();
94 #ifdef RTE_ARCH_64
95         *(uint64_t *)addr = db_be;
96 #else
97         *(uint32_t *)addr = db_be;
98         rte_io_wmb();
99         *((uint32_t *)addr + 1) = db_be >> 32;
100 #endif
101         cq->arm_sn++;
102 }
103
104 static int
105 mlx5_vdpa_cq_create(struct mlx5_vdpa_priv *priv, uint16_t log_desc_n,
106                     int callfd, struct mlx5_vdpa_cq *cq)
107 {
108         struct mlx5_devx_cq_attr attr;
109         size_t pgsize = sysconf(_SC_PAGESIZE);
110         uint32_t umem_size;
111         int ret;
112         uint16_t event_nums[1] = {0};
113
114         cq->log_desc_n = log_desc_n;
115         umem_size = sizeof(struct mlx5_cqe) * (1 << log_desc_n) +
116                                                         sizeof(*cq->db_rec) * 2;
117         cq->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
118         if (!cq->umem_buf) {
119                 DRV_LOG(ERR, "Failed to allocate memory for CQ.");
120                 rte_errno = ENOMEM;
121                 return -ENOMEM;
122         }
123         cq->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
124                                                 (void *)(uintptr_t)cq->umem_buf,
125                                                 umem_size,
126                                                 IBV_ACCESS_LOCAL_WRITE);
127         if (!cq->umem_obj) {
128                 DRV_LOG(ERR, "Failed to register umem for CQ.");
129                 goto error;
130         }
131         attr.q_umem_valid = 1;
132         attr.db_umem_valid = 1;
133         attr.use_first_only = 0;
134         attr.overrun_ignore = 0;
135         attr.uar_page_id = priv->uar->page_id;
136         attr.q_umem_id = cq->umem_obj->umem_id;
137         attr.q_umem_offset = 0;
138         attr.db_umem_id = cq->umem_obj->umem_id;
139         attr.db_umem_offset = sizeof(struct mlx5_cqe) * (1 << log_desc_n);
140         attr.eqn = priv->eqn;
141         attr.log_cq_size = log_desc_n;
142         attr.log_page_size = rte_log2_u32(pgsize);
143         cq->cq = mlx5_devx_cmd_create_cq(priv->ctx, &attr);
144         if (!cq->cq)
145                 goto error;
146         cq->db_rec = RTE_PTR_ADD(cq->umem_buf, (uintptr_t)attr.db_umem_offset);
147         cq->cq_ci = 0;
148         rte_spinlock_init(&cq->sl);
149         /* Subscribe CQ event to the event channel controlled by the driver. */
150         ret = mlx5_glue->devx_subscribe_devx_event(priv->eventc, cq->cq->obj,
151                                                    sizeof(event_nums),
152                                                    event_nums,
153                                                    (uint64_t)(uintptr_t)cq);
154         if (ret) {
155                 DRV_LOG(ERR, "Failed to subscribe CQE event.");
156                 rte_errno = errno;
157                 goto error;
158         }
159         /* Subscribe CQ event to the guest FD only if it is not in poll mode. */
160         if (callfd != -1) {
161                 ret = mlx5_glue->devx_subscribe_devx_event_fd(priv->eventc,
162                                                               callfd,
163                                                               cq->cq->obj, 0);
164                 if (ret) {
165                         DRV_LOG(ERR, "Failed to subscribe CQE event fd.");
166                         rte_errno = errno;
167                         goto error;
168                 }
169         }
170         /* First arming. */
171         mlx5_vdpa_cq_arm(priv, cq);
172         return 0;
173 error:
174         mlx5_vdpa_cq_destroy(cq);
175         return -1;
176 }
177
178 static inline void __rte_unused
179 mlx5_vdpa_cq_poll(struct mlx5_vdpa_priv *priv __rte_unused,
180                   struct mlx5_vdpa_cq *cq)
181 {
182         struct mlx5_vdpa_event_qp *eqp =
183                                 container_of(cq, struct mlx5_vdpa_event_qp, cq);
184         const unsigned int cq_size = 1 << cq->log_desc_n;
185         const unsigned int cq_mask = cq_size - 1;
186         int ret;
187
188         do {
189                 volatile struct mlx5_cqe *cqe = cq->cqes + (cq->cq_ci &
190                                                             cq_mask);
191
192                 ret = check_cqe(cqe, cq_size, cq->cq_ci);
193                 switch (ret) {
194                 case MLX5_CQE_STATUS_ERR:
195                         cq->errors++;
196                         /*fall-through*/
197                 case MLX5_CQE_STATUS_SW_OWN:
198                         cq->cq_ci++;
199                         break;
200                 case MLX5_CQE_STATUS_HW_OWN:
201                 default:
202                         break;
203                 }
204         } while (ret != MLX5_CQE_STATUS_HW_OWN);
205         rte_io_wmb();
206         /* Ring CQ doorbell record. */
207         cq->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci);
208         rte_io_wmb();
209         /* Ring SW QP doorbell record. */
210         eqp->db_rec[0] = rte_cpu_to_be_32(cq->cq_ci + cq_size);
211 }
212
213 static void
214 mlx5_vdpa_interrupt_handler(void *cb_arg)
215 {
216 #ifndef HAVE_IBV_DEVX_EVENT
217         (void)cb_arg;
218         return;
219 #else
220         struct mlx5_vdpa_priv *priv = cb_arg;
221         union {
222                 struct mlx5dv_devx_async_event_hdr event_resp;
223                 uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr) + 128];
224         } out;
225
226         while (mlx5_glue->devx_get_event(priv->eventc, &out.event_resp,
227                                          sizeof(out.buf)) >=
228                                        (ssize_t)sizeof(out.event_resp.cookie)) {
229                 struct mlx5_vdpa_cq *cq = (struct mlx5_vdpa_cq *)
230                                                (uintptr_t)out.event_resp.cookie;
231                 rte_spinlock_lock(&cq->sl);
232                 mlx5_vdpa_cq_poll(priv, cq);
233                 mlx5_vdpa_cq_arm(priv, cq);
234                 rte_spinlock_unlock(&cq->sl);
235                 DRV_LOG(DEBUG, "CQ %d event: new cq_ci = %u.", cq->cq->id,
236                         cq->cq_ci);
237         }
238 #endif /* HAVE_IBV_DEVX_ASYNC */
239 }
240
241 int
242 mlx5_vdpa_cqe_event_setup(struct mlx5_vdpa_priv *priv)
243 {
244         int flags = fcntl(priv->eventc->fd, F_GETFL);
245         int ret = fcntl(priv->eventc->fd, F_SETFL, flags | O_NONBLOCK);
246         if (ret) {
247                 DRV_LOG(ERR, "Failed to change event channel FD.");
248                 rte_errno = errno;
249                 return -rte_errno;
250         }
251         priv->intr_handle.fd = priv->eventc->fd;
252         priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
253         if (rte_intr_callback_register(&priv->intr_handle,
254                                        mlx5_vdpa_interrupt_handler, priv)) {
255                 priv->intr_handle.fd = 0;
256                 DRV_LOG(ERR, "Failed to register CQE interrupt %d.", rte_errno);
257                 return -rte_errno;
258         }
259         return 0;
260 }
261
262 void
263 mlx5_vdpa_cqe_event_unset(struct mlx5_vdpa_priv *priv)
264 {
265         int retries = MLX5_VDPA_INTR_RETRIES;
266         int ret = -EAGAIN;
267
268         if (priv->intr_handle.fd) {
269                 while (retries-- && ret == -EAGAIN) {
270                         ret = rte_intr_callback_unregister(&priv->intr_handle,
271                                                     mlx5_vdpa_interrupt_handler,
272                                                     priv);
273                         if (ret == -EAGAIN) {
274                                 DRV_LOG(DEBUG, "Try again to unregister fd %d "
275                                         "of CQ interrupt, retries = %d.",
276                                         priv->intr_handle.fd, retries);
277                                 usleep(MLX5_VDPA_INTR_RETRIES_USEC);
278                         }
279                 }
280                 memset(&priv->intr_handle, 0, sizeof(priv->intr_handle));
281         }
282 }
283
284 void
285 mlx5_vdpa_event_qp_destroy(struct mlx5_vdpa_event_qp *eqp)
286 {
287         if (eqp->sw_qp)
288                 claim_zero(mlx5_devx_cmd_destroy(eqp->sw_qp));
289         if (eqp->umem_obj)
290                 claim_zero(mlx5_glue->devx_umem_dereg(eqp->umem_obj));
291         if (eqp->umem_buf)
292                 rte_free(eqp->umem_buf);
293         if (eqp->fw_qp)
294                 claim_zero(mlx5_devx_cmd_destroy(eqp->fw_qp));
295         mlx5_vdpa_cq_destroy(&eqp->cq);
296         memset(eqp, 0, sizeof(*eqp));
297 }
298
299 static int
300 mlx5_vdpa_qps2rts(struct mlx5_vdpa_event_qp *eqp)
301 {
302         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RST2INIT_QP,
303                                           eqp->sw_qp->id)) {
304                 DRV_LOG(ERR, "Failed to modify FW QP to INIT state(%u).",
305                         rte_errno);
306                 return -1;
307         }
308         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RST2INIT_QP,
309                                           eqp->fw_qp->id)) {
310                 DRV_LOG(ERR, "Failed to modify SW QP to INIT state(%u).",
311                         rte_errno);
312                 return -1;
313         }
314         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_INIT2RTR_QP,
315                                           eqp->sw_qp->id)) {
316                 DRV_LOG(ERR, "Failed to modify FW QP to RTR state(%u).",
317                         rte_errno);
318                 return -1;
319         }
320         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_INIT2RTR_QP,
321                                           eqp->fw_qp->id)) {
322                 DRV_LOG(ERR, "Failed to modify SW QP to RTR state(%u).",
323                         rte_errno);
324                 return -1;
325         }
326         if (mlx5_devx_cmd_modify_qp_state(eqp->fw_qp, MLX5_CMD_OP_RTR2RTS_QP,
327                                           eqp->sw_qp->id)) {
328                 DRV_LOG(ERR, "Failed to modify FW QP to RTS state(%u).",
329                         rte_errno);
330                 return -1;
331         }
332         if (mlx5_devx_cmd_modify_qp_state(eqp->sw_qp, MLX5_CMD_OP_RTR2RTS_QP,
333                                           eqp->fw_qp->id)) {
334                 DRV_LOG(ERR, "Failed to modify SW QP to RTS state(%u).",
335                         rte_errno);
336                 return -1;
337         }
338         return 0;
339 }
340
341 int
342 mlx5_vdpa_event_qp_create(struct mlx5_vdpa_priv *priv, uint16_t desc_n,
343                           int callfd, struct mlx5_vdpa_event_qp *eqp)
344 {
345         struct mlx5_devx_qp_attr attr = {0};
346         uint16_t log_desc_n = rte_log2_u32(desc_n);
347         uint32_t umem_size = (1 << log_desc_n) * MLX5_WSEG_SIZE +
348                                                        sizeof(*eqp->db_rec) * 2;
349
350         if (mlx5_vdpa_event_qp_global_prepare(priv))
351                 return -1;
352         if (mlx5_vdpa_cq_create(priv, log_desc_n, callfd, &eqp->cq))
353                 return -1;
354         attr.pd = priv->pdn;
355         eqp->fw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
356         if (!eqp->fw_qp) {
357                 DRV_LOG(ERR, "Failed to create FW QP(%u).", rte_errno);
358                 goto error;
359         }
360         eqp->umem_buf = rte_zmalloc(__func__, umem_size, 4096);
361         if (!eqp->umem_buf) {
362                 DRV_LOG(ERR, "Failed to allocate memory for SW QP.");
363                 rte_errno = ENOMEM;
364                 goto error;
365         }
366         eqp->umem_obj = mlx5_glue->devx_umem_reg(priv->ctx,
367                                                (void *)(uintptr_t)eqp->umem_buf,
368                                                umem_size,
369                                                IBV_ACCESS_LOCAL_WRITE);
370         if (!eqp->umem_obj) {
371                 DRV_LOG(ERR, "Failed to register umem for SW QP.");
372                 goto error;
373         }
374         attr.uar_index = priv->uar->page_id;
375         attr.cqn = eqp->cq.cq->id;
376         attr.log_page_size = rte_log2_u32(sysconf(_SC_PAGESIZE));
377         attr.rq_size = 1 << log_desc_n;
378         attr.log_rq_stride = rte_log2_u32(MLX5_WSEG_SIZE);
379         attr.sq_size = 0; /* No need SQ. */
380         attr.dbr_umem_valid = 1;
381         attr.wq_umem_id = eqp->umem_obj->umem_id;
382         attr.wq_umem_offset = 0;
383         attr.dbr_umem_id = eqp->umem_obj->umem_id;
384         attr.dbr_address = (1 << log_desc_n) * MLX5_WSEG_SIZE;
385         eqp->sw_qp = mlx5_devx_cmd_create_qp(priv->ctx, &attr);
386         if (!eqp->sw_qp) {
387                 DRV_LOG(ERR, "Failed to create SW QP(%u).", rte_errno);
388                 goto error;
389         }
390         eqp->db_rec = RTE_PTR_ADD(eqp->umem_buf, (uintptr_t)attr.dbr_address);
391         if (mlx5_vdpa_qps2rts(eqp))
392                 goto error;
393         /* First ringing. */
394         rte_write32(rte_cpu_to_be_32(1 << log_desc_n), &eqp->db_rec[0]);
395         return 0;
396 error:
397         mlx5_vdpa_event_qp_destroy(eqp);
398         return -1;
399 }