b17d109a369fc4e791611a3d93b91817dadc4052
[dpdk.git] / drivers / net / mlx4 / mlx4_intr.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright 2017 6WIND S.A.
5  *   Copyright 2017 Mellanox
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of 6WIND S.A. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 /**
35  * @file
36  * Interrupts handling for mlx4 driver.
37  */
38
39 #include <assert.h>
40 #include <errno.h>
41 #include <stdint.h>
42 #include <stdlib.h>
43
44 /* Verbs headers do not support -pedantic. */
45 #ifdef PEDANTIC
46 #pragma GCC diagnostic ignored "-Wpedantic"
47 #endif
48 #include <infiniband/verbs.h>
49 #ifdef PEDANTIC
50 #pragma GCC diagnostic error "-Wpedantic"
51 #endif
52
53 #include <rte_alarm.h>
54 #include <rte_errno.h>
55 #include <rte_ethdev.h>
56 #include <rte_io.h>
57 #include <rte_interrupts.h>
58
59 #include "mlx4.h"
60 #include "mlx4_rxtx.h"
61 #include "mlx4_utils.h"
62
63 static int mlx4_link_status_check(struct priv *priv);
64
65 /**
66  * Clean up Rx interrupts handler.
67  *
68  * @param priv
69  *   Pointer to private structure.
70  */
71 static void
72 mlx4_rx_intr_vec_disable(struct priv *priv)
73 {
74         struct rte_intr_handle *intr_handle = &priv->intr_handle;
75
76         rte_intr_free_epoll_fd(intr_handle);
77         free(intr_handle->intr_vec);
78         intr_handle->nb_efd = 0;
79         intr_handle->intr_vec = NULL;
80 }
81
82 /**
83  * Allocate queue vector and fill epoll fd list for Rx interrupts.
84  *
85  * @param priv
86  *   Pointer to private structure.
87  *
88  * @return
89  *   0 on success, negative errno value otherwise and rte_errno is set.
90  */
91 static int
92 mlx4_rx_intr_vec_enable(struct priv *priv)
93 {
94         unsigned int i;
95         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
96         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
97         unsigned int count = 0;
98         struct rte_intr_handle *intr_handle = &priv->intr_handle;
99
100         mlx4_rx_intr_vec_disable(priv);
101         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
102         if (intr_handle->intr_vec == NULL) {
103                 rte_errno = ENOMEM;
104                 ERROR("failed to allocate memory for interrupt vector,"
105                       " Rx interrupts will not be supported");
106                 return -rte_errno;
107         }
108         for (i = 0; i != n; ++i) {
109                 struct rxq *rxq = priv->dev->data->rx_queues[i];
110
111                 /* Skip queues that cannot request interrupts. */
112                 if (!rxq || !rxq->channel) {
113                         /* Use invalid intr_vec[] index to disable entry. */
114                         intr_handle->intr_vec[i] =
115                                 RTE_INTR_VEC_RXTX_OFFSET +
116                                 RTE_MAX_RXTX_INTR_VEC_ID;
117                         continue;
118                 }
119                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
120                         rte_errno = E2BIG;
121                         ERROR("too many Rx queues for interrupt vector size"
122                               " (%d), Rx interrupts cannot be enabled",
123                               RTE_MAX_RXTX_INTR_VEC_ID);
124                         mlx4_rx_intr_vec_disable(priv);
125                         return -rte_errno;
126                 }
127                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
128                 intr_handle->efds[count] = rxq->channel->fd;
129                 count++;
130         }
131         if (!count)
132                 mlx4_rx_intr_vec_disable(priv);
133         else
134                 intr_handle->nb_efd = count;
135         return 0;
136 }
137
138 /**
139  * Process scheduled link status check.
140  *
141  * If LSC interrupts are requested, process related callback.
142  *
143  * @param priv
144  *   Pointer to private structure.
145  */
146 static void
147 mlx4_link_status_alarm(struct priv *priv)
148 {
149         const struct rte_intr_conf *const intr_conf =
150                 &priv->dev->data->dev_conf.intr_conf;
151
152         assert(priv->intr_alarm == 1);
153         priv->intr_alarm = 0;
154         if (intr_conf->lsc && !mlx4_link_status_check(priv))
155                 _rte_eth_dev_callback_process(priv->dev,
156                                               RTE_ETH_EVENT_INTR_LSC,
157                                               NULL, NULL);
158 }
159
160 /**
161  * Check link status.
162  *
163  * In case of inconsistency, another check is scheduled.
164  *
165  * @param priv
166  *   Pointer to private structure.
167  *
168  * @return
169  *   0 on success (link status is consistent), negative errno value
170  *   otherwise and rte_errno is set.
171  */
172 static int
173 mlx4_link_status_check(struct priv *priv)
174 {
175         struct rte_eth_link *link = &priv->dev->data->dev_link;
176         int ret = mlx4_link_update(priv->dev, 0);
177
178         if (ret)
179                 return ret;
180         if ((!link->link_speed && link->link_status) ||
181             (link->link_speed && !link->link_status)) {
182                 if (!priv->intr_alarm) {
183                         /* Inconsistent status, check again later. */
184                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
185                                                 (void (*)(void *))
186                                                 mlx4_link_status_alarm,
187                                                 priv);
188                         if (ret)
189                                 return ret;
190                         priv->intr_alarm = 1;
191                 }
192                 rte_errno = EINPROGRESS;
193                 return -rte_errno;
194         }
195         return 0;
196 }
197
198 /**
199  * Handle interrupts from the NIC.
200  *
201  * @param priv
202  *   Pointer to private structure.
203  */
204 static void
205 mlx4_interrupt_handler(struct priv *priv)
206 {
207         enum { LSC, RMV, };
208         static const enum rte_eth_event_type type[] = {
209                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
210                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
211         };
212         uint32_t caught[RTE_DIM(type)] = { 0 };
213         struct ibv_async_event event;
214         const struct rte_intr_conf *const intr_conf =
215                 &priv->dev->data->dev_conf.intr_conf;
216         unsigned int i;
217
218         /* Read all message and acknowledge them. */
219         while (!ibv_get_async_event(priv->ctx, &event)) {
220                 switch (event.event_type) {
221                 case IBV_EVENT_PORT_ACTIVE:
222                 case IBV_EVENT_PORT_ERR:
223                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
224                                 ++caught[LSC];
225                         break;
226                 case IBV_EVENT_DEVICE_FATAL:
227                         if (intr_conf->rmv)
228                                 ++caught[RMV];
229                         break;
230                 default:
231                         DEBUG("event type %d on physical port %d not handled",
232                               event.event_type, event.element.port_num);
233                 }
234                 ibv_ack_async_event(&event);
235         }
236         for (i = 0; i != RTE_DIM(caught); ++i)
237                 if (caught[i])
238                         _rte_eth_dev_callback_process(priv->dev, type[i],
239                                                       NULL, NULL);
240 }
241
242 /**
243  * MLX4 CQ notification .
244  *
245  * @param rxq
246  *   Pointer to receive queue structure.
247  * @param solicited
248  *   Is request solicited or not.
249  */
250 static void
251 mlx4_arm_cq(struct rxq *rxq, int solicited)
252 {
253         struct mlx4_cq *cq = &rxq->mcq;
254         uint64_t doorbell;
255         uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
256         uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
257         uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
258
259         *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
260         /*
261          * Make sure that the doorbell record in host memory is
262          * written before ringing the doorbell via PCI MMIO.
263          */
264         rte_wmb();
265         doorbell = sn << 28 | cmd | cq->cqn;
266         doorbell <<= 32;
267         doorbell |= ci;
268         rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
269 }
270
271 /**
272  * Uninstall interrupt handler.
273  *
274  * @param priv
275  *   Pointer to private structure.
276  *
277  * @return
278  *   0 on success, negative errno value otherwise and rte_errno is set.
279  */
280 int
281 mlx4_intr_uninstall(struct priv *priv)
282 {
283         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
284
285         if (priv->intr_handle.fd != -1) {
286                 rte_intr_callback_unregister(&priv->intr_handle,
287                                              (void (*)(void *))
288                                              mlx4_interrupt_handler,
289                                              priv);
290                 priv->intr_handle.fd = -1;
291         }
292         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
293         priv->intr_alarm = 0;
294         mlx4_rx_intr_vec_disable(priv);
295         rte_errno = err;
296         return 0;
297 }
298
299 /**
300  * Install interrupt handler.
301  *
302  * @param priv
303  *   Pointer to private structure.
304  *
305  * @return
306  *   0 on success, negative errno value otherwise and rte_errno is set.
307  */
308 int
309 mlx4_intr_install(struct priv *priv)
310 {
311         const struct rte_intr_conf *const intr_conf =
312                 &priv->dev->data->dev_conf.intr_conf;
313         int rc;
314
315         mlx4_intr_uninstall(priv);
316         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
317                 goto error;
318         if (intr_conf->lsc | intr_conf->rmv) {
319                 priv->intr_handle.fd = priv->ctx->async_fd;
320                 rc = rte_intr_callback_register(&priv->intr_handle,
321                                                 (void (*)(void *))
322                                                 mlx4_interrupt_handler,
323                                                 priv);
324                 if (rc < 0) {
325                         rte_errno = -rc;
326                         goto error;
327                 }
328         }
329         return 0;
330 error:
331         mlx4_intr_uninstall(priv);
332         return -rte_errno;
333 }
334
335 /**
336  * DPDK callback for Rx queue interrupt disable.
337  *
338  * @param dev
339  *   Pointer to Ethernet device structure.
340  * @param idx
341  *   Rx queue index.
342  *
343  * @return
344  *   0 on success, negative errno value otherwise and rte_errno is set.
345  */
346 int
347 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
348 {
349         struct rxq *rxq = dev->data->rx_queues[idx];
350         struct ibv_cq *ev_cq;
351         void *ev_ctx;
352         int ret;
353
354         if (!rxq || !rxq->channel) {
355                 ret = EINVAL;
356         } else {
357                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
358                 if (ret || ev_cq != rxq->cq)
359                         ret = EINVAL;
360         }
361         if (ret) {
362                 rte_errno = ret;
363                 WARN("unable to disable interrupt on rx queue %d",
364                      idx);
365         } else {
366                 rxq->mcq.arm_sn++;
367                 ibv_ack_cq_events(rxq->cq, 1);
368         }
369         return -ret;
370 }
371
372 /**
373  * DPDK callback for Rx queue interrupt enable.
374  *
375  * @param dev
376  *   Pointer to Ethernet device structure.
377  * @param idx
378  *   Rx queue index.
379  *
380  * @return
381  *   0 on success, negative errno value otherwise and rte_errno is set.
382  */
383 int
384 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
385 {
386         struct rxq *rxq = dev->data->rx_queues[idx];
387         int ret = 0;
388
389         if (!rxq || !rxq->channel) {
390                 ret = EINVAL;
391                 rte_errno = ret;
392                 WARN("unable to arm interrupt on rx queue %d", idx);
393         } else {
394                 mlx4_arm_cq(rxq, 0);
395         }
396         return -ret;
397 }