bcf4d592b6a5a8422aea2817751ee7eb35c8b500
[dpdk.git] / drivers / net / mlx4 / mlx4_intr.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright 2017 6WIND S.A.
5  *   Copyright 2017 Mellanox
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of 6WIND S.A. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 /**
35  * @file
36  * Interrupts handling for mlx4 driver.
37  */
38
39 #include <assert.h>
40 #include <errno.h>
41 #include <stdint.h>
42 #include <stdlib.h>
43
44 /* Verbs headers do not support -pedantic. */
45 #ifdef PEDANTIC
46 #pragma GCC diagnostic ignored "-Wpedantic"
47 #endif
48 #include <infiniband/verbs.h>
49 #ifdef PEDANTIC
50 #pragma GCC diagnostic error "-Wpedantic"
51 #endif
52
53 #include <rte_alarm.h>
54 #include <rte_errno.h>
55 #include <rte_ethdev.h>
56 #include <rte_interrupts.h>
57
58 #include "mlx4.h"
59 #include "mlx4_utils.h"
60
61 static void mlx4_link_status_alarm(struct priv *priv);
62
63 /**
64  * Clean up Rx interrupts handler.
65  *
66  * @param priv
67  *   Pointer to private structure.
68  */
69 static void
70 mlx4_rx_intr_vec_disable(struct priv *priv)
71 {
72         struct rte_intr_handle *intr_handle = &priv->intr_handle;
73
74         rte_intr_free_epoll_fd(intr_handle);
75         free(intr_handle->intr_vec);
76         intr_handle->nb_efd = 0;
77         intr_handle->intr_vec = NULL;
78 }
79
80 /**
81  * Allocate queue vector and fill epoll fd list for Rx interrupts.
82  *
83  * @param priv
84  *   Pointer to private structure.
85  *
86  * @return
87  *   0 on success, negative errno value otherwise and rte_errno is set.
88  */
89 static int
90 mlx4_rx_intr_vec_enable(struct priv *priv)
91 {
92         unsigned int i;
93         unsigned int rxqs_n = priv->rxqs_n;
94         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
95         unsigned int count = 0;
96         struct rte_intr_handle *intr_handle = &priv->intr_handle;
97
98         mlx4_rx_intr_vec_disable(priv);
99         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
100         if (intr_handle->intr_vec == NULL) {
101                 rte_errno = ENOMEM;
102                 ERROR("failed to allocate memory for interrupt vector,"
103                       " Rx interrupts will not be supported");
104                 return -rte_errno;
105         }
106         for (i = 0; i != n; ++i) {
107                 struct rxq *rxq = (*priv->rxqs)[i];
108
109                 /* Skip queues that cannot request interrupts. */
110                 if (!rxq || !rxq->channel) {
111                         /* Use invalid intr_vec[] index to disable entry. */
112                         intr_handle->intr_vec[i] =
113                                 RTE_INTR_VEC_RXTX_OFFSET +
114                                 RTE_MAX_RXTX_INTR_VEC_ID;
115                         continue;
116                 }
117                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
118                         rte_errno = E2BIG;
119                         ERROR("too many Rx queues for interrupt vector size"
120                               " (%d), Rx interrupts cannot be enabled",
121                               RTE_MAX_RXTX_INTR_VEC_ID);
122                         mlx4_rx_intr_vec_disable(priv);
123                         return -rte_errno;
124                 }
125                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
126                 intr_handle->efds[count] = rxq->channel->fd;
127                 count++;
128         }
129         if (!count)
130                 mlx4_rx_intr_vec_disable(priv);
131         else
132                 intr_handle->nb_efd = count;
133         return 0;
134 }
135
136 /**
137  * Collect interrupt events.
138  *
139  * @param priv
140  *   Pointer to private structure.
141  * @param events
142  *   Pointer to event flags holder.
143  *
144  * @return
145  *   Number of events.
146  */
147 static int
148 mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
149 {
150         struct ibv_async_event event;
151         int port_change = 0;
152         struct rte_eth_link *link = &priv->dev->data->dev_link;
153         const struct rte_intr_conf *const intr_conf =
154                 &priv->dev->data->dev_conf.intr_conf;
155         int ret = 0;
156
157         *events = 0;
158         /* Read all message and acknowledge them. */
159         for (;;) {
160                 if (ibv_get_async_event(priv->ctx, &event))
161                         break;
162                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
163                      event.event_type == IBV_EVENT_PORT_ERR) &&
164                     intr_conf->lsc) {
165                         port_change = 1;
166                         ret++;
167                 } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
168                            intr_conf->rmv) {
169                         *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
170                         ret++;
171                 } else {
172                         DEBUG("event type %d on port %d not handled",
173                               event.event_type, event.element.port_num);
174                 }
175                 ibv_ack_async_event(&event);
176         }
177         if (!port_change)
178                 return ret;
179         mlx4_link_update(priv->dev, 0);
180         if (((link->link_speed == 0) && link->link_status) ||
181             ((link->link_speed != 0) && !link->link_status)) {
182                 if (!priv->intr_alarm) {
183                         /* Inconsistent status, check again later. */
184                         priv->intr_alarm = 1;
185                         rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
186                                           (void (*)(void *))
187                                           mlx4_link_status_alarm,
188                                           priv);
189                 }
190         } else {
191                 *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
192         }
193         return ret;
194 }
195
196 /**
197  * Process scheduled link status check.
198  *
199  * @param priv
200  *   Pointer to private structure.
201  */
202 static void
203 mlx4_link_status_alarm(struct priv *priv)
204 {
205         uint32_t events;
206         int ret;
207
208         assert(priv->intr_alarm == 1);
209         priv->intr_alarm = 0;
210         ret = mlx4_collect_interrupt_events(priv, &events);
211         if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
212                 _rte_eth_dev_callback_process(priv->dev,
213                                               RTE_ETH_EVENT_INTR_LSC,
214                                               NULL, NULL);
215 }
216
217 /**
218  * Handle interrupts from the NIC.
219  *
220  * @param priv
221  *   Pointer to private structure.
222  */
223 static void
224 mlx4_interrupt_handler(struct priv *priv)
225 {
226         int ret;
227         uint32_t ev;
228         int i;
229
230         ret = mlx4_collect_interrupt_events(priv, &ev);
231         if (ret > 0) {
232                 for (i = RTE_ETH_EVENT_UNKNOWN;
233                      i < RTE_ETH_EVENT_MAX;
234                      i++) {
235                         if (ev & (1 << i)) {
236                                 ev &= ~(1 << i);
237                                 _rte_eth_dev_callback_process(priv->dev, i,
238                                                               NULL, NULL);
239                                 ret--;
240                         }
241                 }
242                 if (ret)
243                         WARN("%d event%s not processed", ret,
244                              (ret > 1 ? "s were" : " was"));
245         }
246 }
247
248 /**
249  * Uninstall interrupt handler.
250  *
251  * @param priv
252  *   Pointer to private structure.
253  *
254  * @return
255  *   0 on success, negative errno value otherwise and rte_errno is set.
256  */
257 int
258 mlx4_intr_uninstall(struct priv *priv)
259 {
260         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
261
262         if (priv->intr_handle.fd != -1) {
263                 rte_intr_callback_unregister(&priv->intr_handle,
264                                              (void (*)(void *))
265                                              mlx4_interrupt_handler,
266                                              priv);
267                 priv->intr_handle.fd = -1;
268         }
269         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
270         priv->intr_alarm = 0;
271         mlx4_rx_intr_vec_disable(priv);
272         rte_errno = err;
273         return 0;
274 }
275
276 /**
277  * Install interrupt handler.
278  *
279  * @param priv
280  *   Pointer to private structure.
281  *
282  * @return
283  *   0 on success, negative errno value otherwise and rte_errno is set.
284  */
285 int
286 mlx4_intr_install(struct priv *priv)
287 {
288         const struct rte_intr_conf *const intr_conf =
289                 &priv->dev->data->dev_conf.intr_conf;
290         int rc;
291
292         mlx4_intr_uninstall(priv);
293         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
294                 goto error;
295         if (intr_conf->lsc | intr_conf->rmv) {
296                 priv->intr_handle.fd = priv->ctx->async_fd;
297                 rc = rte_intr_callback_register(&priv->intr_handle,
298                                                 (void (*)(void *))
299                                                 mlx4_interrupt_handler,
300                                                 priv);
301                 if (rc < 0) {
302                         rte_errno = -rc;
303                         goto error;
304                 }
305         }
306         return 0;
307 error:
308         mlx4_intr_uninstall(priv);
309         return -rte_errno;
310 }
311
312 /**
313  * DPDK callback for Rx queue interrupt disable.
314  *
315  * @param dev
316  *   Pointer to Ethernet device structure.
317  * @param idx
318  *   Rx queue index.
319  *
320  * @return
321  *   0 on success, negative errno value otherwise and rte_errno is set.
322  */
323 int
324 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
325 {
326         struct priv *priv = dev->data->dev_private;
327         struct rxq *rxq = (*priv->rxqs)[idx];
328         struct ibv_cq *ev_cq;
329         void *ev_ctx;
330         int ret;
331
332         if (!rxq || !rxq->channel) {
333                 ret = EINVAL;
334         } else {
335                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
336                 if (ret || ev_cq != rxq->cq)
337                         ret = EINVAL;
338         }
339         if (ret) {
340                 rte_errno = ret;
341                 WARN("unable to disable interrupt on rx queue %d",
342                      idx);
343         } else {
344                 ibv_ack_cq_events(rxq->cq, 1);
345         }
346         return -ret;
347 }
348
349 /**
350  * DPDK callback for Rx queue interrupt enable.
351  *
352  * @param dev
353  *   Pointer to Ethernet device structure.
354  * @param idx
355  *   Rx queue index.
356  *
357  * @return
358  *   0 on success, negative errno value otherwise and rte_errno is set.
359  */
360 int
361 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
362 {
363         struct priv *priv = dev->data->dev_private;
364         struct rxq *rxq = (*priv->rxqs)[idx];
365         int ret;
366
367         if (!rxq || !rxq->channel)
368                 ret = EINVAL;
369         else
370                 ret = ibv_req_notify_cq(rxq->cq, 0);
371         if (ret) {
372                 rte_errno = ret;
373                 WARN("unable to arm interrupt on rx queue %d", idx);
374         }
375         return -ret;
376 }