net/af_packet: make qdisc bypass configurable
[dpdk.git] / drivers / net / mlx4 / mlx4_intr.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright 2017 6WIND S.A.
5  *   Copyright 2017 Mellanox
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of 6WIND S.A. nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 /**
35  * @file
36  * Interrupts handling for mlx4 driver.
37  */
38
39 #include <assert.h>
40 #include <errno.h>
41 #include <stdint.h>
42 #include <stdlib.h>
43
44 /* Verbs headers do not support -pedantic. */
45 #ifdef PEDANTIC
46 #pragma GCC diagnostic ignored "-Wpedantic"
47 #endif
48 #include <infiniband/verbs.h>
49 #ifdef PEDANTIC
50 #pragma GCC diagnostic error "-Wpedantic"
51 #endif
52
53 #include <rte_alarm.h>
54 #include <rte_errno.h>
55 #include <rte_ethdev.h>
56 #include <rte_interrupts.h>
57
58 #include "mlx4.h"
59 #include "mlx4_rxtx.h"
60 #include "mlx4_utils.h"
61
62 static int mlx4_link_status_check(struct priv *priv);
63
64 /**
65  * Clean up Rx interrupts handler.
66  *
67  * @param priv
68  *   Pointer to private structure.
69  */
70 static void
71 mlx4_rx_intr_vec_disable(struct priv *priv)
72 {
73         struct rte_intr_handle *intr_handle = &priv->intr_handle;
74
75         rte_intr_free_epoll_fd(intr_handle);
76         free(intr_handle->intr_vec);
77         intr_handle->nb_efd = 0;
78         intr_handle->intr_vec = NULL;
79 }
80
81 /**
82  * Allocate queue vector and fill epoll fd list for Rx interrupts.
83  *
84  * @param priv
85  *   Pointer to private structure.
86  *
87  * @return
88  *   0 on success, negative errno value otherwise and rte_errno is set.
89  */
90 static int
91 mlx4_rx_intr_vec_enable(struct priv *priv)
92 {
93         unsigned int i;
94         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
95         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
96         unsigned int count = 0;
97         struct rte_intr_handle *intr_handle = &priv->intr_handle;
98
99         mlx4_rx_intr_vec_disable(priv);
100         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
101         if (intr_handle->intr_vec == NULL) {
102                 rte_errno = ENOMEM;
103                 ERROR("failed to allocate memory for interrupt vector,"
104                       " Rx interrupts will not be supported");
105                 return -rte_errno;
106         }
107         for (i = 0; i != n; ++i) {
108                 struct rxq *rxq = priv->dev->data->rx_queues[i];
109
110                 /* Skip queues that cannot request interrupts. */
111                 if (!rxq || !rxq->channel) {
112                         /* Use invalid intr_vec[] index to disable entry. */
113                         intr_handle->intr_vec[i] =
114                                 RTE_INTR_VEC_RXTX_OFFSET +
115                                 RTE_MAX_RXTX_INTR_VEC_ID;
116                         continue;
117                 }
118                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
119                         rte_errno = E2BIG;
120                         ERROR("too many Rx queues for interrupt vector size"
121                               " (%d), Rx interrupts cannot be enabled",
122                               RTE_MAX_RXTX_INTR_VEC_ID);
123                         mlx4_rx_intr_vec_disable(priv);
124                         return -rte_errno;
125                 }
126                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
127                 intr_handle->efds[count] = rxq->channel->fd;
128                 count++;
129         }
130         if (!count)
131                 mlx4_rx_intr_vec_disable(priv);
132         else
133                 intr_handle->nb_efd = count;
134         return 0;
135 }
136
137 /**
138  * Process scheduled link status check.
139  *
140  * If LSC interrupts are requested, process related callback.
141  *
142  * @param priv
143  *   Pointer to private structure.
144  */
145 static void
146 mlx4_link_status_alarm(struct priv *priv)
147 {
148         const struct rte_intr_conf *const intr_conf =
149                 &priv->dev->data->dev_conf.intr_conf;
150
151         assert(priv->intr_alarm == 1);
152         priv->intr_alarm = 0;
153         if (intr_conf->lsc && !mlx4_link_status_check(priv))
154                 _rte_eth_dev_callback_process(priv->dev,
155                                               RTE_ETH_EVENT_INTR_LSC,
156                                               NULL, NULL);
157 }
158
159 /**
160  * Check link status.
161  *
162  * In case of inconsistency, another check is scheduled.
163  *
164  * @param priv
165  *   Pointer to private structure.
166  *
167  * @return
168  *   0 on success (link status is consistent), negative errno value
169  *   otherwise and rte_errno is set.
170  */
171 static int
172 mlx4_link_status_check(struct priv *priv)
173 {
174         struct rte_eth_link *link = &priv->dev->data->dev_link;
175         int ret = mlx4_link_update(priv->dev, 0);
176
177         if (ret)
178                 return ret;
179         if ((!link->link_speed && link->link_status) ||
180             (link->link_speed && !link->link_status)) {
181                 if (!priv->intr_alarm) {
182                         /* Inconsistent status, check again later. */
183                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
184                                                 (void (*)(void *))
185                                                 mlx4_link_status_alarm,
186                                                 priv);
187                         if (ret)
188                                 return ret;
189                         priv->intr_alarm = 1;
190                 }
191                 rte_errno = EINPROGRESS;
192                 return -rte_errno;
193         }
194         return 0;
195 }
196
197 /**
198  * Handle interrupts from the NIC.
199  *
200  * @param priv
201  *   Pointer to private structure.
202  */
203 static void
204 mlx4_interrupt_handler(struct priv *priv)
205 {
206         enum { LSC, RMV, };
207         static const enum rte_eth_event_type type[] = {
208                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
209                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
210         };
211         uint32_t caught[RTE_DIM(type)] = { 0 };
212         struct ibv_async_event event;
213         const struct rte_intr_conf *const intr_conf =
214                 &priv->dev->data->dev_conf.intr_conf;
215         unsigned int i;
216
217         /* Read all message and acknowledge them. */
218         while (!ibv_get_async_event(priv->ctx, &event)) {
219                 switch (event.event_type) {
220                 case IBV_EVENT_PORT_ACTIVE:
221                 case IBV_EVENT_PORT_ERR:
222                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
223                                 ++caught[LSC];
224                         break;
225                 case IBV_EVENT_DEVICE_FATAL:
226                         if (intr_conf->rmv)
227                                 ++caught[RMV];
228                         break;
229                 default:
230                         DEBUG("event type %d on physical port %d not handled",
231                               event.event_type, event.element.port_num);
232                 }
233                 ibv_ack_async_event(&event);
234         }
235         for (i = 0; i != RTE_DIM(caught); ++i)
236                 if (caught[i])
237                         _rte_eth_dev_callback_process(priv->dev, type[i],
238                                                       NULL, NULL);
239 }
240
241 /**
242  * Uninstall interrupt handler.
243  *
244  * @param priv
245  *   Pointer to private structure.
246  *
247  * @return
248  *   0 on success, negative errno value otherwise and rte_errno is set.
249  */
250 int
251 mlx4_intr_uninstall(struct priv *priv)
252 {
253         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
254
255         if (priv->intr_handle.fd != -1) {
256                 rte_intr_callback_unregister(&priv->intr_handle,
257                                              (void (*)(void *))
258                                              mlx4_interrupt_handler,
259                                              priv);
260                 priv->intr_handle.fd = -1;
261         }
262         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
263         priv->intr_alarm = 0;
264         mlx4_rx_intr_vec_disable(priv);
265         rte_errno = err;
266         return 0;
267 }
268
269 /**
270  * Install interrupt handler.
271  *
272  * @param priv
273  *   Pointer to private structure.
274  *
275  * @return
276  *   0 on success, negative errno value otherwise and rte_errno is set.
277  */
278 int
279 mlx4_intr_install(struct priv *priv)
280 {
281         const struct rte_intr_conf *const intr_conf =
282                 &priv->dev->data->dev_conf.intr_conf;
283         int rc;
284
285         mlx4_intr_uninstall(priv);
286         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
287                 goto error;
288         if (intr_conf->lsc | intr_conf->rmv) {
289                 priv->intr_handle.fd = priv->ctx->async_fd;
290                 rc = rte_intr_callback_register(&priv->intr_handle,
291                                                 (void (*)(void *))
292                                                 mlx4_interrupt_handler,
293                                                 priv);
294                 if (rc < 0) {
295                         rte_errno = -rc;
296                         goto error;
297                 }
298         }
299         return 0;
300 error:
301         mlx4_intr_uninstall(priv);
302         return -rte_errno;
303 }
304
305 /**
306  * DPDK callback for Rx queue interrupt disable.
307  *
308  * @param dev
309  *   Pointer to Ethernet device structure.
310  * @param idx
311  *   Rx queue index.
312  *
313  * @return
314  *   0 on success, negative errno value otherwise and rte_errno is set.
315  */
316 int
317 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
318 {
319         struct rxq *rxq = dev->data->rx_queues[idx];
320         struct ibv_cq *ev_cq;
321         void *ev_ctx;
322         int ret;
323
324         if (!rxq || !rxq->channel) {
325                 ret = EINVAL;
326         } else {
327                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
328                 if (ret || ev_cq != rxq->cq)
329                         ret = EINVAL;
330         }
331         if (ret) {
332                 rte_errno = ret;
333                 WARN("unable to disable interrupt on rx queue %d",
334                      idx);
335         } else {
336                 ibv_ack_cq_events(rxq->cq, 1);
337         }
338         return -ret;
339 }
340
341 /**
342  * DPDK callback for Rx queue interrupt enable.
343  *
344  * @param dev
345  *   Pointer to Ethernet device structure.
346  * @param idx
347  *   Rx queue index.
348  *
349  * @return
350  *   0 on success, negative errno value otherwise and rte_errno is set.
351  */
352 int
353 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
354 {
355         struct rxq *rxq = dev->data->rx_queues[idx];
356         int ret;
357
358         if (!rxq || !rxq->channel)
359                 ret = EINVAL;
360         else
361                 ret = ibv_req_notify_cq(rxq->cq, 0);
362         if (ret) {
363                 rte_errno = ret;
364                 WARN("unable to arm interrupt on rx queue %d", idx);
365         }
366         return -ret;
367 }