4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Interrupts handling for mlx4 driver.
44 /* Verbs headers do not support -pedantic. */
46 #pragma GCC diagnostic ignored "-Wpedantic"
48 #include <infiniband/verbs.h>
50 #pragma GCC diagnostic error "-Wpedantic"
53 #include <rte_alarm.h>
54 #include <rte_errno.h>
55 #include <rte_ethdev.h>
56 #include <rte_interrupts.h>
59 #include "mlx4_rxtx.h"
60 #include "mlx4_utils.h"
62 static void mlx4_link_status_alarm(struct priv *priv);
65 * Clean up Rx interrupts handler.
68 * Pointer to private structure.
71 mlx4_rx_intr_vec_disable(struct priv *priv)
73 struct rte_intr_handle *intr_handle = &priv->intr_handle;
75 rte_intr_free_epoll_fd(intr_handle);
76 free(intr_handle->intr_vec);
77 intr_handle->nb_efd = 0;
78 intr_handle->intr_vec = NULL;
82 * Allocate queue vector and fill epoll fd list for Rx interrupts.
85 * Pointer to private structure.
88 * 0 on success, negative errno value otherwise and rte_errno is set.
91 mlx4_rx_intr_vec_enable(struct priv *priv)
94 unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
95 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
96 unsigned int count = 0;
97 struct rte_intr_handle *intr_handle = &priv->intr_handle;
99 mlx4_rx_intr_vec_disable(priv);
100 intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
101 if (intr_handle->intr_vec == NULL) {
103 ERROR("failed to allocate memory for interrupt vector,"
104 " Rx interrupts will not be supported");
107 for (i = 0; i != n; ++i) {
108 struct rxq *rxq = priv->dev->data->rx_queues[i];
110 /* Skip queues that cannot request interrupts. */
111 if (!rxq || !rxq->channel) {
112 /* Use invalid intr_vec[] index to disable entry. */
113 intr_handle->intr_vec[i] =
114 RTE_INTR_VEC_RXTX_OFFSET +
115 RTE_MAX_RXTX_INTR_VEC_ID;
118 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
120 ERROR("too many Rx queues for interrupt vector size"
121 " (%d), Rx interrupts cannot be enabled",
122 RTE_MAX_RXTX_INTR_VEC_ID);
123 mlx4_rx_intr_vec_disable(priv);
126 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
127 intr_handle->efds[count] = rxq->channel->fd;
131 mlx4_rx_intr_vec_disable(priv);
133 intr_handle->nb_efd = count;
138 * Collect interrupt events.
141 * Pointer to private structure.
143 * Pointer to event flags holder.
149 mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
151 struct ibv_async_event event;
153 struct rte_eth_link *link = &priv->dev->data->dev_link;
154 const struct rte_intr_conf *const intr_conf =
155 &priv->dev->data->dev_conf.intr_conf;
159 /* Read all message and acknowledge them. */
161 if (ibv_get_async_event(priv->ctx, &event))
163 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
164 event.event_type == IBV_EVENT_PORT_ERR) &&
168 } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
170 *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
173 DEBUG("event type %d on port %d not handled",
174 event.event_type, event.element.port_num);
176 ibv_ack_async_event(&event);
180 mlx4_link_update(priv->dev, 0);
181 if (((link->link_speed == 0) && link->link_status) ||
182 ((link->link_speed != 0) && !link->link_status)) {
183 if (!priv->intr_alarm) {
184 /* Inconsistent status, check again later. */
185 priv->intr_alarm = 1;
186 rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
188 mlx4_link_status_alarm,
192 *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
198 * Process scheduled link status check.
201 * Pointer to private structure.
204 mlx4_link_status_alarm(struct priv *priv)
209 assert(priv->intr_alarm == 1);
210 priv->intr_alarm = 0;
211 ret = mlx4_collect_interrupt_events(priv, &events);
212 if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
213 _rte_eth_dev_callback_process(priv->dev,
214 RTE_ETH_EVENT_INTR_LSC,
219 * Handle interrupts from the NIC.
222 * Pointer to private structure.
225 mlx4_interrupt_handler(struct priv *priv)
231 ret = mlx4_collect_interrupt_events(priv, &ev);
233 for (i = RTE_ETH_EVENT_UNKNOWN;
234 i < RTE_ETH_EVENT_MAX;
238 _rte_eth_dev_callback_process(priv->dev, i,
244 WARN("%d event%s not processed", ret,
245 (ret > 1 ? "s were" : " was"));
250 * Uninstall interrupt handler.
253 * Pointer to private structure.
256 * 0 on success, negative errno value otherwise and rte_errno is set.
259 mlx4_intr_uninstall(struct priv *priv)
261 int err = rte_errno; /* Make sure rte_errno remains unchanged. */
263 if (priv->intr_handle.fd != -1) {
264 rte_intr_callback_unregister(&priv->intr_handle,
266 mlx4_interrupt_handler,
268 priv->intr_handle.fd = -1;
270 rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
271 priv->intr_alarm = 0;
272 mlx4_rx_intr_vec_disable(priv);
278 * Install interrupt handler.
281 * Pointer to private structure.
284 * 0 on success, negative errno value otherwise and rte_errno is set.
287 mlx4_intr_install(struct priv *priv)
289 const struct rte_intr_conf *const intr_conf =
290 &priv->dev->data->dev_conf.intr_conf;
293 mlx4_intr_uninstall(priv);
294 if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
296 if (intr_conf->lsc | intr_conf->rmv) {
297 priv->intr_handle.fd = priv->ctx->async_fd;
298 rc = rte_intr_callback_register(&priv->intr_handle,
300 mlx4_interrupt_handler,
309 mlx4_intr_uninstall(priv);
314 * DPDK callback for Rx queue interrupt disable.
317 * Pointer to Ethernet device structure.
322 * 0 on success, negative errno value otherwise and rte_errno is set.
325 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
327 struct rxq *rxq = dev->data->rx_queues[idx];
328 struct ibv_cq *ev_cq;
332 if (!rxq || !rxq->channel) {
335 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
336 if (ret || ev_cq != rxq->cq)
341 WARN("unable to disable interrupt on rx queue %d",
344 ibv_ack_cq_events(rxq->cq, 1);
350 * DPDK callback for Rx queue interrupt enable.
353 * Pointer to Ethernet device structure.
358 * 0 on success, negative errno value otherwise and rte_errno is set.
361 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
363 struct rxq *rxq = dev->data->rx_queues[idx];
366 if (!rxq || !rxq->channel)
369 ret = ibv_req_notify_cq(rxq->cq, 0);
372 WARN("unable to arm interrupt on rx queue %d", idx);