4 * Copyright 2017 6WIND S.A.
5 * Copyright 2017 Mellanox
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Interrupts handling for mlx4 driver.
44 /* Verbs headers do not support -pedantic. */
46 #pragma GCC diagnostic ignored "-Wpedantic"
48 #include <infiniband/verbs.h>
50 #pragma GCC diagnostic error "-Wpedantic"
53 #include <rte_alarm.h>
54 #include <rte_errno.h>
55 #include <rte_ethdev_driver.h>
57 #include <rte_interrupts.h>
60 #include "mlx4_glue.h"
61 #include "mlx4_rxtx.h"
62 #include "mlx4_utils.h"
64 static int mlx4_link_status_check(struct priv *priv);
67 * Clean up Rx interrupts handler.
70 * Pointer to private structure.
73 mlx4_rx_intr_vec_disable(struct priv *priv)
75 struct rte_intr_handle *intr_handle = &priv->intr_handle;
77 rte_intr_free_epoll_fd(intr_handle);
78 free(intr_handle->intr_vec);
79 intr_handle->nb_efd = 0;
80 intr_handle->intr_vec = NULL;
84 * Allocate queue vector and fill epoll fd list for Rx interrupts.
87 * Pointer to private structure.
90 * 0 on success, negative errno value otherwise and rte_errno is set.
93 mlx4_rx_intr_vec_enable(struct priv *priv)
96 unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
97 unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
98 unsigned int count = 0;
99 struct rte_intr_handle *intr_handle = &priv->intr_handle;
101 mlx4_rx_intr_vec_disable(priv);
102 intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
103 if (intr_handle->intr_vec == NULL) {
105 ERROR("failed to allocate memory for interrupt vector,"
106 " Rx interrupts will not be supported");
109 for (i = 0; i != n; ++i) {
110 struct rxq *rxq = priv->dev->data->rx_queues[i];
112 /* Skip queues that cannot request interrupts. */
113 if (!rxq || !rxq->channel) {
114 /* Use invalid intr_vec[] index to disable entry. */
115 intr_handle->intr_vec[i] =
116 RTE_INTR_VEC_RXTX_OFFSET +
117 RTE_MAX_RXTX_INTR_VEC_ID;
120 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
122 ERROR("too many Rx queues for interrupt vector size"
123 " (%d), Rx interrupts cannot be enabled",
124 RTE_MAX_RXTX_INTR_VEC_ID);
125 mlx4_rx_intr_vec_disable(priv);
128 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
129 intr_handle->efds[count] = rxq->channel->fd;
133 mlx4_rx_intr_vec_disable(priv);
135 intr_handle->nb_efd = count;
140 * Process scheduled link status check.
142 * If LSC interrupts are requested, process related callback.
145 * Pointer to private structure.
148 mlx4_link_status_alarm(struct priv *priv)
150 const struct rte_intr_conf *const intr_conf =
151 &priv->dev->data->dev_conf.intr_conf;
153 assert(priv->intr_alarm == 1);
154 priv->intr_alarm = 0;
155 if (intr_conf->lsc && !mlx4_link_status_check(priv))
156 _rte_eth_dev_callback_process(priv->dev,
157 RTE_ETH_EVENT_INTR_LSC,
164 * In case of inconsistency, another check is scheduled.
167 * Pointer to private structure.
170 * 0 on success (link status is consistent), negative errno value
171 * otherwise and rte_errno is set.
174 mlx4_link_status_check(struct priv *priv)
176 struct rte_eth_link *link = &priv->dev->data->dev_link;
177 int ret = mlx4_link_update(priv->dev, 0);
181 if ((!link->link_speed && link->link_status) ||
182 (link->link_speed && !link->link_status)) {
183 if (!priv->intr_alarm) {
184 /* Inconsistent status, check again later. */
185 ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
187 mlx4_link_status_alarm,
191 priv->intr_alarm = 1;
193 rte_errno = EINPROGRESS;
200 * Handle interrupts from the NIC.
203 * Pointer to private structure.
206 mlx4_interrupt_handler(struct priv *priv)
209 static const enum rte_eth_event_type type[] = {
210 [LSC] = RTE_ETH_EVENT_INTR_LSC,
211 [RMV] = RTE_ETH_EVENT_INTR_RMV,
213 uint32_t caught[RTE_DIM(type)] = { 0 };
214 struct ibv_async_event event;
215 const struct rte_intr_conf *const intr_conf =
216 &priv->dev->data->dev_conf.intr_conf;
219 /* Read all message and acknowledge them. */
220 while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
221 switch (event.event_type) {
222 case IBV_EVENT_PORT_ACTIVE:
223 case IBV_EVENT_PORT_ERR:
224 if (intr_conf->lsc && !mlx4_link_status_check(priv))
227 case IBV_EVENT_DEVICE_FATAL:
232 DEBUG("event type %d on physical port %d not handled",
233 event.event_type, event.element.port_num);
235 mlx4_glue->ack_async_event(&event);
237 for (i = 0; i != RTE_DIM(caught); ++i)
239 _rte_eth_dev_callback_process(priv->dev, type[i],
244 * MLX4 CQ notification .
247 * Pointer to receive queue structure.
249 * Is request solicited or not.
252 mlx4_arm_cq(struct rxq *rxq, int solicited)
254 struct mlx4_cq *cq = &rxq->mcq;
256 uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
257 uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
258 uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
260 *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
262 * Make sure that the doorbell record in host memory is
263 * written before ringing the doorbell via PCI MMIO.
266 doorbell = sn << 28 | cmd | cq->cqn;
269 rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
273 * Uninstall interrupt handler.
276 * Pointer to private structure.
279 * 0 on success, negative errno value otherwise and rte_errno is set.
282 mlx4_intr_uninstall(struct priv *priv)
284 int err = rte_errno; /* Make sure rte_errno remains unchanged. */
286 if (priv->intr_handle.fd != -1) {
287 rte_intr_callback_unregister(&priv->intr_handle,
289 mlx4_interrupt_handler,
291 priv->intr_handle.fd = -1;
293 rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
294 priv->intr_alarm = 0;
295 mlx4_rxq_intr_disable(priv);
301 * Install interrupt handler.
304 * Pointer to private structure.
307 * 0 on success, negative errno value otherwise and rte_errno is set.
310 mlx4_intr_install(struct priv *priv)
312 const struct rte_intr_conf *const intr_conf =
313 &priv->dev->data->dev_conf.intr_conf;
316 mlx4_intr_uninstall(priv);
317 if (intr_conf->lsc | intr_conf->rmv) {
318 priv->intr_handle.fd = priv->ctx->async_fd;
319 rc = rte_intr_callback_register(&priv->intr_handle,
321 mlx4_interrupt_handler,
330 mlx4_intr_uninstall(priv);
335 * DPDK callback for Rx queue interrupt disable.
338 * Pointer to Ethernet device structure.
343 * 0 on success, negative errno value otherwise and rte_errno is set.
346 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
348 struct rxq *rxq = dev->data->rx_queues[idx];
349 struct ibv_cq *ev_cq;
353 if (!rxq || !rxq->channel) {
356 ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
358 if (ret || ev_cq != rxq->cq)
363 WARN("unable to disable interrupt on rx queue %d",
367 mlx4_glue->ack_cq_events(rxq->cq, 1);
373 * DPDK callback for Rx queue interrupt enable.
376 * Pointer to Ethernet device structure.
381 * 0 on success, negative errno value otherwise and rte_errno is set.
384 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
386 struct rxq *rxq = dev->data->rx_queues[idx];
389 if (!rxq || !rxq->channel) {
392 WARN("unable to arm interrupt on rx queue %d", idx);
400 * Enable datapath interrupts.
403 * Pointer to private structure.
406 * 0 on success, negative errno value otherwise and rte_errno is set.
409 mlx4_rxq_intr_enable(struct priv *priv)
411 const struct rte_intr_conf *const intr_conf =
412 &priv->dev->data->dev_conf.intr_conf;
414 if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
422 * Disable datapath interrupts, keeping other interrupts intact.
425 * Pointer to private structure.
428 mlx4_rxq_intr_disable(struct priv *priv)
430 int err = rte_errno; /* Make sure rte_errno remains unchanged. */
432 mlx4_rx_intr_vec_disable(priv);