drivers/net/mlx4/mlx4_intr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Interrupts handling for mlx4 driver.
  37  */
  38
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <stdlib.h>
  43
  44 /* Verbs headers do not support -pedantic. */
  45 #ifdef PEDANTIC
  46 #pragma GCC diagnostic ignored "-Wpedantic"
  47 #endif
  48 #include <infiniband/verbs.h>
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic error "-Wpedantic"
  51 #endif
  52
  53 #include <rte_alarm.h>
  54 #include <rte_errno.h>
  55 #include <rte_ethdev.h>
  56 #include <rte_interrupts.h>
  57
  58 #include "mlx4.h"
  59 #include "mlx4_rxtx.h"
  60 #include "mlx4_utils.h"
  61
  62 static void mlx4_link_status_alarm(struct priv *priv);
  63
  64 /**
  65  * Clean up Rx interrupts handler.
  66  *
  67  * @param priv
  68  *   Pointer to private structure.
  69  */
  70 static void
  71 mlx4_rx_intr_vec_disable(struct priv *priv)
  72 {
  73         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  74
  75         rte_intr_free_epoll_fd(intr_handle);
  76         free(intr_handle->intr_vec);
  77         intr_handle->nb_efd = 0;
  78         intr_handle->intr_vec = NULL;
  79 }
  80
  81 /**
  82  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  83  *
  84  * @param priv
  85  *   Pointer to private structure.
  86  *
  87  * @return
  88  *   0 on success, negative errno value otherwise and rte_errno is set.
  89  */
  90 static int
  91 mlx4_rx_intr_vec_enable(struct priv *priv)
  92 {
  93         unsigned int i;
  94         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
  95         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  96         unsigned int count = 0;
  97         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  98
  99         mlx4_rx_intr_vec_disable(priv);
 100         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
 101         if (intr_handle->intr_vec == NULL) {
 102                 rte_errno = ENOMEM;
 103                 ERROR("failed to allocate memory for interrupt vector,"
 104                       " Rx interrupts will not be supported");
 105                 return -rte_errno;
 106         }
 107         for (i = 0; i != n; ++i) {
 108                 struct rxq *rxq = priv->dev->data->rx_queues[i];
 109
 110                 /* Skip queues that cannot request interrupts. */
 111                 if (!rxq || !rxq->channel) {
 112                         /* Use invalid intr_vec[] index to disable entry. */
 113                         intr_handle->intr_vec[i] =
 114                                 RTE_INTR_VEC_RXTX_OFFSET +
 115                                 RTE_MAX_RXTX_INTR_VEC_ID;
 116                         continue;
 117                 }
 118                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 119                         rte_errno = E2BIG;
 120                         ERROR("too many Rx queues for interrupt vector size"
 121                               " (%d), Rx interrupts cannot be enabled",
 122                               RTE_MAX_RXTX_INTR_VEC_ID);
 123                         mlx4_rx_intr_vec_disable(priv);
 124                         return -rte_errno;
 125                 }
 126                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 127                 intr_handle->efds[count] = rxq->channel->fd;
 128                 count++;
 129         }
 130         if (!count)
 131                 mlx4_rx_intr_vec_disable(priv);
 132         else
 133                 intr_handle->nb_efd = count;
 134         return 0;
 135 }
 136
 137 /**
 138  * Collect interrupt events.
 139  *
 140  * @param priv
 141  *   Pointer to private structure.
 142  * @param events
 143  *   Pointer to event flags holder.
 144  *
 145  * @return
 146  *   Number of events.
 147  */
 148 static int
 149 mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
 150 {
 151         struct ibv_async_event event;
 152         int port_change = 0;
 153         struct rte_eth_link *link = &priv->dev->data->dev_link;
 154         const struct rte_intr_conf *const intr_conf =
 155                 &priv->dev->data->dev_conf.intr_conf;
 156         int ret = 0;
 157
 158         *events = 0;
 159         /* Read all message and acknowledge them. */
 160         for (;;) {
 161                 if (ibv_get_async_event(priv->ctx, &event))
 162                         break;
 163                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 164                      event.event_type == IBV_EVENT_PORT_ERR) &&
 165                     intr_conf->lsc) {
 166                         port_change = 1;
 167                         ret++;
 168                 } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
 169                            intr_conf->rmv) {
 170                         *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
 171                         ret++;
 172                 } else {
 173                         DEBUG("event type %d on port %d not handled",
 174                               event.event_type, event.element.port_num);
 175                 }
 176                 ibv_ack_async_event(&event);
 177         }
 178         if (!port_change)
 179                 return ret;
 180         mlx4_link_update(priv->dev, 0);
 181         if (((link->link_speed == 0) && link->link_status) ||
 182             ((link->link_speed != 0) && !link->link_status)) {
 183                 if (!priv->intr_alarm) {
 184                         /* Inconsistent status, check again later. */
 185                         priv->intr_alarm = 1;
 186                         rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 187                                           (void (*)(void *))
 188                                           mlx4_link_status_alarm,
 189                                           priv);
 190                 }
 191         } else {
 192                 *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
 193         }
 194         return ret;
 195 }
 196
 197 /**
 198  * Process scheduled link status check.
 199  *
 200  * @param priv
 201  *   Pointer to private structure.
 202  */
 203 static void
 204 mlx4_link_status_alarm(struct priv *priv)
 205 {
 206         uint32_t events;
 207         int ret;
 208
 209         assert(priv->intr_alarm == 1);
 210         priv->intr_alarm = 0;
 211         ret = mlx4_collect_interrupt_events(priv, &events);
 212         if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
 213                 _rte_eth_dev_callback_process(priv->dev,
 214                                               RTE_ETH_EVENT_INTR_LSC,
 215                                               NULL, NULL);
 216 }
 217
 218 /**
 219  * Handle interrupts from the NIC.
 220  *
 221  * @param priv
 222  *   Pointer to private structure.
 223  */
 224 static void
 225 mlx4_interrupt_handler(struct priv *priv)
 226 {
 227         int ret;
 228         uint32_t ev;
 229         int i;
 230
 231         ret = mlx4_collect_interrupt_events(priv, &ev);
 232         if (ret > 0) {
 233                 for (i = RTE_ETH_EVENT_UNKNOWN;
 234                      i < RTE_ETH_EVENT_MAX;
 235                      i++) {
 236                         if (ev & (1 << i)) {
 237                                 ev &= ~(1 << i);
 238                                 _rte_eth_dev_callback_process(priv->dev, i,
 239                                                               NULL, NULL);
 240                                 ret--;
 241                         }
 242                 }
 243                 if (ret)
 244                         WARN("%d event%s not processed", ret,
 245                              (ret > 1 ? "s were" : " was"));
 246         }
 247 }
 248
 249 /**
 250  * Uninstall interrupt handler.
 251  *
 252  * @param priv
 253  *   Pointer to private structure.
 254  *
 255  * @return
 256  *   0 on success, negative errno value otherwise and rte_errno is set.
 257  */
 258 int
 259 mlx4_intr_uninstall(struct priv *priv)
 260 {
 261         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 262
 263         if (priv->intr_handle.fd != -1) {
 264                 rte_intr_callback_unregister(&priv->intr_handle,
 265                                              (void (*)(void *))
 266                                              mlx4_interrupt_handler,
 267                                              priv);
 268                 priv->intr_handle.fd = -1;
 269         }
 270         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 271         priv->intr_alarm = 0;
 272         mlx4_rx_intr_vec_disable(priv);
 273         rte_errno = err;
 274         return 0;
 275 }
 276
 277 /**
 278  * Install interrupt handler.
 279  *
 280  * @param priv
 281  *   Pointer to private structure.
 282  *
 283  * @return
 284  *   0 on success, negative errno value otherwise and rte_errno is set.
 285  */
 286 int
 287 mlx4_intr_install(struct priv *priv)
 288 {
 289         const struct rte_intr_conf *const intr_conf =
 290                 &priv->dev->data->dev_conf.intr_conf;
 291         int rc;
 292
 293         mlx4_intr_uninstall(priv);
 294         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 295                 goto error;
 296         if (intr_conf->lsc | intr_conf->rmv) {
 297                 priv->intr_handle.fd = priv->ctx->async_fd;
 298                 rc = rte_intr_callback_register(&priv->intr_handle,
 299                                                 (void (*)(void *))
 300                                                 mlx4_interrupt_handler,
 301                                                 priv);
 302                 if (rc < 0) {
 303                         rte_errno = -rc;
 304                         goto error;
 305                 }
 306         }
 307         return 0;
 308 error:
 309         mlx4_intr_uninstall(priv);
 310         return -rte_errno;
 311 }
 312
 313 /**
 314  * DPDK callback for Rx queue interrupt disable.
 315  *
 316  * @param dev
 317  *   Pointer to Ethernet device structure.
 318  * @param idx
 319  *   Rx queue index.
 320  *
 321  * @return
 322  *   0 on success, negative errno value otherwise and rte_errno is set.
 323  */
 324 int
 325 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 326 {
 327         struct rxq *rxq = dev->data->rx_queues[idx];
 328         struct ibv_cq *ev_cq;
 329         void *ev_ctx;
 330         int ret;
 331
 332         if (!rxq || !rxq->channel) {
 333                 ret = EINVAL;
 334         } else {
 335                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
 336                 if (ret || ev_cq != rxq->cq)
 337                         ret = EINVAL;
 338         }
 339         if (ret) {
 340                 rte_errno = ret;
 341                 WARN("unable to disable interrupt on rx queue %d",
 342                      idx);
 343         } else {
 344                 ibv_ack_cq_events(rxq->cq, 1);
 345         }
 346         return -ret;
 347 }
 348
 349 /**
 350  * DPDK callback for Rx queue interrupt enable.
 351  *
 352  * @param dev
 353  *   Pointer to Ethernet device structure.
 354  * @param idx
 355  *   Rx queue index.
 356  *
 357  * @return
 358  *   0 on success, negative errno value otherwise and rte_errno is set.
 359  */
 360 int
 361 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 362 {
 363         struct rxq *rxq = dev->data->rx_queues[idx];
 364         int ret;
 365
 366         if (!rxq || !rxq->channel)
 367                 ret = EINVAL;
 368         else
 369                 ret = ibv_req_notify_cq(rxq->cq, 0);
 370         if (ret) {
 371                 rte_errno = ret;
 372                 WARN("unable to arm interrupt on rx queue %d", idx);
 373         }
 374         return -ret;
 375 }