drivers/net/mlx4/mlx4_intr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Interrupts handling for mlx4 driver.
  37  */
  38
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <stdlib.h>
  43
  44 /* Verbs headers do not support -pedantic. */
  45 #ifdef PEDANTIC
  46 #pragma GCC diagnostic ignored "-Wpedantic"
  47 #endif
  48 #include <infiniband/verbs.h>
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic error "-Wpedantic"
  51 #endif
  52
  53 #include <rte_alarm.h>
  54 #include <rte_errno.h>
  55 #include <rte_ethdev.h>
  56 #include <rte_interrupts.h>
  57
  58 #include "mlx4.h"
  59 #include "mlx4_utils.h"
  60
  61 static void mlx4_link_status_alarm(struct priv *priv);
  62
  63 /**
  64  * Clean up Rx interrupts handler.
  65  *
  66  * @param priv
  67  *   Pointer to private structure.
  68  */
  69 static void
  70 mlx4_rx_intr_vec_disable(struct priv *priv)
  71 {
  72         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  73
  74         rte_intr_free_epoll_fd(intr_handle);
  75         free(intr_handle->intr_vec);
  76         intr_handle->nb_efd = 0;
  77         intr_handle->intr_vec = NULL;
  78 }
  79
  80 /**
  81  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  82  *
  83  * @param priv
  84  *   Pointer to private structure.
  85  *
  86  * @return
  87  *   0 on success, negative errno value otherwise and rte_errno is set.
  88  */
  89 static int
  90 mlx4_rx_intr_vec_enable(struct priv *priv)
  91 {
  92         unsigned int i;
  93         unsigned int rxqs_n = priv->rxqs_n;
  94         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  95         unsigned int count = 0;
  96         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  97
  98         mlx4_rx_intr_vec_disable(priv);
  99         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
 100         if (intr_handle->intr_vec == NULL) {
 101                 rte_errno = ENOMEM;
 102                 ERROR("failed to allocate memory for interrupt vector,"
 103                       " Rx interrupts will not be supported");
 104                 return -rte_errno;
 105         }
 106         for (i = 0; i != n; ++i) {
 107                 struct rxq *rxq = (*priv->rxqs)[i];
 108
 109                 /* Skip queues that cannot request interrupts. */
 110                 if (!rxq || !rxq->channel) {
 111                         /* Use invalid intr_vec[] index to disable entry. */
 112                         intr_handle->intr_vec[i] =
 113                                 RTE_INTR_VEC_RXTX_OFFSET +
 114                                 RTE_MAX_RXTX_INTR_VEC_ID;
 115                         continue;
 116                 }
 117                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 118                         rte_errno = E2BIG;
 119                         ERROR("too many Rx queues for interrupt vector size"
 120                               " (%d), Rx interrupts cannot be enabled",
 121                               RTE_MAX_RXTX_INTR_VEC_ID);
 122                         mlx4_rx_intr_vec_disable(priv);
 123                         return -rte_errno;
 124                 }
 125                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 126                 intr_handle->efds[count] = rxq->channel->fd;
 127                 count++;
 128         }
 129         if (!count)
 130                 mlx4_rx_intr_vec_disable(priv);
 131         else
 132                 intr_handle->nb_efd = count;
 133         return 0;
 134 }
 135
 136 /**
 137  * Collect interrupt events.
 138  *
 139  * @param priv
 140  *   Pointer to private structure.
 141  * @param events
 142  *   Pointer to event flags holder.
 143  *
 144  * @return
 145  *   Number of events.
 146  */
 147 static int
 148 mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
 149 {
 150         struct ibv_async_event event;
 151         int port_change = 0;
 152         struct rte_eth_link *link = &priv->dev->data->dev_link;
 153         const struct rte_intr_conf *const intr_conf =
 154                 &priv->dev->data->dev_conf.intr_conf;
 155         int ret = 0;
 156
 157         *events = 0;
 158         /* Read all message and acknowledge them. */
 159         for (;;) {
 160                 if (ibv_get_async_event(priv->ctx, &event))
 161                         break;
 162                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 163                      event.event_type == IBV_EVENT_PORT_ERR) &&
 164                     intr_conf->lsc) {
 165                         port_change = 1;
 166                         ret++;
 167                 } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
 168                            intr_conf->rmv) {
 169                         *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
 170                         ret++;
 171                 } else {
 172                         DEBUG("event type %d on port %d not handled",
 173                               event.event_type, event.element.port_num);
 174                 }
 175                 ibv_ack_async_event(&event);
 176         }
 177         if (!port_change)
 178                 return ret;
 179         mlx4_link_update(priv->dev, 0);
 180         if (((link->link_speed == 0) && link->link_status) ||
 181             ((link->link_speed != 0) && !link->link_status)) {
 182                 if (!priv->intr_alarm) {
 183                         /* Inconsistent status, check again later. */
 184                         priv->intr_alarm = 1;
 185                         rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 186                                           (void (*)(void *))
 187                                           mlx4_link_status_alarm,
 188                                           priv);
 189                 }
 190         } else {
 191                 *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
 192         }
 193         return ret;
 194 }
 195
 196 /**
 197  * Process scheduled link status check.
 198  *
 199  * @param priv
 200  *   Pointer to private structure.
 201  */
 202 static void
 203 mlx4_link_status_alarm(struct priv *priv)
 204 {
 205         uint32_t events;
 206         int ret;
 207
 208         assert(priv->intr_alarm == 1);
 209         priv->intr_alarm = 0;
 210         ret = mlx4_collect_interrupt_events(priv, &events);
 211         if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
 212                 _rte_eth_dev_callback_process(priv->dev,
 213                                               RTE_ETH_EVENT_INTR_LSC,
 214                                               NULL, NULL);
 215 }
 216
 217 /**
 218  * Handle interrupts from the NIC.
 219  *
 220  * @param priv
 221  *   Pointer to private structure.
 222  */
 223 static void
 224 mlx4_interrupt_handler(struct priv *priv)
 225 {
 226         int ret;
 227         uint32_t ev;
 228         int i;
 229
 230         ret = mlx4_collect_interrupt_events(priv, &ev);
 231         if (ret > 0) {
 232                 for (i = RTE_ETH_EVENT_UNKNOWN;
 233                      i < RTE_ETH_EVENT_MAX;
 234                      i++) {
 235                         if (ev & (1 << i)) {
 236                                 ev &= ~(1 << i);
 237                                 _rte_eth_dev_callback_process(priv->dev, i,
 238                                                               NULL, NULL);
 239                                 ret--;
 240                         }
 241                 }
 242                 if (ret)
 243                         WARN("%d event%s not processed", ret,
 244                              (ret > 1 ? "s were" : " was"));
 245         }
 246 }
 247
 248 /**
 249  * Uninstall interrupt handler.
 250  *
 251  * @param priv
 252  *   Pointer to private structure.
 253  *
 254  * @return
 255  *   0 on success, negative errno value otherwise and rte_errno is set.
 256  */
 257 int
 258 mlx4_intr_uninstall(struct priv *priv)
 259 {
 260         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 261
 262         if (priv->intr_handle.fd != -1) {
 263                 rte_intr_callback_unregister(&priv->intr_handle,
 264                                              (void (*)(void *))
 265                                              mlx4_interrupt_handler,
 266                                              priv);
 267                 priv->intr_handle.fd = -1;
 268         }
 269         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 270         priv->intr_alarm = 0;
 271         mlx4_rx_intr_vec_disable(priv);
 272         rte_errno = err;
 273         return 0;
 274 }
 275
 276 /**
 277  * Install interrupt handler.
 278  *
 279  * @param priv
 280  *   Pointer to private structure.
 281  *
 282  * @return
 283  *   0 on success, negative errno value otherwise and rte_errno is set.
 284  */
 285 int
 286 mlx4_intr_install(struct priv *priv)
 287 {
 288         const struct rte_intr_conf *const intr_conf =
 289                 &priv->dev->data->dev_conf.intr_conf;
 290         int rc;
 291
 292         mlx4_intr_uninstall(priv);
 293         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 294                 goto error;
 295         if (intr_conf->lsc | intr_conf->rmv) {
 296                 priv->intr_handle.fd = priv->ctx->async_fd;
 297                 rc = rte_intr_callback_register(&priv->intr_handle,
 298                                                 (void (*)(void *))
 299                                                 mlx4_interrupt_handler,
 300                                                 priv);
 301                 if (rc < 0) {
 302                         rte_errno = -rc;
 303                         goto error;
 304                 }
 305         }
 306         return 0;
 307 error:
 308         mlx4_intr_uninstall(priv);
 309         return -rte_errno;
 310 }
 311
 312 /**
 313  * DPDK callback for Rx queue interrupt disable.
 314  *
 315  * @param dev
 316  *   Pointer to Ethernet device structure.
 317  * @param idx
 318  *   Rx queue index.
 319  *
 320  * @return
 321  *   0 on success, negative errno value otherwise and rte_errno is set.
 322  */
 323 int
 324 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 325 {
 326         struct priv *priv = dev->data->dev_private;
 327         struct rxq *rxq = (*priv->rxqs)[idx];
 328         struct ibv_cq *ev_cq;
 329         void *ev_ctx;
 330         int ret;
 331
 332         if (!rxq || !rxq->channel) {
 333                 ret = EINVAL;
 334         } else {
 335                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
 336                 if (ret || ev_cq != rxq->cq)
 337                         ret = EINVAL;
 338         }
 339         if (ret) {
 340                 rte_errno = ret;
 341                 WARN("unable to disable interrupt on rx queue %d",
 342                      idx);
 343         } else {
 344                 ibv_ack_cq_events(rxq->cq, 1);
 345         }
 346         return -ret;
 347 }
 348
 349 /**
 350  * DPDK callback for Rx queue interrupt enable.
 351  *
 352  * @param dev
 353  *   Pointer to Ethernet device structure.
 354  * @param idx
 355  *   Rx queue index.
 356  *
 357  * @return
 358  *   0 on success, negative errno value otherwise and rte_errno is set.
 359  */
 360 int
 361 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 362 {
 363         struct priv *priv = dev->data->dev_private;
 364         struct rxq *rxq = (*priv->rxqs)[idx];
 365         int ret;
 366
 367         if (!rxq || !rxq->channel)
 368                 ret = EINVAL;
 369         else
 370                 ret = ibv_req_notify_cq(rxq->cq, 0);
 371         if (ret) {
 372                 rte_errno = ret;
 373                 WARN("unable to arm interrupt on rx queue %d", idx);
 374         }
 375         return -ret;
 376 }