drivers/net/mlx4/mlx4_intr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Interrupts handling for mlx4 driver.
  37  */
  38
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <stdlib.h>
  43
  44 /* Verbs headers do not support -pedantic. */
  45 #ifdef PEDANTIC
  46 #pragma GCC diagnostic ignored "-Wpedantic"
  47 #endif
  48 #include <infiniband/verbs.h>
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic error "-Wpedantic"
  51 #endif
  52
  53 #include <rte_alarm.h>
  54 #include <rte_errno.h>
  55 #include <rte_ethdev.h>
  56 #include <rte_interrupts.h>
  57
  58 #include "mlx4.h"
  59 #include "mlx4_rxtx.h"
  60 #include "mlx4_utils.h"
  61
  62 static int mlx4_link_status_check(struct priv *priv);
  63
  64 /**
  65  * Clean up Rx interrupts handler.
  66  *
  67  * @param priv
  68  *   Pointer to private structure.
  69  */
  70 static void
  71 mlx4_rx_intr_vec_disable(struct priv *priv)
  72 {
  73         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  74
  75         rte_intr_free_epoll_fd(intr_handle);
  76         free(intr_handle->intr_vec);
  77         intr_handle->nb_efd = 0;
  78         intr_handle->intr_vec = NULL;
  79 }
  80
  81 /**
  82  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  83  *
  84  * @param priv
  85  *   Pointer to private structure.
  86  *
  87  * @return
  88  *   0 on success, negative errno value otherwise and rte_errno is set.
  89  */
  90 static int
  91 mlx4_rx_intr_vec_enable(struct priv *priv)
  92 {
  93         unsigned int i;
  94         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
  95         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  96         unsigned int count = 0;
  97         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  98
  99         mlx4_rx_intr_vec_disable(priv);
 100         intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
 101         if (intr_handle->intr_vec == NULL) {
 102                 rte_errno = ENOMEM;
 103                 ERROR("failed to allocate memory for interrupt vector,"
 104                       " Rx interrupts will not be supported");
 105                 return -rte_errno;
 106         }
 107         for (i = 0; i != n; ++i) {
 108                 struct rxq *rxq = priv->dev->data->rx_queues[i];
 109
 110                 /* Skip queues that cannot request interrupts. */
 111                 if (!rxq || !rxq->channel) {
 112                         /* Use invalid intr_vec[] index to disable entry. */
 113                         intr_handle->intr_vec[i] =
 114                                 RTE_INTR_VEC_RXTX_OFFSET +
 115                                 RTE_MAX_RXTX_INTR_VEC_ID;
 116                         continue;
 117                 }
 118                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 119                         rte_errno = E2BIG;
 120                         ERROR("too many Rx queues for interrupt vector size"
 121                               " (%d), Rx interrupts cannot be enabled",
 122                               RTE_MAX_RXTX_INTR_VEC_ID);
 123                         mlx4_rx_intr_vec_disable(priv);
 124                         return -rte_errno;
 125                 }
 126                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 127                 intr_handle->efds[count] = rxq->channel->fd;
 128                 count++;
 129         }
 130         if (!count)
 131                 mlx4_rx_intr_vec_disable(priv);
 132         else
 133                 intr_handle->nb_efd = count;
 134         return 0;
 135 }
 136
 137 /**
 138  * Process scheduled link status check.
 139  *
 140  * If LSC interrupts are requested, process related callback.
 141  *
 142  * @param priv
 143  *   Pointer to private structure.
 144  */
 145 static void
 146 mlx4_link_status_alarm(struct priv *priv)
 147 {
 148         const struct rte_intr_conf *const intr_conf =
 149                 &priv->dev->data->dev_conf.intr_conf;
 150
 151         assert(priv->intr_alarm == 1);
 152         priv->intr_alarm = 0;
 153         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 154                 _rte_eth_dev_callback_process(priv->dev,
 155                                               RTE_ETH_EVENT_INTR_LSC,
 156                                               NULL, NULL);
 157 }
 158
 159 /**
 160  * Check link status.
 161  *
 162  * In case of inconsistency, another check is scheduled.
 163  *
 164  * @param priv
 165  *   Pointer to private structure.
 166  *
 167  * @return
 168  *   0 on success (link status is consistent), negative errno value
 169  *   otherwise and rte_errno is set.
 170  */
 171 static int
 172 mlx4_link_status_check(struct priv *priv)
 173 {
 174         struct rte_eth_link *link = &priv->dev->data->dev_link;
 175         int ret = mlx4_link_update(priv->dev, 0);
 176
 177         if (ret)
 178                 return ret;
 179         if ((!link->link_speed && link->link_status) ||
 180             (link->link_speed && !link->link_status)) {
 181                 if (!priv->intr_alarm) {
 182                         /* Inconsistent status, check again later. */
 183                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 184                                                 (void (*)(void *))
 185                                                 mlx4_link_status_alarm,
 186                                                 priv);
 187                         if (ret)
 188                                 return ret;
 189                         priv->intr_alarm = 1;
 190                 }
 191                 rte_errno = EINPROGRESS;
 192                 return -rte_errno;
 193         }
 194         return 0;
 195 }
 196
 197 /**
 198  * Handle interrupts from the NIC.
 199  *
 200  * @param priv
 201  *   Pointer to private structure.
 202  */
 203 static void
 204 mlx4_interrupt_handler(struct priv *priv)
 205 {
 206         enum { LSC, RMV, };
 207         static const enum rte_eth_event_type type[] = {
 208                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
 209                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
 210         };
 211         uint32_t caught[RTE_DIM(type)] = { 0 };
 212         struct ibv_async_event event;
 213         const struct rte_intr_conf *const intr_conf =
 214                 &priv->dev->data->dev_conf.intr_conf;
 215         unsigned int i;
 216
 217         /* Read all message and acknowledge them. */
 218         while (!ibv_get_async_event(priv->ctx, &event)) {
 219                 switch (event.event_type) {
 220                 case IBV_EVENT_PORT_ACTIVE:
 221                 case IBV_EVENT_PORT_ERR:
 222                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 223                                 ++caught[LSC];
 224                         break;
 225                 case IBV_EVENT_DEVICE_FATAL:
 226                         if (intr_conf->rmv)
 227                                 ++caught[RMV];
 228                         break;
 229                 default:
 230                         DEBUG("event type %d on physical port %d not handled",
 231                               event.event_type, event.element.port_num);
 232                 }
 233                 ibv_ack_async_event(&event);
 234         }
 235         for (i = 0; i != RTE_DIM(caught); ++i)
 236                 if (caught[i])
 237                         _rte_eth_dev_callback_process(priv->dev, type[i],
 238                                                       NULL, NULL);
 239 }
 240
 241 /**
 242  * Uninstall interrupt handler.
 243  *
 244  * @param priv
 245  *   Pointer to private structure.
 246  *
 247  * @return
 248  *   0 on success, negative errno value otherwise and rte_errno is set.
 249  */
 250 int
 251 mlx4_intr_uninstall(struct priv *priv)
 252 {
 253         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 254
 255         if (priv->intr_handle.fd != -1) {
 256                 rte_intr_callback_unregister(&priv->intr_handle,
 257                                              (void (*)(void *))
 258                                              mlx4_interrupt_handler,
 259                                              priv);
 260                 priv->intr_handle.fd = -1;
 261         }
 262         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 263         priv->intr_alarm = 0;
 264         mlx4_rx_intr_vec_disable(priv);
 265         rte_errno = err;
 266         return 0;
 267 }
 268
 269 /**
 270  * Install interrupt handler.
 271  *
 272  * @param priv
 273  *   Pointer to private structure.
 274  *
 275  * @return
 276  *   0 on success, negative errno value otherwise and rte_errno is set.
 277  */
 278 int
 279 mlx4_intr_install(struct priv *priv)
 280 {
 281         const struct rte_intr_conf *const intr_conf =
 282                 &priv->dev->data->dev_conf.intr_conf;
 283         int rc;
 284
 285         mlx4_intr_uninstall(priv);
 286         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 287                 goto error;
 288         if (intr_conf->lsc | intr_conf->rmv) {
 289                 priv->intr_handle.fd = priv->ctx->async_fd;
 290                 rc = rte_intr_callback_register(&priv->intr_handle,
 291                                                 (void (*)(void *))
 292                                                 mlx4_interrupt_handler,
 293                                                 priv);
 294                 if (rc < 0) {
 295                         rte_errno = -rc;
 296                         goto error;
 297                 }
 298         }
 299         return 0;
 300 error:
 301         mlx4_intr_uninstall(priv);
 302         return -rte_errno;
 303 }
 304
 305 /**
 306  * DPDK callback for Rx queue interrupt disable.
 307  *
 308  * @param dev
 309  *   Pointer to Ethernet device structure.
 310  * @param idx
 311  *   Rx queue index.
 312  *
 313  * @return
 314  *   0 on success, negative errno value otherwise and rte_errno is set.
 315  */
 316 int
 317 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 318 {
 319         struct rxq *rxq = dev->data->rx_queues[idx];
 320         struct ibv_cq *ev_cq;
 321         void *ev_ctx;
 322         int ret;
 323
 324         if (!rxq || !rxq->channel) {
 325                 ret = EINVAL;
 326         } else {
 327                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
 328                 if (ret || ev_cq != rxq->cq)
 329                         ret = EINVAL;
 330         }
 331         if (ret) {
 332                 rte_errno = ret;
 333                 WARN("unable to disable interrupt on rx queue %d",
 334                      idx);
 335         } else {
 336                 ibv_ack_cq_events(rxq->cq, 1);
 337         }
 338         return -ret;
 339 }
 340
 341 /**
 342  * DPDK callback for Rx queue interrupt enable.
 343  *
 344  * @param dev
 345  *   Pointer to Ethernet device structure.
 346  * @param idx
 347  *   Rx queue index.
 348  *
 349  * @return
 350  *   0 on success, negative errno value otherwise and rte_errno is set.
 351  */
 352 int
 353 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 354 {
 355         struct rxq *rxq = dev->data->rx_queues[idx];
 356         int ret;
 357
 358         if (!rxq || !rxq->channel)
 359                 ret = EINVAL;
 360         else
 361                 ret = ibv_req_notify_cq(rxq->cq, 0);
 362         if (ret) {
 363                 rte_errno = ret;
 364                 WARN("unable to arm interrupt on rx queue %d", idx);
 365         }
 366         return -ret;
 367 }