drivers/net/mlx4/mlx4_intr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Interrupts handling for mlx4 driver.
  37  */
  38
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <stdlib.h>
  43
  44 /* Verbs headers do not support -pedantic. */
  45 #ifdef PEDANTIC
  46 #pragma GCC diagnostic ignored "-Wpedantic"
  47 #endif
  48 #include <infiniband/verbs.h>
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic error "-Wpedantic"
  51 #endif
  52
  53 #include <rte_alarm.h>
  54 #include <rte_errno.h>
  55 #include <rte_ethdev_driver.h>
  56 #include <rte_io.h>
  57 #include <rte_interrupts.h>
  58
  59 #include "mlx4.h"
  60 #include "mlx4_glue.h"
  61 #include "mlx4_rxtx.h"
  62 #include "mlx4_utils.h"
  63
  64 static int mlx4_link_status_check(struct priv *priv);
  65
  66 /**
  67  * Clean up Rx interrupts handler.
  68  *
  69  * @param priv
  70  *   Pointer to private structure.
  71  */
  72 static void
  73 mlx4_rx_intr_vec_disable(struct priv *priv)
  74 {
  75         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  76
  77         rte_intr_free_epoll_fd(intr_handle);
  78         free(intr_handle->intr_vec);
  79         intr_handle->nb_efd = 0;
  80         intr_handle->intr_vec = NULL;
  81 }
  82
  83 /**
  84  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  85  *
  86  * @param priv
  87  *   Pointer to private structure.
  88  *
  89  * @return
  90  *   0 on success, negative errno value otherwise and rte_errno is set.
  91  */
  92 static int
  93 mlx4_rx_intr_vec_enable(struct priv *priv)
  94 {
  95         unsigned int i;
  96         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
  97         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  98         unsigned int count = 0;
  99         struct rte_intr_handle *intr_handle = &priv->intr_handle;
 100
 101         mlx4_rx_intr_vec_disable(priv);
 102         intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
 103         if (intr_handle->intr_vec == NULL) {
 104                 rte_errno = ENOMEM;
 105                 ERROR("failed to allocate memory for interrupt vector,"
 106                       " Rx interrupts will not be supported");
 107                 return -rte_errno;
 108         }
 109         for (i = 0; i != n; ++i) {
 110                 struct rxq *rxq = priv->dev->data->rx_queues[i];
 111
 112                 /* Skip queues that cannot request interrupts. */
 113                 if (!rxq || !rxq->channel) {
 114                         /* Use invalid intr_vec[] index to disable entry. */
 115                         intr_handle->intr_vec[i] =
 116                                 RTE_INTR_VEC_RXTX_OFFSET +
 117                                 RTE_MAX_RXTX_INTR_VEC_ID;
 118                         continue;
 119                 }
 120                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 121                         rte_errno = E2BIG;
 122                         ERROR("too many Rx queues for interrupt vector size"
 123                               " (%d), Rx interrupts cannot be enabled",
 124                               RTE_MAX_RXTX_INTR_VEC_ID);
 125                         mlx4_rx_intr_vec_disable(priv);
 126                         return -rte_errno;
 127                 }
 128                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 129                 intr_handle->efds[count] = rxq->channel->fd;
 130                 count++;
 131         }
 132         if (!count)
 133                 mlx4_rx_intr_vec_disable(priv);
 134         else
 135                 intr_handle->nb_efd = count;
 136         return 0;
 137 }
 138
 139 /**
 140  * Process scheduled link status check.
 141  *
 142  * If LSC interrupts are requested, process related callback.
 143  *
 144  * @param priv
 145  *   Pointer to private structure.
 146  */
 147 static void
 148 mlx4_link_status_alarm(struct priv *priv)
 149 {
 150         const struct rte_intr_conf *const intr_conf =
 151                 &priv->dev->data->dev_conf.intr_conf;
 152
 153         assert(priv->intr_alarm == 1);
 154         priv->intr_alarm = 0;
 155         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 156                 _rte_eth_dev_callback_process(priv->dev,
 157                                               RTE_ETH_EVENT_INTR_LSC,
 158                                               NULL);
 159 }
 160
 161 /**
 162  * Check link status.
 163  *
 164  * In case of inconsistency, another check is scheduled.
 165  *
 166  * @param priv
 167  *   Pointer to private structure.
 168  *
 169  * @return
 170  *   0 on success (link status is consistent), negative errno value
 171  *   otherwise and rte_errno is set.
 172  */
 173 static int
 174 mlx4_link_status_check(struct priv *priv)
 175 {
 176         struct rte_eth_link *link = &priv->dev->data->dev_link;
 177         int ret = mlx4_link_update(priv->dev, 0);
 178
 179         if (ret)
 180                 return ret;
 181         if ((!link->link_speed && link->link_status) ||
 182             (link->link_speed && !link->link_status)) {
 183                 if (!priv->intr_alarm) {
 184                         /* Inconsistent status, check again later. */
 185                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 186                                                 (void (*)(void *))
 187                                                 mlx4_link_status_alarm,
 188                                                 priv);
 189                         if (ret)
 190                                 return ret;
 191                         priv->intr_alarm = 1;
 192                 }
 193                 rte_errno = EINPROGRESS;
 194                 return -rte_errno;
 195         }
 196         return 0;
 197 }
 198
 199 /**
 200  * Handle interrupts from the NIC.
 201  *
 202  * @param priv
 203  *   Pointer to private structure.
 204  */
 205 static void
 206 mlx4_interrupt_handler(struct priv *priv)
 207 {
 208         enum { LSC, RMV, };
 209         static const enum rte_eth_event_type type[] = {
 210                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
 211                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
 212         };
 213         uint32_t caught[RTE_DIM(type)] = { 0 };
 214         struct ibv_async_event event;
 215         const struct rte_intr_conf *const intr_conf =
 216                 &priv->dev->data->dev_conf.intr_conf;
 217         unsigned int i;
 218
 219         /* Read all message and acknowledge them. */
 220         while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
 221                 switch (event.event_type) {
 222                 case IBV_EVENT_PORT_ACTIVE:
 223                 case IBV_EVENT_PORT_ERR:
 224                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 225                                 ++caught[LSC];
 226                         break;
 227                 case IBV_EVENT_DEVICE_FATAL:
 228                         if (intr_conf->rmv)
 229                                 ++caught[RMV];
 230                         break;
 231                 default:
 232                         DEBUG("event type %d on physical port %d not handled",
 233                               event.event_type, event.element.port_num);
 234                 }
 235                 mlx4_glue->ack_async_event(&event);
 236         }
 237         for (i = 0; i != RTE_DIM(caught); ++i)
 238                 if (caught[i])
 239                         _rte_eth_dev_callback_process(priv->dev, type[i],
 240                                                       NULL);
 241 }
 242
 243 /**
 244  * MLX4 CQ notification .
 245  *
 246  * @param rxq
 247  *   Pointer to receive queue structure.
 248  * @param solicited
 249  *   Is request solicited or not.
 250  */
 251 static void
 252 mlx4_arm_cq(struct rxq *rxq, int solicited)
 253 {
 254         struct mlx4_cq *cq = &rxq->mcq;
 255         uint64_t doorbell;
 256         uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
 257         uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
 258         uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
 259
 260         *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
 261         /*
 262          * Make sure that the doorbell record in host memory is
 263          * written before ringing the doorbell via PCI MMIO.
 264          */
 265         rte_wmb();
 266         doorbell = sn << 28 | cmd | cq->cqn;
 267         doorbell <<= 32;
 268         doorbell |= ci;
 269         rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 270 }
 271
 272 /**
 273  * Uninstall interrupt handler.
 274  *
 275  * @param priv
 276  *   Pointer to private structure.
 277  *
 278  * @return
 279  *   0 on success, negative errno value otherwise and rte_errno is set.
 280  */
 281 int
 282 mlx4_intr_uninstall(struct priv *priv)
 283 {
 284         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 285
 286         if (priv->intr_handle.fd != -1) {
 287                 rte_intr_callback_unregister(&priv->intr_handle,
 288                                              (void (*)(void *))
 289                                              mlx4_interrupt_handler,
 290                                              priv);
 291                 priv->intr_handle.fd = -1;
 292         }
 293         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 294         priv->intr_alarm = 0;
 295         mlx4_rxq_intr_disable(priv);
 296         rte_errno = err;
 297         return 0;
 298 }
 299
 300 /**
 301  * Install interrupt handler.
 302  *
 303  * @param priv
 304  *   Pointer to private structure.
 305  *
 306  * @return
 307  *   0 on success, negative errno value otherwise and rte_errno is set.
 308  */
 309 int
 310 mlx4_intr_install(struct priv *priv)
 311 {
 312         const struct rte_intr_conf *const intr_conf =
 313                 &priv->dev->data->dev_conf.intr_conf;
 314         int rc;
 315
 316         mlx4_intr_uninstall(priv);
 317         if (intr_conf->lsc | intr_conf->rmv) {
 318                 priv->intr_handle.fd = priv->ctx->async_fd;
 319                 rc = rte_intr_callback_register(&priv->intr_handle,
 320                                                 (void (*)(void *))
 321                                                 mlx4_interrupt_handler,
 322                                                 priv);
 323                 if (rc < 0) {
 324                         rte_errno = -rc;
 325                         goto error;
 326                 }
 327         }
 328         return 0;
 329 error:
 330         mlx4_intr_uninstall(priv);
 331         return -rte_errno;
 332 }
 333
 334 /**
 335  * DPDK callback for Rx queue interrupt disable.
 336  *
 337  * @param dev
 338  *   Pointer to Ethernet device structure.
 339  * @param idx
 340  *   Rx queue index.
 341  *
 342  * @return
 343  *   0 on success, negative errno value otherwise and rte_errno is set.
 344  */
 345 int
 346 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 347 {
 348         struct rxq *rxq = dev->data->rx_queues[idx];
 349         struct ibv_cq *ev_cq;
 350         void *ev_ctx;
 351         int ret;
 352
 353         if (!rxq || !rxq->channel) {
 354                 ret = EINVAL;
 355         } else {
 356                 ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
 357                                               &ev_ctx);
 358                 if (ret || ev_cq != rxq->cq)
 359                         ret = EINVAL;
 360         }
 361         if (ret) {
 362                 rte_errno = ret;
 363                 WARN("unable to disable interrupt on rx queue %d",
 364                      idx);
 365         } else {
 366                 rxq->mcq.arm_sn++;
 367                 mlx4_glue->ack_cq_events(rxq->cq, 1);
 368         }
 369         return -ret;
 370 }
 371
 372 /**
 373  * DPDK callback for Rx queue interrupt enable.
 374  *
 375  * @param dev
 376  *   Pointer to Ethernet device structure.
 377  * @param idx
 378  *   Rx queue index.
 379  *
 380  * @return
 381  *   0 on success, negative errno value otherwise and rte_errno is set.
 382  */
 383 int
 384 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 385 {
 386         struct rxq *rxq = dev->data->rx_queues[idx];
 387         int ret = 0;
 388
 389         if (!rxq || !rxq->channel) {
 390                 ret = EINVAL;
 391                 rte_errno = ret;
 392                 WARN("unable to arm interrupt on rx queue %d", idx);
 393         } else {
 394                 mlx4_arm_cq(rxq, 0);
 395         }
 396         return -ret;
 397 }
 398
 399 /**
 400  * Enable datapath interrupts.
 401  *
 402  * @param priv
 403  *   Pointer to private structure.
 404  *
 405  * @return
 406  *   0 on success, negative errno value otherwise and rte_errno is set.
 407  */
 408 int
 409 mlx4_rxq_intr_enable(struct priv *priv)
 410 {
 411         const struct rte_intr_conf *const intr_conf =
 412                 &priv->dev->data->dev_conf.intr_conf;
 413
 414         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 415                 goto error;
 416         return 0;
 417 error:
 418         return -rte_errno;
 419 }
 420
 421 /**
 422  * Disable datapath interrupts, keeping other interrupts intact.
 423  *
 424  * @param priv
 425  *   Pointer to private structure.
 426  */
 427 void
 428 mlx4_rxq_intr_disable(struct priv *priv)
 429 {
 430         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 431
 432         mlx4_rx_intr_vec_disable(priv);
 433         rte_errno = err;
 434 }