drivers/net/mlx4/mlx4_intr.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright 2017 6WIND S.A.
   5  *   Copyright 2017 Mellanox
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of 6WIND S.A. nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /**
  35  * @file
  36  * Interrupts handling for mlx4 driver.
  37  */
  38
  39 #include <assert.h>
  40 #include <errno.h>
  41 #include <stdint.h>
  42 #include <stdlib.h>
  43
  44 /* Verbs headers do not support -pedantic. */
  45 #ifdef PEDANTIC
  46 #pragma GCC diagnostic ignored "-Wpedantic"
  47 #endif
  48 #include <infiniband/verbs.h>
  49 #ifdef PEDANTIC
  50 #pragma GCC diagnostic error "-Wpedantic"
  51 #endif
  52
  53 #include <rte_alarm.h>
  54 #include <rte_errno.h>
  55 #include <rte_ethdev.h>
  56 #include <rte_io.h>
  57 #include <rte_interrupts.h>
  58
  59 #include "mlx4.h"
  60 #include "mlx4_rxtx.h"
  61 #include "mlx4_utils.h"
  62
  63 static int mlx4_link_status_check(struct priv *priv);
  64
  65 /**
  66  * Clean up Rx interrupts handler.
  67  *
  68  * @param priv
  69  *   Pointer to private structure.
  70  */
  71 static void
  72 mlx4_rx_intr_vec_disable(struct priv *priv)
  73 {
  74         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  75
  76         rte_intr_free_epoll_fd(intr_handle);
  77         free(intr_handle->intr_vec);
  78         intr_handle->nb_efd = 0;
  79         intr_handle->intr_vec = NULL;
  80 }
  81
  82 /**
  83  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  84  *
  85  * @param priv
  86  *   Pointer to private structure.
  87  *
  88  * @return
  89  *   0 on success, negative errno value otherwise and rte_errno is set.
  90  */
  91 static int
  92 mlx4_rx_intr_vec_enable(struct priv *priv)
  93 {
  94         unsigned int i;
  95         unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
  96         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  97         unsigned int count = 0;
  98         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  99
 100         mlx4_rx_intr_vec_disable(priv);
 101         intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
 102         if (intr_handle->intr_vec == NULL) {
 103                 rte_errno = ENOMEM;
 104                 ERROR("failed to allocate memory for interrupt vector,"
 105                       " Rx interrupts will not be supported");
 106                 return -rte_errno;
 107         }
 108         for (i = 0; i != n; ++i) {
 109                 struct rxq *rxq = priv->dev->data->rx_queues[i];
 110
 111                 /* Skip queues that cannot request interrupts. */
 112                 if (!rxq || !rxq->channel) {
 113                         /* Use invalid intr_vec[] index to disable entry. */
 114                         intr_handle->intr_vec[i] =
 115                                 RTE_INTR_VEC_RXTX_OFFSET +
 116                                 RTE_MAX_RXTX_INTR_VEC_ID;
 117                         continue;
 118                 }
 119                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 120                         rte_errno = E2BIG;
 121                         ERROR("too many Rx queues for interrupt vector size"
 122                               " (%d), Rx interrupts cannot be enabled",
 123                               RTE_MAX_RXTX_INTR_VEC_ID);
 124                         mlx4_rx_intr_vec_disable(priv);
 125                         return -rte_errno;
 126                 }
 127                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 128                 intr_handle->efds[count] = rxq->channel->fd;
 129                 count++;
 130         }
 131         if (!count)
 132                 mlx4_rx_intr_vec_disable(priv);
 133         else
 134                 intr_handle->nb_efd = count;
 135         return 0;
 136 }
 137
 138 /**
 139  * Process scheduled link status check.
 140  *
 141  * If LSC interrupts are requested, process related callback.
 142  *
 143  * @param priv
 144  *   Pointer to private structure.
 145  */
 146 static void
 147 mlx4_link_status_alarm(struct priv *priv)
 148 {
 149         const struct rte_intr_conf *const intr_conf =
 150                 &priv->dev->data->dev_conf.intr_conf;
 151
 152         assert(priv->intr_alarm == 1);
 153         priv->intr_alarm = 0;
 154         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 155                 _rte_eth_dev_callback_process(priv->dev,
 156                                               RTE_ETH_EVENT_INTR_LSC,
 157                                               NULL, NULL);
 158 }
 159
 160 /**
 161  * Check link status.
 162  *
 163  * In case of inconsistency, another check is scheduled.
 164  *
 165  * @param priv
 166  *   Pointer to private structure.
 167  *
 168  * @return
 169  *   0 on success (link status is consistent), negative errno value
 170  *   otherwise and rte_errno is set.
 171  */
 172 static int
 173 mlx4_link_status_check(struct priv *priv)
 174 {
 175         struct rte_eth_link *link = &priv->dev->data->dev_link;
 176         int ret = mlx4_link_update(priv->dev, 0);
 177
 178         if (ret)
 179                 return ret;
 180         if ((!link->link_speed && link->link_status) ||
 181             (link->link_speed && !link->link_status)) {
 182                 if (!priv->intr_alarm) {
 183                         /* Inconsistent status, check again later. */
 184                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 185                                                 (void (*)(void *))
 186                                                 mlx4_link_status_alarm,
 187                                                 priv);
 188                         if (ret)
 189                                 return ret;
 190                         priv->intr_alarm = 1;
 191                 }
 192                 rte_errno = EINPROGRESS;
 193                 return -rte_errno;
 194         }
 195         return 0;
 196 }
 197
 198 /**
 199  * Handle interrupts from the NIC.
 200  *
 201  * @param priv
 202  *   Pointer to private structure.
 203  */
 204 static void
 205 mlx4_interrupt_handler(struct priv *priv)
 206 {
 207         enum { LSC, RMV, };
 208         static const enum rte_eth_event_type type[] = {
 209                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
 210                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
 211         };
 212         uint32_t caught[RTE_DIM(type)] = { 0 };
 213         struct ibv_async_event event;
 214         const struct rte_intr_conf *const intr_conf =
 215                 &priv->dev->data->dev_conf.intr_conf;
 216         unsigned int i;
 217
 218         /* Read all message and acknowledge them. */
 219         while (!ibv_get_async_event(priv->ctx, &event)) {
 220                 switch (event.event_type) {
 221                 case IBV_EVENT_PORT_ACTIVE:
 222                 case IBV_EVENT_PORT_ERR:
 223                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 224                                 ++caught[LSC];
 225                         break;
 226                 case IBV_EVENT_DEVICE_FATAL:
 227                         if (intr_conf->rmv)
 228                                 ++caught[RMV];
 229                         break;
 230                 default:
 231                         DEBUG("event type %d on physical port %d not handled",
 232                               event.event_type, event.element.port_num);
 233                 }
 234                 ibv_ack_async_event(&event);
 235         }
 236         for (i = 0; i != RTE_DIM(caught); ++i)
 237                 if (caught[i])
 238                         _rte_eth_dev_callback_process(priv->dev, type[i],
 239                                                       NULL, NULL);
 240 }
 241
 242 /**
 243  * MLX4 CQ notification .
 244  *
 245  * @param rxq
 246  *   Pointer to receive queue structure.
 247  * @param solicited
 248  *   Is request solicited or not.
 249  */
 250 static void
 251 mlx4_arm_cq(struct rxq *rxq, int solicited)
 252 {
 253         struct mlx4_cq *cq = &rxq->mcq;
 254         uint64_t doorbell;
 255         uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
 256         uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
 257         uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
 258
 259         *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
 260         /*
 261          * Make sure that the doorbell record in host memory is
 262          * written before ringing the doorbell via PCI MMIO.
 263          */
 264         rte_wmb();
 265         doorbell = sn << 28 | cmd | cq->cqn;
 266         doorbell <<= 32;
 267         doorbell |= ci;
 268         rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 269 }
 270
 271 /**
 272  * Uninstall interrupt handler.
 273  *
 274  * @param priv
 275  *   Pointer to private structure.
 276  *
 277  * @return
 278  *   0 on success, negative errno value otherwise and rte_errno is set.
 279  */
 280 int
 281 mlx4_intr_uninstall(struct priv *priv)
 282 {
 283         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 284
 285         if (priv->intr_handle.fd != -1) {
 286                 rte_intr_callback_unregister(&priv->intr_handle,
 287                                              (void (*)(void *))
 288                                              mlx4_interrupt_handler,
 289                                              priv);
 290                 priv->intr_handle.fd = -1;
 291         }
 292         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 293         priv->intr_alarm = 0;
 294         mlx4_rx_intr_vec_disable(priv);
 295         rte_errno = err;
 296         return 0;
 297 }
 298
 299 /**
 300  * Install interrupt handler.
 301  *
 302  * @param priv
 303  *   Pointer to private structure.
 304  *
 305  * @return
 306  *   0 on success, negative errno value otherwise and rte_errno is set.
 307  */
 308 int
 309 mlx4_intr_install(struct priv *priv)
 310 {
 311         const struct rte_intr_conf *const intr_conf =
 312                 &priv->dev->data->dev_conf.intr_conf;
 313         int rc;
 314
 315         mlx4_intr_uninstall(priv);
 316         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 317                 goto error;
 318         if (intr_conf->lsc | intr_conf->rmv) {
 319                 priv->intr_handle.fd = priv->ctx->async_fd;
 320                 rc = rte_intr_callback_register(&priv->intr_handle,
 321                                                 (void (*)(void *))
 322                                                 mlx4_interrupt_handler,
 323                                                 priv);
 324                 if (rc < 0) {
 325                         rte_errno = -rc;
 326                         goto error;
 327                 }
 328         }
 329         return 0;
 330 error:
 331         mlx4_intr_uninstall(priv);
 332         return -rte_errno;
 333 }
 334
 335 /**
 336  * DPDK callback for Rx queue interrupt disable.
 337  *
 338  * @param dev
 339  *   Pointer to Ethernet device structure.
 340  * @param idx
 341  *   Rx queue index.
 342  *
 343  * @return
 344  *   0 on success, negative errno value otherwise and rte_errno is set.
 345  */
 346 int
 347 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 348 {
 349         struct rxq *rxq = dev->data->rx_queues[idx];
 350         struct ibv_cq *ev_cq;
 351         void *ev_ctx;
 352         int ret;
 353
 354         if (!rxq || !rxq->channel) {
 355                 ret = EINVAL;
 356         } else {
 357                 ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
 358                 if (ret || ev_cq != rxq->cq)
 359                         ret = EINVAL;
 360         }
 361         if (ret) {
 362                 rte_errno = ret;
 363                 WARN("unable to disable interrupt on rx queue %d",
 364                      idx);
 365         } else {
 366                 rxq->mcq.arm_sn++;
 367                 ibv_ack_cq_events(rxq->cq, 1);
 368         }
 369         return -ret;
 370 }
 371
 372 /**
 373  * DPDK callback for Rx queue interrupt enable.
 374  *
 375  * @param dev
 376  *   Pointer to Ethernet device structure.
 377  * @param idx
 378  *   Rx queue index.
 379  *
 380  * @return
 381  *   0 on success, negative errno value otherwise and rte_errno is set.
 382  */
 383 int
 384 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 385 {
 386         struct rxq *rxq = dev->data->rx_queues[idx];
 387         int ret = 0;
 388
 389         if (!rxq || !rxq->channel) {
 390                 ret = EINVAL;
 391                 rte_errno = ret;
 392                 WARN("unable to arm interrupt on rx queue %d", idx);
 393         } else {
 394                 mlx4_arm_cq(rxq, 0);
 395         }
 396         return -ret;
 397 }