drivers/net/mlx4/mlx4_intr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 /**
   7  * @file
   8  * Interrupts handling for mlx4 driver.
   9  */
  10
  11 #include <errno.h>
  12 #include <stdint.h>
  13 #include <stdlib.h>
  14
  15 /* Verbs headers do not support -pedantic. */
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic ignored "-Wpedantic"
  18 #endif
  19 #include <infiniband/verbs.h>
  20 #ifdef PEDANTIC
  21 #pragma GCC diagnostic error "-Wpedantic"
  22 #endif
  23
  24 #include <rte_alarm.h>
  25 #include <rte_errno.h>
  26 #include <rte_ethdev_driver.h>
  27 #include <rte_io.h>
  28 #include <rte_interrupts.h>
  29
  30 #include "mlx4.h"
  31 #include "mlx4_glue.h"
  32 #include "mlx4_rxtx.h"
  33 #include "mlx4_utils.h"
  34
  35 static int mlx4_link_status_check(struct mlx4_priv *priv);
  36
  37 /**
  38  * Clean up Rx interrupts handler.
  39  *
  40  * @param priv
  41  *   Pointer to private structure.
  42  */
  43 static void
  44 mlx4_rx_intr_vec_disable(struct mlx4_priv *priv)
  45 {
  46         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  47
  48         rte_intr_free_epoll_fd(intr_handle);
  49         free(intr_handle->intr_vec);
  50         intr_handle->nb_efd = 0;
  51         intr_handle->intr_vec = NULL;
  52 }
  53
  54 /**
  55  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  56  *
  57  * @param priv
  58  *   Pointer to private structure.
  59  *
  60  * @return
  61  *   0 on success, negative errno value otherwise and rte_errno is set.
  62  */
  63 static int
  64 mlx4_rx_intr_vec_enable(struct mlx4_priv *priv)
  65 {
  66         unsigned int i;
  67         unsigned int rxqs_n = ETH_DEV(priv)->data->nb_rx_queues;
  68         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  69         unsigned int count = 0;
  70         struct rte_intr_handle *intr_handle = &priv->intr_handle;
  71
  72         mlx4_rx_intr_vec_disable(priv);
  73         intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
  74         if (intr_handle->intr_vec == NULL) {
  75                 rte_errno = ENOMEM;
  76                 ERROR("failed to allocate memory for interrupt vector,"
  77                       " Rx interrupts will not be supported");
  78                 return -rte_errno;
  79         }
  80         for (i = 0; i != n; ++i) {
  81                 struct rxq *rxq = ETH_DEV(priv)->data->rx_queues[i];
  82
  83                 /* Skip queues that cannot request interrupts. */
  84                 if (!rxq || !rxq->channel) {
  85                         /* Use invalid intr_vec[] index to disable entry. */
  86                         intr_handle->intr_vec[i] =
  87                                 RTE_INTR_VEC_RXTX_OFFSET +
  88                                 RTE_MAX_RXTX_INTR_VEC_ID;
  89                         continue;
  90                 }
  91                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
  92                         rte_errno = E2BIG;
  93                         ERROR("too many Rx queues for interrupt vector size"
  94                               " (%d), Rx interrupts cannot be enabled",
  95                               RTE_MAX_RXTX_INTR_VEC_ID);
  96                         mlx4_rx_intr_vec_disable(priv);
  97                         return -rte_errno;
  98                 }
  99                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 100                 intr_handle->efds[count] = rxq->channel->fd;
 101                 count++;
 102         }
 103         if (!count)
 104                 mlx4_rx_intr_vec_disable(priv);
 105         else
 106                 intr_handle->nb_efd = count;
 107         return 0;
 108 }
 109
 110 /**
 111  * Process scheduled link status check.
 112  *
 113  * If LSC interrupts are requested, process related callback.
 114  *
 115  * @param priv
 116  *   Pointer to private structure.
 117  */
 118 static void
 119 mlx4_link_status_alarm(struct mlx4_priv *priv)
 120 {
 121         const struct rte_intr_conf *const intr_conf =
 122                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 123
 124         MLX4_ASSERT(priv->intr_alarm == 1);
 125         priv->intr_alarm = 0;
 126         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 127                 _rte_eth_dev_callback_process(ETH_DEV(priv),
 128                                               RTE_ETH_EVENT_INTR_LSC,
 129                                               NULL);
 130 }
 131
 132 /**
 133  * Check link status.
 134  *
 135  * In case of inconsistency, another check is scheduled.
 136  *
 137  * @param priv
 138  *   Pointer to private structure.
 139  *
 140  * @return
 141  *   0 on success (link status is consistent), negative errno value
 142  *   otherwise and rte_errno is set.
 143  */
 144 static int
 145 mlx4_link_status_check(struct mlx4_priv *priv)
 146 {
 147         struct rte_eth_link *link = &ETH_DEV(priv)->data->dev_link;
 148         int ret = mlx4_link_update(ETH_DEV(priv), 0);
 149
 150         if (ret)
 151                 return ret;
 152         if ((!link->link_speed && link->link_status) ||
 153             (link->link_speed && !link->link_status)) {
 154                 if (!priv->intr_alarm) {
 155                         /* Inconsistent status, check again later. */
 156                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 157                                                 (void (*)(void *))
 158                                                 mlx4_link_status_alarm,
 159                                                 priv);
 160                         if (ret)
 161                                 return ret;
 162                         priv->intr_alarm = 1;
 163                 }
 164                 rte_errno = EINPROGRESS;
 165                 return -rte_errno;
 166         }
 167         return 0;
 168 }
 169
 170 /**
 171  * Handle interrupts from the NIC.
 172  *
 173  * @param priv
 174  *   Pointer to private structure.
 175  */
 176 static void
 177 mlx4_interrupt_handler(struct mlx4_priv *priv)
 178 {
 179         enum { LSC, RMV, };
 180         static const enum rte_eth_event_type type[] = {
 181                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
 182                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
 183         };
 184         uint32_t caught[RTE_DIM(type)] = { 0 };
 185         struct ibv_async_event event;
 186         const struct rte_intr_conf *const intr_conf =
 187                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 188         unsigned int i;
 189
 190         /* Read all message and acknowledge them. */
 191         while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
 192                 switch (event.event_type) {
 193                 case IBV_EVENT_PORT_ACTIVE:
 194                 case IBV_EVENT_PORT_ERR:
 195                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 196                                 ++caught[LSC];
 197                         break;
 198                 case IBV_EVENT_DEVICE_FATAL:
 199                         if (intr_conf->rmv)
 200                                 ++caught[RMV];
 201                         break;
 202                 default:
 203                         DEBUG("event type %d on physical port %d not handled",
 204                               event.event_type, event.element.port_num);
 205                 }
 206                 mlx4_glue->ack_async_event(&event);
 207         }
 208         for (i = 0; i != RTE_DIM(caught); ++i)
 209                 if (caught[i])
 210                         _rte_eth_dev_callback_process(ETH_DEV(priv), type[i],
 211                                                       NULL);
 212 }
 213
 214 /**
 215  * MLX4 CQ notification .
 216  *
 217  * @param rxq
 218  *   Pointer to receive queue structure.
 219  * @param solicited
 220  *   Is request solicited or not.
 221  */
 222 static void
 223 mlx4_arm_cq(struct rxq *rxq, int solicited)
 224 {
 225         struct mlx4_cq *cq = &rxq->mcq;
 226         uint64_t doorbell;
 227         uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
 228         uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
 229         uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
 230
 231         *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
 232         /*
 233          * Make sure that the doorbell record in host memory is
 234          * written before ringing the doorbell via PCI MMIO.
 235          */
 236         rte_wmb();
 237         doorbell = sn << 28 | cmd | cq->cqn;
 238         doorbell <<= 32;
 239         doorbell |= ci;
 240         rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 241 }
 242
 243 /**
 244  * Uninstall interrupt handler.
 245  *
 246  * @param priv
 247  *   Pointer to private structure.
 248  *
 249  * @return
 250  *   0 on success, negative errno value otherwise and rte_errno is set.
 251  */
 252 int
 253 mlx4_intr_uninstall(struct mlx4_priv *priv)
 254 {
 255         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 256
 257         if (priv->intr_handle.fd != -1) {
 258                 rte_intr_callback_unregister(&priv->intr_handle,
 259                                              (void (*)(void *))
 260                                              mlx4_interrupt_handler,
 261                                              priv);
 262                 priv->intr_handle.fd = -1;
 263         }
 264         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 265         priv->intr_alarm = 0;
 266         mlx4_rxq_intr_disable(priv);
 267         rte_errno = err;
 268         return 0;
 269 }
 270
 271 /**
 272  * Install interrupt handler.
 273  *
 274  * @param priv
 275  *   Pointer to private structure.
 276  *
 277  * @return
 278  *   0 on success, negative errno value otherwise and rte_errno is set.
 279  */
 280 int
 281 mlx4_intr_install(struct mlx4_priv *priv)
 282 {
 283         const struct rte_intr_conf *const intr_conf =
 284                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 285         int rc;
 286
 287         mlx4_intr_uninstall(priv);
 288         if (intr_conf->lsc | intr_conf->rmv) {
 289                 priv->intr_handle.fd = priv->ctx->async_fd;
 290                 rc = rte_intr_callback_register(&priv->intr_handle,
 291                                                 (void (*)(void *))
 292                                                 mlx4_interrupt_handler,
 293                                                 priv);
 294                 if (rc < 0) {
 295                         rte_errno = -rc;
 296                         goto error;
 297                 }
 298         }
 299         return 0;
 300 error:
 301         mlx4_intr_uninstall(priv);
 302         return -rte_errno;
 303 }
 304
 305 /**
 306  * DPDK callback for Rx queue interrupt disable.
 307  *
 308  * @param dev
 309  *   Pointer to Ethernet device structure.
 310  * @param idx
 311  *   Rx queue index.
 312  *
 313  * @return
 314  *   0 on success, negative errno value otherwise and rte_errno is set.
 315  */
 316 int
 317 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 318 {
 319         struct rxq *rxq = dev->data->rx_queues[idx];
 320         struct ibv_cq *ev_cq;
 321         void *ev_ctx;
 322         int ret;
 323
 324         if (!rxq || !rxq->channel) {
 325                 ret = EINVAL;
 326         } else {
 327                 ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
 328                                               &ev_ctx);
 329                 /** For non-zero ret save the errno (may be EAGAIN
 330                  * which means the get_cq_event function was called before
 331                  * receiving one).
 332                  */
 333                 if (ret)
 334                         ret = errno;
 335                 else if (ev_cq != rxq->cq)
 336                         ret = EINVAL;
 337         }
 338         if (ret) {
 339                 rte_errno = ret;
 340                 if (ret != EAGAIN)
 341                         WARN("unable to disable interrupt on rx queue %d",
 342                              idx);
 343         } else {
 344                 rxq->mcq.arm_sn++;
 345                 mlx4_glue->ack_cq_events(rxq->cq, 1);
 346         }
 347         return -ret;
 348 }
 349
 350 /**
 351  * DPDK callback for Rx queue interrupt enable.
 352  *
 353  * @param dev
 354  *   Pointer to Ethernet device structure.
 355  * @param idx
 356  *   Rx queue index.
 357  *
 358  * @return
 359  *   0 on success, negative errno value otherwise and rte_errno is set.
 360  */
 361 int
 362 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 363 {
 364         struct rxq *rxq = dev->data->rx_queues[idx];
 365         int ret = 0;
 366
 367         if (!rxq || !rxq->channel) {
 368                 ret = EINVAL;
 369                 rte_errno = ret;
 370                 WARN("unable to arm interrupt on rx queue %d", idx);
 371         } else {
 372                 mlx4_arm_cq(rxq, 0);
 373         }
 374         return -ret;
 375 }
 376
 377 /**
 378  * Enable datapath interrupts.
 379  *
 380  * @param priv
 381  *   Pointer to private structure.
 382  *
 383  * @return
 384  *   0 on success, negative errno value otherwise and rte_errno is set.
 385  */
 386 int
 387 mlx4_rxq_intr_enable(struct mlx4_priv *priv)
 388 {
 389         const struct rte_intr_conf *const intr_conf =
 390                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 391
 392         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 393                 goto error;
 394         return 0;
 395 error:
 396         return -rte_errno;
 397 }
 398
 399 /**
 400  * Disable datapath interrupts, keeping other interrupts intact.
 401  *
 402  * @param priv
 403  *   Pointer to private structure.
 404  */
 405 void
 406 mlx4_rxq_intr_disable(struct mlx4_priv *priv)
 407 {
 408         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 409
 410         mlx4_rx_intr_vec_disable(priv);
 411         rte_errno = err;
 412 }