drivers/net/mlx4/mlx4_intr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 /**
   7  * @file
   8  * Interrupts handling for mlx4 driver.
   9  */
  10
  11 #include <errno.h>
  12 #include <stdint.h>
  13 #include <stdlib.h>
  14
  15 /* Verbs headers do not support -pedantic. */
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic ignored "-Wpedantic"
  18 #endif
  19 #include <infiniband/verbs.h>
  20 #ifdef PEDANTIC
  21 #pragma GCC diagnostic error "-Wpedantic"
  22 #endif
  23
  24 #include <rte_alarm.h>
  25 #include <rte_errno.h>
  26 #include <ethdev_driver.h>
  27 #include <rte_io.h>
  28 #include <rte_interrupts.h>
  29
  30 #include "mlx4.h"
  31 #include "mlx4_glue.h"
  32 #include "mlx4_rxtx.h"
  33 #include "mlx4_utils.h"
  34
  35 static int mlx4_link_status_check(struct mlx4_priv *priv);
  36
  37 /**
  38  * Clean up Rx interrupts handler.
  39  *
  40  * @param priv
  41  *   Pointer to private structure.
  42  */
  43 static void
  44 mlx4_rx_intr_vec_disable(struct mlx4_priv *priv)
  45 {
  46         struct rte_intr_handle *intr_handle = priv->intr_handle;
  47
  48         rte_intr_free_epoll_fd(intr_handle);
  49         rte_intr_vec_list_free(intr_handle);
  50
  51         rte_intr_nb_efd_set(intr_handle, 0);
  52 }
  53
  54 /**
  55  * Allocate queue vector and fill epoll fd list for Rx interrupts.
  56  *
  57  * @param priv
  58  *   Pointer to private structure.
  59  *
  60  * @return
  61  *   0 on success, negative errno value otherwise and rte_errno is set.
  62  */
  63 static int
  64 mlx4_rx_intr_vec_enable(struct mlx4_priv *priv)
  65 {
  66         unsigned int i;
  67         unsigned int rxqs_n = ETH_DEV(priv)->data->nb_rx_queues;
  68         unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
  69         unsigned int count = 0;
  70         struct rte_intr_handle *intr_handle = priv->intr_handle;
  71
  72         mlx4_rx_intr_vec_disable(priv);
  73         if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
  74                 rte_errno = ENOMEM;
  75                 ERROR("failed to allocate memory for interrupt vector,"
  76                       " Rx interrupts will not be supported");
  77                 return -rte_errno;
  78         }
  79         for (i = 0; i != n; ++i) {
  80                 struct rxq *rxq = ETH_DEV(priv)->data->rx_queues[i];
  81
  82                 /* Skip queues that cannot request interrupts. */
  83                 if (!rxq || !rxq->channel) {
  84                         /* Use invalid intr_vec[] index to disable entry. */
  85                         if (rte_intr_vec_list_index_set(intr_handle, i,
  86                         RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID))
  87                                 return -rte_errno;
  88                         continue;
  89                 }
  90                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
  91                         rte_errno = E2BIG;
  92                         ERROR("too many Rx queues for interrupt vector size"
  93                               " (%d), Rx interrupts cannot be enabled",
  94                               RTE_MAX_RXTX_INTR_VEC_ID);
  95                         mlx4_rx_intr_vec_disable(priv);
  96                         return -rte_errno;
  97                 }
  98
  99                 if (rte_intr_vec_list_index_set(intr_handle, i,
 100                                         RTE_INTR_VEC_RXTX_OFFSET + count))
 101                         return -rte_errno;
 102
 103                 if (rte_intr_efds_index_set(intr_handle, i,
 104                                                    rxq->channel->fd))
 105                         return -rte_errno;
 106
 107                 count++;
 108         }
 109         if (!count)
 110                 mlx4_rx_intr_vec_disable(priv);
 111         else if (rte_intr_nb_efd_set(intr_handle, count))
 112                 return -rte_errno;
 113         return 0;
 114 }
 115
 116 /**
 117  * Process scheduled link status check.
 118  *
 119  * If LSC interrupts are requested, process related callback.
 120  *
 121  * @param priv
 122  *   Pointer to private structure.
 123  */
 124 static void
 125 mlx4_link_status_alarm(struct mlx4_priv *priv)
 126 {
 127         const struct rte_eth_intr_conf *const intr_conf =
 128                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 129
 130         MLX4_ASSERT(priv->intr_alarm == 1);
 131         priv->intr_alarm = 0;
 132         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 133                 rte_eth_dev_callback_process(ETH_DEV(priv),
 134                                              RTE_ETH_EVENT_INTR_LSC,
 135                                              NULL);
 136 }
 137
 138 /**
 139  * Check link status.
 140  *
 141  * In case of inconsistency, another check is scheduled.
 142  *
 143  * @param priv
 144  *   Pointer to private structure.
 145  *
 146  * @return
 147  *   0 on success (link status is consistent), negative errno value
 148  *   otherwise and rte_errno is set.
 149  */
 150 static int
 151 mlx4_link_status_check(struct mlx4_priv *priv)
 152 {
 153         struct rte_eth_link *link = &ETH_DEV(priv)->data->dev_link;
 154         int ret = mlx4_link_update(ETH_DEV(priv), 0);
 155
 156         if (ret)
 157                 return ret;
 158         if ((!link->link_speed && link->link_status) ||
 159             (link->link_speed && !link->link_status)) {
 160                 if (!priv->intr_alarm) {
 161                         /* Inconsistent status, check again later. */
 162                         ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
 163                                                 (void (*)(void *))
 164                                                 mlx4_link_status_alarm,
 165                                                 priv);
 166                         if (ret)
 167                                 return ret;
 168                         priv->intr_alarm = 1;
 169                 }
 170                 rte_errno = EINPROGRESS;
 171                 return -rte_errno;
 172         }
 173         return 0;
 174 }
 175
 176 /**
 177  * Handle interrupts from the NIC.
 178  *
 179  * @param priv
 180  *   Pointer to private structure.
 181  */
 182 static void
 183 mlx4_interrupt_handler(struct mlx4_priv *priv)
 184 {
 185         enum { LSC, RMV, };
 186         static const enum rte_eth_event_type type[] = {
 187                 [LSC] = RTE_ETH_EVENT_INTR_LSC,
 188                 [RMV] = RTE_ETH_EVENT_INTR_RMV,
 189         };
 190         uint32_t caught[RTE_DIM(type)] = { 0 };
 191         struct ibv_async_event event;
 192         const struct rte_eth_intr_conf *const intr_conf =
 193                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 194         unsigned int i;
 195
 196         /* Read all message and acknowledge them. */
 197         while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
 198                 switch (event.event_type) {
 199                 case IBV_EVENT_PORT_ACTIVE:
 200                 case IBV_EVENT_PORT_ERR:
 201                         if (intr_conf->lsc && !mlx4_link_status_check(priv))
 202                                 ++caught[LSC];
 203                         break;
 204                 case IBV_EVENT_DEVICE_FATAL:
 205                         if (intr_conf->rmv)
 206                                 ++caught[RMV];
 207                         break;
 208                 default:
 209                         DEBUG("event type %d on physical port %d not handled",
 210                               event.event_type, event.element.port_num);
 211                 }
 212                 mlx4_glue->ack_async_event(&event);
 213         }
 214         for (i = 0; i != RTE_DIM(caught); ++i)
 215                 if (caught[i])
 216                         rte_eth_dev_callback_process(ETH_DEV(priv), type[i],
 217                                                      NULL);
 218 }
 219
 220 /**
 221  * MLX4 CQ notification .
 222  *
 223  * @param rxq
 224  *   Pointer to receive queue structure.
 225  * @param solicited
 226  *   Is request solicited or not.
 227  */
 228 static void
 229 mlx4_arm_cq(struct rxq *rxq, int solicited)
 230 {
 231         struct mlx4_cq *cq = &rxq->mcq;
 232         uint64_t doorbell;
 233         uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
 234         uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
 235         uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
 236
 237         *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
 238         /*
 239          * Make sure that the doorbell record in host memory is
 240          * written before ringing the doorbell via PCI MMIO.
 241          */
 242         rte_wmb();
 243         doorbell = sn << 28 | cmd | cq->cqn;
 244         doorbell <<= 32;
 245         doorbell |= ci;
 246         rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 247 }
 248
 249 /**
 250  * Uninstall interrupt handler.
 251  *
 252  * @param priv
 253  *   Pointer to private structure.
 254  *
 255  * @return
 256  *   0 on success, negative errno value otherwise and rte_errno is set.
 257  */
 258 int
 259 mlx4_intr_uninstall(struct mlx4_priv *priv)
 260 {
 261         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 262
 263         if (rte_intr_fd_get(priv->intr_handle) != -1) {
 264                 rte_intr_callback_unregister(priv->intr_handle,
 265                                              (void (*)(void *))
 266                                              mlx4_interrupt_handler,
 267                                              priv);
 268                 if (rte_intr_fd_set(priv->intr_handle, -1))
 269                         return -rte_errno;
 270         }
 271         rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 272         priv->intr_alarm = 0;
 273         mlx4_rxq_intr_disable(priv);
 274         rte_errno = err;
 275         return 0;
 276 }
 277
 278 /**
 279  * Install interrupt handler.
 280  *
 281  * @param priv
 282  *   Pointer to private structure.
 283  *
 284  * @return
 285  *   0 on success, negative errno value otherwise and rte_errno is set.
 286  */
 287 int
 288 mlx4_intr_install(struct mlx4_priv *priv)
 289 {
 290         const struct rte_eth_intr_conf *const intr_conf =
 291                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 292         int rc;
 293
 294         mlx4_intr_uninstall(priv);
 295         if (intr_conf->lsc | intr_conf->rmv) {
 296                 if (rte_intr_fd_set(priv->intr_handle, priv->ctx->async_fd))
 297                         return -rte_errno;
 298
 299                 rc = rte_intr_callback_register(priv->intr_handle,
 300                                                 (void (*)(void *))
 301                                                 mlx4_interrupt_handler,
 302                                                 priv);
 303                 if (rc < 0) {
 304                         rte_errno = -rc;
 305                         goto error;
 306                 }
 307         }
 308         return 0;
 309 error:
 310         mlx4_intr_uninstall(priv);
 311         return -rte_errno;
 312 }
 313
 314 /**
 315  * DPDK callback for Rx queue interrupt disable.
 316  *
 317  * @param dev
 318  *   Pointer to Ethernet device structure.
 319  * @param idx
 320  *   Rx queue index.
 321  *
 322  * @return
 323  *   0 on success, negative errno value otherwise and rte_errno is set.
 324  */
 325 int
 326 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 327 {
 328         struct rxq *rxq = dev->data->rx_queues[idx];
 329         struct ibv_cq *ev_cq;
 330         void *ev_ctx;
 331         int ret;
 332
 333         if (!rxq || !rxq->channel) {
 334                 ret = EINVAL;
 335         } else {
 336                 ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
 337                                               &ev_ctx);
 338                 /** For non-zero ret save the errno (may be EAGAIN
 339                  * which means the get_cq_event function was called before
 340                  * receiving one).
 341                  */
 342                 if (ret)
 343                         ret = errno;
 344                 else if (ev_cq != rxq->cq)
 345                         ret = EINVAL;
 346         }
 347         if (ret) {
 348                 rte_errno = ret;
 349                 if (ret != EAGAIN)
 350                         WARN("unable to disable interrupt on rx queue %d",
 351                              idx);
 352         } else {
 353                 rxq->mcq.arm_sn++;
 354                 mlx4_glue->ack_cq_events(rxq->cq, 1);
 355         }
 356         return -ret;
 357 }
 358
 359 /**
 360  * DPDK callback for Rx queue interrupt enable.
 361  *
 362  * @param dev
 363  *   Pointer to Ethernet device structure.
 364  * @param idx
 365  *   Rx queue index.
 366  *
 367  * @return
 368  *   0 on success, negative errno value otherwise and rte_errno is set.
 369  */
 370 int
 371 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 372 {
 373         struct rxq *rxq = dev->data->rx_queues[idx];
 374         int ret = 0;
 375
 376         if (!rxq || !rxq->channel) {
 377                 ret = EINVAL;
 378                 rte_errno = ret;
 379                 WARN("unable to arm interrupt on rx queue %d", idx);
 380         } else {
 381                 mlx4_arm_cq(rxq, 0);
 382         }
 383         return -ret;
 384 }
 385
 386 /**
 387  * Enable datapath interrupts.
 388  *
 389  * @param priv
 390  *   Pointer to private structure.
 391  *
 392  * @return
 393  *   0 on success, negative errno value otherwise and rte_errno is set.
 394  */
 395 int
 396 mlx4_rxq_intr_enable(struct mlx4_priv *priv)
 397 {
 398         const struct rte_eth_intr_conf *const intr_conf =
 399                 &ETH_DEV(priv)->data->dev_conf.intr_conf;
 400
 401         if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
 402                 goto error;
 403         return 0;
 404 error:
 405         return -rte_errno;
 406 }
 407
 408 /**
 409  * Disable datapath interrupts, keeping other interrupts intact.
 410  *
 411  * @param priv
 412  *   Pointer to private structure.
 413  */
 414 void
 415 mlx4_rxq_intr_disable(struct mlx4_priv *priv)
 416 {
 417         int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 418
 419         mlx4_rx_intr_vec_disable(priv);
 420         rte_errno = err;
 421 }