lib/librte_eal/linuxapp/eal/eal_interrupts.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdint.h>
   7 #include <stdlib.h>
   8 #include <pthread.h>
   9 #include <sys/queue.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <inttypes.h>
  15 #include <sys/epoll.h>
  16 #include <sys/signalfd.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/eventfd.h>
  19 #include <assert.h>
  20 #include <stdbool.h>
  21
  22 #include <rte_common.h>
  23 #include <rte_interrupts.h>
  24 #include <rte_memory.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_debug.h>
  32 #include <rte_log.h>
  33 #include <rte_errno.h>
  34 #include <rte_spinlock.h>
  35 #include <rte_pause.h>
  36
  37 #include "eal_private.h"
  38 #include "eal_vfio.h"
  39 #include "eal_thread.h"
  40
  41 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
  42 #define NB_OTHER_INTR               1
  43
  44 static RTE_DEFINE_PER_LCORE(int, _epfd) = -1; /**< epoll fd per thread */
  45
  46 /**
  47  * union for pipe fds.
  48  */
  49 union intr_pipefds{
  50         struct {
  51                 int pipefd[2];
  52         };
  53         struct {
  54                 int readfd;
  55                 int writefd;
  56         };
  57 };
  58
  59 /**
  60  * union buffer for reading on different devices
  61  */
  62 union rte_intr_read_buffer {
  63         int uio_intr_count;              /* for uio device */
  64 #ifdef VFIO_PRESENT
  65         uint64_t vfio_intr_count;        /* for vfio device */
  66 #endif
  67         uint64_t timerfd_num;            /* for timerfd */
  68         char charbuf[16];                /* for others */
  69 };
  70
  71 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
  72 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
  73
  74 struct rte_intr_callback {
  75         TAILQ_ENTRY(rte_intr_callback) next;
  76         rte_intr_callback_fn cb_fn;  /**< callback address */
  77         void *cb_arg;                /**< parameter for callback */
  78 };
  79
  80 struct rte_intr_source {
  81         TAILQ_ENTRY(rte_intr_source) next;
  82         struct rte_intr_handle intr_handle; /**< interrupt handle */
  83         struct rte_intr_cb_list callbacks;  /**< user callbacks */
  84         uint32_t active;
  85 };
  86
  87 /* global spinlock for interrupt data operation */
  88 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
  89
  90 /* union buffer for pipe read/write */
  91 static union intr_pipefds intr_pipe;
  92
  93 /* interrupt sources list */
  94 static struct rte_intr_source_list intr_sources;
  95
  96 /* interrupt handling thread */
  97 static pthread_t intr_thread;
  98
  99 /* VFIO interrupts */
 100 #ifdef VFIO_PRESENT
 101
 102 #define IRQ_SET_BUF_LEN  (sizeof(struct vfio_irq_set) + sizeof(int))
 103 /* irq set buffer length for queue interrupts and LSC interrupt */
 104 #define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
 105                               sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))
 106
 107 /* enable legacy (INTx) interrupts */
 108 static int
 109 vfio_enable_intx(const struct rte_intr_handle *intr_handle) {
 110         struct vfio_irq_set *irq_set;
 111         char irq_set_buf[IRQ_SET_BUF_LEN];
 112         int len, ret;
 113         int *fd_ptr;
 114
 115         len = sizeof(irq_set_buf);
 116
 117         /* enable INTx */
 118         irq_set = (struct vfio_irq_set *) irq_set_buf;
 119         irq_set->argsz = len;
 120         irq_set->count = 1;
 121         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 122         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 123         irq_set->start = 0;
 124         fd_ptr = (int *) &irq_set->data;
 125         *fd_ptr = intr_handle->fd;
 126
 127         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 128
 129         if (ret) {
 130                 RTE_LOG(ERR, EAL, "Error enabling INTx interrupts for fd %d\n",
 131                                                 intr_handle->fd);
 132                 return -1;
 133         }
 134
 135         /* unmask INTx after enabling */
 136         memset(irq_set, 0, len);
 137         len = sizeof(struct vfio_irq_set);
 138         irq_set->argsz = len;
 139         irq_set->count = 1;
 140         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
 141         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 142         irq_set->start = 0;
 143
 144         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 145
 146         if (ret) {
 147                 RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
 148                                                 intr_handle->fd);
 149                 return -1;
 150         }
 151         return 0;
 152 }
 153
 154 /* disable legacy (INTx) interrupts */
 155 static int
 156 vfio_disable_intx(const struct rte_intr_handle *intr_handle) {
 157         struct vfio_irq_set *irq_set;
 158         char irq_set_buf[IRQ_SET_BUF_LEN];
 159         int len, ret;
 160
 161         len = sizeof(struct vfio_irq_set);
 162
 163         /* mask interrupts before disabling */
 164         irq_set = (struct vfio_irq_set *) irq_set_buf;
 165         irq_set->argsz = len;
 166         irq_set->count = 1;
 167         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK;
 168         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 169         irq_set->start = 0;
 170
 171         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 172
 173         if (ret) {
 174                 RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n",
 175                                                 intr_handle->fd);
 176                 return -1;
 177         }
 178
 179         /* disable INTx*/
 180         memset(irq_set, 0, len);
 181         irq_set->argsz = len;
 182         irq_set->count = 0;
 183         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 184         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 185         irq_set->start = 0;
 186
 187         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 188
 189         if (ret) {
 190                 RTE_LOG(ERR, EAL,
 191                         "Error disabling INTx interrupts for fd %d\n", intr_handle->fd);
 192                 return -1;
 193         }
 194         return 0;
 195 }
 196
 197 /* enable MSI interrupts */
 198 static int
 199 vfio_enable_msi(const struct rte_intr_handle *intr_handle) {
 200         int len, ret;
 201         char irq_set_buf[IRQ_SET_BUF_LEN];
 202         struct vfio_irq_set *irq_set;
 203         int *fd_ptr;
 204
 205         len = sizeof(irq_set_buf);
 206
 207         irq_set = (struct vfio_irq_set *) irq_set_buf;
 208         irq_set->argsz = len;
 209         irq_set->count = 1;
 210         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 211         irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
 212         irq_set->start = 0;
 213         fd_ptr = (int *) &irq_set->data;
 214         *fd_ptr = intr_handle->fd;
 215
 216         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 217
 218         if (ret) {
 219                 RTE_LOG(ERR, EAL, "Error enabling MSI interrupts for fd %d\n",
 220                                                 intr_handle->fd);
 221                 return -1;
 222         }
 223         return 0;
 224 }
 225
 226 /* disable MSI interrupts */
 227 static int
 228 vfio_disable_msi(const struct rte_intr_handle *intr_handle) {
 229         struct vfio_irq_set *irq_set;
 230         char irq_set_buf[IRQ_SET_BUF_LEN];
 231         int len, ret;
 232
 233         len = sizeof(struct vfio_irq_set);
 234
 235         irq_set = (struct vfio_irq_set *) irq_set_buf;
 236         irq_set->argsz = len;
 237         irq_set->count = 0;
 238         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 239         irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
 240         irq_set->start = 0;
 241
 242         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 243
 244         if (ret)
 245                 RTE_LOG(ERR, EAL,
 246                         "Error disabling MSI interrupts for fd %d\n", intr_handle->fd);
 247
 248         return ret;
 249 }
 250
 251 /* enable MSI-X interrupts */
 252 static int
 253 vfio_enable_msix(const struct rte_intr_handle *intr_handle) {
 254         int len, ret;
 255         char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
 256         struct vfio_irq_set *irq_set;
 257         int *fd_ptr;
 258
 259         len = sizeof(irq_set_buf);
 260
 261         irq_set = (struct vfio_irq_set *) irq_set_buf;
 262         irq_set->argsz = len;
 263         /* 0 < irq_set->count < RTE_MAX_RXTX_INTR_VEC_ID + 1 */
 264         irq_set->count = intr_handle->max_intr ?
 265                 (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID + 1 ?
 266                 RTE_MAX_RXTX_INTR_VEC_ID + 1 : intr_handle->max_intr) : 1;
 267         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 268         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
 269         irq_set->start = 0;
 270         fd_ptr = (int *) &irq_set->data;
 271         /* INTR vector offset 0 reserve for non-efds mapping */
 272         fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = intr_handle->fd;
 273         memcpy(&fd_ptr[RTE_INTR_VEC_RXTX_OFFSET], intr_handle->efds,
 274                 sizeof(*intr_handle->efds) * intr_handle->nb_efd);
 275
 276         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 277
 278         if (ret) {
 279                 RTE_LOG(ERR, EAL, "Error enabling MSI-X interrupts for fd %d\n",
 280                                                 intr_handle->fd);
 281                 return -1;
 282         }
 283
 284         return 0;
 285 }
 286
 287 /* disable MSI-X interrupts */
 288 static int
 289 vfio_disable_msix(const struct rte_intr_handle *intr_handle) {
 290         struct vfio_irq_set *irq_set;
 291         char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
 292         int len, ret;
 293
 294         len = sizeof(struct vfio_irq_set);
 295
 296         irq_set = (struct vfio_irq_set *) irq_set_buf;
 297         irq_set->argsz = len;
 298         irq_set->count = 0;
 299         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 300         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
 301         irq_set->start = 0;
 302
 303         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 304
 305         if (ret)
 306                 RTE_LOG(ERR, EAL,
 307                         "Error disabling MSI-X interrupts for fd %d\n", intr_handle->fd);
 308
 309         return ret;
 310 }
 311 #endif
 312
 313 static int
 314 uio_intx_intr_disable(const struct rte_intr_handle *intr_handle)
 315 {
 316         unsigned char command_high;
 317
 318         /* use UIO config file descriptor for uio_pci_generic */
 319         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 320                 RTE_LOG(ERR, EAL,
 321                         "Error reading interrupts status for fd %d\n",
 322                         intr_handle->uio_cfg_fd);
 323                 return -1;
 324         }
 325         /* disable interrupts */
 326         command_high |= 0x4;
 327         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 328                 RTE_LOG(ERR, EAL,
 329                         "Error disabling interrupts for fd %d\n",
 330                         intr_handle->uio_cfg_fd);
 331                 return -1;
 332         }
 333
 334         return 0;
 335 }
 336
 337 static int
 338 uio_intx_intr_enable(const struct rte_intr_handle *intr_handle)
 339 {
 340         unsigned char command_high;
 341
 342         /* use UIO config file descriptor for uio_pci_generic */
 343         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 344                 RTE_LOG(ERR, EAL,
 345                         "Error reading interrupts status for fd %d\n",
 346                         intr_handle->uio_cfg_fd);
 347                 return -1;
 348         }
 349         /* enable interrupts */
 350         command_high &= ~0x4;
 351         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 352                 RTE_LOG(ERR, EAL,
 353                         "Error enabling interrupts for fd %d\n",
 354                         intr_handle->uio_cfg_fd);
 355                 return -1;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static int
 362 uio_intr_disable(const struct rte_intr_handle *intr_handle)
 363 {
 364         const int value = 0;
 365
 366         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 367                 RTE_LOG(ERR, EAL,
 368                         "Error disabling interrupts for fd %d (%s)\n",
 369                         intr_handle->fd, strerror(errno));
 370                 return -1;
 371         }
 372         return 0;
 373 }
 374
 375 static int
 376 uio_intr_enable(const struct rte_intr_handle *intr_handle)
 377 {
 378         const int value = 1;
 379
 380         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 381                 RTE_LOG(ERR, EAL,
 382                         "Error enabling interrupts for fd %d (%s)\n",
 383                         intr_handle->fd, strerror(errno));
 384                 return -1;
 385         }
 386         return 0;
 387 }
 388
 389 int
 390 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 391                         rte_intr_callback_fn cb, void *cb_arg)
 392 {
 393         int ret, wake_thread;
 394         struct rte_intr_source *src;
 395         struct rte_intr_callback *callback;
 396
 397         wake_thread = 0;
 398
 399         /* first do parameter checking */
 400         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
 401                 RTE_LOG(ERR, EAL,
 402                         "Registering with invalid input parameter\n");
 403                 return -EINVAL;
 404         }
 405
 406         /* allocate a new interrupt callback entity */
 407         callback = calloc(1, sizeof(*callback));
 408         if (callback == NULL) {
 409                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 410                 return -ENOMEM;
 411         }
 412         callback->cb_fn = cb;
 413         callback->cb_arg = cb_arg;
 414
 415         rte_spinlock_lock(&intr_lock);
 416
 417         /* check if there is at least one callback registered for the fd */
 418         TAILQ_FOREACH(src, &intr_sources, next) {
 419                 if (src->intr_handle.fd == intr_handle->fd) {
 420                         /* we had no interrupts for this */
 421                         if (TAILQ_EMPTY(&src->callbacks))
 422                                 wake_thread = 1;
 423
 424                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 425                         ret = 0;
 426                         break;
 427                 }
 428         }
 429
 430         /* no existing callbacks for this - add new source */
 431         if (src == NULL) {
 432                 src = calloc(1, sizeof(*src));
 433                 if (src == NULL) {
 434                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 435                         free(callback);
 436                         ret = -ENOMEM;
 437                 } else {
 438                         src->intr_handle = *intr_handle;
 439                         TAILQ_INIT(&src->callbacks);
 440                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 441                         TAILQ_INSERT_TAIL(&intr_sources, src, next);
 442                         wake_thread = 1;
 443                         ret = 0;
 444                 }
 445         }
 446
 447         rte_spinlock_unlock(&intr_lock);
 448
 449         /**
 450          * check if need to notify the pipe fd waited by epoll_wait to
 451          * rebuild the wait list.
 452          */
 453         if (wake_thread)
 454                 if (write(intr_pipe.writefd, "1", 1) < 0)
 455                         return -EPIPE;
 456
 457         return ret;
 458 }
 459
 460 int
 461 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 462                         rte_intr_callback_fn cb_fn, void *cb_arg)
 463 {
 464         int ret;
 465         struct rte_intr_source *src;
 466         struct rte_intr_callback *cb, *next;
 467
 468         /* do parameter checking first */
 469         if (intr_handle == NULL || intr_handle->fd < 0) {
 470                 RTE_LOG(ERR, EAL,
 471                 "Unregistering with invalid input parameter\n");
 472                 return -EINVAL;
 473         }
 474
 475         rte_spinlock_lock(&intr_lock);
 476
 477         /* check if the insterrupt source for the fd is existent */
 478         TAILQ_FOREACH(src, &intr_sources, next)
 479                 if (src->intr_handle.fd == intr_handle->fd)
 480                         break;
 481
 482         /* No interrupt source registered for the fd */
 483         if (src == NULL) {
 484                 ret = -ENOENT;
 485
 486         /* interrupt source has some active callbacks right now. */
 487         } else if (src->active != 0) {
 488                 ret = -EAGAIN;
 489
 490         /* ok to remove. */
 491         } else {
 492                 ret = 0;
 493
 494                 /*walk through the callbacks and remove all that match. */
 495                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 496
 497                         next = TAILQ_NEXT(cb, next);
 498
 499                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 500                                         cb->cb_arg == cb_arg)) {
 501                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 502                                 free(cb);
 503                                 ret++;
 504                         }
 505                 }
 506
 507                 /* all callbacks for that source are removed. */
 508                 if (TAILQ_EMPTY(&src->callbacks)) {
 509                         TAILQ_REMOVE(&intr_sources, src, next);
 510                         free(src);
 511                 }
 512         }
 513
 514         rte_spinlock_unlock(&intr_lock);
 515
 516         /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 517         if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 518                 ret = -EPIPE;
 519         }
 520
 521         return ret;
 522 }
 523
 524 int
 525 rte_intr_enable(const struct rte_intr_handle *intr_handle)
 526 {
 527         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
 528                 return 0;
 529
 530         if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
 531                 return -1;
 532
 533         switch (intr_handle->type){
 534         /* write to the uio fd to enable the interrupt */
 535         case RTE_INTR_HANDLE_UIO:
 536                 if (uio_intr_enable(intr_handle))
 537                         return -1;
 538                 break;
 539         case RTE_INTR_HANDLE_UIO_INTX:
 540                 if (uio_intx_intr_enable(intr_handle))
 541                         return -1;
 542                 break;
 543         /* not used at this moment */
 544         case RTE_INTR_HANDLE_ALARM:
 545                 return -1;
 546 #ifdef VFIO_PRESENT
 547         case RTE_INTR_HANDLE_VFIO_MSIX:
 548                 if (vfio_enable_msix(intr_handle))
 549                         return -1;
 550                 break;
 551         case RTE_INTR_HANDLE_VFIO_MSI:
 552                 if (vfio_enable_msi(intr_handle))
 553                         return -1;
 554                 break;
 555         case RTE_INTR_HANDLE_VFIO_LEGACY:
 556                 if (vfio_enable_intx(intr_handle))
 557                         return -1;
 558                 break;
 559 #endif
 560         /* not used at this moment */
 561         case RTE_INTR_HANDLE_DEV_EVENT:
 562                 return -1;
 563         /* unknown handle type */
 564         default:
 565                 RTE_LOG(ERR, EAL,
 566                         "Unknown handle type of fd %d\n",
 567                                         intr_handle->fd);
 568                 return -1;
 569         }
 570
 571         return 0;
 572 }
 573
 574 int
 575 rte_intr_disable(const struct rte_intr_handle *intr_handle)
 576 {
 577         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
 578                 return 0;
 579
 580         if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
 581                 return -1;
 582
 583         switch (intr_handle->type){
 584         /* write to the uio fd to disable the interrupt */
 585         case RTE_INTR_HANDLE_UIO:
 586                 if (uio_intr_disable(intr_handle))
 587                         return -1;
 588                 break;
 589         case RTE_INTR_HANDLE_UIO_INTX:
 590                 if (uio_intx_intr_disable(intr_handle))
 591                         return -1;
 592                 break;
 593         /* not used at this moment */
 594         case RTE_INTR_HANDLE_ALARM:
 595                 return -1;
 596 #ifdef VFIO_PRESENT
 597         case RTE_INTR_HANDLE_VFIO_MSIX:
 598                 if (vfio_disable_msix(intr_handle))
 599                         return -1;
 600                 break;
 601         case RTE_INTR_HANDLE_VFIO_MSI:
 602                 if (vfio_disable_msi(intr_handle))
 603                         return -1;
 604                 break;
 605         case RTE_INTR_HANDLE_VFIO_LEGACY:
 606                 if (vfio_disable_intx(intr_handle))
 607                         return -1;
 608                 break;
 609 #endif
 610         /* not used at this moment */
 611         case RTE_INTR_HANDLE_DEV_EVENT:
 612                 return -1;
 613         /* unknown handle type */
 614         default:
 615                 RTE_LOG(ERR, EAL,
 616                         "Unknown handle type of fd %d\n",
 617                                         intr_handle->fd);
 618                 return -1;
 619         }
 620
 621         return 0;
 622 }
 623
 624 static int
 625 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 626 {
 627         bool call = false;
 628         int n, bytes_read;
 629         struct rte_intr_source *src;
 630         struct rte_intr_callback *cb;
 631         union rte_intr_read_buffer buf;
 632         struct rte_intr_callback active_cb;
 633
 634         for (n = 0; n < nfds; n++) {
 635
 636                 /**
 637                  * if the pipe fd is ready to read, return out to
 638                  * rebuild the wait list.
 639                  */
 640                 if (events[n].data.fd == intr_pipe.readfd){
 641                         int r = read(intr_pipe.readfd, buf.charbuf,
 642                                         sizeof(buf.charbuf));
 643                         RTE_SET_USED(r);
 644                         return -1;
 645                 }
 646                 rte_spinlock_lock(&intr_lock);
 647                 TAILQ_FOREACH(src, &intr_sources, next)
 648                         if (src->intr_handle.fd ==
 649                                         events[n].data.fd)
 650                                 break;
 651                 if (src == NULL){
 652                         rte_spinlock_unlock(&intr_lock);
 653                         continue;
 654                 }
 655
 656                 /* mark this interrupt source as active and release the lock. */
 657                 src->active = 1;
 658                 rte_spinlock_unlock(&intr_lock);
 659
 660                 /* set the length to be read dor different handle type */
 661                 switch (src->intr_handle.type) {
 662                 case RTE_INTR_HANDLE_UIO:
 663                 case RTE_INTR_HANDLE_UIO_INTX:
 664                         bytes_read = sizeof(buf.uio_intr_count);
 665                         break;
 666                 case RTE_INTR_HANDLE_ALARM:
 667                         bytes_read = sizeof(buf.timerfd_num);
 668                         break;
 669 #ifdef VFIO_PRESENT
 670                 case RTE_INTR_HANDLE_VFIO_MSIX:
 671                 case RTE_INTR_HANDLE_VFIO_MSI:
 672                 case RTE_INTR_HANDLE_VFIO_LEGACY:
 673                         bytes_read = sizeof(buf.vfio_intr_count);
 674                         break;
 675 #endif
 676                 case RTE_INTR_HANDLE_VDEV:
 677                 case RTE_INTR_HANDLE_EXT:
 678                         bytes_read = 0;
 679                         call = true;
 680                         break;
 681                 case RTE_INTR_HANDLE_DEV_EVENT:
 682                         bytes_read = 0;
 683                         call = true;
 684                         break;
 685                 default:
 686                         bytes_read = 1;
 687                         break;
 688                 }
 689
 690                 if (bytes_read > 0) {
 691                         /**
 692                          * read out to clear the ready-to-be-read flag
 693                          * for epoll_wait.
 694                          */
 695                         bytes_read = read(events[n].data.fd, &buf, bytes_read);
 696                         if (bytes_read < 0) {
 697                                 if (errno == EINTR || errno == EWOULDBLOCK)
 698                                         continue;
 699
 700                                 RTE_LOG(ERR, EAL, "Error reading from file "
 701                                         "descriptor %d: %s\n",
 702                                         events[n].data.fd,
 703                                         strerror(errno));
 704                         } else if (bytes_read == 0)
 705                                 RTE_LOG(ERR, EAL, "Read nothing from file "
 706                                         "descriptor %d\n", events[n].data.fd);
 707                         else
 708                                 call = true;
 709                 }
 710
 711                 /* grab a lock, again to call callbacks and update status. */
 712                 rte_spinlock_lock(&intr_lock);
 713
 714                 if (call) {
 715
 716                         /* Finally, call all callbacks. */
 717                         TAILQ_FOREACH(cb, &src->callbacks, next) {
 718
 719                                 /* make a copy and unlock. */
 720                                 active_cb = *cb;
 721                                 rte_spinlock_unlock(&intr_lock);
 722
 723                                 /* call the actual callback */
 724                                 active_cb.cb_fn(active_cb.cb_arg);
 725
 726                                 /*get the lock back. */
 727                                 rte_spinlock_lock(&intr_lock);
 728                         }
 729                 }
 730
 731                 /* we done with that interrupt source, release it. */
 732                 src->active = 0;
 733                 rte_spinlock_unlock(&intr_lock);
 734         }
 735
 736         return 0;
 737 }
 738
 739 /**
 740  * It handles all the interrupts.
 741  *
 742  * @param pfd
 743  *  epoll file descriptor.
 744  * @param totalfds
 745  *  The number of file descriptors added in epoll.
 746  *
 747  * @return
 748  *  void
 749  */
 750 static void
 751 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
 752 {
 753         struct epoll_event events[totalfds];
 754         int nfds = 0;
 755
 756         for(;;) {
 757                 nfds = epoll_wait(pfd, events, totalfds,
 758                         EAL_INTR_EPOLL_WAIT_FOREVER);
 759                 /* epoll_wait fail */
 760                 if (nfds < 0) {
 761                         if (errno == EINTR)
 762                                 continue;
 763                         RTE_LOG(ERR, EAL,
 764                                 "epoll_wait returns with fail\n");
 765                         return;
 766                 }
 767                 /* epoll_wait timeout, will never happens here */
 768                 else if (nfds == 0)
 769                         continue;
 770                 /* epoll_wait has at least one fd ready to read */
 771                 if (eal_intr_process_interrupts(events, nfds) < 0)
 772                         return;
 773         }
 774 }
 775
 776 /**
 777  * It builds/rebuilds up the epoll file descriptor with all the
 778  * file descriptors being waited on. Then handles the interrupts.
 779  *
 780  * @param arg
 781  *  pointer. (unused)
 782  *
 783  * @return
 784  *  never return;
 785  */
 786 static __attribute__((noreturn)) void *
 787 eal_intr_thread_main(__rte_unused void *arg)
 788 {
 789         struct epoll_event ev;
 790
 791         /* host thread, never break out */
 792         for (;;) {
 793                 /* build up the epoll fd with all descriptors we are to
 794                  * wait on then pass it to the handle_interrupts function
 795                  */
 796                 static struct epoll_event pipe_event = {
 797                         .events = EPOLLIN | EPOLLPRI,
 798                 };
 799                 struct rte_intr_source *src;
 800                 unsigned numfds = 0;
 801
 802                 /* create epoll fd */
 803                 int pfd = epoll_create(1);
 804                 if (pfd < 0)
 805                         rte_panic("Cannot create epoll instance\n");
 806
 807                 pipe_event.data.fd = intr_pipe.readfd;
 808                 /**
 809                  * add pipe fd into wait list, this pipe is used to
 810                  * rebuild the wait list.
 811                  */
 812                 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
 813                                                 &pipe_event) < 0) {
 814                         rte_panic("Error adding fd to %d epoll_ctl, %s\n",
 815                                         intr_pipe.readfd, strerror(errno));
 816                 }
 817                 numfds++;
 818
 819                 rte_spinlock_lock(&intr_lock);
 820
 821                 TAILQ_FOREACH(src, &intr_sources, next) {
 822                         if (src->callbacks.tqh_first == NULL)
 823                                 continue; /* skip those with no callbacks */
 824                         ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
 825                         ev.data.fd = src->intr_handle.fd;
 826
 827                         /**
 828                          * add all the uio device file descriptor
 829                          * into wait list.
 830                          */
 831                         if (epoll_ctl(pfd, EPOLL_CTL_ADD,
 832                                         src->intr_handle.fd, &ev) < 0){
 833                                 rte_panic("Error adding fd %d epoll_ctl, %s\n",
 834                                         src->intr_handle.fd, strerror(errno));
 835                         }
 836                         else
 837                                 numfds++;
 838                 }
 839                 rte_spinlock_unlock(&intr_lock);
 840                 /* serve the interrupt */
 841                 eal_intr_handle_interrupts(pfd, numfds);
 842
 843                 /**
 844                  * when we return, we need to rebuild the
 845                  * list of fds to monitor.
 846                  */
 847                 close(pfd);
 848         }
 849 }
 850
 851 int
 852 rte_eal_intr_init(void)
 853 {
 854         int ret = 0;
 855
 856         /* init the global interrupt source head */
 857         TAILQ_INIT(&intr_sources);
 858
 859         /**
 860          * create a pipe which will be waited by epoll and notified to
 861          * rebuild the wait list of epoll.
 862          */
 863         if (pipe(intr_pipe.pipefd) < 0) {
 864                 rte_errno = errno;
 865                 return -1;
 866         }
 867
 868         /* create the host thread to wait/handle the interrupt */
 869         ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
 870                         eal_intr_thread_main, NULL);
 871         if (ret != 0) {
 872                 rte_errno = -ret;
 873                 RTE_LOG(ERR, EAL,
 874                         "Failed to create thread for interrupt handling\n");
 875         }
 876
 877         return ret;
 878 }
 879
 880 static void
 881 eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
 882 {
 883         union rte_intr_read_buffer buf;
 884         int bytes_read = 0;
 885         int nbytes;
 886
 887         switch (intr_handle->type) {
 888         case RTE_INTR_HANDLE_UIO:
 889         case RTE_INTR_HANDLE_UIO_INTX:
 890                 bytes_read = sizeof(buf.uio_intr_count);
 891                 break;
 892 #ifdef VFIO_PRESENT
 893         case RTE_INTR_HANDLE_VFIO_MSIX:
 894         case RTE_INTR_HANDLE_VFIO_MSI:
 895         case RTE_INTR_HANDLE_VFIO_LEGACY:
 896                 bytes_read = sizeof(buf.vfio_intr_count);
 897                 break;
 898 #endif
 899         case RTE_INTR_HANDLE_VDEV:
 900                 bytes_read = intr_handle->efd_counter_size;
 901                 /* For vdev, number of bytes to read is set by driver */
 902                 break;
 903         case RTE_INTR_HANDLE_EXT:
 904                 return;
 905         default:
 906                 bytes_read = 1;
 907                 RTE_LOG(INFO, EAL, "unexpected intr type\n");
 908                 break;
 909         }
 910
 911         /**
 912          * read out to clear the ready-to-be-read flag
 913          * for epoll_wait.
 914          */
 915         if (bytes_read == 0)
 916                 return;
 917         do {
 918                 nbytes = read(fd, &buf, bytes_read);
 919                 if (nbytes < 0) {
 920                         if (errno == EINTR || errno == EWOULDBLOCK ||
 921                             errno == EAGAIN)
 922                                 continue;
 923                         RTE_LOG(ERR, EAL,
 924                                 "Error reading from fd %d: %s\n",
 925                                 fd, strerror(errno));
 926                 } else if (nbytes == 0)
 927                         RTE_LOG(ERR, EAL, "Read nothing from fd %d\n", fd);
 928                 return;
 929         } while (1);
 930 }
 931
 932 static int
 933 eal_epoll_process_event(struct epoll_event *evs, unsigned int n,
 934                         struct rte_epoll_event *events)
 935 {
 936         unsigned int i, count = 0;
 937         struct rte_epoll_event *rev;
 938
 939         for (i = 0; i < n; i++) {
 940                 rev = evs[i].data.ptr;
 941                 if (!rev || !rte_atomic32_cmpset(&rev->status, RTE_EPOLL_VALID,
 942                                                  RTE_EPOLL_EXEC))
 943                         continue;
 944
 945                 events[count].status        = RTE_EPOLL_VALID;
 946                 events[count].fd            = rev->fd;
 947                 events[count].epfd          = rev->epfd;
 948                 events[count].epdata.event  = rev->epdata.event;
 949                 events[count].epdata.data   = rev->epdata.data;
 950                 if (rev->epdata.cb_fun)
 951                         rev->epdata.cb_fun(rev->fd,
 952                                            rev->epdata.cb_arg);
 953
 954                 rte_compiler_barrier();
 955                 rev->status = RTE_EPOLL_VALID;
 956                 count++;
 957         }
 958         return count;
 959 }
 960
 961 static inline int
 962 eal_init_tls_epfd(void)
 963 {
 964         int pfd = epoll_create(255);
 965
 966         if (pfd < 0) {
 967                 RTE_LOG(ERR, EAL,
 968                         "Cannot create epoll instance\n");
 969                 return -1;
 970         }
 971         return pfd;
 972 }
 973
 974 int
 975 rte_intr_tls_epfd(void)
 976 {
 977         if (RTE_PER_LCORE(_epfd) == -1)
 978                 RTE_PER_LCORE(_epfd) = eal_init_tls_epfd();
 979
 980         return RTE_PER_LCORE(_epfd);
 981 }
 982
 983 int
 984 rte_epoll_wait(int epfd, struct rte_epoll_event *events,
 985                int maxevents, int timeout)
 986 {
 987         struct epoll_event evs[maxevents];
 988         int rc;
 989
 990         if (!events) {
 991                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
 992                 return -1;
 993         }
 994
 995         /* using per thread epoll fd */
 996         if (epfd == RTE_EPOLL_PER_THREAD)
 997                 epfd = rte_intr_tls_epfd();
 998
 999         while (1) {
1000                 rc = epoll_wait(epfd, evs, maxevents, timeout);
1001                 if (likely(rc > 0)) {
1002                         /* epoll_wait has at least one fd ready to read */
1003                         rc = eal_epoll_process_event(evs, rc, events);
1004                         break;
1005                 } else if (rc < 0) {
1006                         if (errno == EINTR)
1007                                 continue;
1008                         /* epoll_wait fail */
1009                         RTE_LOG(ERR, EAL, "epoll_wait returns with fail %s\n",
1010                                 strerror(errno));
1011                         rc = -1;
1012                         break;
1013                 } else {
1014                         /* rc == 0, epoll_wait timed out */
1015                         break;
1016                 }
1017         }
1018
1019         return rc;
1020 }
1021
1022 static inline void
1023 eal_epoll_data_safe_free(struct rte_epoll_event *ev)
1024 {
1025         while (!rte_atomic32_cmpset(&ev->status, RTE_EPOLL_VALID,
1026                                     RTE_EPOLL_INVALID))
1027                 while (ev->status != RTE_EPOLL_VALID)
1028                         rte_pause();
1029         memset(&ev->epdata, 0, sizeof(ev->epdata));
1030         ev->fd = -1;
1031         ev->epfd = -1;
1032 }
1033
1034 int
1035 rte_epoll_ctl(int epfd, int op, int fd,
1036               struct rte_epoll_event *event)
1037 {
1038         struct epoll_event ev;
1039
1040         if (!event) {
1041                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
1042                 return -1;
1043         }
1044
1045         /* using per thread epoll fd */
1046         if (epfd == RTE_EPOLL_PER_THREAD)
1047                 epfd = rte_intr_tls_epfd();
1048
1049         if (op == EPOLL_CTL_ADD) {
1050                 event->status = RTE_EPOLL_VALID;
1051                 event->fd = fd;  /* ignore fd in event */
1052                 event->epfd = epfd;
1053                 ev.data.ptr = (void *)event;
1054         }
1055
1056         ev.events = event->epdata.event;
1057         if (epoll_ctl(epfd, op, fd, &ev) < 0) {
1058                 RTE_LOG(ERR, EAL, "Error op %d fd %d epoll_ctl, %s\n",
1059                         op, fd, strerror(errno));
1060                 if (op == EPOLL_CTL_ADD)
1061                         /* rollback status when CTL_ADD fail */
1062                         event->status = RTE_EPOLL_INVALID;
1063                 return -1;
1064         }
1065
1066         if (op == EPOLL_CTL_DEL && event->status != RTE_EPOLL_INVALID)
1067                 eal_epoll_data_safe_free(event);
1068
1069         return 0;
1070 }
1071
1072 int
1073 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
1074                 int op, unsigned int vec, void *data)
1075 {
1076         struct rte_epoll_event *rev;
1077         struct rte_epoll_data *epdata;
1078         int epfd_op;
1079         unsigned int efd_idx;
1080         int rc = 0;
1081
1082         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
1083                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
1084
1085         if (!intr_handle || intr_handle->nb_efd == 0 ||
1086             efd_idx >= intr_handle->nb_efd) {
1087                 RTE_LOG(ERR, EAL, "Wrong intr vector number.\n");
1088                 return -EPERM;
1089         }
1090
1091         switch (op) {
1092         case RTE_INTR_EVENT_ADD:
1093                 epfd_op = EPOLL_CTL_ADD;
1094                 rev = &intr_handle->elist[efd_idx];
1095                 if (rev->status != RTE_EPOLL_INVALID) {
1096                         RTE_LOG(INFO, EAL, "Event already been added.\n");
1097                         return -EEXIST;
1098                 }
1099
1100                 /* attach to intr vector fd */
1101                 epdata = &rev->epdata;
1102                 epdata->event  = EPOLLIN | EPOLLPRI | EPOLLET;
1103                 epdata->data   = data;
1104                 epdata->cb_fun = (rte_intr_event_cb_t)eal_intr_proc_rxtx_intr;
1105                 epdata->cb_arg = (void *)intr_handle;
1106                 rc = rte_epoll_ctl(epfd, epfd_op,
1107                                    intr_handle->efds[efd_idx], rev);
1108                 if (!rc)
1109                         RTE_LOG(DEBUG, EAL,
1110                                 "efd %d associated with vec %d added on epfd %d"
1111                                 "\n", rev->fd, vec, epfd);
1112                 else
1113                         rc = -EPERM;
1114                 break;
1115         case RTE_INTR_EVENT_DEL:
1116                 epfd_op = EPOLL_CTL_DEL;
1117                 rev = &intr_handle->elist[efd_idx];
1118                 if (rev->status == RTE_EPOLL_INVALID) {
1119                         RTE_LOG(INFO, EAL, "Event does not exist.\n");
1120                         return -EPERM;
1121                 }
1122
1123                 rc = rte_epoll_ctl(rev->epfd, epfd_op, rev->fd, rev);
1124                 if (rc)
1125                         rc = -EPERM;
1126                 break;
1127         default:
1128                 RTE_LOG(ERR, EAL, "event op type mismatch\n");
1129                 rc = -EPERM;
1130         }
1131
1132         return rc;
1133 }
1134
1135 void
1136 rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
1137 {
1138         uint32_t i;
1139         struct rte_epoll_event *rev;
1140
1141         for (i = 0; i < intr_handle->nb_efd; i++) {
1142                 rev = &intr_handle->elist[i];
1143                 if (rev->status == RTE_EPOLL_INVALID)
1144                         continue;
1145                 if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
1146                         /* force free if the entry valid */
1147                         eal_epoll_data_safe_free(rev);
1148                         rev->status = RTE_EPOLL_INVALID;
1149                 }
1150         }
1151 }
1152
1153 int
1154 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
1155 {
1156         uint32_t i;
1157         int fd;
1158         uint32_t n = RTE_MIN(nb_efd, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
1159
1160         assert(nb_efd != 0);
1161
1162         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) {
1163                 for (i = 0; i < n; i++) {
1164                         fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
1165                         if (fd < 0) {
1166                                 RTE_LOG(ERR, EAL,
1167                                         "can't setup eventfd, error %i (%s)\n",
1168                                         errno, strerror(errno));
1169                                 return -errno;
1170                         }
1171                         intr_handle->efds[i] = fd;
1172                 }
1173                 intr_handle->nb_efd   = n;
1174                 intr_handle->max_intr = NB_OTHER_INTR + n;
1175         } else if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
1176                 /* only check, initialization would be done in vdev driver.*/
1177                 if (intr_handle->efd_counter_size >
1178                     sizeof(union rte_intr_read_buffer)) {
1179                         RTE_LOG(ERR, EAL, "the efd_counter_size is oversized");
1180                         return -EINVAL;
1181                 }
1182         } else {
1183                 intr_handle->efds[0]  = intr_handle->fd;
1184                 intr_handle->nb_efd   = RTE_MIN(nb_efd, 1U);
1185                 intr_handle->max_intr = NB_OTHER_INTR;
1186         }
1187
1188         return 0;
1189 }
1190
1191 void
1192 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
1193 {
1194         uint32_t i;
1195
1196         rte_intr_free_epoll_fd(intr_handle);
1197         if (intr_handle->max_intr > intr_handle->nb_efd) {
1198                 for (i = 0; i < intr_handle->nb_efd; i++)
1199                         close(intr_handle->efds[i]);
1200         }
1201         intr_handle->nb_efd = 0;
1202         intr_handle->max_intr = 0;
1203 }
1204
1205 int
1206 rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
1207 {
1208         return !(!intr_handle->nb_efd);
1209 }
1210
1211 int
1212 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
1213 {
1214         if (!rte_intr_dp_is_en(intr_handle))
1215                 return 1;
1216         else
1217                 return !!(intr_handle->max_intr - intr_handle->nb_efd);
1218 }
1219
1220 int
1221 rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
1222 {
1223         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX)
1224                 return 1;
1225
1226         if (intr_handle->type == RTE_INTR_HANDLE_VDEV)
1227                 return 1;
1228
1229         return 0;
1230 }