lib/librte_eal/linux/eal/eal_interrupts.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdint.h>
   7 #include <stdlib.h>
   8 #include <pthread.h>
   9 #include <sys/queue.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <inttypes.h>
  15 #include <sys/epoll.h>
  16 #include <sys/signalfd.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/eventfd.h>
  19 #include <assert.h>
  20 #include <stdbool.h>
  21
  22 #include <rte_common.h>
  23 #include <rte_interrupts.h>
  24 #include <rte_memory.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_debug.h>
  32 #include <rte_log.h>
  33 #include <rte_errno.h>
  34 #include <rte_spinlock.h>
  35 #include <rte_pause.h>
  36 #include <rte_vfio.h>
  37
  38 #include "eal_private.h"
  39 #include "eal_vfio.h"
  40 #include "eal_thread.h"
  41
  42 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
  43 #define NB_OTHER_INTR               1
  44
  45 static RTE_DEFINE_PER_LCORE(int, _epfd) = -1; /**< epoll fd per thread */
  46
  47 /**
  48  * union for pipe fds.
  49  */
  50 union intr_pipefds{
  51         struct {
  52                 int pipefd[2];
  53         };
  54         struct {
  55                 int readfd;
  56                 int writefd;
  57         };
  58 };
  59
  60 /**
  61  * union buffer for reading on different devices
  62  */
  63 union rte_intr_read_buffer {
  64         int uio_intr_count;              /* for uio device */
  65 #ifdef VFIO_PRESENT
  66         uint64_t vfio_intr_count;        /* for vfio device */
  67 #endif
  68         uint64_t timerfd_num;            /* for timerfd */
  69         char charbuf[16];                /* for others */
  70 };
  71
  72 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
  73 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
  74
  75 struct rte_intr_callback {
  76         TAILQ_ENTRY(rte_intr_callback) next;
  77         rte_intr_callback_fn cb_fn;  /**< callback address */
  78         void *cb_arg;                /**< parameter for callback */
  79         uint8_t pending_delete;      /**< delete after callback is called */
  80         rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
  81 };
  82
  83 struct rte_intr_source {
  84         TAILQ_ENTRY(rte_intr_source) next;
  85         struct rte_intr_handle intr_handle; /**< interrupt handle */
  86         struct rte_intr_cb_list callbacks;  /**< user callbacks */
  87         uint32_t active;
  88 };
  89
  90 /* global spinlock for interrupt data operation */
  91 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
  92
  93 /* union buffer for pipe read/write */
  94 static union intr_pipefds intr_pipe;
  95
  96 /* interrupt sources list */
  97 static struct rte_intr_source_list intr_sources;
  98
  99 /* interrupt handling thread */
 100 static pthread_t intr_thread;
 101
 102 /* VFIO interrupts */
 103 #ifdef VFIO_PRESENT
 104
 105 #define IRQ_SET_BUF_LEN  (sizeof(struct vfio_irq_set) + sizeof(int))
 106 /* irq set buffer length for queue interrupts and LSC interrupt */
 107 #define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
 108                               sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))
 109
 110 /* enable legacy (INTx) interrupts */
 111 static int
 112 vfio_enable_intx(const struct rte_intr_handle *intr_handle)
 113 {
 114         struct vfio_irq_set irq_set;
 115         int ret;
 116
 117         /* unmask INTx */
 118         irq_set.argsz = sizeof(irq_set);
 119         irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
 120         irq_set.index = VFIO_PCI_INTX_IRQ_INDEX;
 121         irq_set.start = 0;
 122         irq_set.count = 1;
 123
 124         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set);
 125
 126         if (ret) {
 127                 RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
 128                                                 intr_handle->fd);
 129                 return -1;
 130         }
 131         return 0;
 132 }
 133
 134 /* disable legacy (INTx) interrupts */
 135 static int
 136 vfio_disable_intx(const struct rte_intr_handle *intr_handle)
 137 {
 138         struct vfio_irq_set irq_set;
 139         int ret;
 140
 141         /* mask interrupts */
 142         irq_set.argsz = sizeof(irq_set);
 143         irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK;
 144         irq_set.index = VFIO_PCI_INTX_IRQ_INDEX;
 145         irq_set.start = 0;
 146         irq_set.count = 1;
 147
 148         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set);
 149
 150         if (ret) {
 151                 RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n",
 152                                                 intr_handle->fd);
 153                 return -1;
 154         }
 155         return 0;
 156 }
 157
 158 /* enable MSI-X interrupts */
 159 static int
 160 vfio_enable_msix(const struct rte_intr_handle *intr_handle)
 161 {
 162         char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
 163         struct vfio_irq_set *irq_set;
 164         int len, ret;
 165
 166         if (intr_handle->nb_efd == 0)
 167                 return 0;
 168
 169         len = sizeof(irq_set_buf);
 170
 171         irq_set = (struct vfio_irq_set *) irq_set_buf;
 172         irq_set->argsz = len;
 173         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 174         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
 175         irq_set->start = RTE_INTR_VEC_RXTX_OFFSET;
 176         irq_set->count = intr_handle->nb_efd;
 177         memcpy(&irq_set->data, intr_handle->efds,
 178                sizeof(*intr_handle->efds) * intr_handle->nb_efd);
 179
 180         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 181         if (ret) {
 182                 RTE_LOG(ERR, EAL, "Error enabling MSI-X interrupts for fd %d\n",
 183                                                 intr_handle->fd);
 184                 return -1;
 185         }
 186
 187         return 0;
 188 }
 189
 190 /* disable MSI-X interrupts */
 191 static int
 192 vfio_disable_msix(const struct rte_intr_handle *intr_handle)
 193 {
 194         struct vfio_irq_set irq_set;
 195         int ret;
 196
 197         if (intr_handle->nb_efd == 0)
 198                 return 0;
 199
 200         irq_set.argsz = sizeof(irq_set);
 201         irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 202         irq_set.index = VFIO_PCI_MSIX_IRQ_INDEX;
 203         irq_set.start = RTE_INTR_VEC_RXTX_OFFSET;
 204         irq_set.count = intr_handle->nb_efd;
 205
 206         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set);
 207         if (ret)
 208                 RTE_LOG(ERR, EAL,
 209                         "Error disabling MSI-X interrupts for fd %d\n", intr_handle->fd);
 210
 211         return ret;
 212 }
 213
 214 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 215 /* enable req notifier */
 216 static int
 217 vfio_enable_req(const struct rte_intr_handle *intr_handle)
 218 {
 219         int len, ret;
 220         char irq_set_buf[IRQ_SET_BUF_LEN];
 221         struct vfio_irq_set *irq_set;
 222         int *fd_ptr;
 223
 224         len = sizeof(irq_set_buf);
 225
 226         irq_set = (struct vfio_irq_set *) irq_set_buf;
 227         irq_set->argsz = len;
 228         irq_set->count = 1;
 229         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
 230                          VFIO_IRQ_SET_ACTION_TRIGGER;
 231         irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
 232         irq_set->start = 0;
 233         fd_ptr = (int *) &irq_set->data;
 234         *fd_ptr = intr_handle->fd;
 235
 236         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 237
 238         if (ret) {
 239                 RTE_LOG(ERR, EAL, "Error enabling req interrupts for fd %d\n",
 240                                                 intr_handle->fd);
 241                 return -1;
 242         }
 243
 244         return 0;
 245 }
 246
 247 /* disable req notifier */
 248 static int
 249 vfio_disable_req(const struct rte_intr_handle *intr_handle)
 250 {
 251         struct vfio_irq_set *irq_set;
 252         char irq_set_buf[IRQ_SET_BUF_LEN];
 253         int len, ret;
 254
 255         len = sizeof(struct vfio_irq_set);
 256
 257         irq_set = (struct vfio_irq_set *) irq_set_buf;
 258         irq_set->argsz = len;
 259         irq_set->count = 0;
 260         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 261         irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
 262         irq_set->start = 0;
 263
 264         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 265
 266         if (ret)
 267                 RTE_LOG(ERR, EAL, "Error disabling req interrupts for fd %d\n",
 268                         intr_handle->fd);
 269
 270         return ret;
 271 }
 272 #endif
 273 #endif
 274
 275 static int
 276 uio_intx_intr_disable(const struct rte_intr_handle *intr_handle)
 277 {
 278         unsigned char command_high;
 279
 280         /* use UIO config file descriptor for uio_pci_generic */
 281         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 282                 RTE_LOG(ERR, EAL,
 283                         "Error reading interrupts status for fd %d\n",
 284                         intr_handle->uio_cfg_fd);
 285                 return -1;
 286         }
 287         /* disable interrupts */
 288         command_high |= 0x4;
 289         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 290                 RTE_LOG(ERR, EAL,
 291                         "Error disabling interrupts for fd %d\n",
 292                         intr_handle->uio_cfg_fd);
 293                 return -1;
 294         }
 295
 296         return 0;
 297 }
 298
 299 static int
 300 uio_intx_intr_enable(const struct rte_intr_handle *intr_handle)
 301 {
 302         unsigned char command_high;
 303
 304         /* use UIO config file descriptor for uio_pci_generic */
 305         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 306                 RTE_LOG(ERR, EAL,
 307                         "Error reading interrupts status for fd %d\n",
 308                         intr_handle->uio_cfg_fd);
 309                 return -1;
 310         }
 311         /* enable interrupts */
 312         command_high &= ~0x4;
 313         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 314                 RTE_LOG(ERR, EAL,
 315                         "Error enabling interrupts for fd %d\n",
 316                         intr_handle->uio_cfg_fd);
 317                 return -1;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int
 324 uio_intr_disable(const struct rte_intr_handle *intr_handle)
 325 {
 326         const int value = 0;
 327
 328         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 329                 RTE_LOG(ERR, EAL,
 330                         "Error disabling interrupts for fd %d (%s)\n",
 331                         intr_handle->fd, strerror(errno));
 332                 return -1;
 333         }
 334         return 0;
 335 }
 336
 337 static int
 338 uio_intr_enable(const struct rte_intr_handle *intr_handle)
 339 {
 340         const int value = 1;
 341
 342         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 343                 RTE_LOG(ERR, EAL,
 344                         "Error enabling interrupts for fd %d (%s)\n",
 345                         intr_handle->fd, strerror(errno));
 346                 return -1;
 347         }
 348         return 0;
 349 }
 350
 351 int
 352 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 353                         rte_intr_callback_fn cb, void *cb_arg)
 354 {
 355         int ret, wake_thread;
 356         struct rte_intr_source *src;
 357         struct rte_intr_callback *callback;
 358
 359         wake_thread = 0;
 360
 361         /* first do parameter checking */
 362         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
 363                 RTE_LOG(ERR, EAL,
 364                         "Registering with invalid input parameter\n");
 365                 return -EINVAL;
 366         }
 367
 368         /* allocate a new interrupt callback entity */
 369         callback = calloc(1, sizeof(*callback));
 370         if (callback == NULL) {
 371                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 372                 return -ENOMEM;
 373         }
 374         callback->cb_fn = cb;
 375         callback->cb_arg = cb_arg;
 376         callback->pending_delete = 0;
 377         callback->ucb_fn = NULL;
 378
 379         rte_spinlock_lock(&intr_lock);
 380
 381         /* check if there is at least one callback registered for the fd */
 382         TAILQ_FOREACH(src, &intr_sources, next) {
 383                 if (src->intr_handle.fd == intr_handle->fd) {
 384                         /* we had no interrupts for this */
 385                         if (TAILQ_EMPTY(&src->callbacks))
 386                                 wake_thread = 1;
 387
 388                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 389                         ret = 0;
 390                         break;
 391                 }
 392         }
 393
 394         /* no existing callbacks for this - add new source */
 395         if (src == NULL) {
 396                 src = calloc(1, sizeof(*src));
 397                 if (src == NULL) {
 398                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 399                         free(callback);
 400                         ret = -ENOMEM;
 401                 } else {
 402                         src->intr_handle = *intr_handle;
 403                         TAILQ_INIT(&src->callbacks);
 404                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 405                         TAILQ_INSERT_TAIL(&intr_sources, src, next);
 406                         wake_thread = 1;
 407                         ret = 0;
 408                 }
 409         }
 410
 411         rte_spinlock_unlock(&intr_lock);
 412
 413         /**
 414          * check if need to notify the pipe fd waited by epoll_wait to
 415          * rebuild the wait list.
 416          */
 417         if (wake_thread)
 418                 if (write(intr_pipe.writefd, "1", 1) < 0)
 419                         return -EPIPE;
 420
 421         return ret;
 422 }
 423
 424 int
 425 rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
 426                                 rte_intr_callback_fn cb_fn, void *cb_arg,
 427                                 rte_intr_unregister_callback_fn ucb_fn)
 428 {
 429         int ret;
 430         struct rte_intr_source *src;
 431         struct rte_intr_callback *cb, *next;
 432
 433         /* do parameter checking first */
 434         if (intr_handle == NULL || intr_handle->fd < 0) {
 435                 RTE_LOG(ERR, EAL,
 436                 "Unregistering with invalid input parameter\n");
 437                 return -EINVAL;
 438         }
 439
 440         rte_spinlock_lock(&intr_lock);
 441
 442         /* check if the insterrupt source for the fd is existent */
 443         TAILQ_FOREACH(src, &intr_sources, next)
 444                 if (src->intr_handle.fd == intr_handle->fd)
 445                         break;
 446
 447         /* No interrupt source registered for the fd */
 448         if (src == NULL) {
 449                 ret = -ENOENT;
 450
 451         /* only usable if the source is active */
 452         } else if (src->active == 0) {
 453                 ret = -EAGAIN;
 454
 455         } else {
 456                 ret = 0;
 457
 458                 /* walk through the callbacks and mark all that match. */
 459                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 460                         next = TAILQ_NEXT(cb, next);
 461                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 462                                         cb->cb_arg == cb_arg)) {
 463                                 cb->pending_delete = 1;
 464                                 cb->ucb_fn = ucb_fn;
 465                                 ret++;
 466                         }
 467                 }
 468         }
 469
 470         rte_spinlock_unlock(&intr_lock);
 471
 472         return ret;
 473 }
 474
 475 int
 476 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 477                         rte_intr_callback_fn cb_fn, void *cb_arg)
 478 {
 479         int ret;
 480         struct rte_intr_source *src;
 481         struct rte_intr_callback *cb, *next;
 482
 483         /* do parameter checking first */
 484         if (intr_handle == NULL || intr_handle->fd < 0) {
 485                 RTE_LOG(ERR, EAL,
 486                 "Unregistering with invalid input parameter\n");
 487                 return -EINVAL;
 488         }
 489
 490         rte_spinlock_lock(&intr_lock);
 491
 492         /* check if the insterrupt source for the fd is existent */
 493         TAILQ_FOREACH(src, &intr_sources, next)
 494                 if (src->intr_handle.fd == intr_handle->fd)
 495                         break;
 496
 497         /* No interrupt source registered for the fd */
 498         if (src == NULL) {
 499                 ret = -ENOENT;
 500
 501         /* interrupt source has some active callbacks right now. */
 502         } else if (src->active != 0) {
 503                 ret = -EAGAIN;
 504
 505         /* ok to remove. */
 506         } else {
 507                 ret = 0;
 508
 509                 /*walk through the callbacks and remove all that match. */
 510                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 511
 512                         next = TAILQ_NEXT(cb, next);
 513
 514                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 515                                         cb->cb_arg == cb_arg)) {
 516                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 517                                 free(cb);
 518                                 ret++;
 519                         }
 520                 }
 521
 522                 /* all callbacks for that source are removed. */
 523                 if (TAILQ_EMPTY(&src->callbacks)) {
 524                         TAILQ_REMOVE(&intr_sources, src, next);
 525                         free(src);
 526                 }
 527         }
 528
 529         rte_spinlock_unlock(&intr_lock);
 530
 531         /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 532         if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 533                 ret = -EPIPE;
 534         }
 535
 536         return ret;
 537 }
 538
 539 int
 540 rte_intr_enable(const struct rte_intr_handle *intr_handle)
 541 {
 542         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
 543                 return 0;
 544
 545         if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
 546                 return -1;
 547
 548         switch (intr_handle->type){
 549         /* write to the uio fd to enable the interrupt */
 550         case RTE_INTR_HANDLE_UIO:
 551                 if (uio_intr_enable(intr_handle))
 552                         return -1;
 553                 break;
 554         case RTE_INTR_HANDLE_UIO_INTX:
 555                 if (uio_intx_intr_enable(intr_handle))
 556                         return -1;
 557                 break;
 558         /* not used at this moment */
 559         case RTE_INTR_HANDLE_ALARM:
 560                 return -1;
 561 #ifdef VFIO_PRESENT
 562         case RTE_INTR_HANDLE_VFIO_MSIX:
 563                 if (vfio_enable_msix(intr_handle))
 564                         return -1;
 565                 break;
 566         case RTE_INTR_HANDLE_VFIO_MSI:
 567                 return 0;
 568         case RTE_INTR_HANDLE_VFIO_LEGACY:
 569                 if (vfio_enable_intx(intr_handle))
 570                         return -1;
 571                 break;
 572 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 573         case RTE_INTR_HANDLE_VFIO_REQ:
 574                 if (vfio_enable_req(intr_handle))
 575                         return -1;
 576                 break;
 577 #endif
 578 #endif
 579         /* not used at this moment */
 580         case RTE_INTR_HANDLE_DEV_EVENT:
 581                 return -1;
 582         /* unknown handle type */
 583         default:
 584                 RTE_LOG(ERR, EAL,
 585                         "Unknown handle type of fd %d\n",
 586                                         intr_handle->fd);
 587                 return -1;
 588         }
 589
 590         return 0;
 591 }
 592
 593 int
 594 rte_intr_disable(const struct rte_intr_handle *intr_handle)
 595 {
 596         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
 597                 return 0;
 598
 599         if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
 600                 return -1;
 601
 602         switch (intr_handle->type){
 603         /* write to the uio fd to disable the interrupt */
 604         case RTE_INTR_HANDLE_UIO:
 605                 if (uio_intr_disable(intr_handle))
 606                         return -1;
 607                 break;
 608         case RTE_INTR_HANDLE_UIO_INTX:
 609                 if (uio_intx_intr_disable(intr_handle))
 610                         return -1;
 611                 break;
 612         /* not used at this moment */
 613         case RTE_INTR_HANDLE_ALARM:
 614                 return -1;
 615 #ifdef VFIO_PRESENT
 616         case RTE_INTR_HANDLE_VFIO_MSIX:
 617                 if (vfio_disable_msix(intr_handle))
 618                         return -1;
 619                 break;
 620         case RTE_INTR_HANDLE_VFIO_MSI:
 621                 return 0;
 622         case RTE_INTR_HANDLE_VFIO_LEGACY:
 623                 if (vfio_disable_intx(intr_handle))
 624                         return -1;
 625                 break;
 626 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 627         case RTE_INTR_HANDLE_VFIO_REQ:
 628                 if (vfio_disable_req(intr_handle))
 629                         return -1;
 630                 break;
 631 #endif
 632 #endif
 633         /* not used at this moment */
 634         case RTE_INTR_HANDLE_DEV_EVENT:
 635                 return -1;
 636         /* unknown handle type */
 637         default:
 638                 RTE_LOG(ERR, EAL,
 639                         "Unknown handle type of fd %d\n",
 640                                         intr_handle->fd);
 641                 return -1;
 642         }
 643
 644         return 0;
 645 }
 646
 647 static int
 648 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 649 {
 650         bool call = false;
 651         int n, bytes_read, rv;
 652         struct rte_intr_source *src;
 653         struct rte_intr_callback *cb, *next;
 654         union rte_intr_read_buffer buf;
 655         struct rte_intr_callback active_cb;
 656
 657         for (n = 0; n < nfds; n++) {
 658
 659                 /**
 660                  * if the pipe fd is ready to read, return out to
 661                  * rebuild the wait list.
 662                  */
 663                 if (events[n].data.fd == intr_pipe.readfd){
 664                         int r = read(intr_pipe.readfd, buf.charbuf,
 665                                         sizeof(buf.charbuf));
 666                         RTE_SET_USED(r);
 667                         return -1;
 668                 }
 669                 rte_spinlock_lock(&intr_lock);
 670                 TAILQ_FOREACH(src, &intr_sources, next)
 671                         if (src->intr_handle.fd ==
 672                                         events[n].data.fd)
 673                                 break;
 674                 if (src == NULL){
 675                         rte_spinlock_unlock(&intr_lock);
 676                         continue;
 677                 }
 678
 679                 /* mark this interrupt source as active and release the lock. */
 680                 src->active = 1;
 681                 rte_spinlock_unlock(&intr_lock);
 682
 683                 /* set the length to be read dor different handle type */
 684                 switch (src->intr_handle.type) {
 685                 case RTE_INTR_HANDLE_UIO:
 686                 case RTE_INTR_HANDLE_UIO_INTX:
 687                         bytes_read = sizeof(buf.uio_intr_count);
 688                         break;
 689                 case RTE_INTR_HANDLE_ALARM:
 690                         bytes_read = sizeof(buf.timerfd_num);
 691                         break;
 692 #ifdef VFIO_PRESENT
 693                 case RTE_INTR_HANDLE_VFIO_MSIX:
 694                 case RTE_INTR_HANDLE_VFIO_MSI:
 695                 case RTE_INTR_HANDLE_VFIO_LEGACY:
 696                         bytes_read = sizeof(buf.vfio_intr_count);
 697                         break;
 698 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 699                 case RTE_INTR_HANDLE_VFIO_REQ:
 700                         bytes_read = 0;
 701                         call = true;
 702                         break;
 703 #endif
 704 #endif
 705                 case RTE_INTR_HANDLE_VDEV:
 706                 case RTE_INTR_HANDLE_EXT:
 707                         bytes_read = 0;
 708                         call = true;
 709                         break;
 710                 case RTE_INTR_HANDLE_DEV_EVENT:
 711                         bytes_read = 0;
 712                         call = true;
 713                         break;
 714                 default:
 715                         bytes_read = 1;
 716                         break;
 717                 }
 718
 719                 if (bytes_read > 0) {
 720                         /**
 721                          * read out to clear the ready-to-be-read flag
 722                          * for epoll_wait.
 723                          */
 724                         bytes_read = read(events[n].data.fd, &buf, bytes_read);
 725                         if (bytes_read < 0) {
 726                                 if (errno == EINTR || errno == EWOULDBLOCK)
 727                                         continue;
 728
 729                                 RTE_LOG(ERR, EAL, "Error reading from file "
 730                                         "descriptor %d: %s\n",
 731                                         events[n].data.fd,
 732                                         strerror(errno));
 733                                 /*
 734                                  * The device is unplugged or buggy, remove
 735                                  * it as an interrupt source and return to
 736                                  * force the wait list to be rebuilt.
 737                                  */
 738                                 rte_spinlock_lock(&intr_lock);
 739                                 TAILQ_REMOVE(&intr_sources, src, next);
 740                                 rte_spinlock_unlock(&intr_lock);
 741
 742                                 for (cb = TAILQ_FIRST(&src->callbacks); cb;
 743                                                         cb = next) {
 744                                         next = TAILQ_NEXT(cb, next);
 745                                         TAILQ_REMOVE(&src->callbacks, cb, next);
 746                                         free(cb);
 747                                 }
 748                                 free(src);
 749                                 return -1;
 750                         } else if (bytes_read == 0)
 751                                 RTE_LOG(ERR, EAL, "Read nothing from file "
 752                                         "descriptor %d\n", events[n].data.fd);
 753                         else
 754                                 call = true;
 755                 }
 756
 757                 /* grab a lock, again to call callbacks and update status. */
 758                 rte_spinlock_lock(&intr_lock);
 759
 760                 if (call) {
 761
 762                         /* Finally, call all callbacks. */
 763                         TAILQ_FOREACH(cb, &src->callbacks, next) {
 764
 765                                 /* make a copy and unlock. */
 766                                 active_cb = *cb;
 767                                 rte_spinlock_unlock(&intr_lock);
 768
 769                                 /* call the actual callback */
 770                                 active_cb.cb_fn(active_cb.cb_arg);
 771
 772                                 /*get the lock back. */
 773                                 rte_spinlock_lock(&intr_lock);
 774                         }
 775                 }
 776                 /* we done with that interrupt source, release it. */
 777                 src->active = 0;
 778
 779                 rv = 0;
 780
 781                 /* check if any callback are supposed to be removed */
 782                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 783                         next = TAILQ_NEXT(cb, next);
 784                         if (cb->pending_delete) {
 785                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 786                                 if (cb->ucb_fn)
 787                                         cb->ucb_fn(&src->intr_handle, cb->cb_arg);
 788                                 free(cb);
 789                                 rv++;
 790                         }
 791                 }
 792
 793                 /* all callbacks for that source are removed. */
 794                 if (TAILQ_EMPTY(&src->callbacks)) {
 795                         TAILQ_REMOVE(&intr_sources, src, next);
 796                         free(src);
 797                 }
 798
 799                 /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 800                 if (rv >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 801                         rte_spinlock_unlock(&intr_lock);
 802                         return -EPIPE;
 803                 }
 804
 805                 rte_spinlock_unlock(&intr_lock);
 806         }
 807
 808         return 0;
 809 }
 810
 811 /**
 812  * It handles all the interrupts.
 813  *
 814  * @param pfd
 815  *  epoll file descriptor.
 816  * @param totalfds
 817  *  The number of file descriptors added in epoll.
 818  *
 819  * @return
 820  *  void
 821  */
 822 static void
 823 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
 824 {
 825         struct epoll_event events[totalfds];
 826         int nfds = 0;
 827
 828         for(;;) {
 829                 nfds = epoll_wait(pfd, events, totalfds,
 830                         EAL_INTR_EPOLL_WAIT_FOREVER);
 831                 /* epoll_wait fail */
 832                 if (nfds < 0) {
 833                         if (errno == EINTR)
 834                                 continue;
 835                         RTE_LOG(ERR, EAL,
 836                                 "epoll_wait returns with fail\n");
 837                         return;
 838                 }
 839                 /* epoll_wait timeout, will never happens here */
 840                 else if (nfds == 0)
 841                         continue;
 842                 /* epoll_wait has at least one fd ready to read */
 843                 if (eal_intr_process_interrupts(events, nfds) < 0)
 844                         return;
 845         }
 846 }
 847
 848 /**
 849  * It builds/rebuilds up the epoll file descriptor with all the
 850  * file descriptors being waited on. Then handles the interrupts.
 851  *
 852  * @param arg
 853  *  pointer. (unused)
 854  *
 855  * @return
 856  *  never return;
 857  */
 858 static __attribute__((noreturn)) void *
 859 eal_intr_thread_main(__rte_unused void *arg)
 860 {
 861         struct epoll_event ev;
 862
 863         /* host thread, never break out */
 864         for (;;) {
 865                 /* build up the epoll fd with all descriptors we are to
 866                  * wait on then pass it to the handle_interrupts function
 867                  */
 868                 static struct epoll_event pipe_event = {
 869                         .events = EPOLLIN | EPOLLPRI,
 870                 };
 871                 struct rte_intr_source *src;
 872                 unsigned numfds = 0;
 873
 874                 /* create epoll fd */
 875                 int pfd = epoll_create(1);
 876                 if (pfd < 0)
 877                         rte_panic("Cannot create epoll instance\n");
 878
 879                 pipe_event.data.fd = intr_pipe.readfd;
 880                 /**
 881                  * add pipe fd into wait list, this pipe is used to
 882                  * rebuild the wait list.
 883                  */
 884                 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
 885                                                 &pipe_event) < 0) {
 886                         rte_panic("Error adding fd to %d epoll_ctl, %s\n",
 887                                         intr_pipe.readfd, strerror(errno));
 888                 }
 889                 numfds++;
 890
 891                 rte_spinlock_lock(&intr_lock);
 892
 893                 TAILQ_FOREACH(src, &intr_sources, next) {
 894                         if (src->callbacks.tqh_first == NULL)
 895                                 continue; /* skip those with no callbacks */
 896                         ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
 897                         ev.data.fd = src->intr_handle.fd;
 898
 899                         /**
 900                          * add all the uio device file descriptor
 901                          * into wait list.
 902                          */
 903                         if (epoll_ctl(pfd, EPOLL_CTL_ADD,
 904                                         src->intr_handle.fd, &ev) < 0){
 905                                 rte_panic("Error adding fd %d epoll_ctl, %s\n",
 906                                         src->intr_handle.fd, strerror(errno));
 907                         }
 908                         else
 909                                 numfds++;
 910                 }
 911                 rte_spinlock_unlock(&intr_lock);
 912                 /* serve the interrupt */
 913                 eal_intr_handle_interrupts(pfd, numfds);
 914
 915                 /**
 916                  * when we return, we need to rebuild the
 917                  * list of fds to monitor.
 918                  */
 919                 close(pfd);
 920         }
 921 }
 922
 923 int
 924 rte_eal_intr_init(void)
 925 {
 926         int ret = 0;
 927
 928         /* init the global interrupt source head */
 929         TAILQ_INIT(&intr_sources);
 930
 931         /**
 932          * create a pipe which will be waited by epoll and notified to
 933          * rebuild the wait list of epoll.
 934          */
 935         if (pipe(intr_pipe.pipefd) < 0) {
 936                 rte_errno = errno;
 937                 return -1;
 938         }
 939
 940         /* create the host thread to wait/handle the interrupt */
 941         ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
 942                         eal_intr_thread_main, NULL);
 943         if (ret != 0) {
 944                 rte_errno = -ret;
 945                 RTE_LOG(ERR, EAL,
 946                         "Failed to create thread for interrupt handling\n");
 947         }
 948
 949         return ret;
 950 }
 951
 952 static void
 953 eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
 954 {
 955         union rte_intr_read_buffer buf;
 956         int bytes_read = 0;
 957         int nbytes;
 958
 959         switch (intr_handle->type) {
 960         case RTE_INTR_HANDLE_UIO:
 961         case RTE_INTR_HANDLE_UIO_INTX:
 962                 bytes_read = sizeof(buf.uio_intr_count);
 963                 break;
 964 #ifdef VFIO_PRESENT
 965         case RTE_INTR_HANDLE_VFIO_MSIX:
 966         case RTE_INTR_HANDLE_VFIO_MSI:
 967         case RTE_INTR_HANDLE_VFIO_LEGACY:
 968                 bytes_read = sizeof(buf.vfio_intr_count);
 969                 break;
 970 #endif
 971         case RTE_INTR_HANDLE_VDEV:
 972                 bytes_read = intr_handle->efd_counter_size;
 973                 /* For vdev, number of bytes to read is set by driver */
 974                 break;
 975         case RTE_INTR_HANDLE_EXT:
 976                 return;
 977         default:
 978                 bytes_read = 1;
 979                 RTE_LOG(INFO, EAL, "unexpected intr type\n");
 980                 break;
 981         }
 982
 983         /**
 984          * read out to clear the ready-to-be-read flag
 985          * for epoll_wait.
 986          */
 987         if (bytes_read == 0)
 988                 return;
 989         do {
 990                 nbytes = read(fd, &buf, bytes_read);
 991                 if (nbytes < 0) {
 992                         if (errno == EINTR || errno == EWOULDBLOCK ||
 993                             errno == EAGAIN)
 994                                 continue;
 995                         RTE_LOG(ERR, EAL,
 996                                 "Error reading from fd %d: %s\n",
 997                                 fd, strerror(errno));
 998                 } else if (nbytes == 0)
 999                         RTE_LOG(ERR, EAL, "Read nothing from fd %d\n", fd);
1000                 return;
1001         } while (1);
1002 }
1003
1004 static int
1005 eal_epoll_process_event(struct epoll_event *evs, unsigned int n,
1006                         struct rte_epoll_event *events)
1007 {
1008         unsigned int i, count = 0;
1009         struct rte_epoll_event *rev;
1010
1011         for (i = 0; i < n; i++) {
1012                 rev = evs[i].data.ptr;
1013                 if (!rev || !rte_atomic32_cmpset(&rev->status, RTE_EPOLL_VALID,
1014                                                  RTE_EPOLL_EXEC))
1015                         continue;
1016
1017                 events[count].status        = RTE_EPOLL_VALID;
1018                 events[count].fd            = rev->fd;
1019                 events[count].epfd          = rev->epfd;
1020                 events[count].epdata.event  = rev->epdata.event;
1021                 events[count].epdata.data   = rev->epdata.data;
1022                 if (rev->epdata.cb_fun)
1023                         rev->epdata.cb_fun(rev->fd,
1024                                            rev->epdata.cb_arg);
1025
1026                 rte_compiler_barrier();
1027                 rev->status = RTE_EPOLL_VALID;
1028                 count++;
1029         }
1030         return count;
1031 }
1032
1033 static inline int
1034 eal_init_tls_epfd(void)
1035 {
1036         int pfd = epoll_create(255);
1037
1038         if (pfd < 0) {
1039                 RTE_LOG(ERR, EAL,
1040                         "Cannot create epoll instance\n");
1041                 return -1;
1042         }
1043         return pfd;
1044 }
1045
1046 int
1047 rte_intr_tls_epfd(void)
1048 {
1049         if (RTE_PER_LCORE(_epfd) == -1)
1050                 RTE_PER_LCORE(_epfd) = eal_init_tls_epfd();
1051
1052         return RTE_PER_LCORE(_epfd);
1053 }
1054
1055 int
1056 rte_epoll_wait(int epfd, struct rte_epoll_event *events,
1057                int maxevents, int timeout)
1058 {
1059         struct epoll_event evs[maxevents];
1060         int rc;
1061
1062         if (!events) {
1063                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
1064                 return -1;
1065         }
1066
1067         /* using per thread epoll fd */
1068         if (epfd == RTE_EPOLL_PER_THREAD)
1069                 epfd = rte_intr_tls_epfd();
1070
1071         while (1) {
1072                 rc = epoll_wait(epfd, evs, maxevents, timeout);
1073                 if (likely(rc > 0)) {
1074                         /* epoll_wait has at least one fd ready to read */
1075                         rc = eal_epoll_process_event(evs, rc, events);
1076                         break;
1077                 } else if (rc < 0) {
1078                         if (errno == EINTR)
1079                                 continue;
1080                         /* epoll_wait fail */
1081                         RTE_LOG(ERR, EAL, "epoll_wait returns with fail %s\n",
1082                                 strerror(errno));
1083                         rc = -1;
1084                         break;
1085                 } else {
1086                         /* rc == 0, epoll_wait timed out */
1087                         break;
1088                 }
1089         }
1090
1091         return rc;
1092 }
1093
1094 static inline void
1095 eal_epoll_data_safe_free(struct rte_epoll_event *ev)
1096 {
1097         while (!rte_atomic32_cmpset(&ev->status, RTE_EPOLL_VALID,
1098                                     RTE_EPOLL_INVALID))
1099                 while (ev->status != RTE_EPOLL_VALID)
1100                         rte_pause();
1101         memset(&ev->epdata, 0, sizeof(ev->epdata));
1102         ev->fd = -1;
1103         ev->epfd = -1;
1104 }
1105
1106 int
1107 rte_epoll_ctl(int epfd, int op, int fd,
1108               struct rte_epoll_event *event)
1109 {
1110         struct epoll_event ev;
1111
1112         if (!event) {
1113                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
1114                 return -1;
1115         }
1116
1117         /* using per thread epoll fd */
1118         if (epfd == RTE_EPOLL_PER_THREAD)
1119                 epfd = rte_intr_tls_epfd();
1120
1121         if (op == EPOLL_CTL_ADD) {
1122                 event->status = RTE_EPOLL_VALID;
1123                 event->fd = fd;  /* ignore fd in event */
1124                 event->epfd = epfd;
1125                 ev.data.ptr = (void *)event;
1126         }
1127
1128         ev.events = event->epdata.event;
1129         if (epoll_ctl(epfd, op, fd, &ev) < 0) {
1130                 RTE_LOG(ERR, EAL, "Error op %d fd %d epoll_ctl, %s\n",
1131                         op, fd, strerror(errno));
1132                 if (op == EPOLL_CTL_ADD)
1133                         /* rollback status when CTL_ADD fail */
1134                         event->status = RTE_EPOLL_INVALID;
1135                 return -1;
1136         }
1137
1138         if (op == EPOLL_CTL_DEL && event->status != RTE_EPOLL_INVALID)
1139                 eal_epoll_data_safe_free(event);
1140
1141         return 0;
1142 }
1143
1144 int
1145 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
1146                 int op, unsigned int vec, void *data)
1147 {
1148         struct rte_epoll_event *rev;
1149         struct rte_epoll_data *epdata;
1150         int epfd_op;
1151         unsigned int efd_idx;
1152         int rc = 0;
1153
1154         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
1155                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
1156
1157         if (!intr_handle || intr_handle->nb_efd == 0 ||
1158             efd_idx >= intr_handle->nb_efd) {
1159                 RTE_LOG(ERR, EAL, "Wrong intr vector number.\n");
1160                 return -EPERM;
1161         }
1162
1163         switch (op) {
1164         case RTE_INTR_EVENT_ADD:
1165                 epfd_op = EPOLL_CTL_ADD;
1166                 rev = &intr_handle->elist[efd_idx];
1167                 if (rev->status != RTE_EPOLL_INVALID) {
1168                         RTE_LOG(INFO, EAL, "Event already been added.\n");
1169                         return -EEXIST;
1170                 }
1171
1172                 /* attach to intr vector fd */
1173                 epdata = &rev->epdata;
1174                 epdata->event  = EPOLLIN | EPOLLPRI | EPOLLET;
1175                 epdata->data   = data;
1176                 epdata->cb_fun = (rte_intr_event_cb_t)eal_intr_proc_rxtx_intr;
1177                 epdata->cb_arg = (void *)intr_handle;
1178                 rc = rte_epoll_ctl(epfd, epfd_op,
1179                                    intr_handle->efds[efd_idx], rev);
1180                 if (!rc)
1181                         RTE_LOG(DEBUG, EAL,
1182                                 "efd %d associated with vec %d added on epfd %d"
1183                                 "\n", rev->fd, vec, epfd);
1184                 else
1185                         rc = -EPERM;
1186                 break;
1187         case RTE_INTR_EVENT_DEL:
1188                 epfd_op = EPOLL_CTL_DEL;
1189                 rev = &intr_handle->elist[efd_idx];
1190                 if (rev->status == RTE_EPOLL_INVALID) {
1191                         RTE_LOG(INFO, EAL, "Event does not exist.\n");
1192                         return -EPERM;
1193                 }
1194
1195                 rc = rte_epoll_ctl(rev->epfd, epfd_op, rev->fd, rev);
1196                 if (rc)
1197                         rc = -EPERM;
1198                 break;
1199         default:
1200                 RTE_LOG(ERR, EAL, "event op type mismatch\n");
1201                 rc = -EPERM;
1202         }
1203
1204         return rc;
1205 }
1206
1207 void
1208 rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
1209 {
1210         uint32_t i;
1211         struct rte_epoll_event *rev;
1212
1213         for (i = 0; i < intr_handle->nb_efd; i++) {
1214                 rev = &intr_handle->elist[i];
1215                 if (rev->status == RTE_EPOLL_INVALID)
1216                         continue;
1217                 if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
1218                         /* force free if the entry valid */
1219                         eal_epoll_data_safe_free(rev);
1220                         rev->status = RTE_EPOLL_INVALID;
1221                 }
1222         }
1223 }
1224
1225 int
1226 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
1227 {
1228         uint32_t i;
1229         int fd;
1230         uint32_t n = RTE_MIN(nb_efd, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
1231
1232         assert(nb_efd != 0);
1233
1234         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) {
1235                 for (i = 0; i < n; i++) {
1236                         fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
1237                         if (fd < 0) {
1238                                 RTE_LOG(ERR, EAL,
1239                                         "can't setup eventfd, error %i (%s)\n",
1240                                         errno, strerror(errno));
1241                                 return -errno;
1242                         }
1243                         intr_handle->efds[i] = fd;
1244                 }
1245                 intr_handle->nb_efd   = n;
1246                 intr_handle->max_intr = NB_OTHER_INTR + n;
1247         } else if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
1248                 /* only check, initialization would be done in vdev driver.*/
1249                 if (intr_handle->efd_counter_size >
1250                     sizeof(union rte_intr_read_buffer)) {
1251                         RTE_LOG(ERR, EAL, "the efd_counter_size is oversized");
1252                         return -EINVAL;
1253                 }
1254         } else {
1255                 intr_handle->efds[0]  = intr_handle->fd;
1256                 intr_handle->nb_efd   = RTE_MIN(nb_efd, 1U);
1257                 intr_handle->max_intr = NB_OTHER_INTR;
1258         }
1259
1260         return 0;
1261 }
1262
1263 void
1264 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
1265 {
1266         uint32_t i;
1267
1268         rte_intr_free_epoll_fd(intr_handle);
1269         if (intr_handle->max_intr > intr_handle->nb_efd) {
1270                 for (i = 0; i < intr_handle->nb_efd; i++)
1271                         close(intr_handle->efds[i]);
1272         }
1273         intr_handle->nb_efd = 0;
1274         intr_handle->max_intr = 0;
1275 }
1276
1277 int
1278 rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
1279 {
1280         return !(!intr_handle->nb_efd);
1281 }
1282
1283 int
1284 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
1285 {
1286         if (!rte_intr_dp_is_en(intr_handle))
1287                 return 1;
1288         else
1289                 return !!(intr_handle->max_intr - intr_handle->nb_efd);
1290 }
1291
1292 int
1293 rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
1294 {
1295         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX)
1296                 return 1;
1297
1298         if (intr_handle->type == RTE_INTR_HANDLE_VDEV)
1299                 return 1;
1300
1301         return 0;
1302 }