lib/librte_eal/linux/eal_interrupts.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdint.h>
   7 #include <stdlib.h>
   8 #include <pthread.h>
   9 #include <sys/queue.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <string.h>
  13 #include <errno.h>
  14 #include <inttypes.h>
  15 #include <sys/epoll.h>
  16 #include <sys/signalfd.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/eventfd.h>
  19 #include <assert.h>
  20 #include <stdbool.h>
  21
  22 #include <rte_common.h>
  23 #include <rte_interrupts.h>
  24 #include <rte_memory.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_debug.h>
  32 #include <rte_log.h>
  33 #include <rte_errno.h>
  34 #include <rte_spinlock.h>
  35 #include <rte_pause.h>
  36 #include <rte_vfio.h>
  37 #include <rte_eal_trace.h>
  38
  39 #include "eal_private.h"
  40 #include "eal_vfio.h"
  41 #include "eal_thread.h"
  42
  43 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
  44 #define NB_OTHER_INTR               1
  45
  46 static RTE_DEFINE_PER_LCORE(int, _epfd) = -1; /**< epoll fd per thread */
  47
  48 /**
  49  * union for pipe fds.
  50  */
  51 union intr_pipefds{
  52         struct {
  53                 int pipefd[2];
  54         };
  55         struct {
  56                 int readfd;
  57                 int writefd;
  58         };
  59 };
  60
  61 /**
  62  * union buffer for reading on different devices
  63  */
  64 union rte_intr_read_buffer {
  65         int uio_intr_count;              /* for uio device */
  66 #ifdef VFIO_PRESENT
  67         uint64_t vfio_intr_count;        /* for vfio device */
  68 #endif
  69         uint64_t timerfd_num;            /* for timerfd */
  70         char charbuf[16];                /* for others */
  71 };
  72
  73 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
  74 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
  75
  76 struct rte_intr_callback {
  77         TAILQ_ENTRY(rte_intr_callback) next;
  78         rte_intr_callback_fn cb_fn;  /**< callback address */
  79         void *cb_arg;                /**< parameter for callback */
  80         uint8_t pending_delete;      /**< delete after callback is called */
  81         rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
  82 };
  83
  84 struct rte_intr_source {
  85         TAILQ_ENTRY(rte_intr_source) next;
  86         struct rte_intr_handle intr_handle; /**< interrupt handle */
  87         struct rte_intr_cb_list callbacks;  /**< user callbacks */
  88         uint32_t active;
  89 };
  90
  91 /* global spinlock for interrupt data operation */
  92 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
  93
  94 /* union buffer for pipe read/write */
  95 static union intr_pipefds intr_pipe;
  96
  97 /* interrupt sources list */
  98 static struct rte_intr_source_list intr_sources;
  99
 100 /* interrupt handling thread */
 101 static pthread_t intr_thread;
 102
 103 /* VFIO interrupts */
 104 #ifdef VFIO_PRESENT
 105
 106 #define IRQ_SET_BUF_LEN  (sizeof(struct vfio_irq_set) + sizeof(int))
 107 /* irq set buffer length for queue interrupts and LSC interrupt */
 108 #define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
 109                               sizeof(int) * (RTE_MAX_RXTX_INTR_VEC_ID + 1))
 110
 111 /* enable legacy (INTx) interrupts */
 112 static int
 113 vfio_enable_intx(const struct rte_intr_handle *intr_handle) {
 114         struct vfio_irq_set *irq_set;
 115         char irq_set_buf[IRQ_SET_BUF_LEN];
 116         int len, ret;
 117         int *fd_ptr;
 118
 119         len = sizeof(irq_set_buf);
 120
 121         /* enable INTx */
 122         irq_set = (struct vfio_irq_set *) irq_set_buf;
 123         irq_set->argsz = len;
 124         irq_set->count = 1;
 125         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 126         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 127         irq_set->start = 0;
 128         fd_ptr = (int *) &irq_set->data;
 129         *fd_ptr = intr_handle->fd;
 130
 131         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 132
 133         if (ret) {
 134                 RTE_LOG(ERR, EAL, "Error enabling INTx interrupts for fd %d\n",
 135                                                 intr_handle->fd);
 136                 return -1;
 137         }
 138
 139         /* unmask INTx after enabling */
 140         memset(irq_set, 0, len);
 141         len = sizeof(struct vfio_irq_set);
 142         irq_set->argsz = len;
 143         irq_set->count = 1;
 144         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
 145         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 146         irq_set->start = 0;
 147
 148         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 149
 150         if (ret) {
 151                 RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
 152                                                 intr_handle->fd);
 153                 return -1;
 154         }
 155         return 0;
 156 }
 157
 158 /* disable legacy (INTx) interrupts */
 159 static int
 160 vfio_disable_intx(const struct rte_intr_handle *intr_handle) {
 161         struct vfio_irq_set *irq_set;
 162         char irq_set_buf[IRQ_SET_BUF_LEN];
 163         int len, ret;
 164
 165         len = sizeof(struct vfio_irq_set);
 166
 167         /* mask interrupts before disabling */
 168         irq_set = (struct vfio_irq_set *) irq_set_buf;
 169         irq_set->argsz = len;
 170         irq_set->count = 1;
 171         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK;
 172         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 173         irq_set->start = 0;
 174
 175         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 176
 177         if (ret) {
 178                 RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n",
 179                                                 intr_handle->fd);
 180                 return -1;
 181         }
 182
 183         /* disable INTx*/
 184         memset(irq_set, 0, len);
 185         irq_set->argsz = len;
 186         irq_set->count = 0;
 187         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 188         irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
 189         irq_set->start = 0;
 190
 191         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 192
 193         if (ret) {
 194                 RTE_LOG(ERR, EAL,
 195                         "Error disabling INTx interrupts for fd %d\n", intr_handle->fd);
 196                 return -1;
 197         }
 198         return 0;
 199 }
 200
 201 /* unmask/ack legacy (INTx) interrupts */
 202 static int
 203 vfio_ack_intx(const struct rte_intr_handle *intr_handle)
 204 {
 205         struct vfio_irq_set irq_set;
 206
 207         /* unmask INTx */
 208         memset(&irq_set, 0, sizeof(irq_set));
 209         irq_set.argsz = sizeof(irq_set);
 210         irq_set.count = 1;
 211         irq_set.flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
 212         irq_set.index = VFIO_PCI_INTX_IRQ_INDEX;
 213         irq_set.start = 0;
 214
 215         if (ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, &irq_set)) {
 216                 RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
 217                         intr_handle->fd);
 218                 return -1;
 219         }
 220         return 0;
 221 }
 222
 223 /* enable MSI interrupts */
 224 static int
 225 vfio_enable_msi(const struct rte_intr_handle *intr_handle) {
 226         int len, ret;
 227         char irq_set_buf[IRQ_SET_BUF_LEN];
 228         struct vfio_irq_set *irq_set;
 229         int *fd_ptr;
 230
 231         len = sizeof(irq_set_buf);
 232
 233         irq_set = (struct vfio_irq_set *) irq_set_buf;
 234         irq_set->argsz = len;
 235         irq_set->count = 1;
 236         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 237         irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
 238         irq_set->start = 0;
 239         fd_ptr = (int *) &irq_set->data;
 240         *fd_ptr = intr_handle->fd;
 241
 242         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 243
 244         if (ret) {
 245                 RTE_LOG(ERR, EAL, "Error enabling MSI interrupts for fd %d\n",
 246                                                 intr_handle->fd);
 247                 return -1;
 248         }
 249         return 0;
 250 }
 251
 252 /* disable MSI interrupts */
 253 static int
 254 vfio_disable_msi(const struct rte_intr_handle *intr_handle) {
 255         struct vfio_irq_set *irq_set;
 256         char irq_set_buf[IRQ_SET_BUF_LEN];
 257         int len, ret;
 258
 259         len = sizeof(struct vfio_irq_set);
 260
 261         irq_set = (struct vfio_irq_set *) irq_set_buf;
 262         irq_set->argsz = len;
 263         irq_set->count = 0;
 264         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 265         irq_set->index = VFIO_PCI_MSI_IRQ_INDEX;
 266         irq_set->start = 0;
 267
 268         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 269
 270         if (ret)
 271                 RTE_LOG(ERR, EAL,
 272                         "Error disabling MSI interrupts for fd %d\n", intr_handle->fd);
 273
 274         return ret;
 275 }
 276
 277 /* enable MSI-X interrupts */
 278 static int
 279 vfio_enable_msix(const struct rte_intr_handle *intr_handle) {
 280         int len, ret;
 281         char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
 282         struct vfio_irq_set *irq_set;
 283         int *fd_ptr;
 284
 285         len = sizeof(irq_set_buf);
 286
 287         irq_set = (struct vfio_irq_set *) irq_set_buf;
 288         irq_set->argsz = len;
 289         /* 0 < irq_set->count < RTE_MAX_RXTX_INTR_VEC_ID + 1 */
 290         irq_set->count = intr_handle->max_intr ?
 291                 (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID + 1 ?
 292                 RTE_MAX_RXTX_INTR_VEC_ID + 1 : intr_handle->max_intr) : 1;
 293         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
 294         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
 295         irq_set->start = 0;
 296         fd_ptr = (int *) &irq_set->data;
 297         /* INTR vector offset 0 reserve for non-efds mapping */
 298         fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = intr_handle->fd;
 299         memcpy(&fd_ptr[RTE_INTR_VEC_RXTX_OFFSET], intr_handle->efds,
 300                 sizeof(*intr_handle->efds) * intr_handle->nb_efd);
 301
 302         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 303
 304         if (ret) {
 305                 RTE_LOG(ERR, EAL, "Error enabling MSI-X interrupts for fd %d\n",
 306                                                 intr_handle->fd);
 307                 return -1;
 308         }
 309
 310         return 0;
 311 }
 312
 313 /* disable MSI-X interrupts */
 314 static int
 315 vfio_disable_msix(const struct rte_intr_handle *intr_handle) {
 316         struct vfio_irq_set *irq_set;
 317         char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
 318         int len, ret;
 319
 320         len = sizeof(struct vfio_irq_set);
 321
 322         irq_set = (struct vfio_irq_set *) irq_set_buf;
 323         irq_set->argsz = len;
 324         irq_set->count = 0;
 325         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 326         irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
 327         irq_set->start = 0;
 328
 329         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 330
 331         if (ret)
 332                 RTE_LOG(ERR, EAL,
 333                         "Error disabling MSI-X interrupts for fd %d\n", intr_handle->fd);
 334
 335         return ret;
 336 }
 337
 338 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 339 /* enable req notifier */
 340 static int
 341 vfio_enable_req(const struct rte_intr_handle *intr_handle)
 342 {
 343         int len, ret;
 344         char irq_set_buf[IRQ_SET_BUF_LEN];
 345         struct vfio_irq_set *irq_set;
 346         int *fd_ptr;
 347
 348         len = sizeof(irq_set_buf);
 349
 350         irq_set = (struct vfio_irq_set *) irq_set_buf;
 351         irq_set->argsz = len;
 352         irq_set->count = 1;
 353         irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
 354                          VFIO_IRQ_SET_ACTION_TRIGGER;
 355         irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
 356         irq_set->start = 0;
 357         fd_ptr = (int *) &irq_set->data;
 358         *fd_ptr = intr_handle->fd;
 359
 360         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 361
 362         if (ret) {
 363                 RTE_LOG(ERR, EAL, "Error enabling req interrupts for fd %d\n",
 364                                                 intr_handle->fd);
 365                 return -1;
 366         }
 367
 368         return 0;
 369 }
 370
 371 /* disable req notifier */
 372 static int
 373 vfio_disable_req(const struct rte_intr_handle *intr_handle)
 374 {
 375         struct vfio_irq_set *irq_set;
 376         char irq_set_buf[IRQ_SET_BUF_LEN];
 377         int len, ret;
 378
 379         len = sizeof(struct vfio_irq_set);
 380
 381         irq_set = (struct vfio_irq_set *) irq_set_buf;
 382         irq_set->argsz = len;
 383         irq_set->count = 0;
 384         irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
 385         irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
 386         irq_set->start = 0;
 387
 388         ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 389
 390         if (ret)
 391                 RTE_LOG(ERR, EAL, "Error disabling req interrupts for fd %d\n",
 392                         intr_handle->fd);
 393
 394         return ret;
 395 }
 396 #endif
 397 #endif
 398
 399 static int
 400 uio_intx_intr_disable(const struct rte_intr_handle *intr_handle)
 401 {
 402         unsigned char command_high;
 403
 404         /* use UIO config file descriptor for uio_pci_generic */
 405         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 406                 RTE_LOG(ERR, EAL,
 407                         "Error reading interrupts status for fd %d\n",
 408                         intr_handle->uio_cfg_fd);
 409                 return -1;
 410         }
 411         /* disable interrupts */
 412         command_high |= 0x4;
 413         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 414                 RTE_LOG(ERR, EAL,
 415                         "Error disabling interrupts for fd %d\n",
 416                         intr_handle->uio_cfg_fd);
 417                 return -1;
 418         }
 419
 420         return 0;
 421 }
 422
 423 static int
 424 uio_intx_intr_enable(const struct rte_intr_handle *intr_handle)
 425 {
 426         unsigned char command_high;
 427
 428         /* use UIO config file descriptor for uio_pci_generic */
 429         if (pread(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 430                 RTE_LOG(ERR, EAL,
 431                         "Error reading interrupts status for fd %d\n",
 432                         intr_handle->uio_cfg_fd);
 433                 return -1;
 434         }
 435         /* enable interrupts */
 436         command_high &= ~0x4;
 437         if (pwrite(intr_handle->uio_cfg_fd, &command_high, 1, 5) != 1) {
 438                 RTE_LOG(ERR, EAL,
 439                         "Error enabling interrupts for fd %d\n",
 440                         intr_handle->uio_cfg_fd);
 441                 return -1;
 442         }
 443
 444         return 0;
 445 }
 446
 447 static int
 448 uio_intr_disable(const struct rte_intr_handle *intr_handle)
 449 {
 450         const int value = 0;
 451
 452         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 453                 RTE_LOG(ERR, EAL,
 454                         "Error disabling interrupts for fd %d (%s)\n",
 455                         intr_handle->fd, strerror(errno));
 456                 return -1;
 457         }
 458         return 0;
 459 }
 460
 461 static int
 462 uio_intr_enable(const struct rte_intr_handle *intr_handle)
 463 {
 464         const int value = 1;
 465
 466         if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 467                 RTE_LOG(ERR, EAL,
 468                         "Error enabling interrupts for fd %d (%s)\n",
 469                         intr_handle->fd, strerror(errno));
 470                 return -1;
 471         }
 472         return 0;
 473 }
 474
 475 int
 476 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
 477                         rte_intr_callback_fn cb, void *cb_arg)
 478 {
 479         int ret, wake_thread;
 480         struct rte_intr_source *src;
 481         struct rte_intr_callback *callback;
 482
 483         wake_thread = 0;
 484
 485         /* first do parameter checking */
 486         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
 487                 RTE_LOG(ERR, EAL,
 488                         "Registering with invalid input parameter\n");
 489                 return -EINVAL;
 490         }
 491
 492         /* allocate a new interrupt callback entity */
 493         callback = calloc(1, sizeof(*callback));
 494         if (callback == NULL) {
 495                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 496                 return -ENOMEM;
 497         }
 498         callback->cb_fn = cb;
 499         callback->cb_arg = cb_arg;
 500         callback->pending_delete = 0;
 501         callback->ucb_fn = NULL;
 502
 503         rte_spinlock_lock(&intr_lock);
 504
 505         /* check if there is at least one callback registered for the fd */
 506         TAILQ_FOREACH(src, &intr_sources, next) {
 507                 if (src->intr_handle.fd == intr_handle->fd) {
 508                         /* we had no interrupts for this */
 509                         if (TAILQ_EMPTY(&src->callbacks))
 510                                 wake_thread = 1;
 511
 512                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 513                         ret = 0;
 514                         break;
 515                 }
 516         }
 517
 518         /* no existing callbacks for this - add new source */
 519         if (src == NULL) {
 520                 src = calloc(1, sizeof(*src));
 521                 if (src == NULL) {
 522                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 523                         free(callback);
 524                         ret = -ENOMEM;
 525                 } else {
 526                         src->intr_handle = *intr_handle;
 527                         TAILQ_INIT(&src->callbacks);
 528                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 529                         TAILQ_INSERT_TAIL(&intr_sources, src, next);
 530                         wake_thread = 1;
 531                         ret = 0;
 532                 }
 533         }
 534
 535         rte_spinlock_unlock(&intr_lock);
 536
 537         /**
 538          * check if need to notify the pipe fd waited by epoll_wait to
 539          * rebuild the wait list.
 540          */
 541         if (wake_thread)
 542                 if (write(intr_pipe.writefd, "1", 1) < 0)
 543                         ret = -EPIPE;
 544
 545         rte_eal_trace_intr_callback_register(intr_handle, cb, cb_arg, ret);
 546         return ret;
 547 }
 548
 549 int
 550 rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
 551                                 rte_intr_callback_fn cb_fn, void *cb_arg,
 552                                 rte_intr_unregister_callback_fn ucb_fn)
 553 {
 554         int ret;
 555         struct rte_intr_source *src;
 556         struct rte_intr_callback *cb, *next;
 557
 558         /* do parameter checking first */
 559         if (intr_handle == NULL || intr_handle->fd < 0) {
 560                 RTE_LOG(ERR, EAL,
 561                 "Unregistering with invalid input parameter\n");
 562                 return -EINVAL;
 563         }
 564
 565         rte_spinlock_lock(&intr_lock);
 566
 567         /* check if the insterrupt source for the fd is existent */
 568         TAILQ_FOREACH(src, &intr_sources, next)
 569                 if (src->intr_handle.fd == intr_handle->fd)
 570                         break;
 571
 572         /* No interrupt source registered for the fd */
 573         if (src == NULL) {
 574                 ret = -ENOENT;
 575
 576         /* only usable if the source is active */
 577         } else if (src->active == 0) {
 578                 ret = -EAGAIN;
 579
 580         } else {
 581                 ret = 0;
 582
 583                 /* walk through the callbacks and mark all that match. */
 584                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 585                         next = TAILQ_NEXT(cb, next);
 586                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 587                                         cb->cb_arg == cb_arg)) {
 588                                 cb->pending_delete = 1;
 589                                 cb->ucb_fn = ucb_fn;
 590                                 ret++;
 591                         }
 592                 }
 593         }
 594
 595         rte_spinlock_unlock(&intr_lock);
 596
 597         return ret;
 598 }
 599
 600 int
 601 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
 602                         rte_intr_callback_fn cb_fn, void *cb_arg)
 603 {
 604         int ret;
 605         struct rte_intr_source *src;
 606         struct rte_intr_callback *cb, *next;
 607
 608         /* do parameter checking first */
 609         if (intr_handle == NULL || intr_handle->fd < 0) {
 610                 RTE_LOG(ERR, EAL,
 611                 "Unregistering with invalid input parameter\n");
 612                 return -EINVAL;
 613         }
 614
 615         rte_spinlock_lock(&intr_lock);
 616
 617         /* check if the insterrupt source for the fd is existent */
 618         TAILQ_FOREACH(src, &intr_sources, next)
 619                 if (src->intr_handle.fd == intr_handle->fd)
 620                         break;
 621
 622         /* No interrupt source registered for the fd */
 623         if (src == NULL) {
 624                 ret = -ENOENT;
 625
 626         /* interrupt source has some active callbacks right now. */
 627         } else if (src->active != 0) {
 628                 ret = -EAGAIN;
 629
 630         /* ok to remove. */
 631         } else {
 632                 ret = 0;
 633
 634                 /*walk through the callbacks and remove all that match. */
 635                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 636
 637                         next = TAILQ_NEXT(cb, next);
 638
 639                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 640                                         cb->cb_arg == cb_arg)) {
 641                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 642                                 free(cb);
 643                                 ret++;
 644                         }
 645                 }
 646
 647                 /* all callbacks for that source are removed. */
 648                 if (TAILQ_EMPTY(&src->callbacks)) {
 649                         TAILQ_REMOVE(&intr_sources, src, next);
 650                         free(src);
 651                 }
 652         }
 653
 654         rte_spinlock_unlock(&intr_lock);
 655
 656         /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 657         if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 658                 ret = -EPIPE;
 659         }
 660
 661         rte_eal_trace_intr_callback_unregister(intr_handle, cb_fn, cb_arg,
 662                 ret);
 663         return ret;
 664 }
 665
 666 int
 667 rte_intr_enable(const struct rte_intr_handle *intr_handle)
 668 {
 669         int rc = 0;
 670
 671         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV) {
 672                 rc = 0;
 673                 goto out;
 674         }
 675
 676         if (!intr_handle || intr_handle->fd < 0 ||
 677                         intr_handle->uio_cfg_fd < 0) {
 678                 rc = -1;
 679                 goto out;
 680         }
 681
 682         switch (intr_handle->type){
 683         /* write to the uio fd to enable the interrupt */
 684         case RTE_INTR_HANDLE_UIO:
 685                 if (uio_intr_enable(intr_handle))
 686                         rc = -1;
 687                 break;
 688         case RTE_INTR_HANDLE_UIO_INTX:
 689                 if (uio_intx_intr_enable(intr_handle))
 690                         rc = -1;
 691                 break;
 692         /* not used at this moment */
 693         case RTE_INTR_HANDLE_ALARM:
 694                 rc = -1;
 695                 break;
 696 #ifdef VFIO_PRESENT
 697         case RTE_INTR_HANDLE_VFIO_MSIX:
 698                 if (vfio_enable_msix(intr_handle))
 699                         rc = -1;
 700                 break;
 701         case RTE_INTR_HANDLE_VFIO_MSI:
 702                 if (vfio_enable_msi(intr_handle))
 703                         rc = -1;
 704                 break;
 705         case RTE_INTR_HANDLE_VFIO_LEGACY:
 706                 if (vfio_enable_intx(intr_handle))
 707                         rc = -1;
 708                 break;
 709 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 710         case RTE_INTR_HANDLE_VFIO_REQ:
 711                 if (vfio_enable_req(intr_handle))
 712                         rc = -1;
 713                 break;
 714 #endif
 715 #endif
 716         /* not used at this moment */
 717         case RTE_INTR_HANDLE_DEV_EVENT:
 718                 rc = -1;
 719                 break;
 720         /* unknown handle type */
 721         default:
 722                 RTE_LOG(ERR, EAL,
 723                         "Unknown handle type of fd %d\n",
 724                                         intr_handle->fd);
 725                 rc = -1;
 726                 break;
 727         }
 728 out:
 729         rte_eal_trace_intr_enable(intr_handle, rc);
 730         return rc;
 731 }
 732
 733 /**
 734  * PMD generally calls this function at the end of its IRQ callback.
 735  * Internally, it unmasks the interrupt if possible.
 736  *
 737  * For INTx, unmasking is required as the interrupt is auto-masked prior to
 738  * invoking callback.
 739  *
 740  * For MSI/MSI-X, unmasking is typically not needed as the interrupt is not
 741  * auto-masked. In fact, for interrupt handle types VFIO_MSIX and VFIO_MSI,
 742  * this function is no-op.
 743  */
 744 int
 745 rte_intr_ack(const struct rte_intr_handle *intr_handle)
 746 {
 747         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
 748                 return 0;
 749
 750         if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
 751                 return -1;
 752
 753         switch (intr_handle->type) {
 754         /* Both acking and enabling are same for UIO */
 755         case RTE_INTR_HANDLE_UIO:
 756                 if (uio_intr_enable(intr_handle))
 757                         return -1;
 758                 break;
 759         case RTE_INTR_HANDLE_UIO_INTX:
 760                 if (uio_intx_intr_enable(intr_handle))
 761                         return -1;
 762                 break;
 763         /* not used at this moment */
 764         case RTE_INTR_HANDLE_ALARM:
 765                 return -1;
 766 #ifdef VFIO_PRESENT
 767         /* VFIO MSI* is implicitly acked unlike INTx, nothing to do */
 768         case RTE_INTR_HANDLE_VFIO_MSIX:
 769         case RTE_INTR_HANDLE_VFIO_MSI:
 770                 return 0;
 771         case RTE_INTR_HANDLE_VFIO_LEGACY:
 772                 if (vfio_ack_intx(intr_handle))
 773                         return -1;
 774                 break;
 775 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 776         case RTE_INTR_HANDLE_VFIO_REQ:
 777                 return -1;
 778 #endif
 779 #endif
 780         /* not used at this moment */
 781         case RTE_INTR_HANDLE_DEV_EVENT:
 782                 return -1;
 783         /* unknown handle type */
 784         default:
 785                 RTE_LOG(ERR, EAL, "Unknown handle type of fd %d\n",
 786                         intr_handle->fd);
 787                 return -1;
 788         }
 789
 790         return 0;
 791 }
 792
 793 int
 794 rte_intr_disable(const struct rte_intr_handle *intr_handle)
 795 {
 796         int rc = 0;
 797
 798         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV) {
 799                 rc = 0;
 800                 goto out;
 801         }
 802
 803         if (!intr_handle || intr_handle->fd < 0 ||
 804                                         intr_handle->uio_cfg_fd < 0) {
 805                 rc = -1;
 806                 goto out;
 807         }
 808
 809         switch (intr_handle->type){
 810         /* write to the uio fd to disable the interrupt */
 811         case RTE_INTR_HANDLE_UIO:
 812                 if (uio_intr_disable(intr_handle))
 813                         rc = -1;
 814                 break;
 815         case RTE_INTR_HANDLE_UIO_INTX:
 816                 if (uio_intx_intr_disable(intr_handle))
 817                         rc = -1;
 818                 break;
 819         /* not used at this moment */
 820         case RTE_INTR_HANDLE_ALARM:
 821                 rc = -1;
 822                 break;
 823 #ifdef VFIO_PRESENT
 824         case RTE_INTR_HANDLE_VFIO_MSIX:
 825                 if (vfio_disable_msix(intr_handle))
 826                         rc = -1;
 827                 break;
 828         case RTE_INTR_HANDLE_VFIO_MSI:
 829                 if (vfio_disable_msi(intr_handle))
 830                         rc = -1;
 831                 break;
 832         case RTE_INTR_HANDLE_VFIO_LEGACY:
 833                 if (vfio_disable_intx(intr_handle))
 834                         rc = -1;
 835                 break;
 836 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 837         case RTE_INTR_HANDLE_VFIO_REQ:
 838                 if (vfio_disable_req(intr_handle))
 839                         rc = -1;
 840                 break;
 841 #endif
 842 #endif
 843         /* not used at this moment */
 844         case RTE_INTR_HANDLE_DEV_EVENT:
 845                 rc = -1;
 846                 break;
 847         /* unknown handle type */
 848         default:
 849                 RTE_LOG(ERR, EAL,
 850                         "Unknown handle type of fd %d\n",
 851                                         intr_handle->fd);
 852                 rc = -1;
 853                 break;
 854         }
 855 out:
 856         rte_eal_trace_intr_disable(intr_handle, rc);
 857         return rc;
 858 }
 859
 860 static int
 861 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 862 {
 863         bool call = false;
 864         int n, bytes_read, rv;
 865         struct rte_intr_source *src;
 866         struct rte_intr_callback *cb, *next;
 867         union rte_intr_read_buffer buf;
 868         struct rte_intr_callback active_cb;
 869
 870         for (n = 0; n < nfds; n++) {
 871
 872                 /**
 873                  * if the pipe fd is ready to read, return out to
 874                  * rebuild the wait list.
 875                  */
 876                 if (events[n].data.fd == intr_pipe.readfd){
 877                         int r = read(intr_pipe.readfd, buf.charbuf,
 878                                         sizeof(buf.charbuf));
 879                         RTE_SET_USED(r);
 880                         return -1;
 881                 }
 882                 rte_spinlock_lock(&intr_lock);
 883                 TAILQ_FOREACH(src, &intr_sources, next)
 884                         if (src->intr_handle.fd ==
 885                                         events[n].data.fd)
 886                                 break;
 887                 if (src == NULL){
 888                         rte_spinlock_unlock(&intr_lock);
 889                         continue;
 890                 }
 891
 892                 /* mark this interrupt source as active and release the lock. */
 893                 src->active = 1;
 894                 rte_spinlock_unlock(&intr_lock);
 895
 896                 /* set the length to be read dor different handle type */
 897                 switch (src->intr_handle.type) {
 898                 case RTE_INTR_HANDLE_UIO:
 899                 case RTE_INTR_HANDLE_UIO_INTX:
 900                         bytes_read = sizeof(buf.uio_intr_count);
 901                         break;
 902                 case RTE_INTR_HANDLE_ALARM:
 903                         bytes_read = sizeof(buf.timerfd_num);
 904                         break;
 905 #ifdef VFIO_PRESENT
 906                 case RTE_INTR_HANDLE_VFIO_MSIX:
 907                 case RTE_INTR_HANDLE_VFIO_MSI:
 908                 case RTE_INTR_HANDLE_VFIO_LEGACY:
 909                         bytes_read = sizeof(buf.vfio_intr_count);
 910                         break;
 911 #ifdef HAVE_VFIO_DEV_REQ_INTERFACE
 912                 case RTE_INTR_HANDLE_VFIO_REQ:
 913                         bytes_read = 0;
 914                         call = true;
 915                         break;
 916 #endif
 917 #endif
 918                 case RTE_INTR_HANDLE_VDEV:
 919                 case RTE_INTR_HANDLE_EXT:
 920                         bytes_read = 0;
 921                         call = true;
 922                         break;
 923                 case RTE_INTR_HANDLE_DEV_EVENT:
 924                         bytes_read = 0;
 925                         call = true;
 926                         break;
 927                 default:
 928                         bytes_read = 1;
 929                         break;
 930                 }
 931
 932                 if (bytes_read > 0) {
 933                         /**
 934                          * read out to clear the ready-to-be-read flag
 935                          * for epoll_wait.
 936                          */
 937                         bytes_read = read(events[n].data.fd, &buf, bytes_read);
 938                         if (bytes_read < 0) {
 939                                 if (errno == EINTR || errno == EWOULDBLOCK)
 940                                         continue;
 941
 942                                 RTE_LOG(ERR, EAL, "Error reading from file "
 943                                         "descriptor %d: %s\n",
 944                                         events[n].data.fd,
 945                                         strerror(errno));
 946                                 /*
 947                                  * The device is unplugged or buggy, remove
 948                                  * it as an interrupt source and return to
 949                                  * force the wait list to be rebuilt.
 950                                  */
 951                                 rte_spinlock_lock(&intr_lock);
 952                                 TAILQ_REMOVE(&intr_sources, src, next);
 953                                 rte_spinlock_unlock(&intr_lock);
 954
 955                                 for (cb = TAILQ_FIRST(&src->callbacks); cb;
 956                                                         cb = next) {
 957                                         next = TAILQ_NEXT(cb, next);
 958                                         TAILQ_REMOVE(&src->callbacks, cb, next);
 959                                         free(cb);
 960                                 }
 961                                 free(src);
 962                                 return -1;
 963                         } else if (bytes_read == 0)
 964                                 RTE_LOG(ERR, EAL, "Read nothing from file "
 965                                         "descriptor %d\n", events[n].data.fd);
 966                         else
 967                                 call = true;
 968                 }
 969
 970                 /* grab a lock, again to call callbacks and update status. */
 971                 rte_spinlock_lock(&intr_lock);
 972
 973                 if (call) {
 974
 975                         /* Finally, call all callbacks. */
 976                         TAILQ_FOREACH(cb, &src->callbacks, next) {
 977
 978                                 /* make a copy and unlock. */
 979                                 active_cb = *cb;
 980                                 rte_spinlock_unlock(&intr_lock);
 981
 982                                 /* call the actual callback */
 983                                 active_cb.cb_fn(active_cb.cb_arg);
 984
 985                                 /*get the lock back. */
 986                                 rte_spinlock_lock(&intr_lock);
 987                         }
 988                 }
 989                 /* we done with that interrupt source, release it. */
 990                 src->active = 0;
 991
 992                 rv = 0;
 993
 994                 /* check if any callback are supposed to be removed */
 995                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 996                         next = TAILQ_NEXT(cb, next);
 997                         if (cb->pending_delete) {
 998                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 999                                 if (cb->ucb_fn)
1000                                         cb->ucb_fn(&src->intr_handle, cb->cb_arg);
1001                                 free(cb);
1002                                 rv++;
1003                         }
1004                 }
1005
1006                 /* all callbacks for that source are removed. */
1007                 if (TAILQ_EMPTY(&src->callbacks)) {
1008                         TAILQ_REMOVE(&intr_sources, src, next);
1009                         free(src);
1010                 }
1011
1012                 /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
1013                 if (rv > 0 && write(intr_pipe.writefd, "1", 1) < 0) {
1014                         rte_spinlock_unlock(&intr_lock);
1015                         return -EPIPE;
1016                 }
1017
1018                 rte_spinlock_unlock(&intr_lock);
1019         }
1020
1021         return 0;
1022 }
1023
1024 /**
1025  * It handles all the interrupts.
1026  *
1027  * @param pfd
1028  *  epoll file descriptor.
1029  * @param totalfds
1030  *  The number of file descriptors added in epoll.
1031  *
1032  * @return
1033  *  void
1034  */
1035 static void
1036 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
1037 {
1038         struct epoll_event events[totalfds];
1039         int nfds = 0;
1040
1041         for(;;) {
1042                 nfds = epoll_wait(pfd, events, totalfds,
1043                         EAL_INTR_EPOLL_WAIT_FOREVER);
1044                 /* epoll_wait fail */
1045                 if (nfds < 0) {
1046                         if (errno == EINTR)
1047                                 continue;
1048                         RTE_LOG(ERR, EAL,
1049                                 "epoll_wait returns with fail\n");
1050                         return;
1051                 }
1052                 /* epoll_wait timeout, will never happens here */
1053                 else if (nfds == 0)
1054                         continue;
1055                 /* epoll_wait has at least one fd ready to read */
1056                 if (eal_intr_process_interrupts(events, nfds) < 0)
1057                         return;
1058         }
1059 }
1060
1061 /**
1062  * It builds/rebuilds up the epoll file descriptor with all the
1063  * file descriptors being waited on. Then handles the interrupts.
1064  *
1065  * @param arg
1066  *  pointer. (unused)
1067  *
1068  * @return
1069  *  never return;
1070  */
1071 static __rte_noreturn void *
1072 eal_intr_thread_main(__rte_unused void *arg)
1073 {
1074         /* host thread, never break out */
1075         for (;;) {
1076                 /* build up the epoll fd with all descriptors we are to
1077                  * wait on then pass it to the handle_interrupts function
1078                  */
1079                 static struct epoll_event pipe_event = {
1080                         .events = EPOLLIN | EPOLLPRI,
1081                 };
1082                 struct rte_intr_source *src;
1083                 unsigned numfds = 0;
1084
1085                 /* create epoll fd */
1086                 int pfd = epoll_create(1);
1087                 if (pfd < 0)
1088                         rte_panic("Cannot create epoll instance\n");
1089
1090                 pipe_event.data.fd = intr_pipe.readfd;
1091                 /**
1092                  * add pipe fd into wait list, this pipe is used to
1093                  * rebuild the wait list.
1094                  */
1095                 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
1096                                                 &pipe_event) < 0) {
1097                         rte_panic("Error adding fd to %d epoll_ctl, %s\n",
1098                                         intr_pipe.readfd, strerror(errno));
1099                 }
1100                 numfds++;
1101
1102                 rte_spinlock_lock(&intr_lock);
1103
1104                 TAILQ_FOREACH(src, &intr_sources, next) {
1105                         struct epoll_event ev;
1106
1107                         if (src->callbacks.tqh_first == NULL)
1108                                 continue; /* skip those with no callbacks */
1109                         memset(&ev, 0, sizeof(ev));
1110                         ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
1111                         ev.data.fd = src->intr_handle.fd;
1112
1113                         /**
1114                          * add all the uio device file descriptor
1115                          * into wait list.
1116                          */
1117                         if (epoll_ctl(pfd, EPOLL_CTL_ADD,
1118                                         src->intr_handle.fd, &ev) < 0){
1119                                 rte_panic("Error adding fd %d epoll_ctl, %s\n",
1120                                         src->intr_handle.fd, strerror(errno));
1121                         }
1122                         else
1123                                 numfds++;
1124                 }
1125                 rte_spinlock_unlock(&intr_lock);
1126                 /* serve the interrupt */
1127                 eal_intr_handle_interrupts(pfd, numfds);
1128
1129                 /**
1130                  * when we return, we need to rebuild the
1131                  * list of fds to monitor.
1132                  */
1133                 close(pfd);
1134         }
1135 }
1136
1137 int
1138 rte_eal_intr_init(void)
1139 {
1140         int ret = 0;
1141
1142         /* init the global interrupt source head */
1143         TAILQ_INIT(&intr_sources);
1144
1145         /**
1146          * create a pipe which will be waited by epoll and notified to
1147          * rebuild the wait list of epoll.
1148          */
1149         if (pipe(intr_pipe.pipefd) < 0) {
1150                 rte_errno = errno;
1151                 return -1;
1152         }
1153
1154         /* create the host thread to wait/handle the interrupt */
1155         ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
1156                         eal_intr_thread_main, NULL);
1157         if (ret != 0) {
1158                 rte_errno = -ret;
1159                 RTE_LOG(ERR, EAL,
1160                         "Failed to create thread for interrupt handling\n");
1161         }
1162
1163         return ret;
1164 }
1165
1166 static void
1167 eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
1168 {
1169         union rte_intr_read_buffer buf;
1170         int bytes_read = 0;
1171         int nbytes;
1172
1173         switch (intr_handle->type) {
1174         case RTE_INTR_HANDLE_UIO:
1175         case RTE_INTR_HANDLE_UIO_INTX:
1176                 bytes_read = sizeof(buf.uio_intr_count);
1177                 break;
1178 #ifdef VFIO_PRESENT
1179         case RTE_INTR_HANDLE_VFIO_MSIX:
1180         case RTE_INTR_HANDLE_VFIO_MSI:
1181         case RTE_INTR_HANDLE_VFIO_LEGACY:
1182                 bytes_read = sizeof(buf.vfio_intr_count);
1183                 break;
1184 #endif
1185         case RTE_INTR_HANDLE_VDEV:
1186                 bytes_read = intr_handle->efd_counter_size;
1187                 /* For vdev, number of bytes to read is set by driver */
1188                 break;
1189         case RTE_INTR_HANDLE_EXT:
1190                 return;
1191         default:
1192                 bytes_read = 1;
1193                 RTE_LOG(INFO, EAL, "unexpected intr type\n");
1194                 break;
1195         }
1196
1197         /**
1198          * read out to clear the ready-to-be-read flag
1199          * for epoll_wait.
1200          */
1201         if (bytes_read == 0)
1202                 return;
1203         do {
1204                 nbytes = read(fd, &buf, bytes_read);
1205                 if (nbytes < 0) {
1206                         if (errno == EINTR || errno == EWOULDBLOCK ||
1207                             errno == EAGAIN)
1208                                 continue;
1209                         RTE_LOG(ERR, EAL,
1210                                 "Error reading from fd %d: %s\n",
1211                                 fd, strerror(errno));
1212                 } else if (nbytes == 0)
1213                         RTE_LOG(ERR, EAL, "Read nothing from fd %d\n", fd);
1214                 return;
1215         } while (1);
1216 }
1217
1218 static int
1219 eal_epoll_process_event(struct epoll_event *evs, unsigned int n,
1220                         struct rte_epoll_event *events)
1221 {
1222         unsigned int i, count = 0;
1223         struct rte_epoll_event *rev;
1224
1225         for (i = 0; i < n; i++) {
1226                 rev = evs[i].data.ptr;
1227                 if (!rev || !rte_atomic32_cmpset(&rev->status, RTE_EPOLL_VALID,
1228                                                  RTE_EPOLL_EXEC))
1229                         continue;
1230
1231                 events[count].status        = RTE_EPOLL_VALID;
1232                 events[count].fd            = rev->fd;
1233                 events[count].epfd          = rev->epfd;
1234                 events[count].epdata.event  = rev->epdata.event;
1235                 events[count].epdata.data   = rev->epdata.data;
1236                 if (rev->epdata.cb_fun)
1237                         rev->epdata.cb_fun(rev->fd,
1238                                            rev->epdata.cb_arg);
1239
1240                 rte_compiler_barrier();
1241                 rev->status = RTE_EPOLL_VALID;
1242                 count++;
1243         }
1244         return count;
1245 }
1246
1247 static inline int
1248 eal_init_tls_epfd(void)
1249 {
1250         int pfd = epoll_create(255);
1251
1252         if (pfd < 0) {
1253                 RTE_LOG(ERR, EAL,
1254                         "Cannot create epoll instance\n");
1255                 return -1;
1256         }
1257         return pfd;
1258 }
1259
1260 int
1261 rte_intr_tls_epfd(void)
1262 {
1263         if (RTE_PER_LCORE(_epfd) == -1)
1264                 RTE_PER_LCORE(_epfd) = eal_init_tls_epfd();
1265
1266         return RTE_PER_LCORE(_epfd);
1267 }
1268
1269 int
1270 rte_epoll_wait(int epfd, struct rte_epoll_event *events,
1271                int maxevents, int timeout)
1272 {
1273         struct epoll_event evs[maxevents];
1274         int rc;
1275
1276         if (!events) {
1277                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
1278                 return -1;
1279         }
1280
1281         /* using per thread epoll fd */
1282         if (epfd == RTE_EPOLL_PER_THREAD)
1283                 epfd = rte_intr_tls_epfd();
1284
1285         while (1) {
1286                 rc = epoll_wait(epfd, evs, maxevents, timeout);
1287                 if (likely(rc > 0)) {
1288                         /* epoll_wait has at least one fd ready to read */
1289                         rc = eal_epoll_process_event(evs, rc, events);
1290                         break;
1291                 } else if (rc < 0) {
1292                         if (errno == EINTR)
1293                                 continue;
1294                         /* epoll_wait fail */
1295                         RTE_LOG(ERR, EAL, "epoll_wait returns with fail %s\n",
1296                                 strerror(errno));
1297                         rc = -1;
1298                         break;
1299                 } else {
1300                         /* rc == 0, epoll_wait timed out */
1301                         break;
1302                 }
1303         }
1304
1305         return rc;
1306 }
1307
1308 static inline void
1309 eal_epoll_data_safe_free(struct rte_epoll_event *ev)
1310 {
1311         while (!rte_atomic32_cmpset(&ev->status, RTE_EPOLL_VALID,
1312                                     RTE_EPOLL_INVALID))
1313                 while (ev->status != RTE_EPOLL_VALID)
1314                         rte_pause();
1315         memset(&ev->epdata, 0, sizeof(ev->epdata));
1316         ev->fd = -1;
1317         ev->epfd = -1;
1318 }
1319
1320 int
1321 rte_epoll_ctl(int epfd, int op, int fd,
1322               struct rte_epoll_event *event)
1323 {
1324         struct epoll_event ev;
1325
1326         if (!event) {
1327                 RTE_LOG(ERR, EAL, "rte_epoll_event can't be NULL\n");
1328                 return -1;
1329         }
1330
1331         /* using per thread epoll fd */
1332         if (epfd == RTE_EPOLL_PER_THREAD)
1333                 epfd = rte_intr_tls_epfd();
1334
1335         if (op == EPOLL_CTL_ADD) {
1336                 event->status = RTE_EPOLL_VALID;
1337                 event->fd = fd;  /* ignore fd in event */
1338                 event->epfd = epfd;
1339                 ev.data.ptr = (void *)event;
1340         }
1341
1342         ev.events = event->epdata.event;
1343         if (epoll_ctl(epfd, op, fd, &ev) < 0) {
1344                 RTE_LOG(ERR, EAL, "Error op %d fd %d epoll_ctl, %s\n",
1345                         op, fd, strerror(errno));
1346                 if (op == EPOLL_CTL_ADD)
1347                         /* rollback status when CTL_ADD fail */
1348                         event->status = RTE_EPOLL_INVALID;
1349                 return -1;
1350         }
1351
1352         if (op == EPOLL_CTL_DEL && event->status != RTE_EPOLL_INVALID)
1353                 eal_epoll_data_safe_free(event);
1354
1355         return 0;
1356 }
1357
1358 int
1359 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
1360                 int op, unsigned int vec, void *data)
1361 {
1362         struct rte_epoll_event *rev;
1363         struct rte_epoll_data *epdata;
1364         int epfd_op;
1365         unsigned int efd_idx;
1366         int rc = 0;
1367
1368         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
1369                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
1370
1371         if (!intr_handle || intr_handle->nb_efd == 0 ||
1372             efd_idx >= intr_handle->nb_efd) {
1373                 RTE_LOG(ERR, EAL, "Wrong intr vector number.\n");
1374                 return -EPERM;
1375         }
1376
1377         switch (op) {
1378         case RTE_INTR_EVENT_ADD:
1379                 epfd_op = EPOLL_CTL_ADD;
1380                 rev = &intr_handle->elist[efd_idx];
1381                 if (rev->status != RTE_EPOLL_INVALID) {
1382                         RTE_LOG(INFO, EAL, "Event already been added.\n");
1383                         return -EEXIST;
1384                 }
1385
1386                 /* attach to intr vector fd */
1387                 epdata = &rev->epdata;
1388                 epdata->event  = EPOLLIN | EPOLLPRI | EPOLLET;
1389                 epdata->data   = data;
1390                 epdata->cb_fun = (rte_intr_event_cb_t)eal_intr_proc_rxtx_intr;
1391                 epdata->cb_arg = (void *)intr_handle;
1392                 rc = rte_epoll_ctl(epfd, epfd_op,
1393                                    intr_handle->efds[efd_idx], rev);
1394                 if (!rc)
1395                         RTE_LOG(DEBUG, EAL,
1396                                 "efd %d associated with vec %d added on epfd %d"
1397                                 "\n", rev->fd, vec, epfd);
1398                 else
1399                         rc = -EPERM;
1400                 break;
1401         case RTE_INTR_EVENT_DEL:
1402                 epfd_op = EPOLL_CTL_DEL;
1403                 rev = &intr_handle->elist[efd_idx];
1404                 if (rev->status == RTE_EPOLL_INVALID) {
1405                         RTE_LOG(INFO, EAL, "Event does not exist.\n");
1406                         return -EPERM;
1407                 }
1408
1409                 rc = rte_epoll_ctl(rev->epfd, epfd_op, rev->fd, rev);
1410                 if (rc)
1411                         rc = -EPERM;
1412                 break;
1413         default:
1414                 RTE_LOG(ERR, EAL, "event op type mismatch\n");
1415                 rc = -EPERM;
1416         }
1417
1418         return rc;
1419 }
1420
1421 void
1422 rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
1423 {
1424         uint32_t i;
1425         struct rte_epoll_event *rev;
1426
1427         for (i = 0; i < intr_handle->nb_efd; i++) {
1428                 rev = &intr_handle->elist[i];
1429                 if (rev->status == RTE_EPOLL_INVALID)
1430                         continue;
1431                 if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
1432                         /* force free if the entry valid */
1433                         eal_epoll_data_safe_free(rev);
1434                         rev->status = RTE_EPOLL_INVALID;
1435                 }
1436         }
1437 }
1438
1439 int
1440 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
1441 {
1442         uint32_t i;
1443         int fd;
1444         uint32_t n = RTE_MIN(nb_efd, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
1445
1446         assert(nb_efd != 0);
1447
1448         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) {
1449                 for (i = 0; i < n; i++) {
1450                         fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
1451                         if (fd < 0) {
1452                                 RTE_LOG(ERR, EAL,
1453                                         "can't setup eventfd, error %i (%s)\n",
1454                                         errno, strerror(errno));
1455                                 return -errno;
1456                         }
1457                         intr_handle->efds[i] = fd;
1458                 }
1459                 intr_handle->nb_efd   = n;
1460                 intr_handle->max_intr = NB_OTHER_INTR + n;
1461         } else if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
1462                 /* only check, initialization would be done in vdev driver.*/
1463                 if (intr_handle->efd_counter_size >
1464                     sizeof(union rte_intr_read_buffer)) {
1465                         RTE_LOG(ERR, EAL, "the efd_counter_size is oversized");
1466                         return -EINVAL;
1467                 }
1468         } else {
1469                 intr_handle->efds[0]  = intr_handle->fd;
1470                 intr_handle->nb_efd   = RTE_MIN(nb_efd, 1U);
1471                 intr_handle->max_intr = NB_OTHER_INTR;
1472         }
1473
1474         return 0;
1475 }
1476
1477 void
1478 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
1479 {
1480         uint32_t i;
1481
1482         rte_intr_free_epoll_fd(intr_handle);
1483         if (intr_handle->max_intr > intr_handle->nb_efd) {
1484                 for (i = 0; i < intr_handle->nb_efd; i++)
1485                         close(intr_handle->efds[i]);
1486         }
1487         intr_handle->nb_efd = 0;
1488         intr_handle->max_intr = 0;
1489 }
1490
1491 int
1492 rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
1493 {
1494         return !(!intr_handle->nb_efd);
1495 }
1496
1497 int
1498 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
1499 {
1500         if (!rte_intr_dp_is_en(intr_handle))
1501                 return 1;
1502         else
1503                 return !!(intr_handle->max_intr - intr_handle->nb_efd);
1504 }
1505
1506 int
1507 rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
1508 {
1509         if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX)
1510                 return 1;
1511
1512         if (intr_handle->type == RTE_INTR_HANDLE_VDEV)
1513                 return 1;
1514
1515         return 0;
1516 }
1517
1518 int rte_thread_is_intr(void)
1519 {
1520         return pthread_equal(intr_thread, pthread_self());
1521 }