vfio: fix build with Linux < 4.0
[dpdk.git] / lib / librte_eal / linuxapp / eal / eal_interrupts.c
index 3f87875..767b508 100644 (file)
@@ -1,34 +1,5 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2010-2014 Intel Corporation
  */
 
 #include <stdio.h>
 #include <sys/signalfd.h>
 #include <sys/ioctl.h>
 #include <sys/eventfd.h>
+#include <assert.h>
+#include <stdbool.h>
 
 #include <rte_common.h>
 #include <rte_interrupts.h>
 #include <rte_memory.h>
-#include <rte_memzone.h>
 #include <rte_launch.h>
 #include <rte_eal.h>
 #include <rte_per_lcore.h>
 #include <rte_lcore.h>
 #include <rte_atomic.h>
 #include <rte_branch_prediction.h>
-#include <rte_ring.h>
 #include <rte_debug.h>
 #include <rte_log.h>
-#include <rte_mempool.h>
-#include <rte_pci.h>
-#include <rte_malloc.h>
 #include <rte_errno.h>
 #include <rte_spinlock.h>
+#include <rte_pause.h>
 
 #include "eal_private.h"
 #include "eal_vfio.h"
+#include "eal_thread.h"
 
 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
 #define NB_OTHER_INTR               1
@@ -136,7 +106,7 @@ static pthread_t intr_thread;
 
 /* enable legacy (INTx) interrupts */
 static int
-vfio_enable_intx(struct rte_intr_handle *intr_handle) {
+vfio_enable_intx(const struct rte_intr_handle *intr_handle) {
        struct vfio_irq_set *irq_set;
        char irq_set_buf[IRQ_SET_BUF_LEN];
        int len, ret;
@@ -183,7 +153,7 @@ vfio_enable_intx(struct rte_intr_handle *intr_handle) {
 
 /* disable legacy (INTx) interrupts */
 static int
-vfio_disable_intx(struct rte_intr_handle *intr_handle) {
+vfio_disable_intx(const struct rte_intr_handle *intr_handle) {
        struct vfio_irq_set *irq_set;
        char irq_set_buf[IRQ_SET_BUF_LEN];
        int len, ret;
@@ -194,14 +164,14 @@ vfio_disable_intx(struct rte_intr_handle *intr_handle) {
        irq_set = (struct vfio_irq_set *) irq_set_buf;
        irq_set->argsz = len;
        irq_set->count = 1;
-       irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK;
+       irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK;
        irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
        irq_set->start = 0;
 
        ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 
        if (ret) {
-               RTE_LOG(ERR, EAL, "Error unmasking INTx interrupts for fd %d\n",
+               RTE_LOG(ERR, EAL, "Error masking INTx interrupts for fd %d\n",
                                                intr_handle->fd);
                return -1;
        }
@@ -226,7 +196,7 @@ vfio_disable_intx(struct rte_intr_handle *intr_handle) {
 
 /* enable MSI interrupts */
 static int
-vfio_enable_msi(struct rte_intr_handle *intr_handle) {
+vfio_enable_msi(const struct rte_intr_handle *intr_handle) {
        int len, ret;
        char irq_set_buf[IRQ_SET_BUF_LEN];
        struct vfio_irq_set *irq_set;
@@ -255,7 +225,7 @@ vfio_enable_msi(struct rte_intr_handle *intr_handle) {
 
 /* disable MSI interrupts */
 static int
-vfio_disable_msi(struct rte_intr_handle *intr_handle) {
+vfio_disable_msi(const struct rte_intr_handle *intr_handle) {
        struct vfio_irq_set *irq_set;
        char irq_set_buf[IRQ_SET_BUF_LEN];
        int len, ret;
@@ -280,7 +250,7 @@ vfio_disable_msi(struct rte_intr_handle *intr_handle) {
 
 /* enable MSI-X interrupts */
 static int
-vfio_enable_msix(struct rte_intr_handle *intr_handle) {
+vfio_enable_msix(const struct rte_intr_handle *intr_handle) {
        int len, ret;
        char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
        struct vfio_irq_set *irq_set;
@@ -290,26 +260,18 @@ vfio_enable_msix(struct rte_intr_handle *intr_handle) {
 
        irq_set = (struct vfio_irq_set *) irq_set_buf;
        irq_set->argsz = len;
-#ifdef RTE_NEXT_ABI
-       if (!intr_handle->max_intr)
-               intr_handle->max_intr = 1;
-       else if (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID)
-               intr_handle->max_intr = RTE_MAX_RXTX_INTR_VEC_ID + 1;
-
-       irq_set->count = intr_handle->max_intr;
-#else
-       irq_set->count = 1;
-#endif
+       /* 0 < irq_set->count < RTE_MAX_RXTX_INTR_VEC_ID + 1 */
+       irq_set->count = intr_handle->max_intr ?
+               (intr_handle->max_intr > RTE_MAX_RXTX_INTR_VEC_ID + 1 ?
+               RTE_MAX_RXTX_INTR_VEC_ID + 1 : intr_handle->max_intr) : 1;
        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
        irq_set->start = 0;
        fd_ptr = (int *) &irq_set->data;
-#ifdef RTE_NEXT_ABI
-       memcpy(fd_ptr, intr_handle->efds, sizeof(intr_handle->efds));
-       fd_ptr[intr_handle->max_intr - 1] = intr_handle->fd;
-#else
-       fd_ptr[0] = intr_handle->fd;
-#endif
+       /* INTR vector offset 0 reserve for non-efds mapping */
+       fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = intr_handle->fd;
+       memcpy(&fd_ptr[RTE_INTR_VEC_RXTX_OFFSET], intr_handle->efds,
+               sizeof(*intr_handle->efds) * intr_handle->nb_efd);
 
        ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
 
@@ -324,7 +286,7 @@ vfio_enable_msix(struct rte_intr_handle *intr_handle) {
 
 /* disable MSI-X interrupts */
 static int
-vfio_disable_msix(struct rte_intr_handle *intr_handle) {
+vfio_disable_msix(const struct rte_intr_handle *intr_handle) {
        struct vfio_irq_set *irq_set;
        char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];
        int len, ret;
@@ -346,10 +308,70 @@ vfio_disable_msix(struct rte_intr_handle *intr_handle) {
 
        return ret;
 }
+
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+/* enable req notifier */
+static int
+vfio_enable_req(const struct rte_intr_handle *intr_handle)
+{
+       int len, ret;
+       char irq_set_buf[IRQ_SET_BUF_LEN];
+       struct vfio_irq_set *irq_set;
+       int *fd_ptr;
+
+       len = sizeof(irq_set_buf);
+
+       irq_set = (struct vfio_irq_set *) irq_set_buf;
+       irq_set->argsz = len;
+       irq_set->count = 1;
+       irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                        VFIO_IRQ_SET_ACTION_TRIGGER;
+       irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
+       irq_set->start = 0;
+       fd_ptr = (int *) &irq_set->data;
+       *fd_ptr = intr_handle->fd;
+
+       ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+       if (ret) {
+               RTE_LOG(ERR, EAL, "Error enabling req interrupts for fd %d\n",
+                                               intr_handle->fd);
+               return -1;
+       }
+
+       return 0;
+}
+
+/* disable req notifier */
+static int
+vfio_disable_req(const struct rte_intr_handle *intr_handle)
+{
+       struct vfio_irq_set *irq_set;
+       char irq_set_buf[IRQ_SET_BUF_LEN];
+       int len, ret;
+
+       len = sizeof(struct vfio_irq_set);
+
+       irq_set = (struct vfio_irq_set *) irq_set_buf;
+       irq_set->argsz = len;
+       irq_set->count = 0;
+       irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
+       irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
+       irq_set->start = 0;
+
+       ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+       if (ret)
+               RTE_LOG(ERR, EAL, "Error disabling req interrupts for fd %d\n",
+                       intr_handle->fd);
+
+       return ret;
+}
+#endif
 #endif
 
 static int
-uio_intx_intr_disable(struct rte_intr_handle *intr_handle)
+uio_intx_intr_disable(const struct rte_intr_handle *intr_handle)
 {
        unsigned char command_high;
 
@@ -373,7 +395,7 @@ uio_intx_intr_disable(struct rte_intr_handle *intr_handle)
 }
 
 static int
-uio_intx_intr_enable(struct rte_intr_handle *intr_handle)
+uio_intx_intr_enable(const struct rte_intr_handle *intr_handle)
 {
        unsigned char command_high;
 
@@ -397,7 +419,7 @@ uio_intx_intr_enable(struct rte_intr_handle *intr_handle)
 }
 
 static int
-uio_intr_disable(struct rte_intr_handle *intr_handle)
+uio_intr_disable(const struct rte_intr_handle *intr_handle)
 {
        const int value = 0;
 
@@ -411,7 +433,7 @@ uio_intr_disable(struct rte_intr_handle *intr_handle)
 }
 
 static int
-uio_intr_enable(struct rte_intr_handle *intr_handle)
+uio_intr_enable(const struct rte_intr_handle *intr_handle)
 {
        const int value = 1;
 
@@ -425,7 +447,7 @@ uio_intr_enable(struct rte_intr_handle *intr_handle)
 }
 
 int
-rte_intr_callback_register(struct rte_intr_handle *intr_handle,
+rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
                        rte_intr_callback_fn cb, void *cb_arg)
 {
        int ret, wake_thread;
@@ -442,8 +464,7 @@ rte_intr_callback_register(struct rte_intr_handle *intr_handle,
        }
 
        /* allocate a new interrupt callback entity */
-       callback = rte_zmalloc("interrupt callback list",
-                               sizeof(*callback), 0);
+       callback = calloc(1, sizeof(*callback));
        if (callback == NULL) {
                RTE_LOG(ERR, EAL, "Can not allocate memory\n");
                return -ENOMEM;
@@ -457,7 +478,7 @@ rte_intr_callback_register(struct rte_intr_handle *intr_handle,
        TAILQ_FOREACH(src, &intr_sources, next) {
                if (src->intr_handle.fd == intr_handle->fd) {
                        /* we had no interrupts for this */
-                       if TAILQ_EMPTY(&src->callbacks)
+                       if (TAILQ_EMPTY(&src->callbacks))
                                wake_thread = 1;
 
                        TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
@@ -468,10 +489,10 @@ rte_intr_callback_register(struct rte_intr_handle *intr_handle,
 
        /* no existing callbacks for this - add new source */
        if (src == NULL) {
-               if ((src = rte_zmalloc("interrupt source list",
-                               sizeof(*src), 0)) == NULL) {
+               src = calloc(1, sizeof(*src));
+               if (src == NULL) {
                        RTE_LOG(ERR, EAL, "Can not allocate memory\n");
-                       rte_free(callback);
+                       free(callback);
                        ret = -ENOMEM;
                } else {
                        src->intr_handle = *intr_handle;
@@ -497,7 +518,7 @@ rte_intr_callback_register(struct rte_intr_handle *intr_handle,
 }
 
 int
-rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
+rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
                        rte_intr_callback_fn cb_fn, void *cb_arg)
 {
        int ret;
@@ -538,7 +559,7 @@ rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
                        if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
                                        cb->cb_arg == cb_arg)) {
                                TAILQ_REMOVE(&src->callbacks, cb, next);
-                               rte_free(cb);
+                               free(cb);
                                ret++;
                        }
                }
@@ -546,7 +567,7 @@ rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
                /* all callbacks for that source are removed. */
                if (TAILQ_EMPTY(&src->callbacks)) {
                        TAILQ_REMOVE(&intr_sources, src, next);
-                       rte_free(src);
+                       free(src);
                }
        }
 
@@ -561,8 +582,11 @@ rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
 }
 
 int
-rte_intr_enable(struct rte_intr_handle *intr_handle)
+rte_intr_enable(const struct rte_intr_handle *intr_handle)
 {
+       if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
+               return 0;
+
        if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
                return -1;
 
@@ -592,7 +616,16 @@ rte_intr_enable(struct rte_intr_handle *intr_handle)
                if (vfio_enable_intx(intr_handle))
                        return -1;
                break;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+       case RTE_INTR_HANDLE_VFIO_REQ:
+               if (vfio_enable_req(intr_handle))
+                       return -1;
+               break;
+#endif
 #endif
+       /* not used at this moment */
+       case RTE_INTR_HANDLE_DEV_EVENT:
+               return -1;
        /* unknown handle type */
        default:
                RTE_LOG(ERR, EAL,
@@ -605,8 +638,11 @@ rte_intr_enable(struct rte_intr_handle *intr_handle)
 }
 
 int
-rte_intr_disable(struct rte_intr_handle *intr_handle)
+rte_intr_disable(const struct rte_intr_handle *intr_handle)
 {
+       if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
+               return 0;
+
        if (!intr_handle || intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0)
                return -1;
 
@@ -636,7 +672,16 @@ rte_intr_disable(struct rte_intr_handle *intr_handle)
                if (vfio_disable_intx(intr_handle))
                        return -1;
                break;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+       case RTE_INTR_HANDLE_VFIO_REQ:
+               if (vfio_disable_req(intr_handle))
+                       return -1;
+               break;
 #endif
+#endif
+       /* not used at this moment */
+       case RTE_INTR_HANDLE_DEV_EVENT:
+               return -1;
        /* unknown handle type */
        default:
                RTE_LOG(ERR, EAL,
@@ -651,6 +696,7 @@ rte_intr_disable(struct rte_intr_handle *intr_handle)
 static int
 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 {
+       bool call = false;
        int n, bytes_read;
        struct rte_intr_source *src;
        struct rte_intr_callback *cb;
@@ -698,32 +744,52 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
                case RTE_INTR_HANDLE_VFIO_LEGACY:
                        bytes_read = sizeof(buf.vfio_intr_count);
                        break;
+#ifdef HAVE_VFIO_DEV_REQ_INTERFACE
+               case RTE_INTR_HANDLE_VFIO_REQ:
+                       bytes_read = 0;
+                       call = true;
+                       break;
 #endif
+#endif
+               case RTE_INTR_HANDLE_VDEV:
+               case RTE_INTR_HANDLE_EXT:
+                       bytes_read = 0;
+                       call = true;
+                       break;
+               case RTE_INTR_HANDLE_DEV_EVENT:
+                       bytes_read = 0;
+                       call = true;
+                       break;
                default:
                        bytes_read = 1;
                        break;
                }
 
-               /**
-                * read out to clear the ready-to-be-read flag
-                * for epoll_wait.
-                */
-               bytes_read = read(events[n].data.fd, &buf, bytes_read);
-               if (bytes_read < 0) {
-                       if (errno == EINTR || errno == EWOULDBLOCK)
-                               continue;
-
-                       RTE_LOG(ERR, EAL, "Error reading from file "
-                               "descriptor %d: %s\n", events[n].data.fd,
-                                                       strerror(errno));
-               } else if (bytes_read == 0)
-                       RTE_LOG(ERR, EAL, "Read nothing from file "
-                               "descriptor %d\n", events[n].data.fd);
+               if (bytes_read > 0) {
+                       /**
+                        * read out to clear the ready-to-be-read flag
+                        * for epoll_wait.
+                        */
+                       bytes_read = read(events[n].data.fd, &buf, bytes_read);
+                       if (bytes_read < 0) {
+                               if (errno == EINTR || errno == EWOULDBLOCK)
+                                       continue;
+
+                               RTE_LOG(ERR, EAL, "Error reading from file "
+                                       "descriptor %d: %s\n",
+                                       events[n].data.fd,
+                                       strerror(errno));
+                       } else if (bytes_read == 0)
+                               RTE_LOG(ERR, EAL, "Read nothing from file "
+                                       "descriptor %d\n", events[n].data.fd);
+                       else
+                               call = true;
+               }
 
                /* grab a lock, again to call callbacks and update status. */
                rte_spinlock_lock(&intr_lock);
 
-               if (bytes_read > 0) {
+               if (call) {
 
                        /* Finally, call all callbacks. */
                        TAILQ_FOREACH(cb, &src->callbacks, next) {
@@ -733,8 +799,7 @@ eal_intr_process_interrupts(struct epoll_event *events, int nfds)
                                rte_spinlock_unlock(&intr_lock);
 
                                /* call the actual callback */
-                               active_cb.cb_fn(&src->intr_handle,
-                                       active_cb.cb_arg);
+                               active_cb.cb_fn(active_cb.cb_arg);
 
                                /*get the lock back. */
                                rte_spinlock_lock(&intr_lock);
@@ -834,7 +899,7 @@ eal_intr_thread_main(__rte_unused void *arg)
                TAILQ_FOREACH(src, &intr_sources, next) {
                        if (src->callbacks.tqh_first == NULL)
                                continue; /* skip those with no callbacks */
-                       ev.events = EPOLLIN | EPOLLPRI;
+                       ev.events = EPOLLIN | EPOLLPRI | EPOLLRDHUP | EPOLLHUP;
                        ev.data.fd = src->intr_handle.fd;
 
                        /**
@@ -873,25 +938,29 @@ rte_eal_intr_init(void)
         * create a pipe which will be waited by epoll and notified to
         * rebuild the wait list of epoll.
         */
-       if (pipe(intr_pipe.pipefd) < 0)
+       if (pipe(intr_pipe.pipefd) < 0) {
+               rte_errno = errno;
                return -1;
+       }
 
        /* create the host thread to wait/handle the interrupt */
-       ret = pthread_create(&intr_thread, NULL,
+       ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
                        eal_intr_thread_main, NULL);
-       if (ret != 0)
+       if (ret != 0) {
+               rte_errno = -ret;
                RTE_LOG(ERR, EAL,
                        "Failed to create thread for interrupt handling\n");
+       }
 
-       return -ret;
+       return ret;
 }
 
-#ifdef RTE_NEXT_ABI
 static void
 eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
 {
        union rte_intr_read_buffer buf;
-       int bytes_read = 1;
+       int bytes_read = 0;
+       int nbytes;
 
        switch (intr_handle->type) {
        case RTE_INTR_HANDLE_UIO:
@@ -905,6 +974,12 @@ eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
                bytes_read = sizeof(buf.vfio_intr_count);
                break;
 #endif
+       case RTE_INTR_HANDLE_VDEV:
+               bytes_read = intr_handle->efd_counter_size;
+               /* For vdev, number of bytes to read is set by driver */
+               break;
+       case RTE_INTR_HANDLE_EXT:
+               return;
        default:
                bytes_read = 1;
                RTE_LOG(INFO, EAL, "unexpected intr type\n");
@@ -915,21 +990,22 @@ eal_intr_proc_rxtx_intr(int fd, const struct rte_intr_handle *intr_handle)
         * read out to clear the ready-to-be-read flag
         * for epoll_wait.
         */
+       if (bytes_read == 0)
+               return;
        do {
-               bytes_read = read(fd, &buf, bytes_read);
-               if (bytes_read < 0) {
+               nbytes = read(fd, &buf, bytes_read);
+               if (nbytes < 0) {
                        if (errno == EINTR || errno == EWOULDBLOCK ||
                            errno == EAGAIN)
                                continue;
                        RTE_LOG(ERR, EAL,
                                "Error reading from fd %d: %s\n",
                                fd, strerror(errno));
-               } else if (bytes_read == 0)
+               } else if (nbytes == 0)
                        RTE_LOG(ERR, EAL, "Read nothing from fd %d\n", fd);
                return;
        } while (1);
 }
-#endif
 
 static int
 eal_epoll_process_event(struct epoll_event *evs, unsigned int n,
@@ -1012,6 +1088,9 @@ rte_epoll_wait(int epfd, struct rte_epoll_event *events,
                                strerror(errno));
                        rc = -1;
                        break;
+               } else {
+                       /* rc == 0, epoll_wait timed out */
+                       break;
                }
        }
 
@@ -1068,7 +1147,6 @@ rte_epoll_ctl(int epfd, int op, int fd,
        return 0;
 }
 
-#ifdef RTE_NEXT_ABI
 int
 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
                int op, unsigned int vec, void *data)
@@ -1076,10 +1154,14 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
        struct rte_epoll_event *rev;
        struct rte_epoll_data *epdata;
        int epfd_op;
+       unsigned int efd_idx;
        int rc = 0;
 
+       efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
+               (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
+
        if (!intr_handle || intr_handle->nb_efd == 0 ||
-           vec >= intr_handle->nb_efd) {
+           efd_idx >= intr_handle->nb_efd) {
                RTE_LOG(ERR, EAL, "Wrong intr vector number.\n");
                return -EPERM;
        }
@@ -1087,7 +1169,7 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
        switch (op) {
        case RTE_INTR_EVENT_ADD:
                epfd_op = EPOLL_CTL_ADD;
-               rev = &intr_handle->elist[vec];
+               rev = &intr_handle->elist[efd_idx];
                if (rev->status != RTE_EPOLL_INVALID) {
                        RTE_LOG(INFO, EAL, "Event already been added.\n");
                        return -EEXIST;
@@ -1099,7 +1181,8 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
                epdata->data   = data;
                epdata->cb_fun = (rte_intr_event_cb_t)eal_intr_proc_rxtx_intr;
                epdata->cb_arg = (void *)intr_handle;
-               rc = rte_epoll_ctl(epfd, epfd_op, intr_handle->efds[vec], rev);
+               rc = rte_epoll_ctl(epfd, epfd_op,
+                                  intr_handle->efds[efd_idx], rev);
                if (!rc)
                        RTE_LOG(DEBUG, EAL,
                                "efd %d associated with vec %d added on epfd %d"
@@ -1109,7 +1192,7 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
                break;
        case RTE_INTR_EVENT_DEL:
                epfd_op = EPOLL_CTL_DEL;
-               rev = &intr_handle->elist[vec];
+               rev = &intr_handle->elist[efd_idx];
                if (rev->status == RTE_EPOLL_INVALID) {
                        RTE_LOG(INFO, EAL, "Event does not exist.\n");
                        return -EPERM;
@@ -1127,6 +1210,24 @@ rte_intr_rx_ctl(struct rte_intr_handle *intr_handle, int epfd,
        return rc;
 }
 
+void
+rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
+{
+       uint32_t i;
+       struct rte_epoll_event *rev;
+
+       for (i = 0; i < intr_handle->nb_efd; i++) {
+               rev = &intr_handle->elist[i];
+               if (rev->status == RTE_EPOLL_INVALID)
+                       continue;
+               if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
+                       /* force free if the entry valid */
+                       eal_epoll_data_safe_free(rev);
+                       rev->status = RTE_EPOLL_INVALID;
+               }
+       }
+}
+
 int
 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
 {
@@ -1134,6 +1235,8 @@ rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
        int fd;
        uint32_t n = RTE_MIN(nb_efd, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
 
+       assert(nb_efd != 0);
+
        if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX) {
                for (i = 0; i < n; i++) {
                        fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
@@ -1141,12 +1244,19 @@ rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
                                RTE_LOG(ERR, EAL,
                                        "can't setup eventfd, error %i (%s)\n",
                                        errno, strerror(errno));
-                               return -1;
+                               return -errno;
                        }
                        intr_handle->efds[i] = fd;
                }
                intr_handle->nb_efd   = n;
                intr_handle->max_intr = NB_OTHER_INTR + n;
+       } else if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
+               /* only check, initialization would be done in vdev driver.*/
+               if (intr_handle->efd_counter_size >
+                   sizeof(union rte_intr_read_buffer)) {
+                       RTE_LOG(ERR, EAL, "the efd_counter_size is oversized");
+                       return -EINVAL;
+               }
        } else {
                intr_handle->efds[0]  = intr_handle->fd;
                intr_handle->nb_efd   = RTE_MIN(nb_efd, 1U);
@@ -1160,19 +1270,8 @@ void
 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
 {
        uint32_t i;
-       struct rte_epoll_event *rev;
-
-       for (i = 0; i < intr_handle->nb_efd; i++) {
-               rev = &intr_handle->elist[i];
-               if (rev->status == RTE_EPOLL_INVALID)
-                       continue;
-               if (rte_epoll_ctl(rev->epfd, EPOLL_CTL_DEL, rev->fd, rev)) {
-                       /* force free if the entry valid */
-                       eal_epoll_data_safe_free(rev);
-                       rev->status = RTE_EPOLL_INVALID;
-               }
-       }
 
+       rte_intr_free_epoll_fd(intr_handle);
        if (intr_handle->max_intr > intr_handle->nb_efd) {
                for (i = 0; i < intr_handle->nb_efd; i++)
                        close(intr_handle->efds[i]);
@@ -1190,47 +1289,20 @@ rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
 int
 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
 {
-       return !!(intr_handle->max_intr - intr_handle->nb_efd);
-}
-
-#else
-int
-rte_intr_rx_ctl(struct rte_intr_handle *intr_handle,
-               int epfd, int op, unsigned int vec, void *data)
-{
-       RTE_SET_USED(intr_handle);
-       RTE_SET_USED(epfd);
-       RTE_SET_USED(op);
-       RTE_SET_USED(vec);
-       RTE_SET_USED(data);
-       return -ENOTSUP;
+       if (!rte_intr_dp_is_en(intr_handle))
+               return 1;
+       else
+               return !!(intr_handle->max_intr - intr_handle->nb_efd);
 }
 
 int
-rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
+rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
 {
-       RTE_SET_USED(intr_handle);
-       RTE_SET_USED(nb_efd);
-       return 0;
-}
+       if (intr_handle->type == RTE_INTR_HANDLE_VFIO_MSIX)
+               return 1;
 
-void
-rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
-{
-       RTE_SET_USED(intr_handle);
-}
+       if (intr_handle->type == RTE_INTR_HANDLE_VDEV)
+               return 1;
 
-int
-rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
-{
-       RTE_SET_USED(intr_handle);
        return 0;
 }
-
-int
-rte_intr_allow_others(struct rte_intr_handle *intr_handle)
-{
-       RTE_SET_USED(intr_handle);
-       return 1;
-}
-#endif