4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <sys/queue.h>
45 #include <sys/epoll.h>
46 #include <sys/signalfd.h>
48 #include <rte_common.h>
49 #include <rte_interrupts.h>
50 #include <rte_memory.h>
51 #include <rte_memzone.h>
52 #include <rte_launch.h>
53 #include <rte_tailq.h>
55 #include <rte_per_lcore.h>
56 #include <rte_lcore.h>
57 #include <rte_atomic.h>
58 #include <rte_branch_prediction.h>
60 #include <rte_debug.h>
62 #include <rte_mempool.h>
64 #include <rte_malloc.h>
65 #include <rte_errno.h>
66 #include <rte_spinlock.h>
68 #include "eal_private.h"
70 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
86 * union buffer for reading on different devices
88 union rte_intr_read_buffer {
89 int uio_intr_count; /* for uio device */
90 uint64_t timerfd_num; /* for timerfd */
91 char charbuf[16]; /* for others */
94 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
95 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
97 struct rte_intr_callback {
98 TAILQ_ENTRY(rte_intr_callback) next;
99 rte_intr_callback_fn cb_fn; /**< callback address */
100 void *cb_arg; /**< parameter for callback */
103 struct rte_intr_source {
104 TAILQ_ENTRY(rte_intr_source) next;
105 struct rte_intr_handle intr_handle; /**< interrupt handle */
106 struct rte_intr_cb_list callbacks; /**< user callbacks */
110 /* global spinlock for interrupt data operation */
111 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
113 /* union buffer for pipe read/write */
114 static union intr_pipefds intr_pipe;
116 /* interrupt sources list */
117 static struct rte_intr_source_list intr_sources;
119 /* interrupt handling thread */
120 static pthread_t intr_thread;
123 rte_intr_callback_register(struct rte_intr_handle *intr_handle,
124 rte_intr_callback_fn cb, void *cb_arg)
126 int ret, wake_thread;
127 struct rte_intr_source *src;
128 struct rte_intr_callback *callback;
132 /* first do parameter checking */
133 if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
135 "Registering with invalid input parameter\n");
139 /* allocate a new interrupt callback entity */
140 callback = rte_zmalloc("interrupt callback list",
141 sizeof(*callback), 0);
142 if (callback == NULL) {
143 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
146 callback->cb_fn = cb;
147 callback->cb_arg = cb_arg;
149 rte_spinlock_lock(&intr_lock);
151 /* check if there is at least one callback registered for the fd */
152 TAILQ_FOREACH(src, &intr_sources, next) {
153 if (src->intr_handle.fd == intr_handle->fd) {
154 /* we had no interrupts for this */
155 if TAILQ_EMPTY(&src->callbacks)
158 TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
164 /* no existing callbacks for this - add new source */
166 if ((src = rte_zmalloc("interrupt source list",
167 sizeof(*src), 0)) == NULL) {
168 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
172 src->intr_handle = *intr_handle;
173 TAILQ_INIT(&src->callbacks);
174 TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
175 TAILQ_INSERT_TAIL(&intr_sources, src, next);
181 rte_spinlock_unlock(&intr_lock);
184 * check if need to notify the pipe fd waited by epoll_wait to
185 * rebuild the wait list.
188 if (write(intr_pipe.writefd, "1", 1) < 0)
195 rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
196 rte_intr_callback_fn cb_fn, void *cb_arg)
199 struct rte_intr_source *src;
200 struct rte_intr_callback *cb, *next;
202 /* do parameter checking first */
203 if (intr_handle == NULL || intr_handle->fd < 0) {
205 "Unregistering with invalid input parameter\n");
209 rte_spinlock_lock(&intr_lock);
211 /* check if the insterrupt source for the fd is existent */
212 TAILQ_FOREACH(src, &intr_sources, next)
213 if (src->intr_handle.fd == intr_handle->fd)
216 /* No interrupt source registered for the fd */
220 /* interrupt source has some active callbacks right now. */
221 } else if (src->active != 0) {
228 /*walk through the callbacks and remove all that match. */
229 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
231 next = TAILQ_NEXT(cb, next);
233 if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
234 cb->cb_arg == cb_arg)) {
235 TAILQ_REMOVE(&src->callbacks, cb, next);
241 /* all callbacks for that source are removed. */
242 if (TAILQ_EMPTY(&src->callbacks)) {
243 TAILQ_REMOVE(&intr_sources, src, next);
248 rte_spinlock_unlock(&intr_lock);
250 /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
251 if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
259 rte_intr_enable(struct rte_intr_handle *intr_handle)
263 if (!intr_handle || intr_handle->fd < 0)
266 switch (intr_handle->type){
267 /* write to the uio fd to enable the interrupt */
268 case RTE_INTR_HANDLE_UIO:
269 if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
271 "Error enabling interrupts for fd %d\n",
276 /* not used at this moment */
277 case RTE_INTR_HANDLE_ALARM:
279 /* unkown handle type */
282 "Unknown handle type of fd %d\n",
291 rte_intr_disable(struct rte_intr_handle *intr_handle)
295 if (!intr_handle || intr_handle->fd < 0)
298 switch (intr_handle->type){
299 /* write to the uio fd to disable the interrupt */
300 case RTE_INTR_HANDLE_UIO:
301 if (write(intr_handle->fd, &value, sizeof(value)) < 0){
303 "Error enabling interrupts for fd %d\n",
308 /* not used at this moment */
309 case RTE_INTR_HANDLE_ALARM:
311 /* unkown handle type */
314 "Unknown handle type of fd %d\n",
323 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
326 struct rte_intr_source *src;
327 struct rte_intr_callback *cb;
328 union rte_intr_read_buffer buf;
329 struct rte_intr_callback active_cb;
331 for (n = 0; n < nfds; n++) {
334 * if the pipe fd is ready to read, return out to
335 * rebuild the wait list.
337 if (events[n].data.fd == intr_pipe.readfd){
338 int r = read(intr_pipe.readfd, buf.charbuf,
339 sizeof(buf.charbuf));
343 rte_spinlock_lock(&intr_lock);
344 TAILQ_FOREACH(src, &intr_sources, next)
345 if (src->intr_handle.fd ==
349 rte_spinlock_unlock(&intr_lock);
353 /* mark this interrupt source as active and release the lock. */
355 rte_spinlock_unlock(&intr_lock);
357 /* set the length to be read dor different handle type */
358 switch (src->intr_handle.type) {
359 case RTE_INTR_HANDLE_UIO:
362 case RTE_INTR_HANDLE_ALARM:
363 bytes_read = sizeof(uint64_t);
371 * read out to clear the ready-to-be-read flag
374 bytes_read = read(events[n].data.fd, &buf, bytes_read);
377 RTE_LOG(ERR, EAL, "Error reading from file "
378 "descriptor %d: %s\n", events[n].data.fd,
380 else if (bytes_read == 0)
381 RTE_LOG(ERR, EAL, "Read nothing from file "
382 "descriptor %d\n", events[n].data.fd);
384 /* grab a lock, again to call callbacks and update status. */
385 rte_spinlock_lock(&intr_lock);
387 if (bytes_read > 0) {
389 /* Finally, call all callbacks. */
390 TAILQ_FOREACH(cb, &src->callbacks, next) {
392 /* make a copy and unlock. */
394 rte_spinlock_unlock(&intr_lock);
396 /* call the actual callback */
397 active_cb.cb_fn(&src->intr_handle,
400 /*get the lcok back. */
401 rte_spinlock_lock(&intr_lock);
405 /* we done with that interrupt source, release it. */
407 rte_spinlock_unlock(&intr_lock);
414 * It handles all the interrupts.
417 * epoll file descriptor.
419 * The number of file descriptors added in epoll.
425 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
427 struct epoll_event events[totalfds];
431 nfds = epoll_wait(pfd, events, totalfds,
432 EAL_INTR_EPOLL_WAIT_FOREVER);
433 /* epoll_wait fail */
438 "epoll_wait returns with fail\n");
441 /* epoll_wait timeout, will never happens here */
444 /* epoll_wait has at least one fd ready to read */
445 if (eal_intr_process_interrupts(events, nfds) < 0)
451 * It builds/rebuilds up the epoll file descriptor with all the
452 * file descriptors being waited on. Then handles the interrupts.
460 static __attribute__((noreturn)) void *
461 eal_intr_thread_main(__rte_unused void *arg)
463 struct epoll_event ev;
465 /* host thread, never break out */
467 /* build up the epoll fd with all descriptors we are to
468 * wait on then pass it to the handle_interrupts function
470 static struct epoll_event pipe_event = {
471 .events = EPOLLIN | EPOLLPRI,
473 struct rte_intr_source *src;
476 /* create epoll fd */
477 int pfd = epoll_create(1);
479 rte_panic("Cannot create epoll instance\n");
481 pipe_event.data.fd = intr_pipe.readfd;
483 * add pipe fd into wait list, this pipe is used to
484 * rebuild the wait list.
486 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
488 rte_panic("Error adding fd to %d epoll_ctl, %s\n",
489 intr_pipe.readfd, strerror(errno));
493 rte_spinlock_lock(&intr_lock);
495 TAILQ_FOREACH(src, &intr_sources, next) {
496 if (src->callbacks.tqh_first == NULL)
497 continue; /* skip those with no callbacks */
498 ev.events = EPOLLIN | EPOLLPRI;
499 ev.data.fd = src->intr_handle.fd;
502 * add all the uio device file descriptor
505 if (epoll_ctl(pfd, EPOLL_CTL_ADD,
506 src->intr_handle.fd, &ev) < 0){
507 rte_panic("Error adding fd %d epoll_ctl, %s\n",
508 src->intr_handle.fd, strerror(errno));
513 rte_spinlock_unlock(&intr_lock);
514 /* serve the interrupt */
515 eal_intr_handle_interrupts(pfd, numfds);
518 * when we return, we need to rebuild the
519 * list of fds to monitor.
526 rte_eal_intr_init(void)
530 /* init the global interrupt source head */
531 TAILQ_INIT(&intr_sources);
534 * create a pipe which will be waited by epoll and notified to
535 * rebuild the wait list of epoll.
537 if (pipe(intr_pipe.pipefd) < 0)
540 /* create the host thread to wait/handle the interrupt */
541 ret = pthread_create(&intr_thread, NULL,
542 eal_intr_thread_main, NULL);
545 "Failed to create thread for interrupt handling\n");