1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2018 Intel Corporation
11 #include <rte_errno.h>
12 #include <rte_lcore.h>
13 #include <rte_spinlock.h>
14 #include <rte_common.h>
15 #include <rte_interrupts.h>
16 #include <rte_eal_trace.h>
18 #include "eal_private.h"
19 #include "eal_alarm_private.h"
21 #define MAX_INTR_EVENTS 16
24 * union buffer for reading on different devices
26 union rte_intr_read_buffer {
27 char charbuf[16]; /* for others */
30 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
31 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
33 struct rte_intr_callback {
34 TAILQ_ENTRY(rte_intr_callback) next;
35 rte_intr_callback_fn cb_fn; /**< callback address */
36 void *cb_arg; /**< parameter for callback */
37 uint8_t pending_delete; /**< delete after callback is called */
38 rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
41 struct rte_intr_source {
42 TAILQ_ENTRY(rte_intr_source) next;
43 struct rte_intr_handle *intr_handle; /**< interrupt handle */
44 struct rte_intr_cb_list callbacks; /**< user callbacks */
48 /* global spinlock for interrupt data operation */
49 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
51 /* interrupt sources list */
52 static struct rte_intr_source_list intr_sources;
54 /* interrupt handling thread */
55 static pthread_t intr_thread;
57 static volatile int kq = -1;
60 intr_source_to_kevent(const struct rte_intr_handle *ih, struct kevent *ke)
62 /* alarm callbacks are special case */
63 if (rte_intr_type_get(ih) == RTE_INTR_HANDLE_ALARM) {
66 /* get soonest alarm timeout */
67 if (eal_alarm_get_timeout_ns(&timeout_ns) < 0)
70 ke->filter = EVFILT_TIMER;
71 /* timers are one shot */
72 ke->flags |= EV_ONESHOT;
73 ke->fflags = NOTE_NSECONDS;
74 ke->data = timeout_ns;
76 ke->filter = EVFILT_READ;
78 ke->ident = rte_intr_fd_get(ih);
84 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
85 rte_intr_callback_fn cb, void *cb_arg)
87 struct rte_intr_callback *callback;
88 struct rte_intr_source *src;
89 int ret = 0, add_event = 0;
91 /* first do parameter checking */
92 if (rte_intr_fd_get(intr_handle) < 0 || cb == NULL) {
94 "Registering with invalid input parameter\n");
98 RTE_LOG(ERR, EAL, "Kqueue is not active: %d\n", kq);
102 rte_spinlock_lock(&intr_lock);
104 /* find the source for this intr_handle */
105 TAILQ_FOREACH(src, &intr_sources, next) {
106 if (rte_intr_fd_get(src->intr_handle) == rte_intr_fd_get(intr_handle))
110 /* if this is an alarm interrupt and it already has a callback,
111 * then we don't want to create a new callback because the only
112 * thing on the list should be eal_alarm_callback() and we may
113 * be called just to reset the timer.
116 rte_intr_type_get(src->intr_handle) == RTE_INTR_HANDLE_ALARM &&
117 !TAILQ_EMPTY(&src->callbacks)) {
120 /* allocate a new interrupt callback entity */
121 callback = calloc(1, sizeof(*callback));
122 if (callback == NULL) {
123 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
127 callback->cb_fn = cb;
128 callback->cb_arg = cb_arg;
129 callback->pending_delete = 0;
130 callback->ucb_fn = NULL;
133 src = calloc(1, sizeof(*src));
135 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
139 src->intr_handle = rte_intr_instance_dup(intr_handle);
140 if (src->intr_handle == NULL) {
141 RTE_LOG(ERR, EAL, "Can not create intr instance\n");
147 TAILQ_INIT(&src->callbacks);
148 TAILQ_INSERT_TAIL(&intr_sources, src, next);
152 /* we had no interrupts for this */
153 if (TAILQ_EMPTY(&src->callbacks))
156 TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
159 /* add events to the queue. timer events are special as we need to
163 rte_intr_type_get(src->intr_handle) == RTE_INTR_HANDLE_ALARM) {
166 memset(&ke, 0, sizeof(ke));
167 ke.flags = EV_ADD; /* mark for addition to the queue */
169 if (intr_source_to_kevent(intr_handle, &ke) < 0) {
170 RTE_LOG(ERR, EAL, "Cannot convert interrupt handle to kevent\n");
176 * add the intr file descriptor into wait list.
178 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
179 /* currently, nic_uio does not support interrupts, so
180 * this error will always be triggered and output to the
181 * user. so, don't output it unless debug log level set.
184 RTE_LOG(DEBUG, EAL, "Interrupt handle %d not supported\n",
185 rte_intr_fd_get(src->intr_handle));
187 RTE_LOG(ERR, EAL, "Error adding fd %d kevent, %s\n",
188 rte_intr_fd_get(src->intr_handle),
194 rte_eal_trace_intr_callback_register(intr_handle, cb, cb_arg, ret);
195 rte_spinlock_unlock(&intr_lock);
201 if (callback != NULL)
202 TAILQ_REMOVE(&(src->callbacks), callback, next);
203 if (TAILQ_EMPTY(&(src->callbacks))) {
204 TAILQ_REMOVE(&intr_sources, src, next);
209 rte_eal_trace_intr_callback_register(intr_handle, cb, cb_arg, ret);
210 rte_spinlock_unlock(&intr_lock);
215 rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
216 rte_intr_callback_fn cb_fn, void *cb_arg,
217 rte_intr_unregister_callback_fn ucb_fn)
220 struct rte_intr_source *src;
221 struct rte_intr_callback *cb, *next;
223 /* do parameter checking first */
224 if (rte_intr_fd_get(intr_handle) < 0) {
226 "Unregistering with invalid input parameter\n");
231 RTE_LOG(ERR, EAL, "Kqueue is not active\n");
235 rte_spinlock_lock(&intr_lock);
237 /* check if the interrupt source for the fd is existent */
238 TAILQ_FOREACH(src, &intr_sources, next)
239 if (rte_intr_fd_get(src->intr_handle) == rte_intr_fd_get(intr_handle))
242 /* No interrupt source registered for the fd */
246 /* only usable if the source is active */
247 } else if (src->active == 0) {
253 /* walk through the callbacks and mark all that match. */
254 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
255 next = TAILQ_NEXT(cb, next);
256 if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
257 cb->cb_arg == cb_arg)) {
258 cb->pending_delete = 1;
265 rte_spinlock_unlock(&intr_lock);
271 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
272 rte_intr_callback_fn cb_fn, void *cb_arg)
275 struct rte_intr_source *src;
276 struct rte_intr_callback *cb, *next;
278 /* do parameter checking first */
279 if (rte_intr_fd_get(intr_handle) < 0) {
281 "Unregistering with invalid input parameter\n");
285 RTE_LOG(ERR, EAL, "Kqueue is not active\n");
289 rte_spinlock_lock(&intr_lock);
291 /* check if the interrupt source for the fd is existent */
292 TAILQ_FOREACH(src, &intr_sources, next)
293 if (rte_intr_fd_get(src->intr_handle) == rte_intr_fd_get(intr_handle))
296 /* No interrupt source registered for the fd */
300 /* interrupt source has some active callbacks right now. */
301 } else if (src->active != 0) {
310 /* remove it from the kqueue */
311 memset(&ke, 0, sizeof(ke));
312 ke.flags = EV_DELETE; /* mark for deletion from the queue */
314 if (intr_source_to_kevent(intr_handle, &ke) < 0) {
315 RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
321 * remove intr file descriptor from wait list.
323 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
324 RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
325 rte_intr_fd_get(src->intr_handle),
327 /* removing non-existent even is an expected condition
328 * in some circumstances (e.g. oneshot events).
332 /*walk through the callbacks and remove all that match. */
333 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
334 next = TAILQ_NEXT(cb, next);
335 if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
336 cb->cb_arg == cb_arg)) {
337 TAILQ_REMOVE(&src->callbacks, cb, next);
343 /* all callbacks for that source are removed. */
344 if (TAILQ_EMPTY(&src->callbacks)) {
345 TAILQ_REMOVE(&intr_sources, src, next);
350 rte_eal_trace_intr_callback_unregister(intr_handle, cb_fn, cb_arg,
352 rte_spinlock_unlock(&intr_lock);
358 rte_intr_callback_unregister_sync(const struct rte_intr_handle *intr_handle,
359 rte_intr_callback_fn cb_fn, void *cb_arg)
363 while ((ret = rte_intr_callback_unregister(intr_handle, cb_fn, cb_arg)) == -EAGAIN)
370 rte_intr_enable(const struct rte_intr_handle *intr_handle)
374 if (intr_handle == NULL)
377 if (rte_intr_type_get(intr_handle) == RTE_INTR_HANDLE_VDEV) {
382 if (rte_intr_fd_get(intr_handle) < 0 ||
383 rte_intr_dev_fd_get(intr_handle) < 0) {
388 switch (rte_intr_type_get(intr_handle)) {
389 /* not used at this moment */
390 case RTE_INTR_HANDLE_ALARM:
393 /* not used at this moment */
394 case RTE_INTR_HANDLE_DEV_EVENT:
397 /* unknown handle type */
399 RTE_LOG(ERR, EAL, "Unknown handle type of fd %d\n",
400 rte_intr_fd_get(intr_handle));
406 rte_eal_trace_intr_enable(intr_handle, rc);
411 rte_intr_disable(const struct rte_intr_handle *intr_handle)
415 if (intr_handle == NULL)
418 if (rte_intr_type_get(intr_handle) == RTE_INTR_HANDLE_VDEV) {
423 if (rte_intr_fd_get(intr_handle) < 0 ||
424 rte_intr_dev_fd_get(intr_handle) < 0) {
429 switch (rte_intr_type_get(intr_handle)) {
430 /* not used at this moment */
431 case RTE_INTR_HANDLE_ALARM:
434 /* not used at this moment */
435 case RTE_INTR_HANDLE_DEV_EVENT:
438 /* unknown handle type */
440 RTE_LOG(ERR, EAL, "Unknown handle type of fd %d\n",
441 rte_intr_fd_get(intr_handle));
446 rte_eal_trace_intr_disable(intr_handle, rc);
451 rte_intr_ack(const struct rte_intr_handle *intr_handle)
453 if (rte_intr_type_get(intr_handle) == RTE_INTR_HANDLE_VDEV)
460 eal_intr_process_interrupts(struct kevent *events, int nfds)
462 struct rte_intr_callback active_cb;
463 union rte_intr_read_buffer buf;
464 struct rte_intr_callback *cb, *next;
465 struct rte_intr_source *src;
470 for (n = 0; n < nfds; n++) {
471 int event_fd = events[n].ident;
473 rte_spinlock_lock(&intr_lock);
474 TAILQ_FOREACH(src, &intr_sources, next)
475 if (rte_intr_fd_get(src->intr_handle) == event_fd)
478 rte_spinlock_unlock(&intr_lock);
482 /* mark this interrupt source as active and release the lock. */
484 rte_spinlock_unlock(&intr_lock);
486 /* set the length to be read dor different handle type */
487 switch (rte_intr_type_get(src->intr_handle)) {
488 case RTE_INTR_HANDLE_ALARM:
492 case RTE_INTR_HANDLE_VDEV:
493 case RTE_INTR_HANDLE_EXT:
497 case RTE_INTR_HANDLE_DEV_EVENT:
506 if (bytes_read > 0) {
508 * read out to clear the ready-to-be-read flag
511 bytes_read = read(event_fd, &buf, bytes_read);
512 if (bytes_read < 0) {
513 if (errno == EINTR || errno == EWOULDBLOCK)
516 RTE_LOG(ERR, EAL, "Error reading from file "
517 "descriptor %d: %s\n",
520 } else if (bytes_read == 0)
521 RTE_LOG(ERR, EAL, "Read nothing from file "
522 "descriptor %d\n", event_fd);
527 /* grab a lock, again to call callbacks and update status. */
528 rte_spinlock_lock(&intr_lock);
531 /* Finally, call all callbacks. */
532 TAILQ_FOREACH(cb, &src->callbacks, next) {
534 /* make a copy and unlock. */
536 rte_spinlock_unlock(&intr_lock);
538 /* call the actual callback */
539 active_cb.cb_fn(active_cb.cb_arg);
541 /*get the lock back. */
542 rte_spinlock_lock(&intr_lock);
546 /* we done with that interrupt source, release it. */
549 /* check if any callback are supposed to be removed */
550 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
551 next = TAILQ_NEXT(cb, next);
552 if (cb->pending_delete) {
553 /* remove it from the kqueue */
554 memset(&ke, 0, sizeof(ke));
555 /* mark for deletion from the queue */
556 ke.flags = EV_DELETE;
558 if (intr_source_to_kevent(src->intr_handle, &ke) < 0) {
559 RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
560 rte_spinlock_unlock(&intr_lock);
565 * remove intr file descriptor from wait list.
567 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
568 RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
569 rte_intr_fd_get(src->intr_handle),
571 /* removing non-existent even is an expected
572 * condition in some circumstances
573 * (e.g. oneshot events).
577 TAILQ_REMOVE(&src->callbacks, cb, next);
579 cb->ucb_fn(src->intr_handle, cb->cb_arg);
584 /* all callbacks for that source are removed. */
585 if (TAILQ_EMPTY(&src->callbacks)) {
586 TAILQ_REMOVE(&intr_sources, src, next);
590 rte_spinlock_unlock(&intr_lock);
595 eal_intr_thread_main(void *arg __rte_unused)
597 struct kevent events[MAX_INTR_EVENTS];
600 /* host thread, never break out */
602 /* do not change anything, just wait */
603 nfds = kevent(kq, NULL, 0, events, MAX_INTR_EVENTS, NULL);
610 "kevent returns with fail\n");
613 /* kevent timeout, will never happen here */
617 /* kevent has at least one fd ready to read */
618 eal_intr_process_interrupts(events, nfds);
626 rte_eal_intr_init(void)
630 /* init the global interrupt source head */
631 TAILQ_INIT(&intr_sources);
635 RTE_LOG(ERR, EAL, "Cannot create kqueue instance\n");
639 /* create the host thread to wait/handle the interrupt */
640 ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
641 eal_intr_thread_main, NULL);
645 "Failed to create thread for interrupt handling\n");
652 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle,
653 int epfd, int op, unsigned int vec, void *data)
655 RTE_SET_USED(intr_handle);
665 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
667 RTE_SET_USED(intr_handle);
668 RTE_SET_USED(nb_efd);
674 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
676 RTE_SET_USED(intr_handle);
680 rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
682 RTE_SET_USED(intr_handle);
687 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
689 RTE_SET_USED(intr_handle);
694 rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
696 RTE_SET_USED(intr_handle);
701 rte_epoll_wait(int epfd, struct rte_epoll_event *events,
702 int maxevents, int timeout)
705 RTE_SET_USED(events);
706 RTE_SET_USED(maxevents);
707 RTE_SET_USED(timeout);
713 rte_epoll_wait_interruptible(int epfd, struct rte_epoll_event *events,
714 int maxevents, int timeout)
717 RTE_SET_USED(events);
718 RTE_SET_USED(maxevents);
719 RTE_SET_USED(timeout);
725 rte_epoll_ctl(int epfd, int op, int fd, struct rte_epoll_event *event)
736 rte_intr_tls_epfd(void)
742 rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
744 RTE_SET_USED(intr_handle);
747 int rte_thread_is_intr(void)
749 return pthread_equal(intr_thread, pthread_self());