vdpa/mlx5: fix minsize build
[dpdk.git] / lib / eal / freebsd / eal_interrupts.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2018 Intel Corporation
3  */
4
5 #include <string.h>
6 #include <sys/types.h>
7 #include <sys/event.h>
8 #include <sys/queue.h>
9 #include <unistd.h>
10
11 #include <rte_errno.h>
12 #include <rte_lcore.h>
13 #include <rte_spinlock.h>
14 #include <rte_common.h>
15 #include <rte_interrupts.h>
16 #include <rte_eal_trace.h>
17
18 #include "eal_private.h"
19 #include "eal_alarm_private.h"
20
21 #define MAX_INTR_EVENTS 16
22
23 /**
24  * union buffer for reading on different devices
25  */
26 union rte_intr_read_buffer {
27         char charbuf[16];                /* for others */
28 };
29
30 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
31 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
32
33 struct rte_intr_callback {
34         TAILQ_ENTRY(rte_intr_callback) next;
35         rte_intr_callback_fn cb_fn;  /**< callback address */
36         void *cb_arg;                /**< parameter for callback */
37         uint8_t pending_delete;      /**< delete after callback is called */
38         rte_intr_unregister_callback_fn ucb_fn; /**< fn to call before cb is deleted */
39 };
40
41 struct rte_intr_source {
42         TAILQ_ENTRY(rte_intr_source) next;
43         struct rte_intr_handle intr_handle; /**< interrupt handle */
44         struct rte_intr_cb_list callbacks;  /**< user callbacks */
45         uint32_t active;
46 };
47
48 /* global spinlock for interrupt data operation */
49 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
50
51 /* interrupt sources list */
52 static struct rte_intr_source_list intr_sources;
53
54 /* interrupt handling thread */
55 static pthread_t intr_thread;
56
57 static volatile int kq = -1;
58
59 static int
60 intr_source_to_kevent(const struct rte_intr_handle *ih, struct kevent *ke)
61 {
62         /* alarm callbacks are special case */
63         if (ih->type == RTE_INTR_HANDLE_ALARM) {
64                 uint64_t timeout_ns;
65
66                 /* get soonest alarm timeout */
67                 if (eal_alarm_get_timeout_ns(&timeout_ns) < 0)
68                         return -1;
69
70                 ke->filter = EVFILT_TIMER;
71                 /* timers are one shot */
72                 ke->flags |= EV_ONESHOT;
73                 ke->fflags = NOTE_NSECONDS;
74                 ke->data = timeout_ns;
75         } else {
76                 ke->filter = EVFILT_READ;
77         }
78         ke->ident = ih->fd;
79
80         return 0;
81 }
82
83 int
84 rte_intr_callback_register(const struct rte_intr_handle *intr_handle,
85                 rte_intr_callback_fn cb, void *cb_arg)
86 {
87         struct rte_intr_callback *callback;
88         struct rte_intr_source *src;
89         int ret = 0, add_event = 0;
90
91         /* first do parameter checking */
92         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
93                 RTE_LOG(ERR, EAL,
94                         "Registering with invalid input parameter\n");
95                 return -EINVAL;
96         }
97         if (kq < 0) {
98                 RTE_LOG(ERR, EAL, "Kqueue is not active: %d\n", kq);
99                 return -ENODEV;
100         }
101
102         rte_spinlock_lock(&intr_lock);
103
104         /* find the source for this intr_handle */
105         TAILQ_FOREACH(src, &intr_sources, next) {
106                 if (src->intr_handle.fd == intr_handle->fd)
107                         break;
108         }
109
110         /* if this is an alarm interrupt and it already has a callback,
111          * then we don't want to create a new callback because the only
112          * thing on the list should be eal_alarm_callback() and we may
113          * be called just to reset the timer.
114          */
115         if (src != NULL && src->intr_handle.type == RTE_INTR_HANDLE_ALARM &&
116                  !TAILQ_EMPTY(&src->callbacks)) {
117                 callback = NULL;
118         } else {
119                 /* allocate a new interrupt callback entity */
120                 callback = calloc(1, sizeof(*callback));
121                 if (callback == NULL) {
122                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
123                         ret = -ENOMEM;
124                         goto fail;
125                 }
126                 callback->cb_fn = cb;
127                 callback->cb_arg = cb_arg;
128                 callback->pending_delete = 0;
129                 callback->ucb_fn = NULL;
130
131                 if (src == NULL) {
132                         src = calloc(1, sizeof(*src));
133                         if (src == NULL) {
134                                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
135                                 ret = -ENOMEM;
136                                 goto fail;
137                         } else {
138                                 src->intr_handle = *intr_handle;
139                                 TAILQ_INIT(&src->callbacks);
140                                 TAILQ_INSERT_TAIL(&intr_sources, src, next);
141                         }
142                 }
143
144                 /* we had no interrupts for this */
145                 if (TAILQ_EMPTY(&src->callbacks))
146                         add_event = 1;
147
148                 TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
149         }
150
151         /* add events to the queue. timer events are special as we need to
152          * re-set the timer.
153          */
154         if (add_event || src->intr_handle.type == RTE_INTR_HANDLE_ALARM) {
155                 struct kevent ke;
156
157                 memset(&ke, 0, sizeof(ke));
158                 ke.flags = EV_ADD; /* mark for addition to the queue */
159
160                 if (intr_source_to_kevent(intr_handle, &ke) < 0) {
161                         RTE_LOG(ERR, EAL, "Cannot convert interrupt handle to kevent\n");
162                         ret = -ENODEV;
163                         goto fail;
164                 }
165
166                 /**
167                  * add the intr file descriptor into wait list.
168                  */
169                 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
170                         /* currently, nic_uio does not support interrupts, so
171                          * this error will always be triggered and output to the
172                          * user. so, don't output it unless debug log level set.
173                          */
174                         if (errno == ENODEV)
175                                 RTE_LOG(DEBUG, EAL, "Interrupt handle %d not supported\n",
176                                         src->intr_handle.fd);
177                         else
178                                 RTE_LOG(ERR, EAL, "Error adding fd %d "
179                                                 "kevent, %s\n",
180                                                 src->intr_handle.fd,
181                                                 strerror(errno));
182                         ret = -errno;
183                         goto fail;
184                 }
185         }
186         rte_eal_trace_intr_callback_register(intr_handle, cb, cb_arg, ret);
187         rte_spinlock_unlock(&intr_lock);
188
189         return 0;
190 fail:
191         /* clean up */
192         if (src != NULL) {
193                 if (callback != NULL)
194                         TAILQ_REMOVE(&(src->callbacks), callback, next);
195                 if (TAILQ_EMPTY(&(src->callbacks))) {
196                         TAILQ_REMOVE(&intr_sources, src, next);
197                         free(src);
198                 }
199         }
200         free(callback);
201         rte_eal_trace_intr_callback_register(intr_handle, cb, cb_arg, ret);
202         rte_spinlock_unlock(&intr_lock);
203         return ret;
204 }
205
206 int
207 rte_intr_callback_unregister_pending(const struct rte_intr_handle *intr_handle,
208                                 rte_intr_callback_fn cb_fn, void *cb_arg,
209                                 rte_intr_unregister_callback_fn ucb_fn)
210 {
211         int ret;
212         struct rte_intr_source *src;
213         struct rte_intr_callback *cb, *next;
214
215         /* do parameter checking first */
216         if (intr_handle == NULL || intr_handle->fd < 0) {
217                 RTE_LOG(ERR, EAL,
218                 "Unregistering with invalid input parameter\n");
219                 return -EINVAL;
220         }
221
222         if (kq < 0) {
223                 RTE_LOG(ERR, EAL, "Kqueue is not active\n");
224                 return -ENODEV;
225         }
226
227         rte_spinlock_lock(&intr_lock);
228
229         /* check if the insterrupt source for the fd is existent */
230         TAILQ_FOREACH(src, &intr_sources, next)
231                 if (src->intr_handle.fd == intr_handle->fd)
232                         break;
233
234         /* No interrupt source registered for the fd */
235         if (src == NULL) {
236                 ret = -ENOENT;
237
238         /* only usable if the source is active */
239         } else if (src->active == 0) {
240                 ret = -EAGAIN;
241
242         } else {
243                 ret = 0;
244
245                 /* walk through the callbacks and mark all that match. */
246                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
247                         next = TAILQ_NEXT(cb, next);
248                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
249                                         cb->cb_arg == cb_arg)) {
250                                 cb->pending_delete = 1;
251                                 cb->ucb_fn = ucb_fn;
252                                 ret++;
253                         }
254                 }
255         }
256
257         rte_spinlock_unlock(&intr_lock);
258
259         return ret;
260 }
261
262 int
263 rte_intr_callback_unregister(const struct rte_intr_handle *intr_handle,
264                 rte_intr_callback_fn cb_fn, void *cb_arg)
265 {
266         int ret;
267         struct rte_intr_source *src;
268         struct rte_intr_callback *cb, *next;
269
270         /* do parameter checking first */
271         if (intr_handle == NULL || intr_handle->fd < 0) {
272                 RTE_LOG(ERR, EAL,
273                 "Unregistering with invalid input parameter\n");
274                 return -EINVAL;
275         }
276         if (kq < 0) {
277                 RTE_LOG(ERR, EAL, "Kqueue is not active\n");
278                 return -ENODEV;
279         }
280
281         rte_spinlock_lock(&intr_lock);
282
283         /* check if the insterrupt source for the fd is existent */
284         TAILQ_FOREACH(src, &intr_sources, next)
285                 if (src->intr_handle.fd == intr_handle->fd)
286                         break;
287
288         /* No interrupt source registered for the fd */
289         if (src == NULL) {
290                 ret = -ENOENT;
291
292         /* interrupt source has some active callbacks right now. */
293         } else if (src->active != 0) {
294                 ret = -EAGAIN;
295
296         /* ok to remove. */
297         } else {
298                 struct kevent ke;
299
300                 ret = 0;
301
302                 /* remove it from the kqueue */
303                 memset(&ke, 0, sizeof(ke));
304                 ke.flags = EV_DELETE; /* mark for deletion from the queue */
305
306                 if (intr_source_to_kevent(intr_handle, &ke) < 0) {
307                         RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
308                         ret = -ENODEV;
309                         goto out;
310                 }
311
312                 /**
313                  * remove intr file descriptor from wait list.
314                  */
315                 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
316                         RTE_LOG(ERR, EAL, "Error removing fd %d kevent, %s\n",
317                                 src->intr_handle.fd, strerror(errno));
318                         /* removing non-existent even is an expected condition
319                          * in some circumstances (e.g. oneshot events).
320                          */
321                 }
322
323                 /*walk through the callbacks and remove all that match. */
324                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
325                         next = TAILQ_NEXT(cb, next);
326                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
327                                         cb->cb_arg == cb_arg)) {
328                                 TAILQ_REMOVE(&src->callbacks, cb, next);
329                                 free(cb);
330                                 ret++;
331                         }
332                 }
333
334                 /* all callbacks for that source are removed. */
335                 if (TAILQ_EMPTY(&src->callbacks)) {
336                         TAILQ_REMOVE(&intr_sources, src, next);
337                         free(src);
338                 }
339         }
340 out:
341         rte_eal_trace_intr_callback_unregister(intr_handle, cb_fn, cb_arg,
342                 ret);
343         rte_spinlock_unlock(&intr_lock);
344
345         return ret;
346 }
347
348 int
349 rte_intr_callback_unregister_sync(const struct rte_intr_handle *intr_handle,
350                 rte_intr_callback_fn cb_fn, void *cb_arg)
351 {
352         int ret = 0;
353
354         while ((ret = rte_intr_callback_unregister(intr_handle, cb_fn, cb_arg)) == -EAGAIN)
355                 rte_pause();
356
357         return ret;
358 }
359
360 int
361 rte_intr_enable(const struct rte_intr_handle *intr_handle)
362 {
363         int rc = 0;
364
365         if (intr_handle == NULL)
366                 return -1;
367
368         if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
369                 rc = 0;
370                 goto out;
371         }
372
373         if (intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0) {
374                 rc = -1;
375                 goto out;
376         }
377
378         switch (intr_handle->type) {
379         /* not used at this moment */
380         case RTE_INTR_HANDLE_ALARM:
381                 rc = -1;
382                 break;
383         /* not used at this moment */
384         case RTE_INTR_HANDLE_DEV_EVENT:
385                 rc = -1;
386                 break;
387         /* unknown handle type */
388         default:
389                 RTE_LOG(ERR, EAL,
390                         "Unknown handle type of fd %d\n",
391                                         intr_handle->fd);
392                 rc = -1;
393                 break;
394         }
395
396 out:
397         rte_eal_trace_intr_enable(intr_handle, rc);
398         return rc;
399 }
400
401 int
402 rte_intr_disable(const struct rte_intr_handle *intr_handle)
403 {
404         int rc = 0;
405
406         if (intr_handle == NULL)
407                 return -1;
408
409         if (intr_handle->type == RTE_INTR_HANDLE_VDEV) {
410                 rc = 0;
411                 goto out;
412         }
413
414         if (intr_handle->fd < 0 || intr_handle->uio_cfg_fd < 0) {
415                 rc = -1;
416                 goto out;
417         }
418
419         switch (intr_handle->type) {
420         /* not used at this moment */
421         case RTE_INTR_HANDLE_ALARM:
422                 rc = -1;
423                 break;
424         /* not used at this moment */
425         case RTE_INTR_HANDLE_DEV_EVENT:
426                 rc = -1;
427                 break;
428         /* unknown handle type */
429         default:
430                 RTE_LOG(ERR, EAL,
431                         "Unknown handle type of fd %d\n",
432                                         intr_handle->fd);
433                 rc = -1;
434                 break;
435         }
436 out:
437         rte_eal_trace_intr_disable(intr_handle, rc);
438         return rc;
439 }
440
441 int
442 rte_intr_ack(const struct rte_intr_handle *intr_handle)
443 {
444         if (intr_handle && intr_handle->type == RTE_INTR_HANDLE_VDEV)
445                 return 0;
446
447         return -1;
448 }
449
450 static void
451 eal_intr_process_interrupts(struct kevent *events, int nfds)
452 {
453         struct rte_intr_callback active_cb;
454         union rte_intr_read_buffer buf;
455         struct rte_intr_callback *cb, *next;
456         struct rte_intr_source *src;
457         bool call = false;
458         int n, bytes_read;
459         struct kevent ke;
460
461         for (n = 0; n < nfds; n++) {
462                 int event_fd = events[n].ident;
463
464                 rte_spinlock_lock(&intr_lock);
465                 TAILQ_FOREACH(src, &intr_sources, next)
466                         if (src->intr_handle.fd == event_fd)
467                                 break;
468                 if (src == NULL) {
469                         rte_spinlock_unlock(&intr_lock);
470                         continue;
471                 }
472
473                 /* mark this interrupt source as active and release the lock. */
474                 src->active = 1;
475                 rte_spinlock_unlock(&intr_lock);
476
477                 /* set the length to be read dor different handle type */
478                 switch (src->intr_handle.type) {
479                 case RTE_INTR_HANDLE_ALARM:
480                         bytes_read = 0;
481                         call = true;
482                         break;
483                 case RTE_INTR_HANDLE_VDEV:
484                 case RTE_INTR_HANDLE_EXT:
485                         bytes_read = 0;
486                         call = true;
487                         break;
488                 case RTE_INTR_HANDLE_DEV_EVENT:
489                         bytes_read = 0;
490                         call = true;
491                         break;
492                 default:
493                         bytes_read = 1;
494                         break;
495                 }
496
497                 if (bytes_read > 0) {
498                         /**
499                          * read out to clear the ready-to-be-read flag
500                          * for epoll_wait.
501                          */
502                         bytes_read = read(event_fd, &buf, bytes_read);
503                         if (bytes_read < 0) {
504                                 if (errno == EINTR || errno == EWOULDBLOCK)
505                                         continue;
506
507                                 RTE_LOG(ERR, EAL, "Error reading from file "
508                                         "descriptor %d: %s\n",
509                                         event_fd,
510                                         strerror(errno));
511                         } else if (bytes_read == 0)
512                                 RTE_LOG(ERR, EAL, "Read nothing from file "
513                                         "descriptor %d\n", event_fd);
514                         else
515                                 call = true;
516                 }
517
518                 /* grab a lock, again to call callbacks and update status. */
519                 rte_spinlock_lock(&intr_lock);
520
521                 if (call) {
522                         /* Finally, call all callbacks. */
523                         TAILQ_FOREACH(cb, &src->callbacks, next) {
524
525                                 /* make a copy and unlock. */
526                                 active_cb = *cb;
527                                 rte_spinlock_unlock(&intr_lock);
528
529                                 /* call the actual callback */
530                                 active_cb.cb_fn(active_cb.cb_arg);
531
532                                 /*get the lock back. */
533                                 rte_spinlock_lock(&intr_lock);
534                         }
535                 }
536
537                 /* we done with that interrupt source, release it. */
538                 src->active = 0;
539
540                 /* check if any callback are supposed to be removed */
541                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
542                         next = TAILQ_NEXT(cb, next);
543                         if (cb->pending_delete) {
544                                 /* remove it from the kqueue */
545                                 memset(&ke, 0, sizeof(ke));
546                                 /* mark for deletion from the queue */
547                                 ke.flags = EV_DELETE;
548
549                                 if (intr_source_to_kevent(&src->intr_handle, &ke) < 0) {
550                                         RTE_LOG(ERR, EAL, "Cannot convert to kevent\n");
551                                         rte_spinlock_unlock(&intr_lock);
552                                         return;
553                                 }
554
555                                 /**
556                                  * remove intr file descriptor from wait list.
557                                  */
558                                 if (kevent(kq, &ke, 1, NULL, 0, NULL) < 0) {
559                                         RTE_LOG(ERR, EAL, "Error removing fd %d kevent, "
560                                                 "%s\n", src->intr_handle.fd,
561                                                 strerror(errno));
562                                         /* removing non-existent even is an expected
563                                          * condition in some circumstances
564                                          * (e.g. oneshot events).
565                                          */
566                                 }
567
568                                 TAILQ_REMOVE(&src->callbacks, cb, next);
569                                 if (cb->ucb_fn)
570                                         cb->ucb_fn(&src->intr_handle, cb->cb_arg);
571                                 free(cb);
572                         }
573                 }
574
575                 /* all callbacks for that source are removed. */
576                 if (TAILQ_EMPTY(&src->callbacks)) {
577                         TAILQ_REMOVE(&intr_sources, src, next);
578                         free(src);
579                 }
580
581                 rte_spinlock_unlock(&intr_lock);
582         }
583 }
584
585 static void *
586 eal_intr_thread_main(void *arg __rte_unused)
587 {
588         struct kevent events[MAX_INTR_EVENTS];
589         int nfds;
590
591         /* host thread, never break out */
592         for (;;) {
593                 /* do not change anything, just wait */
594                 nfds = kevent(kq, NULL, 0, events, MAX_INTR_EVENTS, NULL);
595
596                 /* kevent fail */
597                 if (nfds < 0) {
598                         if (errno == EINTR)
599                                 continue;
600                         RTE_LOG(ERR, EAL,
601                                 "kevent returns with fail\n");
602                         break;
603                 }
604                 /* kevent timeout, will never happen here */
605                 else if (nfds == 0)
606                         continue;
607
608                 /* kevent has at least one fd ready to read */
609                 eal_intr_process_interrupts(events, nfds);
610         }
611         close(kq);
612         kq = -1;
613         return NULL;
614 }
615
616 int
617 rte_eal_intr_init(void)
618 {
619         int ret = 0;
620
621         /* init the global interrupt source head */
622         TAILQ_INIT(&intr_sources);
623
624         kq = kqueue();
625         if (kq < 0) {
626                 RTE_LOG(ERR, EAL, "Cannot create kqueue instance\n");
627                 return -1;
628         }
629
630         /* create the host thread to wait/handle the interrupt */
631         ret = rte_ctrl_thread_create(&intr_thread, "eal-intr-thread", NULL,
632                         eal_intr_thread_main, NULL);
633         if (ret != 0) {
634                 rte_errno = -ret;
635                 RTE_LOG(ERR, EAL,
636                         "Failed to create thread for interrupt handling\n");
637         }
638
639         return ret;
640 }
641
642 int
643 rte_intr_rx_ctl(struct rte_intr_handle *intr_handle,
644                 int epfd, int op, unsigned int vec, void *data)
645 {
646         RTE_SET_USED(intr_handle);
647         RTE_SET_USED(epfd);
648         RTE_SET_USED(op);
649         RTE_SET_USED(vec);
650         RTE_SET_USED(data);
651
652         return -ENOTSUP;
653 }
654
655 int
656 rte_intr_efd_enable(struct rte_intr_handle *intr_handle, uint32_t nb_efd)
657 {
658         RTE_SET_USED(intr_handle);
659         RTE_SET_USED(nb_efd);
660
661         return 0;
662 }
663
664 void
665 rte_intr_efd_disable(struct rte_intr_handle *intr_handle)
666 {
667         RTE_SET_USED(intr_handle);
668 }
669
670 int
671 rte_intr_dp_is_en(struct rte_intr_handle *intr_handle)
672 {
673         RTE_SET_USED(intr_handle);
674         return 0;
675 }
676
677 int
678 rte_intr_allow_others(struct rte_intr_handle *intr_handle)
679 {
680         RTE_SET_USED(intr_handle);
681         return 1;
682 }
683
684 int
685 rte_intr_cap_multiple(struct rte_intr_handle *intr_handle)
686 {
687         RTE_SET_USED(intr_handle);
688         return 0;
689 }
690
691 int
692 rte_epoll_wait(int epfd, struct rte_epoll_event *events,
693                 int maxevents, int timeout)
694 {
695         RTE_SET_USED(epfd);
696         RTE_SET_USED(events);
697         RTE_SET_USED(maxevents);
698         RTE_SET_USED(timeout);
699
700         return -ENOTSUP;
701 }
702
703 int
704 rte_epoll_wait_interruptible(int epfd, struct rte_epoll_event *events,
705                              int maxevents, int timeout)
706 {
707         RTE_SET_USED(epfd);
708         RTE_SET_USED(events);
709         RTE_SET_USED(maxevents);
710         RTE_SET_USED(timeout);
711
712         return -ENOTSUP;
713 }
714
715 int
716 rte_epoll_ctl(int epfd, int op, int fd, struct rte_epoll_event *event)
717 {
718         RTE_SET_USED(epfd);
719         RTE_SET_USED(op);
720         RTE_SET_USED(fd);
721         RTE_SET_USED(event);
722
723         return -ENOTSUP;
724 }
725
726 int
727 rte_intr_tls_epfd(void)
728 {
729         return -ENOTSUP;
730 }
731
732 void
733 rte_intr_free_epoll_fd(struct rte_intr_handle *intr_handle)
734 {
735         RTE_SET_USED(intr_handle);
736 }
737
738 int rte_thread_is_intr(void)
739 {
740         return pthread_equal(intr_thread, pthread_self());
741 }