lib/librte_eal/linuxapp/eal/eal_interrupts.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdint.h>
  36 #include <stdlib.h>
  37 #include <pthread.h>
  38 #include <sys/queue.h>
  39 #include <malloc.h>
  40 #include <stdarg.h>
  41 #include <unistd.h>
  42 #include <string.h>
  43 #include <errno.h>
  44 #include <inttypes.h>
  45 #include <sys/epoll.h>
  46 #include <sys/signalfd.h>
  47
  48 #include <rte_common.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_memory.h>
  51 #include <rte_memzone.h>
  52 #include <rte_launch.h>
  53 #include <rte_tailq.h>
  54 #include <rte_eal.h>
  55 #include <rte_per_lcore.h>
  56 #include <rte_lcore.h>
  57 #include <rte_atomic.h>
  58 #include <rte_branch_prediction.h>
  59 #include <rte_ring.h>
  60 #include <rte_debug.h>
  61 #include <rte_log.h>
  62 #include <rte_mempool.h>
  63 #include <rte_pci.h>
  64 #include <rte_malloc.h>
  65 #include <rte_errno.h>
  66 #include <rte_spinlock.h>
  67
  68 #include "eal_private.h"
  69
  70 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
  71
  72 /**
  73  * union for pipe fds.
  74  */
  75 union intr_pipefds{
  76         struct {
  77                 int pipefd[2];
  78         };
  79         struct {
  80                 int readfd;
  81                 int writefd;
  82         };
  83 };
  84
  85 /**
  86  * union buffer for reading on different devices
  87  */
  88 union rte_intr_read_buffer {
  89         int uio_intr_count;              /* for uio device */
  90         uint64_t timerfd_num;            /* for timerfd */
  91         char charbuf[16];                /* for others */
  92 };
  93
  94 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
  95 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
  96
  97 struct rte_intr_callback {
  98         TAILQ_ENTRY(rte_intr_callback) next;
  99         rte_intr_callback_fn cb_fn;  /**< callback address */
 100         void *cb_arg;                /**< parameter for callback */
 101 };
 102
 103 struct rte_intr_source {
 104         TAILQ_ENTRY(rte_intr_source) next;
 105         struct rte_intr_handle intr_handle; /**< interrupt handle */
 106         struct rte_intr_cb_list callbacks;  /**< user callbacks */
 107         uint32_t active;
 108 };
 109
 110 /* global spinlock for interrupt data operation */
 111 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
 112
 113 /* union buffer for pipe read/write */
 114 static union intr_pipefds intr_pipe;
 115
 116 /* interrupt sources list */
 117 static struct rte_intr_source_list intr_sources;
 118
 119 /* interrupt handling thread */
 120 static pthread_t intr_thread;
 121
 122 int
 123 rte_intr_callback_register(struct rte_intr_handle *intr_handle,
 124                         rte_intr_callback_fn cb, void *cb_arg)
 125 {
 126         int ret, wake_thread;
 127         struct rte_intr_source *src;
 128         struct rte_intr_callback *callback;
 129
 130         wake_thread = 0;
 131
 132         /* first do parameter checking */
 133         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
 134                 RTE_LOG(ERR, EAL,
 135                         "Registering with invalid input parameter\n");
 136                 return -EINVAL;
 137         }
 138
 139         /* allocate a new interrupt callback entity */
 140         callback = rte_zmalloc("interrupt callback list",
 141                                 sizeof(*callback), 0);
 142         if (callback == NULL) {
 143                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 144                 return -ENOMEM;
 145         }
 146         callback->cb_fn = cb;
 147         callback->cb_arg = cb_arg;
 148
 149         rte_spinlock_lock(&intr_lock);
 150
 151         /* check if there is at least one callback registered for the fd */
 152         TAILQ_FOREACH(src, &intr_sources, next) {
 153                 if (src->intr_handle.fd == intr_handle->fd) {
 154                         /* we had no interrupts for this */
 155                         if TAILQ_EMPTY(&src->callbacks)
 156                                 wake_thread = 1;
 157
 158                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 159                         ret = 0;
 160                         break;
 161                 }
 162         }
 163
 164         /* no existing callbacks for this - add new source */
 165         if (src == NULL) {
 166                 if ((src = rte_zmalloc("interrupt source list",
 167                                 sizeof(*src), 0)) == NULL) {
 168                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 169                         rte_free(callback);
 170                         ret = -ENOMEM;
 171                 } else {
 172                         src->intr_handle = *intr_handle;
 173                         TAILQ_INIT(&src->callbacks);
 174                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 175                         TAILQ_INSERT_TAIL(&intr_sources, src, next);
 176                         wake_thread = 1;
 177                         ret = 0;
 178                 }
 179         }
 180
 181         rte_spinlock_unlock(&intr_lock);
 182
 183         /**
 184          * check if need to notify the pipe fd waited by epoll_wait to
 185          * rebuild the wait list.
 186          */
 187         if (wake_thread)
 188                 if (write(intr_pipe.writefd, "1", 1) < 0)
 189                         return -EPIPE;
 190
 191         return (ret);
 192 }
 193
 194 int
 195 rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
 196                         rte_intr_callback_fn cb_fn, void *cb_arg)
 197 {
 198         int ret;
 199         struct rte_intr_source *src;
 200         struct rte_intr_callback *cb, *next;
 201
 202         /* do parameter checking first */
 203         if (intr_handle == NULL || intr_handle->fd < 0) {
 204                 RTE_LOG(ERR, EAL,
 205                 "Unregistering with invalid input parameter\n");
 206                 return -EINVAL;
 207         }
 208
 209         rte_spinlock_lock(&intr_lock);
 210
 211         /* check if the insterrupt source for the fd is existent */
 212         TAILQ_FOREACH(src, &intr_sources, next)
 213                 if (src->intr_handle.fd == intr_handle->fd)
 214                         break;
 215
 216         /* No interrupt source registered for the fd */
 217         if (src == NULL) {
 218                 ret = -ENOENT;
 219
 220         /* interrupt source has some active callbacks right now. */
 221         } else if (src->active != 0) {
 222                 ret = -EAGAIN;
 223
 224         /* ok to remove. */
 225         } else {
 226                 ret = 0;
 227
 228                 /*walk through the callbacks and remove all that match. */
 229                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 230
 231                         next = TAILQ_NEXT(cb, next);
 232
 233                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 234                                         cb->cb_arg == cb_arg)) {
 235                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 236                                 rte_free(cb);
 237                                 ret++;
 238                         }
 239                 }
 240
 241                 /* all callbacks for that source are removed. */
 242                 if (TAILQ_EMPTY(&src->callbacks)) {
 243                         TAILQ_REMOVE(&intr_sources, src, next);
 244                         rte_free(src);
 245                 }
 246         }
 247
 248         rte_spinlock_unlock(&intr_lock);
 249
 250         /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 251         if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 252                 ret = -EPIPE;
 253         }
 254
 255         return (ret);
 256 }
 257
 258 int
 259 rte_intr_enable(struct rte_intr_handle *intr_handle)
 260 {
 261         const int value = 1;
 262
 263         if (!intr_handle || intr_handle->fd < 0)
 264                 return -1;
 265
 266         switch (intr_handle->type){
 267         /* write to the uio fd to enable the interrupt */
 268         case RTE_INTR_HANDLE_UIO:
 269                 if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 270                         RTE_LOG(ERR, EAL,
 271                                 "Error enabling interrupts for fd %d\n",
 272                                                         intr_handle->fd);
 273                         return -1;
 274                 }
 275                 break;
 276         /* not used at this moment */
 277         case RTE_INTR_HANDLE_ALARM:
 278                 return -1;
 279         /* unkown handle type */
 280         default:
 281                 RTE_LOG(ERR, EAL,
 282                         "Unknown handle type of fd %d\n",
 283                                         intr_handle->fd);
 284                 return -1;
 285         }
 286
 287         return 0;
 288 }
 289
 290 int
 291 rte_intr_disable(struct rte_intr_handle *intr_handle)
 292 {
 293         const int value = 0;
 294
 295         if (!intr_handle || intr_handle->fd < 0)
 296                 return -1;
 297
 298         switch (intr_handle->type){
 299         /* write to the uio fd to disable the interrupt */
 300         case RTE_INTR_HANDLE_UIO:
 301                 if (write(intr_handle->fd, &value, sizeof(value)) < 0){
 302                         RTE_LOG(ERR, EAL,
 303                                 "Error enabling interrupts for fd %d\n",
 304                                                         intr_handle->fd);
 305                         return -1;
 306                 }
 307                 break;
 308         /* not used at this moment */
 309         case RTE_INTR_HANDLE_ALARM:
 310                 return -1;
 311         /* unkown handle type */
 312         default:
 313                 RTE_LOG(ERR, EAL,
 314                         "Unknown handle type of fd %d\n",
 315                                         intr_handle->fd);
 316                 return -1;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int
 323 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 324 {
 325         int n, bytes_read;
 326         struct rte_intr_source *src;
 327         struct rte_intr_callback *cb;
 328         union rte_intr_read_buffer buf;
 329         struct rte_intr_callback active_cb;
 330
 331         for (n = 0; n < nfds; n++) {
 332
 333                 /**
 334                  * if the pipe fd is ready to read, return out to
 335                  * rebuild the wait list.
 336                  */
 337                 if (events[n].data.fd == intr_pipe.readfd){
 338                         int r = read(intr_pipe.readfd, buf.charbuf,
 339                                         sizeof(buf.charbuf));
 340                         RTE_SET_USED(r);
 341                         return -1;
 342                 }
 343                 rte_spinlock_lock(&intr_lock);
 344                 TAILQ_FOREACH(src, &intr_sources, next)
 345                         if (src->intr_handle.fd ==
 346                                         events[n].data.fd)
 347                                 break;
 348                 if (src == NULL){
 349                         rte_spinlock_unlock(&intr_lock);
 350                         continue;
 351                 }
 352
 353                 /* mark this interrupt source as active and release the lock. */
 354                 src->active = 1;
 355                 rte_spinlock_unlock(&intr_lock);
 356
 357                 /* set the length to be read dor different handle type */
 358                 switch (src->intr_handle.type) {
 359                 case RTE_INTR_HANDLE_UIO:
 360                         bytes_read = 4;
 361                         break;
 362                 case RTE_INTR_HANDLE_ALARM:
 363                         bytes_read = sizeof(uint64_t);
 364                         break;
 365                 default:
 366                         bytes_read = 1;
 367                         break;
 368                 }
 369
 370                 /**
 371                  * read out to clear the ready-to-be-read flag
 372                  * for epoll_wait.
 373                  */
 374                 bytes_read = read(events[n].data.fd, &buf, bytes_read);
 375
 376                 if (bytes_read < 0) {
 377                         RTE_LOG(ERR, EAL, "Error reading from file descriptor"
 378                                 " %d, error: %d\n", events[n].data.fd, errno);
 379                 }
 380                 else if (bytes_read == 0) {
 381                         RTE_LOG(ERR, EAL,
 382                                 "Read nothing from file descriptor %d.\n",
 383                                                         events[n].data.fd);
 384                 }
 385
 386                 /* grab a lock, again to call callbacks and update status. */
 387                 rte_spinlock_lock(&intr_lock);
 388
 389                 if (bytes_read > 0) {
 390
 391                         /* Finally, call all callbacks. */
 392                         TAILQ_FOREACH(cb, &src->callbacks, next) {
 393
 394                                 /* make a copy and unlock. */
 395                                 active_cb = *cb;
 396                                 rte_spinlock_unlock(&intr_lock);
 397
 398                                 /* call the actual callback */
 399                                 active_cb.cb_fn(&src->intr_handle,
 400                                         active_cb.cb_arg);
 401
 402                                 /*get the lcok back. */
 403                                 rte_spinlock_lock(&intr_lock);
 404                         }
 405                 }
 406
 407                 /* we done with that interrupt source, release it. */
 408                 src->active = 0;
 409                 rte_spinlock_unlock(&intr_lock);
 410         }
 411
 412         return 0;
 413 }
 414
 415 /**
 416  * It handles all the interrupts.
 417  *
 418  * @param pfd
 419  *  epoll file descriptor.
 420  * @param totalfds
 421  *  The number of file descriptors added in epoll.
 422  *
 423  * @return
 424  *  void
 425  */
 426 static void
 427 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
 428 {
 429         struct epoll_event events[totalfds];
 430         int nfds = 0;
 431
 432         for(;;) {
 433                 nfds = epoll_wait(pfd, events, totalfds,
 434                         EAL_INTR_EPOLL_WAIT_FOREVER);
 435                 /* epoll_wait fail */
 436                 if (nfds < 0) {
 437                         if (errno == EINTR)
 438                                 continue;
 439                         RTE_LOG(ERR, EAL,
 440                                 "epoll_wait returns with fail\n");
 441                         return;
 442                 }
 443                 /* epoll_wait timeout, will never happens here */
 444                 else if (nfds == 0)
 445                         continue;
 446                 /* epoll_wait has at least one fd ready to read */
 447                 if (eal_intr_process_interrupts(events, nfds) < 0)
 448                         return;
 449         }
 450 }
 451
 452 /**
 453  * It builds/rebuilds up the epoll file descriptor with all the
 454  * file descriptors being waited on. Then handles the interrupts.
 455  *
 456  * @param arg
 457  *  pointer. (unused)
 458  *
 459  * @return
 460  *  never return;
 461  */
 462 static __attribute__((noreturn)) void *
 463 eal_intr_thread_main(__rte_unused void *arg)
 464 {
 465         struct epoll_event ev;
 466
 467         /* host thread, never break out */
 468         for (;;) {
 469                 /* build up the epoll fd with all descriptors we are to
 470                  * wait on then pass it to the handle_interrupts function
 471                  */
 472                 static struct epoll_event pipe_event = {
 473                         .events = EPOLLIN | EPOLLPRI,
 474                 };
 475                 struct rte_intr_source *src;
 476                 unsigned numfds = 0;
 477
 478                 /* create epoll fd */
 479                 int pfd = epoll_create(1);
 480                 if (pfd < 0)
 481                         rte_panic("Cannot create epoll instance\n");
 482
 483                 pipe_event.data.fd = intr_pipe.readfd;
 484                 /**
 485                  * add pipe fd into wait list, this pipe is used to
 486                  * rebuild the wait list.
 487                  */
 488                 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
 489                                                 &pipe_event) < 0) {
 490                         rte_panic("Error adding fd to %d epoll_ctl, %s\n",
 491                                         intr_pipe.readfd, strerror(errno));
 492                 }
 493                 numfds++;
 494
 495                 rte_spinlock_lock(&intr_lock);
 496
 497                 TAILQ_FOREACH(src, &intr_sources, next) {
 498                         if (src->callbacks.tqh_first == NULL)
 499                                 continue; /* skip those with no callbacks */
 500                         ev.events = EPOLLIN | EPOLLPRI;
 501                         ev.data.fd = src->intr_handle.fd;
 502
 503                         /**
 504                          * add all the uio device file descriptor
 505                          * into wait list.
 506                          */
 507                         if (epoll_ctl(pfd, EPOLL_CTL_ADD,
 508                                         src->intr_handle.fd, &ev) < 0){
 509                                 rte_panic("Error adding fd %d epoll_ctl, %s\n",
 510                                         src->intr_handle.fd, strerror(errno));
 511                         }
 512                         else
 513                                 numfds++;
 514                 }
 515                 rte_spinlock_unlock(&intr_lock);
 516                 /* serve the interrupt */
 517                 eal_intr_handle_interrupts(pfd, numfds);
 518
 519                 /**
 520                  * when we return, we need to rebuild the
 521                  * list of fds to monitor.
 522                  */
 523                 close(pfd);
 524         }
 525 }
 526
 527 int
 528 rte_eal_intr_init(void)
 529 {
 530         int ret = 0;
 531
 532         /* init the global interrupt source head */
 533         TAILQ_INIT(&intr_sources);
 534
 535         /**
 536          * create a pipe which will be waited by epoll and notified to
 537          * rebuild the wait list of epoll.
 538          */
 539         if (pipe(intr_pipe.pipefd) < 0)
 540                 return -1;
 541
 542         /* create the host thread to wait/handle the interrupt */
 543         ret = pthread_create(&intr_thread, NULL,
 544                         eal_intr_thread_main, NULL);
 545         if (ret != 0)
 546                 RTE_LOG(ERR, EAL,
 547                         "Failed to create thread for interrupt handling\n");
 548
 549         return -ret;
 550 }
 551