lib/librte_eal/linuxapp/eal/eal_interrupts.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2013 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <stdio.h>
  36 #include <stdint.h>
  37 #include <stdlib.h>
  38 #include <pthread.h>
  39 #include <sys/queue.h>
  40 #include <malloc.h>
  41 #include <stdarg.h>
  42 #include <unistd.h>
  43 #include <string.h>
  44 #include <errno.h>
  45 #include <inttypes.h>
  46 #include <sys/epoll.h>
  47 #include <sys/signalfd.h>
  48
  49 #include <rte_common.h>
  50 #include <rte_interrupts.h>
  51 #include <rte_memory.h>
  52 #include <rte_memzone.h>
  53 #include <rte_launch.h>
  54 #include <rte_tailq.h>
  55 #include <rte_eal.h>
  56 #include <rte_per_lcore.h>
  57 #include <rte_lcore.h>
  58 #include <rte_atomic.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_ring.h>
  61 #include <rte_debug.h>
  62 #include <rte_log.h>
  63 #include <rte_mempool.h>
  64 #include <rte_pci.h>
  65 #include <rte_malloc.h>
  66 #include <rte_errno.h>
  67 #include <rte_spinlock.h>
  68
  69 #include "eal_private.h"
  70
  71 #define EAL_INTR_EPOLL_WAIT_FOREVER (-1)
  72
  73 /**
  74  * union for pipe fds.
  75  */
  76 union intr_pipefds{
  77         struct {
  78                 int pipefd[2];
  79         };
  80         struct {
  81                 int readfd;
  82                 int writefd;
  83         };
  84 };
  85
  86 /**
  87  * union buffer for reading on different devices
  88  */
  89 union rte_intr_read_buffer {
  90         int uio_intr_count;              /* for uio device */
  91         uint64_t timerfd_num;            /* for timerfd */
  92         char charbuf[16];                /* for others */
  93 };
  94
  95 TAILQ_HEAD(rte_intr_cb_list, rte_intr_callback);
  96 TAILQ_HEAD(rte_intr_source_list, rte_intr_source);
  97
  98 struct rte_intr_callback {
  99         TAILQ_ENTRY(rte_intr_callback) next;
 100         rte_intr_callback_fn cb_fn;  /**< callback address */
 101         void *cb_arg;                /**< parameter for callback */
 102 };
 103
 104 struct rte_intr_source {
 105         TAILQ_ENTRY(rte_intr_source) next;
 106         struct rte_intr_handle intr_handle; /**< interrupt handle */
 107         struct rte_intr_cb_list callbacks;  /**< user callbacks */
 108         uint32_t active;
 109 };
 110
 111 /* global spinlock for interrupt data operation */
 112 static rte_spinlock_t intr_lock = RTE_SPINLOCK_INITIALIZER;
 113
 114 /* union buffer for pipe read/write */
 115 static union intr_pipefds intr_pipe;
 116
 117 /* interrupt sources list */
 118 static struct rte_intr_source_list intr_sources;
 119
 120 /* interrupt handling thread */
 121 static pthread_t intr_thread;
 122
 123 int
 124 rte_intr_callback_register(struct rte_intr_handle *intr_handle,
 125                         rte_intr_callback_fn cb, void *cb_arg)
 126 {
 127         int ret, wake_thread;
 128         struct rte_intr_source *src;
 129         struct rte_intr_callback *callback;
 130
 131         wake_thread = 0;
 132
 133         /* first do parameter checking */
 134         if (intr_handle == NULL || intr_handle->fd < 0 || cb == NULL) {
 135                 RTE_LOG(ERR, EAL,
 136                         "Registering with invalid input parameter\n");
 137                 return -EINVAL;
 138         }
 139
 140         /* allocate a new interrupt callback entity */
 141         callback = rte_zmalloc("interrupt callback list",
 142                                 sizeof(*callback), 0);
 143         if (callback == NULL) {
 144                 RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 145                 return -ENOMEM;
 146         }
 147         callback->cb_fn = cb;
 148         callback->cb_arg = cb_arg;
 149
 150         rte_spinlock_lock(&intr_lock);
 151
 152         /* check if there is at least one callback registered for the fd */
 153         TAILQ_FOREACH(src, &intr_sources, next) {
 154                 if (src->intr_handle.fd == intr_handle->fd) {
 155                         /* we had no interrupts for this */
 156                         if TAILQ_EMPTY(&src->callbacks)
 157                                 wake_thread = 1;
 158
 159                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 160                         ret = 0;
 161                         break;
 162                 }
 163         }
 164
 165         /* no existing callbacks for this - add new source */
 166         if (src == NULL) {
 167                 if ((src = rte_zmalloc("interrupt source list",
 168                                 sizeof(*src), 0)) == NULL) {
 169                         RTE_LOG(ERR, EAL, "Can not allocate memory\n");
 170                         rte_free(callback);
 171                         ret = -ENOMEM;
 172                 } else {
 173                         src->intr_handle = *intr_handle;
 174                         TAILQ_INIT(&src->callbacks);
 175                         TAILQ_INSERT_TAIL(&(src->callbacks), callback, next);
 176                         TAILQ_INSERT_TAIL(&intr_sources, src, next);
 177                         wake_thread = 1;
 178                         ret = 0;
 179                 }
 180         }
 181
 182         rte_spinlock_unlock(&intr_lock);
 183
 184         /**
 185          * check if need to notify the pipe fd waited by epoll_wait to
 186          * rebuild the wait list.
 187          */
 188         if (wake_thread)
 189                 if (write(intr_pipe.writefd, "1", 1) < 0)
 190                         return -EPIPE;
 191
 192         return (ret);
 193 }
 194
 195 int
 196 rte_intr_callback_unregister(struct rte_intr_handle *intr_handle,
 197                         rte_intr_callback_fn cb_fn, void *cb_arg)
 198 {
 199         int ret;
 200         struct rte_intr_source *src;
 201         struct rte_intr_callback *cb, *next;
 202
 203         /* do parameter checking first */
 204         if (intr_handle == NULL || intr_handle->fd < 0) {
 205                 RTE_LOG(ERR, EAL,
 206                 "Unregistering with invalid input parameter\n");
 207                 return -EINVAL;
 208         }
 209
 210         rte_spinlock_lock(&intr_lock);
 211
 212         /* check if the insterrupt source for the fd is existent */
 213         TAILQ_FOREACH(src, &intr_sources, next)
 214                 if (src->intr_handle.fd == intr_handle->fd)
 215                         break;
 216
 217         /* No interrupt source registered for the fd */
 218         if (src == NULL) {
 219                 ret = -ENOENT;
 220
 221         /* interrupt source has some active callbacks right now. */
 222         } else if (src->active != 0) {
 223                 ret = -EAGAIN;
 224
 225         /* ok to remove. */
 226         } else {
 227                 ret = 0;
 228
 229                 /*walk through the callbacks and remove all that match. */
 230                 for (cb = TAILQ_FIRST(&src->callbacks); cb != NULL; cb = next) {
 231
 232                         next = TAILQ_NEXT(cb, next);
 233
 234                         if (cb->cb_fn == cb_fn && (cb_arg == (void *)-1 ||
 235                                         cb->cb_arg == cb_arg)) {
 236                                 TAILQ_REMOVE(&src->callbacks, cb, next);
 237                                 rte_free(cb);
 238                                 ret++;
 239                         }
 240                 }
 241
 242                 /* all callbacks for that source are removed. */
 243                 if (TAILQ_EMPTY(&src->callbacks)) {
 244                         TAILQ_REMOVE(&intr_sources, src, next);
 245                         rte_free(src);
 246                 }
 247         }
 248
 249         rte_spinlock_unlock(&intr_lock);
 250
 251         /* notify the pipe fd waited by epoll_wait to rebuild the wait list */
 252         if (ret >= 0 && write(intr_pipe.writefd, "1", 1) < 0) {
 253                 ret = -EPIPE;
 254         }
 255
 256         return (ret);
 257 }
 258
 259 int
 260 rte_intr_enable(struct rte_intr_handle *intr_handle)
 261 {
 262         const int value = 1;
 263
 264         if (!intr_handle || intr_handle->fd < 0)
 265                 return -1;
 266
 267         switch (intr_handle->type){
 268         /* write to the uio fd to enable the interrupt */
 269         case RTE_INTR_HANDLE_UIO:
 270                 if (write(intr_handle->fd, &value, sizeof(value)) < 0) {
 271                         RTE_LOG(ERR, EAL,
 272                                 "Error enabling interrupts for fd %d\n",
 273                                                         intr_handle->fd);
 274                         return -1;
 275                 }
 276                 break;
 277         /* not used at this moment */
 278         case RTE_INTR_HANDLE_ALARM:
 279                 return -1;
 280         /* unkown handle type */
 281         default:
 282                 RTE_LOG(ERR, EAL,
 283                         "Unknown handle type of fd %d\n",
 284                                         intr_handle->fd);
 285                 return -1;
 286         }
 287
 288         return 0;
 289 }
 290
 291 int
 292 rte_intr_disable(struct rte_intr_handle *intr_handle)
 293 {
 294         const int value = 0;
 295
 296         if (!intr_handle || intr_handle->fd < 0)
 297                 return -1;
 298
 299         switch (intr_handle->type){
 300         /* write to the uio fd to disable the interrupt */
 301         case RTE_INTR_HANDLE_UIO:
 302                 if (write(intr_handle->fd, &value, sizeof(value)) < 0){
 303                         RTE_LOG(ERR, EAL,
 304                                 "Error enabling interrupts for fd %d\n",
 305                                                         intr_handle->fd);
 306                         return -1;
 307                 }
 308                 break;
 309         /* not used at this moment */
 310         case RTE_INTR_HANDLE_ALARM:
 311                 return -1;
 312         /* unkown handle type */
 313         default:
 314                 RTE_LOG(ERR, EAL,
 315                         "Unknown handle type of fd %d\n",
 316                                         intr_handle->fd);
 317                 return -1;
 318         }
 319
 320         return 0;
 321 }
 322
 323 static int
 324 eal_intr_process_interrupts(struct epoll_event *events, int nfds)
 325 {
 326         int n, bytes_read;
 327         struct rte_intr_source *src;
 328         struct rte_intr_callback *cb;
 329         union rte_intr_read_buffer buf;
 330         struct rte_intr_callback active_cb;
 331
 332         for (n = 0; n < nfds; n++) {
 333
 334                 /**
 335                  * if the pipe fd is ready to read, return out to
 336                  * rebuild the wait list.
 337                  */
 338                 if (events[n].data.fd == intr_pipe.readfd){
 339                         int r = read(intr_pipe.readfd, buf.charbuf,
 340                                         sizeof(buf.charbuf));
 341                         RTE_SET_USED(r);
 342                         return -1;
 343                 }
 344                 rte_spinlock_lock(&intr_lock);
 345                 TAILQ_FOREACH(src, &intr_sources, next)
 346                         if (src->intr_handle.fd ==
 347                                         events[n].data.fd)
 348                                 break;
 349                 if (src == NULL){
 350                         rte_spinlock_unlock(&intr_lock);
 351                         continue;
 352                 }
 353
 354                 /* mark this interrupt source as active and release the lock. */
 355                 src->active = 1;
 356                 rte_spinlock_unlock(&intr_lock);
 357
 358                 /* set the length to be read dor different handle type */
 359                 switch (src->intr_handle.type) {
 360                 case RTE_INTR_HANDLE_UIO:
 361                         bytes_read = 4;
 362                         break;
 363                 case RTE_INTR_HANDLE_ALARM:
 364                         bytes_read = sizeof(uint64_t);
 365                         break;
 366                 default:
 367                         bytes_read = 1;
 368                         break;
 369                 }
 370
 371                 /**
 372                  * read out to clear the ready-to-be-read flag
 373                  * for epoll_wait.
 374                  */
 375                 bytes_read = read(events[n].data.fd, &buf, bytes_read);
 376
 377                 if (bytes_read < 0) {
 378                         RTE_LOG(ERR, EAL, "Error reading from file descriptor"
 379                                 " %d, error: %d\n", events[n].data.fd, errno);
 380                 }
 381                 else if (bytes_read == 0) {
 382                         RTE_LOG(ERR, EAL,
 383                                 "Read nothing from file descriptor %d.\n",
 384                                                         events[n].data.fd);
 385                 }
 386
 387                 /* grab a lock, again to call callbacks and update status. */
 388                 rte_spinlock_lock(&intr_lock);
 389
 390                 if (bytes_read > 0) {
 391
 392                         /* Finally, call all callbacks. */
 393                         TAILQ_FOREACH(cb, &src->callbacks, next) {
 394
 395                                 /* make a copy and unlock. */
 396                                 active_cb = *cb;
 397                                 rte_spinlock_unlock(&intr_lock);
 398
 399                                 /* call the actual callback */
 400                                 active_cb.cb_fn(&src->intr_handle,
 401                                         active_cb.cb_arg);
 402
 403                                 /*get the lcok back. */
 404                                 rte_spinlock_lock(&intr_lock);
 405                         }
 406                 }
 407
 408                 /* we done with that interrupt source, release it. */
 409                 src->active = 0;
 410                 rte_spinlock_unlock(&intr_lock);
 411         }
 412
 413         return 0;
 414 }
 415
 416 /**
 417  * It handles all the interrupts.
 418  *
 419  * @param pfd
 420  *  epoll file descriptor.
 421  * @param totalfds
 422  *  The number of file descriptors added in epoll.
 423  *
 424  * @return
 425  *  void
 426  */
 427 static void
 428 eal_intr_handle_interrupts(int pfd, unsigned totalfds)
 429 {
 430         struct epoll_event events[totalfds];
 431         int nfds = 0;
 432
 433         for(;;) {
 434                 nfds = epoll_wait(pfd, events, totalfds,
 435                         EAL_INTR_EPOLL_WAIT_FOREVER);
 436                 /* epoll_wait fail */
 437                 if (nfds < 0) {
 438                         if (errno == EINTR)
 439                                 continue;
 440                         RTE_LOG(ERR, EAL,
 441                                 "epoll_wait returns with fail\n");
 442                         return;
 443                 }
 444                 /* epoll_wait timeout, will never happens here */
 445                 else if (nfds == 0)
 446                         continue;
 447                 /* epoll_wait has at least one fd ready to read */
 448                 if (eal_intr_process_interrupts(events, nfds) < 0)
 449                         return;
 450         }
 451 }
 452
 453 /**
 454  * It builds/rebuilds up the epoll file descriptor with all the
 455  * file descriptors being waited on. Then handles the interrupts.
 456  *
 457  * @param arg
 458  *  pointer. (unused)
 459  *
 460  * @return
 461  *  never return;
 462  */
 463 static __attribute__((noreturn)) void *
 464 eal_intr_thread_main(__rte_unused void *arg)
 465 {
 466         struct epoll_event ev;
 467
 468         /* host thread, never break out */
 469         for (;;) {
 470                 /* build up the epoll fd with all descriptors we are to
 471                  * wait on then pass it to the handle_interrupts function
 472                  */
 473                 static struct epoll_event pipe_event = {
 474                         .events = EPOLLIN | EPOLLPRI,
 475                 };
 476                 struct rte_intr_source *src;
 477                 unsigned numfds = 0;
 478
 479                 /* create epoll fd */
 480                 int pfd = epoll_create(1);
 481                 if (pfd < 0)
 482                         rte_panic("Cannot create epoll instance\n");
 483
 484                 pipe_event.data.fd = intr_pipe.readfd;
 485                 /**
 486                  * add pipe fd into wait list, this pipe is used to
 487                  * rebuild the wait list.
 488                  */
 489                 if (epoll_ctl(pfd, EPOLL_CTL_ADD, intr_pipe.readfd,
 490                                                 &pipe_event) < 0) {
 491                         rte_panic("Error adding fd to %d epoll_ctl, %s\n",
 492                                         intr_pipe.readfd, strerror(errno));
 493                 }
 494                 numfds++;
 495
 496                 rte_spinlock_lock(&intr_lock);
 497
 498                 TAILQ_FOREACH(src, &intr_sources, next) {
 499                         if (src->callbacks.tqh_first == NULL)
 500                                 continue; /* skip those with no callbacks */
 501                         ev.events = EPOLLIN | EPOLLPRI;
 502                         ev.data.fd = src->intr_handle.fd;
 503
 504                         /**
 505                          * add all the uio device file descriptor
 506                          * into wait list.
 507                          */
 508                         if (epoll_ctl(pfd, EPOLL_CTL_ADD,
 509                                         src->intr_handle.fd, &ev) < 0){
 510                                 rte_panic("Error adding fd %d epoll_ctl, %s\n",
 511                                         src->intr_handle.fd, strerror(errno));
 512                         }
 513                         else
 514                                 numfds++;
 515                 }
 516                 rte_spinlock_unlock(&intr_lock);
 517                 /* serve the interrupt */
 518                 eal_intr_handle_interrupts(pfd, numfds);
 519
 520                 /**
 521                  * when we return, we need to rebuild the
 522                  * list of fds to monitor.
 523                  */
 524                 close(pfd);
 525         }
 526 }
 527
 528 int
 529 rte_eal_intr_init(void)
 530 {
 531         int ret = 0;
 532
 533         /* init the global interrupt source head */
 534         TAILQ_INIT(&intr_sources);
 535
 536         /**
 537          * create a pipe which will be waited by epoll and notified to
 538          * rebuild the wait list of epoll.
 539          */
 540         if (pipe(intr_pipe.pipefd) < 0)
 541                 return -1;
 542
 543         /* create the host thread to wait/handle the interrupt */
 544         ret = pthread_create(&intr_thread, NULL,
 545                         eal_intr_thread_main, NULL);
 546         if (ret != 0)
 547                 RTE_LOG(ERR, EAL,
 548                         "Failed to create thread for interrupt handling\n");
 549
 550         return -ret;
 551 }
 552