drivers/net/failsafe/failsafe_intr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 /**
   6  * @file
   7  * Interrupts handling for failsafe driver.
   8  */
   9
  10 #if defined(LINUX)
  11 #include <sys/epoll.h>
  12 #endif
  13 #include <unistd.h>
  14
  15 #include <rte_alarm.h>
  16 #include <rte_config.h>
  17 #include <rte_errno.h>
  18 #include <rte_ethdev.h>
  19 #include <rte_interrupts.h>
  20 #include <rte_io.h>
  21 #include <rte_service_component.h>
  22
  23 #include "failsafe_private.h"
  24
  25 #define NUM_RX_PROXIES (FAILSAFE_MAX_ETHPORTS * RTE_MAX_RXTX_INTR_VEC_ID)
  26
  27
  28 /**
  29  * Open an epoll file descriptor.
  30  *
  31  * @param flags
  32  *   Flags for defining epoll behavior.
  33  * @return
  34  *   0 on success, negative errno value otherwise.
  35  */
  36 static int
  37 fs_epoll_create1(int flags)
  38 {
  39 #if defined(LINUX)
  40         return epoll_create1(flags);
  41 #elif defined(BSD)
  42         RTE_SET_USED(flags);
  43         return -ENOTSUP;
  44 #endif
  45 }
  46
  47 /**
  48  * Install failsafe Rx event proxy service.
  49  * The Rx event proxy is the service that listens to Rx events from the
  50  * subdevices and triggers failsafe Rx events accordingly.
  51  *
  52  * @param priv
  53  *   Pointer to failsafe private structure.
  54  * @return
  55  *   0 on success, negative errno value otherwise.
  56  */
  57 static int
  58 fs_rx_event_proxy_routine(void *data)
  59 {
  60         struct fs_priv *priv;
  61         struct rxq *rxq;
  62         struct rte_epoll_event *events;
  63         uint64_t u64;
  64         int i, n;
  65         int rc = 0;
  66
  67         u64 = 1;
  68         priv = data;
  69         events = priv->rxp.evec;
  70         n = rte_epoll_wait(priv->rxp.efd, events, NUM_RX_PROXIES, -1);
  71         for (i = 0; i < n; i++) {
  72                 rxq = events[i].epdata.data;
  73                 if (rxq->enable_events && rxq->event_fd != -1) {
  74                         if (write(rxq->event_fd, &u64, sizeof(u64)) !=
  75                             sizeof(u64)) {
  76                                 ERROR("Failed to proxy Rx event to socket %d",
  77                                        rxq->event_fd);
  78                                 rc = -EIO;
  79                         }
  80                 }
  81         }
  82         return rc;
  83 }
  84
  85 /**
  86  * Uninstall failsafe Rx event proxy service.
  87  *
  88  * @param priv
  89  *   Pointer to failsafe private structure.
  90  */
  91 static void
  92 fs_rx_event_proxy_service_uninstall(struct fs_priv *priv)
  93 {
  94         /* Unregister the event service. */
  95         switch (priv->rxp.sstate) {
  96         case SS_RUNNING:
  97                 rte_service_map_lcore_set(priv->rxp.sid, priv->rxp.scid, 0);
  98                 /* fall through */
  99         case SS_READY:
 100                 rte_service_runstate_set(priv->rxp.sid, 0);
 101                 rte_service_set_stats_enable(priv->rxp.sid, 0);
 102                 rte_service_component_runstate_set(priv->rxp.sid, 0);
 103                 /* fall through */
 104         case SS_REGISTERED:
 105                 rte_service_component_unregister(priv->rxp.sid);
 106                 /* fall through */
 107         default:
 108                 break;
 109         }
 110 }
 111
 112 /**
 113  * Install the failsafe Rx event proxy service.
 114  *
 115  * @param priv
 116  *   Pointer to failsafe private structure.
 117  * @return
 118  *   0 on success, negative errno value otherwise.
 119  */
 120 static int
 121 fs_rx_event_proxy_service_install(struct fs_priv *priv)
 122 {
 123         struct rte_service_spec service;
 124         int32_t num_service_cores;
 125         int ret = 0;
 126
 127         num_service_cores = rte_service_lcore_count();
 128         if (num_service_cores <= 0) {
 129                 ERROR("Failed to install Rx interrupts, "
 130                       "no service core found");
 131                 return -ENOTSUP;
 132         }
 133         /* prepare service info */
 134         memset(&service, 0, sizeof(struct rte_service_spec));
 135         snprintf(service.name, sizeof(service.name), "%s_Rx_service",
 136                  priv->data->name);
 137         service.socket_id = priv->data->numa_node;
 138         service.callback = fs_rx_event_proxy_routine;
 139         service.callback_userdata = priv;
 140
 141         if (priv->rxp.sstate == SS_NO_SERVICE) {
 142                 uint32_t service_core_list[num_service_cores];
 143
 144                 /* get a service core to work with */
 145                 ret = rte_service_lcore_list(service_core_list,
 146                                              num_service_cores);
 147                 if (ret <= 0) {
 148                         ERROR("Failed to install Rx interrupts, "
 149                               "service core list empty or corrupted");
 150                         return -ENOTSUP;
 151                 }
 152                 priv->rxp.scid = service_core_list[0];
 153                 ret = rte_service_lcore_add(priv->rxp.scid);
 154                 if (ret && ret != -EALREADY) {
 155                         ERROR("Failed adding service core");
 156                         return ret;
 157                 }
 158                 /* service core may be in "stopped" state, start it */
 159                 ret = rte_service_lcore_start(priv->rxp.scid);
 160                 if (ret && (ret != -EALREADY)) {
 161                         ERROR("Failed to install Rx interrupts, "
 162                               "service core not started");
 163                         return ret;
 164                 }
 165                 /* register our service */
 166                 int32_t ret = rte_service_component_register(&service,
 167                                                              &priv->rxp.sid);
 168                 if (ret) {
 169                         ERROR("service register() failed");
 170                         return -ENOEXEC;
 171                 }
 172                 priv->rxp.sstate = SS_REGISTERED;
 173                 /* run the service */
 174                 ret = rte_service_component_runstate_set(priv->rxp.sid, 1);
 175                 if (ret < 0) {
 176                         ERROR("Failed Setting component runstate\n");
 177                         return ret;
 178                 }
 179                 ret = rte_service_set_stats_enable(priv->rxp.sid, 1);
 180                 if (ret < 0) {
 181                         ERROR("Failed enabling stats\n");
 182                         return ret;
 183                 }
 184                 ret = rte_service_runstate_set(priv->rxp.sid, 1);
 185                 if (ret < 0) {
 186                         ERROR("Failed to run service\n");
 187                         return ret;
 188                 }
 189                 priv->rxp.sstate = SS_READY;
 190                 /* map the service with the service core */
 191                 ret = rte_service_map_lcore_set(priv->rxp.sid,
 192                                                 priv->rxp.scid, 1);
 193                 if (ret) {
 194                         ERROR("Failed to install Rx interrupts, "
 195                               "could not map service core");
 196                         return ret;
 197                 }
 198                 priv->rxp.sstate = SS_RUNNING;
 199         }
 200         return 0;
 201 }
 202
 203 /**
 204  * Install failsafe Rx event proxy subsystem.
 205  * This is the way the failsafe PMD generates Rx events on behalf of its
 206  * subdevices.
 207  *
 208  * @param priv
 209  *   Pointer to failsafe private structure.
 210  * @return
 211  *   0 on success, negative errno value otherwise and rte_errno is set.
 212  */
 213 static int
 214 fs_rx_event_proxy_install(struct fs_priv *priv)
 215 {
 216         int rc = 0;
 217
 218         /*
 219          * Create the epoll fd and event vector for the proxy service to
 220          * wait on for Rx events generated by the subdevices.
 221          */
 222         priv->rxp.efd = fs_epoll_create1(0);
 223         if (priv->rxp.efd < 0) {
 224                 rte_errno = errno;
 225                 ERROR("Failed to create epoll,"
 226                       " Rx interrupts will not be supported");
 227                 return -rte_errno;
 228         }
 229         priv->rxp.evec = calloc(NUM_RX_PROXIES, sizeof(*priv->rxp.evec));
 230         if (priv->rxp.evec == NULL) {
 231                 ERROR("Failed to allocate memory for event vectors,"
 232                       " Rx interrupts will not be supported");
 233                 rc = -ENOMEM;
 234                 goto error;
 235         }
 236         rc = fs_rx_event_proxy_service_install(priv);
 237         if (rc < 0)
 238                 goto error;
 239         return 0;
 240 error:
 241         if (priv->rxp.efd >= 0) {
 242                 close(priv->rxp.efd);
 243                 priv->rxp.efd = -1;
 244         }
 245         if (priv->rxp.evec != NULL) {
 246                 free(priv->rxp.evec);
 247                 priv->rxp.evec = NULL;
 248         }
 249         rte_errno = -rc;
 250         return rc;
 251 }
 252
 253 /**
 254  * RX Interrupt control per subdevice.
 255  *
 256  * @param sdev
 257  *   Pointer to sub-device structure.
 258  * @param op
 259  *   The operation be performed for the vector.
 260  *   Operation type of {RTE_INTR_EVENT_ADD, RTE_INTR_EVENT_DEL}.
 261  * @return
 262  *   - On success, zero.
 263  *   - On failure, a negative value.
 264  */
 265 static int
 266 failsafe_eth_rx_intr_ctl_subdevice(struct sub_device *sdev, int op)
 267 {
 268         struct rte_eth_dev *dev;
 269         struct rte_eth_dev *fsdev;
 270         int epfd;
 271         uint16_t pid;
 272         uint16_t qid;
 273         struct rxq *fsrxq;
 274         int rc;
 275         int ret = 0;
 276
 277         fsdev = fs_dev(sdev);
 278         if (sdev == NULL || (ETH(sdev) == NULL) ||
 279                 fsdev == NULL || (PRIV(fsdev) == NULL)) {
 280                 ERROR("Called with invalid arguments");
 281                 return -EINVAL;
 282         }
 283         dev = ETH(sdev);
 284         epfd = PRIV(fsdev)->rxp.efd;
 285         pid = PORT_ID(sdev);
 286
 287         if (epfd <= 0) {
 288                 if (op == RTE_INTR_EVENT_ADD) {
 289                         ERROR("Proxy events are not initialized");
 290                         return -EBADF;
 291                 } else {
 292                         return 0;
 293                 }
 294         }
 295         if (dev->data->nb_rx_queues > fsdev->data->nb_rx_queues) {
 296                 ERROR("subdevice has too many queues,"
 297                       " Interrupts will not be enabled");
 298                         return -E2BIG;
 299         }
 300         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
 301                 fsrxq = fsdev->data->rx_queues[qid];
 302                 rc = rte_eth_dev_rx_intr_ctl_q(pid, qid, epfd,
 303                                                op, (void *)fsrxq);
 304                 if (rc) {
 305                         ERROR("rte_eth_dev_rx_intr_ctl_q failed for "
 306                               "port %d  queue %d, epfd %d, error %d",
 307                               pid, qid, epfd, rc);
 308                         ret = rc;
 309                 }
 310         }
 311         return ret;
 312 }
 313
 314 /**
 315  * Install Rx interrupts subsystem for a subdevice.
 316  * This is a support for dynamically adding subdevices.
 317  *
 318  * @param sdev
 319  *   Pointer to subdevice structure.
 320  *
 321  * @return
 322  *   0 on success, negative errno value otherwise and rte_errno is set.
 323  */
 324 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev)
 325 {
 326         int rc;
 327         int qid;
 328         struct rte_eth_dev *fsdev;
 329         struct rxq **rxq;
 330         const struct rte_intr_conf *const intr_conf =
 331                                 &ETH(sdev)->data->dev_conf.intr_conf;
 332
 333         fsdev = fs_dev(sdev);
 334         rxq = (struct rxq **)fsdev->data->rx_queues;
 335         if (intr_conf->rxq == 0)
 336                 return 0;
 337         rc = failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_ADD);
 338         if (rc)
 339                 return rc;
 340         /* enable interrupts on already-enabled queues */
 341         for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) {
 342                 if (rxq[qid]->enable_events) {
 343                         int ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev),
 344                                                              qid);
 345                         if (ret && (ret != -ENOTSUP)) {
 346                                 ERROR("Failed to enable interrupts on "
 347                                       "port %d queue %d", PORT_ID(sdev), qid);
 348                                 rc = ret;
 349                         }
 350                 }
 351         }
 352         return rc;
 353 }
 354
 355 /**
 356  * Uninstall Rx interrupts subsystem for a subdevice.
 357  * This is a support for dynamically removing subdevices.
 358  *
 359  * @param sdev
 360  *   Pointer to subdevice structure.
 361  *
 362  * @return
 363  *   0 on success, negative errno value otherwise and rte_errno is set.
 364  */
 365 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev)
 366 {
 367         int qid;
 368         struct rte_eth_dev *fsdev;
 369         struct rxq *fsrxq;
 370
 371         fsdev = fs_dev(sdev);
 372         for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) {
 373                 if (qid < fsdev->data->nb_rx_queues) {
 374                         fsrxq = fsdev->data->rx_queues[qid];
 375                         if (fsrxq != NULL && fsrxq->enable_events)
 376                                 rte_eth_dev_rx_intr_disable(PORT_ID(sdev),
 377                                                             qid);
 378                 }
 379         }
 380         failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_DEL);
 381 }
 382
 383 /**
 384  * Uninstall failsafe Rx event proxy.
 385  *
 386  * @param priv
 387  *   Pointer to failsafe private structure.
 388  */
 389 static void
 390 fs_rx_event_proxy_uninstall(struct fs_priv *priv)
 391 {
 392         fs_rx_event_proxy_service_uninstall(priv);
 393         if (priv->rxp.evec != NULL) {
 394                 free(priv->rxp.evec);
 395                 priv->rxp.evec = NULL;
 396         }
 397         if (priv->rxp.efd > 0) {
 398                 close(priv->rxp.efd);
 399                 priv->rxp.efd = -1;
 400         }
 401 }
 402
 403 /**
 404  * Uninstall failsafe interrupt vector.
 405  *
 406  * @param priv
 407  *   Pointer to failsafe private structure.
 408  */
 409 static void
 410 fs_rx_intr_vec_uninstall(struct fs_priv *priv)
 411 {
 412         struct rte_intr_handle *intr_handle;
 413
 414         intr_handle = &priv->intr_handle;
 415         if (intr_handle->intr_vec != NULL) {
 416                 free(intr_handle->intr_vec);
 417                 intr_handle->intr_vec = NULL;
 418         }
 419         intr_handle->nb_efd = 0;
 420 }
 421
 422 /**
 423  * Installs failsafe interrupt vector to be registered with EAL later on.
 424  *
 425  * @param priv
 426  *   Pointer to failsafe private structure.
 427  *
 428  * @return
 429  *   0 on success, negative errno value otherwise and rte_errno is set.
 430  */
 431 static int
 432 fs_rx_intr_vec_install(struct fs_priv *priv)
 433 {
 434         unsigned int i;
 435         unsigned int rxqs_n;
 436         unsigned int n;
 437         unsigned int count;
 438         struct rte_intr_handle *intr_handle;
 439
 440         rxqs_n = priv->data->nb_rx_queues;
 441         n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
 442         count = 0;
 443         intr_handle = &priv->intr_handle;
 444         RTE_ASSERT(intr_handle->intr_vec == NULL);
 445         /* Allocate the interrupt vector of the failsafe Rx proxy interrupts */
 446         intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
 447         if (intr_handle->intr_vec == NULL) {
 448                 fs_rx_intr_vec_uninstall(priv);
 449                 rte_errno = ENOMEM;
 450                 ERROR("Failed to allocate memory for interrupt vector,"
 451                       " Rx interrupts will not be supported");
 452                 return -rte_errno;
 453         }
 454         for (i = 0; i < n; i++) {
 455                 struct rxq *rxq = priv->data->rx_queues[i];
 456
 457                 /* Skip queues that cannot request interrupts. */
 458                 if (rxq == NULL || rxq->event_fd < 0) {
 459                         /* Use invalid intr_vec[] index to disable entry. */
 460                         intr_handle->intr_vec[i] =
 461                                 RTE_INTR_VEC_RXTX_OFFSET +
 462                                 RTE_MAX_RXTX_INTR_VEC_ID;
 463                         continue;
 464                 }
 465                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 466                         rte_errno = E2BIG;
 467                         ERROR("Too many Rx queues for interrupt vector size"
 468                               " (%d), Rx interrupts cannot be enabled",
 469                               RTE_MAX_RXTX_INTR_VEC_ID);
 470                         fs_rx_intr_vec_uninstall(priv);
 471                         return -rte_errno;
 472                 }
 473                 intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
 474                 intr_handle->efds[count] = rxq->event_fd;
 475                 count++;
 476         }
 477         if (count == 0) {
 478                 fs_rx_intr_vec_uninstall(priv);
 479         } else {
 480                 intr_handle->nb_efd = count;
 481                 intr_handle->efd_counter_size = sizeof(uint64_t);
 482         }
 483         return 0;
 484 }
 485
 486
 487 /**
 488  * Uninstall failsafe Rx interrupts subsystem.
 489  *
 490  * @param priv
 491  *   Pointer to private structure.
 492  *
 493  * @return
 494  *   0 on success, negative errno value otherwise and rte_errno is set.
 495  */
 496 void
 497 failsafe_rx_intr_uninstall(struct rte_eth_dev *dev)
 498 {
 499         struct fs_priv *priv;
 500         struct rte_intr_handle *intr_handle;
 501
 502         priv = PRIV(dev);
 503         intr_handle = &priv->intr_handle;
 504         rte_intr_free_epoll_fd(intr_handle);
 505         fs_rx_event_proxy_uninstall(priv);
 506         fs_rx_intr_vec_uninstall(priv);
 507         dev->intr_handle = NULL;
 508 }
 509
 510 /**
 511  * Install failsafe Rx interrupts subsystem.
 512  *
 513  * @param priv
 514  *   Pointer to private structure.
 515  *
 516  * @return
 517  *   0 on success, negative errno value otherwise and rte_errno is set.
 518  */
 519 int
 520 failsafe_rx_intr_install(struct rte_eth_dev *dev)
 521 {
 522         struct fs_priv *priv = PRIV(dev);
 523         const struct rte_intr_conf *const intr_conf =
 524                         &priv->data->dev_conf.intr_conf;
 525
 526         if (intr_conf->rxq == 0 || dev->intr_handle != NULL)
 527                 return 0;
 528         if (fs_rx_intr_vec_install(priv) < 0)
 529                 return -rte_errno;
 530         if (fs_rx_event_proxy_install(priv) < 0) {
 531                 fs_rx_intr_vec_uninstall(priv);
 532                 return -rte_errno;
 533         }
 534         dev->intr_handle = &priv->intr_handle;
 535         return 0;
 536 }