drivers/net/failsafe/failsafe_intr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018 Mellanox Technologies, Ltd
   3  */
   4
   5 /**
   6  * @file
   7  * Interrupts handling for failsafe driver.
   8  */
   9
  10 #if defined(LINUX)
  11 #include <sys/epoll.h>
  12 #endif
  13 #include <unistd.h>
  14
  15 #include <rte_alarm.h>
  16 #include <rte_errno.h>
  17 #include <rte_ethdev.h>
  18 #include <rte_interrupts.h>
  19 #include <rte_io.h>
  20 #include <rte_service_component.h>
  21
  22 #include "failsafe_private.h"
  23
  24 #define NUM_RX_PROXIES (FAILSAFE_MAX_ETHPORTS * RTE_MAX_RXTX_INTR_VEC_ID)
  25
  26
  27 /**
  28  * Open an epoll file descriptor.
  29  *
  30  * @param flags
  31  *   Flags for defining epoll behavior.
  32  * @return
  33  *   0 on success, negative errno value otherwise.
  34  */
  35 static int
  36 fs_epoll_create1(int flags)
  37 {
  38 #if defined(LINUX)
  39         return epoll_create1(flags);
  40 #elif defined(BSD)
  41         RTE_SET_USED(flags);
  42         return -ENOTSUP;
  43 #endif
  44 }
  45
  46 /**
  47  * Install failsafe Rx event proxy service.
  48  * The Rx event proxy is the service that listens to Rx events from the
  49  * subdevices and triggers failsafe Rx events accordingly.
  50  *
  51  * @param priv
  52  *   Pointer to failsafe private structure.
  53  * @return
  54  *   0 on success, negative errno value otherwise.
  55  */
  56 static int
  57 fs_rx_event_proxy_routine(void *data)
  58 {
  59         struct fs_priv *priv;
  60         struct rxq *rxq;
  61         struct rte_epoll_event *events;
  62         uint64_t u64;
  63         int i, n;
  64         int rc = 0;
  65
  66         u64 = 1;
  67         priv = data;
  68         events = priv->rxp.evec;
  69         n = rte_epoll_wait(priv->rxp.efd, events, NUM_RX_PROXIES, -1);
  70         for (i = 0; i < n; i++) {
  71                 rxq = events[i].epdata.data;
  72                 if (rxq->enable_events && rxq->event_fd != -1) {
  73                         if (write(rxq->event_fd, &u64, sizeof(u64)) !=
  74                             sizeof(u64)) {
  75                                 ERROR("Failed to proxy Rx event to socket %d",
  76                                        rxq->event_fd);
  77                                 rc = -EIO;
  78                         }
  79                 }
  80         }
  81         return rc;
  82 }
  83
  84 /**
  85  * Uninstall failsafe Rx event proxy service.
  86  *
  87  * @param priv
  88  *   Pointer to failsafe private structure.
  89  */
  90 static void
  91 fs_rx_event_proxy_service_uninstall(struct fs_priv *priv)
  92 {
  93         /* Unregister the event service. */
  94         switch (priv->rxp.sstate) {
  95         case SS_RUNNING:
  96                 rte_service_map_lcore_set(priv->rxp.sid, priv->rxp.scid, 0);
  97                 /* fall through */
  98         case SS_READY:
  99                 rte_service_runstate_set(priv->rxp.sid, 0);
 100                 rte_service_set_stats_enable(priv->rxp.sid, 0);
 101                 rte_service_component_runstate_set(priv->rxp.sid, 0);
 102                 /* fall through */
 103         case SS_REGISTERED:
 104                 rte_service_component_unregister(priv->rxp.sid);
 105                 /* fall through */
 106         default:
 107                 break;
 108         }
 109 }
 110
 111 /**
 112  * Install the failsafe Rx event proxy service.
 113  *
 114  * @param priv
 115  *   Pointer to failsafe private structure.
 116  * @return
 117  *   0 on success, negative errno value otherwise.
 118  */
 119 static int
 120 fs_rx_event_proxy_service_install(struct fs_priv *priv)
 121 {
 122         struct rte_service_spec service;
 123         int32_t num_service_cores;
 124         int ret = 0;
 125
 126         num_service_cores = rte_service_lcore_count();
 127         if (num_service_cores <= 0) {
 128                 ERROR("Failed to install Rx interrupts, "
 129                       "no service core found");
 130                 return -ENOTSUP;
 131         }
 132         /* prepare service info */
 133         memset(&service, 0, sizeof(struct rte_service_spec));
 134         snprintf(service.name, sizeof(service.name), "%s_Rx_service",
 135                  priv->data->name);
 136         service.socket_id = priv->data->numa_node;
 137         service.callback = fs_rx_event_proxy_routine;
 138         service.callback_userdata = priv;
 139
 140         if (priv->rxp.sstate == SS_NO_SERVICE) {
 141                 uint32_t service_core_list[num_service_cores];
 142
 143                 /* get a service core to work with */
 144                 ret = rte_service_lcore_list(service_core_list,
 145                                              num_service_cores);
 146                 if (ret <= 0) {
 147                         ERROR("Failed to install Rx interrupts, "
 148                               "service core list empty or corrupted");
 149                         return -ENOTSUP;
 150                 }
 151                 priv->rxp.scid = service_core_list[0];
 152                 ret = rte_service_lcore_add(priv->rxp.scid);
 153                 if (ret && ret != -EALREADY) {
 154                         ERROR("Failed adding service core");
 155                         return ret;
 156                 }
 157                 /* service core may be in "stopped" state, start it */
 158                 ret = rte_service_lcore_start(priv->rxp.scid);
 159                 if (ret && (ret != -EALREADY)) {
 160                         ERROR("Failed to install Rx interrupts, "
 161                               "service core not started");
 162                         return ret;
 163                 }
 164                 /* register our service */
 165                 int32_t ret = rte_service_component_register(&service,
 166                                                              &priv->rxp.sid);
 167                 if (ret) {
 168                         ERROR("service register() failed");
 169                         return -ENOEXEC;
 170                 }
 171                 priv->rxp.sstate = SS_REGISTERED;
 172                 /* run the service */
 173                 ret = rte_service_component_runstate_set(priv->rxp.sid, 1);
 174                 if (ret < 0) {
 175                         ERROR("Failed Setting component runstate\n");
 176                         return ret;
 177                 }
 178                 ret = rte_service_set_stats_enable(priv->rxp.sid, 1);
 179                 if (ret < 0) {
 180                         ERROR("Failed enabling stats\n");
 181                         return ret;
 182                 }
 183                 ret = rte_service_runstate_set(priv->rxp.sid, 1);
 184                 if (ret < 0) {
 185                         ERROR("Failed to run service\n");
 186                         return ret;
 187                 }
 188                 priv->rxp.sstate = SS_READY;
 189                 /* map the service with the service core */
 190                 ret = rte_service_map_lcore_set(priv->rxp.sid,
 191                                                 priv->rxp.scid, 1);
 192                 if (ret) {
 193                         ERROR("Failed to install Rx interrupts, "
 194                               "could not map service core");
 195                         return ret;
 196                 }
 197                 priv->rxp.sstate = SS_RUNNING;
 198         }
 199         return 0;
 200 }
 201
 202 /**
 203  * Install failsafe Rx event proxy subsystem.
 204  * This is the way the failsafe PMD generates Rx events on behalf of its
 205  * subdevices.
 206  *
 207  * @param priv
 208  *   Pointer to failsafe private structure.
 209  * @return
 210  *   0 on success, negative errno value otherwise and rte_errno is set.
 211  */
 212 static int
 213 fs_rx_event_proxy_install(struct fs_priv *priv)
 214 {
 215         int rc = 0;
 216
 217         /*
 218          * Create the epoll fd and event vector for the proxy service to
 219          * wait on for Rx events generated by the subdevices.
 220          */
 221         priv->rxp.efd = fs_epoll_create1(0);
 222         if (priv->rxp.efd < 0) {
 223                 rte_errno = errno;
 224                 ERROR("Failed to create epoll,"
 225                       " Rx interrupts will not be supported");
 226                 return -rte_errno;
 227         }
 228         priv->rxp.evec = calloc(NUM_RX_PROXIES, sizeof(*priv->rxp.evec));
 229         if (priv->rxp.evec == NULL) {
 230                 ERROR("Failed to allocate memory for event vectors,"
 231                       " Rx interrupts will not be supported");
 232                 rc = -ENOMEM;
 233                 goto error;
 234         }
 235         rc = fs_rx_event_proxy_service_install(priv);
 236         if (rc < 0)
 237                 goto error;
 238         return 0;
 239 error:
 240         if (priv->rxp.efd >= 0) {
 241                 close(priv->rxp.efd);
 242                 priv->rxp.efd = -1;
 243         }
 244         if (priv->rxp.evec != NULL) {
 245                 free(priv->rxp.evec);
 246                 priv->rxp.evec = NULL;
 247         }
 248         rte_errno = -rc;
 249         return rc;
 250 }
 251
 252 /**
 253  * RX Interrupt control per subdevice.
 254  *
 255  * @param sdev
 256  *   Pointer to sub-device structure.
 257  * @param op
 258  *   The operation be performed for the vector.
 259  *   Operation type of {RTE_INTR_EVENT_ADD, RTE_INTR_EVENT_DEL}.
 260  * @return
 261  *   - On success, zero.
 262  *   - On failure, a negative value.
 263  */
 264 static int
 265 failsafe_eth_rx_intr_ctl_subdevice(struct sub_device *sdev, int op)
 266 {
 267         struct rte_eth_dev *dev;
 268         struct rte_eth_dev *fsdev;
 269         int epfd;
 270         uint16_t pid;
 271         uint16_t qid;
 272         struct rxq *fsrxq;
 273         int rc;
 274         int ret = 0;
 275
 276         fsdev = fs_dev(sdev);
 277         if (sdev == NULL || (ETH(sdev) == NULL) ||
 278                 fsdev == NULL || (PRIV(fsdev) == NULL)) {
 279                 ERROR("Called with invalid arguments");
 280                 return -EINVAL;
 281         }
 282         dev = ETH(sdev);
 283         epfd = PRIV(fsdev)->rxp.efd;
 284         pid = PORT_ID(sdev);
 285
 286         if (epfd <= 0) {
 287                 if (op == RTE_INTR_EVENT_ADD) {
 288                         ERROR("Proxy events are not initialized");
 289                         return -EBADF;
 290                 } else {
 291                         return 0;
 292                 }
 293         }
 294         if (dev->data->nb_rx_queues > fsdev->data->nb_rx_queues) {
 295                 ERROR("subdevice has too many queues,"
 296                       " Interrupts will not be enabled");
 297                         return -E2BIG;
 298         }
 299         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
 300                 fsrxq = fsdev->data->rx_queues[qid];
 301                 rc = rte_eth_dev_rx_intr_ctl_q(pid, qid, epfd,
 302                                                op, (void *)fsrxq);
 303                 if (rc) {
 304                         ERROR("rte_eth_dev_rx_intr_ctl_q failed for "
 305                               "port %d  queue %d, epfd %d, error %d",
 306                               pid, qid, epfd, rc);
 307                         ret = rc;
 308                 }
 309         }
 310         return ret;
 311 }
 312
 313 /**
 314  * Install Rx interrupts subsystem for a subdevice.
 315  * This is a support for dynamically adding subdevices.
 316  *
 317  * @param sdev
 318  *   Pointer to subdevice structure.
 319  *
 320  * @return
 321  *   0 on success, negative errno value otherwise and rte_errno is set.
 322  */
 323 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev)
 324 {
 325         int rc;
 326         int qid;
 327         struct rte_eth_dev *fsdev;
 328         struct rxq **rxq;
 329         const struct rte_eth_intr_conf *const intr_conf =
 330                                 &ETH(sdev)->data->dev_conf.intr_conf;
 331
 332         fsdev = fs_dev(sdev);
 333         rxq = (struct rxq **)fsdev->data->rx_queues;
 334         if (intr_conf->rxq == 0)
 335                 return 0;
 336         rc = failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_ADD);
 337         if (rc)
 338                 return rc;
 339         /* enable interrupts on already-enabled queues */
 340         for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) {
 341                 if (rxq[qid]->enable_events) {
 342                         int ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev),
 343                                                              qid);
 344                         if (ret && (ret != -ENOTSUP)) {
 345                                 ERROR("Failed to enable interrupts on "
 346                                       "port %d queue %d", PORT_ID(sdev), qid);
 347                                 rc = ret;
 348                         }
 349                 }
 350         }
 351         return rc;
 352 }
 353
 354 /**
 355  * Uninstall Rx interrupts subsystem for a subdevice.
 356  * This is a support for dynamically removing subdevices.
 357  *
 358  * @param sdev
 359  *   Pointer to subdevice structure.
 360  *
 361  * @return
 362  *   0 on success, negative errno value otherwise and rte_errno is set.
 363  */
 364 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev)
 365 {
 366         int qid;
 367         struct rte_eth_dev *fsdev;
 368         struct rxq *fsrxq;
 369
 370         fsdev = fs_dev(sdev);
 371         for (qid = 0; qid < ETH(sdev)->data->nb_rx_queues; qid++) {
 372                 if (qid < fsdev->data->nb_rx_queues) {
 373                         fsrxq = fsdev->data->rx_queues[qid];
 374                         if (fsrxq != NULL && fsrxq->enable_events)
 375                                 rte_eth_dev_rx_intr_disable(PORT_ID(sdev),
 376                                                             qid);
 377                 }
 378         }
 379         failsafe_eth_rx_intr_ctl_subdevice(sdev, RTE_INTR_EVENT_DEL);
 380 }
 381
 382 /**
 383  * Uninstall failsafe Rx event proxy.
 384  *
 385  * @param priv
 386  *   Pointer to failsafe private structure.
 387  */
 388 static void
 389 fs_rx_event_proxy_uninstall(struct fs_priv *priv)
 390 {
 391         fs_rx_event_proxy_service_uninstall(priv);
 392         if (priv->rxp.evec != NULL) {
 393                 free(priv->rxp.evec);
 394                 priv->rxp.evec = NULL;
 395         }
 396         if (priv->rxp.efd >= 0) {
 397                 close(priv->rxp.efd);
 398                 priv->rxp.efd = -1;
 399         }
 400 }
 401
 402 /**
 403  * Uninstall failsafe interrupt vector.
 404  *
 405  * @param priv
 406  *   Pointer to failsafe private structure.
 407  */
 408 static void
 409 fs_rx_intr_vec_uninstall(struct fs_priv *priv)
 410 {
 411         struct rte_intr_handle *intr_handle;
 412
 413         intr_handle = priv->intr_handle;
 414         rte_intr_vec_list_free(intr_handle);
 415
 416         rte_intr_nb_efd_set(intr_handle, 0);
 417 }
 418
 419 /**
 420  * Installs failsafe interrupt vector to be registered with EAL later on.
 421  *
 422  * @param priv
 423  *   Pointer to failsafe private structure.
 424  *
 425  * @return
 426  *   0 on success, negative errno value otherwise and rte_errno is set.
 427  */
 428 static int
 429 fs_rx_intr_vec_install(struct fs_priv *priv)
 430 {
 431         unsigned int i;
 432         unsigned int rxqs_n;
 433         unsigned int n;
 434         unsigned int count;
 435         struct rte_intr_handle *intr_handle;
 436
 437         rxqs_n = priv->data->nb_rx_queues;
 438         n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
 439         count = 0;
 440         intr_handle = priv->intr_handle;
 441         /* Allocate the interrupt vector of the failsafe Rx proxy interrupts */
 442         if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
 443                 fs_rx_intr_vec_uninstall(priv);
 444                 rte_errno = ENOMEM;
 445                 ERROR("Failed to allocate memory for interrupt vector,"
 446                       " Rx interrupts will not be supported");
 447                 return -rte_errno;
 448         }
 449         for (i = 0; i < n; i++) {
 450                 struct rxq *rxq = priv->data->rx_queues[i];
 451
 452                 /* Skip queues that cannot request interrupts. */
 453                 if (rxq == NULL || rxq->event_fd < 0) {
 454                         /* Use invalid intr_vec[] index to disable entry. */
 455                         if (rte_intr_vec_list_index_set(intr_handle, i,
 456                         RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID))
 457                                 return -rte_errno;
 458                         continue;
 459                 }
 460                 if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
 461                         rte_errno = E2BIG;
 462                         ERROR("Too many Rx queues for interrupt vector size"
 463                               " (%d), Rx interrupts cannot be enabled",
 464                               RTE_MAX_RXTX_INTR_VEC_ID);
 465                         fs_rx_intr_vec_uninstall(priv);
 466                         return -rte_errno;
 467                 }
 468                 if (rte_intr_vec_list_index_set(intr_handle, i,
 469                                         RTE_INTR_VEC_RXTX_OFFSET + count))
 470                         return -rte_errno;
 471
 472                 if (rte_intr_efds_index_set(intr_handle, count,
 473                                                    rxq->event_fd))
 474                         return -rte_errno;
 475                 count++;
 476         }
 477         if (count == 0) {
 478                 fs_rx_intr_vec_uninstall(priv);
 479         } else {
 480                 if (rte_intr_nb_efd_set(intr_handle, count))
 481                         return -rte_errno;
 482
 483                 if (rte_intr_efd_counter_size_set(intr_handle,
 484                                 sizeof(uint64_t)))
 485                         return -rte_errno;
 486         }
 487         return 0;
 488 }
 489
 490
 491 /**
 492  * Uninstall failsafe Rx interrupts subsystem.
 493  *
 494  * @param priv
 495  *   Pointer to private structure.
 496  *
 497  * @return
 498  *   0 on success, negative errno value otherwise and rte_errno is set.
 499  */
 500 void
 501 failsafe_rx_intr_uninstall(struct rte_eth_dev *dev)
 502 {
 503         struct fs_priv *priv;
 504         struct rte_intr_handle *intr_handle;
 505
 506         priv = PRIV(dev);
 507         intr_handle = priv->intr_handle;
 508         rte_intr_free_epoll_fd(intr_handle);
 509         fs_rx_event_proxy_uninstall(priv);
 510         fs_rx_intr_vec_uninstall(priv);
 511         dev->intr_handle = NULL;
 512 }
 513
 514 /**
 515  * Install failsafe Rx interrupts subsystem.
 516  *
 517  * @param priv
 518  *   Pointer to private structure.
 519  *
 520  * @return
 521  *   0 on success, negative errno value otherwise and rte_errno is set.
 522  */
 523 int
 524 failsafe_rx_intr_install(struct rte_eth_dev *dev)
 525 {
 526         struct fs_priv *priv = PRIV(dev);
 527         const struct rte_eth_intr_conf *const intr_conf =
 528                         &priv->data->dev_conf.intr_conf;
 529
 530         if (intr_conf->rxq == 0 || dev->intr_handle != NULL)
 531                 return 0;
 532         if (fs_rx_intr_vec_install(priv) < 0)
 533                 return -rte_errno;
 534         if (fs_rx_event_proxy_install(priv) < 0) {
 535                 fs_rx_intr_vec_uninstall(priv);
 536                 return -rte_errno;
 537         }
 538         dev->intr_handle = priv->intr_handle;
 539         return 0;
 540 }