lib/eventdev/rte_event_timer_adapter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2018 Intel Corporation.
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <inttypes.h>
   8 #include <stdbool.h>
   9 #include <sys/queue.h>
  10
  11 #include <rte_memzone.h>
  12 #include <rte_memory.h>
  13 #include <rte_dev.h>
  14 #include <rte_errno.h>
  15 #include <rte_malloc.h>
  16 #include <rte_ring.h>
  17 #include <rte_mempool.h>
  18 #include <rte_common.h>
  19 #include <rte_timer.h>
  20 #include <rte_service_component.h>
  21 #include <rte_cycles.h>
  22
  23 #include "event_timer_adapter_pmd.h"
  24 #include "eventdev_pmd.h"
  25 #include "rte_event_timer_adapter.h"
  26 #include "rte_eventdev.h"
  27 #include "eventdev_trace.h"
  28
  29 #define DATA_MZ_NAME_MAX_LEN 64
  30 #define DATA_MZ_NAME_FORMAT "rte_event_timer_adapter_data_%d"
  31
  32 RTE_LOG_REGISTER_SUFFIX(evtim_logtype, adapter.timer, NOTICE);
  33 RTE_LOG_REGISTER_SUFFIX(evtim_buffer_logtype, adapter.timer, NOTICE);
  34 RTE_LOG_REGISTER_SUFFIX(evtim_svc_logtype, adapter.timer.svc, NOTICE);
  35
  36 static struct rte_event_timer_adapter *adapters;
  37
  38 static const struct event_timer_adapter_ops swtim_ops;
  39
  40 #define EVTIM_LOG(level, logtype, ...) \
  41         rte_log(RTE_LOG_ ## level, logtype, \
  42                 RTE_FMT("EVTIMER: %s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) \
  43                         "\n", __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,)))
  44
  45 #define EVTIM_LOG_ERR(...) EVTIM_LOG(ERR, evtim_logtype, __VA_ARGS__)
  46
  47 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
  48 #define EVTIM_LOG_DBG(...) \
  49         EVTIM_LOG(DEBUG, evtim_logtype, __VA_ARGS__)
  50 #define EVTIM_BUF_LOG_DBG(...) \
  51         EVTIM_LOG(DEBUG, evtim_buffer_logtype, __VA_ARGS__)
  52 #define EVTIM_SVC_LOG_DBG(...) \
  53         EVTIM_LOG(DEBUG, evtim_svc_logtype, __VA_ARGS__)
  54 #else
  55 #define EVTIM_LOG_DBG(...) (void)0
  56 #define EVTIM_BUF_LOG_DBG(...) (void)0
  57 #define EVTIM_SVC_LOG_DBG(...) (void)0
  58 #endif
  59
  60 static int
  61 default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id,
  62                      void *conf_arg)
  63 {
  64         struct rte_event_timer_adapter *adapter;
  65         struct rte_eventdev *dev;
  66         struct rte_event_dev_config dev_conf;
  67         struct rte_event_port_conf *port_conf, def_port_conf = {0};
  68         int started;
  69         uint8_t port_id;
  70         uint8_t dev_id;
  71         int ret;
  72
  73         RTE_SET_USED(event_dev_id);
  74
  75         adapter = &adapters[id];
  76         dev = &rte_eventdevs[adapter->data->event_dev_id];
  77         dev_id = dev->data->dev_id;
  78         dev_conf = dev->data->dev_conf;
  79
  80         started = dev->data->dev_started;
  81         if (started)
  82                 rte_event_dev_stop(dev_id);
  83
  84         port_id = dev_conf.nb_event_ports;
  85         dev_conf.nb_event_ports += 1;
  86         ret = rte_event_dev_configure(dev_id, &dev_conf);
  87         if (ret < 0) {
  88                 EVTIM_LOG_ERR("failed to configure event dev %u\n", dev_id);
  89                 if (started)
  90                         if (rte_event_dev_start(dev_id))
  91                                 return -EIO;
  92
  93                 return ret;
  94         }
  95
  96         if (conf_arg != NULL)
  97                 port_conf = conf_arg;
  98         else {
  99                 port_conf = &def_port_conf;
 100                 ret = rte_event_port_default_conf_get(dev_id, port_id,
 101                                                       port_conf);
 102                 if (ret < 0)
 103                         return ret;
 104         }
 105
 106         ret = rte_event_port_setup(dev_id, port_id, port_conf);
 107         if (ret < 0) {
 108                 EVTIM_LOG_ERR("failed to setup event port %u on event dev %u\n",
 109                               port_id, dev_id);
 110                 return ret;
 111         }
 112
 113         *event_port_id = port_id;
 114
 115         if (started)
 116                 ret = rte_event_dev_start(dev_id);
 117
 118         return ret;
 119 }
 120
 121 struct rte_event_timer_adapter *
 122 rte_event_timer_adapter_create(const struct rte_event_timer_adapter_conf *conf)
 123 {
 124         return rte_event_timer_adapter_create_ext(conf, default_port_conf_cb,
 125                                                   NULL);
 126 }
 127
 128 struct rte_event_timer_adapter *
 129 rte_event_timer_adapter_create_ext(
 130                 const struct rte_event_timer_adapter_conf *conf,
 131                 rte_event_timer_adapter_port_conf_cb_t conf_cb,
 132                 void *conf_arg)
 133 {
 134         uint16_t adapter_id;
 135         struct rte_event_timer_adapter *adapter;
 136         const struct rte_memzone *mz;
 137         char mz_name[DATA_MZ_NAME_MAX_LEN];
 138         int n, ret;
 139         struct rte_eventdev *dev;
 140
 141         if (adapters == NULL) {
 142                 adapters = rte_zmalloc("Eventdev",
 143                                        sizeof(struct rte_event_timer_adapter) *
 144                                                RTE_EVENT_TIMER_ADAPTER_NUM_MAX,
 145                                        RTE_CACHE_LINE_SIZE);
 146                 if (adapters == NULL) {
 147                         rte_errno = ENOMEM;
 148                         return NULL;
 149                 }
 150         }
 151
 152         if (conf == NULL) {
 153                 rte_errno = EINVAL;
 154                 return NULL;
 155         }
 156
 157         /* Check eventdev ID */
 158         if (!rte_event_pmd_is_valid_dev(conf->event_dev_id)) {
 159                 rte_errno = EINVAL;
 160                 return NULL;
 161         }
 162         dev = &rte_eventdevs[conf->event_dev_id];
 163
 164         adapter_id = conf->timer_adapter_id;
 165
 166         /* Check that adapter_id is in range */
 167         if (adapter_id >= RTE_EVENT_TIMER_ADAPTER_NUM_MAX) {
 168                 rte_errno = EINVAL;
 169                 return NULL;
 170         }
 171
 172         /* Check adapter ID not already allocated */
 173         adapter = &adapters[adapter_id];
 174         if (adapter->allocated) {
 175                 rte_errno = EEXIST;
 176                 return NULL;
 177         }
 178
 179         /* Create shared data area. */
 180         n = snprintf(mz_name, sizeof(mz_name), DATA_MZ_NAME_FORMAT, adapter_id);
 181         if (n >= (int)sizeof(mz_name)) {
 182                 rte_errno = EINVAL;
 183                 return NULL;
 184         }
 185         mz = rte_memzone_reserve(mz_name,
 186                                  sizeof(struct rte_event_timer_adapter_data),
 187                                  conf->socket_id, 0);
 188         if (mz == NULL)
 189                 /* rte_errno set by rte_memzone_reserve */
 190                 return NULL;
 191
 192         adapter->data = mz->addr;
 193         memset(adapter->data, 0, sizeof(struct rte_event_timer_adapter_data));
 194
 195         adapter->data->mz = mz;
 196         adapter->data->event_dev_id = conf->event_dev_id;
 197         adapter->data->id = adapter_id;
 198         adapter->data->socket_id = conf->socket_id;
 199         adapter->data->conf = *conf;  /* copy conf structure */
 200
 201         /* Query eventdev PMD for timer adapter capabilities and ops */
 202         ret = dev->dev_ops->timer_adapter_caps_get(dev,
 203                                                    adapter->data->conf.flags,
 204                                                    &adapter->data->caps,
 205                                                    &adapter->ops);
 206         if (ret < 0) {
 207                 rte_errno = -ret;
 208                 goto free_memzone;
 209         }
 210
 211         if (!(adapter->data->caps &
 212               RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
 213                 FUNC_PTR_OR_NULL_RET_WITH_ERRNO(conf_cb, EINVAL);
 214                 ret = conf_cb(adapter->data->id, adapter->data->event_dev_id,
 215                               &adapter->data->event_port_id, conf_arg);
 216                 if (ret < 0) {
 217                         rte_errno = -ret;
 218                         goto free_memzone;
 219                 }
 220         }
 221
 222         /* If eventdev PMD did not provide ops, use default software
 223          * implementation.
 224          */
 225         if (adapter->ops == NULL)
 226                 adapter->ops = &swtim_ops;
 227
 228         /* Allow driver to do some setup */
 229         FUNC_PTR_OR_NULL_RET_WITH_ERRNO(adapter->ops->init, ENOTSUP);
 230         ret = adapter->ops->init(adapter);
 231         if (ret < 0) {
 232                 rte_errno = -ret;
 233                 goto free_memzone;
 234         }
 235
 236         /* Set fast-path function pointers */
 237         adapter->arm_burst = adapter->ops->arm_burst;
 238         adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
 239         adapter->cancel_burst = adapter->ops->cancel_burst;
 240
 241         adapter->allocated = 1;
 242
 243         rte_eventdev_trace_timer_adapter_create(adapter_id, adapter, conf,
 244                 conf_cb);
 245         return adapter;
 246
 247 free_memzone:
 248         rte_memzone_free(adapter->data->mz);
 249         return NULL;
 250 }
 251
 252 int
 253 rte_event_timer_adapter_get_info(const struct rte_event_timer_adapter *adapter,
 254                 struct rte_event_timer_adapter_info *adapter_info)
 255 {
 256         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 257
 258         if (adapter->ops->get_info)
 259                 /* let driver set values it knows */
 260                 adapter->ops->get_info(adapter, adapter_info);
 261
 262         /* Set common values */
 263         adapter_info->conf = adapter->data->conf;
 264         adapter_info->event_dev_port_id = adapter->data->event_port_id;
 265         adapter_info->caps = adapter->data->caps;
 266
 267         return 0;
 268 }
 269
 270 int
 271 rte_event_timer_adapter_start(const struct rte_event_timer_adapter *adapter)
 272 {
 273         int ret;
 274
 275         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 276         FUNC_PTR_OR_ERR_RET(adapter->ops->start, -EINVAL);
 277
 278         if (adapter->data->started) {
 279                 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already started",
 280                               adapter->data->id);
 281                 return -EALREADY;
 282         }
 283
 284         ret = adapter->ops->start(adapter);
 285         if (ret < 0)
 286                 return ret;
 287
 288         adapter->data->started = 1;
 289         rte_eventdev_trace_timer_adapter_start(adapter);
 290         return 0;
 291 }
 292
 293 int
 294 rte_event_timer_adapter_stop(const struct rte_event_timer_adapter *adapter)
 295 {
 296         int ret;
 297
 298         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 299         FUNC_PTR_OR_ERR_RET(adapter->ops->stop, -EINVAL);
 300
 301         if (adapter->data->started == 0) {
 302                 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already stopped",
 303                               adapter->data->id);
 304                 return 0;
 305         }
 306
 307         ret = adapter->ops->stop(adapter);
 308         if (ret < 0)
 309                 return ret;
 310
 311         adapter->data->started = 0;
 312         rte_eventdev_trace_timer_adapter_stop(adapter);
 313         return 0;
 314 }
 315
 316 struct rte_event_timer_adapter *
 317 rte_event_timer_adapter_lookup(uint16_t adapter_id)
 318 {
 319         char name[DATA_MZ_NAME_MAX_LEN];
 320         const struct rte_memzone *mz;
 321         struct rte_event_timer_adapter_data *data;
 322         struct rte_event_timer_adapter *adapter;
 323         int ret;
 324         struct rte_eventdev *dev;
 325
 326         if (adapters == NULL) {
 327                 adapters = rte_zmalloc("Eventdev",
 328                                        sizeof(struct rte_event_timer_adapter) *
 329                                                RTE_EVENT_TIMER_ADAPTER_NUM_MAX,
 330                                        RTE_CACHE_LINE_SIZE);
 331                 if (adapters == NULL) {
 332                         rte_errno = ENOMEM;
 333                         return NULL;
 334                 }
 335         }
 336
 337         if (adapters[adapter_id].allocated)
 338                 return &adapters[adapter_id]; /* Adapter is already loaded */
 339
 340         snprintf(name, DATA_MZ_NAME_MAX_LEN, DATA_MZ_NAME_FORMAT, adapter_id);
 341         mz = rte_memzone_lookup(name);
 342         if (mz == NULL) {
 343                 rte_errno = ENOENT;
 344                 return NULL;
 345         }
 346
 347         data = mz->addr;
 348
 349         adapter = &adapters[data->id];
 350         adapter->data = data;
 351
 352         dev = &rte_eventdevs[adapter->data->event_dev_id];
 353
 354         /* Query eventdev PMD for timer adapter capabilities and ops */
 355         ret = dev->dev_ops->timer_adapter_caps_get(dev,
 356                                                    adapter->data->conf.flags,
 357                                                    &adapter->data->caps,
 358                                                    &adapter->ops);
 359         if (ret < 0) {
 360                 rte_errno = EINVAL;
 361                 return NULL;
 362         }
 363
 364         /* If eventdev PMD did not provide ops, use default software
 365          * implementation.
 366          */
 367         if (adapter->ops == NULL)
 368                 adapter->ops = &swtim_ops;
 369
 370         /* Set fast-path function pointers */
 371         adapter->arm_burst = adapter->ops->arm_burst;
 372         adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
 373         adapter->cancel_burst = adapter->ops->cancel_burst;
 374
 375         adapter->allocated = 1;
 376
 377         return adapter;
 378 }
 379
 380 int
 381 rte_event_timer_adapter_free(struct rte_event_timer_adapter *adapter)
 382 {
 383         int i, ret;
 384
 385         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 386         FUNC_PTR_OR_ERR_RET(adapter->ops->uninit, -EINVAL);
 387
 388         if (adapter->data->started == 1) {
 389                 EVTIM_LOG_ERR("event timer adapter %"PRIu8" must be stopped "
 390                               "before freeing", adapter->data->id);
 391                 return -EBUSY;
 392         }
 393
 394         /* free impl priv data */
 395         ret = adapter->ops->uninit(adapter);
 396         if (ret < 0)
 397                 return ret;
 398
 399         /* free shared data area */
 400         ret = rte_memzone_free(adapter->data->mz);
 401         if (ret < 0)
 402                 return ret;
 403
 404         adapter->data = NULL;
 405         adapter->allocated = 0;
 406
 407         ret = 0;
 408         for (i = 0; i < RTE_EVENT_TIMER_ADAPTER_NUM_MAX; i++)
 409                 if (adapters[i].allocated)
 410                         ret = adapters[i].allocated;
 411
 412         if (!ret) {
 413                 rte_free(adapters);
 414                 adapters = NULL;
 415         }
 416
 417         rte_eventdev_trace_timer_adapter_free(adapter);
 418         return 0;
 419 }
 420
 421 int
 422 rte_event_timer_adapter_service_id_get(struct rte_event_timer_adapter *adapter,
 423                                        uint32_t *service_id)
 424 {
 425         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 426
 427         if (adapter->data->service_inited && service_id != NULL)
 428                 *service_id = adapter->data->service_id;
 429
 430         return adapter->data->service_inited ? 0 : -ESRCH;
 431 }
 432
 433 int
 434 rte_event_timer_adapter_stats_get(struct rte_event_timer_adapter *adapter,
 435                                   struct rte_event_timer_adapter_stats *stats)
 436 {
 437         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 438         FUNC_PTR_OR_ERR_RET(adapter->ops->stats_get, -EINVAL);
 439         if (stats == NULL)
 440                 return -EINVAL;
 441
 442         return adapter->ops->stats_get(adapter, stats);
 443 }
 444
 445 int
 446 rte_event_timer_adapter_stats_reset(struct rte_event_timer_adapter *adapter)
 447 {
 448         ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
 449         FUNC_PTR_OR_ERR_RET(adapter->ops->stats_reset, -EINVAL);
 450         return adapter->ops->stats_reset(adapter);
 451 }
 452
 453 /*
 454  * Software event timer adapter buffer helper functions
 455  */
 456
 457 #define NSECPERSEC 1E9
 458
 459 /* Optimizations used to index into the buffer require that the buffer size
 460  * be a power of 2.
 461  */
 462 #define EVENT_BUFFER_SZ 4096
 463 #define EVENT_BUFFER_BATCHSZ 32
 464 #define EVENT_BUFFER_MASK (EVENT_BUFFER_SZ - 1)
 465
 466 #define EXP_TIM_BUF_SZ 128
 467
 468 struct event_buffer {
 469         size_t head;
 470         size_t tail;
 471         struct rte_event events[EVENT_BUFFER_SZ];
 472 } __rte_cache_aligned;
 473
 474 static inline bool
 475 event_buffer_full(struct event_buffer *bufp)
 476 {
 477         return (bufp->head - bufp->tail) == EVENT_BUFFER_SZ;
 478 }
 479
 480 static inline bool
 481 event_buffer_batch_ready(struct event_buffer *bufp)
 482 {
 483         return (bufp->head - bufp->tail) >= EVENT_BUFFER_BATCHSZ;
 484 }
 485
 486 static void
 487 event_buffer_init(struct event_buffer *bufp)
 488 {
 489         bufp->head = bufp->tail = 0;
 490         memset(&bufp->events, 0, sizeof(struct rte_event) * EVENT_BUFFER_SZ);
 491 }
 492
 493 static int
 494 event_buffer_add(struct event_buffer *bufp, struct rte_event *eventp)
 495 {
 496         size_t head_idx;
 497         struct rte_event *buf_eventp;
 498
 499         if (event_buffer_full(bufp))
 500                 return -1;
 501
 502         /* Instead of modulus, bitwise AND with mask to get head_idx. */
 503         head_idx = bufp->head & EVENT_BUFFER_MASK;
 504         buf_eventp = &bufp->events[head_idx];
 505         rte_memcpy(buf_eventp, eventp, sizeof(struct rte_event));
 506
 507         /* Wrap automatically when overflow occurs. */
 508         bufp->head++;
 509
 510         return 0;
 511 }
 512
 513 static void
 514 event_buffer_flush(struct event_buffer *bufp, uint8_t dev_id, uint8_t port_id,
 515                    uint16_t *nb_events_flushed,
 516                    uint16_t *nb_events_inv)
 517 {
 518         struct rte_event *events = bufp->events;
 519         size_t head_idx, tail_idx;
 520         uint16_t n = 0;
 521
 522         /* Instead of modulus, bitwise AND with mask to get index. */
 523         head_idx = bufp->head & EVENT_BUFFER_MASK;
 524         tail_idx = bufp->tail & EVENT_BUFFER_MASK;
 525
 526         RTE_ASSERT(head_idx < EVENT_BUFFER_SZ && tail_idx < EVENT_BUFFER_SZ);
 527
 528         /* Determine the largest contigous run we can attempt to enqueue to the
 529          * event device.
 530          */
 531         if (head_idx > tail_idx)
 532                 n = head_idx - tail_idx;
 533         else if (head_idx < tail_idx)
 534                 n = EVENT_BUFFER_SZ - tail_idx;
 535         else if (event_buffer_full(bufp))
 536                 n = EVENT_BUFFER_SZ - tail_idx;
 537         else {
 538                 *nb_events_flushed = 0;
 539                 return;
 540         }
 541
 542         n = RTE_MIN(EVENT_BUFFER_BATCHSZ, n);
 543         *nb_events_inv = 0;
 544
 545         *nb_events_flushed = rte_event_enqueue_burst(dev_id, port_id,
 546                                                      &events[tail_idx], n);
 547         if (*nb_events_flushed != n) {
 548                 if (rte_errno == EINVAL) {
 549                         EVTIM_LOG_ERR("failed to enqueue invalid event - "
 550                                       "dropping it");
 551                         (*nb_events_inv)++;
 552                 } else if (rte_errno == ENOSPC)
 553                         rte_pause();
 554         }
 555
 556         if (*nb_events_flushed > 0)
 557                 EVTIM_BUF_LOG_DBG("enqueued %"PRIu16" timer events to event "
 558                                   "device", *nb_events_flushed);
 559
 560         bufp->tail = bufp->tail + *nb_events_flushed + *nb_events_inv;
 561 }
 562
 563 /*
 564  * Software event timer adapter implementation
 565  */
 566 struct swtim {
 567         /* Identifier of service executing timer management logic. */
 568         uint32_t service_id;
 569         /* The cycle count at which the adapter should next tick */
 570         uint64_t next_tick_cycles;
 571         /* The tick resolution used by adapter instance. May have been
 572          * adjusted from what user requested
 573          */
 574         uint64_t timer_tick_ns;
 575         /* Maximum timeout in nanoseconds allowed by adapter instance. */
 576         uint64_t max_tmo_ns;
 577         /* Buffered timer expiry events to be enqueued to an event device. */
 578         struct event_buffer buffer;
 579         /* Statistics */
 580         struct rte_event_timer_adapter_stats stats;
 581         /* Mempool of timer objects */
 582         struct rte_mempool *tim_pool;
 583         /* Back pointer for convenience */
 584         struct rte_event_timer_adapter *adapter;
 585         /* Identifier of timer data instance */
 586         uint32_t timer_data_id;
 587         /* Track which cores have actually armed a timer */
 588         struct {
 589                 uint16_t v;
 590         } __rte_cache_aligned in_use[RTE_MAX_LCORE];
 591         /* Track which cores' timer lists should be polled */
 592         unsigned int poll_lcores[RTE_MAX_LCORE];
 593         /* The number of lists that should be polled */
 594         int n_poll_lcores;
 595         /* Timers which have expired and can be returned to a mempool */
 596         struct rte_timer *expired_timers[EXP_TIM_BUF_SZ];
 597         /* The number of timers that can be returned to a mempool */
 598         size_t n_expired_timers;
 599 };
 600
 601 static inline struct swtim *
 602 swtim_pmd_priv(const struct rte_event_timer_adapter *adapter)
 603 {
 604         return adapter->data->adapter_priv;
 605 }
 606
 607 static void
 608 swtim_callback(struct rte_timer *tim)
 609 {
 610         struct rte_event_timer *evtim = tim->arg;
 611         struct rte_event_timer_adapter *adapter;
 612         unsigned int lcore = rte_lcore_id();
 613         struct swtim *sw;
 614         uint16_t nb_evs_flushed = 0;
 615         uint16_t nb_evs_invalid = 0;
 616         uint64_t opaque;
 617         int ret;
 618         int n_lcores;
 619
 620         opaque = evtim->impl_opaque[1];
 621         adapter = (struct rte_event_timer_adapter *)(uintptr_t)opaque;
 622         sw = swtim_pmd_priv(adapter);
 623
 624         ret = event_buffer_add(&sw->buffer, &evtim->ev);
 625         if (ret < 0) {
 626                 /* If event buffer is full, put timer back in list with
 627                  * immediate expiry value, so that we process it again on the
 628                  * next iteration.
 629                  */
 630                 ret = rte_timer_alt_reset(sw->timer_data_id, tim, 0, SINGLE,
 631                                           lcore, NULL, evtim);
 632                 if (ret < 0) {
 633                         EVTIM_LOG_DBG("event buffer full, failed to reset "
 634                                       "timer with immediate expiry value");
 635                 } else {
 636                         sw->stats.evtim_retry_count++;
 637                         EVTIM_LOG_DBG("event buffer full, resetting rte_timer "
 638                                       "with immediate expiry value");
 639                 }
 640
 641                 if (unlikely(sw->in_use[lcore].v == 0)) {
 642                         sw->in_use[lcore].v = 1;
 643                         n_lcores = __atomic_fetch_add(&sw->n_poll_lcores, 1,
 644                                                      __ATOMIC_RELAXED);
 645                         __atomic_store_n(&sw->poll_lcores[n_lcores], lcore,
 646                                         __ATOMIC_RELAXED);
 647                 }
 648         } else {
 649                 EVTIM_BUF_LOG_DBG("buffered an event timer expiry event");
 650
 651                 /* Empty the buffer here, if necessary, to free older expired
 652                  * timers only
 653                  */
 654                 if (unlikely(sw->n_expired_timers == EXP_TIM_BUF_SZ)) {
 655                         rte_mempool_put_bulk(sw->tim_pool,
 656                                              (void **)sw->expired_timers,
 657                                              sw->n_expired_timers);
 658                         sw->n_expired_timers = 0;
 659                 }
 660
 661                 sw->expired_timers[sw->n_expired_timers++] = tim;
 662                 sw->stats.evtim_exp_count++;
 663
 664                 __atomic_store_n(&evtim->state, RTE_EVENT_TIMER_NOT_ARMED,
 665                                 __ATOMIC_RELEASE);
 666         }
 667
 668         if (event_buffer_batch_ready(&sw->buffer)) {
 669                 event_buffer_flush(&sw->buffer,
 670                                    adapter->data->event_dev_id,
 671                                    adapter->data->event_port_id,
 672                                    &nb_evs_flushed,
 673                                    &nb_evs_invalid);
 674
 675                 sw->stats.ev_enq_count += nb_evs_flushed;
 676                 sw->stats.ev_inv_count += nb_evs_invalid;
 677         }
 678 }
 679
 680 static __rte_always_inline uint64_t
 681 get_timeout_cycles(struct rte_event_timer *evtim,
 682                    const struct rte_event_timer_adapter *adapter)
 683 {
 684         struct swtim *sw = swtim_pmd_priv(adapter);
 685         uint64_t timeout_ns = evtim->timeout_ticks * sw->timer_tick_ns;
 686         return timeout_ns * rte_get_timer_hz() / NSECPERSEC;
 687 }
 688
 689 /* This function returns true if one or more (adapter) ticks have occurred since
 690  * the last time it was called.
 691  */
 692 static inline bool
 693 swtim_did_tick(struct swtim *sw)
 694 {
 695         uint64_t cycles_per_adapter_tick, start_cycles;
 696         uint64_t *next_tick_cyclesp;
 697
 698         next_tick_cyclesp = &sw->next_tick_cycles;
 699         cycles_per_adapter_tick = sw->timer_tick_ns *
 700                         (rte_get_timer_hz() / NSECPERSEC);
 701         start_cycles = rte_get_timer_cycles();
 702
 703         /* Note: initially, *next_tick_cyclesp == 0, so the clause below will
 704          * execute, and set things going.
 705          */
 706
 707         if (start_cycles >= *next_tick_cyclesp) {
 708                 /* Snap the current cycle count to the preceding adapter tick
 709                  * boundary.
 710                  */
 711                 start_cycles -= start_cycles % cycles_per_adapter_tick;
 712                 *next_tick_cyclesp = start_cycles + cycles_per_adapter_tick;
 713
 714                 return true;
 715         }
 716
 717         return false;
 718 }
 719
 720 /* Check that event timer timeout value is in range */
 721 static __rte_always_inline int
 722 check_timeout(struct rte_event_timer *evtim,
 723               const struct rte_event_timer_adapter *adapter)
 724 {
 725         uint64_t tmo_nsec;
 726         struct swtim *sw = swtim_pmd_priv(adapter);
 727
 728         tmo_nsec = evtim->timeout_ticks * sw->timer_tick_ns;
 729         if (tmo_nsec > sw->max_tmo_ns)
 730                 return -1;
 731         if (tmo_nsec < sw->timer_tick_ns)
 732                 return -2;
 733
 734         return 0;
 735 }
 736
 737 /* Check that event timer event queue sched type matches destination event queue
 738  * sched type
 739  */
 740 static __rte_always_inline int
 741 check_destination_event_queue(struct rte_event_timer *evtim,
 742                               const struct rte_event_timer_adapter *adapter)
 743 {
 744         int ret;
 745         uint32_t sched_type;
 746
 747         ret = rte_event_queue_attr_get(adapter->data->event_dev_id,
 748                                        evtim->ev.queue_id,
 749                                        RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE,
 750                                        &sched_type);
 751
 752         if ((ret == 0 && evtim->ev.sched_type == sched_type) ||
 753             ret == -EOVERFLOW)
 754                 return 0;
 755
 756         return -1;
 757 }
 758
 759 static int
 760 swtim_service_func(void *arg)
 761 {
 762         struct rte_event_timer_adapter *adapter = arg;
 763         struct swtim *sw = swtim_pmd_priv(adapter);
 764         uint16_t nb_evs_flushed = 0;
 765         uint16_t nb_evs_invalid = 0;
 766
 767         if (swtim_did_tick(sw)) {
 768                 rte_timer_alt_manage(sw->timer_data_id,
 769                                      sw->poll_lcores,
 770                                      sw->n_poll_lcores,
 771                                      swtim_callback);
 772
 773                 /* Return expired timer objects back to mempool */
 774                 rte_mempool_put_bulk(sw->tim_pool, (void **)sw->expired_timers,
 775                                      sw->n_expired_timers);
 776                 sw->n_expired_timers = 0;
 777
 778                 event_buffer_flush(&sw->buffer,
 779                                    adapter->data->event_dev_id,
 780                                    adapter->data->event_port_id,
 781                                    &nb_evs_flushed,
 782                                    &nb_evs_invalid);
 783
 784                 sw->stats.ev_enq_count += nb_evs_flushed;
 785                 sw->stats.ev_inv_count += nb_evs_invalid;
 786                 sw->stats.adapter_tick_count++;
 787         }
 788
 789         rte_event_maintain(adapter->data->event_dev_id,
 790                            adapter->data->event_port_id, 0);
 791
 792         return 0;
 793 }
 794
 795 /* The adapter initialization function rounds the mempool size up to the next
 796  * power of 2, so we can take the difference between that value and what the
 797  * user requested, and use the space for caches.  This avoids a scenario where a
 798  * user can't arm the number of timers the adapter was configured with because
 799  * mempool objects have been lost to caches.
 800  *
 801  * nb_actual should always be a power of 2, so we can iterate over the powers
 802  * of 2 to see what the largest cache size we can use is.
 803  */
 804 static int
 805 compute_msg_mempool_cache_size(uint64_t nb_requested, uint64_t nb_actual)
 806 {
 807         int i;
 808         int size;
 809         int cache_size = 0;
 810
 811         for (i = 0;; i++) {
 812                 size = 1 << i;
 813
 814                 if (RTE_MAX_LCORE * size < (int)(nb_actual - nb_requested) &&
 815                     size < RTE_MEMPOOL_CACHE_MAX_SIZE &&
 816                     size <= nb_actual / 1.5)
 817                         cache_size = size;
 818                 else
 819                         break;
 820         }
 821
 822         return cache_size;
 823 }
 824
 825 static int
 826 swtim_init(struct rte_event_timer_adapter *adapter)
 827 {
 828         int i, ret;
 829         struct swtim *sw;
 830         unsigned int flags;
 831         struct rte_service_spec service;
 832
 833         /* Allocate storage for private data area */
 834 #define SWTIM_NAMESIZE 32
 835         char swtim_name[SWTIM_NAMESIZE];
 836         snprintf(swtim_name, SWTIM_NAMESIZE, "swtim_%"PRIu8,
 837                         adapter->data->id);
 838         sw = rte_zmalloc_socket(swtim_name, sizeof(*sw), RTE_CACHE_LINE_SIZE,
 839                         adapter->data->socket_id);
 840         if (sw == NULL) {
 841                 EVTIM_LOG_ERR("failed to allocate space for private data");
 842                 rte_errno = ENOMEM;
 843                 return -1;
 844         }
 845
 846         /* Connect storage to adapter instance */
 847         adapter->data->adapter_priv = sw;
 848         sw->adapter = adapter;
 849
 850         sw->timer_tick_ns = adapter->data->conf.timer_tick_ns;
 851         sw->max_tmo_ns = adapter->data->conf.max_tmo_ns;
 852
 853         /* Create a timer pool */
 854         char pool_name[SWTIM_NAMESIZE];
 855         snprintf(pool_name, SWTIM_NAMESIZE, "swtim_pool_%"PRIu8,
 856                  adapter->data->id);
 857         /* Optimal mempool size is a power of 2 minus one */
 858         uint64_t nb_timers = rte_align64pow2(adapter->data->conf.nb_timers);
 859         int pool_size = nb_timers - 1;
 860         int cache_size = compute_msg_mempool_cache_size(
 861                                 adapter->data->conf.nb_timers, nb_timers);
 862         flags = 0; /* pool is multi-producer, multi-consumer */
 863         sw->tim_pool = rte_mempool_create(pool_name, pool_size,
 864                         sizeof(struct rte_timer), cache_size, 0, NULL, NULL,
 865                         NULL, NULL, adapter->data->socket_id, flags);
 866         if (sw->tim_pool == NULL) {
 867                 EVTIM_LOG_ERR("failed to create timer object mempool");
 868                 rte_errno = ENOMEM;
 869                 goto free_alloc;
 870         }
 871
 872         /* Initialize the variables that track in-use timer lists */
 873         for (i = 0; i < RTE_MAX_LCORE; i++)
 874                 sw->in_use[i].v = 0;
 875
 876         /* Initialize the timer subsystem and allocate timer data instance */
 877         ret = rte_timer_subsystem_init();
 878         if (ret < 0) {
 879                 if (ret != -EALREADY) {
 880                         EVTIM_LOG_ERR("failed to initialize timer subsystem");
 881                         rte_errno = -ret;
 882                         goto free_mempool;
 883                 }
 884         }
 885
 886         ret = rte_timer_data_alloc(&sw->timer_data_id);
 887         if (ret < 0) {
 888                 EVTIM_LOG_ERR("failed to allocate timer data instance");
 889                 rte_errno = -ret;
 890                 goto free_mempool;
 891         }
 892
 893         /* Initialize timer event buffer */
 894         event_buffer_init(&sw->buffer);
 895
 896         sw->adapter = adapter;
 897
 898         /* Register a service component to run adapter logic */
 899         memset(&service, 0, sizeof(service));
 900         snprintf(service.name, RTE_SERVICE_NAME_MAX,
 901                  "swtim_svc_%"PRIu8, adapter->data->id);
 902         service.socket_id = adapter->data->socket_id;
 903         service.callback = swtim_service_func;
 904         service.callback_userdata = adapter;
 905         service.capabilities &= ~(RTE_SERVICE_CAP_MT_SAFE);
 906         ret = rte_service_component_register(&service, &sw->service_id);
 907         if (ret < 0) {
 908                 EVTIM_LOG_ERR("failed to register service %s with id %"PRIu32
 909                               ": err = %d", service.name, sw->service_id,
 910                               ret);
 911
 912                 rte_errno = ENOSPC;
 913                 goto free_mempool;
 914         }
 915
 916         EVTIM_LOG_DBG("registered service %s with id %"PRIu32, service.name,
 917                       sw->service_id);
 918
 919         adapter->data->service_id = sw->service_id;
 920         adapter->data->service_inited = 1;
 921
 922         return 0;
 923 free_mempool:
 924         rte_mempool_free(sw->tim_pool);
 925 free_alloc:
 926         rte_free(sw);
 927         return -1;
 928 }
 929
 930 static void
 931 swtim_free_tim(struct rte_timer *tim, void *arg)
 932 {
 933         struct swtim *sw = arg;
 934
 935         rte_mempool_put(sw->tim_pool, tim);
 936 }
 937
 938 /* Traverse the list of outstanding timers and put them back in the mempool
 939  * before freeing the adapter to avoid leaking the memory.
 940  */
 941 static int
 942 swtim_uninit(struct rte_event_timer_adapter *adapter)
 943 {
 944         int ret;
 945         struct swtim *sw = swtim_pmd_priv(adapter);
 946
 947         /* Free outstanding timers */
 948         rte_timer_stop_all(sw->timer_data_id,
 949                            sw->poll_lcores,
 950                            sw->n_poll_lcores,
 951                            swtim_free_tim,
 952                            sw);
 953
 954         ret = rte_service_component_unregister(sw->service_id);
 955         if (ret < 0) {
 956                 EVTIM_LOG_ERR("failed to unregister service component");
 957                 return ret;
 958         }
 959
 960         rte_mempool_free(sw->tim_pool);
 961         rte_free(sw);
 962         adapter->data->adapter_priv = NULL;
 963
 964         return 0;
 965 }
 966
 967 static inline int32_t
 968 get_mapped_count_for_service(uint32_t service_id)
 969 {
 970         int32_t core_count, i, mapped_count = 0;
 971         uint32_t lcore_arr[RTE_MAX_LCORE];
 972
 973         core_count = rte_service_lcore_list(lcore_arr, RTE_MAX_LCORE);
 974
 975         for (i = 0; i < core_count; i++)
 976                 if (rte_service_map_lcore_get(service_id, lcore_arr[i]) == 1)
 977                         mapped_count++;
 978
 979         return mapped_count;
 980 }
 981
 982 static int
 983 swtim_start(const struct rte_event_timer_adapter *adapter)
 984 {
 985         int mapped_count;
 986         struct swtim *sw = swtim_pmd_priv(adapter);
 987
 988         /* Mapping the service to more than one service core can introduce
 989          * delays while one thread is waiting to acquire a lock, so only allow
 990          * one core to be mapped to the service.
 991          *
 992          * Note: the service could be modified such that it spreads cores to
 993          * poll over multiple service instances.
 994          */
 995         mapped_count = get_mapped_count_for_service(sw->service_id);
 996
 997         if (mapped_count != 1)
 998                 return mapped_count < 1 ? -ENOENT : -ENOTSUP;
 999
1000         return rte_service_component_runstate_set(sw->service_id, 1);
1001 }
1002
1003 static int
1004 swtim_stop(const struct rte_event_timer_adapter *adapter)
1005 {
1006         int ret;
1007         struct swtim *sw = swtim_pmd_priv(adapter);
1008
1009         ret = rte_service_component_runstate_set(sw->service_id, 0);
1010         if (ret < 0)
1011                 return ret;
1012
1013         /* Wait for the service to complete its final iteration */
1014         while (rte_service_may_be_active(sw->service_id))
1015                 rte_pause();
1016
1017         return 0;
1018 }
1019
1020 static void
1021 swtim_get_info(const struct rte_event_timer_adapter *adapter,
1022                 struct rte_event_timer_adapter_info *adapter_info)
1023 {
1024         struct swtim *sw = swtim_pmd_priv(adapter);
1025         adapter_info->min_resolution_ns = sw->timer_tick_ns;
1026         adapter_info->max_tmo_ns = sw->max_tmo_ns;
1027 }
1028
1029 static int
1030 swtim_stats_get(const struct rte_event_timer_adapter *adapter,
1031                 struct rte_event_timer_adapter_stats *stats)
1032 {
1033         struct swtim *sw = swtim_pmd_priv(adapter);
1034         *stats = sw->stats; /* structure copy */
1035         return 0;
1036 }
1037
1038 static int
1039 swtim_stats_reset(const struct rte_event_timer_adapter *adapter)
1040 {
1041         struct swtim *sw = swtim_pmd_priv(adapter);
1042         memset(&sw->stats, 0, sizeof(sw->stats));
1043         return 0;
1044 }
1045
1046 static uint16_t
1047 __swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1048                 struct rte_event_timer **evtims,
1049                 uint16_t nb_evtims)
1050 {
1051         int i, ret;
1052         struct swtim *sw = swtim_pmd_priv(adapter);
1053         uint32_t lcore_id = rte_lcore_id();
1054         struct rte_timer *tim, *tims[nb_evtims];
1055         uint64_t cycles;
1056         int n_lcores;
1057         /* Timer list for this lcore is not in use. */
1058         uint16_t exp_state = 0;
1059         enum rte_event_timer_state n_state;
1060
1061 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1062         /* Check that the service is running. */
1063         if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1064                 rte_errno = EINVAL;
1065                 return 0;
1066         }
1067 #endif
1068
1069         /* Adjust lcore_id if non-EAL thread. Arbitrarily pick the timer list of
1070          * the highest lcore to insert such timers into
1071          */
1072         if (lcore_id == LCORE_ID_ANY)
1073                 lcore_id = RTE_MAX_LCORE - 1;
1074
1075         /* If this is the first time we're arming an event timer on this lcore,
1076          * mark this lcore as "in use"; this will cause the service
1077          * function to process the timer list that corresponds to this lcore.
1078          * The atomic compare-and-swap operation can prevent the race condition
1079          * on in_use flag between multiple non-EAL threads.
1080          */
1081         if (unlikely(__atomic_compare_exchange_n(&sw->in_use[lcore_id].v,
1082                         &exp_state, 1, 0,
1083                         __ATOMIC_RELAXED, __ATOMIC_RELAXED))) {
1084                 EVTIM_LOG_DBG("Adding lcore id = %u to list of lcores to poll",
1085                               lcore_id);
1086                 n_lcores = __atomic_fetch_add(&sw->n_poll_lcores, 1,
1087                                              __ATOMIC_RELAXED);
1088                 __atomic_store_n(&sw->poll_lcores[n_lcores], lcore_id,
1089                                 __ATOMIC_RELAXED);
1090         }
1091
1092         ret = rte_mempool_get_bulk(sw->tim_pool, (void **)tims,
1093                                    nb_evtims);
1094         if (ret < 0) {
1095                 rte_errno = ENOSPC;
1096                 return 0;
1097         }
1098
1099         for (i = 0; i < nb_evtims; i++) {
1100                 n_state = __atomic_load_n(&evtims[i]->state, __ATOMIC_ACQUIRE);
1101                 if (n_state == RTE_EVENT_TIMER_ARMED) {
1102                         rte_errno = EALREADY;
1103                         break;
1104                 } else if (!(n_state == RTE_EVENT_TIMER_NOT_ARMED ||
1105                              n_state == RTE_EVENT_TIMER_CANCELED)) {
1106                         rte_errno = EINVAL;
1107                         break;
1108                 }
1109
1110                 ret = check_timeout(evtims[i], adapter);
1111                 if (unlikely(ret == -1)) {
1112                         __atomic_store_n(&evtims[i]->state,
1113                                         RTE_EVENT_TIMER_ERROR_TOOLATE,
1114                                         __ATOMIC_RELAXED);
1115                         rte_errno = EINVAL;
1116                         break;
1117                 } else if (unlikely(ret == -2)) {
1118                         __atomic_store_n(&evtims[i]->state,
1119                                         RTE_EVENT_TIMER_ERROR_TOOEARLY,
1120                                         __ATOMIC_RELAXED);
1121                         rte_errno = EINVAL;
1122                         break;
1123                 }
1124
1125                 if (unlikely(check_destination_event_queue(evtims[i],
1126                                                            adapter) < 0)) {
1127                         __atomic_store_n(&evtims[i]->state,
1128                                         RTE_EVENT_TIMER_ERROR,
1129                                         __ATOMIC_RELAXED);
1130                         rte_errno = EINVAL;
1131                         break;
1132                 }
1133
1134                 tim = tims[i];
1135                 rte_timer_init(tim);
1136
1137                 evtims[i]->impl_opaque[0] = (uintptr_t)tim;
1138                 evtims[i]->impl_opaque[1] = (uintptr_t)adapter;
1139
1140                 cycles = get_timeout_cycles(evtims[i], adapter);
1141                 ret = rte_timer_alt_reset(sw->timer_data_id, tim, cycles,
1142                                           SINGLE, lcore_id, NULL, evtims[i]);
1143                 if (ret < 0) {
1144                         /* tim was in RUNNING or CONFIG state */
1145                         __atomic_store_n(&evtims[i]->state,
1146                                         RTE_EVENT_TIMER_ERROR,
1147                                         __ATOMIC_RELEASE);
1148                         break;
1149                 }
1150
1151                 EVTIM_LOG_DBG("armed an event timer");
1152                 /* RELEASE ordering guarantees the adapter specific value
1153                  * changes observed before the update of state.
1154                  */
1155                 __atomic_store_n(&evtims[i]->state, RTE_EVENT_TIMER_ARMED,
1156                                 __ATOMIC_RELEASE);
1157         }
1158
1159         if (i < nb_evtims)
1160                 rte_mempool_put_bulk(sw->tim_pool,
1161                                      (void **)&tims[i], nb_evtims - i);
1162
1163         return i;
1164 }
1165
1166 static uint16_t
1167 swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1168                 struct rte_event_timer **evtims,
1169                 uint16_t nb_evtims)
1170 {
1171         return __swtim_arm_burst(adapter, evtims, nb_evtims);
1172 }
1173
1174 static uint16_t
1175 swtim_cancel_burst(const struct rte_event_timer_adapter *adapter,
1176                    struct rte_event_timer **evtims,
1177                    uint16_t nb_evtims)
1178 {
1179         int i, ret;
1180         struct rte_timer *timp;
1181         uint64_t opaque;
1182         struct swtim *sw = swtim_pmd_priv(adapter);
1183         enum rte_event_timer_state n_state;
1184
1185 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1186         /* Check that the service is running. */
1187         if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1188                 rte_errno = EINVAL;
1189                 return 0;
1190         }
1191 #endif
1192
1193         for (i = 0; i < nb_evtims; i++) {
1194                 /* Don't modify the event timer state in these cases */
1195                 /* ACQUIRE ordering guarantees the access of implementation
1196                  * specific opaque data under the correct state.
1197                  */
1198                 n_state = __atomic_load_n(&evtims[i]->state, __ATOMIC_ACQUIRE);
1199                 if (n_state == RTE_EVENT_TIMER_CANCELED) {
1200                         rte_errno = EALREADY;
1201                         break;
1202                 } else if (n_state != RTE_EVENT_TIMER_ARMED) {
1203                         rte_errno = EINVAL;
1204                         break;
1205                 }
1206
1207                 opaque = evtims[i]->impl_opaque[0];
1208                 timp = (struct rte_timer *)(uintptr_t)opaque;
1209                 RTE_ASSERT(timp != NULL);
1210
1211                 ret = rte_timer_alt_stop(sw->timer_data_id, timp);
1212                 if (ret < 0) {
1213                         /* Timer is running or being configured */
1214                         rte_errno = EAGAIN;
1215                         break;
1216                 }
1217
1218                 rte_mempool_put(sw->tim_pool, (void **)timp);
1219
1220                 /* The RELEASE ordering here pairs with atomic ordering
1221                  * to make sure the state update data observed between
1222                  * threads.
1223                  */
1224                 __atomic_store_n(&evtims[i]->state, RTE_EVENT_TIMER_CANCELED,
1225                                 __ATOMIC_RELEASE);
1226         }
1227
1228         return i;
1229 }
1230
1231 static uint16_t
1232 swtim_arm_tmo_tick_burst(const struct rte_event_timer_adapter *adapter,
1233                          struct rte_event_timer **evtims,
1234                          uint64_t timeout_ticks,
1235                          uint16_t nb_evtims)
1236 {
1237         int i;
1238
1239         for (i = 0; i < nb_evtims; i++)
1240                 evtims[i]->timeout_ticks = timeout_ticks;
1241
1242         return __swtim_arm_burst(adapter, evtims, nb_evtims);
1243 }
1244
1245 static const struct event_timer_adapter_ops swtim_ops = {
1246         .init = swtim_init,
1247         .uninit = swtim_uninit,
1248         .start = swtim_start,
1249         .stop = swtim_stop,
1250         .get_info = swtim_get_info,
1251         .stats_get = swtim_stats_get,
1252         .stats_reset = swtim_stats_reset,
1253         .arm_burst = swtim_arm_burst,
1254         .arm_tmo_tick_burst = swtim_arm_tmo_tick_burst,
1255         .cancel_burst = swtim_cancel_burst,
1256 };