1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation.
11 #include <rte_memzone.h>
12 #include <rte_memory.h>
14 #include <rte_errno.h>
15 #include <rte_malloc.h>
17 #include <rte_mempool.h>
18 #include <rte_common.h>
19 #include <rte_timer.h>
20 #include <rte_service_component.h>
21 #include <rte_cycles.h>
23 #include "event_timer_adapter_pmd.h"
24 #include "eventdev_pmd.h"
25 #include "rte_event_timer_adapter.h"
26 #include "rte_eventdev.h"
27 #include "eventdev_trace.h"
29 #define DATA_MZ_NAME_MAX_LEN 64
30 #define DATA_MZ_NAME_FORMAT "rte_event_timer_adapter_data_%d"
32 RTE_LOG_REGISTER_SUFFIX(evtim_logtype, adapter.timer, NOTICE);
33 RTE_LOG_REGISTER_SUFFIX(evtim_buffer_logtype, adapter.timer, NOTICE);
34 RTE_LOG_REGISTER_SUFFIX(evtim_svc_logtype, adapter.timer.svc, NOTICE);
36 static struct rte_event_timer_adapter *adapters;
38 static const struct event_timer_adapter_ops swtim_ops;
40 #define EVTIM_LOG(level, logtype, ...) \
41 rte_log(RTE_LOG_ ## level, logtype, \
42 RTE_FMT("EVTIMER: %s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) \
43 "\n", __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,)))
45 #define EVTIM_LOG_ERR(...) EVTIM_LOG(ERR, evtim_logtype, __VA_ARGS__)
47 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
48 #define EVTIM_LOG_DBG(...) \
49 EVTIM_LOG(DEBUG, evtim_logtype, __VA_ARGS__)
50 #define EVTIM_BUF_LOG_DBG(...) \
51 EVTIM_LOG(DEBUG, evtim_buffer_logtype, __VA_ARGS__)
52 #define EVTIM_SVC_LOG_DBG(...) \
53 EVTIM_LOG(DEBUG, evtim_svc_logtype, __VA_ARGS__)
55 #define EVTIM_LOG_DBG(...) (void)0
56 #define EVTIM_BUF_LOG_DBG(...) (void)0
57 #define EVTIM_SVC_LOG_DBG(...) (void)0
61 default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id,
64 struct rte_event_timer_adapter *adapter;
65 struct rte_eventdev *dev;
66 struct rte_event_dev_config dev_conf;
67 struct rte_event_port_conf *port_conf, def_port_conf = {0};
73 RTE_SET_USED(event_dev_id);
75 adapter = &adapters[id];
76 dev = &rte_eventdevs[adapter->data->event_dev_id];
77 dev_id = dev->data->dev_id;
78 dev_conf = dev->data->dev_conf;
80 started = dev->data->dev_started;
82 rte_event_dev_stop(dev_id);
84 port_id = dev_conf.nb_event_ports;
85 dev_conf.nb_event_ports += 1;
86 ret = rte_event_dev_configure(dev_id, &dev_conf);
88 EVTIM_LOG_ERR("failed to configure event dev %u\n", dev_id);
90 if (rte_event_dev_start(dev_id))
99 port_conf = &def_port_conf;
100 ret = rte_event_port_default_conf_get(dev_id, port_id,
106 ret = rte_event_port_setup(dev_id, port_id, port_conf);
108 EVTIM_LOG_ERR("failed to setup event port %u on event dev %u\n",
113 *event_port_id = port_id;
116 ret = rte_event_dev_start(dev_id);
121 struct rte_event_timer_adapter *
122 rte_event_timer_adapter_create(const struct rte_event_timer_adapter_conf *conf)
124 return rte_event_timer_adapter_create_ext(conf, default_port_conf_cb,
128 struct rte_event_timer_adapter *
129 rte_event_timer_adapter_create_ext(
130 const struct rte_event_timer_adapter_conf *conf,
131 rte_event_timer_adapter_port_conf_cb_t conf_cb,
135 struct rte_event_timer_adapter *adapter;
136 const struct rte_memzone *mz;
137 char mz_name[DATA_MZ_NAME_MAX_LEN];
139 struct rte_eventdev *dev;
141 if (adapters == NULL) {
142 adapters = rte_zmalloc("Eventdev",
143 sizeof(struct rte_event_timer_adapter) *
144 RTE_EVENT_TIMER_ADAPTER_NUM_MAX,
145 RTE_CACHE_LINE_SIZE);
146 if (adapters == NULL) {
157 /* Check eventdev ID */
158 if (!rte_event_pmd_is_valid_dev(conf->event_dev_id)) {
162 dev = &rte_eventdevs[conf->event_dev_id];
164 adapter_id = conf->timer_adapter_id;
166 /* Check that adapter_id is in range */
167 if (adapter_id >= RTE_EVENT_TIMER_ADAPTER_NUM_MAX) {
172 /* Check adapter ID not already allocated */
173 adapter = &adapters[adapter_id];
174 if (adapter->allocated) {
179 /* Create shared data area. */
180 n = snprintf(mz_name, sizeof(mz_name), DATA_MZ_NAME_FORMAT, adapter_id);
181 if (n >= (int)sizeof(mz_name)) {
185 mz = rte_memzone_reserve(mz_name,
186 sizeof(struct rte_event_timer_adapter_data),
189 /* rte_errno set by rte_memzone_reserve */
192 adapter->data = mz->addr;
193 memset(adapter->data, 0, sizeof(struct rte_event_timer_adapter_data));
195 adapter->data->mz = mz;
196 adapter->data->event_dev_id = conf->event_dev_id;
197 adapter->data->id = adapter_id;
198 adapter->data->socket_id = conf->socket_id;
199 adapter->data->conf = *conf; /* copy conf structure */
201 /* Query eventdev PMD for timer adapter capabilities and ops */
202 ret = dev->dev_ops->timer_adapter_caps_get(dev,
203 adapter->data->conf.flags,
204 &adapter->data->caps,
211 if (!(adapter->data->caps &
212 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
213 FUNC_PTR_OR_NULL_RET_WITH_ERRNO(conf_cb, EINVAL);
214 ret = conf_cb(adapter->data->id, adapter->data->event_dev_id,
215 &adapter->data->event_port_id, conf_arg);
222 /* If eventdev PMD did not provide ops, use default software
225 if (adapter->ops == NULL)
226 adapter->ops = &swtim_ops;
228 /* Allow driver to do some setup */
229 FUNC_PTR_OR_NULL_RET_WITH_ERRNO(adapter->ops->init, ENOTSUP);
230 ret = adapter->ops->init(adapter);
236 /* Set fast-path function pointers */
237 adapter->arm_burst = adapter->ops->arm_burst;
238 adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
239 adapter->cancel_burst = adapter->ops->cancel_burst;
241 adapter->allocated = 1;
243 rte_eventdev_trace_timer_adapter_create(adapter_id, adapter, conf,
248 rte_memzone_free(adapter->data->mz);
253 rte_event_timer_adapter_get_info(const struct rte_event_timer_adapter *adapter,
254 struct rte_event_timer_adapter_info *adapter_info)
256 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
258 if (adapter->ops->get_info)
259 /* let driver set values it knows */
260 adapter->ops->get_info(adapter, adapter_info);
262 /* Set common values */
263 adapter_info->conf = adapter->data->conf;
264 adapter_info->event_dev_port_id = adapter->data->event_port_id;
265 adapter_info->caps = adapter->data->caps;
271 rte_event_timer_adapter_start(const struct rte_event_timer_adapter *adapter)
275 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
276 FUNC_PTR_OR_ERR_RET(adapter->ops->start, -EINVAL);
278 if (adapter->data->started) {
279 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already started",
284 ret = adapter->ops->start(adapter);
288 adapter->data->started = 1;
289 rte_eventdev_trace_timer_adapter_start(adapter);
294 rte_event_timer_adapter_stop(const struct rte_event_timer_adapter *adapter)
298 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
299 FUNC_PTR_OR_ERR_RET(adapter->ops->stop, -EINVAL);
301 if (adapter->data->started == 0) {
302 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already stopped",
307 ret = adapter->ops->stop(adapter);
311 adapter->data->started = 0;
312 rte_eventdev_trace_timer_adapter_stop(adapter);
316 struct rte_event_timer_adapter *
317 rte_event_timer_adapter_lookup(uint16_t adapter_id)
319 char name[DATA_MZ_NAME_MAX_LEN];
320 const struct rte_memzone *mz;
321 struct rte_event_timer_adapter_data *data;
322 struct rte_event_timer_adapter *adapter;
324 struct rte_eventdev *dev;
326 if (adapters == NULL) {
327 adapters = rte_zmalloc("Eventdev",
328 sizeof(struct rte_event_timer_adapter) *
329 RTE_EVENT_TIMER_ADAPTER_NUM_MAX,
330 RTE_CACHE_LINE_SIZE);
331 if (adapters == NULL) {
337 if (adapters[adapter_id].allocated)
338 return &adapters[adapter_id]; /* Adapter is already loaded */
340 snprintf(name, DATA_MZ_NAME_MAX_LEN, DATA_MZ_NAME_FORMAT, adapter_id);
341 mz = rte_memzone_lookup(name);
349 adapter = &adapters[data->id];
350 adapter->data = data;
352 dev = &rte_eventdevs[adapter->data->event_dev_id];
354 /* Query eventdev PMD for timer adapter capabilities and ops */
355 ret = dev->dev_ops->timer_adapter_caps_get(dev,
356 adapter->data->conf.flags,
357 &adapter->data->caps,
364 /* If eventdev PMD did not provide ops, use default software
367 if (adapter->ops == NULL)
368 adapter->ops = &swtim_ops;
370 /* Set fast-path function pointers */
371 adapter->arm_burst = adapter->ops->arm_burst;
372 adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
373 adapter->cancel_burst = adapter->ops->cancel_burst;
375 adapter->allocated = 1;
381 rte_event_timer_adapter_free(struct rte_event_timer_adapter *adapter)
385 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
386 FUNC_PTR_OR_ERR_RET(adapter->ops->uninit, -EINVAL);
388 if (adapter->data->started == 1) {
389 EVTIM_LOG_ERR("event timer adapter %"PRIu8" must be stopped "
390 "before freeing", adapter->data->id);
394 /* free impl priv data */
395 ret = adapter->ops->uninit(adapter);
399 /* free shared data area */
400 ret = rte_memzone_free(adapter->data->mz);
404 adapter->data = NULL;
405 adapter->allocated = 0;
408 for (i = 0; i < RTE_EVENT_TIMER_ADAPTER_NUM_MAX; i++)
409 if (adapters[i].allocated)
410 ret = adapters[i].allocated;
417 rte_eventdev_trace_timer_adapter_free(adapter);
422 rte_event_timer_adapter_service_id_get(struct rte_event_timer_adapter *adapter,
423 uint32_t *service_id)
425 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
427 if (adapter->data->service_inited && service_id != NULL)
428 *service_id = adapter->data->service_id;
430 return adapter->data->service_inited ? 0 : -ESRCH;
434 rte_event_timer_adapter_stats_get(struct rte_event_timer_adapter *adapter,
435 struct rte_event_timer_adapter_stats *stats)
437 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
438 FUNC_PTR_OR_ERR_RET(adapter->ops->stats_get, -EINVAL);
442 return adapter->ops->stats_get(adapter, stats);
446 rte_event_timer_adapter_stats_reset(struct rte_event_timer_adapter *adapter)
448 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
449 FUNC_PTR_OR_ERR_RET(adapter->ops->stats_reset, -EINVAL);
450 return adapter->ops->stats_reset(adapter);
454 * Software event timer adapter buffer helper functions
457 #define NSECPERSEC 1E9
459 /* Optimizations used to index into the buffer require that the buffer size
462 #define EVENT_BUFFER_SZ 4096
463 #define EVENT_BUFFER_BATCHSZ 32
464 #define EVENT_BUFFER_MASK (EVENT_BUFFER_SZ - 1)
466 #define EXP_TIM_BUF_SZ 128
468 struct event_buffer {
471 struct rte_event events[EVENT_BUFFER_SZ];
472 } __rte_cache_aligned;
475 event_buffer_full(struct event_buffer *bufp)
477 return (bufp->head - bufp->tail) == EVENT_BUFFER_SZ;
481 event_buffer_batch_ready(struct event_buffer *bufp)
483 return (bufp->head - bufp->tail) >= EVENT_BUFFER_BATCHSZ;
487 event_buffer_init(struct event_buffer *bufp)
489 bufp->head = bufp->tail = 0;
490 memset(&bufp->events, 0, sizeof(struct rte_event) * EVENT_BUFFER_SZ);
494 event_buffer_add(struct event_buffer *bufp, struct rte_event *eventp)
497 struct rte_event *buf_eventp;
499 if (event_buffer_full(bufp))
502 /* Instead of modulus, bitwise AND with mask to get head_idx. */
503 head_idx = bufp->head & EVENT_BUFFER_MASK;
504 buf_eventp = &bufp->events[head_idx];
505 rte_memcpy(buf_eventp, eventp, sizeof(struct rte_event));
507 /* Wrap automatically when overflow occurs. */
514 event_buffer_flush(struct event_buffer *bufp, uint8_t dev_id, uint8_t port_id,
515 uint16_t *nb_events_flushed,
516 uint16_t *nb_events_inv)
518 struct rte_event *events = bufp->events;
519 size_t head_idx, tail_idx;
522 /* Instead of modulus, bitwise AND with mask to get index. */
523 head_idx = bufp->head & EVENT_BUFFER_MASK;
524 tail_idx = bufp->tail & EVENT_BUFFER_MASK;
526 RTE_ASSERT(head_idx < EVENT_BUFFER_SZ && tail_idx < EVENT_BUFFER_SZ);
528 /* Determine the largest contigous run we can attempt to enqueue to the
531 if (head_idx > tail_idx)
532 n = head_idx - tail_idx;
533 else if (head_idx < tail_idx)
534 n = EVENT_BUFFER_SZ - tail_idx;
535 else if (event_buffer_full(bufp))
536 n = EVENT_BUFFER_SZ - tail_idx;
538 *nb_events_flushed = 0;
542 n = RTE_MIN(EVENT_BUFFER_BATCHSZ, n);
545 *nb_events_flushed = rte_event_enqueue_burst(dev_id, port_id,
546 &events[tail_idx], n);
547 if (*nb_events_flushed != n) {
548 if (rte_errno == EINVAL) {
549 EVTIM_LOG_ERR("failed to enqueue invalid event - "
552 } else if (rte_errno == ENOSPC)
556 if (*nb_events_flushed > 0)
557 EVTIM_BUF_LOG_DBG("enqueued %"PRIu16" timer events to event "
558 "device", *nb_events_flushed);
560 bufp->tail = bufp->tail + *nb_events_flushed + *nb_events_inv;
564 * Software event timer adapter implementation
567 /* Identifier of service executing timer management logic. */
569 /* The cycle count at which the adapter should next tick */
570 uint64_t next_tick_cycles;
571 /* The tick resolution used by adapter instance. May have been
572 * adjusted from what user requested
574 uint64_t timer_tick_ns;
575 /* Maximum timeout in nanoseconds allowed by adapter instance. */
577 /* Buffered timer expiry events to be enqueued to an event device. */
578 struct event_buffer buffer;
580 struct rte_event_timer_adapter_stats stats;
581 /* Mempool of timer objects */
582 struct rte_mempool *tim_pool;
583 /* Back pointer for convenience */
584 struct rte_event_timer_adapter *adapter;
585 /* Identifier of timer data instance */
586 uint32_t timer_data_id;
587 /* Track which cores have actually armed a timer */
590 } __rte_cache_aligned in_use[RTE_MAX_LCORE];
591 /* Track which cores' timer lists should be polled */
592 unsigned int poll_lcores[RTE_MAX_LCORE];
593 /* The number of lists that should be polled */
595 /* Timers which have expired and can be returned to a mempool */
596 struct rte_timer *expired_timers[EXP_TIM_BUF_SZ];
597 /* The number of timers that can be returned to a mempool */
598 size_t n_expired_timers;
601 static inline struct swtim *
602 swtim_pmd_priv(const struct rte_event_timer_adapter *adapter)
604 return adapter->data->adapter_priv;
608 swtim_callback(struct rte_timer *tim)
610 struct rte_event_timer *evtim = tim->arg;
611 struct rte_event_timer_adapter *adapter;
612 unsigned int lcore = rte_lcore_id();
614 uint16_t nb_evs_flushed = 0;
615 uint16_t nb_evs_invalid = 0;
620 opaque = evtim->impl_opaque[1];
621 adapter = (struct rte_event_timer_adapter *)(uintptr_t)opaque;
622 sw = swtim_pmd_priv(adapter);
624 ret = event_buffer_add(&sw->buffer, &evtim->ev);
626 /* If event buffer is full, put timer back in list with
627 * immediate expiry value, so that we process it again on the
630 ret = rte_timer_alt_reset(sw->timer_data_id, tim, 0, SINGLE,
633 EVTIM_LOG_DBG("event buffer full, failed to reset "
634 "timer with immediate expiry value");
636 sw->stats.evtim_retry_count++;
637 EVTIM_LOG_DBG("event buffer full, resetting rte_timer "
638 "with immediate expiry value");
641 if (unlikely(sw->in_use[lcore].v == 0)) {
642 sw->in_use[lcore].v = 1;
643 n_lcores = __atomic_fetch_add(&sw->n_poll_lcores, 1,
645 __atomic_store_n(&sw->poll_lcores[n_lcores], lcore,
649 EVTIM_BUF_LOG_DBG("buffered an event timer expiry event");
651 /* Empty the buffer here, if necessary, to free older expired
654 if (unlikely(sw->n_expired_timers == EXP_TIM_BUF_SZ)) {
655 rte_mempool_put_bulk(sw->tim_pool,
656 (void **)sw->expired_timers,
657 sw->n_expired_timers);
658 sw->n_expired_timers = 0;
661 sw->expired_timers[sw->n_expired_timers++] = tim;
662 sw->stats.evtim_exp_count++;
664 __atomic_store_n(&evtim->state, RTE_EVENT_TIMER_NOT_ARMED,
668 if (event_buffer_batch_ready(&sw->buffer)) {
669 event_buffer_flush(&sw->buffer,
670 adapter->data->event_dev_id,
671 adapter->data->event_port_id,
675 sw->stats.ev_enq_count += nb_evs_flushed;
676 sw->stats.ev_inv_count += nb_evs_invalid;
680 static __rte_always_inline uint64_t
681 get_timeout_cycles(struct rte_event_timer *evtim,
682 const struct rte_event_timer_adapter *adapter)
684 struct swtim *sw = swtim_pmd_priv(adapter);
685 uint64_t timeout_ns = evtim->timeout_ticks * sw->timer_tick_ns;
686 return timeout_ns * rte_get_timer_hz() / NSECPERSEC;
689 /* This function returns true if one or more (adapter) ticks have occurred since
690 * the last time it was called.
693 swtim_did_tick(struct swtim *sw)
695 uint64_t cycles_per_adapter_tick, start_cycles;
696 uint64_t *next_tick_cyclesp;
698 next_tick_cyclesp = &sw->next_tick_cycles;
699 cycles_per_adapter_tick = sw->timer_tick_ns *
700 (rte_get_timer_hz() / NSECPERSEC);
701 start_cycles = rte_get_timer_cycles();
703 /* Note: initially, *next_tick_cyclesp == 0, so the clause below will
704 * execute, and set things going.
707 if (start_cycles >= *next_tick_cyclesp) {
708 /* Snap the current cycle count to the preceding adapter tick
711 start_cycles -= start_cycles % cycles_per_adapter_tick;
712 *next_tick_cyclesp = start_cycles + cycles_per_adapter_tick;
720 /* Check that event timer timeout value is in range */
721 static __rte_always_inline int
722 check_timeout(struct rte_event_timer *evtim,
723 const struct rte_event_timer_adapter *adapter)
726 struct swtim *sw = swtim_pmd_priv(adapter);
728 tmo_nsec = evtim->timeout_ticks * sw->timer_tick_ns;
729 if (tmo_nsec > sw->max_tmo_ns)
731 if (tmo_nsec < sw->timer_tick_ns)
737 /* Check that event timer event queue sched type matches destination event queue
740 static __rte_always_inline int
741 check_destination_event_queue(struct rte_event_timer *evtim,
742 const struct rte_event_timer_adapter *adapter)
747 ret = rte_event_queue_attr_get(adapter->data->event_dev_id,
749 RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE,
752 if ((ret == 0 && evtim->ev.sched_type == sched_type) ||
760 swtim_service_func(void *arg)
762 struct rte_event_timer_adapter *adapter = arg;
763 struct swtim *sw = swtim_pmd_priv(adapter);
764 uint16_t nb_evs_flushed = 0;
765 uint16_t nb_evs_invalid = 0;
767 if (swtim_did_tick(sw)) {
768 rte_timer_alt_manage(sw->timer_data_id,
773 /* Return expired timer objects back to mempool */
774 rte_mempool_put_bulk(sw->tim_pool, (void **)sw->expired_timers,
775 sw->n_expired_timers);
776 sw->n_expired_timers = 0;
778 event_buffer_flush(&sw->buffer,
779 adapter->data->event_dev_id,
780 adapter->data->event_port_id,
784 sw->stats.ev_enq_count += nb_evs_flushed;
785 sw->stats.ev_inv_count += nb_evs_invalid;
786 sw->stats.adapter_tick_count++;
789 rte_event_maintain(adapter->data->event_dev_id,
790 adapter->data->event_port_id, 0);
795 /* The adapter initialization function rounds the mempool size up to the next
796 * power of 2, so we can take the difference between that value and what the
797 * user requested, and use the space for caches. This avoids a scenario where a
798 * user can't arm the number of timers the adapter was configured with because
799 * mempool objects have been lost to caches.
801 * nb_actual should always be a power of 2, so we can iterate over the powers
802 * of 2 to see what the largest cache size we can use is.
805 compute_msg_mempool_cache_size(uint64_t nb_requested, uint64_t nb_actual)
814 if (RTE_MAX_LCORE * size < (int)(nb_actual - nb_requested) &&
815 size < RTE_MEMPOOL_CACHE_MAX_SIZE &&
816 size <= nb_actual / 1.5)
826 swtim_init(struct rte_event_timer_adapter *adapter)
831 struct rte_service_spec service;
833 /* Allocate storage for private data area */
834 #define SWTIM_NAMESIZE 32
835 char swtim_name[SWTIM_NAMESIZE];
836 snprintf(swtim_name, SWTIM_NAMESIZE, "swtim_%"PRIu8,
838 sw = rte_zmalloc_socket(swtim_name, sizeof(*sw), RTE_CACHE_LINE_SIZE,
839 adapter->data->socket_id);
841 EVTIM_LOG_ERR("failed to allocate space for private data");
846 /* Connect storage to adapter instance */
847 adapter->data->adapter_priv = sw;
848 sw->adapter = adapter;
850 sw->timer_tick_ns = adapter->data->conf.timer_tick_ns;
851 sw->max_tmo_ns = adapter->data->conf.max_tmo_ns;
853 /* Create a timer pool */
854 char pool_name[SWTIM_NAMESIZE];
855 snprintf(pool_name, SWTIM_NAMESIZE, "swtim_pool_%"PRIu8,
857 /* Optimal mempool size is a power of 2 minus one */
858 uint64_t nb_timers = rte_align64pow2(adapter->data->conf.nb_timers);
859 int pool_size = nb_timers - 1;
860 int cache_size = compute_msg_mempool_cache_size(
861 adapter->data->conf.nb_timers, nb_timers);
862 flags = 0; /* pool is multi-producer, multi-consumer */
863 sw->tim_pool = rte_mempool_create(pool_name, pool_size,
864 sizeof(struct rte_timer), cache_size, 0, NULL, NULL,
865 NULL, NULL, adapter->data->socket_id, flags);
866 if (sw->tim_pool == NULL) {
867 EVTIM_LOG_ERR("failed to create timer object mempool");
872 /* Initialize the variables that track in-use timer lists */
873 for (i = 0; i < RTE_MAX_LCORE; i++)
876 /* Initialize the timer subsystem and allocate timer data instance */
877 ret = rte_timer_subsystem_init();
879 if (ret != -EALREADY) {
880 EVTIM_LOG_ERR("failed to initialize timer subsystem");
886 ret = rte_timer_data_alloc(&sw->timer_data_id);
888 EVTIM_LOG_ERR("failed to allocate timer data instance");
893 /* Initialize timer event buffer */
894 event_buffer_init(&sw->buffer);
896 sw->adapter = adapter;
898 /* Register a service component to run adapter logic */
899 memset(&service, 0, sizeof(service));
900 snprintf(service.name, RTE_SERVICE_NAME_MAX,
901 "swtim_svc_%"PRIu8, adapter->data->id);
902 service.socket_id = adapter->data->socket_id;
903 service.callback = swtim_service_func;
904 service.callback_userdata = adapter;
905 service.capabilities &= ~(RTE_SERVICE_CAP_MT_SAFE);
906 ret = rte_service_component_register(&service, &sw->service_id);
908 EVTIM_LOG_ERR("failed to register service %s with id %"PRIu32
909 ": err = %d", service.name, sw->service_id,
916 EVTIM_LOG_DBG("registered service %s with id %"PRIu32, service.name,
919 adapter->data->service_id = sw->service_id;
920 adapter->data->service_inited = 1;
924 rte_mempool_free(sw->tim_pool);
931 swtim_free_tim(struct rte_timer *tim, void *arg)
933 struct swtim *sw = arg;
935 rte_mempool_put(sw->tim_pool, tim);
938 /* Traverse the list of outstanding timers and put them back in the mempool
939 * before freeing the adapter to avoid leaking the memory.
942 swtim_uninit(struct rte_event_timer_adapter *adapter)
945 struct swtim *sw = swtim_pmd_priv(adapter);
947 /* Free outstanding timers */
948 rte_timer_stop_all(sw->timer_data_id,
954 ret = rte_service_component_unregister(sw->service_id);
956 EVTIM_LOG_ERR("failed to unregister service component");
960 rte_mempool_free(sw->tim_pool);
962 adapter->data->adapter_priv = NULL;
967 static inline int32_t
968 get_mapped_count_for_service(uint32_t service_id)
970 int32_t core_count, i, mapped_count = 0;
971 uint32_t lcore_arr[RTE_MAX_LCORE];
973 core_count = rte_service_lcore_list(lcore_arr, RTE_MAX_LCORE);
975 for (i = 0; i < core_count; i++)
976 if (rte_service_map_lcore_get(service_id, lcore_arr[i]) == 1)
983 swtim_start(const struct rte_event_timer_adapter *adapter)
986 struct swtim *sw = swtim_pmd_priv(adapter);
988 /* Mapping the service to more than one service core can introduce
989 * delays while one thread is waiting to acquire a lock, so only allow
990 * one core to be mapped to the service.
992 * Note: the service could be modified such that it spreads cores to
993 * poll over multiple service instances.
995 mapped_count = get_mapped_count_for_service(sw->service_id);
997 if (mapped_count != 1)
998 return mapped_count < 1 ? -ENOENT : -ENOTSUP;
1000 return rte_service_component_runstate_set(sw->service_id, 1);
1004 swtim_stop(const struct rte_event_timer_adapter *adapter)
1007 struct swtim *sw = swtim_pmd_priv(adapter);
1009 ret = rte_service_component_runstate_set(sw->service_id, 0);
1013 /* Wait for the service to complete its final iteration */
1014 while (rte_service_may_be_active(sw->service_id))
1021 swtim_get_info(const struct rte_event_timer_adapter *adapter,
1022 struct rte_event_timer_adapter_info *adapter_info)
1024 struct swtim *sw = swtim_pmd_priv(adapter);
1025 adapter_info->min_resolution_ns = sw->timer_tick_ns;
1026 adapter_info->max_tmo_ns = sw->max_tmo_ns;
1030 swtim_stats_get(const struct rte_event_timer_adapter *adapter,
1031 struct rte_event_timer_adapter_stats *stats)
1033 struct swtim *sw = swtim_pmd_priv(adapter);
1034 *stats = sw->stats; /* structure copy */
1039 swtim_stats_reset(const struct rte_event_timer_adapter *adapter)
1041 struct swtim *sw = swtim_pmd_priv(adapter);
1042 memset(&sw->stats, 0, sizeof(sw->stats));
1047 __swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1048 struct rte_event_timer **evtims,
1052 struct swtim *sw = swtim_pmd_priv(adapter);
1053 uint32_t lcore_id = rte_lcore_id();
1054 struct rte_timer *tim, *tims[nb_evtims];
1057 /* Timer list for this lcore is not in use. */
1058 uint16_t exp_state = 0;
1059 enum rte_event_timer_state n_state;
1061 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1062 /* Check that the service is running. */
1063 if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1069 /* Adjust lcore_id if non-EAL thread. Arbitrarily pick the timer list of
1070 * the highest lcore to insert such timers into
1072 if (lcore_id == LCORE_ID_ANY)
1073 lcore_id = RTE_MAX_LCORE - 1;
1075 /* If this is the first time we're arming an event timer on this lcore,
1076 * mark this lcore as "in use"; this will cause the service
1077 * function to process the timer list that corresponds to this lcore.
1078 * The atomic compare-and-swap operation can prevent the race condition
1079 * on in_use flag between multiple non-EAL threads.
1081 if (unlikely(__atomic_compare_exchange_n(&sw->in_use[lcore_id].v,
1083 __ATOMIC_RELAXED, __ATOMIC_RELAXED))) {
1084 EVTIM_LOG_DBG("Adding lcore id = %u to list of lcores to poll",
1086 n_lcores = __atomic_fetch_add(&sw->n_poll_lcores, 1,
1088 __atomic_store_n(&sw->poll_lcores[n_lcores], lcore_id,
1092 ret = rte_mempool_get_bulk(sw->tim_pool, (void **)tims,
1099 for (i = 0; i < nb_evtims; i++) {
1100 n_state = __atomic_load_n(&evtims[i]->state, __ATOMIC_ACQUIRE);
1101 if (n_state == RTE_EVENT_TIMER_ARMED) {
1102 rte_errno = EALREADY;
1104 } else if (!(n_state == RTE_EVENT_TIMER_NOT_ARMED ||
1105 n_state == RTE_EVENT_TIMER_CANCELED)) {
1110 ret = check_timeout(evtims[i], adapter);
1111 if (unlikely(ret == -1)) {
1112 __atomic_store_n(&evtims[i]->state,
1113 RTE_EVENT_TIMER_ERROR_TOOLATE,
1117 } else if (unlikely(ret == -2)) {
1118 __atomic_store_n(&evtims[i]->state,
1119 RTE_EVENT_TIMER_ERROR_TOOEARLY,
1125 if (unlikely(check_destination_event_queue(evtims[i],
1127 __atomic_store_n(&evtims[i]->state,
1128 RTE_EVENT_TIMER_ERROR,
1135 rte_timer_init(tim);
1137 evtims[i]->impl_opaque[0] = (uintptr_t)tim;
1138 evtims[i]->impl_opaque[1] = (uintptr_t)adapter;
1140 cycles = get_timeout_cycles(evtims[i], adapter);
1141 ret = rte_timer_alt_reset(sw->timer_data_id, tim, cycles,
1142 SINGLE, lcore_id, NULL, evtims[i]);
1144 /* tim was in RUNNING or CONFIG state */
1145 __atomic_store_n(&evtims[i]->state,
1146 RTE_EVENT_TIMER_ERROR,
1151 EVTIM_LOG_DBG("armed an event timer");
1152 /* RELEASE ordering guarantees the adapter specific value
1153 * changes observed before the update of state.
1155 __atomic_store_n(&evtims[i]->state, RTE_EVENT_TIMER_ARMED,
1160 rte_mempool_put_bulk(sw->tim_pool,
1161 (void **)&tims[i], nb_evtims - i);
1167 swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1168 struct rte_event_timer **evtims,
1171 return __swtim_arm_burst(adapter, evtims, nb_evtims);
1175 swtim_cancel_burst(const struct rte_event_timer_adapter *adapter,
1176 struct rte_event_timer **evtims,
1180 struct rte_timer *timp;
1182 struct swtim *sw = swtim_pmd_priv(adapter);
1183 enum rte_event_timer_state n_state;
1185 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1186 /* Check that the service is running. */
1187 if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1193 for (i = 0; i < nb_evtims; i++) {
1194 /* Don't modify the event timer state in these cases */
1195 /* ACQUIRE ordering guarantees the access of implementation
1196 * specific opaque data under the correct state.
1198 n_state = __atomic_load_n(&evtims[i]->state, __ATOMIC_ACQUIRE);
1199 if (n_state == RTE_EVENT_TIMER_CANCELED) {
1200 rte_errno = EALREADY;
1202 } else if (n_state != RTE_EVENT_TIMER_ARMED) {
1207 opaque = evtims[i]->impl_opaque[0];
1208 timp = (struct rte_timer *)(uintptr_t)opaque;
1209 RTE_ASSERT(timp != NULL);
1211 ret = rte_timer_alt_stop(sw->timer_data_id, timp);
1213 /* Timer is running or being configured */
1218 rte_mempool_put(sw->tim_pool, (void **)timp);
1220 /* The RELEASE ordering here pairs with atomic ordering
1221 * to make sure the state update data observed between
1224 __atomic_store_n(&evtims[i]->state, RTE_EVENT_TIMER_CANCELED,
1232 swtim_arm_tmo_tick_burst(const struct rte_event_timer_adapter *adapter,
1233 struct rte_event_timer **evtims,
1234 uint64_t timeout_ticks,
1239 for (i = 0; i < nb_evtims; i++)
1240 evtims[i]->timeout_ticks = timeout_ticks;
1242 return __swtim_arm_burst(adapter, evtims, nb_evtims);
1245 static const struct event_timer_adapter_ops swtim_ops = {
1247 .uninit = swtim_uninit,
1248 .start = swtim_start,
1250 .get_info = swtim_get_info,
1251 .stats_get = swtim_stats_get,
1252 .stats_reset = swtim_stats_reset,
1253 .arm_burst = swtim_arm_burst,
1254 .arm_tmo_tick_burst = swtim_arm_tmo_tick_burst,
1255 .cancel_burst = swtim_cancel_burst,