1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2018 Intel Corporation.
11 #include <rte_memzone.h>
12 #include <rte_memory.h>
14 #include <rte_errno.h>
15 #include <rte_malloc.h>
17 #include <rte_mempool.h>
18 #include <rte_common.h>
19 #include <rte_timer.h>
20 #include <rte_service_component.h>
21 #include <rte_cycles.h>
23 #include "rte_eventdev.h"
24 #include "rte_eventdev_pmd.h"
25 #include "rte_eventdev_trace.h"
26 #include "rte_event_timer_adapter.h"
27 #include "rte_event_timer_adapter_pmd.h"
29 #define DATA_MZ_NAME_MAX_LEN 64
30 #define DATA_MZ_NAME_FORMAT "rte_event_timer_adapter_data_%d"
32 static int evtim_logtype;
33 static int evtim_svc_logtype;
34 static int evtim_buffer_logtype;
36 static struct rte_event_timer_adapter adapters[RTE_EVENT_TIMER_ADAPTER_NUM_MAX];
38 static const struct rte_event_timer_adapter_ops swtim_ops;
40 #define EVTIM_LOG(level, logtype, ...) \
41 rte_log(RTE_LOG_ ## level, logtype, \
42 RTE_FMT("EVTIMER: %s() line %u: " RTE_FMT_HEAD(__VA_ARGS__,) \
43 "\n", __func__, __LINE__, RTE_FMT_TAIL(__VA_ARGS__,)))
45 #define EVTIM_LOG_ERR(...) EVTIM_LOG(ERR, evtim_logtype, __VA_ARGS__)
47 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
48 #define EVTIM_LOG_DBG(...) \
49 EVTIM_LOG(DEBUG, evtim_logtype, __VA_ARGS__)
50 #define EVTIM_BUF_LOG_DBG(...) \
51 EVTIM_LOG(DEBUG, evtim_buffer_logtype, __VA_ARGS__)
52 #define EVTIM_SVC_LOG_DBG(...) \
53 EVTIM_LOG(DEBUG, evtim_svc_logtype, __VA_ARGS__)
55 #define EVTIM_LOG_DBG(...) (void)0
56 #define EVTIM_BUF_LOG_DBG(...) (void)0
57 #define EVTIM_SVC_LOG_DBG(...) (void)0
61 default_port_conf_cb(uint16_t id, uint8_t event_dev_id, uint8_t *event_port_id,
64 struct rte_event_timer_adapter *adapter;
65 struct rte_eventdev *dev;
66 struct rte_event_dev_config dev_conf;
67 struct rte_event_port_conf *port_conf, def_port_conf = {0};
73 RTE_SET_USED(event_dev_id);
75 adapter = &adapters[id];
76 dev = &rte_eventdevs[adapter->data->event_dev_id];
77 dev_id = dev->data->dev_id;
78 dev_conf = dev->data->dev_conf;
80 started = dev->data->dev_started;
82 rte_event_dev_stop(dev_id);
84 port_id = dev_conf.nb_event_ports;
85 dev_conf.nb_event_ports += 1;
86 ret = rte_event_dev_configure(dev_id, &dev_conf);
88 EVTIM_LOG_ERR("failed to configure event dev %u\n", dev_id);
90 if (rte_event_dev_start(dev_id))
99 port_conf = &def_port_conf;
100 ret = rte_event_port_default_conf_get(dev_id, port_id,
106 ret = rte_event_port_setup(dev_id, port_id, port_conf);
108 EVTIM_LOG_ERR("failed to setup event port %u on event dev %u\n",
113 *event_port_id = port_id;
116 ret = rte_event_dev_start(dev_id);
121 struct rte_event_timer_adapter *
122 rte_event_timer_adapter_create(const struct rte_event_timer_adapter_conf *conf)
124 return rte_event_timer_adapter_create_ext(conf, default_port_conf_cb,
128 struct rte_event_timer_adapter *
129 rte_event_timer_adapter_create_ext(
130 const struct rte_event_timer_adapter_conf *conf,
131 rte_event_timer_adapter_port_conf_cb_t conf_cb,
135 struct rte_event_timer_adapter *adapter;
136 const struct rte_memzone *mz;
137 char mz_name[DATA_MZ_NAME_MAX_LEN];
139 struct rte_eventdev *dev;
146 /* Check eventdev ID */
147 if (!rte_event_pmd_is_valid_dev(conf->event_dev_id)) {
151 dev = &rte_eventdevs[conf->event_dev_id];
153 adapter_id = conf->timer_adapter_id;
155 /* Check that adapter_id is in range */
156 if (adapter_id >= RTE_EVENT_TIMER_ADAPTER_NUM_MAX) {
161 /* Check adapter ID not already allocated */
162 adapter = &adapters[adapter_id];
163 if (adapter->allocated) {
168 /* Create shared data area. */
169 n = snprintf(mz_name, sizeof(mz_name), DATA_MZ_NAME_FORMAT, adapter_id);
170 if (n >= (int)sizeof(mz_name)) {
174 mz = rte_memzone_reserve(mz_name,
175 sizeof(struct rte_event_timer_adapter_data),
178 /* rte_errno set by rte_memzone_reserve */
181 adapter->data = mz->addr;
182 memset(adapter->data, 0, sizeof(struct rte_event_timer_adapter_data));
184 adapter->data->mz = mz;
185 adapter->data->event_dev_id = conf->event_dev_id;
186 adapter->data->id = adapter_id;
187 adapter->data->socket_id = conf->socket_id;
188 adapter->data->conf = *conf; /* copy conf structure */
190 /* Query eventdev PMD for timer adapter capabilities and ops */
191 ret = dev->dev_ops->timer_adapter_caps_get(dev,
192 adapter->data->conf.flags,
193 &adapter->data->caps,
200 if (!(adapter->data->caps &
201 RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) {
202 FUNC_PTR_OR_NULL_RET_WITH_ERRNO(conf_cb, EINVAL);
203 ret = conf_cb(adapter->data->id, adapter->data->event_dev_id,
204 &adapter->data->event_port_id, conf_arg);
211 /* If eventdev PMD did not provide ops, use default software
214 if (adapter->ops == NULL)
215 adapter->ops = &swtim_ops;
217 /* Allow driver to do some setup */
218 FUNC_PTR_OR_NULL_RET_WITH_ERRNO(adapter->ops->init, ENOTSUP);
219 ret = adapter->ops->init(adapter);
225 /* Set fast-path function pointers */
226 adapter->arm_burst = adapter->ops->arm_burst;
227 adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
228 adapter->cancel_burst = adapter->ops->cancel_burst;
230 adapter->allocated = 1;
232 rte_eventdev_trace_timer_adapter_create(adapter_id, adapter, conf,
237 rte_memzone_free(adapter->data->mz);
242 rte_event_timer_adapter_get_info(const struct rte_event_timer_adapter *adapter,
243 struct rte_event_timer_adapter_info *adapter_info)
245 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
247 if (adapter->ops->get_info)
248 /* let driver set values it knows */
249 adapter->ops->get_info(adapter, adapter_info);
251 /* Set common values */
252 adapter_info->conf = adapter->data->conf;
253 adapter_info->event_dev_port_id = adapter->data->event_port_id;
254 adapter_info->caps = adapter->data->caps;
260 rte_event_timer_adapter_start(const struct rte_event_timer_adapter *adapter)
264 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
265 FUNC_PTR_OR_ERR_RET(adapter->ops->start, -EINVAL);
267 if (adapter->data->started) {
268 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already started",
273 ret = adapter->ops->start(adapter);
277 adapter->data->started = 1;
278 rte_eventdev_trace_timer_adapter_start(adapter);
283 rte_event_timer_adapter_stop(const struct rte_event_timer_adapter *adapter)
287 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
288 FUNC_PTR_OR_ERR_RET(adapter->ops->stop, -EINVAL);
290 if (adapter->data->started == 0) {
291 EVTIM_LOG_ERR("event timer adapter %"PRIu8" already stopped",
296 ret = adapter->ops->stop(adapter);
300 adapter->data->started = 0;
301 rte_eventdev_trace_timer_adapter_stop(adapter);
305 struct rte_event_timer_adapter *
306 rte_event_timer_adapter_lookup(uint16_t adapter_id)
308 char name[DATA_MZ_NAME_MAX_LEN];
309 const struct rte_memzone *mz;
310 struct rte_event_timer_adapter_data *data;
311 struct rte_event_timer_adapter *adapter;
313 struct rte_eventdev *dev;
315 if (adapters[adapter_id].allocated)
316 return &adapters[adapter_id]; /* Adapter is already loaded */
318 snprintf(name, DATA_MZ_NAME_MAX_LEN, DATA_MZ_NAME_FORMAT, adapter_id);
319 mz = rte_memzone_lookup(name);
327 adapter = &adapters[data->id];
328 adapter->data = data;
330 dev = &rte_eventdevs[adapter->data->event_dev_id];
332 /* Query eventdev PMD for timer adapter capabilities and ops */
333 ret = dev->dev_ops->timer_adapter_caps_get(dev,
334 adapter->data->conf.flags,
335 &adapter->data->caps,
342 /* If eventdev PMD did not provide ops, use default software
345 if (adapter->ops == NULL)
346 adapter->ops = &swtim_ops;
348 /* Set fast-path function pointers */
349 adapter->arm_burst = adapter->ops->arm_burst;
350 adapter->arm_tmo_tick_burst = adapter->ops->arm_tmo_tick_burst;
351 adapter->cancel_burst = adapter->ops->cancel_burst;
353 adapter->allocated = 1;
359 rte_event_timer_adapter_free(struct rte_event_timer_adapter *adapter)
363 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
364 FUNC_PTR_OR_ERR_RET(adapter->ops->uninit, -EINVAL);
366 if (adapter->data->started == 1) {
367 EVTIM_LOG_ERR("event timer adapter %"PRIu8" must be stopped "
368 "before freeing", adapter->data->id);
372 /* free impl priv data */
373 ret = adapter->ops->uninit(adapter);
377 /* free shared data area */
378 ret = rte_memzone_free(adapter->data->mz);
382 adapter->data = NULL;
383 adapter->allocated = 0;
385 rte_eventdev_trace_timer_adapter_free(adapter);
390 rte_event_timer_adapter_service_id_get(struct rte_event_timer_adapter *adapter,
391 uint32_t *service_id)
393 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
395 if (adapter->data->service_inited && service_id != NULL)
396 *service_id = adapter->data->service_id;
398 return adapter->data->service_inited ? 0 : -ESRCH;
402 rte_event_timer_adapter_stats_get(struct rte_event_timer_adapter *adapter,
403 struct rte_event_timer_adapter_stats *stats)
405 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
406 FUNC_PTR_OR_ERR_RET(adapter->ops->stats_get, -EINVAL);
410 return adapter->ops->stats_get(adapter, stats);
414 rte_event_timer_adapter_stats_reset(struct rte_event_timer_adapter *adapter)
416 ADAPTER_VALID_OR_ERR_RET(adapter, -EINVAL);
417 FUNC_PTR_OR_ERR_RET(adapter->ops->stats_reset, -EINVAL);
418 return adapter->ops->stats_reset(adapter);
422 * Software event timer adapter buffer helper functions
425 #define NSECPERSEC 1E9
427 /* Optimizations used to index into the buffer require that the buffer size
430 #define EVENT_BUFFER_SZ 4096
431 #define EVENT_BUFFER_BATCHSZ 32
432 #define EVENT_BUFFER_MASK (EVENT_BUFFER_SZ - 1)
434 #define EXP_TIM_BUF_SZ 128
436 struct event_buffer {
439 struct rte_event events[EVENT_BUFFER_SZ];
440 } __rte_cache_aligned;
443 event_buffer_full(struct event_buffer *bufp)
445 return (bufp->head - bufp->tail) == EVENT_BUFFER_SZ;
449 event_buffer_batch_ready(struct event_buffer *bufp)
451 return (bufp->head - bufp->tail) >= EVENT_BUFFER_BATCHSZ;
455 event_buffer_init(struct event_buffer *bufp)
457 bufp->head = bufp->tail = 0;
458 memset(&bufp->events, 0, sizeof(struct rte_event) * EVENT_BUFFER_SZ);
462 event_buffer_add(struct event_buffer *bufp, struct rte_event *eventp)
465 struct rte_event *buf_eventp;
467 if (event_buffer_full(bufp))
470 /* Instead of modulus, bitwise AND with mask to get head_idx. */
471 head_idx = bufp->head & EVENT_BUFFER_MASK;
472 buf_eventp = &bufp->events[head_idx];
473 rte_memcpy(buf_eventp, eventp, sizeof(struct rte_event));
475 /* Wrap automatically when overflow occurs. */
482 event_buffer_flush(struct event_buffer *bufp, uint8_t dev_id, uint8_t port_id,
483 uint16_t *nb_events_flushed,
484 uint16_t *nb_events_inv)
486 struct rte_event *events = bufp->events;
487 size_t head_idx, tail_idx;
490 /* Instead of modulus, bitwise AND with mask to get index. */
491 head_idx = bufp->head & EVENT_BUFFER_MASK;
492 tail_idx = bufp->tail & EVENT_BUFFER_MASK;
494 RTE_ASSERT(head_idx < EVENT_BUFFER_SZ && tail_idx < EVENT_BUFFER_SZ);
496 /* Determine the largest contigous run we can attempt to enqueue to the
499 if (head_idx > tail_idx)
500 n = head_idx - tail_idx;
501 else if (head_idx < tail_idx)
502 n = EVENT_BUFFER_SZ - tail_idx;
503 else if (event_buffer_full(bufp))
504 n = EVENT_BUFFER_SZ - tail_idx;
506 *nb_events_flushed = 0;
510 n = RTE_MIN(EVENT_BUFFER_BATCHSZ, n);
513 *nb_events_flushed = rte_event_enqueue_burst(dev_id, port_id,
514 &events[tail_idx], n);
515 if (*nb_events_flushed != n) {
516 if (rte_errno == EINVAL) {
517 EVTIM_LOG_ERR("failed to enqueue invalid event - "
520 } else if (rte_errno == ENOSPC)
524 if (*nb_events_flushed > 0)
525 EVTIM_BUF_LOG_DBG("enqueued %"PRIu16" timer events to event "
526 "device", *nb_events_flushed);
528 bufp->tail = bufp->tail + *nb_events_flushed + *nb_events_inv;
532 * Software event timer adapter implementation
535 /* Identifier of service executing timer management logic. */
537 /* The cycle count at which the adapter should next tick */
538 uint64_t next_tick_cycles;
539 /* The tick resolution used by adapter instance. May have been
540 * adjusted from what user requested
542 uint64_t timer_tick_ns;
543 /* Maximum timeout in nanoseconds allowed by adapter instance. */
545 /* Buffered timer expiry events to be enqueued to an event device. */
546 struct event_buffer buffer;
548 struct rte_event_timer_adapter_stats stats;
549 /* Mempool of timer objects */
550 struct rte_mempool *tim_pool;
551 /* Back pointer for convenience */
552 struct rte_event_timer_adapter *adapter;
553 /* Identifier of timer data instance */
554 uint32_t timer_data_id;
555 /* Track which cores have actually armed a timer */
558 } __rte_cache_aligned in_use[RTE_MAX_LCORE];
559 /* Track which cores' timer lists should be polled */
560 unsigned int poll_lcores[RTE_MAX_LCORE];
561 /* The number of lists that should be polled */
563 /* Timers which have expired and can be returned to a mempool */
564 struct rte_timer *expired_timers[EXP_TIM_BUF_SZ];
565 /* The number of timers that can be returned to a mempool */
566 size_t n_expired_timers;
569 static inline struct swtim *
570 swtim_pmd_priv(const struct rte_event_timer_adapter *adapter)
572 return adapter->data->adapter_priv;
576 swtim_callback(struct rte_timer *tim)
578 struct rte_event_timer *evtim = tim->arg;
579 struct rte_event_timer_adapter *adapter;
580 unsigned int lcore = rte_lcore_id();
582 uint16_t nb_evs_flushed = 0;
583 uint16_t nb_evs_invalid = 0;
587 opaque = evtim->impl_opaque[1];
588 adapter = (struct rte_event_timer_adapter *)(uintptr_t)opaque;
589 sw = swtim_pmd_priv(adapter);
591 ret = event_buffer_add(&sw->buffer, &evtim->ev);
593 /* If event buffer is full, put timer back in list with
594 * immediate expiry value, so that we process it again on the
597 ret = rte_timer_alt_reset(sw->timer_data_id, tim, 0, SINGLE,
600 EVTIM_LOG_DBG("event buffer full, failed to reset "
601 "timer with immediate expiry value");
603 sw->stats.evtim_retry_count++;
604 EVTIM_LOG_DBG("event buffer full, resetting rte_timer "
605 "with immediate expiry value");
608 if (unlikely(rte_atomic16_test_and_set(&sw->in_use[lcore].v)))
609 sw->poll_lcores[sw->n_poll_lcores++] = lcore;
611 EVTIM_BUF_LOG_DBG("buffered an event timer expiry event");
613 /* Empty the buffer here, if necessary, to free older expired
616 if (unlikely(sw->n_expired_timers == EXP_TIM_BUF_SZ)) {
617 rte_mempool_put_bulk(sw->tim_pool,
618 (void **)sw->expired_timers,
619 sw->n_expired_timers);
620 sw->n_expired_timers = 0;
623 sw->expired_timers[sw->n_expired_timers++] = tim;
624 sw->stats.evtim_exp_count++;
626 evtim->state = RTE_EVENT_TIMER_NOT_ARMED;
629 if (event_buffer_batch_ready(&sw->buffer)) {
630 event_buffer_flush(&sw->buffer,
631 adapter->data->event_dev_id,
632 adapter->data->event_port_id,
636 sw->stats.ev_enq_count += nb_evs_flushed;
637 sw->stats.ev_inv_count += nb_evs_invalid;
641 static __rte_always_inline uint64_t
642 get_timeout_cycles(struct rte_event_timer *evtim,
643 const struct rte_event_timer_adapter *adapter)
645 struct swtim *sw = swtim_pmd_priv(adapter);
646 uint64_t timeout_ns = evtim->timeout_ticks * sw->timer_tick_ns;
647 return timeout_ns * rte_get_timer_hz() / NSECPERSEC;
650 /* This function returns true if one or more (adapter) ticks have occurred since
651 * the last time it was called.
654 swtim_did_tick(struct swtim *sw)
656 uint64_t cycles_per_adapter_tick, start_cycles;
657 uint64_t *next_tick_cyclesp;
659 next_tick_cyclesp = &sw->next_tick_cycles;
660 cycles_per_adapter_tick = sw->timer_tick_ns *
661 (rte_get_timer_hz() / NSECPERSEC);
662 start_cycles = rte_get_timer_cycles();
664 /* Note: initially, *next_tick_cyclesp == 0, so the clause below will
665 * execute, and set things going.
668 if (start_cycles >= *next_tick_cyclesp) {
669 /* Snap the current cycle count to the preceding adapter tick
672 start_cycles -= start_cycles % cycles_per_adapter_tick;
673 *next_tick_cyclesp = start_cycles + cycles_per_adapter_tick;
681 /* Check that event timer timeout value is in range */
682 static __rte_always_inline int
683 check_timeout(struct rte_event_timer *evtim,
684 const struct rte_event_timer_adapter *adapter)
687 struct swtim *sw = swtim_pmd_priv(adapter);
689 tmo_nsec = evtim->timeout_ticks * sw->timer_tick_ns;
690 if (tmo_nsec > sw->max_tmo_ns)
692 if (tmo_nsec < sw->timer_tick_ns)
698 /* Check that event timer event queue sched type matches destination event queue
701 static __rte_always_inline int
702 check_destination_event_queue(struct rte_event_timer *evtim,
703 const struct rte_event_timer_adapter *adapter)
708 ret = rte_event_queue_attr_get(adapter->data->event_dev_id,
710 RTE_EVENT_QUEUE_ATTR_SCHEDULE_TYPE,
713 if ((ret == 0 && evtim->ev.sched_type == sched_type) ||
721 swtim_service_func(void *arg)
723 struct rte_event_timer_adapter *adapter = arg;
724 struct swtim *sw = swtim_pmd_priv(adapter);
725 uint16_t nb_evs_flushed = 0;
726 uint16_t nb_evs_invalid = 0;
728 if (swtim_did_tick(sw)) {
729 rte_timer_alt_manage(sw->timer_data_id,
734 /* Return expired timer objects back to mempool */
735 rte_mempool_put_bulk(sw->tim_pool, (void **)sw->expired_timers,
736 sw->n_expired_timers);
737 sw->n_expired_timers = 0;
739 event_buffer_flush(&sw->buffer,
740 adapter->data->event_dev_id,
741 adapter->data->event_port_id,
745 sw->stats.ev_enq_count += nb_evs_flushed;
746 sw->stats.ev_inv_count += nb_evs_invalid;
747 sw->stats.adapter_tick_count++;
753 /* The adapter initialization function rounds the mempool size up to the next
754 * power of 2, so we can take the difference between that value and what the
755 * user requested, and use the space for caches. This avoids a scenario where a
756 * user can't arm the number of timers the adapter was configured with because
757 * mempool objects have been lost to caches.
759 * nb_actual should always be a power of 2, so we can iterate over the powers
760 * of 2 to see what the largest cache size we can use is.
763 compute_msg_mempool_cache_size(uint64_t nb_requested, uint64_t nb_actual)
772 if (RTE_MAX_LCORE * size < (int)(nb_actual - nb_requested) &&
773 size < RTE_MEMPOOL_CACHE_MAX_SIZE &&
774 size <= nb_actual / 1.5)
784 swtim_init(struct rte_event_timer_adapter *adapter)
789 struct rte_service_spec service;
791 /* Allocate storage for private data area */
792 #define SWTIM_NAMESIZE 32
793 char swtim_name[SWTIM_NAMESIZE];
794 snprintf(swtim_name, SWTIM_NAMESIZE, "swtim_%"PRIu8,
796 sw = rte_zmalloc_socket(swtim_name, sizeof(*sw), RTE_CACHE_LINE_SIZE,
797 adapter->data->socket_id);
799 EVTIM_LOG_ERR("failed to allocate space for private data");
804 /* Connect storage to adapter instance */
805 adapter->data->adapter_priv = sw;
806 sw->adapter = adapter;
808 sw->timer_tick_ns = adapter->data->conf.timer_tick_ns;
809 sw->max_tmo_ns = adapter->data->conf.max_tmo_ns;
811 /* Create a timer pool */
812 char pool_name[SWTIM_NAMESIZE];
813 snprintf(pool_name, SWTIM_NAMESIZE, "swtim_pool_%"PRIu8,
815 /* Optimal mempool size is a power of 2 minus one */
816 uint64_t nb_timers = rte_align64pow2(adapter->data->conf.nb_timers);
817 int pool_size = nb_timers - 1;
818 int cache_size = compute_msg_mempool_cache_size(
819 adapter->data->conf.nb_timers, nb_timers);
820 flags = 0; /* pool is multi-producer, multi-consumer */
821 sw->tim_pool = rte_mempool_create(pool_name, pool_size,
822 sizeof(struct rte_timer), cache_size, 0, NULL, NULL,
823 NULL, NULL, adapter->data->socket_id, flags);
824 if (sw->tim_pool == NULL) {
825 EVTIM_LOG_ERR("failed to create timer object mempool");
830 /* Initialize the variables that track in-use timer lists */
831 for (i = 0; i < RTE_MAX_LCORE; i++)
832 rte_atomic16_init(&sw->in_use[i].v);
834 /* Initialize the timer subsystem and allocate timer data instance */
835 ret = rte_timer_subsystem_init();
837 if (ret != -EALREADY) {
838 EVTIM_LOG_ERR("failed to initialize timer subsystem");
844 ret = rte_timer_data_alloc(&sw->timer_data_id);
846 EVTIM_LOG_ERR("failed to allocate timer data instance");
851 /* Initialize timer event buffer */
852 event_buffer_init(&sw->buffer);
854 sw->adapter = adapter;
856 /* Register a service component to run adapter logic */
857 memset(&service, 0, sizeof(service));
858 snprintf(service.name, RTE_SERVICE_NAME_MAX,
859 "swtim_svc_%"PRIu8, adapter->data->id);
860 service.socket_id = adapter->data->socket_id;
861 service.callback = swtim_service_func;
862 service.callback_userdata = adapter;
863 service.capabilities &= ~(RTE_SERVICE_CAP_MT_SAFE);
864 ret = rte_service_component_register(&service, &sw->service_id);
866 EVTIM_LOG_ERR("failed to register service %s with id %"PRIu32
867 ": err = %d", service.name, sw->service_id,
874 EVTIM_LOG_DBG("registered service %s with id %"PRIu32, service.name,
877 adapter->data->service_id = sw->service_id;
878 adapter->data->service_inited = 1;
882 rte_mempool_free(sw->tim_pool);
889 swtim_free_tim(struct rte_timer *tim, void *arg)
891 struct swtim *sw = arg;
893 rte_mempool_put(sw->tim_pool, tim);
896 /* Traverse the list of outstanding timers and put them back in the mempool
897 * before freeing the adapter to avoid leaking the memory.
900 swtim_uninit(struct rte_event_timer_adapter *adapter)
903 struct swtim *sw = swtim_pmd_priv(adapter);
905 /* Free outstanding timers */
906 rte_timer_stop_all(sw->timer_data_id,
912 ret = rte_service_component_unregister(sw->service_id);
914 EVTIM_LOG_ERR("failed to unregister service component");
918 rte_mempool_free(sw->tim_pool);
920 adapter->data->adapter_priv = NULL;
925 static inline int32_t
926 get_mapped_count_for_service(uint32_t service_id)
928 int32_t core_count, i, mapped_count = 0;
929 uint32_t lcore_arr[RTE_MAX_LCORE];
931 core_count = rte_service_lcore_list(lcore_arr, RTE_MAX_LCORE);
933 for (i = 0; i < core_count; i++)
934 if (rte_service_map_lcore_get(service_id, lcore_arr[i]) == 1)
941 swtim_start(const struct rte_event_timer_adapter *adapter)
944 struct swtim *sw = swtim_pmd_priv(adapter);
946 /* Mapping the service to more than one service core can introduce
947 * delays while one thread is waiting to acquire a lock, so only allow
948 * one core to be mapped to the service.
950 * Note: the service could be modified such that it spreads cores to
951 * poll over multiple service instances.
953 mapped_count = get_mapped_count_for_service(sw->service_id);
955 if (mapped_count != 1)
956 return mapped_count < 1 ? -ENOENT : -ENOTSUP;
958 return rte_service_component_runstate_set(sw->service_id, 1);
962 swtim_stop(const struct rte_event_timer_adapter *adapter)
965 struct swtim *sw = swtim_pmd_priv(adapter);
967 ret = rte_service_component_runstate_set(sw->service_id, 0);
971 /* Wait for the service to complete its final iteration */
972 while (rte_service_may_be_active(sw->service_id))
979 swtim_get_info(const struct rte_event_timer_adapter *adapter,
980 struct rte_event_timer_adapter_info *adapter_info)
982 struct swtim *sw = swtim_pmd_priv(adapter);
983 adapter_info->min_resolution_ns = sw->timer_tick_ns;
984 adapter_info->max_tmo_ns = sw->max_tmo_ns;
988 swtim_stats_get(const struct rte_event_timer_adapter *adapter,
989 struct rte_event_timer_adapter_stats *stats)
991 struct swtim *sw = swtim_pmd_priv(adapter);
992 *stats = sw->stats; /* structure copy */
997 swtim_stats_reset(const struct rte_event_timer_adapter *adapter)
999 struct swtim *sw = swtim_pmd_priv(adapter);
1000 memset(&sw->stats, 0, sizeof(sw->stats));
1005 __swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1006 struct rte_event_timer **evtims,
1010 struct swtim *sw = swtim_pmd_priv(adapter);
1011 uint32_t lcore_id = rte_lcore_id();
1012 struct rte_timer *tim, *tims[nb_evtims];
1015 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1016 /* Check that the service is running. */
1017 if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1023 /* Adjust lcore_id if non-EAL thread. Arbitrarily pick the timer list of
1024 * the highest lcore to insert such timers into
1026 if (lcore_id == LCORE_ID_ANY)
1027 lcore_id = RTE_MAX_LCORE - 1;
1029 /* If this is the first time we're arming an event timer on this lcore,
1030 * mark this lcore as "in use"; this will cause the service
1031 * function to process the timer list that corresponds to this lcore.
1033 if (unlikely(rte_atomic16_test_and_set(&sw->in_use[lcore_id].v))) {
1034 EVTIM_LOG_DBG("Adding lcore id = %u to list of lcores to poll",
1036 sw->poll_lcores[sw->n_poll_lcores] = lcore_id;
1037 ++sw->n_poll_lcores;
1040 ret = rte_mempool_get_bulk(sw->tim_pool, (void **)tims,
1047 for (i = 0; i < nb_evtims; i++) {
1048 /* Don't modify the event timer state in these cases */
1049 if (evtims[i]->state == RTE_EVENT_TIMER_ARMED) {
1050 rte_errno = EALREADY;
1052 } else if (!(evtims[i]->state == RTE_EVENT_TIMER_NOT_ARMED ||
1053 evtims[i]->state == RTE_EVENT_TIMER_CANCELED)) {
1058 ret = check_timeout(evtims[i], adapter);
1059 if (unlikely(ret == -1)) {
1060 evtims[i]->state = RTE_EVENT_TIMER_ERROR_TOOLATE;
1063 } else if (unlikely(ret == -2)) {
1064 evtims[i]->state = RTE_EVENT_TIMER_ERROR_TOOEARLY;
1069 if (unlikely(check_destination_event_queue(evtims[i],
1071 evtims[i]->state = RTE_EVENT_TIMER_ERROR;
1077 rte_timer_init(tim);
1079 evtims[i]->impl_opaque[0] = (uintptr_t)tim;
1080 evtims[i]->impl_opaque[1] = (uintptr_t)adapter;
1082 cycles = get_timeout_cycles(evtims[i], adapter);
1083 ret = rte_timer_alt_reset(sw->timer_data_id, tim, cycles,
1084 SINGLE, lcore_id, NULL, evtims[i]);
1086 /* tim was in RUNNING or CONFIG state */
1087 evtims[i]->state = RTE_EVENT_TIMER_ERROR;
1092 EVTIM_LOG_DBG("armed an event timer");
1093 evtims[i]->state = RTE_EVENT_TIMER_ARMED;
1097 rte_mempool_put_bulk(sw->tim_pool,
1098 (void **)&tims[i], nb_evtims - i);
1104 swtim_arm_burst(const struct rte_event_timer_adapter *adapter,
1105 struct rte_event_timer **evtims,
1108 return __swtim_arm_burst(adapter, evtims, nb_evtims);
1112 swtim_cancel_burst(const struct rte_event_timer_adapter *adapter,
1113 struct rte_event_timer **evtims,
1117 struct rte_timer *timp;
1119 struct swtim *sw = swtim_pmd_priv(adapter);
1121 #ifdef RTE_LIBRTE_EVENTDEV_DEBUG
1122 /* Check that the service is running. */
1123 if (rte_service_runstate_get(adapter->data->service_id) != 1) {
1129 for (i = 0; i < nb_evtims; i++) {
1130 /* Don't modify the event timer state in these cases */
1131 if (evtims[i]->state == RTE_EVENT_TIMER_CANCELED) {
1132 rte_errno = EALREADY;
1134 } else if (evtims[i]->state != RTE_EVENT_TIMER_ARMED) {
1141 opaque = evtims[i]->impl_opaque[0];
1142 timp = (struct rte_timer *)(uintptr_t)opaque;
1143 RTE_ASSERT(timp != NULL);
1145 ret = rte_timer_alt_stop(sw->timer_data_id, timp);
1147 /* Timer is running or being configured */
1152 rte_mempool_put(sw->tim_pool, (void **)timp);
1154 evtims[i]->state = RTE_EVENT_TIMER_CANCELED;
1155 evtims[i]->impl_opaque[0] = 0;
1156 evtims[i]->impl_opaque[1] = 0;
1165 swtim_arm_tmo_tick_burst(const struct rte_event_timer_adapter *adapter,
1166 struct rte_event_timer **evtims,
1167 uint64_t timeout_ticks,
1172 for (i = 0; i < nb_evtims; i++)
1173 evtims[i]->timeout_ticks = timeout_ticks;
1175 return __swtim_arm_burst(adapter, evtims, nb_evtims);
1178 static const struct rte_event_timer_adapter_ops swtim_ops = {
1180 .uninit = swtim_uninit,
1181 .start = swtim_start,
1183 .get_info = swtim_get_info,
1184 .stats_get = swtim_stats_get,
1185 .stats_reset = swtim_stats_reset,
1186 .arm_burst = swtim_arm_burst,
1187 .arm_tmo_tick_burst = swtim_arm_tmo_tick_burst,
1188 .cancel_burst = swtim_cancel_burst,
1191 RTE_INIT(event_timer_adapter_init_log)
1193 evtim_logtype = rte_log_register("lib.eventdev.adapter.timer");
1194 if (evtim_logtype >= 0)
1195 rte_log_set_level(evtim_logtype, RTE_LOG_NOTICE);
1197 evtim_buffer_logtype = rte_log_register("lib.eventdev.adapter.timer."
1199 if (evtim_buffer_logtype >= 0)
1200 rte_log_set_level(evtim_buffer_logtype, RTE_LOG_NOTICE);
1202 evtim_svc_logtype = rte_log_register("lib.eventdev.adapter.timer.svc");
1203 if (evtim_svc_logtype >= 0)
1204 rte_log_set_level(evtim_svc_logtype, RTE_LOG_NOTICE);