1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <sys/queue.h>
13 #include <rte_atomic.h>
14 #include <rte_common.h>
15 #include <rte_cycles.h>
16 #include <rte_per_lcore.h>
17 #include <rte_memory.h>
18 #include <rte_launch.h>
20 #include <rte_lcore.h>
21 #include <rte_branch_prediction.h>
22 #include <rte_spinlock.h>
23 #include <rte_random.h>
24 #include <rte_pause.h>
25 #include <rte_memzone.h>
26 #include <rte_malloc.h>
27 #include <rte_compat.h>
29 #include "rte_timer.h"
32 * Per-lcore info for timers.
35 struct rte_timer pending_head; /**< dummy timer instance to head up list */
36 rte_spinlock_t list_lock; /**< lock to protect list access */
38 /** per-core variable that true if a timer was updated on this
39 * core since last reset of the variable */
42 /** track the current depth of the skiplist */
43 unsigned curr_skiplist_depth;
45 unsigned prev_lcore; /**< used for lcore round robin */
47 /** running timer on this lcore now */
48 struct rte_timer *running_tim;
50 #ifdef RTE_LIBRTE_TIMER_DEBUG
51 /** per-lcore statistics */
52 struct rte_timer_debug_stats stats;
54 } __rte_cache_aligned;
56 #define FL_ALLOCATED (1 << 0)
57 struct rte_timer_data {
58 struct priv_timer priv_timer[RTE_MAX_LCORE];
59 uint8_t internal_flags;
62 #define RTE_MAX_DATA_ELS 64
63 static struct rte_timer_data *rte_timer_data_arr;
64 static const uint32_t default_data_id;
65 static uint32_t rte_timer_subsystem_initialized;
67 /* For maintaining older interfaces for a period */
68 static struct rte_timer_data default_timer_data;
70 /* when debug is enabled, store some statistics */
71 #ifdef RTE_LIBRTE_TIMER_DEBUG
72 #define __TIMER_STAT_ADD(priv_timer, name, n) do { \
73 unsigned __lcore_id = rte_lcore_id(); \
74 if (__lcore_id < RTE_MAX_LCORE) \
75 priv_timer[__lcore_id].stats.name += (n); \
78 #define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0)
82 timer_data_valid(uint32_t id)
84 return !!(rte_timer_data_arr[id].internal_flags & FL_ALLOCATED);
87 /* validate ID and retrieve timer data pointer, or return error value */
88 #define TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, retval) do { \
89 if (id >= RTE_MAX_DATA_ELS || !timer_data_valid(id)) \
91 timer_data = &rte_timer_data_arr[id]; \
94 int __rte_experimental
95 rte_timer_data_alloc(uint32_t *id_ptr)
98 struct rte_timer_data *data;
100 if (!rte_timer_subsystem_initialized)
103 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
104 data = &rte_timer_data_arr[i];
105 if (!(data->internal_flags & FL_ALLOCATED)) {
106 data->internal_flags |= FL_ALLOCATED;
118 int __rte_experimental
119 rte_timer_data_dealloc(uint32_t id)
121 struct rte_timer_data *timer_data;
122 TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, -EINVAL);
124 timer_data->internal_flags &= ~(FL_ALLOCATED);
130 rte_timer_subsystem_init_v20(void)
133 struct priv_timer *priv_timer = default_timer_data.priv_timer;
135 /* since priv_timer is static, it's zeroed by default, so only init some
138 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) {
139 rte_spinlock_init(&priv_timer[lcore_id].list_lock);
140 priv_timer[lcore_id].prev_lcore = lcore_id;
143 VERSION_SYMBOL(rte_timer_subsystem_init, _v20, 2.0);
145 /* Init the timer library. Allocate an array of timer data structs in shared
146 * memory, and allocate the zeroth entry for use with original timer
147 * APIs. Since the intersection of the sets of lcore ids in primary and
148 * secondary processes should be empty, the zeroth entry can be shared by
149 * multiple processes.
152 rte_timer_subsystem_init_v1905(void)
154 const struct rte_memzone *mz;
155 struct rte_timer_data *data;
157 static const char *mz_name = "rte_timer_mz";
159 if (rte_timer_subsystem_initialized)
162 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
163 mz = rte_memzone_lookup(mz_name);
167 rte_timer_data_arr = mz->addr;
169 rte_timer_data_arr[default_data_id].internal_flags |=
172 rte_timer_subsystem_initialized = 1;
177 mz = rte_memzone_reserve_aligned(mz_name,
178 RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr),
179 SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
183 rte_timer_data_arr = mz->addr;
185 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
186 data = &rte_timer_data_arr[i];
188 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
190 &data->priv_timer[lcore_id].list_lock);
191 data->priv_timer[lcore_id].prev_lcore = lcore_id;
195 rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
197 rte_timer_subsystem_initialized = 1;
201 MAP_STATIC_SYMBOL(int rte_timer_subsystem_init(void),
202 rte_timer_subsystem_init_v1905);
203 BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
205 void __rte_experimental
206 rte_timer_subsystem_finalize(void)
208 if (rte_timer_data_arr)
209 rte_free(rte_timer_data_arr);
211 rte_timer_subsystem_initialized = 0;
214 /* Initialize the timer handle tim for use */
216 rte_timer_init(struct rte_timer *tim)
218 union rte_timer_status status;
220 status.state = RTE_TIMER_STOP;
221 status.owner = RTE_TIMER_NO_OWNER;
222 tim->status.u32 = status.u32;
226 * if timer is pending or stopped (or running on the same core than
227 * us), mark timer as configuring, and on success return the previous
228 * status of the timer
231 timer_set_config_state(struct rte_timer *tim,
232 union rte_timer_status *ret_prev_status,
233 struct priv_timer *priv_timer)
235 union rte_timer_status prev_status, status;
239 lcore_id = rte_lcore_id();
241 /* wait that the timer is in correct status before update,
242 * and mark it as being configured */
243 while (success == 0) {
244 prev_status.u32 = tim->status.u32;
246 /* timer is running on another core
247 * or ready to run on local core, exit
249 if (prev_status.state == RTE_TIMER_RUNNING &&
250 (prev_status.owner != (uint16_t)lcore_id ||
251 tim != priv_timer[lcore_id].running_tim))
254 /* timer is being configured on another core */
255 if (prev_status.state == RTE_TIMER_CONFIG)
258 /* here, we know that timer is stopped or pending,
259 * mark it atomically as being configured */
260 status.state = RTE_TIMER_CONFIG;
261 status.owner = (int16_t)lcore_id;
262 success = rte_atomic32_cmpset(&tim->status.u32,
267 ret_prev_status->u32 = prev_status.u32;
272 * if timer is pending, mark timer as running
275 timer_set_running_state(struct rte_timer *tim)
277 union rte_timer_status prev_status, status;
278 unsigned lcore_id = rte_lcore_id();
281 /* wait that the timer is in correct status before update,
282 * and mark it as running */
283 while (success == 0) {
284 prev_status.u32 = tim->status.u32;
286 /* timer is not pending anymore */
287 if (prev_status.state != RTE_TIMER_PENDING)
290 /* here, we know that timer is stopped or pending,
291 * mark it atomically as being configured */
292 status.state = RTE_TIMER_RUNNING;
293 status.owner = (int16_t)lcore_id;
294 success = rte_atomic32_cmpset(&tim->status.u32,
303 * Return a skiplist level for a new entry.
304 * This probabilistically gives a level with p=1/4 that an entry at level n
305 * will also appear at level n+1.
308 timer_get_skiplist_level(unsigned curr_depth)
310 #ifdef RTE_LIBRTE_TIMER_DEBUG
311 static uint32_t i, count = 0;
312 static uint32_t levels[MAX_SKIPLIST_DEPTH] = {0};
315 /* probability value is 1/4, i.e. all at level 0, 1 in 4 is at level 1,
316 * 1 in 16 at level 2, 1 in 64 at level 3, etc. Calculated using lowest
317 * bit position of a (pseudo)random number.
319 uint32_t rand = rte_rand() & (UINT32_MAX - 1);
320 uint32_t level = rand == 0 ? MAX_SKIPLIST_DEPTH : (rte_bsf32(rand)-1) / 2;
322 /* limit the levels used to one above our current level, so we don't,
323 * for instance, have a level 0 and a level 7 without anything between
325 if (level > curr_depth)
327 if (level >= MAX_SKIPLIST_DEPTH)
328 level = MAX_SKIPLIST_DEPTH-1;
329 #ifdef RTE_LIBRTE_TIMER_DEBUG
332 if (count % 10000 == 0)
333 for (i = 0; i < MAX_SKIPLIST_DEPTH; i++)
334 printf("Level %u: %u\n", (unsigned)i, (unsigned)levels[i]);
340 * For a given time value, get the entries at each level which
341 * are <= that time value.
344 timer_get_prev_entries(uint64_t time_val, unsigned tim_lcore,
345 struct rte_timer **prev, struct priv_timer *priv_timer)
347 unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth;
348 prev[lvl] = &priv_timer[tim_lcore].pending_head;
351 prev[lvl] = prev[lvl+1];
352 while (prev[lvl]->sl_next[lvl] &&
353 prev[lvl]->sl_next[lvl]->expire <= time_val)
354 prev[lvl] = prev[lvl]->sl_next[lvl];
359 * Given a timer node in the skiplist, find the previous entries for it at
360 * all skiplist levels.
363 timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore,
364 struct rte_timer **prev,
365 struct priv_timer *priv_timer)
369 /* to get a specific entry in the list, look for just lower than the time
370 * values, and then increment on each level individually if necessary
372 timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);
373 for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) {
374 while (prev[i]->sl_next[i] != NULL &&
375 prev[i]->sl_next[i] != tim &&
376 prev[i]->sl_next[i]->expire <= tim->expire)
377 prev[i] = prev[i]->sl_next[i];
381 /* call with lock held as necessary
383 * timer must be in config state
384 * timer must not be in a list
387 timer_add(struct rte_timer *tim, unsigned int tim_lcore,
388 struct priv_timer *priv_timer)
391 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
393 /* find where exactly this element goes in the list of elements
395 timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);
397 /* now assign it a new level and add at that level */
398 const unsigned tim_level = timer_get_skiplist_level(
399 priv_timer[tim_lcore].curr_skiplist_depth);
400 if (tim_level == priv_timer[tim_lcore].curr_skiplist_depth)
401 priv_timer[tim_lcore].curr_skiplist_depth++;
405 tim->sl_next[lvl] = prev[lvl]->sl_next[lvl];
406 prev[lvl]->sl_next[lvl] = tim;
409 tim->sl_next[0] = prev[0]->sl_next[0];
410 prev[0]->sl_next[0] = tim;
412 /* save the lowest list entry into the expire field of the dummy hdr
413 * NOTE: this is not atomic on 32-bit*/
414 priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\
415 pending_head.sl_next[0]->expire;
419 * del from list, lock if needed
420 * timer must be in config state
421 * timer must be in a list
424 timer_del(struct rte_timer *tim, union rte_timer_status prev_status,
425 int local_is_locked, struct priv_timer *priv_timer)
427 unsigned lcore_id = rte_lcore_id();
428 unsigned prev_owner = prev_status.owner;
430 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
432 /* if timer needs is pending another core, we need to lock the
433 * list; if it is on local core, we need to lock if we are not
434 * called from rte_timer_manage() */
435 if (prev_owner != lcore_id || !local_is_locked)
436 rte_spinlock_lock(&priv_timer[prev_owner].list_lock);
438 /* save the lowest list entry into the expire field of the dummy hdr.
439 * NOTE: this is not atomic on 32-bit */
440 if (tim == priv_timer[prev_owner].pending_head.sl_next[0])
441 priv_timer[prev_owner].pending_head.expire =
442 ((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire);
444 /* adjust pointers from previous entries to point past this */
445 timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);
446 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) {
447 if (prev[i]->sl_next[i] == tim)
448 prev[i]->sl_next[i] = tim->sl_next[i];
451 /* in case we deleted last entry at a level, adjust down max level */
452 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--)
453 if (priv_timer[prev_owner].pending_head.sl_next[i] == NULL)
454 priv_timer[prev_owner].curr_skiplist_depth --;
458 if (prev_owner != lcore_id || !local_is_locked)
459 rte_spinlock_unlock(&priv_timer[prev_owner].list_lock);
462 /* Reset and start the timer associated with the timer handle (private func) */
464 __rte_timer_reset(struct rte_timer *tim, uint64_t expire,
465 uint64_t period, unsigned tim_lcore,
466 rte_timer_cb_t fct, void *arg,
468 struct rte_timer_data *timer_data)
470 union rte_timer_status prev_status, status;
472 unsigned lcore_id = rte_lcore_id();
473 struct priv_timer *priv_timer = timer_data->priv_timer;
475 /* round robin for tim_lcore */
476 if (tim_lcore == (unsigned)LCORE_ID_ANY) {
477 if (lcore_id < RTE_MAX_LCORE) {
478 /* EAL thread with valid lcore_id */
479 tim_lcore = rte_get_next_lcore(
480 priv_timer[lcore_id].prev_lcore,
482 priv_timer[lcore_id].prev_lcore = tim_lcore;
484 /* non-EAL thread do not run rte_timer_manage(),
485 * so schedule the timer on the first enabled lcore. */
486 tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0, 1);
489 /* wait that the timer is in correct status before update,
490 * and mark it as being configured */
491 ret = timer_set_config_state(tim, &prev_status, priv_timer);
495 __TIMER_STAT_ADD(priv_timer, reset, 1);
496 if (prev_status.state == RTE_TIMER_RUNNING &&
497 lcore_id < RTE_MAX_LCORE) {
498 priv_timer[lcore_id].updated = 1;
501 /* remove it from list */
502 if (prev_status.state == RTE_TIMER_PENDING) {
503 timer_del(tim, prev_status, local_is_locked, priv_timer);
504 __TIMER_STAT_ADD(priv_timer, pending, -1);
507 tim->period = period;
508 tim->expire = expire;
512 /* if timer needs to be scheduled on another core, we need to
513 * lock the destination list; if it is on local core, we need to lock if
514 * we are not called from rte_timer_manage()
516 if (tim_lcore != lcore_id || !local_is_locked)
517 rte_spinlock_lock(&priv_timer[tim_lcore].list_lock);
519 __TIMER_STAT_ADD(priv_timer, pending, 1);
520 timer_add(tim, tim_lcore, priv_timer);
522 /* update state: as we are in CONFIG state, only us can modify
523 * the state so we don't need to use cmpset() here */
525 status.state = RTE_TIMER_PENDING;
526 status.owner = (int16_t)tim_lcore;
527 tim->status.u32 = status.u32;
529 if (tim_lcore != lcore_id || !local_is_locked)
530 rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
535 /* Reset and start the timer associated with the timer handle tim */
537 rte_timer_reset_v20(struct rte_timer *tim, uint64_t ticks,
538 enum rte_timer_type type, unsigned int tim_lcore,
539 rte_timer_cb_t fct, void *arg)
541 uint64_t cur_time = rte_get_timer_cycles();
544 if (unlikely((tim_lcore != (unsigned)LCORE_ID_ANY) &&
545 !(rte_lcore_is_enabled(tim_lcore) ||
546 rte_lcore_has_role(tim_lcore, ROLE_SERVICE))))
549 if (type == PERIODICAL)
554 return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
555 fct, arg, 0, &default_timer_data);
557 VERSION_SYMBOL(rte_timer_reset, _v20, 2.0);
560 rte_timer_reset_v1905(struct rte_timer *tim, uint64_t ticks,
561 enum rte_timer_type type, unsigned int tim_lcore,
562 rte_timer_cb_t fct, void *arg)
564 return rte_timer_alt_reset(default_data_id, tim, ticks, type,
565 tim_lcore, fct, arg);
567 MAP_STATIC_SYMBOL(int rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
568 enum rte_timer_type type,
569 unsigned int tim_lcore,
570 rte_timer_cb_t fct, void *arg),
571 rte_timer_reset_v1905);
572 BIND_DEFAULT_SYMBOL(rte_timer_reset, _v1905, 19.05);
574 int __rte_experimental
575 rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,
576 uint64_t ticks, enum rte_timer_type type,
577 unsigned int tim_lcore, rte_timer_cb_t fct, void *arg)
579 uint64_t cur_time = rte_get_timer_cycles();
581 struct rte_timer_data *timer_data;
583 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
585 if (type == PERIODICAL)
590 return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
591 fct, arg, 0, timer_data);
594 /* loop until rte_timer_reset() succeed */
596 rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,
597 enum rte_timer_type type, unsigned tim_lcore,
598 rte_timer_cb_t fct, void *arg)
600 while (rte_timer_reset(tim, ticks, type, tim_lcore,
606 __rte_timer_stop(struct rte_timer *tim, int local_is_locked,
607 struct rte_timer_data *timer_data)
609 union rte_timer_status prev_status, status;
610 unsigned lcore_id = rte_lcore_id();
612 struct priv_timer *priv_timer = timer_data->priv_timer;
614 /* wait that the timer is in correct status before update,
615 * and mark it as being configured */
616 ret = timer_set_config_state(tim, &prev_status, priv_timer);
620 __TIMER_STAT_ADD(priv_timer, stop, 1);
621 if (prev_status.state == RTE_TIMER_RUNNING &&
622 lcore_id < RTE_MAX_LCORE) {
623 priv_timer[lcore_id].updated = 1;
626 /* remove it from list */
627 if (prev_status.state == RTE_TIMER_PENDING) {
628 timer_del(tim, prev_status, local_is_locked, priv_timer);
629 __TIMER_STAT_ADD(priv_timer, pending, -1);
632 /* mark timer as stopped */
634 status.state = RTE_TIMER_STOP;
635 status.owner = RTE_TIMER_NO_OWNER;
636 tim->status.u32 = status.u32;
641 /* Stop the timer associated with the timer handle tim */
643 rte_timer_stop_v20(struct rte_timer *tim)
645 return __rte_timer_stop(tim, 0, &default_timer_data);
647 VERSION_SYMBOL(rte_timer_stop, _v20, 2.0);
650 rte_timer_stop_v1905(struct rte_timer *tim)
652 return rte_timer_alt_stop(default_data_id, tim);
654 MAP_STATIC_SYMBOL(int rte_timer_stop(struct rte_timer *tim),
655 rte_timer_stop_v1905);
656 BIND_DEFAULT_SYMBOL(rte_timer_stop, _v1905, 19.05);
658 int __rte_experimental
659 rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)
661 struct rte_timer_data *timer_data;
663 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
665 return __rte_timer_stop(tim, 0, timer_data);
668 /* loop until rte_timer_stop() succeed */
670 rte_timer_stop_sync(struct rte_timer *tim)
672 while (rte_timer_stop(tim) != 0)
676 /* Test the PENDING status of the timer handle tim */
678 rte_timer_pending(struct rte_timer *tim)
680 return tim->status.state == RTE_TIMER_PENDING;
683 /* must be called periodically, run all timer that expired */
685 __rte_timer_manage(struct rte_timer_data *timer_data)
687 union rte_timer_status status;
688 struct rte_timer *tim, *next_tim;
689 struct rte_timer *run_first_tim, **pprev;
690 unsigned lcore_id = rte_lcore_id();
691 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
694 struct priv_timer *priv_timer = timer_data->priv_timer;
696 /* timer manager only runs on EAL thread with valid lcore_id */
697 assert(lcore_id < RTE_MAX_LCORE);
699 __TIMER_STAT_ADD(priv_timer, manage, 1);
700 /* optimize for the case where per-cpu list is empty */
701 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL)
703 cur_time = rte_get_timer_cycles();
706 /* on 64-bit the value cached in the pending_head.expired will be
707 * updated atomically, so we can consult that for a quick check here
708 * outside the lock */
709 if (likely(priv_timer[lcore_id].pending_head.expire > cur_time))
713 /* browse ordered list, add expired timers in 'expired' list */
714 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
716 /* if nothing to do just unlock and return */
717 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL ||
718 priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) {
719 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
723 /* save start of list of expired timers */
724 tim = priv_timer[lcore_id].pending_head.sl_next[0];
726 /* break the existing list at current time point */
727 timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer);
728 for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) {
729 if (prev[i] == &priv_timer[lcore_id].pending_head)
731 priv_timer[lcore_id].pending_head.sl_next[i] =
733 if (prev[i]->sl_next[i] == NULL)
734 priv_timer[lcore_id].curr_skiplist_depth--;
735 prev[i] ->sl_next[i] = NULL;
738 /* transition run-list from PENDING to RUNNING */
740 pprev = &run_first_tim;
742 for ( ; tim != NULL; tim = next_tim) {
743 next_tim = tim->sl_next[0];
745 ret = timer_set_running_state(tim);
746 if (likely(ret == 0)) {
747 pprev = &tim->sl_next[0];
749 /* another core is trying to re-config this one,
750 * remove it from local expired list
756 /* update the next to expire timer value */
757 priv_timer[lcore_id].pending_head.expire =
758 (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 :
759 priv_timer[lcore_id].pending_head.sl_next[0]->expire;
761 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
763 /* now scan expired list and call callbacks */
764 for (tim = run_first_tim; tim != NULL; tim = next_tim) {
765 next_tim = tim->sl_next[0];
766 priv_timer[lcore_id].updated = 0;
767 priv_timer[lcore_id].running_tim = tim;
769 /* execute callback function with list unlocked */
770 tim->f(tim, tim->arg);
772 __TIMER_STAT_ADD(priv_timer, pending, -1);
773 /* the timer was stopped or reloaded by the callback
774 * function, we have nothing to do here */
775 if (priv_timer[lcore_id].updated == 1)
778 if (tim->period == 0) {
779 /* remove from done list and mark timer as stopped */
780 status.state = RTE_TIMER_STOP;
781 status.owner = RTE_TIMER_NO_OWNER;
783 tim->status.u32 = status.u32;
786 /* keep it in list and mark timer as pending */
787 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
788 status.state = RTE_TIMER_PENDING;
789 __TIMER_STAT_ADD(priv_timer, pending, 1);
790 status.owner = (int16_t)lcore_id;
792 tim->status.u32 = status.u32;
793 __rte_timer_reset(tim, tim->expire + tim->period,
794 tim->period, lcore_id, tim->f, tim->arg, 1,
796 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
799 priv_timer[lcore_id].running_tim = NULL;
803 rte_timer_manage_v20(void)
805 __rte_timer_manage(&default_timer_data);
807 VERSION_SYMBOL(rte_timer_manage, _v20, 2.0);
810 rte_timer_manage_v1905(void)
812 struct rte_timer_data *timer_data;
814 TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
816 __rte_timer_manage(timer_data);
820 MAP_STATIC_SYMBOL(int rte_timer_manage(void), rte_timer_manage_v1905);
821 BIND_DEFAULT_SYMBOL(rte_timer_manage, _v1905, 19.05);
823 int __rte_experimental
824 rte_timer_alt_manage(uint32_t timer_data_id,
825 unsigned int *poll_lcores,
827 rte_timer_alt_manage_cb_t f)
829 unsigned int default_poll_lcores[] = {rte_lcore_id()};
830 union rte_timer_status status;
831 struct rte_timer *tim, *next_tim, **pprev;
832 struct rte_timer *run_first_tims[RTE_MAX_LCORE];
833 unsigned int runlist_lcore_ids[RTE_MAX_LCORE];
834 unsigned int this_lcore = rte_lcore_id();
835 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
839 struct rte_timer_data *data;
840 struct priv_timer *privp;
843 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, data, -EINVAL);
845 /* timer manager only runs on EAL thread with valid lcore_id */
846 assert(this_lcore < RTE_MAX_LCORE);
848 __TIMER_STAT_ADD(data->priv_timer, manage, 1);
850 if (poll_lcores == NULL) {
851 poll_lcores = default_poll_lcores;
852 nb_poll_lcores = RTE_DIM(default_poll_lcores);
855 for (i = 0; i < nb_poll_lcores; i++) {
856 poll_lcore = poll_lcores[i];
857 privp = &data->priv_timer[poll_lcore];
859 /* optimize for the case where per-cpu list is empty */
860 if (privp->pending_head.sl_next[0] == NULL)
862 cur_time = rte_get_timer_cycles();
865 /* on 64-bit the value cached in the pending_head.expired will
866 * be updated atomically, so we can consult that for a quick
867 * check here outside the lock
869 if (likely(privp->pending_head.expire > cur_time))
873 /* browse ordered list, add expired timers in 'expired' list */
874 rte_spinlock_lock(&privp->list_lock);
876 /* if nothing to do just unlock and return */
877 if (privp->pending_head.sl_next[0] == NULL ||
878 privp->pending_head.sl_next[0]->expire > cur_time) {
879 rte_spinlock_unlock(&privp->list_lock);
883 /* save start of list of expired timers */
884 tim = privp->pending_head.sl_next[0];
886 /* break the existing list at current time point */
887 timer_get_prev_entries(cur_time, poll_lcore, prev,
889 for (j = privp->curr_skiplist_depth - 1; j >= 0; j--) {
890 if (prev[j] == &privp->pending_head)
892 privp->pending_head.sl_next[j] =
894 if (prev[j]->sl_next[j] == NULL)
895 privp->curr_skiplist_depth--;
897 prev[j]->sl_next[j] = NULL;
900 /* transition run-list from PENDING to RUNNING */
901 run_first_tims[nb_runlists] = tim;
902 runlist_lcore_ids[nb_runlists] = poll_lcore;
903 pprev = &run_first_tims[nb_runlists];
906 for ( ; tim != NULL; tim = next_tim) {
907 next_tim = tim->sl_next[0];
909 ret = timer_set_running_state(tim);
910 if (likely(ret == 0)) {
911 pprev = &tim->sl_next[0];
913 /* another core is trying to re-config this one,
914 * remove it from local expired list
920 /* update the next to expire timer value */
921 privp->pending_head.expire =
922 (privp->pending_head.sl_next[0] == NULL) ? 0 :
923 privp->pending_head.sl_next[0]->expire;
925 rte_spinlock_unlock(&privp->list_lock);
928 /* Now process the run lists */
931 uint64_t min_expire = UINT64_MAX;
934 /* Find the next oldest timer to process */
935 for (i = 0; i < nb_runlists; i++) {
936 tim = run_first_tims[i];
938 if (tim != NULL && tim->expire < min_expire) {
939 min_expire = tim->expire;
948 tim = run_first_tims[min_idx];
949 privp = &data->priv_timer[runlist_lcore_ids[min_idx]];
951 /* Move down the runlist from which we picked a timer to
954 run_first_tims[min_idx] = run_first_tims[min_idx]->sl_next[0];
957 privp->running_tim = tim;
959 /* Call the provided callback function */
962 __TIMER_STAT_ADD(privp, pending, -1);
964 /* the timer was stopped or reloaded by the callback
965 * function, we have nothing to do here
967 if (privp->updated == 1)
970 if (tim->period == 0) {
971 /* remove from done list and mark timer as stopped */
972 status.state = RTE_TIMER_STOP;
973 status.owner = RTE_TIMER_NO_OWNER;
975 tim->status.u32 = status.u32;
977 /* keep it in list and mark timer as pending */
979 &data->priv_timer[this_lcore].list_lock);
980 status.state = RTE_TIMER_PENDING;
981 __TIMER_STAT_ADD(data->priv_timer, pending, 1);
982 status.owner = (int16_t)this_lcore;
984 tim->status.u32 = status.u32;
985 __rte_timer_reset(tim, tim->expire + tim->period,
986 tim->period, this_lcore, tim->f, tim->arg, 1,
989 &data->priv_timer[this_lcore].list_lock);
992 privp->running_tim = NULL;
998 /* Walk pending lists, stopping timers and calling user-specified function */
999 int __rte_experimental
1000 rte_timer_stop_all(uint32_t timer_data_id, unsigned int *walk_lcores,
1002 rte_timer_stop_all_cb_t f, void *f_arg)
1005 struct priv_timer *priv_timer;
1006 uint32_t walk_lcore;
1007 struct rte_timer *tim, *next_tim;
1008 struct rte_timer_data *timer_data;
1010 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
1012 for (i = 0; i < nb_walk_lcores; i++) {
1013 walk_lcore = walk_lcores[i];
1014 priv_timer = &timer_data->priv_timer[walk_lcore];
1016 rte_spinlock_lock(&priv_timer->list_lock);
1018 for (tim = priv_timer->pending_head.sl_next[0];
1021 next_tim = tim->sl_next[0];
1023 /* Call timer_stop with lock held */
1024 __rte_timer_stop(tim, 1, timer_data);
1030 rte_spinlock_unlock(&priv_timer->list_lock);
1036 /* dump statistics about timers */
1038 __rte_timer_dump_stats(struct rte_timer_data *timer_data __rte_unused, FILE *f)
1040 #ifdef RTE_LIBRTE_TIMER_DEBUG
1041 struct rte_timer_debug_stats sum;
1043 struct priv_timer *priv_timer = timer_data->priv_timer;
1045 memset(&sum, 0, sizeof(sum));
1046 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1047 sum.reset += priv_timer[lcore_id].stats.reset;
1048 sum.stop += priv_timer[lcore_id].stats.stop;
1049 sum.manage += priv_timer[lcore_id].stats.manage;
1050 sum.pending += priv_timer[lcore_id].stats.pending;
1052 fprintf(f, "Timer statistics:\n");
1053 fprintf(f, " reset = %"PRIu64"\n", sum.reset);
1054 fprintf(f, " stop = %"PRIu64"\n", sum.stop);
1055 fprintf(f, " manage = %"PRIu64"\n", sum.manage);
1056 fprintf(f, " pending = %"PRIu64"\n", sum.pending);
1058 fprintf(f, "No timer statistics, RTE_LIBRTE_TIMER_DEBUG is disabled\n");
1063 rte_timer_dump_stats_v20(FILE *f)
1065 __rte_timer_dump_stats(&default_timer_data, f);
1067 VERSION_SYMBOL(rte_timer_dump_stats, _v20, 2.0);
1070 rte_timer_dump_stats_v1905(FILE *f)
1072 return rte_timer_alt_dump_stats(default_data_id, f);
1074 MAP_STATIC_SYMBOL(int rte_timer_dump_stats(FILE *f),
1075 rte_timer_dump_stats_v1905);
1076 BIND_DEFAULT_SYMBOL(rte_timer_dump_stats, _v1905, 19.05);
1078 int __rte_experimental
1079 rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f)
1081 struct rte_timer_data *timer_data;
1083 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
1085 __rte_timer_dump_stats(timer_data, f);