1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <sys/queue.h>
13 #include <rte_atomic.h>
14 #include <rte_common.h>
15 #include <rte_cycles.h>
16 #include <rte_eal_memconfig.h>
17 #include <rte_per_lcore.h>
18 #include <rte_memory.h>
19 #include <rte_launch.h>
21 #include <rte_lcore.h>
22 #include <rte_branch_prediction.h>
23 #include <rte_spinlock.h>
24 #include <rte_random.h>
25 #include <rte_pause.h>
26 #include <rte_memzone.h>
27 #include <rte_malloc.h>
28 #include <rte_compat.h>
29 #include <rte_errno.h>
31 #include "rte_timer.h"
34 * Per-lcore info for timers.
37 struct rte_timer pending_head; /**< dummy timer instance to head up list */
38 rte_spinlock_t list_lock; /**< lock to protect list access */
40 /** per-core variable that true if a timer was updated on this
41 * core since last reset of the variable */
44 /** track the current depth of the skiplist */
45 unsigned curr_skiplist_depth;
47 unsigned prev_lcore; /**< used for lcore round robin */
49 /** running timer on this lcore now */
50 struct rte_timer *running_tim;
52 #ifdef RTE_LIBRTE_TIMER_DEBUG
53 /** per-lcore statistics */
54 struct rte_timer_debug_stats stats;
56 } __rte_cache_aligned;
58 #define FL_ALLOCATED (1 << 0)
59 struct rte_timer_data {
60 struct priv_timer priv_timer[RTE_MAX_LCORE];
61 uint8_t internal_flags;
64 #define RTE_MAX_DATA_ELS 64
65 static const struct rte_memzone *rte_timer_data_mz;
66 static int *volatile rte_timer_mz_refcnt;
67 static struct rte_timer_data *rte_timer_data_arr;
68 static const uint32_t default_data_id;
69 static uint32_t rte_timer_subsystem_initialized;
71 /* For maintaining older interfaces for a period */
72 static struct rte_timer_data default_timer_data;
74 /* when debug is enabled, store some statistics */
75 #ifdef RTE_LIBRTE_TIMER_DEBUG
76 #define __TIMER_STAT_ADD(priv_timer, name, n) do { \
77 unsigned __lcore_id = rte_lcore_id(); \
78 if (__lcore_id < RTE_MAX_LCORE) \
79 priv_timer[__lcore_id].stats.name += (n); \
82 #define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0)
86 timer_data_valid(uint32_t id)
88 return rte_timer_data_arr &&
89 (rte_timer_data_arr[id].internal_flags & FL_ALLOCATED);
92 /* validate ID and retrieve timer data pointer, or return error value */
93 #define TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, retval) do { \
94 if (id >= RTE_MAX_DATA_ELS || !timer_data_valid(id)) \
96 timer_data = &rte_timer_data_arr[id]; \
100 rte_timer_data_alloc(uint32_t *id_ptr)
103 struct rte_timer_data *data;
105 if (!rte_timer_subsystem_initialized)
108 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
109 data = &rte_timer_data_arr[i];
110 if (!(data->internal_flags & FL_ALLOCATED)) {
111 data->internal_flags |= FL_ALLOCATED;
124 rte_timer_data_dealloc(uint32_t id)
126 struct rte_timer_data *timer_data;
127 TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, -EINVAL);
129 timer_data->internal_flags &= ~(FL_ALLOCATED);
135 rte_timer_subsystem_init_v20(void)
138 struct priv_timer *priv_timer = default_timer_data.priv_timer;
140 /* since priv_timer is static, it's zeroed by default, so only init some
143 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) {
144 rte_spinlock_init(&priv_timer[lcore_id].list_lock);
145 priv_timer[lcore_id].prev_lcore = lcore_id;
148 VERSION_SYMBOL(rte_timer_subsystem_init, _v20, 2.0);
150 /* Init the timer library. Allocate an array of timer data structs in shared
151 * memory, and allocate the zeroth entry for use with original timer
152 * APIs. Since the intersection of the sets of lcore ids in primary and
153 * secondary processes should be empty, the zeroth entry can be shared by
154 * multiple processes.
157 rte_timer_subsystem_init_v1905(void)
159 const struct rte_memzone *mz;
160 struct rte_timer_data *data;
162 static const char *mz_name = "rte_timer_mz";
163 const size_t data_arr_size =
164 RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
165 const size_t mem_size = data_arr_size + sizeof(*rte_timer_mz_refcnt);
166 bool do_full_init = true;
168 if (rte_timer_subsystem_initialized)
171 rte_mcfg_timer_lock();
173 mz = rte_memzone_lookup(mz_name);
175 mz = rte_memzone_reserve_aligned(mz_name, mem_size,
176 SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
178 rte_mcfg_timer_unlock();
183 do_full_init = false;
185 rte_timer_data_mz = mz;
186 rte_timer_data_arr = mz->addr;
187 rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
190 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
191 data = &rte_timer_data_arr[i];
193 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE;
196 &data->priv_timer[lcore_id].list_lock);
197 data->priv_timer[lcore_id].prev_lcore =
203 rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
204 (*rte_timer_mz_refcnt)++;
206 rte_mcfg_timer_unlock();
208 rte_timer_subsystem_initialized = 1;
212 MAP_STATIC_SYMBOL(int rte_timer_subsystem_init(void),
213 rte_timer_subsystem_init_v1905);
214 BIND_DEFAULT_SYMBOL(rte_timer_subsystem_init, _v1905, 19.05);
217 rte_timer_subsystem_finalize(void)
219 if (!rte_timer_subsystem_initialized)
222 rte_mcfg_timer_lock();
224 if (--(*rte_timer_mz_refcnt) == 0)
225 rte_memzone_free(rte_timer_data_mz);
227 rte_mcfg_timer_unlock();
229 rte_timer_subsystem_initialized = 0;
232 /* Initialize the timer handle tim for use */
234 rte_timer_init(struct rte_timer *tim)
236 union rte_timer_status status;
238 status.state = RTE_TIMER_STOP;
239 status.owner = RTE_TIMER_NO_OWNER;
240 tim->status.u32 = status.u32;
244 * if timer is pending or stopped (or running on the same core than
245 * us), mark timer as configuring, and on success return the previous
246 * status of the timer
249 timer_set_config_state(struct rte_timer *tim,
250 union rte_timer_status *ret_prev_status,
251 struct priv_timer *priv_timer)
253 union rte_timer_status prev_status, status;
257 lcore_id = rte_lcore_id();
259 /* wait that the timer is in correct status before update,
260 * and mark it as being configured */
261 while (success == 0) {
262 prev_status.u32 = tim->status.u32;
264 /* timer is running on another core
265 * or ready to run on local core, exit
267 if (prev_status.state == RTE_TIMER_RUNNING &&
268 (prev_status.owner != (uint16_t)lcore_id ||
269 tim != priv_timer[lcore_id].running_tim))
272 /* timer is being configured on another core */
273 if (prev_status.state == RTE_TIMER_CONFIG)
276 /* here, we know that timer is stopped or pending,
277 * mark it atomically as being configured */
278 status.state = RTE_TIMER_CONFIG;
279 status.owner = (int16_t)lcore_id;
280 success = rte_atomic32_cmpset(&tim->status.u32,
285 ret_prev_status->u32 = prev_status.u32;
290 * if timer is pending, mark timer as running
293 timer_set_running_state(struct rte_timer *tim)
295 union rte_timer_status prev_status, status;
296 unsigned lcore_id = rte_lcore_id();
299 /* wait that the timer is in correct status before update,
300 * and mark it as running */
301 while (success == 0) {
302 prev_status.u32 = tim->status.u32;
304 /* timer is not pending anymore */
305 if (prev_status.state != RTE_TIMER_PENDING)
308 /* here, we know that timer is stopped or pending,
309 * mark it atomically as being configured */
310 status.state = RTE_TIMER_RUNNING;
311 status.owner = (int16_t)lcore_id;
312 success = rte_atomic32_cmpset(&tim->status.u32,
321 * Return a skiplist level for a new entry.
322 * This probabilistically gives a level with p=1/4 that an entry at level n
323 * will also appear at level n+1.
326 timer_get_skiplist_level(unsigned curr_depth)
328 #ifdef RTE_LIBRTE_TIMER_DEBUG
329 static uint32_t i, count = 0;
330 static uint32_t levels[MAX_SKIPLIST_DEPTH] = {0};
333 /* probability value is 1/4, i.e. all at level 0, 1 in 4 is at level 1,
334 * 1 in 16 at level 2, 1 in 64 at level 3, etc. Calculated using lowest
335 * bit position of a (pseudo)random number.
337 uint32_t rand = rte_rand() & (UINT32_MAX - 1);
338 uint32_t level = rand == 0 ? MAX_SKIPLIST_DEPTH : (rte_bsf32(rand)-1) / 2;
340 /* limit the levels used to one above our current level, so we don't,
341 * for instance, have a level 0 and a level 7 without anything between
343 if (level > curr_depth)
345 if (level >= MAX_SKIPLIST_DEPTH)
346 level = MAX_SKIPLIST_DEPTH-1;
347 #ifdef RTE_LIBRTE_TIMER_DEBUG
350 if (count % 10000 == 0)
351 for (i = 0; i < MAX_SKIPLIST_DEPTH; i++)
352 printf("Level %u: %u\n", (unsigned)i, (unsigned)levels[i]);
358 * For a given time value, get the entries at each level which
359 * are <= that time value.
362 timer_get_prev_entries(uint64_t time_val, unsigned tim_lcore,
363 struct rte_timer **prev, struct priv_timer *priv_timer)
365 unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth;
366 prev[lvl] = &priv_timer[tim_lcore].pending_head;
369 prev[lvl] = prev[lvl+1];
370 while (prev[lvl]->sl_next[lvl] &&
371 prev[lvl]->sl_next[lvl]->expire <= time_val)
372 prev[lvl] = prev[lvl]->sl_next[lvl];
377 * Given a timer node in the skiplist, find the previous entries for it at
378 * all skiplist levels.
381 timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore,
382 struct rte_timer **prev,
383 struct priv_timer *priv_timer)
387 /* to get a specific entry in the list, look for just lower than the time
388 * values, and then increment on each level individually if necessary
390 timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);
391 for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) {
392 while (prev[i]->sl_next[i] != NULL &&
393 prev[i]->sl_next[i] != tim &&
394 prev[i]->sl_next[i]->expire <= tim->expire)
395 prev[i] = prev[i]->sl_next[i];
399 /* call with lock held as necessary
401 * timer must be in config state
402 * timer must not be in a list
405 timer_add(struct rte_timer *tim, unsigned int tim_lcore,
406 struct priv_timer *priv_timer)
409 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
411 /* find where exactly this element goes in the list of elements
413 timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);
415 /* now assign it a new level and add at that level */
416 const unsigned tim_level = timer_get_skiplist_level(
417 priv_timer[tim_lcore].curr_skiplist_depth);
418 if (tim_level == priv_timer[tim_lcore].curr_skiplist_depth)
419 priv_timer[tim_lcore].curr_skiplist_depth++;
423 tim->sl_next[lvl] = prev[lvl]->sl_next[lvl];
424 prev[lvl]->sl_next[lvl] = tim;
427 tim->sl_next[0] = prev[0]->sl_next[0];
428 prev[0]->sl_next[0] = tim;
430 /* save the lowest list entry into the expire field of the dummy hdr
431 * NOTE: this is not atomic on 32-bit*/
432 priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\
433 pending_head.sl_next[0]->expire;
437 * del from list, lock if needed
438 * timer must be in config state
439 * timer must be in a list
442 timer_del(struct rte_timer *tim, union rte_timer_status prev_status,
443 int local_is_locked, struct priv_timer *priv_timer)
445 unsigned lcore_id = rte_lcore_id();
446 unsigned prev_owner = prev_status.owner;
448 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
450 /* if timer needs is pending another core, we need to lock the
451 * list; if it is on local core, we need to lock if we are not
452 * called from rte_timer_manage() */
453 if (prev_owner != lcore_id || !local_is_locked)
454 rte_spinlock_lock(&priv_timer[prev_owner].list_lock);
456 /* save the lowest list entry into the expire field of the dummy hdr.
457 * NOTE: this is not atomic on 32-bit */
458 if (tim == priv_timer[prev_owner].pending_head.sl_next[0])
459 priv_timer[prev_owner].pending_head.expire =
460 ((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire);
462 /* adjust pointers from previous entries to point past this */
463 timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);
464 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) {
465 if (prev[i]->sl_next[i] == tim)
466 prev[i]->sl_next[i] = tim->sl_next[i];
469 /* in case we deleted last entry at a level, adjust down max level */
470 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--)
471 if (priv_timer[prev_owner].pending_head.sl_next[i] == NULL)
472 priv_timer[prev_owner].curr_skiplist_depth --;
476 if (prev_owner != lcore_id || !local_is_locked)
477 rte_spinlock_unlock(&priv_timer[prev_owner].list_lock);
480 /* Reset and start the timer associated with the timer handle (private func) */
482 __rte_timer_reset(struct rte_timer *tim, uint64_t expire,
483 uint64_t period, unsigned tim_lcore,
484 rte_timer_cb_t fct, void *arg,
486 struct rte_timer_data *timer_data)
488 union rte_timer_status prev_status, status;
490 unsigned lcore_id = rte_lcore_id();
491 struct priv_timer *priv_timer = timer_data->priv_timer;
493 /* round robin for tim_lcore */
494 if (tim_lcore == (unsigned)LCORE_ID_ANY) {
495 if (lcore_id < RTE_MAX_LCORE) {
496 /* EAL thread with valid lcore_id */
497 tim_lcore = rte_get_next_lcore(
498 priv_timer[lcore_id].prev_lcore,
500 priv_timer[lcore_id].prev_lcore = tim_lcore;
502 /* non-EAL thread do not run rte_timer_manage(),
503 * so schedule the timer on the first enabled lcore. */
504 tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0, 1);
507 /* wait that the timer is in correct status before update,
508 * and mark it as being configured */
509 ret = timer_set_config_state(tim, &prev_status, priv_timer);
513 __TIMER_STAT_ADD(priv_timer, reset, 1);
514 if (prev_status.state == RTE_TIMER_RUNNING &&
515 lcore_id < RTE_MAX_LCORE) {
516 priv_timer[lcore_id].updated = 1;
519 /* remove it from list */
520 if (prev_status.state == RTE_TIMER_PENDING) {
521 timer_del(tim, prev_status, local_is_locked, priv_timer);
522 __TIMER_STAT_ADD(priv_timer, pending, -1);
525 tim->period = period;
526 tim->expire = expire;
530 /* if timer needs to be scheduled on another core, we need to
531 * lock the destination list; if it is on local core, we need to lock if
532 * we are not called from rte_timer_manage()
534 if (tim_lcore != lcore_id || !local_is_locked)
535 rte_spinlock_lock(&priv_timer[tim_lcore].list_lock);
537 __TIMER_STAT_ADD(priv_timer, pending, 1);
538 timer_add(tim, tim_lcore, priv_timer);
540 /* update state: as we are in CONFIG state, only us can modify
541 * the state so we don't need to use cmpset() here */
543 status.state = RTE_TIMER_PENDING;
544 status.owner = (int16_t)tim_lcore;
545 tim->status.u32 = status.u32;
547 if (tim_lcore != lcore_id || !local_is_locked)
548 rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
553 /* Reset and start the timer associated with the timer handle tim */
555 rte_timer_reset_v20(struct rte_timer *tim, uint64_t ticks,
556 enum rte_timer_type type, unsigned int tim_lcore,
557 rte_timer_cb_t fct, void *arg)
559 uint64_t cur_time = rte_get_timer_cycles();
562 if (unlikely((tim_lcore != (unsigned)LCORE_ID_ANY) &&
563 !(rte_lcore_is_enabled(tim_lcore) ||
564 rte_lcore_has_role(tim_lcore, ROLE_SERVICE))))
567 if (type == PERIODICAL)
572 return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
573 fct, arg, 0, &default_timer_data);
575 VERSION_SYMBOL(rte_timer_reset, _v20, 2.0);
578 rte_timer_reset_v1905(struct rte_timer *tim, uint64_t ticks,
579 enum rte_timer_type type, unsigned int tim_lcore,
580 rte_timer_cb_t fct, void *arg)
582 return rte_timer_alt_reset(default_data_id, tim, ticks, type,
583 tim_lcore, fct, arg);
585 MAP_STATIC_SYMBOL(int rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
586 enum rte_timer_type type,
587 unsigned int tim_lcore,
588 rte_timer_cb_t fct, void *arg),
589 rte_timer_reset_v1905);
590 BIND_DEFAULT_SYMBOL(rte_timer_reset, _v1905, 19.05);
593 rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,
594 uint64_t ticks, enum rte_timer_type type,
595 unsigned int tim_lcore, rte_timer_cb_t fct, void *arg)
597 uint64_t cur_time = rte_get_timer_cycles();
599 struct rte_timer_data *timer_data;
601 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
603 if (type == PERIODICAL)
608 return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
609 fct, arg, 0, timer_data);
612 /* loop until rte_timer_reset() succeed */
614 rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,
615 enum rte_timer_type type, unsigned tim_lcore,
616 rte_timer_cb_t fct, void *arg)
618 while (rte_timer_reset(tim, ticks, type, tim_lcore,
624 __rte_timer_stop(struct rte_timer *tim, int local_is_locked,
625 struct rte_timer_data *timer_data)
627 union rte_timer_status prev_status, status;
628 unsigned lcore_id = rte_lcore_id();
630 struct priv_timer *priv_timer = timer_data->priv_timer;
632 /* wait that the timer is in correct status before update,
633 * and mark it as being configured */
634 ret = timer_set_config_state(tim, &prev_status, priv_timer);
638 __TIMER_STAT_ADD(priv_timer, stop, 1);
639 if (prev_status.state == RTE_TIMER_RUNNING &&
640 lcore_id < RTE_MAX_LCORE) {
641 priv_timer[lcore_id].updated = 1;
644 /* remove it from list */
645 if (prev_status.state == RTE_TIMER_PENDING) {
646 timer_del(tim, prev_status, local_is_locked, priv_timer);
647 __TIMER_STAT_ADD(priv_timer, pending, -1);
650 /* mark timer as stopped */
652 status.state = RTE_TIMER_STOP;
653 status.owner = RTE_TIMER_NO_OWNER;
654 tim->status.u32 = status.u32;
659 /* Stop the timer associated with the timer handle tim */
661 rte_timer_stop_v20(struct rte_timer *tim)
663 return __rte_timer_stop(tim, 0, &default_timer_data);
665 VERSION_SYMBOL(rte_timer_stop, _v20, 2.0);
668 rte_timer_stop_v1905(struct rte_timer *tim)
670 return rte_timer_alt_stop(default_data_id, tim);
672 MAP_STATIC_SYMBOL(int rte_timer_stop(struct rte_timer *tim),
673 rte_timer_stop_v1905);
674 BIND_DEFAULT_SYMBOL(rte_timer_stop, _v1905, 19.05);
677 rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)
679 struct rte_timer_data *timer_data;
681 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
683 return __rte_timer_stop(tim, 0, timer_data);
686 /* loop until rte_timer_stop() succeed */
688 rte_timer_stop_sync(struct rte_timer *tim)
690 while (rte_timer_stop(tim) != 0)
694 /* Test the PENDING status of the timer handle tim */
696 rte_timer_pending(struct rte_timer *tim)
698 return tim->status.state == RTE_TIMER_PENDING;
701 /* must be called periodically, run all timer that expired */
703 __rte_timer_manage(struct rte_timer_data *timer_data)
705 union rte_timer_status status;
706 struct rte_timer *tim, *next_tim;
707 struct rte_timer *run_first_tim, **pprev;
708 unsigned lcore_id = rte_lcore_id();
709 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
712 struct priv_timer *priv_timer = timer_data->priv_timer;
714 /* timer manager only runs on EAL thread with valid lcore_id */
715 assert(lcore_id < RTE_MAX_LCORE);
717 __TIMER_STAT_ADD(priv_timer, manage, 1);
718 /* optimize for the case where per-cpu list is empty */
719 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL)
721 cur_time = rte_get_timer_cycles();
724 /* on 64-bit the value cached in the pending_head.expired will be
725 * updated atomically, so we can consult that for a quick check here
726 * outside the lock */
727 if (likely(priv_timer[lcore_id].pending_head.expire > cur_time))
731 /* browse ordered list, add expired timers in 'expired' list */
732 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
734 /* if nothing to do just unlock and return */
735 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL ||
736 priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) {
737 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
741 /* save start of list of expired timers */
742 tim = priv_timer[lcore_id].pending_head.sl_next[0];
744 /* break the existing list at current time point */
745 timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer);
746 for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) {
747 if (prev[i] == &priv_timer[lcore_id].pending_head)
749 priv_timer[lcore_id].pending_head.sl_next[i] =
751 if (prev[i]->sl_next[i] == NULL)
752 priv_timer[lcore_id].curr_skiplist_depth--;
753 prev[i] ->sl_next[i] = NULL;
756 /* transition run-list from PENDING to RUNNING */
758 pprev = &run_first_tim;
760 for ( ; tim != NULL; tim = next_tim) {
761 next_tim = tim->sl_next[0];
763 ret = timer_set_running_state(tim);
764 if (likely(ret == 0)) {
765 pprev = &tim->sl_next[0];
767 /* another core is trying to re-config this one,
768 * remove it from local expired list
774 /* update the next to expire timer value */
775 priv_timer[lcore_id].pending_head.expire =
776 (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 :
777 priv_timer[lcore_id].pending_head.sl_next[0]->expire;
779 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
781 /* now scan expired list and call callbacks */
782 for (tim = run_first_tim; tim != NULL; tim = next_tim) {
783 next_tim = tim->sl_next[0];
784 priv_timer[lcore_id].updated = 0;
785 priv_timer[lcore_id].running_tim = tim;
787 /* execute callback function with list unlocked */
788 tim->f(tim, tim->arg);
790 __TIMER_STAT_ADD(priv_timer, pending, -1);
791 /* the timer was stopped or reloaded by the callback
792 * function, we have nothing to do here */
793 if (priv_timer[lcore_id].updated == 1)
796 if (tim->period == 0) {
797 /* remove from done list and mark timer as stopped */
798 status.state = RTE_TIMER_STOP;
799 status.owner = RTE_TIMER_NO_OWNER;
801 tim->status.u32 = status.u32;
804 /* keep it in list and mark timer as pending */
805 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
806 status.state = RTE_TIMER_PENDING;
807 __TIMER_STAT_ADD(priv_timer, pending, 1);
808 status.owner = (int16_t)lcore_id;
810 tim->status.u32 = status.u32;
811 __rte_timer_reset(tim, tim->expire + tim->period,
812 tim->period, lcore_id, tim->f, tim->arg, 1,
814 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
817 priv_timer[lcore_id].running_tim = NULL;
821 rte_timer_manage_v20(void)
823 __rte_timer_manage(&default_timer_data);
825 VERSION_SYMBOL(rte_timer_manage, _v20, 2.0);
828 rte_timer_manage_v1905(void)
830 struct rte_timer_data *timer_data;
832 TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
834 __rte_timer_manage(timer_data);
838 MAP_STATIC_SYMBOL(int rte_timer_manage(void), rte_timer_manage_v1905);
839 BIND_DEFAULT_SYMBOL(rte_timer_manage, _v1905, 19.05);
842 rte_timer_alt_manage(uint32_t timer_data_id,
843 unsigned int *poll_lcores,
845 rte_timer_alt_manage_cb_t f)
847 unsigned int default_poll_lcores[] = {rte_lcore_id()};
848 union rte_timer_status status;
849 struct rte_timer *tim, *next_tim, **pprev;
850 struct rte_timer *run_first_tims[RTE_MAX_LCORE];
851 unsigned int this_lcore = rte_lcore_id();
852 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
856 struct rte_timer_data *data;
857 struct priv_timer *privp;
860 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, data, -EINVAL);
862 /* timer manager only runs on EAL thread with valid lcore_id */
863 assert(this_lcore < RTE_MAX_LCORE);
865 __TIMER_STAT_ADD(data->priv_timer, manage, 1);
867 if (poll_lcores == NULL) {
868 poll_lcores = default_poll_lcores;
869 nb_poll_lcores = RTE_DIM(default_poll_lcores);
872 for (i = 0; i < nb_poll_lcores; i++) {
873 poll_lcore = poll_lcores[i];
874 privp = &data->priv_timer[poll_lcore];
876 /* optimize for the case where per-cpu list is empty */
877 if (privp->pending_head.sl_next[0] == NULL)
879 cur_time = rte_get_timer_cycles();
882 /* on 64-bit the value cached in the pending_head.expired will
883 * be updated atomically, so we can consult that for a quick
884 * check here outside the lock
886 if (likely(privp->pending_head.expire > cur_time))
890 /* browse ordered list, add expired timers in 'expired' list */
891 rte_spinlock_lock(&privp->list_lock);
893 /* if nothing to do just unlock and return */
894 if (privp->pending_head.sl_next[0] == NULL ||
895 privp->pending_head.sl_next[0]->expire > cur_time) {
896 rte_spinlock_unlock(&privp->list_lock);
900 /* save start of list of expired timers */
901 tim = privp->pending_head.sl_next[0];
903 /* break the existing list at current time point */
904 timer_get_prev_entries(cur_time, poll_lcore, prev,
906 for (j = privp->curr_skiplist_depth - 1; j >= 0; j--) {
907 if (prev[j] == &privp->pending_head)
909 privp->pending_head.sl_next[j] =
911 if (prev[j]->sl_next[j] == NULL)
912 privp->curr_skiplist_depth--;
914 prev[j]->sl_next[j] = NULL;
917 /* transition run-list from PENDING to RUNNING */
918 run_first_tims[nb_runlists] = tim;
919 pprev = &run_first_tims[nb_runlists];
922 for ( ; tim != NULL; tim = next_tim) {
923 next_tim = tim->sl_next[0];
925 ret = timer_set_running_state(tim);
926 if (likely(ret == 0)) {
927 pprev = &tim->sl_next[0];
929 /* another core is trying to re-config this one,
930 * remove it from local expired list
936 /* update the next to expire timer value */
937 privp->pending_head.expire =
938 (privp->pending_head.sl_next[0] == NULL) ? 0 :
939 privp->pending_head.sl_next[0]->expire;
941 rte_spinlock_unlock(&privp->list_lock);
944 /* Now process the run lists */
947 uint64_t min_expire = UINT64_MAX;
950 /* Find the next oldest timer to process */
951 for (i = 0; i < nb_runlists; i++) {
952 tim = run_first_tims[i];
954 if (tim != NULL && tim->expire < min_expire) {
955 min_expire = tim->expire;
964 tim = run_first_tims[min_idx];
966 /* Move down the runlist from which we picked a timer to
969 run_first_tims[min_idx] = run_first_tims[min_idx]->sl_next[0];
971 data->priv_timer[this_lcore].updated = 0;
972 data->priv_timer[this_lcore].running_tim = tim;
974 /* Call the provided callback function */
977 __TIMER_STAT_ADD(data->priv_timer, pending, -1);
979 /* the timer was stopped or reloaded by the callback
980 * function, we have nothing to do here
982 if (data->priv_timer[this_lcore].updated == 1)
985 if (tim->period == 0) {
986 /* remove from done list and mark timer as stopped */
987 status.state = RTE_TIMER_STOP;
988 status.owner = RTE_TIMER_NO_OWNER;
990 tim->status.u32 = status.u32;
992 /* keep it in list and mark timer as pending */
994 &data->priv_timer[this_lcore].list_lock);
995 status.state = RTE_TIMER_PENDING;
996 __TIMER_STAT_ADD(data->priv_timer, pending, 1);
997 status.owner = (int16_t)this_lcore;
999 tim->status.u32 = status.u32;
1000 __rte_timer_reset(tim, tim->expire + tim->period,
1001 tim->period, this_lcore, tim->f, tim->arg, 1,
1003 rte_spinlock_unlock(
1004 &data->priv_timer[this_lcore].list_lock);
1007 data->priv_timer[this_lcore].running_tim = NULL;
1013 /* Walk pending lists, stopping timers and calling user-specified function */
1015 rte_timer_stop_all(uint32_t timer_data_id, unsigned int *walk_lcores,
1017 rte_timer_stop_all_cb_t f, void *f_arg)
1020 struct priv_timer *priv_timer;
1021 uint32_t walk_lcore;
1022 struct rte_timer *tim, *next_tim;
1023 struct rte_timer_data *timer_data;
1025 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
1027 for (i = 0; i < nb_walk_lcores; i++) {
1028 walk_lcore = walk_lcores[i];
1029 priv_timer = &timer_data->priv_timer[walk_lcore];
1031 rte_spinlock_lock(&priv_timer->list_lock);
1033 for (tim = priv_timer->pending_head.sl_next[0];
1036 next_tim = tim->sl_next[0];
1038 /* Call timer_stop with lock held */
1039 __rte_timer_stop(tim, 1, timer_data);
1045 rte_spinlock_unlock(&priv_timer->list_lock);
1051 /* dump statistics about timers */
1053 __rte_timer_dump_stats(struct rte_timer_data *timer_data __rte_unused, FILE *f)
1055 #ifdef RTE_LIBRTE_TIMER_DEBUG
1056 struct rte_timer_debug_stats sum;
1058 struct priv_timer *priv_timer = timer_data->priv_timer;
1060 memset(&sum, 0, sizeof(sum));
1061 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1062 sum.reset += priv_timer[lcore_id].stats.reset;
1063 sum.stop += priv_timer[lcore_id].stats.stop;
1064 sum.manage += priv_timer[lcore_id].stats.manage;
1065 sum.pending += priv_timer[lcore_id].stats.pending;
1067 fprintf(f, "Timer statistics:\n");
1068 fprintf(f, " reset = %"PRIu64"\n", sum.reset);
1069 fprintf(f, " stop = %"PRIu64"\n", sum.stop);
1070 fprintf(f, " manage = %"PRIu64"\n", sum.manage);
1071 fprintf(f, " pending = %"PRIu64"\n", sum.pending);
1073 fprintf(f, "No timer statistics, RTE_LIBRTE_TIMER_DEBUG is disabled\n");
1078 rte_timer_dump_stats_v20(FILE *f)
1080 __rte_timer_dump_stats(&default_timer_data, f);
1082 VERSION_SYMBOL(rte_timer_dump_stats, _v20, 2.0);
1085 rte_timer_dump_stats_v1905(FILE *f)
1087 return rte_timer_alt_dump_stats(default_data_id, f);
1089 MAP_STATIC_SYMBOL(int rte_timer_dump_stats(FILE *f),
1090 rte_timer_dump_stats_v1905);
1091 BIND_DEFAULT_SYMBOL(rte_timer_dump_stats, _v1905, 19.05);
1094 rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f)
1096 struct rte_timer_data *timer_data;
1098 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
1100 __rte_timer_dump_stats(timer_data, f);