1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
11 #include <sys/queue.h>
13 #include <rte_atomic.h>
14 #include <rte_common.h>
15 #include <rte_cycles.h>
16 #include <rte_eal_memconfig.h>
17 #include <rte_per_lcore.h>
18 #include <rte_memory.h>
19 #include <rte_launch.h>
21 #include <rte_lcore.h>
22 #include <rte_branch_prediction.h>
23 #include <rte_spinlock.h>
24 #include <rte_random.h>
25 #include <rte_pause.h>
26 #include <rte_memzone.h>
27 #include <rte_malloc.h>
28 #include <rte_errno.h>
29 #include <rte_function_versioning.h>
31 #include "rte_timer.h"
34 * Per-lcore info for timers.
37 struct rte_timer pending_head; /**< dummy timer instance to head up list */
38 rte_spinlock_t list_lock; /**< lock to protect list access */
40 /** per-core variable that true if a timer was updated on this
41 * core since last reset of the variable */
44 /** track the current depth of the skiplist */
45 unsigned curr_skiplist_depth;
47 unsigned prev_lcore; /**< used for lcore round robin */
49 /** running timer on this lcore now */
50 struct rte_timer *running_tim;
52 #ifdef RTE_LIBRTE_TIMER_DEBUG
53 /** per-lcore statistics */
54 struct rte_timer_debug_stats stats;
56 } __rte_cache_aligned;
58 #define FL_ALLOCATED (1 << 0)
59 struct rte_timer_data {
60 struct priv_timer priv_timer[RTE_MAX_LCORE];
61 uint8_t internal_flags;
64 #define RTE_MAX_DATA_ELS 64
65 static const struct rte_memzone *rte_timer_data_mz;
66 static int *volatile rte_timer_mz_refcnt;
67 static struct rte_timer_data *rte_timer_data_arr;
68 static const uint32_t default_data_id;
69 static uint32_t rte_timer_subsystem_initialized;
71 /* when debug is enabled, store some statistics */
72 #ifdef RTE_LIBRTE_TIMER_DEBUG
73 #define __TIMER_STAT_ADD(priv_timer, name, n) do { \
74 unsigned __lcore_id = rte_lcore_id(); \
75 if (__lcore_id < RTE_MAX_LCORE) \
76 priv_timer[__lcore_id].stats.name += (n); \
79 #define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0)
83 timer_data_valid(uint32_t id)
85 return rte_timer_data_arr &&
86 (rte_timer_data_arr[id].internal_flags & FL_ALLOCATED);
89 /* validate ID and retrieve timer data pointer, or return error value */
90 #define TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, retval) do { \
91 if (id >= RTE_MAX_DATA_ELS || !timer_data_valid(id)) \
93 timer_data = &rte_timer_data_arr[id]; \
97 rte_timer_data_alloc(uint32_t *id_ptr)
100 struct rte_timer_data *data;
102 if (!rte_timer_subsystem_initialized)
105 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
106 data = &rte_timer_data_arr[i];
107 if (!(data->internal_flags & FL_ALLOCATED)) {
108 data->internal_flags |= FL_ALLOCATED;
121 rte_timer_data_dealloc(uint32_t id)
123 struct rte_timer_data *timer_data;
124 TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, -EINVAL);
126 timer_data->internal_flags &= ~(FL_ALLOCATED);
131 /* Init the timer library. Allocate an array of timer data structs in shared
132 * memory, and allocate the zeroth entry for use with original timer
133 * APIs. Since the intersection of the sets of lcore ids in primary and
134 * secondary processes should be empty, the zeroth entry can be shared by
135 * multiple processes.
138 rte_timer_subsystem_init(void)
140 const struct rte_memzone *mz;
141 struct rte_timer_data *data;
143 static const char *mz_name = "rte_timer_mz";
144 const size_t data_arr_size =
145 RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr);
146 const size_t mem_size = data_arr_size + sizeof(*rte_timer_mz_refcnt);
147 bool do_full_init = true;
149 if (rte_timer_subsystem_initialized)
152 rte_mcfg_timer_lock();
154 mz = rte_memzone_lookup(mz_name);
156 mz = rte_memzone_reserve_aligned(mz_name, mem_size,
157 SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE);
159 rte_mcfg_timer_unlock();
164 do_full_init = false;
166 rte_timer_data_mz = mz;
167 rte_timer_data_arr = mz->addr;
168 rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size);
171 for (i = 0; i < RTE_MAX_DATA_ELS; i++) {
172 data = &rte_timer_data_arr[i];
174 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE;
177 &data->priv_timer[lcore_id].list_lock);
178 data->priv_timer[lcore_id].prev_lcore =
184 rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED;
185 (*rte_timer_mz_refcnt)++;
187 rte_mcfg_timer_unlock();
189 rte_timer_subsystem_initialized = 1;
195 rte_timer_subsystem_finalize(void)
197 if (!rte_timer_subsystem_initialized)
200 rte_mcfg_timer_lock();
202 if (--(*rte_timer_mz_refcnt) == 0)
203 rte_memzone_free(rte_timer_data_mz);
205 rte_mcfg_timer_unlock();
207 rte_timer_subsystem_initialized = 0;
210 /* Initialize the timer handle tim for use */
212 rte_timer_init(struct rte_timer *tim)
214 union rte_timer_status status;
216 status.state = RTE_TIMER_STOP;
217 status.owner = RTE_TIMER_NO_OWNER;
218 tim->status.u32 = status.u32;
222 * if timer is pending or stopped (or running on the same core than
223 * us), mark timer as configuring, and on success return the previous
224 * status of the timer
227 timer_set_config_state(struct rte_timer *tim,
228 union rte_timer_status *ret_prev_status,
229 struct priv_timer *priv_timer)
231 union rte_timer_status prev_status, status;
235 lcore_id = rte_lcore_id();
237 /* wait that the timer is in correct status before update,
238 * and mark it as being configured */
239 while (success == 0) {
240 prev_status.u32 = tim->status.u32;
242 /* timer is running on another core
243 * or ready to run on local core, exit
245 if (prev_status.state == RTE_TIMER_RUNNING &&
246 (prev_status.owner != (uint16_t)lcore_id ||
247 tim != priv_timer[lcore_id].running_tim))
250 /* timer is being configured on another core */
251 if (prev_status.state == RTE_TIMER_CONFIG)
254 /* here, we know that timer is stopped or pending,
255 * mark it atomically as being configured */
256 status.state = RTE_TIMER_CONFIG;
257 status.owner = (int16_t)lcore_id;
258 success = rte_atomic32_cmpset(&tim->status.u32,
263 ret_prev_status->u32 = prev_status.u32;
268 * if timer is pending, mark timer as running
271 timer_set_running_state(struct rte_timer *tim)
273 union rte_timer_status prev_status, status;
274 unsigned lcore_id = rte_lcore_id();
277 /* wait that the timer is in correct status before update,
278 * and mark it as running */
279 while (success == 0) {
280 prev_status.u32 = tim->status.u32;
282 /* timer is not pending anymore */
283 if (prev_status.state != RTE_TIMER_PENDING)
286 /* here, we know that timer is stopped or pending,
287 * mark it atomically as being configured */
288 status.state = RTE_TIMER_RUNNING;
289 status.owner = (int16_t)lcore_id;
290 success = rte_atomic32_cmpset(&tim->status.u32,
299 * Return a skiplist level for a new entry.
300 * This probabilistically gives a level with p=1/4 that an entry at level n
301 * will also appear at level n+1.
304 timer_get_skiplist_level(unsigned curr_depth)
306 #ifdef RTE_LIBRTE_TIMER_DEBUG
307 static uint32_t i, count = 0;
308 static uint32_t levels[MAX_SKIPLIST_DEPTH] = {0};
311 /* probability value is 1/4, i.e. all at level 0, 1 in 4 is at level 1,
312 * 1 in 16 at level 2, 1 in 64 at level 3, etc. Calculated using lowest
313 * bit position of a (pseudo)random number.
315 uint32_t rand = rte_rand() & (UINT32_MAX - 1);
316 uint32_t level = rand == 0 ? MAX_SKIPLIST_DEPTH : (rte_bsf32(rand)-1) / 2;
318 /* limit the levels used to one above our current level, so we don't,
319 * for instance, have a level 0 and a level 7 without anything between
321 if (level > curr_depth)
323 if (level >= MAX_SKIPLIST_DEPTH)
324 level = MAX_SKIPLIST_DEPTH-1;
325 #ifdef RTE_LIBRTE_TIMER_DEBUG
328 if (count % 10000 == 0)
329 for (i = 0; i < MAX_SKIPLIST_DEPTH; i++)
330 printf("Level %u: %u\n", (unsigned)i, (unsigned)levels[i]);
336 * For a given time value, get the entries at each level which
337 * are <= that time value.
340 timer_get_prev_entries(uint64_t time_val, unsigned tim_lcore,
341 struct rte_timer **prev, struct priv_timer *priv_timer)
343 unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth;
344 prev[lvl] = &priv_timer[tim_lcore].pending_head;
347 prev[lvl] = prev[lvl+1];
348 while (prev[lvl]->sl_next[lvl] &&
349 prev[lvl]->sl_next[lvl]->expire <= time_val)
350 prev[lvl] = prev[lvl]->sl_next[lvl];
355 * Given a timer node in the skiplist, find the previous entries for it at
356 * all skiplist levels.
359 timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore,
360 struct rte_timer **prev,
361 struct priv_timer *priv_timer)
365 /* to get a specific entry in the list, look for just lower than the time
366 * values, and then increment on each level individually if necessary
368 timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer);
369 for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) {
370 while (prev[i]->sl_next[i] != NULL &&
371 prev[i]->sl_next[i] != tim &&
372 prev[i]->sl_next[i]->expire <= tim->expire)
373 prev[i] = prev[i]->sl_next[i];
377 /* call with lock held as necessary
379 * timer must be in config state
380 * timer must not be in a list
383 timer_add(struct rte_timer *tim, unsigned int tim_lcore,
384 struct priv_timer *priv_timer)
387 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
389 /* find where exactly this element goes in the list of elements
391 timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer);
393 /* now assign it a new level and add at that level */
394 const unsigned tim_level = timer_get_skiplist_level(
395 priv_timer[tim_lcore].curr_skiplist_depth);
396 if (tim_level == priv_timer[tim_lcore].curr_skiplist_depth)
397 priv_timer[tim_lcore].curr_skiplist_depth++;
401 tim->sl_next[lvl] = prev[lvl]->sl_next[lvl];
402 prev[lvl]->sl_next[lvl] = tim;
405 tim->sl_next[0] = prev[0]->sl_next[0];
406 prev[0]->sl_next[0] = tim;
408 /* save the lowest list entry into the expire field of the dummy hdr
409 * NOTE: this is not atomic on 32-bit*/
410 priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\
411 pending_head.sl_next[0]->expire;
415 * del from list, lock if needed
416 * timer must be in config state
417 * timer must be in a list
420 timer_del(struct rte_timer *tim, union rte_timer_status prev_status,
421 int local_is_locked, struct priv_timer *priv_timer)
423 unsigned lcore_id = rte_lcore_id();
424 unsigned prev_owner = prev_status.owner;
426 struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1];
428 /* if timer needs is pending another core, we need to lock the
429 * list; if it is on local core, we need to lock if we are not
430 * called from rte_timer_manage() */
431 if (prev_owner != lcore_id || !local_is_locked)
432 rte_spinlock_lock(&priv_timer[prev_owner].list_lock);
434 /* save the lowest list entry into the expire field of the dummy hdr.
435 * NOTE: this is not atomic on 32-bit */
436 if (tim == priv_timer[prev_owner].pending_head.sl_next[0])
437 priv_timer[prev_owner].pending_head.expire =
438 ((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire);
440 /* adjust pointers from previous entries to point past this */
441 timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer);
442 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) {
443 if (prev[i]->sl_next[i] == tim)
444 prev[i]->sl_next[i] = tim->sl_next[i];
447 /* in case we deleted last entry at a level, adjust down max level */
448 for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--)
449 if (priv_timer[prev_owner].pending_head.sl_next[i] == NULL)
450 priv_timer[prev_owner].curr_skiplist_depth --;
454 if (prev_owner != lcore_id || !local_is_locked)
455 rte_spinlock_unlock(&priv_timer[prev_owner].list_lock);
458 /* Reset and start the timer associated with the timer handle (private func) */
460 __rte_timer_reset(struct rte_timer *tim, uint64_t expire,
461 uint64_t period, unsigned tim_lcore,
462 rte_timer_cb_t fct, void *arg,
464 struct rte_timer_data *timer_data)
466 union rte_timer_status prev_status, status;
468 unsigned lcore_id = rte_lcore_id();
469 struct priv_timer *priv_timer = timer_data->priv_timer;
471 /* round robin for tim_lcore */
472 if (tim_lcore == (unsigned)LCORE_ID_ANY) {
473 if (lcore_id < RTE_MAX_LCORE) {
474 /* EAL thread with valid lcore_id */
475 tim_lcore = rte_get_next_lcore(
476 priv_timer[lcore_id].prev_lcore,
478 priv_timer[lcore_id].prev_lcore = tim_lcore;
480 /* non-EAL thread do not run rte_timer_manage(),
481 * so schedule the timer on the first enabled lcore. */
482 tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0, 1);
485 /* wait that the timer is in correct status before update,
486 * and mark it as being configured */
487 ret = timer_set_config_state(tim, &prev_status, priv_timer);
491 __TIMER_STAT_ADD(priv_timer, reset, 1);
492 if (prev_status.state == RTE_TIMER_RUNNING &&
493 lcore_id < RTE_MAX_LCORE) {
494 priv_timer[lcore_id].updated = 1;
497 /* remove it from list */
498 if (prev_status.state == RTE_TIMER_PENDING) {
499 timer_del(tim, prev_status, local_is_locked, priv_timer);
500 __TIMER_STAT_ADD(priv_timer, pending, -1);
503 tim->period = period;
504 tim->expire = expire;
508 /* if timer needs to be scheduled on another core, we need to
509 * lock the destination list; if it is on local core, we need to lock if
510 * we are not called from rte_timer_manage()
512 if (tim_lcore != lcore_id || !local_is_locked)
513 rte_spinlock_lock(&priv_timer[tim_lcore].list_lock);
515 __TIMER_STAT_ADD(priv_timer, pending, 1);
516 timer_add(tim, tim_lcore, priv_timer);
518 /* update state: as we are in CONFIG state, only us can modify
519 * the state so we don't need to use cmpset() here */
521 status.state = RTE_TIMER_PENDING;
522 status.owner = (int16_t)tim_lcore;
523 tim->status.u32 = status.u32;
525 if (tim_lcore != lcore_id || !local_is_locked)
526 rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock);
531 /* Reset and start the timer associated with the timer handle tim */
533 rte_timer_reset(struct rte_timer *tim, uint64_t ticks,
534 enum rte_timer_type type, unsigned int tim_lcore,
535 rte_timer_cb_t fct, void *arg)
537 return rte_timer_alt_reset(default_data_id, tim, ticks, type,
538 tim_lcore, fct, arg);
542 rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim,
543 uint64_t ticks, enum rte_timer_type type,
544 unsigned int tim_lcore, rte_timer_cb_t fct, void *arg)
546 uint64_t cur_time = rte_get_timer_cycles();
548 struct rte_timer_data *timer_data;
550 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
552 if (type == PERIODICAL)
557 return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore,
558 fct, arg, 0, timer_data);
561 /* loop until rte_timer_reset() succeed */
563 rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks,
564 enum rte_timer_type type, unsigned tim_lcore,
565 rte_timer_cb_t fct, void *arg)
567 while (rte_timer_reset(tim, ticks, type, tim_lcore,
573 __rte_timer_stop(struct rte_timer *tim, int local_is_locked,
574 struct rte_timer_data *timer_data)
576 union rte_timer_status prev_status, status;
577 unsigned lcore_id = rte_lcore_id();
579 struct priv_timer *priv_timer = timer_data->priv_timer;
581 /* wait that the timer is in correct status before update,
582 * and mark it as being configured */
583 ret = timer_set_config_state(tim, &prev_status, priv_timer);
587 __TIMER_STAT_ADD(priv_timer, stop, 1);
588 if (prev_status.state == RTE_TIMER_RUNNING &&
589 lcore_id < RTE_MAX_LCORE) {
590 priv_timer[lcore_id].updated = 1;
593 /* remove it from list */
594 if (prev_status.state == RTE_TIMER_PENDING) {
595 timer_del(tim, prev_status, local_is_locked, priv_timer);
596 __TIMER_STAT_ADD(priv_timer, pending, -1);
599 /* mark timer as stopped */
601 status.state = RTE_TIMER_STOP;
602 status.owner = RTE_TIMER_NO_OWNER;
603 tim->status.u32 = status.u32;
608 /* Stop the timer associated with the timer handle tim */
610 rte_timer_stop(struct rte_timer *tim)
612 return rte_timer_alt_stop(default_data_id, tim);
616 rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim)
618 struct rte_timer_data *timer_data;
620 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
622 return __rte_timer_stop(tim, 0, timer_data);
625 /* loop until rte_timer_stop() succeed */
627 rte_timer_stop_sync(struct rte_timer *tim)
629 while (rte_timer_stop(tim) != 0)
633 /* Test the PENDING status of the timer handle tim */
635 rte_timer_pending(struct rte_timer *tim)
637 return tim->status.state == RTE_TIMER_PENDING;
640 /* must be called periodically, run all timer that expired */
642 __rte_timer_manage(struct rte_timer_data *timer_data)
644 union rte_timer_status status;
645 struct rte_timer *tim, *next_tim;
646 struct rte_timer *run_first_tim, **pprev;
647 unsigned lcore_id = rte_lcore_id();
648 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
651 struct priv_timer *priv_timer = timer_data->priv_timer;
653 /* timer manager only runs on EAL thread with valid lcore_id */
654 assert(lcore_id < RTE_MAX_LCORE);
656 __TIMER_STAT_ADD(priv_timer, manage, 1);
657 /* optimize for the case where per-cpu list is empty */
658 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL)
660 cur_time = rte_get_timer_cycles();
663 /* on 64-bit the value cached in the pending_head.expired will be
664 * updated atomically, so we can consult that for a quick check here
665 * outside the lock */
666 if (likely(priv_timer[lcore_id].pending_head.expire > cur_time))
670 /* browse ordered list, add expired timers in 'expired' list */
671 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
673 /* if nothing to do just unlock and return */
674 if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL ||
675 priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) {
676 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
680 /* save start of list of expired timers */
681 tim = priv_timer[lcore_id].pending_head.sl_next[0];
683 /* break the existing list at current time point */
684 timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer);
685 for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) {
686 if (prev[i] == &priv_timer[lcore_id].pending_head)
688 priv_timer[lcore_id].pending_head.sl_next[i] =
690 if (prev[i]->sl_next[i] == NULL)
691 priv_timer[lcore_id].curr_skiplist_depth--;
692 prev[i] ->sl_next[i] = NULL;
695 /* transition run-list from PENDING to RUNNING */
697 pprev = &run_first_tim;
699 for ( ; tim != NULL; tim = next_tim) {
700 next_tim = tim->sl_next[0];
702 ret = timer_set_running_state(tim);
703 if (likely(ret == 0)) {
704 pprev = &tim->sl_next[0];
706 /* another core is trying to re-config this one,
707 * remove it from local expired list
713 /* update the next to expire timer value */
714 priv_timer[lcore_id].pending_head.expire =
715 (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 :
716 priv_timer[lcore_id].pending_head.sl_next[0]->expire;
718 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
720 /* now scan expired list and call callbacks */
721 for (tim = run_first_tim; tim != NULL; tim = next_tim) {
722 next_tim = tim->sl_next[0];
723 priv_timer[lcore_id].updated = 0;
724 priv_timer[lcore_id].running_tim = tim;
726 /* execute callback function with list unlocked */
727 tim->f(tim, tim->arg);
729 __TIMER_STAT_ADD(priv_timer, pending, -1);
730 /* the timer was stopped or reloaded by the callback
731 * function, we have nothing to do here */
732 if (priv_timer[lcore_id].updated == 1)
735 if (tim->period == 0) {
736 /* remove from done list and mark timer as stopped */
737 status.state = RTE_TIMER_STOP;
738 status.owner = RTE_TIMER_NO_OWNER;
740 tim->status.u32 = status.u32;
743 /* keep it in list and mark timer as pending */
744 rte_spinlock_lock(&priv_timer[lcore_id].list_lock);
745 status.state = RTE_TIMER_PENDING;
746 __TIMER_STAT_ADD(priv_timer, pending, 1);
747 status.owner = (int16_t)lcore_id;
749 tim->status.u32 = status.u32;
750 __rte_timer_reset(tim, tim->expire + tim->period,
751 tim->period, lcore_id, tim->f, tim->arg, 1,
753 rte_spinlock_unlock(&priv_timer[lcore_id].list_lock);
756 priv_timer[lcore_id].running_tim = NULL;
760 rte_timer_manage(void)
762 struct rte_timer_data *timer_data;
764 TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL);
766 __rte_timer_manage(timer_data);
772 rte_timer_alt_manage(uint32_t timer_data_id,
773 unsigned int *poll_lcores,
775 rte_timer_alt_manage_cb_t f)
777 unsigned int default_poll_lcores[] = {rte_lcore_id()};
778 union rte_timer_status status;
779 struct rte_timer *tim, *next_tim, **pprev;
780 struct rte_timer *run_first_tims[RTE_MAX_LCORE];
781 unsigned int this_lcore = rte_lcore_id();
782 struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1];
786 struct rte_timer_data *data;
787 struct priv_timer *privp;
790 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, data, -EINVAL);
792 /* timer manager only runs on EAL thread with valid lcore_id */
793 assert(this_lcore < RTE_MAX_LCORE);
795 __TIMER_STAT_ADD(data->priv_timer, manage, 1);
797 if (poll_lcores == NULL) {
798 poll_lcores = default_poll_lcores;
799 nb_poll_lcores = RTE_DIM(default_poll_lcores);
802 for (i = 0; i < nb_poll_lcores; i++) {
803 poll_lcore = poll_lcores[i];
804 privp = &data->priv_timer[poll_lcore];
806 /* optimize for the case where per-cpu list is empty */
807 if (privp->pending_head.sl_next[0] == NULL)
809 cur_time = rte_get_timer_cycles();
812 /* on 64-bit the value cached in the pending_head.expired will
813 * be updated atomically, so we can consult that for a quick
814 * check here outside the lock
816 if (likely(privp->pending_head.expire > cur_time))
820 /* browse ordered list, add expired timers in 'expired' list */
821 rte_spinlock_lock(&privp->list_lock);
823 /* if nothing to do just unlock and return */
824 if (privp->pending_head.sl_next[0] == NULL ||
825 privp->pending_head.sl_next[0]->expire > cur_time) {
826 rte_spinlock_unlock(&privp->list_lock);
830 /* save start of list of expired timers */
831 tim = privp->pending_head.sl_next[0];
833 /* break the existing list at current time point */
834 timer_get_prev_entries(cur_time, poll_lcore, prev,
836 for (j = privp->curr_skiplist_depth - 1; j >= 0; j--) {
837 if (prev[j] == &privp->pending_head)
839 privp->pending_head.sl_next[j] =
841 if (prev[j]->sl_next[j] == NULL)
842 privp->curr_skiplist_depth--;
844 prev[j]->sl_next[j] = NULL;
847 /* transition run-list from PENDING to RUNNING */
848 run_first_tims[nb_runlists] = tim;
849 pprev = &run_first_tims[nb_runlists];
852 for ( ; tim != NULL; tim = next_tim) {
853 next_tim = tim->sl_next[0];
855 ret = timer_set_running_state(tim);
856 if (likely(ret == 0)) {
857 pprev = &tim->sl_next[0];
859 /* another core is trying to re-config this one,
860 * remove it from local expired list
866 /* update the next to expire timer value */
867 privp->pending_head.expire =
868 (privp->pending_head.sl_next[0] == NULL) ? 0 :
869 privp->pending_head.sl_next[0]->expire;
871 rte_spinlock_unlock(&privp->list_lock);
874 /* Now process the run lists */
877 uint64_t min_expire = UINT64_MAX;
880 /* Find the next oldest timer to process */
881 for (i = 0; i < nb_runlists; i++) {
882 tim = run_first_tims[i];
884 if (tim != NULL && tim->expire < min_expire) {
885 min_expire = tim->expire;
894 tim = run_first_tims[min_idx];
896 /* Move down the runlist from which we picked a timer to
899 run_first_tims[min_idx] = run_first_tims[min_idx]->sl_next[0];
901 data->priv_timer[this_lcore].updated = 0;
902 data->priv_timer[this_lcore].running_tim = tim;
904 /* Call the provided callback function */
907 __TIMER_STAT_ADD(data->priv_timer, pending, -1);
909 /* the timer was stopped or reloaded by the callback
910 * function, we have nothing to do here
912 if (data->priv_timer[this_lcore].updated == 1)
915 if (tim->period == 0) {
916 /* remove from done list and mark timer as stopped */
917 status.state = RTE_TIMER_STOP;
918 status.owner = RTE_TIMER_NO_OWNER;
920 tim->status.u32 = status.u32;
922 /* keep it in list and mark timer as pending */
924 &data->priv_timer[this_lcore].list_lock);
925 status.state = RTE_TIMER_PENDING;
926 __TIMER_STAT_ADD(data->priv_timer, pending, 1);
927 status.owner = (int16_t)this_lcore;
929 tim->status.u32 = status.u32;
930 __rte_timer_reset(tim, tim->expire + tim->period,
931 tim->period, this_lcore, tim->f, tim->arg, 1,
934 &data->priv_timer[this_lcore].list_lock);
937 data->priv_timer[this_lcore].running_tim = NULL;
943 /* Walk pending lists, stopping timers and calling user-specified function */
945 rte_timer_stop_all(uint32_t timer_data_id, unsigned int *walk_lcores,
947 rte_timer_stop_all_cb_t f, void *f_arg)
950 struct priv_timer *priv_timer;
952 struct rte_timer *tim, *next_tim;
953 struct rte_timer_data *timer_data;
955 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
957 for (i = 0; i < nb_walk_lcores; i++) {
958 walk_lcore = walk_lcores[i];
959 priv_timer = &timer_data->priv_timer[walk_lcore];
961 rte_spinlock_lock(&priv_timer->list_lock);
963 for (tim = priv_timer->pending_head.sl_next[0];
966 next_tim = tim->sl_next[0];
968 /* Call timer_stop with lock held */
969 __rte_timer_stop(tim, 1, timer_data);
975 rte_spinlock_unlock(&priv_timer->list_lock);
981 /* dump statistics about timers */
983 __rte_timer_dump_stats(struct rte_timer_data *timer_data __rte_unused, FILE *f)
985 #ifdef RTE_LIBRTE_TIMER_DEBUG
986 struct rte_timer_debug_stats sum;
988 struct priv_timer *priv_timer = timer_data->priv_timer;
990 memset(&sum, 0, sizeof(sum));
991 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
992 sum.reset += priv_timer[lcore_id].stats.reset;
993 sum.stop += priv_timer[lcore_id].stats.stop;
994 sum.manage += priv_timer[lcore_id].stats.manage;
995 sum.pending += priv_timer[lcore_id].stats.pending;
997 fprintf(f, "Timer statistics:\n");
998 fprintf(f, " reset = %"PRIu64"\n", sum.reset);
999 fprintf(f, " stop = %"PRIu64"\n", sum.stop);
1000 fprintf(f, " manage = %"PRIu64"\n", sum.manage);
1001 fprintf(f, " pending = %"PRIu64"\n", sum.pending);
1003 fprintf(f, "No timer statistics, RTE_LIBRTE_TIMER_DEBUG is disabled\n");
1008 rte_timer_dump_stats(FILE *f)
1010 return rte_timer_alt_dump_stats(default_data_id, f);
1014 rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f)
1016 struct rte_timer_data *timer_data;
1018 TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL);
1020 __rte_timer_dump_stats(timer_data, f);