X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_timer%2Frte_timer.c;h=4680a91b58cf2c3fa3979692c63b5e8286a39c40;hb=9160d6386472b70503e504df765440ecdb604d7d;hp=269a992b981fdb72a554d2ac143890a2f90be017;hpb=add720fce99710cda6a2a7aeba59a9da85ef9a33;p=dpdk.git diff --git a/lib/librte_timer/rte_timer.c b/lib/librte_timer/rte_timer.c index 269a992b98..4680a91b58 100644 --- a/lib/librte_timer/rte_timer.c +++ b/lib/librte_timer/rte_timer.c @@ -1,60 +1,37 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2014 Intel Corporation */ #include #include #include +#include #include +#include +#include #include #include #include +#include #include #include -#include #include -#include #include -#include #include #include #include #include +#include +#include +#include +#include #include "rte_timer.h" -LIST_HEAD(rte_timer_list, rte_timer); - +/** + * Per-lcore info for timers. + */ struct priv_timer { struct rte_timer pending_head; /**< dummy timer instance to head up list */ rte_spinlock_t list_lock; /**< lock to protect list access */ @@ -68,38 +45,165 @@ struct priv_timer { unsigned prev_lcore; /**< used for lcore round robin */ + /** running timer on this lcore now */ + struct rte_timer *running_tim; + #ifdef RTE_LIBRTE_TIMER_DEBUG /** per-lcore statistics */ struct rte_timer_debug_stats stats; #endif } __rte_cache_aligned; -/** per-lcore private info for timers */ -static struct priv_timer priv_timer[RTE_MAX_LCORE]; +#define FL_ALLOCATED (1 << 0) +struct rte_timer_data { + struct priv_timer priv_timer[RTE_MAX_LCORE]; + uint8_t internal_flags; +}; + +#define RTE_MAX_DATA_ELS 64 +static const struct rte_memzone *rte_timer_data_mz; +static int *volatile rte_timer_mz_refcnt; +static struct rte_timer_data *rte_timer_data_arr; +static const uint32_t default_data_id; +static uint32_t rte_timer_subsystem_initialized; /* when debug is enabled, store some statistics */ #ifdef RTE_LIBRTE_TIMER_DEBUG -#define __TIMER_STAT_ADD(name, n) do { \ - unsigned __lcore_id = rte_lcore_id(); \ - priv_timer[__lcore_id].stats.name += (n); \ +#define __TIMER_STAT_ADD(priv_timer, name, n) do { \ + unsigned __lcore_id = rte_lcore_id(); \ + if (__lcore_id < RTE_MAX_LCORE) \ + priv_timer[__lcore_id].stats.name += (n); \ } while(0) #else -#define __TIMER_STAT_ADD(name, n) do {} while(0) +#define __TIMER_STAT_ADD(priv_timer, name, n) do {} while (0) #endif -/* Init the timer library. */ -void -rte_timer_subsystem_init(void) +static inline int +timer_data_valid(uint32_t id) { - unsigned lcore_id; + return rte_timer_data_arr && + (rte_timer_data_arr[id].internal_flags & FL_ALLOCATED); +} - /* since priv_timer is static, it's zeroed by default, so only init some - * fields. - */ - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id ++) { - rte_spinlock_init(&priv_timer[lcore_id].list_lock); - priv_timer[lcore_id].prev_lcore = lcore_id; +/* validate ID and retrieve timer data pointer, or return error value */ +#define TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, retval) do { \ + if (id >= RTE_MAX_DATA_ELS || !timer_data_valid(id)) \ + return retval; \ + timer_data = &rte_timer_data_arr[id]; \ +} while (0) + +int +rte_timer_data_alloc(uint32_t *id_ptr) +{ + int i; + struct rte_timer_data *data; + + if (!rte_timer_subsystem_initialized) + return -ENOMEM; + + for (i = 0; i < RTE_MAX_DATA_ELS; i++) { + data = &rte_timer_data_arr[i]; + if (!(data->internal_flags & FL_ALLOCATED)) { + data->internal_flags |= FL_ALLOCATED; + + if (id_ptr) + *id_ptr = i; + + return 0; + } + } + + return -ENOSPC; +} + +int +rte_timer_data_dealloc(uint32_t id) +{ + struct rte_timer_data *timer_data; + TIMER_DATA_VALID_GET_OR_ERR_RET(id, timer_data, -EINVAL); + + timer_data->internal_flags &= ~(FL_ALLOCATED); + + return 0; +} + +/* Init the timer library. Allocate an array of timer data structs in shared + * memory, and allocate the zeroth entry for use with original timer + * APIs. Since the intersection of the sets of lcore ids in primary and + * secondary processes should be empty, the zeroth entry can be shared by + * multiple processes. + */ +int +rte_timer_subsystem_init(void) +{ + const struct rte_memzone *mz; + struct rte_timer_data *data; + int i, lcore_id; + static const char *mz_name = "rte_timer_mz"; + const size_t data_arr_size = + RTE_MAX_DATA_ELS * sizeof(*rte_timer_data_arr); + const size_t mem_size = data_arr_size + sizeof(*rte_timer_mz_refcnt); + bool do_full_init = true; + + if (rte_timer_subsystem_initialized) + return -EALREADY; + + rte_mcfg_timer_lock(); + + mz = rte_memzone_lookup(mz_name); + if (mz == NULL) { + mz = rte_memzone_reserve_aligned(mz_name, mem_size, + SOCKET_ID_ANY, 0, RTE_CACHE_LINE_SIZE); + if (mz == NULL) { + rte_mcfg_timer_unlock(); + return -ENOMEM; + } + do_full_init = true; + } else + do_full_init = false; + + rte_timer_data_mz = mz; + rte_timer_data_arr = mz->addr; + rte_timer_mz_refcnt = (void *)((char *)mz->addr + data_arr_size); + + if (do_full_init) { + for (i = 0; i < RTE_MAX_DATA_ELS; i++) { + data = &rte_timer_data_arr[i]; + + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; + lcore_id++) { + rte_spinlock_init( + &data->priv_timer[lcore_id].list_lock); + data->priv_timer[lcore_id].prev_lcore = + lcore_id; + } + } } + + rte_timer_data_arr[default_data_id].internal_flags |= FL_ALLOCATED; + (*rte_timer_mz_refcnt)++; + + rte_mcfg_timer_unlock(); + + rte_timer_subsystem_initialized = 1; + + return 0; +} + +void +rte_timer_subsystem_finalize(void) +{ + if (!rte_timer_subsystem_initialized) + return; + + rte_mcfg_timer_lock(); + + if (--(*rte_timer_mz_refcnt) == 0) + rte_memzone_free(rte_timer_data_mz); + + rte_mcfg_timer_unlock(); + + rte_timer_subsystem_initialized = 0; } /* Initialize the timer handle tim for use */ @@ -120,7 +224,8 @@ rte_timer_init(struct rte_timer *tim) */ static int timer_set_config_state(struct rte_timer *tim, - union rte_timer_status *ret_prev_status) + union rte_timer_status *ret_prev_status, + struct priv_timer *priv_timer) { union rte_timer_status prev_status, status; int success = 0; @@ -133,9 +238,12 @@ timer_set_config_state(struct rte_timer *tim, while (success == 0) { prev_status.u32 = tim->status.u32; - /* timer is running on another core, exit */ + /* timer is running on another core + * or ready to run on local core, exit + */ if (prev_status.state == RTE_TIMER_RUNNING && - (unsigned)prev_status.owner != lcore_id) + (prev_status.owner != (uint16_t)lcore_id || + tim != priv_timer[lcore_id].running_tim)) return -1; /* timer is being configured on another core */ @@ -175,7 +283,7 @@ timer_set_running_state(struct rte_timer *tim) return -1; /* here, we know that timer is stopped or pending, - * mark it atomically as beeing configured */ + * mark it atomically as being configured */ status.state = RTE_TIMER_RUNNING; status.owner = (int16_t)lcore_id; success = rte_atomic32_cmpset(&tim->status.u32, @@ -188,7 +296,7 @@ timer_set_running_state(struct rte_timer *tim) /* * Return a skiplist level for a new entry. - * This probabalistically gives a level with p=1/4 that an entry at level n + * This probabilistically gives a level with p=1/4 that an entry at level n * will also appear at level n+1. */ static uint32_t @@ -229,7 +337,7 @@ timer_get_skiplist_level(unsigned curr_depth) */ static void timer_get_prev_entries(uint64_t time_val, unsigned tim_lcore, - struct rte_timer **prev) + struct rte_timer **prev, struct priv_timer *priv_timer) { unsigned lvl = priv_timer[tim_lcore].curr_skiplist_depth; prev[lvl] = &priv_timer[tim_lcore].pending_head; @@ -248,13 +356,15 @@ timer_get_prev_entries(uint64_t time_val, unsigned tim_lcore, */ static void timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore, - struct rte_timer **prev) + struct rte_timer **prev, + struct priv_timer *priv_timer) { int i; + /* to get a specific entry in the list, look for just lower than the time * values, and then increment on each level individually if necessary */ - timer_get_prev_entries(tim->expire - 1, tim_lcore, prev); + timer_get_prev_entries(tim->expire - 1, tim_lcore, prev, priv_timer); for (i = priv_timer[tim_lcore].curr_skiplist_depth - 1; i >= 0; i--) { while (prev[i]->sl_next[i] != NULL && prev[i]->sl_next[i] != tim && @@ -263,27 +373,21 @@ timer_get_prev_entries_for_node(struct rte_timer *tim, unsigned tim_lcore, } } -/* - * add in list, lock if needed +/* call with lock held as necessary + * add in list * timer must be in config state * timer must not be in a list */ static void -timer_add(struct rte_timer *tim, unsigned tim_lcore, int local_is_locked) +timer_add(struct rte_timer *tim, unsigned int tim_lcore, + struct priv_timer *priv_timer) { - unsigned lcore_id = rte_lcore_id(); unsigned lvl; struct rte_timer *prev[MAX_SKIPLIST_DEPTH+1]; - /* if timer needs to be scheduled on another core, we need to - * lock the list; if it is on local core, we need to lock if - * we are not called from rte_timer_manage() */ - if (tim_lcore != lcore_id || !local_is_locked) - rte_spinlock_lock(&priv_timer[tim_lcore].list_lock); - /* find where exactly this element goes in the list of elements * for each depth. */ - timer_get_prev_entries(tim->expire, tim_lcore, prev); + timer_get_prev_entries(tim->expire, tim_lcore, prev, priv_timer); /* now assign it a new level and add at that level */ const unsigned tim_level = timer_get_skiplist_level( @@ -304,9 +408,6 @@ timer_add(struct rte_timer *tim, unsigned tim_lcore, int local_is_locked) * NOTE: this is not atomic on 32-bit*/ priv_timer[tim_lcore].pending_head.expire = priv_timer[tim_lcore].\ pending_head.sl_next[0]->expire; - - if (tim_lcore != lcore_id || !local_is_locked) - rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock); } /* @@ -316,7 +417,7 @@ timer_add(struct rte_timer *tim, unsigned tim_lcore, int local_is_locked) */ static void timer_del(struct rte_timer *tim, union rte_timer_status prev_status, - int local_is_locked) + int local_is_locked, struct priv_timer *priv_timer) { unsigned lcore_id = rte_lcore_id(); unsigned prev_owner = prev_status.owner; @@ -336,7 +437,7 @@ timer_del(struct rte_timer *tim, union rte_timer_status prev_status, ((tim->sl_next[0] == NULL) ? 0 : tim->sl_next[0]->expire); /* adjust pointers from previous entries to point past this */ - timer_get_prev_entries_for_node(tim, prev_owner, prev); + timer_get_prev_entries_for_node(tim, prev_owner, prev, priv_timer); for (i = priv_timer[prev_owner].curr_skiplist_depth - 1; i >= 0; i--) { if (prev[i]->sl_next[i] == tim) prev[i]->sl_next[i] = tim->sl_next[i]; @@ -358,34 +459,44 @@ static int __rte_timer_reset(struct rte_timer *tim, uint64_t expire, uint64_t period, unsigned tim_lcore, rte_timer_cb_t fct, void *arg, - int local_is_locked) + int local_is_locked, + struct rte_timer_data *timer_data) { union rte_timer_status prev_status, status; int ret; unsigned lcore_id = rte_lcore_id(); + struct priv_timer *priv_timer = timer_data->priv_timer; /* round robin for tim_lcore */ if (tim_lcore == (unsigned)LCORE_ID_ANY) { - tim_lcore = rte_get_next_lcore(priv_timer[lcore_id].prev_lcore, - 0, 1); - priv_timer[lcore_id].prev_lcore = tim_lcore; + if (lcore_id < RTE_MAX_LCORE) { + /* EAL thread with valid lcore_id */ + tim_lcore = rte_get_next_lcore( + priv_timer[lcore_id].prev_lcore, + 0, 1); + priv_timer[lcore_id].prev_lcore = tim_lcore; + } else + /* non-EAL thread do not run rte_timer_manage(), + * so schedule the timer on the first enabled lcore. */ + tim_lcore = rte_get_next_lcore(LCORE_ID_ANY, 0, 1); } /* wait that the timer is in correct status before update, * and mark it as being configured */ - ret = timer_set_config_state(tim, &prev_status); + ret = timer_set_config_state(tim, &prev_status, priv_timer); if (ret < 0) return -1; - __TIMER_STAT_ADD(reset, 1); - if (prev_status.state == RTE_TIMER_RUNNING) { + __TIMER_STAT_ADD(priv_timer, reset, 1); + if (prev_status.state == RTE_TIMER_RUNNING && + lcore_id < RTE_MAX_LCORE) { priv_timer[lcore_id].updated = 1; } /* remove it from list */ if (prev_status.state == RTE_TIMER_PENDING) { - timer_del(tim, prev_status, local_is_locked); - __TIMER_STAT_ADD(pending, -1); + timer_del(tim, prev_status, local_is_locked, priv_timer); + __TIMER_STAT_ADD(priv_timer, pending, -1); } tim->period = period; @@ -393,8 +504,15 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire, tim->f = fct; tim->arg = arg; - __TIMER_STAT_ADD(pending, 1); - timer_add(tim, tim_lcore, local_is_locked); + /* if timer needs to be scheduled on another core, we need to + * lock the destination list; if it is on local core, we need to lock if + * we are not called from rte_timer_manage() + */ + if (tim_lcore != lcore_id || !local_is_locked) + rte_spinlock_lock(&priv_timer[tim_lcore].list_lock); + + __TIMER_STAT_ADD(priv_timer, pending, 1); + timer_add(tim, tim_lcore, priv_timer); /* update state: as we are in CONFIG state, only us can modify * the state so we don't need to use cmpset() here */ @@ -403,31 +521,40 @@ __rte_timer_reset(struct rte_timer *tim, uint64_t expire, status.owner = (int16_t)tim_lcore; tim->status.u32 = status.u32; + if (tim_lcore != lcore_id || !local_is_locked) + rte_spinlock_unlock(&priv_timer[tim_lcore].list_lock); + return 0; } /* Reset and start the timer associated with the timer handle tim */ int rte_timer_reset(struct rte_timer *tim, uint64_t ticks, - enum rte_timer_type type, unsigned tim_lcore, - rte_timer_cb_t fct, void *arg) + enum rte_timer_type type, unsigned int tim_lcore, + rte_timer_cb_t fct, void *arg) +{ + return rte_timer_alt_reset(default_data_id, tim, ticks, type, + tim_lcore, fct, arg); +} + +int +rte_timer_alt_reset(uint32_t timer_data_id, struct rte_timer *tim, + uint64_t ticks, enum rte_timer_type type, + unsigned int tim_lcore, rte_timer_cb_t fct, void *arg) { uint64_t cur_time = rte_get_timer_cycles(); uint64_t period; + struct rte_timer_data *timer_data; - if (unlikely((tim_lcore != (unsigned)LCORE_ID_ANY) && - !rte_lcore_is_enabled(tim_lcore))) - return -1; + TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL); if (type == PERIODICAL) period = ticks; else period = 0; - __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore, - fct, arg, 0); - - return 0; + return __rte_timer_reset(tim, cur_time + ticks, period, tim_lcore, + fct, arg, 0, timer_data); } /* loop until rte_timer_reset() succeed */ @@ -437,32 +564,35 @@ rte_timer_reset_sync(struct rte_timer *tim, uint64_t ticks, rte_timer_cb_t fct, void *arg) { while (rte_timer_reset(tim, ticks, type, tim_lcore, - fct, arg) != 0); + fct, arg) != 0) + rte_pause(); } -/* Stop the timer associated with the timer handle tim */ -int -rte_timer_stop(struct rte_timer *tim) +static int +__rte_timer_stop(struct rte_timer *tim, int local_is_locked, + struct rte_timer_data *timer_data) { union rte_timer_status prev_status, status; unsigned lcore_id = rte_lcore_id(); int ret; + struct priv_timer *priv_timer = timer_data->priv_timer; /* wait that the timer is in correct status before update, * and mark it as being configured */ - ret = timer_set_config_state(tim, &prev_status); + ret = timer_set_config_state(tim, &prev_status, priv_timer); if (ret < 0) return -1; - __TIMER_STAT_ADD(stop, 1); - if (prev_status.state == RTE_TIMER_RUNNING) { + __TIMER_STAT_ADD(priv_timer, stop, 1); + if (prev_status.state == RTE_TIMER_RUNNING && + lcore_id < RTE_MAX_LCORE) { priv_timer[lcore_id].updated = 1; } /* remove it from list */ if (prev_status.state == RTE_TIMER_PENDING) { - timer_del(tim, prev_status, 0); - __TIMER_STAT_ADD(pending, -1); + timer_del(tim, prev_status, local_is_locked, priv_timer); + __TIMER_STAT_ADD(priv_timer, pending, -1); } /* mark timer as stopped */ @@ -474,6 +604,23 @@ rte_timer_stop(struct rte_timer *tim) return 0; } +/* Stop the timer associated with the timer handle tim */ +int +rte_timer_stop(struct rte_timer *tim) +{ + return rte_timer_alt_stop(default_data_id, tim); +} + +int +rte_timer_alt_stop(uint32_t timer_data_id, struct rte_timer *tim) +{ + struct rte_timer_data *timer_data; + + TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL); + + return __rte_timer_stop(tim, 0, timer_data); +} + /* loop until rte_timer_stop() succeed */ void rte_timer_stop_sync(struct rte_timer *tim) @@ -490,25 +637,31 @@ rte_timer_pending(struct rte_timer *tim) } /* must be called periodically, run all timer that expired */ -void rte_timer_manage(void) +static void +__rte_timer_manage(struct rte_timer_data *timer_data) { union rte_timer_status status; struct rte_timer *tim, *next_tim; + struct rte_timer *run_first_tim, **pprev; unsigned lcore_id = rte_lcore_id(); struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1]; uint64_t cur_time; int i, ret; + struct priv_timer *priv_timer = timer_data->priv_timer; + + /* timer manager only runs on EAL thread with valid lcore_id */ + assert(lcore_id < RTE_MAX_LCORE); - __TIMER_STAT_ADD(manage, 1); + __TIMER_STAT_ADD(priv_timer, manage, 1); /* optimize for the case where per-cpu list is empty */ if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) return; cur_time = rte_get_timer_cycles(); -#ifdef RTE_ARCH_X86_64 - /* on 64-bit the value cached in the pending_head.expired will be updated - * atomically, so we can consult that for a quick check here outside the - * lock */ +#ifdef RTE_ARCH_64 + /* on 64-bit the value cached in the pending_head.expired will be + * updated atomically, so we can consult that for a quick check here + * outside the lock */ if (likely(priv_timer[lcore_id].pending_head.expire > cur_time)) return; #endif @@ -518,40 +671,61 @@ void rte_timer_manage(void) /* if nothing to do just unlock and return */ if (priv_timer[lcore_id].pending_head.sl_next[0] == NULL || - priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) - goto done; + priv_timer[lcore_id].pending_head.sl_next[0]->expire > cur_time) { + rte_spinlock_unlock(&priv_timer[lcore_id].list_lock); + return; + } /* save start of list of expired timers */ tim = priv_timer[lcore_id].pending_head.sl_next[0]; /* break the existing list at current time point */ - timer_get_prev_entries(cur_time, lcore_id, prev); + timer_get_prev_entries(cur_time, lcore_id, prev, priv_timer); for (i = priv_timer[lcore_id].curr_skiplist_depth -1; i >= 0; i--) { - priv_timer[lcore_id].pending_head.sl_next[i] = prev[i]->sl_next[i]; + if (prev[i] == &priv_timer[lcore_id].pending_head) + continue; + priv_timer[lcore_id].pending_head.sl_next[i] = + prev[i]->sl_next[i]; if (prev[i]->sl_next[i] == NULL) priv_timer[lcore_id].curr_skiplist_depth--; prev[i] ->sl_next[i] = NULL; } - /* now scan expired list and call callbacks */ + /* transition run-list from PENDING to RUNNING */ + run_first_tim = tim; + pprev = &run_first_tim; + for ( ; tim != NULL; tim = next_tim) { next_tim = tim->sl_next[0]; ret = timer_set_running_state(tim); + if (likely(ret == 0)) { + pprev = &tim->sl_next[0]; + } else { + /* another core is trying to re-config this one, + * remove it from local expired list + */ + *pprev = next_tim; + } + } - /* this timer was not pending, continue */ - if (ret < 0) - continue; + /* update the next to expire timer value */ + priv_timer[lcore_id].pending_head.expire = + (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 : + priv_timer[lcore_id].pending_head.sl_next[0]->expire; - rte_spinlock_unlock(&priv_timer[lcore_id].list_lock); + rte_spinlock_unlock(&priv_timer[lcore_id].list_lock); + /* now scan expired list and call callbacks */ + for (tim = run_first_tim; tim != NULL; tim = next_tim) { + next_tim = tim->sl_next[0]; priv_timer[lcore_id].updated = 0; + priv_timer[lcore_id].running_tim = tim; /* execute callback function with list unlocked */ tim->f(tim, tim->arg); - rte_spinlock_lock(&priv_timer[lcore_id].list_lock); - __TIMER_STAT_ADD(pending, -1); + __TIMER_STAT_ADD(priv_timer, pending, -1); /* the timer was stopped or reloaded by the callback * function, we have nothing to do here */ if (priv_timer[lcore_id].updated == 1) @@ -566,31 +740,251 @@ void rte_timer_manage(void) } else { /* keep it in list and mark timer as pending */ + rte_spinlock_lock(&priv_timer[lcore_id].list_lock); status.state = RTE_TIMER_PENDING; - __TIMER_STAT_ADD(pending, 1); + __TIMER_STAT_ADD(priv_timer, pending, 1); status.owner = (int16_t)lcore_id; rte_wmb(); tim->status.u32 = status.u32; - __rte_timer_reset(tim, cur_time + tim->period, - tim->period, lcore_id, tim->f, tim->arg, 1); + __rte_timer_reset(tim, tim->expire + tim->period, + tim->period, lcore_id, tim->f, tim->arg, 1, + timer_data); + rte_spinlock_unlock(&priv_timer[lcore_id].list_lock); } } + priv_timer[lcore_id].running_tim = NULL; +} - /* update the next to expire timer value */ - priv_timer[lcore_id].pending_head.expire = - (priv_timer[lcore_id].pending_head.sl_next[0] == NULL) ? 0 : - priv_timer[lcore_id].pending_head.sl_next[0]->expire; -done: - /* job finished, unlock the list lock */ - rte_spinlock_unlock(&priv_timer[lcore_id].list_lock); +int +rte_timer_manage(void) +{ + struct rte_timer_data *timer_data; + + TIMER_DATA_VALID_GET_OR_ERR_RET(default_data_id, timer_data, -EINVAL); + + __rte_timer_manage(timer_data); + + return 0; +} + +int +rte_timer_alt_manage(uint32_t timer_data_id, + unsigned int *poll_lcores, + int nb_poll_lcores, + rte_timer_alt_manage_cb_t f) +{ + unsigned int default_poll_lcores[] = {rte_lcore_id()}; + union rte_timer_status status; + struct rte_timer *tim, *next_tim, **pprev; + struct rte_timer *run_first_tims[RTE_MAX_LCORE]; + unsigned int this_lcore = rte_lcore_id(); + struct rte_timer *prev[MAX_SKIPLIST_DEPTH + 1]; + uint64_t cur_time; + int i, j, ret; + int nb_runlists = 0; + struct rte_timer_data *data; + struct priv_timer *privp; + uint32_t poll_lcore; + + TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, data, -EINVAL); + + /* timer manager only runs on EAL thread with valid lcore_id */ + assert(this_lcore < RTE_MAX_LCORE); + + __TIMER_STAT_ADD(data->priv_timer, manage, 1); + + if (poll_lcores == NULL) { + poll_lcores = default_poll_lcores; + nb_poll_lcores = RTE_DIM(default_poll_lcores); + } + + for (i = 0; i < nb_poll_lcores; i++) { + poll_lcore = poll_lcores[i]; + privp = &data->priv_timer[poll_lcore]; + + /* optimize for the case where per-cpu list is empty */ + if (privp->pending_head.sl_next[0] == NULL) + continue; + cur_time = rte_get_timer_cycles(); + +#ifdef RTE_ARCH_64 + /* on 64-bit the value cached in the pending_head.expired will + * be updated atomically, so we can consult that for a quick + * check here outside the lock + */ + if (likely(privp->pending_head.expire > cur_time)) + continue; +#endif + + /* browse ordered list, add expired timers in 'expired' list */ + rte_spinlock_lock(&privp->list_lock); + + /* if nothing to do just unlock and return */ + if (privp->pending_head.sl_next[0] == NULL || + privp->pending_head.sl_next[0]->expire > cur_time) { + rte_spinlock_unlock(&privp->list_lock); + continue; + } + + /* save start of list of expired timers */ + tim = privp->pending_head.sl_next[0]; + + /* break the existing list at current time point */ + timer_get_prev_entries(cur_time, poll_lcore, prev, + data->priv_timer); + for (j = privp->curr_skiplist_depth - 1; j >= 0; j--) { + if (prev[j] == &privp->pending_head) + continue; + privp->pending_head.sl_next[j] = + prev[j]->sl_next[j]; + if (prev[j]->sl_next[j] == NULL) + privp->curr_skiplist_depth--; + + prev[j]->sl_next[j] = NULL; + } + + /* transition run-list from PENDING to RUNNING */ + run_first_tims[nb_runlists] = tim; + pprev = &run_first_tims[nb_runlists]; + nb_runlists++; + + for ( ; tim != NULL; tim = next_tim) { + next_tim = tim->sl_next[0]; + + ret = timer_set_running_state(tim); + if (likely(ret == 0)) { + pprev = &tim->sl_next[0]; + } else { + /* another core is trying to re-config this one, + * remove it from local expired list + */ + *pprev = next_tim; + } + } + + /* update the next to expire timer value */ + privp->pending_head.expire = + (privp->pending_head.sl_next[0] == NULL) ? 0 : + privp->pending_head.sl_next[0]->expire; + + rte_spinlock_unlock(&privp->list_lock); + } + + /* Now process the run lists */ + while (1) { + bool done = true; + uint64_t min_expire = UINT64_MAX; + int min_idx = 0; + + /* Find the next oldest timer to process */ + for (i = 0; i < nb_runlists; i++) { + tim = run_first_tims[i]; + + if (tim != NULL && tim->expire < min_expire) { + min_expire = tim->expire; + min_idx = i; + done = false; + } + } + + if (done) + break; + + tim = run_first_tims[min_idx]; + + /* Move down the runlist from which we picked a timer to + * execute + */ + run_first_tims[min_idx] = run_first_tims[min_idx]->sl_next[0]; + + data->priv_timer[this_lcore].updated = 0; + data->priv_timer[this_lcore].running_tim = tim; + + /* Call the provided callback function */ + f(tim); + + __TIMER_STAT_ADD(data->priv_timer, pending, -1); + + /* the timer was stopped or reloaded by the callback + * function, we have nothing to do here + */ + if (data->priv_timer[this_lcore].updated == 1) + continue; + + if (tim->period == 0) { + /* remove from done list and mark timer as stopped */ + status.state = RTE_TIMER_STOP; + status.owner = RTE_TIMER_NO_OWNER; + rte_wmb(); + tim->status.u32 = status.u32; + } else { + /* keep it in list and mark timer as pending */ + rte_spinlock_lock( + &data->priv_timer[this_lcore].list_lock); + status.state = RTE_TIMER_PENDING; + __TIMER_STAT_ADD(data->priv_timer, pending, 1); + status.owner = (int16_t)this_lcore; + rte_wmb(); + tim->status.u32 = status.u32; + __rte_timer_reset(tim, tim->expire + tim->period, + tim->period, this_lcore, tim->f, tim->arg, 1, + data); + rte_spinlock_unlock( + &data->priv_timer[this_lcore].list_lock); + } + + data->priv_timer[this_lcore].running_tim = NULL; + } + + return 0; +} + +/* Walk pending lists, stopping timers and calling user-specified function */ +int +rte_timer_stop_all(uint32_t timer_data_id, unsigned int *walk_lcores, + int nb_walk_lcores, + rte_timer_stop_all_cb_t f, void *f_arg) +{ + int i; + struct priv_timer *priv_timer; + uint32_t walk_lcore; + struct rte_timer *tim, *next_tim; + struct rte_timer_data *timer_data; + + TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL); + + for (i = 0; i < nb_walk_lcores; i++) { + walk_lcore = walk_lcores[i]; + priv_timer = &timer_data->priv_timer[walk_lcore]; + + rte_spinlock_lock(&priv_timer->list_lock); + + for (tim = priv_timer->pending_head.sl_next[0]; + tim != NULL; + tim = next_tim) { + next_tim = tim->sl_next[0]; + + /* Call timer_stop with lock held */ + __rte_timer_stop(tim, 1, timer_data); + + if (f) + f(tim, f_arg); + } + + rte_spinlock_unlock(&priv_timer->list_lock); + } + + return 0; } /* dump statistics about timers */ -void rte_timer_dump_stats(FILE *f) +static void +__rte_timer_dump_stats(struct rte_timer_data *timer_data __rte_unused, FILE *f) { #ifdef RTE_LIBRTE_TIMER_DEBUG struct rte_timer_debug_stats sum; unsigned lcore_id; + struct priv_timer *priv_timer = timer_data->priv_timer; memset(&sum, 0, sizeof(sum)); for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { @@ -608,3 +1002,21 @@ void rte_timer_dump_stats(FILE *f) fprintf(f, "No timer statistics, RTE_LIBRTE_TIMER_DEBUG is disabled\n"); #endif } + +int +rte_timer_dump_stats(FILE *f) +{ + return rte_timer_alt_dump_stats(default_data_id, f); +} + +int +rte_timer_alt_dump_stats(uint32_t timer_data_id __rte_unused, FILE *f) +{ + struct rte_timer_data *timer_data; + + TIMER_DATA_VALID_GET_OR_ERR_RET(timer_data_id, timer_data, -EINVAL); + + __rte_timer_dump_stats(timer_data, f); + + return 0; +}