4 * Copyright(c) 2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 * Some portions of this software is derived from the
36 * https://github.com/halayli/lthread which carrys the following license.
38 * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
40 * Redistribution and use in source and binary forms, with or without
41 * modification, are permitted provided that the following conditions
43 * 1. Redistributions of source code must retain the above copyright
44 * notice, this list of conditions and the following disclaimer.
45 * 2. Redistributions in binary form must reproduce the above copyright
46 * notice, this list of conditions and the following disclaimer in the
47 * documentation and/or other materials provided with the distribution.
49 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
77 #include <rte_config.h>
81 #include "lthread_api.h"
83 #include "lthread_timer.h"
84 #include "lthread_tls.h"
85 #include "lthread_objcache.h"
86 #include "lthread_diag.h"
90 * This function gets called after an lthread function has returned.
92 void _lthread_exit_handler(struct lthread *lt)
95 lt->state |= BIT(ST_LT_EXITED);
97 if (!(lt->state & BIT(ST_LT_DETACH))) {
98 /* thread is this not explicitly detached
99 * it must be joinable, so we call lthread_exit().
104 /* if we get here the thread is detached so we can reschedule it,
105 * allowing the scheduler to free it
112 * Free resources allocated to an lthread
114 void _lthread_free(struct lthread *lt)
117 DIAG_EVENT(lt, LT_DIAG_LTHREAD_FREE, lt, 0);
119 /* invoke any user TLS destructor functions */
120 _lthread_tls_destroy(lt);
122 /* free memory allocated for TLS defined using RTE_PER_LTHREAD macros */
123 if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE)
124 _lthread_objcache_free(lt->tls->root_sched->per_lthread_cache,
125 lt->per_lthread_data);
127 /* free pthread style TLS memory */
128 _lthread_objcache_free(lt->tls->root_sched->tls_cache, lt->tls);
131 _lthread_objcache_free(lt->stack_container->root_sched->stack_cache,
132 lt->stack_container);
134 /* now free the thread */
135 _lthread_objcache_free(lt->root_sched->lthread_cache, lt);
140 * Allocate a stack and maintain a cache of stacks
142 struct lthread_stack *_stack_alloc(void)
144 struct lthread_stack *s;
146 s = _lthread_objcache_alloc((THIS_SCHED)->stack_cache);
147 LTHREAD_ASSERT(s != NULL);
149 s->root_sched = THIS_SCHED;
150 s->stack_size = LTHREAD_MAX_STACK_SIZE;
155 * Execute a ctx by invoking the start function
156 * On return call an exit handler if the user has provided one
158 static void _lthread_exec(void *arg)
160 struct lthread *lt = (struct lthread *)arg;
162 /* invoke the contexts function */
164 /* do exit handling */
165 if (lt->exit_handler != NULL)
166 lt->exit_handler(lt);
170 * Initialize an lthread
171 * Set its function, args, and exit handler
174 _lthread_init(struct lthread *lt,
175 lthread_func_t fun, void *arg, lthread_exit_func exit_handler)
178 /* set ctx func and args */
181 lt->exit_handler = exit_handler;
183 /* set initial state */
184 lt->birth = _sched_now();
185 lt->state = BIT(ST_LT_INIT);
186 lt->join = LT_JOIN_INITIAL;
190 * set the lthread stack
192 void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size)
194 char *stack_top = (char *)stack + stack_size;
195 void **s = (void **)stack_top;
199 lt->stack_size = stack_size;
201 /* set initial context */
204 lt->ctx.rsp = (void *)(stack_top - (4 * sizeof(void *)));
205 lt->ctx.rbp = (void *)(stack_top - (3 * sizeof(void *)));
206 lt->ctx.rip = (void *)_lthread_exec;
210 * Create an lthread on the current scheduler
211 * If there is no current scheduler on this pthread then first create one
214 lthread_create(struct lthread **new_lt, int lcore_id,
215 lthread_func_t fun, void *arg)
217 if ((new_lt == NULL) || (fun == NULL))
218 return POSIX_ERRNO(EINVAL);
221 lcore_id = rte_lcore_id();
222 else if (lcore_id > LTHREAD_MAX_LCORES)
223 return POSIX_ERRNO(EINVAL);
225 struct lthread *lt = NULL;
227 if (THIS_SCHED == NULL) {
228 THIS_SCHED = _lthread_sched_create(0);
229 if (THIS_SCHED == NULL) {
230 perror("Failed to create scheduler");
231 return POSIX_ERRNO(EAGAIN);
235 /* allocate a thread structure */
236 lt = _lthread_objcache_alloc((THIS_SCHED)->lthread_cache);
238 return POSIX_ERRNO(EAGAIN);
240 bzero(lt, sizeof(struct lthread));
241 lt->root_sched = THIS_SCHED;
243 /* set the function args and exit handlder */
244 _lthread_init(lt, fun, arg, _lthread_exit_handler);
246 /* put it in the ready queue */
250 lcore_id = rte_lcore_id();
252 DIAG_CREATE_EVENT(lt, LT_DIAG_LTHREAD_CREATE);
255 _ready_queue_insert(_lthread_sched_get(lcore_id), lt);
260 * Schedules lthread to sleep for `nsecs`
261 * setting the lthread state to LT_ST_SLEEPING.
262 * lthread state is cleared upon resumption or expiry.
264 static inline void _lthread_sched_sleep(struct lthread *lt, uint64_t nsecs)
266 uint64_t state = lt->state;
267 uint64_t clks = _ns_to_clks(nsecs);
270 _timer_start(lt, clks);
271 lt->state = state | BIT(ST_LT_SLEEPING);
273 DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
280 * Cancels any running timer.
281 * This can be called multiple times on the same lthread regardless if it was
284 int _lthread_desched_sleep(struct lthread *lt)
286 uint64_t state = lt->state;
288 if (state & BIT(ST_LT_SLEEPING)) {
290 state &= (CLEARBIT(ST_LT_SLEEPING) & CLEARBIT(ST_LT_EXPIRED));
291 lt->state = state | BIT(ST_LT_READY);
298 * set user data pointer in an lthread
300 void lthread_set_data(void *data)
302 if (sizeof(void *) == RTE_PER_LTHREAD_SECTION_SIZE)
303 THIS_LTHREAD->per_lthread_data = data;
307 * Retrieve user data pointer from an lthread
309 void *lthread_get_data(void)
311 return THIS_LTHREAD->per_lthread_data;
315 * Return the current lthread handle
317 struct lthread *lthread_current(void)
319 struct lthread_sched *sched = THIS_SCHED;
322 return sched->current_lthread;
329 * Tasklet to cancel a thread
334 struct lthread *lt = (struct lthread *) arg;
336 lt->state |= BIT(ST_LT_CANCELLED);
342 * Mark the specified as canceled
344 int lthread_cancel(struct lthread *cancel_lt)
348 if ((cancel_lt == NULL) || (cancel_lt == THIS_LTHREAD))
349 return POSIX_ERRNO(EINVAL);
351 DIAG_EVENT(cancel_lt, LT_DIAG_LTHREAD_CANCEL, cancel_lt, 0);
353 if (cancel_lt->sched != THIS_SCHED) {
355 /* spawn task-let to cancel the thread */
357 cancel_lt->sched->lcore_id,
362 cancel_lt->state |= BIT(ST_LT_CANCELLED);
367 * Suspend the current lthread for specified time
369 void lthread_sleep(uint64_t nsecs)
371 struct lthread *lt = THIS_LTHREAD;
373 _lthread_sched_sleep(lt, nsecs);
378 * Suspend the current lthread for specified time
380 void lthread_sleep_clks(uint64_t clks)
382 struct lthread *lt = THIS_LTHREAD;
383 uint64_t state = lt->state;
386 _timer_start(lt, clks);
387 lt->state = state | BIT(ST_LT_SLEEPING);
389 DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
394 * Requeue the current thread to the back of the ready queue
396 void lthread_yield(void)
398 struct lthread *lt = THIS_LTHREAD;
400 DIAG_EVENT(lt, LT_DIAG_LTHREAD_YIELD, 0, 0);
402 _ready_queue_insert(THIS_SCHED, lt);
403 ctx_switch(&(THIS_SCHED)->ctx, <->ctx);
407 * Exit the current lthread
408 * If a thread is joining pass the user pointer to it
410 void lthread_exit(void *ptr)
412 struct lthread *lt = THIS_LTHREAD;
414 /* if thread is detached (this is not valid) just exit */
415 if (lt->state & BIT(ST_LT_DETACH))
418 /* There is a race between lthread_join() and lthread_exit()
419 * - if exit before join then we suspend and resume on join
420 * - if join before exit then we resume the joining thread
422 if ((lt->join == LT_JOIN_INITIAL)
423 && rte_atomic64_cmpset(<->join, LT_JOIN_INITIAL,
426 DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 1, 0);
428 /* set the exit value */
429 if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
430 *(lt->lt_join->lt_exit_ptr) = ptr;
432 /* let the joining thread know we have set the exit value */
433 lt->join = LT_JOIN_EXIT_VAL_SET;
436 DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 0, 0);
437 /* set the exit value */
438 if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
439 *(lt->lt_join->lt_exit_ptr) = ptr;
440 /* let the joining thread know we have set the exit value */
441 lt->join = LT_JOIN_EXIT_VAL_SET;
442 _ready_queue_insert(lt->lt_join->sched,
443 (struct lthread *)lt->lt_join);
447 /* wait until the joinging thread has collected the exit value */
448 while (lt->join != LT_JOIN_EXIT_VAL_READ)
451 /* reset join state */
452 lt->join = LT_JOIN_INITIAL;
454 /* detach it so its resources can be released */
455 lt->state |= (BIT(ST_LT_DETACH) | BIT(ST_LT_EXITED));
460 * Suspend until the joined thread returns
462 int lthread_join(struct lthread *lt, void **ptr)
465 return POSIX_ERRNO(EINVAL);
467 struct lthread *current = THIS_LTHREAD;
468 uint64_t lt_state = lt->state;
470 /* invalid to join a detached thread, or a thread that is joined */
471 if ((lt_state & BIT(ST_LT_DETACH)) || (lt->join == LT_JOIN_THREAD_SET))
472 return POSIX_ERRNO(EINVAL);
473 /* pointer to the joining thread and a poingter to return a value */
474 lt->lt_join = current;
475 current->lt_exit_ptr = ptr;
476 /* There is a race between lthread_join() and lthread_exit()
477 * - if join before exit we suspend and will resume when exit is called
478 * - if exit before join we resume the exiting thread
480 if ((lt->join == LT_JOIN_INITIAL)
481 && rte_atomic64_cmpset(<->join, LT_JOIN_INITIAL,
482 LT_JOIN_THREAD_SET)) {
484 DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 1);
487 DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 0);
488 _ready_queue_insert(lt->sched, lt);
491 /* wait for exiting thread to set return value */
492 while (lt->join != LT_JOIN_EXIT_VAL_SET)
495 /* collect the return value */
497 *ptr = *current->lt_exit_ptr;
499 /* let the exiting thread proceed to exit */
500 lt->join = LT_JOIN_EXIT_VAL_READ;
506 * Detach current lthread
507 * A detached thread cannot be joined
509 void lthread_detach(void)
511 struct lthread *lt = THIS_LTHREAD;
513 DIAG_EVENT(lt, LT_DIAG_LTHREAD_DETACH, 0, 0);
515 uint64_t state = lt->state;
517 lt->state = state | BIT(ST_LT_DETACH);
521 * Set function name of an lthread
522 * this is a debug aid
524 void lthread_set_funcname(const char *f)
526 struct lthread *lt = THIS_LTHREAD;
528 strncpy(lt->funcname, f, sizeof(lt->funcname));
529 lt->funcname[sizeof(lt->funcname)-1] = 0;