4 * Copyright(c) 2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <sys/types.h>
44 #include "lthread_api.h"
45 #include "pthread_shim.h"
47 #define RTE_LOGTYPE_PTHREAD_SHIM RTE_LOGTYPE_USER3
49 #define POSIX_ERRNO(x) (x)
51 /* some releases of FreeBSD 10, e.g. 10.0, don't have CPU_COUNT macro */
53 #define CPU_COUNT(x) __cpu_count(x)
55 static inline unsigned int
56 __cpu_count(const rte_cpuset_t *cpuset)
58 unsigned int i, count = 0;
59 for (i = 0; i < RTE_MAX_LCORE; i++)
60 if (CPU_ISSET(i, cpuset))
67 * this flag determines at run time if we override pthread
68 * calls and map then to equivalent lthread calls
69 * or of we call the standard pthread function
71 static __thread int override;
75 * this structures contains function pointers that will be
76 * initialised to the loaded address of the real
77 * pthread library API functions
79 struct pthread_lib_funcs {
80 int (*f_pthread_barrier_destroy)
81 (pthread_barrier_t *);
82 int (*f_pthread_barrier_init)
83 (pthread_barrier_t *, const pthread_barrierattr_t *, unsigned);
84 int (*f_pthread_barrier_wait)
85 (pthread_barrier_t *);
86 int (*f_pthread_cond_broadcast)
88 int (*f_pthread_cond_destroy)
90 int (*f_pthread_cond_init)
91 (pthread_cond_t *, const pthread_condattr_t *);
92 int (*f_pthread_cond_signal)
94 int (*f_pthread_cond_timedwait)
95 (pthread_cond_t *, pthread_mutex_t *, const struct timespec *);
96 int (*f_pthread_cond_wait)
97 (pthread_cond_t *, pthread_mutex_t *);
98 int (*f_pthread_create)
99 (pthread_t *, const pthread_attr_t *, void *(*)(void *), void *);
100 int (*f_pthread_detach)
102 int (*f_pthread_equal)
103 (pthread_t, pthread_t);
104 void (*f_pthread_exit)
106 void * (*f_pthread_getspecific)
108 int (*f_pthread_getcpuclockid)
109 (pthread_t, clockid_t *);
110 int (*f_pthread_join)
111 (pthread_t, void **);
112 int (*f_pthread_key_create)
113 (pthread_key_t *, void (*) (void *));
114 int (*f_pthread_key_delete)
116 int (*f_pthread_mutex_destroy)
117 (pthread_mutex_t *__mutex);
118 int (*f_pthread_mutex_init)
119 (pthread_mutex_t *__mutex, const pthread_mutexattr_t *);
120 int (*f_pthread_mutex_lock)
121 (pthread_mutex_t *__mutex);
122 int (*f_pthread_mutex_trylock)
123 (pthread_mutex_t *__mutex);
124 int (*f_pthread_mutex_timedlock)
125 (pthread_mutex_t *__mutex, const struct timespec *);
126 int (*f_pthread_mutex_unlock)
127 (pthread_mutex_t *__mutex);
128 int (*f_pthread_once)
129 (pthread_once_t *, void (*) (void));
130 int (*f_pthread_rwlock_destroy)
131 (pthread_rwlock_t *__rwlock);
132 int (*f_pthread_rwlock_init)
133 (pthread_rwlock_t *__rwlock, const pthread_rwlockattr_t *);
134 int (*f_pthread_rwlock_rdlock)
135 (pthread_rwlock_t *__rwlock);
136 int (*f_pthread_rwlock_timedrdlock)
137 (pthread_rwlock_t *__rwlock, const struct timespec *);
138 int (*f_pthread_rwlock_timedwrlock)
139 (pthread_rwlock_t *__rwlock, const struct timespec *);
140 int (*f_pthread_rwlock_tryrdlock)
141 (pthread_rwlock_t *__rwlock);
142 int (*f_pthread_rwlock_trywrlock)
143 (pthread_rwlock_t *__rwlock);
144 int (*f_pthread_rwlock_unlock)
145 (pthread_rwlock_t *__rwlock);
146 int (*f_pthread_rwlock_wrlock)
147 (pthread_rwlock_t *__rwlock);
148 pthread_t (*f_pthread_self)
150 int (*f_pthread_setspecific)
151 (pthread_key_t, const void *);
152 int (*f_pthread_spin_init)
153 (pthread_spinlock_t *__spin, int);
154 int (*f_pthread_spin_destroy)
155 (pthread_spinlock_t *__spin);
156 int (*f_pthread_spin_lock)
157 (pthread_spinlock_t *__spin);
158 int (*f_pthread_spin_trylock)
159 (pthread_spinlock_t *__spin);
160 int (*f_pthread_spin_unlock)
161 (pthread_spinlock_t *__spin);
162 int (*f_pthread_cancel)
164 int (*f_pthread_setcancelstate)
166 int (*f_pthread_setcanceltype)
168 void (*f_pthread_testcancel)
170 int (*f_pthread_getschedparam)
171 (pthread_t pthread, int *, struct sched_param *);
172 int (*f_pthread_setschedparam)
173 (pthread_t, int, const struct sched_param *);
174 int (*f_pthread_yield)
176 int (*f_pthread_setaffinity_np)
177 (pthread_t thread, size_t cpusetsize, const rte_cpuset_t *cpuset);
179 (const struct timespec *req, struct timespec *rem);
180 } _sys_pthread_funcs = {
181 .f_pthread_barrier_destroy = NULL,
186 * this macro obtains the loaded address of a library function
189 static void *__libc_dl_handle = RTLD_NEXT;
191 #define get_addr_of_loaded_symbol(name) do { \
193 _sys_pthread_funcs.f_##name = dlsym(__libc_dl_handle, (#name)); \
194 error_str = dlerror(); \
195 if (error_str != NULL) { \
196 fprintf(stderr, "%s\n", error_str); \
202 * The constructor function initialises the
203 * function pointers for pthread library functions
206 pthread_intercept_ctor(void)__attribute__((constructor));
208 pthread_intercept_ctor(void)
212 * Get the original functions
214 get_addr_of_loaded_symbol(pthread_barrier_destroy);
215 get_addr_of_loaded_symbol(pthread_barrier_init);
216 get_addr_of_loaded_symbol(pthread_barrier_wait);
217 get_addr_of_loaded_symbol(pthread_cond_broadcast);
218 get_addr_of_loaded_symbol(pthread_cond_destroy);
219 get_addr_of_loaded_symbol(pthread_cond_init);
220 get_addr_of_loaded_symbol(pthread_cond_signal);
221 get_addr_of_loaded_symbol(pthread_cond_timedwait);
222 get_addr_of_loaded_symbol(pthread_cond_wait);
223 get_addr_of_loaded_symbol(pthread_create);
224 get_addr_of_loaded_symbol(pthread_detach);
225 get_addr_of_loaded_symbol(pthread_equal);
226 get_addr_of_loaded_symbol(pthread_exit);
227 get_addr_of_loaded_symbol(pthread_getspecific);
228 get_addr_of_loaded_symbol(pthread_getcpuclockid);
229 get_addr_of_loaded_symbol(pthread_join);
230 get_addr_of_loaded_symbol(pthread_key_create);
231 get_addr_of_loaded_symbol(pthread_key_delete);
232 get_addr_of_loaded_symbol(pthread_mutex_destroy);
233 get_addr_of_loaded_symbol(pthread_mutex_init);
234 get_addr_of_loaded_symbol(pthread_mutex_lock);
235 get_addr_of_loaded_symbol(pthread_mutex_trylock);
236 get_addr_of_loaded_symbol(pthread_mutex_timedlock);
237 get_addr_of_loaded_symbol(pthread_mutex_unlock);
238 get_addr_of_loaded_symbol(pthread_once);
239 get_addr_of_loaded_symbol(pthread_rwlock_destroy);
240 get_addr_of_loaded_symbol(pthread_rwlock_init);
241 get_addr_of_loaded_symbol(pthread_rwlock_rdlock);
242 get_addr_of_loaded_symbol(pthread_rwlock_timedrdlock);
243 get_addr_of_loaded_symbol(pthread_rwlock_timedwrlock);
244 get_addr_of_loaded_symbol(pthread_rwlock_tryrdlock);
245 get_addr_of_loaded_symbol(pthread_rwlock_trywrlock);
246 get_addr_of_loaded_symbol(pthread_rwlock_unlock);
247 get_addr_of_loaded_symbol(pthread_rwlock_wrlock);
248 get_addr_of_loaded_symbol(pthread_self);
249 get_addr_of_loaded_symbol(pthread_setspecific);
250 get_addr_of_loaded_symbol(pthread_spin_init);
251 get_addr_of_loaded_symbol(pthread_spin_destroy);
252 get_addr_of_loaded_symbol(pthread_spin_lock);
253 get_addr_of_loaded_symbol(pthread_spin_trylock);
254 get_addr_of_loaded_symbol(pthread_spin_unlock);
255 get_addr_of_loaded_symbol(pthread_cancel);
256 get_addr_of_loaded_symbol(pthread_setcancelstate);
257 get_addr_of_loaded_symbol(pthread_setcanceltype);
258 get_addr_of_loaded_symbol(pthread_testcancel);
259 get_addr_of_loaded_symbol(pthread_getschedparam);
260 get_addr_of_loaded_symbol(pthread_setschedparam);
261 get_addr_of_loaded_symbol(pthread_yield);
262 get_addr_of_loaded_symbol(pthread_setaffinity_np);
263 get_addr_of_loaded_symbol(nanosleep);
268 * Enable/Disable pthread override
273 void pthread_override_set(int state)
280 * Return pthread override state
285 int pthread_override_get(void)
291 * This macro is used to catch and log
292 * invocation of stubs for unimplemented pthread
295 #define NOT_IMPLEMENTED do { \
299 "WARNING %s NOT IMPLEMENTED\n", \
305 * pthread API override functions follow
306 * Note in this example code only a subset of functions are
309 * The stub functions provided will issue a warning log
310 * message if an unimplemented function is invoked
314 int pthread_barrier_destroy(pthread_barrier_t *a)
317 return _sys_pthread_funcs.f_pthread_barrier_destroy(a);
321 pthread_barrier_init(pthread_barrier_t *a,
322 const pthread_barrierattr_t *b, unsigned c)
325 return _sys_pthread_funcs.f_pthread_barrier_init(a, b, c);
328 int pthread_barrier_wait(pthread_barrier_t *a)
331 return _sys_pthread_funcs.f_pthread_barrier_wait(a);
334 int pthread_cond_broadcast(pthread_cond_t *cond)
338 lthread_cond_broadcast(*(struct lthread_cond **)cond);
341 return _sys_pthread_funcs.f_pthread_cond_broadcast(cond);
344 int pthread_mutex_destroy(pthread_mutex_t *mutex)
347 return lthread_mutex_destroy(*(struct lthread_mutex **)mutex);
348 return _sys_pthread_funcs.f_pthread_mutex_destroy(mutex);
351 int pthread_cond_destroy(pthread_cond_t *cond)
354 return lthread_cond_destroy(*(struct lthread_cond **)cond);
355 return _sys_pthread_funcs.f_pthread_cond_destroy(cond);
358 int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
361 return lthread_cond_init(NULL,
362 (struct lthread_cond **)cond,
363 (const struct lthread_condattr *) attr);
364 return _sys_pthread_funcs.f_pthread_cond_init(cond, attr);
367 int pthread_cond_signal(pthread_cond_t *cond)
370 lthread_cond_signal(*(struct lthread_cond **)cond);
373 return _sys_pthread_funcs.f_pthread_cond_signal(cond);
377 pthread_cond_timedwait(pthread_cond_t *__restrict cond,
378 pthread_mutex_t *__restrict mutex,
379 const struct timespec *__restrict time)
382 return _sys_pthread_funcs.f_pthread_cond_timedwait(cond, mutex, time);
385 int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
388 pthread_mutex_unlock(mutex);
389 int rv = lthread_cond_wait(*(struct lthread_cond **)cond, 0);
391 pthread_mutex_lock(mutex);
394 return _sys_pthread_funcs.f_pthread_cond_wait(cond, mutex);
398 pthread_create(pthread_t *__restrict tid,
399 const pthread_attr_t *__restrict attr,
400 void *(func) (void *),
401 void *__restrict arg)
407 /* determine CPU being requested */
411 pthread_attr_getaffinity_np(attr,
412 sizeof(rte_cpuset_t),
415 if (CPU_COUNT(&cpuset) != 1)
416 return POSIX_ERRNO(EINVAL);
418 for (lcore = 0; lcore < LTHREAD_MAX_LCORES; lcore++) {
419 if (!CPU_ISSET(lcore, &cpuset))
424 return lthread_create((struct lthread **)tid, lcore,
425 (void (*)(void *))func, arg);
427 return _sys_pthread_funcs.f_pthread_create(tid, attr, func, arg);
430 int pthread_detach(pthread_t tid)
433 struct lthread *lt = (struct lthread *)tid;
435 if (lt == lthread_current()) {
441 return _sys_pthread_funcs.f_pthread_detach(tid);
444 int pthread_equal(pthread_t a, pthread_t b)
447 return _sys_pthread_funcs.f_pthread_equal(a, b);
450 void pthread_exit_override(void *v)
456 _sys_pthread_funcs.f_pthread_exit(v);
460 *pthread_getspecific(pthread_key_t key)
463 return lthread_getspecific((unsigned int) key);
464 return _sys_pthread_funcs.f_pthread_getspecific(key);
467 int pthread_getcpuclockid(pthread_t a, clockid_t *b)
470 return _sys_pthread_funcs.f_pthread_getcpuclockid(a, b);
473 int pthread_join(pthread_t tid, void **val)
476 return lthread_join((struct lthread *)tid, val);
477 return _sys_pthread_funcs.f_pthread_join(tid, val);
480 int pthread_key_create(pthread_key_t *keyptr, void (*dtor) (void *))
483 return lthread_key_create((unsigned int *)keyptr, dtor);
484 return _sys_pthread_funcs.f_pthread_key_create(keyptr, dtor);
487 int pthread_key_delete(pthread_key_t key)
490 lthread_key_delete((unsigned int) key);
493 return _sys_pthread_funcs.f_pthread_key_delete(key);
498 pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
501 return lthread_mutex_init(NULL,
502 (struct lthread_mutex **)mutex,
503 (const struct lthread_mutexattr *)attr);
504 return _sys_pthread_funcs.f_pthread_mutex_init(mutex, attr);
507 int pthread_mutex_lock(pthread_mutex_t *mutex)
510 return lthread_mutex_lock(*(struct lthread_mutex **)mutex);
511 return _sys_pthread_funcs.f_pthread_mutex_lock(mutex);
514 int pthread_mutex_trylock(pthread_mutex_t *mutex)
517 return lthread_mutex_trylock(*(struct lthread_mutex **)mutex);
518 return _sys_pthread_funcs.f_pthread_mutex_trylock(mutex);
521 int pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *b)
524 return _sys_pthread_funcs.f_pthread_mutex_timedlock(mutex, b);
527 int pthread_mutex_unlock(pthread_mutex_t *mutex)
530 return lthread_mutex_unlock(*(struct lthread_mutex **)mutex);
531 return _sys_pthread_funcs.f_pthread_mutex_unlock(mutex);
534 int pthread_once(pthread_once_t *a, void (b) (void))
537 return _sys_pthread_funcs.f_pthread_once(a, b);
540 int pthread_rwlock_destroy(pthread_rwlock_t *a)
543 return _sys_pthread_funcs.f_pthread_rwlock_destroy(a);
546 int pthread_rwlock_init(pthread_rwlock_t *a, const pthread_rwlockattr_t *b)
549 return _sys_pthread_funcs.f_pthread_rwlock_init(a, b);
552 int pthread_rwlock_rdlock(pthread_rwlock_t *a)
555 return _sys_pthread_funcs.f_pthread_rwlock_rdlock(a);
558 int pthread_rwlock_timedrdlock(pthread_rwlock_t *a, const struct timespec *b)
561 return _sys_pthread_funcs.f_pthread_rwlock_timedrdlock(a, b);
564 int pthread_rwlock_timedwrlock(pthread_rwlock_t *a, const struct timespec *b)
567 return _sys_pthread_funcs.f_pthread_rwlock_timedwrlock(a, b);
570 int pthread_rwlock_tryrdlock(pthread_rwlock_t *a)
573 return _sys_pthread_funcs.f_pthread_rwlock_tryrdlock(a);
576 int pthread_rwlock_trywrlock(pthread_rwlock_t *a)
579 return _sys_pthread_funcs.f_pthread_rwlock_trywrlock(a);
582 int pthread_rwlock_unlock(pthread_rwlock_t *a)
585 return _sys_pthread_funcs.f_pthread_rwlock_unlock(a);
588 int pthread_rwlock_wrlock(pthread_rwlock_t *a)
591 return _sys_pthread_funcs.f_pthread_rwlock_wrlock(a);
594 #ifdef RTE_EXEC_ENV_LINUXAPP
602 return _sys_pthread_funcs.f_pthread_yield();
611 _sys_pthread_funcs.f_pthread_yield();
615 pthread_t pthread_self(void)
618 return (pthread_t) lthread_current();
619 return _sys_pthread_funcs.f_pthread_self();
622 int pthread_setspecific(pthread_key_t key, const void *data)
625 int rv = lthread_setspecific((unsigned int)key, data);
628 return _sys_pthread_funcs.f_pthread_setspecific(key, data);
631 int pthread_spin_init(pthread_spinlock_t *a, int b)
634 return _sys_pthread_funcs.f_pthread_spin_init(a, b);
637 int pthread_spin_destroy(pthread_spinlock_t *a)
640 return _sys_pthread_funcs.f_pthread_spin_destroy(a);
643 int pthread_spin_lock(pthread_spinlock_t *a)
646 return _sys_pthread_funcs.f_pthread_spin_lock(a);
649 int pthread_spin_trylock(pthread_spinlock_t *a)
652 return _sys_pthread_funcs.f_pthread_spin_trylock(a);
655 int pthread_spin_unlock(pthread_spinlock_t *a)
658 return _sys_pthread_funcs.f_pthread_spin_unlock(a);
661 int pthread_cancel(pthread_t tid)
664 lthread_cancel(*(struct lthread **)tid);
667 return _sys_pthread_funcs.f_pthread_cancel(tid);
670 int pthread_setcancelstate(int a, int *b)
673 return _sys_pthread_funcs.f_pthread_setcancelstate(a, b);
676 int pthread_setcanceltype(int a, int *b)
679 return _sys_pthread_funcs.f_pthread_setcanceltype(a, b);
682 void pthread_testcancel(void)
685 return _sys_pthread_funcs.f_pthread_testcancel();
689 int pthread_getschedparam(pthread_t tid, int *a, struct sched_param *b)
692 return _sys_pthread_funcs.f_pthread_getschedparam(tid, a, b);
695 int pthread_setschedparam(pthread_t a, int b, const struct sched_param *c)
698 return _sys_pthread_funcs.f_pthread_setschedparam(a, b, c);
702 int nanosleep(const struct timespec *req, struct timespec *rem)
705 uint64_t ns = req->tv_sec * 1000000000 + req->tv_nsec;
710 return _sys_pthread_funcs.f_nanosleep(req, rem);
714 pthread_setaffinity_np(pthread_t thread, size_t cpusetsize,
715 const rte_cpuset_t *cpuset)
718 /* we only allow affinity with a single CPU */
719 if (CPU_COUNT(cpuset) != 1)
720 return POSIX_ERRNO(EINVAL);
722 /* we only allow the current thread to sets its own affinity */
723 struct lthread *lt = (struct lthread *)thread;
725 if (lthread_current() != lt)
726 return POSIX_ERRNO(EINVAL);
728 /* determine the CPU being requested */
731 for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
732 if (!CPU_ISSET(i, cpuset))
736 /* check requested core is allowed */
737 if (i == LTHREAD_MAX_LCORES)
738 return POSIX_ERRNO(EINVAL);
740 /* finally we can set affinity to the requested lcore */
741 lthread_set_affinity(i);
744 return _sys_pthread_funcs.f_pthread_setaffinity_np(thread, cpusetsize,