1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015 Intel Corporation
14 #include "lthread_api.h"
15 #include "pthread_shim.h"
17 #define RTE_LOGTYPE_PTHREAD_SHIM RTE_LOGTYPE_USER3
19 #define POSIX_ERRNO(x) (x)
21 /* some releases of FreeBSD 10, e.g. 10.0, don't have CPU_COUNT macro */
23 #define CPU_COUNT(x) __cpu_count(x)
25 static inline unsigned int
26 __cpu_count(const rte_cpuset_t *cpuset)
28 unsigned int i, count = 0;
29 for (i = 0; i < RTE_MAX_LCORE; i++)
30 if (CPU_ISSET(i, cpuset))
37 * this flag determines at run time if we override pthread
38 * calls and map then to equivalent lthread calls
39 * or of we call the standard pthread function
41 static __thread int override;
45 * this structures contains function pointers that will be
46 * initialised to the loaded address of the real
47 * pthread library API functions
49 struct pthread_lib_funcs {
50 int (*f_pthread_barrier_destroy)
51 (pthread_barrier_t *);
52 int (*f_pthread_barrier_init)
53 (pthread_barrier_t *, const pthread_barrierattr_t *, unsigned);
54 int (*f_pthread_barrier_wait)
55 (pthread_barrier_t *);
56 int (*f_pthread_cond_broadcast)
58 int (*f_pthread_cond_destroy)
60 int (*f_pthread_cond_init)
61 (pthread_cond_t *, const pthread_condattr_t *);
62 int (*f_pthread_cond_signal)
64 int (*f_pthread_cond_timedwait)
65 (pthread_cond_t *, pthread_mutex_t *, const struct timespec *);
66 int (*f_pthread_cond_wait)
67 (pthread_cond_t *, pthread_mutex_t *);
68 int (*f_pthread_create)
69 (pthread_t *, const pthread_attr_t *, void *(*)(void *), void *);
70 int (*f_pthread_detach)
72 int (*f_pthread_equal)
73 (pthread_t, pthread_t);
74 void (*f_pthread_exit)
76 void * (*f_pthread_getspecific)
78 int (*f_pthread_getcpuclockid)
79 (pthread_t, clockid_t *);
82 int (*f_pthread_key_create)
83 (pthread_key_t *, void (*) (void *));
84 int (*f_pthread_key_delete)
86 int (*f_pthread_mutex_destroy)
87 (pthread_mutex_t *__mutex);
88 int (*f_pthread_mutex_init)
89 (pthread_mutex_t *__mutex, const pthread_mutexattr_t *);
90 int (*f_pthread_mutex_lock)
91 (pthread_mutex_t *__mutex);
92 int (*f_pthread_mutex_trylock)
93 (pthread_mutex_t *__mutex);
94 int (*f_pthread_mutex_timedlock)
95 (pthread_mutex_t *__mutex, const struct timespec *);
96 int (*f_pthread_mutex_unlock)
97 (pthread_mutex_t *__mutex);
99 (pthread_once_t *, void (*) (void));
100 int (*f_pthread_rwlock_destroy)
101 (pthread_rwlock_t *__rwlock);
102 int (*f_pthread_rwlock_init)
103 (pthread_rwlock_t *__rwlock, const pthread_rwlockattr_t *);
104 int (*f_pthread_rwlock_rdlock)
105 (pthread_rwlock_t *__rwlock);
106 int (*f_pthread_rwlock_timedrdlock)
107 (pthread_rwlock_t *__rwlock, const struct timespec *);
108 int (*f_pthread_rwlock_timedwrlock)
109 (pthread_rwlock_t *__rwlock, const struct timespec *);
110 int (*f_pthread_rwlock_tryrdlock)
111 (pthread_rwlock_t *__rwlock);
112 int (*f_pthread_rwlock_trywrlock)
113 (pthread_rwlock_t *__rwlock);
114 int (*f_pthread_rwlock_unlock)
115 (pthread_rwlock_t *__rwlock);
116 int (*f_pthread_rwlock_wrlock)
117 (pthread_rwlock_t *__rwlock);
118 pthread_t (*f_pthread_self)
120 int (*f_pthread_setspecific)
121 (pthread_key_t, const void *);
122 int (*f_pthread_spin_init)
123 (pthread_spinlock_t *__spin, int);
124 int (*f_pthread_spin_destroy)
125 (pthread_spinlock_t *__spin);
126 int (*f_pthread_spin_lock)
127 (pthread_spinlock_t *__spin);
128 int (*f_pthread_spin_trylock)
129 (pthread_spinlock_t *__spin);
130 int (*f_pthread_spin_unlock)
131 (pthread_spinlock_t *__spin);
132 int (*f_pthread_cancel)
134 int (*f_pthread_setcancelstate)
136 int (*f_pthread_setcanceltype)
138 void (*f_pthread_testcancel)
140 int (*f_pthread_getschedparam)
141 (pthread_t pthread, int *, struct sched_param *);
142 int (*f_pthread_setschedparam)
143 (pthread_t, int, const struct sched_param *);
144 int (*f_pthread_yield)
146 int (*f_pthread_setaffinity_np)
147 (pthread_t thread, size_t cpusetsize, const rte_cpuset_t *cpuset);
149 (const struct timespec *req, struct timespec *rem);
150 } _sys_pthread_funcs = {
151 .f_pthread_barrier_destroy = NULL,
156 * this macro obtains the loaded address of a library function
159 static void *__libc_dl_handle = RTLD_NEXT;
161 #define get_addr_of_loaded_symbol(name) do { \
163 _sys_pthread_funcs.f_##name = dlsym(__libc_dl_handle, (#name)); \
164 error_str = dlerror(); \
165 if (error_str != NULL) { \
166 fprintf(stderr, "%s\n", error_str); \
172 * The constructor function initialises the
173 * function pointers for pthread library functions
175 RTE_INIT(pthread_intercept_ctor)
179 * Get the original functions
181 get_addr_of_loaded_symbol(pthread_barrier_destroy);
182 get_addr_of_loaded_symbol(pthread_barrier_init);
183 get_addr_of_loaded_symbol(pthread_barrier_wait);
184 get_addr_of_loaded_symbol(pthread_cond_broadcast);
185 get_addr_of_loaded_symbol(pthread_cond_destroy);
186 get_addr_of_loaded_symbol(pthread_cond_init);
187 get_addr_of_loaded_symbol(pthread_cond_signal);
188 get_addr_of_loaded_symbol(pthread_cond_timedwait);
189 get_addr_of_loaded_symbol(pthread_cond_wait);
190 get_addr_of_loaded_symbol(pthread_create);
191 get_addr_of_loaded_symbol(pthread_detach);
192 get_addr_of_loaded_symbol(pthread_equal);
193 get_addr_of_loaded_symbol(pthread_exit);
194 get_addr_of_loaded_symbol(pthread_getspecific);
195 get_addr_of_loaded_symbol(pthread_getcpuclockid);
196 get_addr_of_loaded_symbol(pthread_join);
197 get_addr_of_loaded_symbol(pthread_key_create);
198 get_addr_of_loaded_symbol(pthread_key_delete);
199 get_addr_of_loaded_symbol(pthread_mutex_destroy);
200 get_addr_of_loaded_symbol(pthread_mutex_init);
201 get_addr_of_loaded_symbol(pthread_mutex_lock);
202 get_addr_of_loaded_symbol(pthread_mutex_trylock);
203 get_addr_of_loaded_symbol(pthread_mutex_timedlock);
204 get_addr_of_loaded_symbol(pthread_mutex_unlock);
205 get_addr_of_loaded_symbol(pthread_once);
206 get_addr_of_loaded_symbol(pthread_rwlock_destroy);
207 get_addr_of_loaded_symbol(pthread_rwlock_init);
208 get_addr_of_loaded_symbol(pthread_rwlock_rdlock);
209 get_addr_of_loaded_symbol(pthread_rwlock_timedrdlock);
210 get_addr_of_loaded_symbol(pthread_rwlock_timedwrlock);
211 get_addr_of_loaded_symbol(pthread_rwlock_tryrdlock);
212 get_addr_of_loaded_symbol(pthread_rwlock_trywrlock);
213 get_addr_of_loaded_symbol(pthread_rwlock_unlock);
214 get_addr_of_loaded_symbol(pthread_rwlock_wrlock);
215 get_addr_of_loaded_symbol(pthread_self);
216 get_addr_of_loaded_symbol(pthread_setspecific);
217 get_addr_of_loaded_symbol(pthread_spin_init);
218 get_addr_of_loaded_symbol(pthread_spin_destroy);
219 get_addr_of_loaded_symbol(pthread_spin_lock);
220 get_addr_of_loaded_symbol(pthread_spin_trylock);
221 get_addr_of_loaded_symbol(pthread_spin_unlock);
222 get_addr_of_loaded_symbol(pthread_cancel);
223 get_addr_of_loaded_symbol(pthread_setcancelstate);
224 get_addr_of_loaded_symbol(pthread_setcanceltype);
225 get_addr_of_loaded_symbol(pthread_testcancel);
226 get_addr_of_loaded_symbol(pthread_getschedparam);
227 get_addr_of_loaded_symbol(pthread_setschedparam);
228 get_addr_of_loaded_symbol(pthread_yield);
229 get_addr_of_loaded_symbol(pthread_setaffinity_np);
230 get_addr_of_loaded_symbol(nanosleep);
235 * Enable/Disable pthread override
240 void pthread_override_set(int state)
247 * Return pthread override state
252 int pthread_override_get(void)
258 * This macro is used to catch and log
259 * invocation of stubs for unimplemented pthread
262 #define NOT_IMPLEMENTED do { \
266 "WARNING %s NOT IMPLEMENTED\n", \
272 * pthread API override functions follow
273 * Note in this example code only a subset of functions are
276 * The stub functions provided will issue a warning log
277 * message if an unimplemented function is invoked
281 int pthread_barrier_destroy(pthread_barrier_t *a)
284 return _sys_pthread_funcs.f_pthread_barrier_destroy(a);
288 pthread_barrier_init(pthread_barrier_t *a,
289 const pthread_barrierattr_t *b, unsigned c)
292 return _sys_pthread_funcs.f_pthread_barrier_init(a, b, c);
295 int pthread_barrier_wait(pthread_barrier_t *a)
298 return _sys_pthread_funcs.f_pthread_barrier_wait(a);
301 int pthread_cond_broadcast(pthread_cond_t *cond)
305 lthread_cond_broadcast(*(struct lthread_cond **)cond);
308 return _sys_pthread_funcs.f_pthread_cond_broadcast(cond);
311 int pthread_mutex_destroy(pthread_mutex_t *mutex)
314 return lthread_mutex_destroy(*(struct lthread_mutex **)mutex);
315 return _sys_pthread_funcs.f_pthread_mutex_destroy(mutex);
318 int pthread_cond_destroy(pthread_cond_t *cond)
321 return lthread_cond_destroy(*(struct lthread_cond **)cond);
322 return _sys_pthread_funcs.f_pthread_cond_destroy(cond);
325 int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
328 return lthread_cond_init(NULL,
329 (struct lthread_cond **)cond,
330 (const struct lthread_condattr *) attr);
331 return _sys_pthread_funcs.f_pthread_cond_init(cond, attr);
334 int pthread_cond_signal(pthread_cond_t *cond)
337 lthread_cond_signal(*(struct lthread_cond **)cond);
340 return _sys_pthread_funcs.f_pthread_cond_signal(cond);
344 pthread_cond_timedwait(pthread_cond_t *__rte_restrict cond,
345 pthread_mutex_t *__rte_restrict mutex,
346 const struct timespec *__rte_restrict time)
349 return _sys_pthread_funcs.f_pthread_cond_timedwait(cond, mutex, time);
352 int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
355 pthread_mutex_unlock(mutex);
356 int rv = lthread_cond_wait(*(struct lthread_cond **)cond, 0);
358 pthread_mutex_lock(mutex);
361 return _sys_pthread_funcs.f_pthread_cond_wait(cond, mutex);
365 pthread_create(pthread_t *__rte_restrict tid,
366 const pthread_attr_t *__rte_restrict attr,
368 void *__rte_restrict arg)
374 /* determine CPU being requested */
378 pthread_attr_getaffinity_np(attr,
379 sizeof(rte_cpuset_t),
382 if (CPU_COUNT(&cpuset) != 1)
383 return POSIX_ERRNO(EINVAL);
385 for (lcore = 0; lcore < LTHREAD_MAX_LCORES; lcore++) {
386 if (!CPU_ISSET(lcore, &cpuset))
391 return lthread_create((struct lthread **)tid, lcore,
394 return _sys_pthread_funcs.f_pthread_create(tid, attr, func, arg);
397 int pthread_detach(pthread_t tid)
400 struct lthread *lt = (struct lthread *)tid;
402 if (lt == lthread_current()) {
408 return _sys_pthread_funcs.f_pthread_detach(tid);
411 int pthread_equal(pthread_t a, pthread_t b)
414 return _sys_pthread_funcs.f_pthread_equal(a, b);
417 void pthread_exit_override(void *v)
423 _sys_pthread_funcs.f_pthread_exit(v);
427 *pthread_getspecific(pthread_key_t key)
430 return lthread_getspecific((unsigned int) key);
431 return _sys_pthread_funcs.f_pthread_getspecific(key);
434 int pthread_getcpuclockid(pthread_t a, clockid_t *b)
437 return _sys_pthread_funcs.f_pthread_getcpuclockid(a, b);
440 int pthread_join(pthread_t tid, void **val)
443 return lthread_join((struct lthread *)tid, val);
444 return _sys_pthread_funcs.f_pthread_join(tid, val);
447 int pthread_key_create(pthread_key_t *keyptr, void (*dtor) (void *))
450 return lthread_key_create((unsigned int *)keyptr, dtor);
451 return _sys_pthread_funcs.f_pthread_key_create(keyptr, dtor);
454 int pthread_key_delete(pthread_key_t key)
457 lthread_key_delete((unsigned int) key);
460 return _sys_pthread_funcs.f_pthread_key_delete(key);
465 pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
468 return lthread_mutex_init(NULL,
469 (struct lthread_mutex **)mutex,
470 (const struct lthread_mutexattr *)attr);
471 return _sys_pthread_funcs.f_pthread_mutex_init(mutex, attr);
474 int pthread_mutex_lock(pthread_mutex_t *mutex)
477 return lthread_mutex_lock(*(struct lthread_mutex **)mutex);
478 return _sys_pthread_funcs.f_pthread_mutex_lock(mutex);
481 int pthread_mutex_trylock(pthread_mutex_t *mutex)
484 return lthread_mutex_trylock(*(struct lthread_mutex **)mutex);
485 return _sys_pthread_funcs.f_pthread_mutex_trylock(mutex);
488 int pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *b)
491 return _sys_pthread_funcs.f_pthread_mutex_timedlock(mutex, b);
494 int pthread_mutex_unlock(pthread_mutex_t *mutex)
497 return lthread_mutex_unlock(*(struct lthread_mutex **)mutex);
498 return _sys_pthread_funcs.f_pthread_mutex_unlock(mutex);
501 int pthread_once(pthread_once_t *a, void (b) (void))
504 return _sys_pthread_funcs.f_pthread_once(a, b);
507 int pthread_rwlock_destroy(pthread_rwlock_t *a)
510 return _sys_pthread_funcs.f_pthread_rwlock_destroy(a);
513 int pthread_rwlock_init(pthread_rwlock_t *a, const pthread_rwlockattr_t *b)
516 return _sys_pthread_funcs.f_pthread_rwlock_init(a, b);
519 int pthread_rwlock_rdlock(pthread_rwlock_t *a)
522 return _sys_pthread_funcs.f_pthread_rwlock_rdlock(a);
525 int pthread_rwlock_timedrdlock(pthread_rwlock_t *a, const struct timespec *b)
528 return _sys_pthread_funcs.f_pthread_rwlock_timedrdlock(a, b);
531 int pthread_rwlock_timedwrlock(pthread_rwlock_t *a, const struct timespec *b)
534 return _sys_pthread_funcs.f_pthread_rwlock_timedwrlock(a, b);
537 int pthread_rwlock_tryrdlock(pthread_rwlock_t *a)
540 return _sys_pthread_funcs.f_pthread_rwlock_tryrdlock(a);
543 int pthread_rwlock_trywrlock(pthread_rwlock_t *a)
546 return _sys_pthread_funcs.f_pthread_rwlock_trywrlock(a);
549 int pthread_rwlock_unlock(pthread_rwlock_t *a)
552 return _sys_pthread_funcs.f_pthread_rwlock_unlock(a);
555 int pthread_rwlock_wrlock(pthread_rwlock_t *a)
558 return _sys_pthread_funcs.f_pthread_rwlock_wrlock(a);
561 #ifdef RTE_EXEC_ENV_LINUX
569 return _sys_pthread_funcs.f_pthread_yield();
578 _sys_pthread_funcs.f_pthread_yield();
582 pthread_t pthread_self(void)
585 return (pthread_t) lthread_current();
586 return _sys_pthread_funcs.f_pthread_self();
589 int pthread_setspecific(pthread_key_t key, const void *data)
592 int rv = lthread_setspecific((unsigned int)key, data);
595 return _sys_pthread_funcs.f_pthread_setspecific(key, data);
598 int pthread_spin_init(pthread_spinlock_t *a, int b)
601 return _sys_pthread_funcs.f_pthread_spin_init(a, b);
604 int pthread_spin_destroy(pthread_spinlock_t *a)
607 return _sys_pthread_funcs.f_pthread_spin_destroy(a);
610 int pthread_spin_lock(pthread_spinlock_t *a)
613 return _sys_pthread_funcs.f_pthread_spin_lock(a);
616 int pthread_spin_trylock(pthread_spinlock_t *a)
619 return _sys_pthread_funcs.f_pthread_spin_trylock(a);
622 int pthread_spin_unlock(pthread_spinlock_t *a)
625 return _sys_pthread_funcs.f_pthread_spin_unlock(a);
628 int pthread_cancel(pthread_t tid)
631 lthread_cancel(*(struct lthread **)tid);
634 return _sys_pthread_funcs.f_pthread_cancel(tid);
637 int pthread_setcancelstate(int a, int *b)
640 return _sys_pthread_funcs.f_pthread_setcancelstate(a, b);
643 int pthread_setcanceltype(int a, int *b)
646 return _sys_pthread_funcs.f_pthread_setcanceltype(a, b);
649 void pthread_testcancel(void)
652 return _sys_pthread_funcs.f_pthread_testcancel();
656 int pthread_getschedparam(pthread_t tid, int *a, struct sched_param *b)
659 return _sys_pthread_funcs.f_pthread_getschedparam(tid, a, b);
662 int pthread_setschedparam(pthread_t a, int b, const struct sched_param *c)
665 return _sys_pthread_funcs.f_pthread_setschedparam(a, b, c);
669 int nanosleep(const struct timespec *req, struct timespec *rem)
672 uint64_t ns = req->tv_sec * 1000000000 + req->tv_nsec;
677 return _sys_pthread_funcs.f_nanosleep(req, rem);
681 pthread_setaffinity_np(pthread_t thread, size_t cpusetsize,
682 const rte_cpuset_t *cpuset)
685 /* we only allow affinity with a single CPU */
686 if (CPU_COUNT(cpuset) != 1)
687 return POSIX_ERRNO(EINVAL);
689 /* we only allow the current thread to sets its own affinity */
690 struct lthread *lt = (struct lthread *)thread;
692 if (lthread_current() != lt)
693 return POSIX_ERRNO(EINVAL);
695 /* determine the CPU being requested */
698 for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
699 if (!CPU_ISSET(i, cpuset))
703 /* check requested core is allowed */
704 if (i == LTHREAD_MAX_LCORES)
705 return POSIX_ERRNO(EINVAL);
707 /* finally we can set affinity to the requested lcore */
708 lthread_set_affinity(i);
711 return _sys_pthread_funcs.f_pthread_setaffinity_np(thread, cpusetsize,