5 * Copyright(c) 2015 Intel Corporation. All rights reserved.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the
18 * * Neither the name of Intel Corporation nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/types.h>
42 #include <sys/queue.h>
50 #include <rte_common.h>
51 #include <rte_lcore.h>
52 #include <rte_per_lcore.h>
53 #include <rte_timer.h>
55 #include "lthread_api.h"
56 #include "lthread_diag_api.h"
57 #include "pthread_shim.h"
60 #define HELLOW_WORLD_MAX_LTHREADS 10
62 #ifndef __GLIBC__ /* sched_getcpu() is glibc-specific */
63 #define sched_getcpu() rte_lcore_id()
66 __thread int print_count;
67 __thread pthread_mutex_t print_lock;
69 __thread pthread_mutex_t exit_lock;
70 __thread pthread_cond_t exit_cond;
73 * A simple thread that demonstrates use of a mutex, a condition
74 * variable, thread local storage, explicit yield, and thread exit.
76 * The thread uses a mutex to protect a shared counter which is incremented
77 * and then it waits on condition variable before exiting.
79 * The thread argument is stored in and retrieved from TLS, using
80 * the pthread key create, get and set specific APIs.
82 * The thread yields while holding the mutex, to provide opportunity
83 * for other threads to contend.
85 * All of the pthread API functions used by this thread are actually
86 * resolved to corresponding lthread functions by the pthread shim
87 * implemented in pthread_shim.c
89 void *helloworld_pthread(void *arg);
90 void *helloworld_pthread(void *arg)
94 /* create a key for TLS */
95 pthread_key_create(&key, NULL);
97 /* store the arg in TLS */
98 pthread_setspecific(key, arg);
100 /* grab lock and increment shared counter */
101 pthread_mutex_lock(&print_lock);
104 /* yield thread to give opportunity for lock contention */
107 /* retrieve arg from TLS */
108 uint64_t thread_no = (uint64_t) pthread_getspecific(key);
110 printf("Hello - lcore = %d count = %d thread_no = %d thread_id = %p\n",
114 (void *)pthread_self());
116 /* release the lock */
117 pthread_mutex_unlock(&print_lock);
120 * wait on condition variable
123 pthread_mutex_lock(&exit_lock);
124 pthread_cond_wait(&exit_cond, &exit_lock);
125 pthread_mutex_unlock(&exit_lock);
128 pthread_exit((void *) thread_no);
133 * This is the initial thread
135 * It demonstrates pthread, mutex and condition variable creation,
136 * broadcast and pthread join APIs.
138 * This initial thread must always start life as an lthread.
140 * This thread creates many more threads then waits a short time
141 * before signalling them to exit using a broadcast.
143 * All of the pthread API functions used by this thread are actually
144 * resolved to corresponding lthread functions by the pthread shim
145 * implemented in pthread_shim.c
147 * After all threads have finished the lthread scheduler is shutdown
148 * and normal pthread operation is restored
150 __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS];
152 static void initial_lthread(void *args);
153 static void initial_lthread(void *args __attribute__((unused)))
155 int lcore = (int) rte_lcore_id();
158 * We can now enable pthread API override
159 * and start to use the pthread APIs
161 pthread_override_set(1);
165 /* initialize mutex for shared counter */
167 pthread_mutex_init(&print_lock, NULL);
169 /* initialize mutex and condition variable controlling thread exit */
170 pthread_mutex_init(&exit_lock, NULL);
171 pthread_cond_init(&exit_cond, NULL);
173 /* spawn a number of threads */
174 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
177 * Not strictly necessary but
178 * for the sake of this example
179 * use an attribute to pass the desired lcore
185 CPU_SET(lcore, &cpuset);
186 pthread_attr_init(&attr);
187 pthread_attr_setaffinity_np(&attr, sizeof(rte_cpuset_t), &cpuset);
189 /* create the thread */
190 pthread_create(&tid[i], &attr, helloworld_pthread, (void *) i);
193 /* wait for 1s to allow threads
194 * to block on the condition variable
195 * N.B. nanosleep() is resolved to lthread_sleep()
198 struct timespec time;
202 nanosleep(&time, NULL);
204 /* wake up all the threads */
205 pthread_cond_broadcast(&exit_cond);
207 /* wait for them to finish */
208 for (i = 0; i < HELLOW_WORLD_MAX_LTHREADS; i++) {
212 pthread_join(tid[i], (void *) &thread_no);
214 printf("error on thread exit\n");
217 pthread_cond_destroy(&exit_cond);
218 pthread_mutex_destroy(&print_lock);
219 pthread_mutex_destroy(&exit_lock);
221 /* shutdown the lthread scheduler */
222 lthread_scheduler_shutdown(rte_lcore_id());
228 /* This thread creates a single initial lthread
229 * and then runs the scheduler
230 * An instance of this thread is created on each thread
234 lthread_scheduler(void *args);
236 lthread_scheduler(void *args __attribute__((unused)))
238 /* create initial thread */
241 lthread_create(<, -1, initial_lthread, (void *) NULL);
243 /* run the lthread scheduler */
246 /* restore genuine pthread operation */
247 pthread_override_set(0);
251 int main(int argc, char **argv)
255 /* basic DPDK initialization is all that is necessary to run lthreads*/
256 int ret = rte_eal_init(argc, argv);
259 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
261 /* enable timer subsystem */
262 rte_timer_subsystem_init();
265 lthread_diagnostic_set_mask(LT_DIAG_ALL);
268 /* create a scheduler on every core in the core mask
269 * and launch an initial lthread that will spawn many more.
273 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
274 if (rte_lcore_is_enabled(lcore_id))
278 /* set the number of schedulers, this forces all schedulers synchronize
279 * before entering their main loop
281 lthread_num_schedulers_set(num_sched);
283 /* launch all threads */
284 rte_eal_mp_remote_launch(lthread_scheduler, (void *)NULL, CALL_MASTER);
286 /* wait for threads to stop */
287 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
288 rte_eal_wait_lcore(lcore_id);