lib/eal/common/eal_common_thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <pthread.h>
   8 #include <sched.h>
   9 #include <assert.h>
  10 #include <string.h>
  11
  12 #include <rte_errno.h>
  13 #include <rte_lcore.h>
  14 #include <rte_log.h>
  15 #include <rte_memory.h>
  16 #include <rte_trace_point.h>
  17
  18 #include "eal_internal_cfg.h"
  19 #include "eal_private.h"
  20 #include "eal_thread.h"
  21 #include "eal_trace.h"
  22
  23 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
  24 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1;
  25 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) =
  26         (unsigned int)SOCKET_ID_ANY;
  27 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset);
  28
  29 unsigned rte_socket_id(void)
  30 {
  31         return RTE_PER_LCORE(_socket_id);
  32 }
  33
  34 static int
  35 eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
  36 {
  37         unsigned cpu = 0;
  38         int socket_id = SOCKET_ID_ANY;
  39         int sid;
  40
  41         if (cpusetp == NULL)
  42                 return SOCKET_ID_ANY;
  43
  44         do {
  45                 if (!CPU_ISSET(cpu, cpusetp))
  46                         continue;
  47
  48                 if (socket_id == SOCKET_ID_ANY)
  49                         socket_id = eal_cpu_socket_id(cpu);
  50
  51                 sid = eal_cpu_socket_id(cpu);
  52                 if (socket_id != sid) {
  53                         socket_id = SOCKET_ID_ANY;
  54                         break;
  55                 }
  56
  57         } while (++cpu < CPU_SETSIZE);
  58
  59         return socket_id;
  60 }
  61
  62 static void
  63 thread_update_affinity(rte_cpuset_t *cpusetp)
  64 {
  65         unsigned int lcore_id = rte_lcore_id();
  66
  67         /* store socket_id in TLS for quick access */
  68         RTE_PER_LCORE(_socket_id) =
  69                 eal_cpuset_socket_id(cpusetp);
  70
  71         /* store cpuset in TLS for quick access */
  72         memmove(&RTE_PER_LCORE(_cpuset), cpusetp,
  73                 sizeof(rte_cpuset_t));
  74
  75         if (lcore_id != (unsigned)LCORE_ID_ANY) {
  76                 /* EAL thread will update lcore_config */
  77                 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id);
  78                 memmove(&lcore_config[lcore_id].cpuset, cpusetp,
  79                         sizeof(rte_cpuset_t));
  80         }
  81 }
  82
  83 int
  84 rte_thread_set_affinity(rte_cpuset_t *cpusetp)
  85 {
  86         if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
  87                         cpusetp) != 0) {
  88                 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n");
  89                 return -1;
  90         }
  91
  92         thread_update_affinity(cpusetp);
  93         return 0;
  94 }
  95
  96 void
  97 rte_thread_get_affinity(rte_cpuset_t *cpusetp)
  98 {
  99         assert(cpusetp);
 100         memmove(cpusetp, &RTE_PER_LCORE(_cpuset),
 101                 sizeof(rte_cpuset_t));
 102 }
 103
 104 int
 105 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size)
 106 {
 107         unsigned cpu;
 108         int ret;
 109         unsigned int out = 0;
 110
 111         for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
 112                 if (!CPU_ISSET(cpu, cpuset))
 113                         continue;
 114
 115                 ret = snprintf(str + out,
 116                                size - out, "%u,", cpu);
 117                 if (ret < 0 || (unsigned)ret >= size - out) {
 118                         /* string will be truncated */
 119                         ret = -1;
 120                         goto exit;
 121                 }
 122
 123                 out += ret;
 124         }
 125
 126         ret = 0;
 127 exit:
 128         /* remove the last separator */
 129         if (out > 0)
 130                 str[out - 1] = '\0';
 131
 132         return ret;
 133 }
 134
 135 int
 136 eal_thread_dump_current_affinity(char *str, unsigned int size)
 137 {
 138         rte_cpuset_t cpuset;
 139
 140         rte_thread_get_affinity(&cpuset);
 141         return eal_thread_dump_affinity(&cpuset, str, size);
 142 }
 143
 144 void
 145 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset)
 146 {
 147         /* set the lcore ID in per-lcore memory area */
 148         RTE_PER_LCORE(_lcore_id) = lcore_id;
 149
 150         /* acquire system unique id */
 151         rte_gettid();
 152
 153         thread_update_affinity(cpuset);
 154
 155         __rte_trace_mem_per_thread_alloc();
 156 }
 157
 158 void
 159 __rte_thread_uninit(void)
 160 {
 161         trace_mem_per_thread_free();
 162
 163         RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
 164 }
 165
 166 enum __rte_ctrl_thread_status {
 167         CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */
 168         CTRL_THREAD_RUNNING, /* Control thread is running successfully */
 169         CTRL_THREAD_ERROR /* Control thread encountered an error */
 170 };
 171
 172 struct rte_thread_ctrl_params {
 173         void *(*start_routine)(void *);
 174         void *arg;
 175         int ret;
 176         /* Control thread status.
 177          * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
 178          */
 179         enum __rte_ctrl_thread_status ctrl_thread_status;
 180 };
 181
 182 static void *ctrl_thread_init(void *arg)
 183 {
 184         struct internal_config *internal_conf =
 185                 eal_get_internal_configuration();
 186         rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
 187         struct rte_thread_ctrl_params *params = arg;
 188         void *(*start_routine)(void *) = params->start_routine;
 189         void *routine_arg = params->arg;
 190
 191         __rte_thread_init(rte_lcore_id(), cpuset);
 192         params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset),
 193                 cpuset);
 194         if (params->ret != 0) {
 195                 __atomic_store_n(&params->ctrl_thread_status,
 196                         CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
 197                 return NULL;
 198         }
 199
 200         __atomic_store_n(&params->ctrl_thread_status,
 201                 CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
 202
 203         return start_routine(routine_arg);
 204 }
 205
 206 int
 207 rte_ctrl_thread_create(pthread_t *thread, const char *name,
 208                 const pthread_attr_t *attr,
 209                 void *(*start_routine)(void *), void *arg)
 210 {
 211         struct rte_thread_ctrl_params *params;
 212         enum __rte_ctrl_thread_status ctrl_thread_status;
 213         int ret;
 214
 215         params = malloc(sizeof(*params));
 216         if (!params)
 217                 return -ENOMEM;
 218
 219         params->start_routine = start_routine;
 220         params->arg = arg;
 221         params->ret = 0;
 222         params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;
 223
 224         ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
 225         if (ret != 0) {
 226                 free(params);
 227                 return -ret;
 228         }
 229
 230         if (name != NULL) {
 231                 ret = rte_thread_setname(*thread, name);
 232                 if (ret < 0)
 233                         RTE_LOG(DEBUG, EAL,
 234                                 "Cannot set name for ctrl thread\n");
 235         }
 236
 237         /* Wait for the control thread to initialize successfully */
 238         while ((ctrl_thread_status =
 239                         __atomic_load_n(&params->ctrl_thread_status,
 240                         __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
 241                 /* Yield the CPU. Using sched_yield call requires maintaining
 242                  * another implementation for Windows as sched_yield is not
 243                  * supported on Windows.
 244                  */
 245                 rte_delay_us_sleep(1);
 246         }
 247
 248         /* Check if the control thread encountered an error */
 249         if (ctrl_thread_status == CTRL_THREAD_ERROR) {
 250                 /* ctrl thread is exiting */
 251                 pthread_join(*thread, NULL);
 252         }
 253
 254         ret = params->ret;
 255         free(params);
 256
 257         return -ret;
 258 }
 259
 260 int
 261 rte_thread_register(void)
 262 {
 263         unsigned int lcore_id;
 264         rte_cpuset_t cpuset;
 265
 266         /* EAL init flushes all lcores, we can't register before. */
 267         if (eal_get_internal_configuration()->init_complete != 1) {
 268                 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__);
 269                 rte_errno = EINVAL;
 270                 return -1;
 271         }
 272         if (!rte_mp_disable()) {
 273                 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n");
 274                 rte_errno = EINVAL;
 275                 return -1;
 276         }
 277         if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset),
 278                         &cpuset) != 0)
 279                 CPU_ZERO(&cpuset);
 280         lcore_id = eal_lcore_non_eal_allocate();
 281         if (lcore_id >= RTE_MAX_LCORE)
 282                 lcore_id = LCORE_ID_ANY;
 283         __rte_thread_init(lcore_id, &cpuset);
 284         if (lcore_id == LCORE_ID_ANY) {
 285                 rte_errno = ENOMEM;
 286                 return -1;
 287         }
 288         RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n",
 289                 lcore_id);
 290         return 0;
 291 }
 292
 293 void
 294 rte_thread_unregister(void)
 295 {
 296         unsigned int lcore_id = rte_lcore_id();
 297
 298         if (lcore_id != LCORE_ID_ANY)
 299                 eal_lcore_non_eal_release(lcore_id);
 300         __rte_thread_uninit();
 301         if (lcore_id != LCORE_ID_ANY)
 302                 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n",
 303                         lcore_id);
 304 }