lib/eal/common/eal_common_thread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9 #include <pthread.h>
  10 #include <signal.h>
  11 #include <sched.h>
  12 #include <assert.h>
  13 #include <string.h>
  14
  15 #include <rte_errno.h>
  16 #include <rte_lcore.h>
  17 #include <rte_log.h>
  18 #include <rte_memory.h>
  19 #include <rte_trace_point.h>
  20
  21 #include "eal_internal_cfg.h"
  22 #include "eal_private.h"
  23 #include "eal_thread.h"
  24 #include "eal_trace.h"
  25
  26 RTE_DEFINE_PER_LCORE(unsigned int, _lcore_id) = LCORE_ID_ANY;
  27 RTE_DEFINE_PER_LCORE(int, _thread_id) = -1;
  28 static RTE_DEFINE_PER_LCORE(unsigned int, _socket_id) =
  29         (unsigned int)SOCKET_ID_ANY;
  30 static RTE_DEFINE_PER_LCORE(rte_cpuset_t, _cpuset);
  31
  32 unsigned rte_socket_id(void)
  33 {
  34         return RTE_PER_LCORE(_socket_id);
  35 }
  36
  37 static int
  38 eal_cpuset_socket_id(rte_cpuset_t *cpusetp)
  39 {
  40         unsigned cpu = 0;
  41         int socket_id = SOCKET_ID_ANY;
  42         int sid;
  43
  44         if (cpusetp == NULL)
  45                 return SOCKET_ID_ANY;
  46
  47         do {
  48                 if (!CPU_ISSET(cpu, cpusetp))
  49                         continue;
  50
  51                 if (socket_id == SOCKET_ID_ANY)
  52                         socket_id = eal_cpu_socket_id(cpu);
  53
  54                 sid = eal_cpu_socket_id(cpu);
  55                 if (socket_id != sid) {
  56                         socket_id = SOCKET_ID_ANY;
  57                         break;
  58                 }
  59
  60         } while (++cpu < CPU_SETSIZE);
  61
  62         return socket_id;
  63 }
  64
  65 static void
  66 thread_update_affinity(rte_cpuset_t *cpusetp)
  67 {
  68         unsigned int lcore_id = rte_lcore_id();
  69
  70         /* store socket_id in TLS for quick access */
  71         RTE_PER_LCORE(_socket_id) =
  72                 eal_cpuset_socket_id(cpusetp);
  73
  74         /* store cpuset in TLS for quick access */
  75         memmove(&RTE_PER_LCORE(_cpuset), cpusetp,
  76                 sizeof(rte_cpuset_t));
  77
  78         if (lcore_id != (unsigned)LCORE_ID_ANY) {
  79                 /* EAL thread will update lcore_config */
  80                 lcore_config[lcore_id].socket_id = RTE_PER_LCORE(_socket_id);
  81                 memmove(&lcore_config[lcore_id].cpuset, cpusetp,
  82                         sizeof(rte_cpuset_t));
  83         }
  84 }
  85
  86 int
  87 rte_thread_set_affinity(rte_cpuset_t *cpusetp)
  88 {
  89         if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
  90                         cpusetp) != 0) {
  91                 RTE_LOG(ERR, EAL, "pthread_setaffinity_np failed\n");
  92                 return -1;
  93         }
  94
  95         thread_update_affinity(cpusetp);
  96         return 0;
  97 }
  98
  99 void
 100 rte_thread_get_affinity(rte_cpuset_t *cpusetp)
 101 {
 102         assert(cpusetp);
 103         memmove(cpusetp, &RTE_PER_LCORE(_cpuset),
 104                 sizeof(rte_cpuset_t));
 105 }
 106
 107 int
 108 eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size)
 109 {
 110         unsigned cpu;
 111         int ret;
 112         unsigned int out = 0;
 113
 114         for (cpu = 0; cpu < CPU_SETSIZE; cpu++) {
 115                 if (!CPU_ISSET(cpu, cpuset))
 116                         continue;
 117
 118                 ret = snprintf(str + out,
 119                                size - out, "%u,", cpu);
 120                 if (ret < 0 || (unsigned)ret >= size - out) {
 121                         /* string will be truncated */
 122                         ret = -1;
 123                         goto exit;
 124                 }
 125
 126                 out += ret;
 127         }
 128
 129         ret = 0;
 130 exit:
 131         /* remove the last separator */
 132         if (out > 0)
 133                 str[out - 1] = '\0';
 134
 135         return ret;
 136 }
 137
 138 int
 139 eal_thread_dump_current_affinity(char *str, unsigned int size)
 140 {
 141         rte_cpuset_t cpuset;
 142
 143         rte_thread_get_affinity(&cpuset);
 144         return eal_thread_dump_affinity(&cpuset, str, size);
 145 }
 146
 147 void
 148 __rte_thread_init(unsigned int lcore_id, rte_cpuset_t *cpuset)
 149 {
 150         /* set the lcore ID in per-lcore memory area */
 151         RTE_PER_LCORE(_lcore_id) = lcore_id;
 152
 153         /* acquire system unique id */
 154         rte_gettid();
 155
 156         thread_update_affinity(cpuset);
 157
 158         __rte_trace_mem_per_thread_alloc();
 159 }
 160
 161 void
 162 __rte_thread_uninit(void)
 163 {
 164         trace_mem_per_thread_free();
 165
 166         RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
 167 }
 168
 169 enum __rte_ctrl_thread_status {
 170         CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */
 171         CTRL_THREAD_RUNNING, /* Control thread is running successfully */
 172         CTRL_THREAD_ERROR /* Control thread encountered an error */
 173 };
 174
 175 struct rte_thread_ctrl_params {
 176         void *(*start_routine)(void *);
 177         void *arg;
 178         int ret;
 179         /* Control thread status.
 180          * If the status is CTRL_THREAD_ERROR, 'ret' has the error code.
 181          */
 182         enum __rte_ctrl_thread_status ctrl_thread_status;
 183 };
 184
 185 static void *ctrl_thread_init(void *arg)
 186 {
 187         struct internal_config *internal_conf =
 188                 eal_get_internal_configuration();
 189         rte_cpuset_t *cpuset = &internal_conf->ctrl_cpuset;
 190         struct rte_thread_ctrl_params *params = arg;
 191         void *(*start_routine)(void *) = params->start_routine;
 192         void *routine_arg = params->arg;
 193
 194         __rte_thread_init(rte_lcore_id(), cpuset);
 195         params->ret = pthread_setaffinity_np(pthread_self(), sizeof(*cpuset),
 196                 cpuset);
 197         if (params->ret != 0) {
 198                 __atomic_store_n(&params->ctrl_thread_status,
 199                         CTRL_THREAD_ERROR, __ATOMIC_RELEASE);
 200                 return NULL;
 201         }
 202
 203         __atomic_store_n(&params->ctrl_thread_status,
 204                 CTRL_THREAD_RUNNING, __ATOMIC_RELEASE);
 205
 206         return start_routine(routine_arg);
 207 }
 208
 209 int
 210 rte_ctrl_thread_create(pthread_t *thread, const char *name,
 211                 const pthread_attr_t *attr,
 212                 void *(*start_routine)(void *), void *arg)
 213 {
 214         struct rte_thread_ctrl_params *params;
 215         enum __rte_ctrl_thread_status ctrl_thread_status;
 216         int ret;
 217
 218         params = malloc(sizeof(*params));
 219         if (!params)
 220                 return -ENOMEM;
 221
 222         params->start_routine = start_routine;
 223         params->arg = arg;
 224         params->ret = 0;
 225         params->ctrl_thread_status = CTRL_THREAD_LAUNCHING;
 226
 227         ret = pthread_create(thread, attr, ctrl_thread_init, (void *)params);
 228         if (ret != 0) {
 229                 free(params);
 230                 return -ret;
 231         }
 232
 233         if (name != NULL) {
 234                 ret = rte_thread_setname(*thread, name);
 235                 if (ret < 0)
 236                         RTE_LOG(DEBUG, EAL,
 237                                 "Cannot set name for ctrl thread\n");
 238         }
 239
 240         /* Wait for the control thread to initialize successfully */
 241         while ((ctrl_thread_status =
 242                         __atomic_load_n(&params->ctrl_thread_status,
 243                         __ATOMIC_ACQUIRE)) == CTRL_THREAD_LAUNCHING) {
 244                 /* Yield the CPU. Using sched_yield call requires maintaining
 245                  * another implementation for Windows as sched_yield is not
 246                  * supported on Windows.
 247                  */
 248                 rte_delay_us_sleep(1);
 249         }
 250
 251         /* Check if the control thread encountered an error */
 252         if (ctrl_thread_status == CTRL_THREAD_ERROR) {
 253                 /* ctrl thread is exiting */
 254                 pthread_join(*thread, NULL);
 255         }
 256
 257         ret = params->ret;
 258         free(params);
 259
 260         return -ret;
 261 }
 262
 263 int
 264 rte_thread_register(void)
 265 {
 266         unsigned int lcore_id;
 267         rte_cpuset_t cpuset;
 268
 269         /* EAL init flushes all lcores, we can't register before. */
 270         if (eal_get_internal_configuration()->init_complete != 1) {
 271                 RTE_LOG(DEBUG, EAL, "Called %s before EAL init.\n", __func__);
 272                 rte_errno = EINVAL;
 273                 return -1;
 274         }
 275         if (!rte_mp_disable()) {
 276                 RTE_LOG(ERR, EAL, "Multiprocess in use, registering non-EAL threads is not supported.\n");
 277                 rte_errno = EINVAL;
 278                 return -1;
 279         }
 280         if (pthread_getaffinity_np(pthread_self(), sizeof(cpuset),
 281                         &cpuset) != 0)
 282                 CPU_ZERO(&cpuset);
 283         lcore_id = eal_lcore_non_eal_allocate();
 284         if (lcore_id >= RTE_MAX_LCORE)
 285                 lcore_id = LCORE_ID_ANY;
 286         __rte_thread_init(lcore_id, &cpuset);
 287         if (lcore_id == LCORE_ID_ANY) {
 288                 rte_errno = ENOMEM;
 289                 return -1;
 290         }
 291         RTE_LOG(DEBUG, EAL, "Registered non-EAL thread as lcore %u.\n",
 292                 lcore_id);
 293         return 0;
 294 }
 295
 296 void
 297 rte_thread_unregister(void)
 298 {
 299         unsigned int lcore_id = rte_lcore_id();
 300
 301         if (lcore_id != LCORE_ID_ANY)
 302                 eal_lcore_non_eal_release(lcore_id);
 303         __rte_thread_uninit();
 304         if (lcore_id != LCORE_ID_ANY)
 305                 RTE_LOG(DEBUG, EAL, "Unregistered non-EAL thread (was lcore %u).\n",
 306                         lcore_id);
 307 }