lib/gpudev/gpudev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_eal.h>
   6 #include <rte_tailq.h>
   7 #include <rte_rwlock.h>
   8 #include <rte_string_fns.h>
   9 #include <rte_errno.h>
  10 #include <rte_log.h>
  11
  12 #include "rte_gpudev.h"
  13 #include "gpudev_driver.h"
  14
  15 /* Logging */
  16 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
  17 #define GPU_LOG(level, ...) \
  18         rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \
  19                 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, )))
  20
  21 /* Set any driver error as EPERM */
  22 #define GPU_DRV_RET(function) \
  23         ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
  24
  25 /* Array of devices */
  26 static struct rte_gpu *gpus;
  27 /* Number of currently valid devices */
  28 static int16_t gpu_max;
  29 /* Number of currently valid devices */
  30 static int16_t gpu_count;
  31
  32 /* Event callback object */
  33 struct rte_gpu_callback {
  34         TAILQ_ENTRY(rte_gpu_callback) next;
  35         rte_gpu_callback_t *function;
  36         void *user_data;
  37         enum rte_gpu_event event;
  38 };
  39 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
  40 static void gpu_free_callbacks(struct rte_gpu *dev);
  41
  42 int
  43 rte_gpu_init(size_t dev_max)
  44 {
  45         if (dev_max == 0 || dev_max > INT16_MAX) {
  46                 GPU_LOG(ERR, "invalid array size");
  47                 rte_errno = EINVAL;
  48                 return -rte_errno;
  49         }
  50
  51         /* No lock, it must be called before or during first probing. */
  52         if (gpus != NULL) {
  53                 GPU_LOG(ERR, "already initialized");
  54                 rte_errno = EBUSY;
  55                 return -rte_errno;
  56         }
  57
  58         gpus = calloc(dev_max, sizeof(struct rte_gpu));
  59         if (gpus == NULL) {
  60                 GPU_LOG(ERR, "cannot initialize library");
  61                 rte_errno = ENOMEM;
  62                 return -rte_errno;
  63         }
  64
  65         gpu_max = dev_max;
  66         return 0;
  67 }
  68
  69 uint16_t
  70 rte_gpu_count_avail(void)
  71 {
  72         return gpu_count;
  73 }
  74
  75 bool
  76 rte_gpu_is_valid(int16_t dev_id)
  77 {
  78         if (dev_id >= 0 && dev_id < gpu_max &&
  79                 gpus[dev_id].state == RTE_GPU_STATE_INITIALIZED)
  80                 return true;
  81         return false;
  82 }
  83
  84 int16_t
  85 rte_gpu_find_next(int16_t dev_id)
  86 {
  87         if (dev_id < 0)
  88                 dev_id = 0;
  89         while (dev_id < gpu_max &&
  90                         gpus[dev_id].state == RTE_GPU_STATE_UNUSED)
  91                 dev_id++;
  92
  93         if (dev_id >= gpu_max)
  94                 return RTE_GPU_ID_NONE;
  95         return dev_id;
  96 }
  97
  98 static int16_t
  99 gpu_find_free_id(void)
 100 {
 101         int16_t dev_id;
 102
 103         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 104                 if (gpus[dev_id].state == RTE_GPU_STATE_UNUSED)
 105                         return dev_id;
 106         }
 107         return RTE_GPU_ID_NONE;
 108 }
 109
 110 static struct rte_gpu *
 111 gpu_get_by_id(int16_t dev_id)
 112 {
 113         if (!rte_gpu_is_valid(dev_id))
 114                 return NULL;
 115         return &gpus[dev_id];
 116 }
 117
 118 struct rte_gpu *
 119 rte_gpu_get_by_name(const char *name)
 120 {
 121         int16_t dev_id;
 122         struct rte_gpu *dev;
 123
 124         if (name == NULL) {
 125                 rte_errno = EINVAL;
 126                 return NULL;
 127         }
 128
 129         RTE_GPU_FOREACH(dev_id) {
 130                 dev = &gpus[dev_id];
 131                 if (strncmp(name, dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
 132                         return dev;
 133         }
 134         return NULL;
 135 }
 136
 137 struct rte_gpu *
 138 rte_gpu_allocate(const char *name)
 139 {
 140         int16_t dev_id;
 141         struct rte_gpu *dev;
 142
 143         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 144                 GPU_LOG(ERR, "only primary process can allocate device");
 145                 rte_errno = EPERM;
 146                 return NULL;
 147         }
 148         if (name == NULL) {
 149                 GPU_LOG(ERR, "allocate device without a name");
 150                 rte_errno = EINVAL;
 151                 return NULL;
 152         }
 153
 154         /* implicit initialization of library before adding first device */
 155         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 156                 return NULL;
 157
 158         if (rte_gpu_get_by_name(name) != NULL) {
 159                 GPU_LOG(ERR, "device with name %s already exists", name);
 160                 rte_errno = EEXIST;
 161                 return NULL;
 162         }
 163         dev_id = gpu_find_free_id();
 164         if (dev_id == RTE_GPU_ID_NONE) {
 165                 GPU_LOG(ERR, "reached maximum number of devices");
 166                 rte_errno = ENOENT;
 167                 return NULL;
 168         }
 169
 170         dev = &gpus[dev_id];
 171         memset(dev, 0, sizeof(*dev));
 172
 173         if (rte_strscpy(dev->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
 174                 GPU_LOG(ERR, "device name too long: %s", name);
 175                 rte_errno = ENAMETOOLONG;
 176                 return NULL;
 177         }
 178         dev->info.name = dev->name;
 179         dev->info.dev_id = dev_id;
 180         dev->info.numa_node = -1;
 181         TAILQ_INIT(&dev->callbacks);
 182
 183         gpu_count++;
 184         GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
 185                         name, dev_id, gpu_count);
 186         return dev;
 187 }
 188
 189 void
 190 rte_gpu_complete_new(struct rte_gpu *dev)
 191 {
 192         if (dev == NULL)
 193                 return;
 194
 195         dev->state = RTE_GPU_STATE_INITIALIZED;
 196         dev->state = RTE_GPU_STATE_INITIALIZED;
 197         rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
 198 }
 199
 200 int
 201 rte_gpu_release(struct rte_gpu *dev)
 202 {
 203         if (dev == NULL) {
 204                 rte_errno = ENODEV;
 205                 return -rte_errno;
 206         }
 207
 208         GPU_LOG(DEBUG, "free device %s (id %d)",
 209                         dev->info.name, dev->info.dev_id);
 210         rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
 211
 212         gpu_free_callbacks(dev);
 213         dev->state = RTE_GPU_STATE_UNUSED;
 214         gpu_count--;
 215
 216         return 0;
 217 }
 218
 219 int
 220 rte_gpu_close(int16_t dev_id)
 221 {
 222         int firsterr, binerr;
 223         int *lasterr = &firsterr;
 224         struct rte_gpu *dev;
 225
 226         dev = gpu_get_by_id(dev_id);
 227         if (dev == NULL) {
 228                 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
 229                 rte_errno = ENODEV;
 230                 return -rte_errno;
 231         }
 232
 233         if (dev->ops.dev_close != NULL) {
 234                 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
 235                 if (*lasterr != 0)
 236                         lasterr = &binerr;
 237         }
 238
 239         *lasterr = rte_gpu_release(dev);
 240
 241         rte_errno = -firsterr;
 242         return firsterr;
 243 }
 244
 245 int
 246 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
 247                 rte_gpu_callback_t *function, void *user_data)
 248 {
 249         int16_t next_dev, last_dev;
 250         struct rte_gpu_callback_list *callbacks;
 251         struct rte_gpu_callback *callback;
 252
 253         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 254                 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
 255                 rte_errno = ENODEV;
 256                 return -rte_errno;
 257         }
 258         if (function == NULL) {
 259                 GPU_LOG(ERR, "cannot register callback without function");
 260                 rte_errno = EINVAL;
 261                 return -rte_errno;
 262         }
 263
 264         if (dev_id == RTE_GPU_ID_ANY) {
 265                 next_dev = 0;
 266                 last_dev = gpu_max - 1;
 267         } else {
 268                 next_dev = last_dev = dev_id;
 269         }
 270
 271         rte_rwlock_write_lock(&gpu_callback_lock);
 272         do {
 273                 callbacks = &gpus[next_dev].callbacks;
 274
 275                 /* check if not already registered */
 276                 TAILQ_FOREACH(callback, callbacks, next) {
 277                         if (callback->event == event &&
 278                                         callback->function == function &&
 279                                         callback->user_data == user_data) {
 280                                 GPU_LOG(INFO, "callback already registered");
 281                                 return 0;
 282                         }
 283                 }
 284
 285                 callback = malloc(sizeof(*callback));
 286                 if (callback == NULL) {
 287                         GPU_LOG(ERR, "cannot allocate callback");
 288                         return -ENOMEM;
 289                 }
 290                 callback->function = function;
 291                 callback->user_data = user_data;
 292                 callback->event = event;
 293                 TAILQ_INSERT_TAIL(callbacks, callback, next);
 294
 295         } while (++next_dev <= last_dev);
 296         rte_rwlock_write_unlock(&gpu_callback_lock);
 297
 298         return 0;
 299 }
 300
 301 int
 302 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
 303                 rte_gpu_callback_t *function, void *user_data)
 304 {
 305         int16_t next_dev, last_dev;
 306         struct rte_gpu_callback_list *callbacks;
 307         struct rte_gpu_callback *callback, *nextcb;
 308
 309         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 310                 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
 311                 rte_errno = ENODEV;
 312                 return -rte_errno;
 313         }
 314         if (function == NULL) {
 315                 GPU_LOG(ERR, "cannot unregister callback without function");
 316                 rte_errno = EINVAL;
 317                 return -rte_errno;
 318         }
 319
 320         if (dev_id == RTE_GPU_ID_ANY) {
 321                 next_dev = 0;
 322                 last_dev = gpu_max - 1;
 323         } else {
 324                 next_dev = last_dev = dev_id;
 325         }
 326
 327         rte_rwlock_write_lock(&gpu_callback_lock);
 328         do {
 329                 callbacks = &gpus[next_dev].callbacks;
 330                 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 331                         if (callback->event != event ||
 332                                         callback->function != function ||
 333                                         (callback->user_data != user_data &&
 334                                         user_data != (void *)-1))
 335                                 continue;
 336                         TAILQ_REMOVE(callbacks, callback, next);
 337                         free(callback);
 338                 }
 339         } while (++next_dev <= last_dev);
 340         rte_rwlock_write_unlock(&gpu_callback_lock);
 341
 342         return 0;
 343 }
 344
 345 static void
 346 gpu_free_callbacks(struct rte_gpu *dev)
 347 {
 348         struct rte_gpu_callback_list *callbacks;
 349         struct rte_gpu_callback *callback, *nextcb;
 350
 351         callbacks = &dev->callbacks;
 352         rte_rwlock_write_lock(&gpu_callback_lock);
 353         RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 354                 TAILQ_REMOVE(callbacks, callback, next);
 355                 free(callback);
 356         }
 357         rte_rwlock_write_unlock(&gpu_callback_lock);
 358 }
 359
 360 void
 361 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
 362 {
 363         int16_t dev_id;
 364         struct rte_gpu_callback *callback;
 365
 366         dev_id = dev->info.dev_id;
 367         rte_rwlock_read_lock(&gpu_callback_lock);
 368         TAILQ_FOREACH(callback, &dev->callbacks, next) {
 369                 if (callback->event != event || callback->function == NULL)
 370                         continue;
 371                 callback->function(dev_id, event, callback->user_data);
 372         }
 373         rte_rwlock_read_unlock(&gpu_callback_lock);
 374 }
 375
 376 int
 377 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
 378 {
 379         struct rte_gpu *dev;
 380
 381         dev = gpu_get_by_id(dev_id);
 382         if (dev == NULL) {
 383                 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
 384                 rte_errno = ENODEV;
 385                 return -rte_errno;
 386         }
 387         if (info == NULL) {
 388                 GPU_LOG(ERR, "query without storage");
 389                 rte_errno = EINVAL;
 390                 return -rte_errno;
 391         }
 392
 393         if (dev->ops.dev_info_get == NULL) {
 394                 *info = dev->info;
 395                 return 0;
 396         }
 397         return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));
 398 }