lib/gpudev/gpudev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2021 NVIDIA Corporation & Affiliates
   3  */
   4
   5 #include <rte_eal.h>
   6 #include <rte_tailq.h>
   7 #include <rte_rwlock.h>
   8 #include <rte_string_fns.h>
   9 #include <rte_errno.h>
  10 #include <rte_log.h>
  11
  12 #include "rte_gpudev.h"
  13 #include "gpudev_driver.h"
  14
  15 /* Logging */
  16 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
  17 #define GPU_LOG(level, ...) \
  18         rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \
  19                 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, )))
  20
  21 /* Set any driver error as EPERM */
  22 #define GPU_DRV_RET(function) \
  23         ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
  24
  25 /* Array of devices */
  26 static struct rte_gpu *gpus;
  27 /* Number of currently valid devices */
  28 static int16_t gpu_max;
  29 /* Number of currently valid devices */
  30 static int16_t gpu_count;
  31
  32 /* Event callback object */
  33 struct rte_gpu_callback {
  34         TAILQ_ENTRY(rte_gpu_callback) next;
  35         rte_gpu_callback_t *function;
  36         void *user_data;
  37         enum rte_gpu_event event;
  38 };
  39 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
  40 static void gpu_free_callbacks(struct rte_gpu *dev);
  41
  42 int
  43 rte_gpu_init(size_t dev_max)
  44 {
  45         if (dev_max == 0 || dev_max > INT16_MAX) {
  46                 GPU_LOG(ERR, "invalid array size");
  47                 rte_errno = EINVAL;
  48                 return -rte_errno;
  49         }
  50
  51         /* No lock, it must be called before or during first probing. */
  52         if (gpus != NULL) {
  53                 GPU_LOG(ERR, "already initialized");
  54                 rte_errno = EBUSY;
  55                 return -rte_errno;
  56         }
  57
  58         gpus = calloc(dev_max, sizeof(struct rte_gpu));
  59         if (gpus == NULL) {
  60                 GPU_LOG(ERR, "cannot initialize library");
  61                 rte_errno = ENOMEM;
  62                 return -rte_errno;
  63         }
  64
  65         gpu_max = dev_max;
  66         return 0;
  67 }
  68
  69 uint16_t
  70 rte_gpu_count_avail(void)
  71 {
  72         return gpu_count;
  73 }
  74
  75 bool
  76 rte_gpu_is_valid(int16_t dev_id)
  77 {
  78         if (dev_id >= 0 && dev_id < gpu_max &&
  79                 gpus[dev_id].state == RTE_GPU_STATE_INITIALIZED)
  80                 return true;
  81         return false;
  82 }
  83
  84 static bool
  85 gpu_match_parent(int16_t dev_id, int16_t parent)
  86 {
  87         if (parent == RTE_GPU_ID_ANY)
  88                 return true;
  89         return gpus[dev_id].info.parent == parent;
  90 }
  91
  92 int16_t
  93 rte_gpu_find_next(int16_t dev_id, int16_t parent)
  94 {
  95         if (dev_id < 0)
  96                 dev_id = 0;
  97         while (dev_id < gpu_max &&
  98                         (gpus[dev_id].state == RTE_GPU_STATE_UNUSED ||
  99                         !gpu_match_parent(dev_id, parent)))
 100                 dev_id++;
 101
 102         if (dev_id >= gpu_max)
 103                 return RTE_GPU_ID_NONE;
 104         return dev_id;
 105 }
 106
 107 static int16_t
 108 gpu_find_free_id(void)
 109 {
 110         int16_t dev_id;
 111
 112         for (dev_id = 0; dev_id < gpu_max; dev_id++) {
 113                 if (gpus[dev_id].state == RTE_GPU_STATE_UNUSED)
 114                         return dev_id;
 115         }
 116         return RTE_GPU_ID_NONE;
 117 }
 118
 119 static struct rte_gpu *
 120 gpu_get_by_id(int16_t dev_id)
 121 {
 122         if (!rte_gpu_is_valid(dev_id))
 123                 return NULL;
 124         return &gpus[dev_id];
 125 }
 126
 127 struct rte_gpu *
 128 rte_gpu_get_by_name(const char *name)
 129 {
 130         int16_t dev_id;
 131         struct rte_gpu *dev;
 132
 133         if (name == NULL) {
 134                 rte_errno = EINVAL;
 135                 return NULL;
 136         }
 137
 138         RTE_GPU_FOREACH(dev_id) {
 139                 dev = &gpus[dev_id];
 140                 if (strncmp(name, dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
 141                         return dev;
 142         }
 143         return NULL;
 144 }
 145
 146 struct rte_gpu *
 147 rte_gpu_allocate(const char *name)
 148 {
 149         int16_t dev_id;
 150         struct rte_gpu *dev;
 151
 152         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 153                 GPU_LOG(ERR, "only primary process can allocate device");
 154                 rte_errno = EPERM;
 155                 return NULL;
 156         }
 157         if (name == NULL) {
 158                 GPU_LOG(ERR, "allocate device without a name");
 159                 rte_errno = EINVAL;
 160                 return NULL;
 161         }
 162
 163         /* implicit initialization of library before adding first device */
 164         if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
 165                 return NULL;
 166
 167         if (rte_gpu_get_by_name(name) != NULL) {
 168                 GPU_LOG(ERR, "device with name %s already exists", name);
 169                 rte_errno = EEXIST;
 170                 return NULL;
 171         }
 172         dev_id = gpu_find_free_id();
 173         if (dev_id == RTE_GPU_ID_NONE) {
 174                 GPU_LOG(ERR, "reached maximum number of devices");
 175                 rte_errno = ENOENT;
 176                 return NULL;
 177         }
 178
 179         dev = &gpus[dev_id];
 180         memset(dev, 0, sizeof(*dev));
 181
 182         if (rte_strscpy(dev->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
 183                 GPU_LOG(ERR, "device name too long: %s", name);
 184                 rte_errno = ENAMETOOLONG;
 185                 return NULL;
 186         }
 187         dev->info.name = dev->name;
 188         dev->info.dev_id = dev_id;
 189         dev->info.numa_node = -1;
 190         dev->info.parent = RTE_GPU_ID_NONE;
 191         TAILQ_INIT(&dev->callbacks);
 192
 193         gpu_count++;
 194         GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
 195                         name, dev_id, gpu_count);
 196         return dev;
 197 }
 198
 199 int16_t
 200 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
 201 {
 202         struct rte_gpu *dev;
 203
 204         if (!rte_gpu_is_valid(parent)) {
 205                 GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
 206                 rte_errno = ENODEV;
 207                 return -rte_errno;
 208         }
 209
 210         dev = rte_gpu_allocate(name);
 211         if (dev == NULL)
 212                 return -rte_errno;
 213
 214         dev->info.parent = parent;
 215         dev->info.context = child_context;
 216
 217         rte_gpu_complete_new(dev);
 218         return dev->info.dev_id;
 219 }
 220
 221 void
 222 rte_gpu_complete_new(struct rte_gpu *dev)
 223 {
 224         if (dev == NULL)
 225                 return;
 226
 227         dev->state = RTE_GPU_STATE_INITIALIZED;
 228         dev->state = RTE_GPU_STATE_INITIALIZED;
 229         rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
 230 }
 231
 232 int
 233 rte_gpu_release(struct rte_gpu *dev)
 234 {
 235         int16_t dev_id, child;
 236
 237         if (dev == NULL) {
 238                 rte_errno = ENODEV;
 239                 return -rte_errno;
 240         }
 241         dev_id = dev->info.dev_id;
 242         RTE_GPU_FOREACH_CHILD(child, dev_id) {
 243                 GPU_LOG(ERR, "cannot release device %d with child %d",
 244                                 dev_id, child);
 245                 rte_errno = EBUSY;
 246                 return -rte_errno;
 247         }
 248
 249         GPU_LOG(DEBUG, "free device %s (id %d)",
 250                         dev->info.name, dev->info.dev_id);
 251         rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
 252
 253         gpu_free_callbacks(dev);
 254         dev->state = RTE_GPU_STATE_UNUSED;
 255         gpu_count--;
 256
 257         return 0;
 258 }
 259
 260 int
 261 rte_gpu_close(int16_t dev_id)
 262 {
 263         int firsterr, binerr;
 264         int *lasterr = &firsterr;
 265         struct rte_gpu *dev;
 266
 267         dev = gpu_get_by_id(dev_id);
 268         if (dev == NULL) {
 269                 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
 270                 rte_errno = ENODEV;
 271                 return -rte_errno;
 272         }
 273
 274         if (dev->ops.dev_close != NULL) {
 275                 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
 276                 if (*lasterr != 0)
 277                         lasterr = &binerr;
 278         }
 279
 280         *lasterr = rte_gpu_release(dev);
 281
 282         rte_errno = -firsterr;
 283         return firsterr;
 284 }
 285
 286 int
 287 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
 288                 rte_gpu_callback_t *function, void *user_data)
 289 {
 290         int16_t next_dev, last_dev;
 291         struct rte_gpu_callback_list *callbacks;
 292         struct rte_gpu_callback *callback;
 293
 294         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 295                 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
 296                 rte_errno = ENODEV;
 297                 return -rte_errno;
 298         }
 299         if (function == NULL) {
 300                 GPU_LOG(ERR, "cannot register callback without function");
 301                 rte_errno = EINVAL;
 302                 return -rte_errno;
 303         }
 304
 305         if (dev_id == RTE_GPU_ID_ANY) {
 306                 next_dev = 0;
 307                 last_dev = gpu_max - 1;
 308         } else {
 309                 next_dev = last_dev = dev_id;
 310         }
 311
 312         rte_rwlock_write_lock(&gpu_callback_lock);
 313         do {
 314                 callbacks = &gpus[next_dev].callbacks;
 315
 316                 /* check if not already registered */
 317                 TAILQ_FOREACH(callback, callbacks, next) {
 318                         if (callback->event == event &&
 319                                         callback->function == function &&
 320                                         callback->user_data == user_data) {
 321                                 GPU_LOG(INFO, "callback already registered");
 322                                 return 0;
 323                         }
 324                 }
 325
 326                 callback = malloc(sizeof(*callback));
 327                 if (callback == NULL) {
 328                         GPU_LOG(ERR, "cannot allocate callback");
 329                         return -ENOMEM;
 330                 }
 331                 callback->function = function;
 332                 callback->user_data = user_data;
 333                 callback->event = event;
 334                 TAILQ_INSERT_TAIL(callbacks, callback, next);
 335
 336         } while (++next_dev <= last_dev);
 337         rte_rwlock_write_unlock(&gpu_callback_lock);
 338
 339         return 0;
 340 }
 341
 342 int
 343 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
 344                 rte_gpu_callback_t *function, void *user_data)
 345 {
 346         int16_t next_dev, last_dev;
 347         struct rte_gpu_callback_list *callbacks;
 348         struct rte_gpu_callback *callback, *nextcb;
 349
 350         if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
 351                 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
 352                 rte_errno = ENODEV;
 353                 return -rte_errno;
 354         }
 355         if (function == NULL) {
 356                 GPU_LOG(ERR, "cannot unregister callback without function");
 357                 rte_errno = EINVAL;
 358                 return -rte_errno;
 359         }
 360
 361         if (dev_id == RTE_GPU_ID_ANY) {
 362                 next_dev = 0;
 363                 last_dev = gpu_max - 1;
 364         } else {
 365                 next_dev = last_dev = dev_id;
 366         }
 367
 368         rte_rwlock_write_lock(&gpu_callback_lock);
 369         do {
 370                 callbacks = &gpus[next_dev].callbacks;
 371                 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 372                         if (callback->event != event ||
 373                                         callback->function != function ||
 374                                         (callback->user_data != user_data &&
 375                                         user_data != (void *)-1))
 376                                 continue;
 377                         TAILQ_REMOVE(callbacks, callback, next);
 378                         free(callback);
 379                 }
 380         } while (++next_dev <= last_dev);
 381         rte_rwlock_write_unlock(&gpu_callback_lock);
 382
 383         return 0;
 384 }
 385
 386 static void
 387 gpu_free_callbacks(struct rte_gpu *dev)
 388 {
 389         struct rte_gpu_callback_list *callbacks;
 390         struct rte_gpu_callback *callback, *nextcb;
 391
 392         callbacks = &dev->callbacks;
 393         rte_rwlock_write_lock(&gpu_callback_lock);
 394         RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
 395                 TAILQ_REMOVE(callbacks, callback, next);
 396                 free(callback);
 397         }
 398         rte_rwlock_write_unlock(&gpu_callback_lock);
 399 }
 400
 401 void
 402 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
 403 {
 404         int16_t dev_id;
 405         struct rte_gpu_callback *callback;
 406
 407         dev_id = dev->info.dev_id;
 408         rte_rwlock_read_lock(&gpu_callback_lock);
 409         TAILQ_FOREACH(callback, &dev->callbacks, next) {
 410                 if (callback->event != event || callback->function == NULL)
 411                         continue;
 412                 callback->function(dev_id, event, callback->user_data);
 413         }
 414         rte_rwlock_read_unlock(&gpu_callback_lock);
 415 }
 416
 417 int
 418 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
 419 {
 420         struct rte_gpu *dev;
 421
 422         dev = gpu_get_by_id(dev_id);
 423         if (dev == NULL) {
 424                 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
 425                 rte_errno = ENODEV;
 426                 return -rte_errno;
 427         }
 428         if (info == NULL) {
 429                 GPU_LOG(ERR, "query without storage");
 430                 rte_errno = EINVAL;
 431                 return -rte_errno;
 432         }
 433
 434         if (dev->ops.dev_info_get == NULL) {
 435                 *info = dev->info;
 436                 return 0;
 437         }
 438         return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));
 439 }