1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2021 NVIDIA Corporation & Affiliates
7 #include <rte_rwlock.h>
8 #include <rte_string_fns.h>
12 #include "rte_gpudev.h"
13 #include "gpudev_driver.h"
16 RTE_LOG_REGISTER_DEFAULT(gpu_logtype, NOTICE);
17 #define GPU_LOG(level, ...) \
18 rte_log(RTE_LOG_ ## level, gpu_logtype, RTE_FMT("gpu: " \
19 RTE_FMT_HEAD(__VA_ARGS__, ) "\n", RTE_FMT_TAIL(__VA_ARGS__, )))
21 /* Set any driver error as EPERM */
22 #define GPU_DRV_RET(function) \
23 ((function != 0) ? -(rte_errno = EPERM) : (rte_errno = 0))
25 /* Array of devices */
26 static struct rte_gpu *gpus;
27 /* Number of currently valid devices */
28 static int16_t gpu_max;
29 /* Number of currently valid devices */
30 static int16_t gpu_count;
32 /* Event callback object */
33 struct rte_gpu_callback {
34 TAILQ_ENTRY(rte_gpu_callback) next;
35 rte_gpu_callback_t *function;
37 enum rte_gpu_event event;
39 static rte_rwlock_t gpu_callback_lock = RTE_RWLOCK_INITIALIZER;
40 static void gpu_free_callbacks(struct rte_gpu *dev);
43 rte_gpu_init(size_t dev_max)
45 if (dev_max == 0 || dev_max > INT16_MAX) {
46 GPU_LOG(ERR, "invalid array size");
51 /* No lock, it must be called before or during first probing. */
53 GPU_LOG(ERR, "already initialized");
58 gpus = calloc(dev_max, sizeof(struct rte_gpu));
60 GPU_LOG(ERR, "cannot initialize library");
70 rte_gpu_count_avail(void)
76 rte_gpu_is_valid(int16_t dev_id)
78 if (dev_id >= 0 && dev_id < gpu_max &&
79 gpus[dev_id].state == RTE_GPU_STATE_INITIALIZED)
85 gpu_match_parent(int16_t dev_id, int16_t parent)
87 if (parent == RTE_GPU_ID_ANY)
89 return gpus[dev_id].info.parent == parent;
93 rte_gpu_find_next(int16_t dev_id, int16_t parent)
97 while (dev_id < gpu_max &&
98 (gpus[dev_id].state == RTE_GPU_STATE_UNUSED ||
99 !gpu_match_parent(dev_id, parent)))
102 if (dev_id >= gpu_max)
103 return RTE_GPU_ID_NONE;
108 gpu_find_free_id(void)
112 for (dev_id = 0; dev_id < gpu_max; dev_id++) {
113 if (gpus[dev_id].state == RTE_GPU_STATE_UNUSED)
116 return RTE_GPU_ID_NONE;
119 static struct rte_gpu *
120 gpu_get_by_id(int16_t dev_id)
122 if (!rte_gpu_is_valid(dev_id))
124 return &gpus[dev_id];
128 rte_gpu_get_by_name(const char *name)
138 RTE_GPU_FOREACH(dev_id) {
140 if (strncmp(name, dev->name, RTE_DEV_NAME_MAX_LEN) == 0)
147 rte_gpu_allocate(const char *name)
152 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
153 GPU_LOG(ERR, "only primary process can allocate device");
158 GPU_LOG(ERR, "allocate device without a name");
163 /* implicit initialization of library before adding first device */
164 if (gpus == NULL && rte_gpu_init(RTE_GPU_DEFAULT_MAX) < 0)
167 if (rte_gpu_get_by_name(name) != NULL) {
168 GPU_LOG(ERR, "device with name %s already exists", name);
172 dev_id = gpu_find_free_id();
173 if (dev_id == RTE_GPU_ID_NONE) {
174 GPU_LOG(ERR, "reached maximum number of devices");
180 memset(dev, 0, sizeof(*dev));
182 if (rte_strscpy(dev->name, name, RTE_DEV_NAME_MAX_LEN) < 0) {
183 GPU_LOG(ERR, "device name too long: %s", name);
184 rte_errno = ENAMETOOLONG;
187 dev->info.name = dev->name;
188 dev->info.dev_id = dev_id;
189 dev->info.numa_node = -1;
190 dev->info.parent = RTE_GPU_ID_NONE;
191 TAILQ_INIT(&dev->callbacks);
194 GPU_LOG(DEBUG, "new device %s (id %d) of total %d",
195 name, dev_id, gpu_count);
200 rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
204 if (!rte_gpu_is_valid(parent)) {
205 GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
210 dev = rte_gpu_allocate(name);
214 dev->info.parent = parent;
215 dev->info.context = child_context;
217 rte_gpu_complete_new(dev);
218 return dev->info.dev_id;
222 rte_gpu_complete_new(struct rte_gpu *dev)
227 dev->state = RTE_GPU_STATE_INITIALIZED;
228 dev->state = RTE_GPU_STATE_INITIALIZED;
229 rte_gpu_notify(dev, RTE_GPU_EVENT_NEW);
233 rte_gpu_release(struct rte_gpu *dev)
235 int16_t dev_id, child;
241 dev_id = dev->info.dev_id;
242 RTE_GPU_FOREACH_CHILD(child, dev_id) {
243 GPU_LOG(ERR, "cannot release device %d with child %d",
249 GPU_LOG(DEBUG, "free device %s (id %d)",
250 dev->info.name, dev->info.dev_id);
251 rte_gpu_notify(dev, RTE_GPU_EVENT_DEL);
253 gpu_free_callbacks(dev);
254 dev->state = RTE_GPU_STATE_UNUSED;
261 rte_gpu_close(int16_t dev_id)
263 int firsterr, binerr;
264 int *lasterr = &firsterr;
267 dev = gpu_get_by_id(dev_id);
269 GPU_LOG(ERR, "close invalid device ID %d", dev_id);
274 if (dev->ops.dev_close != NULL) {
275 *lasterr = GPU_DRV_RET(dev->ops.dev_close(dev));
280 *lasterr = rte_gpu_release(dev);
282 rte_errno = -firsterr;
287 rte_gpu_callback_register(int16_t dev_id, enum rte_gpu_event event,
288 rte_gpu_callback_t *function, void *user_data)
290 int16_t next_dev, last_dev;
291 struct rte_gpu_callback_list *callbacks;
292 struct rte_gpu_callback *callback;
294 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
295 GPU_LOG(ERR, "register callback of invalid ID %d", dev_id);
299 if (function == NULL) {
300 GPU_LOG(ERR, "cannot register callback without function");
305 if (dev_id == RTE_GPU_ID_ANY) {
307 last_dev = gpu_max - 1;
309 next_dev = last_dev = dev_id;
312 rte_rwlock_write_lock(&gpu_callback_lock);
314 callbacks = &gpus[next_dev].callbacks;
316 /* check if not already registered */
317 TAILQ_FOREACH(callback, callbacks, next) {
318 if (callback->event == event &&
319 callback->function == function &&
320 callback->user_data == user_data) {
321 GPU_LOG(INFO, "callback already registered");
326 callback = malloc(sizeof(*callback));
327 if (callback == NULL) {
328 GPU_LOG(ERR, "cannot allocate callback");
331 callback->function = function;
332 callback->user_data = user_data;
333 callback->event = event;
334 TAILQ_INSERT_TAIL(callbacks, callback, next);
336 } while (++next_dev <= last_dev);
337 rte_rwlock_write_unlock(&gpu_callback_lock);
343 rte_gpu_callback_unregister(int16_t dev_id, enum rte_gpu_event event,
344 rte_gpu_callback_t *function, void *user_data)
346 int16_t next_dev, last_dev;
347 struct rte_gpu_callback_list *callbacks;
348 struct rte_gpu_callback *callback, *nextcb;
350 if (!rte_gpu_is_valid(dev_id) && dev_id != RTE_GPU_ID_ANY) {
351 GPU_LOG(ERR, "unregister callback of invalid ID %d", dev_id);
355 if (function == NULL) {
356 GPU_LOG(ERR, "cannot unregister callback without function");
361 if (dev_id == RTE_GPU_ID_ANY) {
363 last_dev = gpu_max - 1;
365 next_dev = last_dev = dev_id;
368 rte_rwlock_write_lock(&gpu_callback_lock);
370 callbacks = &gpus[next_dev].callbacks;
371 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
372 if (callback->event != event ||
373 callback->function != function ||
374 (callback->user_data != user_data &&
375 user_data != (void *)-1))
377 TAILQ_REMOVE(callbacks, callback, next);
380 } while (++next_dev <= last_dev);
381 rte_rwlock_write_unlock(&gpu_callback_lock);
387 gpu_free_callbacks(struct rte_gpu *dev)
389 struct rte_gpu_callback_list *callbacks;
390 struct rte_gpu_callback *callback, *nextcb;
392 callbacks = &dev->callbacks;
393 rte_rwlock_write_lock(&gpu_callback_lock);
394 RTE_TAILQ_FOREACH_SAFE(callback, callbacks, next, nextcb) {
395 TAILQ_REMOVE(callbacks, callback, next);
398 rte_rwlock_write_unlock(&gpu_callback_lock);
402 rte_gpu_notify(struct rte_gpu *dev, enum rte_gpu_event event)
405 struct rte_gpu_callback *callback;
407 dev_id = dev->info.dev_id;
408 rte_rwlock_read_lock(&gpu_callback_lock);
409 TAILQ_FOREACH(callback, &dev->callbacks, next) {
410 if (callback->event != event || callback->function == NULL)
412 callback->function(dev_id, event, callback->user_data);
414 rte_rwlock_read_unlock(&gpu_callback_lock);
418 rte_gpu_info_get(int16_t dev_id, struct rte_gpu_info *info)
422 dev = gpu_get_by_id(dev_id);
424 GPU_LOG(ERR, "query invalid device ID %d", dev_id);
429 GPU_LOG(ERR, "query without storage");
434 if (dev->ops.dev_info_get == NULL) {
438 return GPU_DRV_RET(dev->ops.dev_info_get(dev, info));