gpudev: add child device representing a device context

The computing device may operate in some isolated contexts.
Memory and processing are isolated in a silo represented by
a child device.
The context is provided as an opaque by the caller of
rte_gpu_add_child().

Signed-off-by: Thomas Monjalon <thomas@monjalon.net>

diff --git a/app/test-gpudev/main.c b/app/test-gpudev/main.c
index 15b6e4e..438cfda 100644 (file)
--- a/app/test-gpudev/main.c
+++ b/app/test-gpudev/main.c
@@ -88,8 +88,9 @@ main(int argc, char **argv)
                if (rte_gpu_info_get(gpu_id, &ginfo))
                        rte_exit(EXIT_FAILURE, "rte_gpu_info_get error - bye\n");
 
-               printf("\tGPU ID %d\n\t\tGPU Bus ID %s NUMA node %d Tot memory %.02f MB, Tot processors %d\n",
+               printf("\tGPU ID %d\n\t\tparent ID %d GPU Bus ID %s NUMA node %d Tot memory %.02f MB, Tot processors %d\n",
                                ginfo.dev_id,
+                               ginfo.parent,
                                ginfo.name,
                                ginfo.numa_node,
                                (((float)ginfo.total_memory)/(float)1024)/(float)1024,

diff --git a/doc/guides/prog_guide/gpudev.rst b/doc/guides/prog_guide/gpudev.rst
index 6ea7239..7694639 100644 (file)
--- a/doc/guides/prog_guide/gpudev.rst
+++ b/doc/guides/prog_guide/gpudev.rst
@@ -34,3 +34,15 @@ This library provides a number of features:
 
 API Overview
 ------------
+
+Child Device
+~~~~~~~~~~~~
+
+By default, DPDK PCIe module detects and registers physical GPU devices
+in the system.
+With the gpudev library is also possible to add additional non-physical devices
+through an ``uint64_t`` generic handler (e.g. CUDA Driver context)
+that will be registered internally by the driver as an additional device (child)
+connected to a physical device (parent).
+Each device (parent or child) is represented through a ID
+required to indicate which device a given operation should be executed on.

diff --git a/lib/gpudev/gpudev.c b/lib/gpudev/gpudev.c
index c25f9ed..4ed8426 100644 (file)
--- a/lib/gpudev/gpudev.c
+++ b/lib/gpudev/gpudev.c
@@ -81,13 +81,22 @@ rte_gpu_is_valid(int16_t dev_id)
        return false;
 }
 
+static bool
+gpu_match_parent(int16_t dev_id, int16_t parent)
+{
+       if (parent == RTE_GPU_ID_ANY)
+               return true;
+       return gpus[dev_id].info.parent == parent;
+}
+
 int16_t
-rte_gpu_find_next(int16_t dev_id)
+rte_gpu_find_next(int16_t dev_id, int16_t parent)
 {
        if (dev_id < 0)
                dev_id = 0;
        while (dev_id < gpu_max &&
-                       gpus[dev_id].state == RTE_GPU_STATE_UNUSED)
+                       (gpus[dev_id].state == RTE_GPU_STATE_UNUSED ||
+                       !gpu_match_parent(dev_id, parent)))
                dev_id++;
 
        if (dev_id >= gpu_max)
@@ -178,6 +187,7 @@ rte_gpu_allocate(const char *name)
        dev->info.name = dev->name;
        dev->info.dev_id = dev_id;
        dev->info.numa_node = -1;
+       dev->info.parent = RTE_GPU_ID_NONE;
        TAILQ_INIT(&dev->callbacks);
 
        gpu_count++;
@@ -186,6 +196,28 @@ rte_gpu_allocate(const char *name)
        return dev;
 }
 
+int16_t
+rte_gpu_add_child(const char *name, int16_t parent, uint64_t child_context)
+{
+       struct rte_gpu *dev;
+
+       if (!rte_gpu_is_valid(parent)) {
+               GPU_LOG(ERR, "add child to invalid parent ID %d", parent);
+               rte_errno = ENODEV;
+               return -rte_errno;
+       }
+
+       dev = rte_gpu_allocate(name);
+       if (dev == NULL)
+               return -rte_errno;
+
+       dev->info.parent = parent;
+       dev->info.context = child_context;
+
+       rte_gpu_complete_new(dev);
+       return dev->info.dev_id;
+}
+
 void
 rte_gpu_complete_new(struct rte_gpu *dev)
 {
@@ -200,10 +232,19 @@ rte_gpu_complete_new(struct rte_gpu *dev)
 int
 rte_gpu_release(struct rte_gpu *dev)
 {
+       int16_t dev_id, child;
+
        if (dev == NULL) {
                rte_errno = ENODEV;
                return -rte_errno;
        }
+       dev_id = dev->info.dev_id;
+       RTE_GPU_FOREACH_CHILD(child, dev_id) {
+               GPU_LOG(ERR, "cannot release device %d with child %d",
+                               dev_id, child);
+               rte_errno = EBUSY;
+               return -rte_errno;
+       }
 
        GPU_LOG(DEBUG, "free device %s (id %d)",
                        dev->info.name, dev->info.dev_id);

diff --git a/lib/gpudev/gpudev_driver.h b/lib/gpudev/gpudev_driver.h
index 2a7089a..4d00771 100644 (file)
--- a/lib/gpudev/gpudev_driver.h
+++ b/lib/gpudev/gpudev_driver.h
@@ -31,7 +31,7 @@ typedef int (rte_gpu_info_get_t)(struct rte_gpu *dev, struct rte_gpu_info *info)
 struct rte_gpu_ops {
        /* Get device info. If NULL, info is just copied. */
        rte_gpu_info_get_t *dev_info_get;
-       /* Close device. */
+       /* Close device or child context. */
        rte_gpu_close_t *dev_close;
 };
 

diff --git a/lib/gpudev/rte_gpudev.h b/lib/gpudev/rte_gpudev.h
index e1702fb..df75dbd 100644 (file)
--- a/lib/gpudev/rte_gpudev.h
+++ b/lib/gpudev/rte_gpudev.h
@@ -41,8 +41,12 @@ extern "C" {
 struct rte_gpu_info {
        /** Unique identifier name. */
        const char *name;
+       /** Opaque handler of the device context. */
+       uint64_t context;
        /** Device ID. */
        int16_t dev_id;
+       /** ID of the parent device, RTE_GPU_ID_NONE if no parent */
+       int16_t parent;
        /** Total processors available on device. */
        uint32_t processor_count;
        /** Total memory available on device. */
@@ -110,6 +114,33 @@ uint16_t rte_gpu_count_avail(void);
 __rte_experimental
 bool rte_gpu_is_valid(int16_t dev_id);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Create a virtual device representing a context in the parent device.
+ *
+ * @param name
+ *   Unique string to identify the device.
+ * @param parent
+ *   Device ID of the parent.
+ * @param child_context
+ *   Opaque context handler.
+ *
+ * @return
+ *   Device ID of the new created child, -rte_errno otherwise:
+ *   - EINVAL if empty name
+ *   - ENAMETOOLONG if long name
+ *   - EEXIST if existing device name
+ *   - ENODEV if invalid parent
+ *   - EPERM if secondary process
+ *   - ENOENT if too many devices
+ *   - ENOMEM if out of space
+ */
+__rte_experimental
+int16_t rte_gpu_add_child(const char *name,
+               int16_t parent, uint64_t child_context);
+
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
@@ -118,13 +149,17 @@ bool rte_gpu_is_valid(int16_t dev_id);
  *
  * @param dev_id
  *   The initial device ID to start the research.
+ * @param parent
+ *   The device ID of the parent.
+ *   RTE_GPU_ID_NONE means no parent.
+ *   RTE_GPU_ID_ANY means no or any parent.
  *
  * @return
  *   Next device ID corresponding to a valid and initialized computing device,
  *   RTE_GPU_ID_NONE if there is none.
  */
 __rte_experimental
-int16_t rte_gpu_find_next(int16_t dev_id);
+int16_t rte_gpu_find_next(int16_t dev_id, int16_t parent);
 
 /**
  * @warning
@@ -136,15 +171,41 @@ int16_t rte_gpu_find_next(int16_t dev_id);
  *   The ID of the next possible valid device, usually 0 to iterate all.
  */
 #define RTE_GPU_FOREACH(dev_id) \
-       for (dev_id = rte_gpu_find_next(0); \
-            dev_id > 0; \
-            dev_id = rte_gpu_find_next(dev_id + 1))
+       RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_ANY)
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Macro to iterate over all valid computing devices having no parent.
+ *
+ * @param dev_id
+ *   The ID of the next possible valid device, usually 0 to iterate all.
+ */
+#define RTE_GPU_FOREACH_PARENT(dev_id) \
+       RTE_GPU_FOREACH_CHILD(dev_id, RTE_GPU_ID_NONE)
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Macro to iterate over all valid children of a computing device parent.
+ *
+ * @param dev_id
+ *   The ID of the next possible valid device, usually 0 to iterate all.
+ * @param parent
+ *   The device ID of the parent.
+ */
+#define RTE_GPU_FOREACH_CHILD(dev_id, parent) \
+       for (dev_id = rte_gpu_find_next(0, parent); \
+            dev_id >= 0; \
+            dev_id = rte_gpu_find_next(dev_id + 1, parent))
 
 /**
  * @warning
  * @b EXPERIMENTAL: this API may change without prior notice.
  *
- * Close device.
+ * Close device or child context.
  * All resources are released.
  *
  * @param dev_id

diff --git a/lib/gpudev/version.map b/lib/gpudev/version.map
index b3b6b76..4a934ed 100644 (file)
--- a/lib/gpudev/version.map
+++ b/lib/gpudev/version.map
@@ -2,6 +2,7 @@ EXPERIMENTAL {
        global:
 
        # added in 21.11
+       rte_gpu_add_child;
        rte_gpu_callback_register;
        rte_gpu_callback_unregister;
        rte_gpu_close;