1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 RehiveTech. All rights reserved.
11 #include <sys/queue.h>
16 #include <rte_common.h>
17 #include <rte_devargs.h>
18 #include <rte_memory.h>
19 #include <rte_tailq.h>
20 #include <rte_spinlock.h>
21 #include <rte_string_fns.h>
22 #include <rte_errno.h>
24 #include "rte_bus_vdev.h"
25 #include "vdev_logs.h"
26 #include "vdev_private.h"
28 #define VDEV_MP_KEY "bus_vdev_mp"
30 /* Forward declare to access virtual bus name */
31 static struct rte_bus rte_vdev_bus;
33 /** Double linked list of virtual device drivers. */
34 TAILQ_HEAD(vdev_device_list, rte_vdev_device);
36 static struct vdev_device_list vdev_device_list =
37 TAILQ_HEAD_INITIALIZER(vdev_device_list);
38 /* The lock needs to be recursive because a vdev can manage another vdev. */
39 static rte_spinlock_recursive_t vdev_device_list_lock =
40 RTE_SPINLOCK_RECURSIVE_INITIALIZER;
42 static struct vdev_driver_list vdev_driver_list =
43 TAILQ_HEAD_INITIALIZER(vdev_driver_list);
45 struct vdev_custom_scan {
46 TAILQ_ENTRY(vdev_custom_scan) next;
47 rte_vdev_scan_callback callback;
50 TAILQ_HEAD(vdev_custom_scans, vdev_custom_scan);
51 static struct vdev_custom_scans vdev_custom_scans =
52 TAILQ_HEAD_INITIALIZER(vdev_custom_scans);
53 static rte_spinlock_t vdev_custom_scan_lock = RTE_SPINLOCK_INITIALIZER;
55 /* register a driver */
57 rte_vdev_register(struct rte_vdev_driver *driver)
59 TAILQ_INSERT_TAIL(&vdev_driver_list, driver, next);
62 /* unregister a driver */
64 rte_vdev_unregister(struct rte_vdev_driver *driver)
66 TAILQ_REMOVE(&vdev_driver_list, driver, next);
70 rte_vdev_add_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
72 struct vdev_custom_scan *custom_scan;
74 rte_spinlock_lock(&vdev_custom_scan_lock);
76 /* check if already registered */
77 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
78 if (custom_scan->callback == callback &&
79 custom_scan->user_arg == user_arg)
83 if (custom_scan == NULL) {
84 custom_scan = malloc(sizeof(struct vdev_custom_scan));
85 if (custom_scan != NULL) {
86 custom_scan->callback = callback;
87 custom_scan->user_arg = user_arg;
88 TAILQ_INSERT_TAIL(&vdev_custom_scans, custom_scan, next);
92 rte_spinlock_unlock(&vdev_custom_scan_lock);
94 return (custom_scan == NULL) ? -1 : 0;
98 rte_vdev_remove_custom_scan(rte_vdev_scan_callback callback, void *user_arg)
100 struct vdev_custom_scan *custom_scan, *tmp_scan;
102 rte_spinlock_lock(&vdev_custom_scan_lock);
103 TAILQ_FOREACH_SAFE(custom_scan, &vdev_custom_scans, next, tmp_scan) {
104 if (custom_scan->callback != callback ||
105 (custom_scan->user_arg != (void *)-1 &&
106 custom_scan->user_arg != user_arg))
108 TAILQ_REMOVE(&vdev_custom_scans, custom_scan, next);
111 rte_spinlock_unlock(&vdev_custom_scan_lock);
117 vdev_parse(const char *name, void *addr)
119 struct rte_vdev_driver **out = addr;
120 struct rte_vdev_driver *driver = NULL;
122 TAILQ_FOREACH(driver, &vdev_driver_list, next) {
123 if (strncmp(driver->driver.name, name,
124 strlen(driver->driver.name)) == 0)
126 if (driver->driver.alias &&
127 strncmp(driver->driver.alias, name,
128 strlen(driver->driver.alias)) == 0)
131 if (driver != NULL &&
134 return driver == NULL;
138 vdev_dma_map(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
140 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
141 const struct rte_vdev_driver *driver;
148 if (!vdev->device.driver) {
149 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
153 driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
157 return driver->dma_map(vdev, addr, iova, len);
163 vdev_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)
165 struct rte_vdev_device *vdev = RTE_DEV_TO_VDEV(dev);
166 const struct rte_vdev_driver *driver;
173 if (!vdev->device.driver) {
174 VDEV_LOG(DEBUG, "no driver attach to device %s", dev->name);
178 driver = container_of(vdev->device.driver, const struct rte_vdev_driver,
181 if (driver->dma_unmap)
182 return driver->dma_unmap(vdev, addr, iova, len);
188 vdev_probe_all_drivers(struct rte_vdev_device *dev)
191 struct rte_vdev_driver *driver;
192 enum rte_iova_mode iova_mode;
195 if (rte_dev_is_probed(&dev->device))
198 name = rte_vdev_device_name(dev);
199 VDEV_LOG(DEBUG, "Search driver to probe device %s", name);
201 if (vdev_parse(name, &driver))
204 iova_mode = rte_eal_iova_mode();
205 if ((driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA) && (iova_mode == RTE_IOVA_PA)) {
206 VDEV_LOG(ERR, "%s requires VA IOVA mode but current mode is PA, not initializing",
211 ret = driver->probe(dev);
213 dev->device.driver = &driver->driver;
217 /* The caller shall be responsible for thread-safe */
218 static struct rte_vdev_device *
219 find_vdev(const char *name)
221 struct rte_vdev_device *dev;
226 TAILQ_FOREACH(dev, &vdev_device_list, next) {
227 const char *devname = rte_vdev_device_name(dev);
229 if (!strcmp(devname, name))
236 static struct rte_devargs *
237 alloc_devargs(const char *name, const char *args)
239 struct rte_devargs *devargs;
242 devargs = calloc(1, sizeof(*devargs));
246 devargs->bus = &rte_vdev_bus;
248 devargs->data = strdup(args);
250 devargs->data = strdup("");
251 devargs->args = devargs->data;
253 ret = strlcpy(devargs->name, name, sizeof(devargs->name));
254 if (ret < 0 || ret >= (int)sizeof(devargs->name)) {
255 rte_devargs_reset(devargs);
264 insert_vdev(const char *name, const char *args,
265 struct rte_vdev_device **p_dev,
268 struct rte_vdev_device *dev;
269 struct rte_devargs *devargs;
275 devargs = alloc_devargs(name, args);
279 dev = calloc(1, sizeof(*dev));
285 dev->device.bus = &rte_vdev_bus;
286 dev->device.numa_node = SOCKET_ID_ANY;
287 dev->device.name = devargs->name;
289 if (find_vdev(name)) {
291 * A vdev is expected to have only one port.
292 * So there is no reason to try probing again,
293 * even with new arguments.
300 rte_devargs_insert(&devargs);
301 dev->device.devargs = devargs;
302 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
309 rte_devargs_reset(devargs);
316 rte_vdev_init(const char *name, const char *args)
318 struct rte_vdev_device *dev;
321 rte_spinlock_recursive_lock(&vdev_device_list_lock);
322 ret = insert_vdev(name, args, &dev, true);
324 ret = vdev_probe_all_drivers(dev);
327 VDEV_LOG(ERR, "no driver found for %s", name);
328 /* If fails, remove it from vdev list */
329 TAILQ_REMOVE(&vdev_device_list, dev, next);
330 rte_devargs_remove(dev->device.devargs);
334 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
339 vdev_remove_driver(struct rte_vdev_device *dev)
341 const char *name = rte_vdev_device_name(dev);
342 const struct rte_vdev_driver *driver;
344 if (!dev->device.driver) {
345 VDEV_LOG(DEBUG, "no driver attach to device %s", name);
349 driver = container_of(dev->device.driver, const struct rte_vdev_driver,
351 return driver->remove(dev);
355 rte_vdev_uninit(const char *name)
357 struct rte_vdev_device *dev;
363 rte_spinlock_recursive_lock(&vdev_device_list_lock);
365 dev = find_vdev(name);
371 ret = vdev_remove_driver(dev);
375 TAILQ_REMOVE(&vdev_device_list, dev, next);
376 rte_devargs_remove(dev->device.devargs);
380 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
385 #define VDEV_SCAN_REQ 1
386 #define VDEV_SCAN_ONE 2
387 #define VDEV_SCAN_REP 3
390 char name[RTE_DEV_NAME_MAX_LEN];
393 static int vdev_plug(struct rte_device *dev);
396 * This function works as the action for both primary and secondary process
397 * for static vdev discovery when a secondary process is booting.
399 * step 1, secondary process sends a sync request to ask for vdev in primary;
400 * step 2, primary process receives the request, and send vdevs one by one;
401 * step 3, primary process sends back reply, which indicates how many vdevs
405 vdev_action(const struct rte_mp_msg *mp_msg, const void *peer)
407 struct rte_vdev_device *dev;
408 struct rte_mp_msg mp_resp;
409 struct vdev_param *ou = (struct vdev_param *)&mp_resp.param;
410 const struct vdev_param *in = (const struct vdev_param *)mp_msg->param;
415 strlcpy(mp_resp.name, VDEV_MP_KEY, sizeof(mp_resp.name));
416 mp_resp.len_param = sizeof(*ou);
421 ou->type = VDEV_SCAN_ONE;
425 rte_spinlock_recursive_lock(&vdev_device_list_lock);
426 TAILQ_FOREACH(dev, &vdev_device_list, next) {
427 devname = rte_vdev_device_name(dev);
428 if (strlen(devname) == 0) {
429 VDEV_LOG(INFO, "vdev with no name is not sent");
432 VDEV_LOG(INFO, "send vdev, %s", devname);
433 strlcpy(ou->name, devname, RTE_DEV_NAME_MAX_LEN);
434 if (rte_mp_sendmsg(&mp_resp) < 0)
435 VDEV_LOG(ERR, "send vdev, %s, failed, %s",
436 devname, strerror(rte_errno));
439 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
441 ou->type = VDEV_SCAN_REP;
443 if (rte_mp_reply(&mp_resp, peer) < 0)
444 VDEV_LOG(ERR, "Failed to reply a scan request");
447 VDEV_LOG(INFO, "receive vdev, %s", in->name);
448 ret = insert_vdev(in->name, NULL, NULL, false);
450 VDEV_LOG(DEBUG, "device already exist, %s", in->name);
452 VDEV_LOG(ERR, "failed to add vdev, %s", in->name);
455 VDEV_LOG(ERR, "vdev cannot recognize this message");
464 struct rte_vdev_device *dev;
465 struct rte_devargs *devargs;
466 struct vdev_custom_scan *custom_scan;
468 if (rte_mp_action_register(VDEV_MP_KEY, vdev_action) < 0 &&
469 rte_errno != EEXIST) {
470 /* for primary, unsupported IPC is not an error */
471 if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
472 rte_errno == ENOTSUP)
474 VDEV_LOG(ERR, "Failed to add vdev mp action");
478 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
479 struct rte_mp_msg mp_req, *mp_rep;
480 struct rte_mp_reply mp_reply;
481 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
482 struct vdev_param *req = (struct vdev_param *)mp_req.param;
483 struct vdev_param *resp;
485 strlcpy(mp_req.name, VDEV_MP_KEY, sizeof(mp_req.name));
486 mp_req.len_param = sizeof(*req);
488 req->type = VDEV_SCAN_REQ;
489 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
490 mp_reply.nb_received == 1) {
491 mp_rep = &mp_reply.msgs[0];
492 resp = (struct vdev_param *)mp_rep->param;
493 VDEV_LOG(INFO, "Received %d vdevs", resp->num);
496 VDEV_LOG(ERR, "Failed to request vdev from primary");
498 /* Fall through to allow private vdevs in secondary process */
502 /* call custom scan callbacks if any */
503 rte_spinlock_lock(&vdev_custom_scan_lock);
504 TAILQ_FOREACH(custom_scan, &vdev_custom_scans, next) {
505 if (custom_scan->callback != NULL)
507 * the callback should update devargs list
508 * by calling rte_devargs_insert() with
509 * devargs.bus = rte_bus_find_by_name("vdev");
510 * devargs.type = RTE_DEVTYPE_VIRTUAL;
511 * devargs.policy = RTE_DEV_ALLOWED;
513 custom_scan->callback(custom_scan->user_arg);
515 rte_spinlock_unlock(&vdev_custom_scan_lock);
517 /* for virtual devices we scan the devargs_list populated via cmdline */
518 RTE_EAL_DEVARGS_FOREACH("vdev", devargs) {
520 dev = calloc(1, sizeof(*dev));
524 rte_spinlock_recursive_lock(&vdev_device_list_lock);
526 if (find_vdev(devargs->name)) {
527 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
532 dev->device.bus = &rte_vdev_bus;
533 dev->device.devargs = devargs;
534 dev->device.numa_node = SOCKET_ID_ANY;
535 dev->device.name = devargs->name;
537 TAILQ_INSERT_TAIL(&vdev_device_list, dev, next);
539 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
548 struct rte_vdev_device *dev;
551 /* call the init function for each virtual device */
552 TAILQ_FOREACH(dev, &vdev_device_list, next) {
553 /* we don't use the vdev lock here, as it's only used in DPDK
554 * initialization; and we don't want to hold such a lock when
555 * we call each driver probe.
558 r = vdev_probe_all_drivers(dev);
562 VDEV_LOG(ERR, "failed to initialize %s device",
563 rte_vdev_device_name(dev));
572 rte_vdev_find_device(const struct rte_device *start, rte_dev_cmp_t cmp,
575 const struct rte_vdev_device *vstart;
576 struct rte_vdev_device *dev;
578 rte_spinlock_recursive_lock(&vdev_device_list_lock);
580 vstart = RTE_DEV_TO_VDEV_CONST(start);
581 dev = TAILQ_NEXT(vstart, next);
583 dev = TAILQ_FIRST(&vdev_device_list);
585 while (dev != NULL) {
586 if (cmp(&dev->device, data) == 0)
588 dev = TAILQ_NEXT(dev, next);
590 rte_spinlock_recursive_unlock(&vdev_device_list_lock);
592 return dev ? &dev->device : NULL;
596 vdev_plug(struct rte_device *dev)
598 return vdev_probe_all_drivers(RTE_DEV_TO_VDEV(dev));
602 vdev_unplug(struct rte_device *dev)
604 return rte_vdev_uninit(dev->name);
607 static enum rte_iova_mode
608 vdev_get_iommu_class(void)
611 struct rte_vdev_device *dev;
612 struct rte_vdev_driver *driver;
614 TAILQ_FOREACH(dev, &vdev_device_list, next) {
615 name = rte_vdev_device_name(dev);
616 if (vdev_parse(name, &driver))
619 if (driver->drv_flags & RTE_VDEV_DRV_NEED_IOVA_AS_VA)
626 static struct rte_bus rte_vdev_bus = {
629 .find_device = rte_vdev_find_device,
631 .unplug = vdev_unplug,
633 .dma_map = vdev_dma_map,
634 .dma_unmap = vdev_dma_unmap,
635 .get_iommu_class = vdev_get_iommu_class,
636 .dev_iterate = rte_vdev_dev_iterate,
639 RTE_REGISTER_BUS(vdev, rte_vdev_bus);
640 RTE_LOG_REGISTER_DEFAULT(vdev_logtype_bus, NOTICE);