1 /* SPDX-License-Identifier: BSD-3-Clause
3 * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved.
4 * Copyright 2016-2018 NXP
10 #include <sys/types.h>
15 #include <sys/ioctl.h>
21 #include <sys/eventfd.h>
23 #include <eal_filesystem.h>
25 #include <rte_ethdev_driver.h>
26 #include <rte_malloc.h>
27 #include <rte_memcpy.h>
28 #include <rte_string_fns.h>
29 #include <rte_cycles.h>
30 #include <rte_kvargs.h>
33 #include <rte_eal_memconfig.h>
35 #include "rte_fslmc.h"
36 #include "fslmc_vfio.h"
37 #include "fslmc_logs.h"
38 #include <mc/fsl_dpmng.h>
40 #include "portal/dpaa2_hw_pvt.h"
41 #include "portal/dpaa2_hw_dpio.h"
43 /** Pathname of FSL-MC devices directory. */
44 #define SYSFS_FSL_MC_DEVICES "/sys/bus/fsl-mc/devices"
46 #define FSLMC_CONTAINER_MAX_LEN 8 /**< Of the format dprc.XX */
48 /* Number of VFIO containers & groups with in */
49 static struct fslmc_vfio_group vfio_group;
50 static struct fslmc_vfio_container vfio_container;
51 static int container_device_fd;
52 static char *g_container;
53 static uint32_t *msi_intr_vaddr;
54 void *(*rte_mcp_ptr_list);
56 static struct rte_dpaa2_object_list dpaa2_obj_list =
57 TAILQ_HEAD_INITIALIZER(dpaa2_obj_list);
59 /*register a fslmc bus based dpaa2 driver */
61 rte_fslmc_object_register(struct rte_dpaa2_object *object)
65 TAILQ_INSERT_TAIL(&dpaa2_obj_list, object, next);
69 fslmc_get_container_group(int *groupid)
75 container = getenv("DPRC");
76 if (container == NULL) {
77 DPAA2_BUS_DEBUG("DPAA2: DPRC not available");
81 if (strlen(container) >= FSLMC_CONTAINER_MAX_LEN) {
82 DPAA2_BUS_ERR("Invalid container name: %s", container);
86 g_container = strdup(container);
88 DPAA2_BUS_ERR("Mem alloc failure; Container name");
93 /* get group number */
94 ret = rte_vfio_get_group_num(SYSFS_FSL_MC_DEVICES,
95 g_container, groupid);
97 DPAA2_BUS_ERR("Unable to find %s IOMMU group", g_container);
101 DPAA2_BUS_DEBUG("Container: %s has VFIO iommu group id = %d",
102 g_container, *groupid);
108 vfio_connect_container(void)
112 if (vfio_container.used) {
113 DPAA2_BUS_DEBUG("No container available");
117 /* Try connecting to vfio container if already created */
118 if (!ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER,
119 &vfio_container.fd)) {
121 "Container pre-exists with FD[0x%x] for this group",
123 vfio_group.container = &vfio_container;
127 /* Opens main vfio file descriptor which represents the "container" */
128 fd = rte_vfio_get_container_fd();
130 DPAA2_BUS_ERR("Failed to open VFIO container");
134 /* Check whether support for SMMU type IOMMU present or not */
135 if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
136 /* Connect group to container */
137 ret = ioctl(vfio_group.fd, VFIO_GROUP_SET_CONTAINER, &fd);
139 DPAA2_BUS_ERR("Failed to setup group container");
144 ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
146 DPAA2_BUS_ERR("Failed to setup VFIO iommu");
151 DPAA2_BUS_ERR("No supported IOMMU available");
156 vfio_container.used = 1;
157 vfio_container.fd = fd;
158 vfio_container.group = &vfio_group;
159 vfio_group.container = &vfio_container;
164 static int vfio_map_irq_region(struct fslmc_vfio_group *group)
167 unsigned long *vaddr = NULL;
168 struct vfio_iommu_type1_dma_map map = {
169 .argsz = sizeof(map),
170 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
176 vaddr = (unsigned long *)mmap(NULL, 0x1000, PROT_WRITE |
177 PROT_READ, MAP_SHARED, container_device_fd, 0x6030000);
178 if (vaddr == MAP_FAILED) {
179 DPAA2_BUS_INFO("Unable to map region (errno = %d)", errno);
183 msi_intr_vaddr = (uint32_t *)((char *)(vaddr) + 64);
184 map.vaddr = (unsigned long)vaddr;
185 ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &map);
189 DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)", errno);
193 static int fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
194 static int fslmc_unmap_dma(uint64_t vaddr, rte_iova_t iovaddr, size_t len);
197 fslmc_memevent_cb(enum rte_mem_event type, const void *addr, size_t len,
198 void *arg __rte_unused)
200 struct rte_memseg_list *msl;
201 struct rte_memseg *ms;
202 size_t cur_len = 0, map_len = 0;
204 rte_iova_t iova_addr;
207 msl = rte_mem_virt2memseg_list(addr);
209 while (cur_len < len) {
210 const void *va = RTE_PTR_ADD(addr, cur_len);
212 ms = rte_mem_virt2memseg(va, msl);
213 iova_addr = ms->iova;
214 virt_addr = ms->addr_64;
217 DPAA2_BUS_DEBUG("Request for %s, va=%p, "
218 "virt_addr=0x%" PRIx64 ", "
219 "iova=0x%" PRIx64 ", map_len=%zu",
220 type == RTE_MEM_EVENT_ALLOC ?
222 va, virt_addr, iova_addr, map_len);
224 /* iova_addr may be set to RTE_BAD_IOVA */
225 if (iova_addr == RTE_BAD_IOVA) {
226 DPAA2_BUS_DEBUG("Segment has invalid iova, skipping\n");
231 if (type == RTE_MEM_EVENT_ALLOC)
232 ret = fslmc_map_dma(virt_addr, iova_addr, map_len);
234 ret = fslmc_unmap_dma(virt_addr, iova_addr, map_len);
237 DPAA2_BUS_ERR("DMA Mapping/Unmapping failed. "
238 "Map=%d, addr=%p, len=%zu, err:(%d)",
239 type, va, map_len, ret);
246 if (type == RTE_MEM_EVENT_ALLOC)
247 DPAA2_BUS_DEBUG("Total Mapped: addr=%p, len=%zu",
250 DPAA2_BUS_DEBUG("Total Unmapped: addr=%p, len=%zu",
255 fslmc_map_dma(uint64_t vaddr, rte_iova_t iovaddr __rte_unused, size_t len)
257 struct fslmc_vfio_group *group;
258 struct vfio_iommu_type1_dma_map dma_map = {
259 .argsz = sizeof(struct vfio_iommu_type1_dma_map),
260 .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
265 dma_map.vaddr = vaddr;
267 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
268 dma_map.iova = iovaddr;
270 dma_map.iova = dma_map.vaddr;
273 /* SET DMA MAP for IOMMU */
276 if (!group->container) {
277 DPAA2_BUS_ERR("Container is not connected ");
281 DPAA2_BUS_DEBUG("--> Map address: 0x%"PRIx64", size: %"PRIu64"",
282 (uint64_t)dma_map.vaddr, (uint64_t)dma_map.size);
283 ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA, &dma_map);
285 DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
294 fslmc_unmap_dma(uint64_t vaddr, uint64_t iovaddr __rte_unused, size_t len)
296 struct fslmc_vfio_group *group;
297 struct vfio_iommu_type1_dma_unmap dma_unmap = {
298 .argsz = sizeof(struct vfio_iommu_type1_dma_unmap),
303 dma_unmap.size = len;
304 dma_unmap.iova = vaddr;
306 /* SET DMA MAP for IOMMU */
309 if (!group->container) {
310 DPAA2_BUS_ERR("Container is not connected ");
314 DPAA2_BUS_DEBUG("--> Unmap address: 0x%"PRIx64", size: %"PRIu64"",
315 (uint64_t)dma_unmap.iova, (uint64_t)dma_unmap.size);
316 ret = ioctl(group->container->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
318 DPAA2_BUS_ERR("VFIO_IOMMU_UNMAP_DMA API(errno = %d)",
327 fslmc_dmamap_seg(const struct rte_memseg_list *msl __rte_unused,
328 const struct rte_memseg *ms, void *arg)
333 /* if IOVA address is invalid, skip */
334 if (ms->iova == RTE_BAD_IOVA)
337 ret = fslmc_map_dma(ms->addr_64, ms->iova, ms->len);
339 DPAA2_BUS_ERR("Unable to VFIO map (addr=%p, len=%zu)",
347 int rte_fslmc_vfio_dmamap(void)
351 /* Lock before parsing and registering callback to memory subsystem */
352 rte_mcfg_mem_read_lock();
354 if (rte_memseg_walk(fslmc_dmamap_seg, &i) < 0) {
355 rte_mcfg_mem_read_unlock();
359 ret = rte_mem_event_callback_register("fslmc_memevent_clb",
360 fslmc_memevent_cb, NULL);
361 if (ret && rte_errno == ENOTSUP)
362 DPAA2_BUS_DEBUG("Memory event callbacks not supported");
364 DPAA2_BUS_DEBUG("Unable to install memory handler");
366 DPAA2_BUS_DEBUG("Installed memory callback handler");
368 DPAA2_BUS_DEBUG("Total %d segments found.", i);
370 /* TODO - This is a W.A. as VFIO currently does not add the mapping of
371 * the interrupt region to SMMU. This should be removed once the
372 * support is added in the Kernel.
374 vfio_map_irq_region(&vfio_group);
376 /* Existing segments have been mapped and memory callback for hotplug
377 * has been installed.
379 rte_mcfg_mem_read_unlock();
384 static int64_t vfio_map_mcp_obj(struct fslmc_vfio_group *group, char *mcp_obj)
386 intptr_t v_addr = (intptr_t)MAP_FAILED;
389 struct vfio_device_info d_info = { .argsz = sizeof(d_info) };
390 struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
392 /* getting the mcp object's fd*/
393 mc_fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, mcp_obj);
395 DPAA2_BUS_ERR("Error in VFIO get dev %s fd from group %d",
400 /* getting device info*/
401 ret = ioctl(mc_fd, VFIO_DEVICE_GET_INFO, &d_info);
403 DPAA2_BUS_ERR("Error in VFIO getting DEVICE_INFO");
407 /* getting device region info*/
408 ret = ioctl(mc_fd, VFIO_DEVICE_GET_REGION_INFO, ®_info);
410 DPAA2_BUS_ERR("Error in VFIO getting REGION_INFO");
414 v_addr = (size_t)mmap(NULL, reg_info.size,
415 PROT_WRITE | PROT_READ, MAP_SHARED,
416 mc_fd, reg_info.offset);
424 #define IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + sizeof(int))
426 int rte_dpaa2_intr_enable(struct rte_intr_handle *intr_handle, int index)
429 char irq_set_buf[IRQ_SET_BUF_LEN];
430 struct vfio_irq_set *irq_set;
433 len = sizeof(irq_set_buf);
435 irq_set = (struct vfio_irq_set *)irq_set_buf;
436 irq_set->argsz = len;
439 VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
440 irq_set->index = index;
442 fd_ptr = (int *)&irq_set->data;
443 *fd_ptr = intr_handle->fd;
445 ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
447 DPAA2_BUS_ERR("Error:dpaa2 SET IRQs fd=%d, err = %d(%s)",
448 intr_handle->fd, errno, strerror(errno));
455 int rte_dpaa2_intr_disable(struct rte_intr_handle *intr_handle, int index)
457 struct vfio_irq_set *irq_set;
458 char irq_set_buf[IRQ_SET_BUF_LEN];
461 len = sizeof(struct vfio_irq_set);
463 irq_set = (struct vfio_irq_set *)irq_set_buf;
464 irq_set->argsz = len;
465 irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
466 irq_set->index = index;
470 ret = ioctl(intr_handle->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
473 "Error disabling dpaa2 interrupts for fd %d",
479 /* set up interrupt support (but not enable interrupts) */
481 rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,
487 /* start from MSI-X interrupt type */
488 for (i = 0; i < num_irqs; i++) {
489 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
494 ret = ioctl(vfio_dev_fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
496 DPAA2_BUS_ERR("Cannot get IRQ(%d) info, error %i (%s)",
497 i, errno, strerror(errno));
501 /* if this vector cannot be used with eventfd,
502 * fail if we explicitly
503 * specified interrupt type, otherwise continue
505 if ((irq_info.flags & VFIO_IRQ_INFO_EVENTFD) == 0)
508 /* set up an eventfd for interrupts */
509 fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
511 DPAA2_BUS_ERR("Cannot set up eventfd, error %i (%s)",
512 errno, strerror(errno));
516 intr_handle->fd = fd;
517 intr_handle->type = RTE_INTR_HANDLE_VFIO_MSI;
518 intr_handle->vfio_dev_fd = vfio_dev_fd;
523 /* if we're here, we haven't found a suitable interrupt vector */
528 * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
532 fslmc_process_iodevices(struct rte_dpaa2_device *dev)
535 struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
536 struct rte_dpaa2_object *object = NULL;
538 dev_fd = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD,
541 DPAA2_BUS_ERR("Unable to obtain device FD for device:%s",
546 if (ioctl(dev_fd, VFIO_DEVICE_GET_INFO, &device_info)) {
547 DPAA2_BUS_ERR("Unable to obtain information for device:%s",
552 switch (dev->dev_type) {
554 rte_dpaa2_vfio_setup_intr(&dev->intr_handle, dev_fd,
555 device_info.num_irqs);
562 TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
563 if (dev->dev_type == object->dev_type)
564 object->create(dev_fd, &device_info,
574 DPAA2_BUS_LOG(DEBUG, "Device (%s) abstracted from VFIO",
580 fslmc_process_mcp(struct rte_dpaa2_device *dev)
584 char *dev_name = NULL;
585 struct fsl_mc_io dpmng = {0};
586 struct mc_version mc_ver_info = {0};
588 rte_mcp_ptr_list = malloc(sizeof(void *) * 1);
589 if (!rte_mcp_ptr_list) {
590 DPAA2_BUS_ERR("Unable to allocate MC portal memory");
595 dev_name = strdup(dev->device.name);
597 DPAA2_BUS_ERR("Unable to allocate MC device name memory");
602 v_addr = vfio_map_mcp_obj(&vfio_group, dev_name);
603 if (v_addr == (intptr_t)MAP_FAILED) {
604 DPAA2_BUS_ERR("Error mapping region (errno = %d)", errno);
609 /* check the MC version compatibility */
610 dpmng.regs = (void *)v_addr;
612 /* In case of secondary processes, MC version check is no longer
615 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
616 rte_mcp_ptr_list[0] = (void *)v_addr;
620 if (mc_get_version(&dpmng, CMD_PRI_LOW, &mc_ver_info)) {
621 DPAA2_BUS_ERR("Unable to obtain MC version");
626 if ((mc_ver_info.major != MC_VER_MAJOR) ||
627 (mc_ver_info.minor < MC_VER_MINOR)) {
628 DPAA2_BUS_ERR("DPAA2 MC version not compatible!"
629 " Expected %d.%d.x, Detected %d.%d.%d",
630 MC_VER_MAJOR, MC_VER_MINOR,
631 mc_ver_info.major, mc_ver_info.minor,
632 mc_ver_info.revision);
636 rte_mcp_ptr_list[0] = (void *)v_addr;
645 if (rte_mcp_ptr_list) {
646 free(rte_mcp_ptr_list);
647 rte_mcp_ptr_list = NULL;
654 fslmc_vfio_process_group(void)
657 int found_mportal = 0;
658 struct rte_dpaa2_device *dev, *dev_temp;
660 /* Search the MCP as that should be initialized first. */
661 TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
662 if (dev->dev_type == DPAA2_MPORTAL) {
663 if (dev->device.devargs &&
664 dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
665 DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
667 TAILQ_REMOVE(&rte_fslmc_bus.device_list,
672 ret = fslmc_process_mcp(dev);
674 DPAA2_BUS_ERR("Unable to map MC Portal");
680 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
683 /* Ideally there is only a single dpmcp, but in case
684 * multiple exists, looping on remaining devices.
689 /* Cannot continue if there is not even a single mportal */
690 if (!found_mportal) {
691 DPAA2_BUS_ERR("No MC Portal device found. Not continuing");
695 TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
696 if (dev->device.devargs &&
697 dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
698 DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
700 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
703 switch (dev->dev_type) {
707 ret = fslmc_process_iodevices(dev);
709 DPAA2_BUS_DEBUG("Dev (%s) init failed",
718 /* IN case of secondary processes, all control objects
719 * like dpbp, dpcon, dpci are not initialized/required
720 * - all of these are assumed to be initialized and made
721 * available by primary.
723 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
726 /* Call the object creation routine and remove the
727 * device entry from device list
729 ret = fslmc_process_iodevices(dev);
731 DPAA2_BUS_DEBUG("Dev (%s) init failed",
738 ret = fslmc_process_iodevices(dev);
740 DPAA2_BUS_DEBUG("Dev (%s) init failed",
748 /* Unknown - ignore */
749 DPAA2_BUS_DEBUG("Found unknown device (%s)",
751 TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
761 fslmc_vfio_setup_group(void)
765 struct vfio_group_status status = { .argsz = sizeof(status) };
767 /* if already done once */
768 if (container_device_fd)
771 ret = fslmc_get_container_group(&groupid);
775 /* In case this group was already opened, continue without any
778 if (vfio_group.groupid == groupid) {
779 DPAA2_BUS_ERR("groupid already exists %d", groupid);
783 /* Get the actual group fd */
784 ret = rte_vfio_get_group_fd(groupid);
789 /* Check group viability */
790 ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_STATUS, &status);
792 DPAA2_BUS_ERR("VFIO error getting group status");
793 close(vfio_group.fd);
794 rte_vfio_clear_group(vfio_group.fd);
798 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
799 DPAA2_BUS_ERR("VFIO group not viable");
800 close(vfio_group.fd);
801 rte_vfio_clear_group(vfio_group.fd);
804 /* Since Group is VIABLE, Store the groupid */
805 vfio_group.groupid = groupid;
807 /* check if group does not have a container yet */
808 if (!(status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET)) {
809 /* Now connect this IOMMU group to given container */
810 ret = vfio_connect_container();
813 "Error connecting container with groupid %d",
815 close(vfio_group.fd);
816 rte_vfio_clear_group(vfio_group.fd);
821 /* Get Device information */
822 ret = ioctl(vfio_group.fd, VFIO_GROUP_GET_DEVICE_FD, g_container);
824 DPAA2_BUS_ERR("Error getting device %s fd from group %d",
825 g_container, vfio_group.groupid);
826 close(vfio_group.fd);
827 rte_vfio_clear_group(vfio_group.fd);
830 container_device_fd = ret;
831 DPAA2_BUS_DEBUG("VFIO Container FD is [0x%X]",
832 container_device_fd);