1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2018 Intel Corporation
11 #include <rte_errno.h>
13 #include <rte_memory.h>
14 #include <rte_eal_memconfig.h>
17 #include "eal_filesystem.h"
18 #include "eal_memcfg.h"
20 #include "eal_private.h"
21 #include "eal_internal_cfg.h"
23 #define VFIO_MEM_EVENT_CLB_NAME "vfio_mem_event_clb"
25 /* hot plug/unplug of VFIO groups may cause all DMA maps to be dropped. we can
26 * recreate the mappings for DPDK segments, but we cannot do so for memory that
27 * was registered by the user themselves, so we need to store the user mappings
28 * somewhere, to recreate them later.
30 #define VFIO_MAX_USER_MEM_MAPS 256
32 uint64_t addr; /**< start VA */
33 uint64_t iova; /**< start IOVA */
34 uint64_t len; /**< total length of the mapping */
35 uint64_t chunk; /**< this mapping can be split in chunks of this size */
38 struct user_mem_maps {
39 rte_spinlock_recursive_t lock;
41 struct user_mem_map maps[VFIO_MAX_USER_MEM_MAPS];
46 int vfio_container_fd;
47 int vfio_active_groups;
48 const struct vfio_iommu_type *vfio_iommu_type;
49 struct vfio_group vfio_groups[VFIO_MAX_GROUPS];
50 struct user_mem_maps mem_maps;
53 /* per-process VFIO config */
54 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
55 static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
57 static int vfio_type1_dma_map(int);
58 static int vfio_type1_dma_mem_map(int, uint64_t, uint64_t, uint64_t, int);
59 static int vfio_spapr_dma_map(int);
60 static int vfio_spapr_dma_mem_map(int, uint64_t, uint64_t, uint64_t, int);
61 static int vfio_noiommu_dma_map(int);
62 static int vfio_noiommu_dma_mem_map(int, uint64_t, uint64_t, uint64_t, int);
63 static int vfio_dma_mem_map(struct vfio_config *vfio_cfg, uint64_t vaddr,
64 uint64_t iova, uint64_t len, int do_map);
66 /* IOMMU types we support */
67 static const struct vfio_iommu_type iommu_types[] = {
68 /* x86 IOMMU, otherwise known as type 1 */
70 .type_id = RTE_VFIO_TYPE1,
72 .partial_unmap = false,
73 .dma_map_func = &vfio_type1_dma_map,
74 .dma_user_map_func = &vfio_type1_dma_mem_map
76 /* ppc64 IOMMU, otherwise known as spapr */
78 .type_id = RTE_VFIO_SPAPR,
80 .partial_unmap = true,
81 .dma_map_func = &vfio_spapr_dma_map,
82 .dma_user_map_func = &vfio_spapr_dma_mem_map
86 .type_id = RTE_VFIO_NOIOMMU,
88 .partial_unmap = true,
89 .dma_map_func = &vfio_noiommu_dma_map,
90 .dma_user_map_func = &vfio_noiommu_dma_mem_map
95 is_null_map(const struct user_mem_map *map)
97 return map->addr == 0 && map->iova == 0 &&
98 map->len == 0 && map->chunk == 0;
101 /* we may need to merge user mem maps together in case of user mapping/unmapping
102 * chunks of memory, so we'll need a comparator function to sort segments.
105 user_mem_map_cmp(const void *a, const void *b)
107 const struct user_mem_map *umm_a = a;
108 const struct user_mem_map *umm_b = b;
110 /* move null entries to end */
111 if (is_null_map(umm_a))
113 if (is_null_map(umm_b))
116 /* sort by iova first */
117 if (umm_a->iova < umm_b->iova)
119 if (umm_a->iova > umm_b->iova)
122 if (umm_a->addr < umm_b->addr)
124 if (umm_a->addr > umm_b->addr)
127 if (umm_a->len < umm_b->len)
129 if (umm_a->len > umm_b->len)
132 if (umm_a->chunk < umm_b->chunk)
134 if (umm_a->chunk > umm_b->chunk)
141 * Take in an address range and list of current mappings, and produce a list of
142 * mappings that will be kept.
145 process_maps(struct user_mem_map *src, size_t src_len,
146 struct user_mem_map newmap[2], uint64_t vaddr, uint64_t len)
148 struct user_mem_map *src_first = &src[0];
149 struct user_mem_map *src_last = &src[src_len - 1];
150 struct user_mem_map *dst_first = &newmap[0];
151 /* we can get at most two new segments */
152 struct user_mem_map *dst_last = &newmap[1];
153 uint64_t first_off = vaddr - src_first->addr;
154 uint64_t last_off = (src_last->addr + src_last->len) - (vaddr + len);
157 if (first_off != 0) {
158 dst_first->addr = src_first->addr;
159 dst_first->iova = src_first->iova;
160 dst_first->len = first_off;
161 dst_first->chunk = src_first->chunk;
166 /* if we had start offset, we have two segments */
167 struct user_mem_map *last =
168 first_off == 0 ? dst_first : dst_last;
169 last->addr = (src_last->addr + src_last->len) - last_off;
170 last->iova = (src_last->iova + src_last->len) - last_off;
171 last->len = last_off;
172 last->chunk = src_last->chunk;
179 /* erase certain maps from the list */
181 delete_maps(struct user_mem_maps *user_mem_maps, struct user_mem_map *del_maps,
187 for (i = 0, j = 0; i < VFIO_MAX_USER_MEM_MAPS && j < n_del; i++) {
188 struct user_mem_map *left = &user_mem_maps->maps[i];
189 struct user_mem_map *right = &del_maps[j];
191 if (user_mem_map_cmp(left, right) == 0) {
192 memset(left, 0, sizeof(*left));
194 user_mem_maps->n_maps--;
200 copy_maps(struct user_mem_maps *user_mem_maps, struct user_mem_map *add_maps,
206 for (i = 0, j = 0; i < VFIO_MAX_USER_MEM_MAPS && j < n_add; i++) {
207 struct user_mem_map *left = &user_mem_maps->maps[i];
208 struct user_mem_map *right = &add_maps[j];
210 /* insert into empty space */
211 if (is_null_map(left)) {
212 memcpy(left, right, sizeof(*left));
214 user_mem_maps->n_maps++;
219 /* try merging two maps into one, return 1 if succeeded */
221 merge_map(struct user_mem_map *left, struct user_mem_map *right)
223 /* merge the same maps into one */
224 if (memcmp(left, right, sizeof(struct user_mem_map)) == 0)
227 if (left->addr + left->len != right->addr)
229 if (left->iova + left->len != right->iova)
231 if (left->chunk != right->chunk)
233 left->len += right->len;
236 memset(right, 0, sizeof(*right));
242 addr_is_chunk_aligned(struct user_mem_map *maps, size_t n_maps,
243 uint64_t vaddr, uint64_t iova)
247 for (i = 0; i < n_maps; i++) {
248 struct user_mem_map *map = &maps[i];
249 uint64_t map_va_end = map->addr + map->len;
250 uint64_t map_iova_end = map->iova + map->len;
251 uint64_t map_va_off = vaddr - map->addr;
252 uint64_t map_iova_off = iova - map->iova;
254 /* we include end of the segment in comparison as well */
255 bool addr_in_map = (vaddr >= map->addr) && (vaddr <= map_va_end);
256 bool iova_in_map = (iova >= map->iova) && (iova <= map_iova_end);
257 /* chunk may not be power of two, so use modulo */
258 bool addr_is_aligned = (map_va_off % map->chunk) == 0;
259 bool iova_is_aligned = (map_iova_off % map->chunk) == 0;
261 if (addr_in_map && iova_in_map &&
262 addr_is_aligned && iova_is_aligned)
269 find_user_mem_maps(struct user_mem_maps *user_mem_maps, uint64_t addr,
270 uint64_t iova, uint64_t len, struct user_mem_map *dst,
273 uint64_t va_end = addr + len;
274 uint64_t iova_end = iova + len;
279 for (i = 0, j = 0; i < user_mem_maps->n_maps; i++) {
280 struct user_mem_map *map = &user_mem_maps->maps[i];
281 uint64_t map_va_end = map->addr + map->len;
282 uint64_t map_iova_end = map->iova + map->len;
284 bool start_addr_in_map = (addr >= map->addr) &&
286 bool end_addr_in_map = (va_end > map->addr) &&
287 (va_end <= map_va_end);
288 bool start_iova_in_map = (iova >= map->iova) &&
289 (iova < map_iova_end);
290 bool end_iova_in_map = (iova_end > map->iova) &&
291 (iova_end <= map_iova_end);
293 /* do we have space in temporary map? */
298 /* check if current map is start of our segment */
299 if (!found && start_addr_in_map && start_iova_in_map)
301 /* if we have previously found a segment, add it to the map */
303 /* copy the segment into our temporary map */
304 memcpy(&dst[j++], map, sizeof(*map));
306 /* if we match end of segment, quit */
307 if (end_addr_in_map && end_iova_in_map)
311 /* we didn't find anything */
314 memset(dst, 0, sizeof(*dst) * dst_len);
318 /* this will sort all user maps, and merge/compact any adjacent maps */
320 compact_user_maps(struct user_mem_maps *user_mem_maps)
324 qsort(user_mem_maps->maps, VFIO_MAX_USER_MEM_MAPS,
325 sizeof(user_mem_maps->maps[0]), user_mem_map_cmp);
327 /* we'll go over the list backwards when merging */
328 for (i = VFIO_MAX_USER_MEM_MAPS - 2; i >= 0; i--) {
329 struct user_mem_map *l, *r;
331 l = &user_mem_maps->maps[i];
332 r = &user_mem_maps->maps[i + 1];
334 if (is_null_map(l) || is_null_map(r))
337 /* try and merge the maps */
339 user_mem_maps->n_maps--;
342 /* the entries are still sorted, but now they have holes in them, so
343 * sort the list again.
345 qsort(user_mem_maps->maps, VFIO_MAX_USER_MEM_MAPS,
346 sizeof(user_mem_maps->maps[0]), user_mem_map_cmp);
350 vfio_open_group_fd(int iommu_group_num)
353 char filename[PATH_MAX];
354 struct rte_mp_msg mp_req, *mp_rep;
355 struct rte_mp_reply mp_reply = {0};
356 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
357 struct vfio_mp_param *p = (struct vfio_mp_param *)mp_req.param;
358 const struct internal_config *internal_conf =
359 eal_get_internal_configuration();
361 /* if primary, try to open the group */
362 if (internal_conf->process_type == RTE_PROC_PRIMARY) {
363 /* try regular group format */
364 snprintf(filename, sizeof(filename),
365 VFIO_GROUP_FMT, iommu_group_num);
366 vfio_group_fd = open(filename, O_RDWR);
367 if (vfio_group_fd < 0) {
368 /* if file not found, it's not an error */
369 if (errno != ENOENT) {
370 RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
371 filename, strerror(errno));
375 /* special case: try no-IOMMU path as well */
376 snprintf(filename, sizeof(filename),
377 VFIO_NOIOMMU_GROUP_FMT,
379 vfio_group_fd = open(filename, O_RDWR);
380 if (vfio_group_fd < 0) {
381 if (errno != ENOENT) {
383 "Cannot open %s: %s\n",
384 filename, strerror(errno));
389 /* noiommu group found */
392 return vfio_group_fd;
394 /* if we're in a secondary process, request group fd from the primary
395 * process via mp channel.
397 p->req = SOCKET_REQ_GROUP;
398 p->group_num = iommu_group_num;
399 strcpy(mp_req.name, EAL_VFIO_MP);
400 mp_req.len_param = sizeof(*p);
404 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
405 mp_reply.nb_received == 1) {
406 mp_rep = &mp_reply.msgs[0];
407 p = (struct vfio_mp_param *)mp_rep->param;
408 if (p->result == SOCKET_OK && mp_rep->num_fds == 1) {
409 vfio_group_fd = mp_rep->fds[0];
410 } else if (p->result == SOCKET_NO_FD) {
411 RTE_LOG(ERR, EAL, "Bad VFIO group fd\n");
412 vfio_group_fd = -ENOENT;
417 if (vfio_group_fd < 0 && vfio_group_fd != -ENOENT)
418 RTE_LOG(ERR, EAL, "Cannot request VFIO group fd\n");
419 return vfio_group_fd;
422 static struct vfio_config *
423 get_vfio_cfg_by_group_num(int iommu_group_num)
425 struct vfio_config *vfio_cfg;
428 for (i = 0; i < VFIO_MAX_CONTAINERS; i++) {
429 vfio_cfg = &vfio_cfgs[i];
430 for (j = 0; j < VFIO_MAX_GROUPS; j++) {
431 if (vfio_cfg->vfio_groups[j].group_num ==
441 vfio_get_group_fd(struct vfio_config *vfio_cfg,
446 struct vfio_group *cur_grp;
448 /* check if we already have the group descriptor open */
449 for (i = 0; i < VFIO_MAX_GROUPS; i++)
450 if (vfio_cfg->vfio_groups[i].group_num == iommu_group_num)
451 return vfio_cfg->vfio_groups[i].fd;
453 /* Lets see first if there is room for a new group */
454 if (vfio_cfg->vfio_active_groups == VFIO_MAX_GROUPS) {
455 RTE_LOG(ERR, EAL, "Maximum number of VFIO groups reached!\n");
459 /* Now lets get an index for the new group */
460 for (i = 0; i < VFIO_MAX_GROUPS; i++)
461 if (vfio_cfg->vfio_groups[i].group_num == -1) {
462 cur_grp = &vfio_cfg->vfio_groups[i];
466 /* This should not happen */
467 if (i == VFIO_MAX_GROUPS) {
468 RTE_LOG(ERR, EAL, "No VFIO group free slot found\n");
472 vfio_group_fd = vfio_open_group_fd(iommu_group_num);
473 if (vfio_group_fd < 0) {
474 RTE_LOG(ERR, EAL, "Failed to open VFIO group %d\n",
476 return vfio_group_fd;
479 cur_grp->group_num = iommu_group_num;
480 cur_grp->fd = vfio_group_fd;
481 vfio_cfg->vfio_active_groups++;
483 return vfio_group_fd;
486 static struct vfio_config *
487 get_vfio_cfg_by_group_fd(int vfio_group_fd)
489 struct vfio_config *vfio_cfg;
492 for (i = 0; i < VFIO_MAX_CONTAINERS; i++) {
493 vfio_cfg = &vfio_cfgs[i];
494 for (j = 0; j < VFIO_MAX_GROUPS; j++)
495 if (vfio_cfg->vfio_groups[j].fd == vfio_group_fd)
502 static struct vfio_config *
503 get_vfio_cfg_by_container_fd(int container_fd)
507 if (container_fd == RTE_VFIO_DEFAULT_CONTAINER_FD)
508 return default_vfio_cfg;
510 for (i = 0; i < VFIO_MAX_CONTAINERS; i++) {
511 if (vfio_cfgs[i].vfio_container_fd == container_fd)
512 return &vfio_cfgs[i];
519 rte_vfio_get_group_fd(int iommu_group_num)
521 struct vfio_config *vfio_cfg;
523 /* get the vfio_config it belongs to */
524 vfio_cfg = get_vfio_cfg_by_group_num(iommu_group_num);
525 vfio_cfg = vfio_cfg ? vfio_cfg : default_vfio_cfg;
527 return vfio_get_group_fd(vfio_cfg, iommu_group_num);
531 get_vfio_group_idx(int vfio_group_fd)
533 struct vfio_config *vfio_cfg;
536 for (i = 0; i < VFIO_MAX_CONTAINERS; i++) {
537 vfio_cfg = &vfio_cfgs[i];
538 for (j = 0; j < VFIO_MAX_GROUPS; j++)
539 if (vfio_cfg->vfio_groups[j].fd == vfio_group_fd)
547 vfio_group_device_get(int vfio_group_fd)
549 struct vfio_config *vfio_cfg;
552 vfio_cfg = get_vfio_cfg_by_group_fd(vfio_group_fd);
553 if (vfio_cfg == NULL) {
554 RTE_LOG(ERR, EAL, "Invalid VFIO group fd!\n");
558 i = get_vfio_group_idx(vfio_group_fd);
559 if (i < 0 || i > (VFIO_MAX_GROUPS - 1))
560 RTE_LOG(ERR, EAL, "Wrong VFIO group index (%d)\n", i);
562 vfio_cfg->vfio_groups[i].devices++;
566 vfio_group_device_put(int vfio_group_fd)
568 struct vfio_config *vfio_cfg;
571 vfio_cfg = get_vfio_cfg_by_group_fd(vfio_group_fd);
572 if (vfio_cfg == NULL) {
573 RTE_LOG(ERR, EAL, "Invalid VFIO group fd!\n");
577 i = get_vfio_group_idx(vfio_group_fd);
578 if (i < 0 || i > (VFIO_MAX_GROUPS - 1))
579 RTE_LOG(ERR, EAL, "Wrong VFIO group index (%d)\n", i);
581 vfio_cfg->vfio_groups[i].devices--;
585 vfio_group_device_count(int vfio_group_fd)
587 struct vfio_config *vfio_cfg;
590 vfio_cfg = get_vfio_cfg_by_group_fd(vfio_group_fd);
591 if (vfio_cfg == NULL) {
592 RTE_LOG(ERR, EAL, "Invalid VFIO group fd!\n");
596 i = get_vfio_group_idx(vfio_group_fd);
597 if (i < 0 || i > (VFIO_MAX_GROUPS - 1)) {
598 RTE_LOG(ERR, EAL, "Wrong VFIO group index (%d)\n", i);
602 return vfio_cfg->vfio_groups[i].devices;
606 vfio_mem_event_callback(enum rte_mem_event type, const void *addr, size_t len,
607 void *arg __rte_unused)
609 struct rte_memseg_list *msl;
610 struct rte_memseg *ms;
613 msl = rte_mem_virt2memseg_list(addr);
615 /* for IOVA as VA mode, no need to care for IOVA addresses */
616 if (rte_eal_iova_mode() == RTE_IOVA_VA && msl->external == 0) {
617 uint64_t vfio_va = (uint64_t)(uintptr_t)addr;
618 uint64_t page_sz = msl->page_sz;
620 /* Maintain granularity of DMA map/unmap to memseg size */
621 for (; cur_len < len; cur_len += page_sz) {
622 if (type == RTE_MEM_EVENT_ALLOC)
623 vfio_dma_mem_map(default_vfio_cfg, vfio_va,
624 vfio_va, page_sz, 1);
626 vfio_dma_mem_map(default_vfio_cfg, vfio_va,
627 vfio_va, page_sz, 0);
634 /* memsegs are contiguous in memory */
635 ms = rte_mem_virt2memseg(addr, msl);
636 while (cur_len < len) {
637 /* some memory segments may have invalid IOVA */
638 if (ms->iova == RTE_BAD_IOVA) {
640 "Memory segment at %p has bad IOVA, skipping\n",
644 if (type == RTE_MEM_EVENT_ALLOC)
645 vfio_dma_mem_map(default_vfio_cfg, ms->addr_64,
646 ms->iova, ms->len, 1);
648 vfio_dma_mem_map(default_vfio_cfg, ms->addr_64,
649 ms->iova, ms->len, 0);
657 vfio_sync_default_container(void)
659 struct rte_mp_msg mp_req, *mp_rep;
660 struct rte_mp_reply mp_reply = {0};
661 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
662 struct vfio_mp_param *p = (struct vfio_mp_param *)mp_req.param;
666 /* cannot be called from primary */
667 if (rte_eal_process_type() != RTE_PROC_SECONDARY)
670 /* default container fd should have been opened in rte_vfio_enable() */
671 if (!default_vfio_cfg->vfio_enabled ||
672 default_vfio_cfg->vfio_container_fd < 0) {
673 RTE_LOG(ERR, EAL, "VFIO support is not initialized\n");
677 /* find default container's IOMMU type */
678 p->req = SOCKET_REQ_IOMMU_TYPE;
679 strcpy(mp_req.name, EAL_VFIO_MP);
680 mp_req.len_param = sizeof(*p);
684 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
685 mp_reply.nb_received == 1) {
686 mp_rep = &mp_reply.msgs[0];
687 p = (struct vfio_mp_param *)mp_rep->param;
688 if (p->result == SOCKET_OK)
689 iommu_type_id = p->iommu_type_id;
692 if (iommu_type_id < 0) {
694 "Could not get IOMMU type for default container\n");
698 /* we now have an fd for default container, as well as its IOMMU type.
699 * now, set up default VFIO container config to match.
701 for (i = 0; i < RTE_DIM(iommu_types); i++) {
702 const struct vfio_iommu_type *t = &iommu_types[i];
703 if (t->type_id != iommu_type_id)
706 /* we found our IOMMU type */
707 default_vfio_cfg->vfio_iommu_type = t;
711 RTE_LOG(ERR, EAL, "Could not find IOMMU type id (%i)\n",
717 rte_vfio_clear_group(int vfio_group_fd)
720 struct vfio_config *vfio_cfg;
722 vfio_cfg = get_vfio_cfg_by_group_fd(vfio_group_fd);
723 if (vfio_cfg == NULL) {
724 RTE_LOG(ERR, EAL, "Invalid VFIO group fd!\n");
728 i = get_vfio_group_idx(vfio_group_fd);
731 vfio_cfg->vfio_groups[i].group_num = -1;
732 vfio_cfg->vfio_groups[i].fd = -1;
733 vfio_cfg->vfio_groups[i].devices = 0;
734 vfio_cfg->vfio_active_groups--;
740 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
741 int *vfio_dev_fd, struct vfio_device_info *device_info)
743 struct vfio_group_status group_status = {
744 .argsz = sizeof(group_status)
746 struct vfio_config *vfio_cfg;
747 struct user_mem_maps *user_mem_maps;
748 int vfio_container_fd;
753 const struct internal_config *internal_conf =
754 eal_get_internal_configuration();
756 /* get group number */
757 ret = rte_vfio_get_group_num(sysfs_base, dev_addr, &iommu_group_num);
760 "%s not managed by VFIO driver, skipping\n",
765 /* if negative, something failed */
769 /* get the actual group fd */
770 vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
771 if (vfio_group_fd < 0 && vfio_group_fd != -ENOENT)
775 * if vfio_group_fd == -ENOENT, that means the device
776 * isn't managed by VFIO
778 if (vfio_group_fd == -ENOENT) {
780 "%s not managed by VFIO driver, skipping\n",
786 * at this point, we know that this group is viable (meaning, all devices
787 * are either bound to VFIO or not bound to anything)
790 /* check if the group is viable */
791 ret = ioctl(vfio_group_fd, VFIO_GROUP_GET_STATUS, &group_status);
793 RTE_LOG(ERR, EAL, "%s cannot get VFIO group status, "
794 "error %i (%s)\n", dev_addr, errno, strerror(errno));
795 close(vfio_group_fd);
796 rte_vfio_clear_group(vfio_group_fd);
798 } else if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
799 RTE_LOG(ERR, EAL, "%s VFIO group is not viable! "
800 "Not all devices in IOMMU group bound to VFIO or unbound\n",
802 close(vfio_group_fd);
803 rte_vfio_clear_group(vfio_group_fd);
807 /* get the vfio_config it belongs to */
808 vfio_cfg = get_vfio_cfg_by_group_num(iommu_group_num);
809 vfio_cfg = vfio_cfg ? vfio_cfg : default_vfio_cfg;
810 vfio_container_fd = vfio_cfg->vfio_container_fd;
811 user_mem_maps = &vfio_cfg->mem_maps;
813 /* check if group does not have a container yet */
814 if (!(group_status.flags & VFIO_GROUP_FLAGS_CONTAINER_SET)) {
816 /* add group to a container */
817 ret = ioctl(vfio_group_fd, VFIO_GROUP_SET_CONTAINER,
821 "%s cannot add VFIO group to container, error "
822 "%i (%s)\n", dev_addr, errno, strerror(errno));
823 close(vfio_group_fd);
824 rte_vfio_clear_group(vfio_group_fd);
829 * pick an IOMMU type and set up DMA mappings for container
831 * needs to be done only once, only when first group is
832 * assigned to a container and only in primary process.
833 * Note this can happen several times with the hotplug
836 if (internal_conf->process_type == RTE_PROC_PRIMARY &&
837 vfio_cfg->vfio_active_groups == 1 &&
838 vfio_group_device_count(vfio_group_fd) == 0) {
839 const struct vfio_iommu_type *t;
841 /* select an IOMMU type which we will be using */
842 t = vfio_set_iommu_type(vfio_container_fd);
845 "%s failed to select IOMMU type\n",
847 close(vfio_group_fd);
848 rte_vfio_clear_group(vfio_group_fd);
851 /* lock memory hotplug before mapping and release it
852 * after registering callback, to prevent races
854 rte_mcfg_mem_read_lock();
855 if (vfio_cfg == default_vfio_cfg)
856 ret = t->dma_map_func(vfio_container_fd);
861 "%s DMA remapping failed, error "
863 dev_addr, errno, strerror(errno));
864 close(vfio_group_fd);
865 rte_vfio_clear_group(vfio_group_fd);
866 rte_mcfg_mem_read_unlock();
870 vfio_cfg->vfio_iommu_type = t;
872 /* re-map all user-mapped segments */
873 rte_spinlock_recursive_lock(&user_mem_maps->lock);
875 /* this IOMMU type may not support DMA mapping, but
876 * if we have mappings in the list - that means we have
877 * previously mapped something successfully, so we can
878 * be sure that DMA mapping is supported.
880 for (i = 0; i < user_mem_maps->n_maps; i++) {
881 struct user_mem_map *map;
882 map = &user_mem_maps->maps[i];
884 ret = t->dma_user_map_func(
886 map->addr, map->iova, map->len,
889 RTE_LOG(ERR, EAL, "Couldn't map user memory for DMA: "
891 "iova: 0x%" PRIx64 " "
892 "len: 0x%" PRIu64 "\n",
893 map->addr, map->iova,
895 rte_spinlock_recursive_unlock(
896 &user_mem_maps->lock);
897 rte_mcfg_mem_read_unlock();
901 rte_spinlock_recursive_unlock(&user_mem_maps->lock);
903 /* register callback for mem events */
904 if (vfio_cfg == default_vfio_cfg)
905 ret = rte_mem_event_callback_register(
906 VFIO_MEM_EVENT_CLB_NAME,
907 vfio_mem_event_callback, NULL);
910 /* unlock memory hotplug */
911 rte_mcfg_mem_read_unlock();
913 if (ret && rte_errno != ENOTSUP) {
914 RTE_LOG(ERR, EAL, "Could not install memory event callback for VFIO\n");
918 RTE_LOG(DEBUG, EAL, "Memory event callbacks not supported\n");
920 RTE_LOG(DEBUG, EAL, "Installed memory event callback for VFIO\n");
922 } else if (rte_eal_process_type() != RTE_PROC_PRIMARY &&
923 vfio_cfg == default_vfio_cfg &&
924 vfio_cfg->vfio_iommu_type == NULL) {
925 /* if we're not a primary process, we do not set up the VFIO
926 * container because it's already been set up by the primary
927 * process. instead, we simply ask the primary about VFIO type
928 * we are using, and set the VFIO config up appropriately.
930 ret = vfio_sync_default_container();
932 RTE_LOG(ERR, EAL, "Could not sync default VFIO container\n");
933 close(vfio_group_fd);
934 rte_vfio_clear_group(vfio_group_fd);
937 /* we have successfully initialized VFIO, notify user */
938 const struct vfio_iommu_type *t =
939 default_vfio_cfg->vfio_iommu_type;
940 RTE_LOG(INFO, EAL, "Using IOMMU type %d (%s)\n",
941 t->type_id, t->name);
944 rte_eal_vfio_get_vf_token(vf_token);
946 /* get a file descriptor for the device with VF token firstly */
947 if (!rte_uuid_is_null(vf_token)) {
948 char vf_token_str[RTE_UUID_STRLEN];
951 rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
952 snprintf(dev, sizeof(dev),
953 "%s vf_token=%s", dev_addr, vf_token_str);
955 *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
957 if (*vfio_dev_fd >= 0)
961 /* get a file descriptor for the device */
962 *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
963 if (*vfio_dev_fd < 0) {
964 /* if we cannot get a device fd, this implies a problem with
965 * the VFIO group or the container not having IOMMU configured.
968 RTE_LOG(WARNING, EAL, "Getting a vfio_dev_fd for %s failed\n",
970 close(vfio_group_fd);
971 rte_vfio_clear_group(vfio_group_fd);
975 /* test and setup the device */
977 ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
979 RTE_LOG(ERR, EAL, "%s cannot get device info, "
980 "error %i (%s)\n", dev_addr, errno,
983 close(vfio_group_fd);
984 rte_vfio_clear_group(vfio_group_fd);
987 vfio_group_device_get(vfio_group_fd);
993 rte_vfio_release_device(const char *sysfs_base, const char *dev_addr,
996 struct vfio_config *vfio_cfg;
1001 /* we don't want any DMA mapping messages to come while we're detaching
1002 * VFIO device, because this might be the last device and we might need
1003 * to unregister the callback.
1005 rte_mcfg_mem_read_lock();
1007 /* get group number */
1008 ret = rte_vfio_get_group_num(sysfs_base, dev_addr, &iommu_group_num);
1010 RTE_LOG(WARNING, EAL, "%s not managed by VFIO driver\n",
1012 /* This is an error at this point. */
1017 /* get the actual group fd */
1018 vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
1019 if (vfio_group_fd < 0) {
1020 RTE_LOG(INFO, EAL, "rte_vfio_get_group_fd failed for %s\n",
1022 ret = vfio_group_fd;
1026 /* get the vfio_config it belongs to */
1027 vfio_cfg = get_vfio_cfg_by_group_num(iommu_group_num);
1028 vfio_cfg = vfio_cfg ? vfio_cfg : default_vfio_cfg;
1030 /* At this point we got an active group. Closing it will make the
1031 * container detachment. If this is the last active group, VFIO kernel
1032 * code will unset the container and the IOMMU mappings.
1035 /* Closing a device */
1036 if (close(vfio_dev_fd) < 0) {
1037 RTE_LOG(INFO, EAL, "Error when closing vfio_dev_fd for %s\n",
1043 /* An VFIO group can have several devices attached. Just when there is
1044 * no devices remaining should the group be closed.
1046 vfio_group_device_put(vfio_group_fd);
1047 if (!vfio_group_device_count(vfio_group_fd)) {
1049 if (close(vfio_group_fd) < 0) {
1050 RTE_LOG(INFO, EAL, "Error when closing vfio_group_fd for %s\n",
1056 if (rte_vfio_clear_group(vfio_group_fd) < 0) {
1057 RTE_LOG(INFO, EAL, "Error when clearing group for %s\n",
1064 /* if there are no active device groups, unregister the callback to
1065 * avoid spurious attempts to map/unmap memory from VFIO.
1067 if (vfio_cfg == default_vfio_cfg && vfio_cfg->vfio_active_groups == 0 &&
1068 rte_eal_process_type() != RTE_PROC_SECONDARY)
1069 rte_mem_event_callback_unregister(VFIO_MEM_EVENT_CLB_NAME,
1076 rte_mcfg_mem_read_unlock();
1081 rte_vfio_enable(const char *modname)
1083 /* initialize group list */
1086 const struct internal_config *internal_conf =
1087 eal_get_internal_configuration();
1089 rte_spinlock_recursive_t lock = RTE_SPINLOCK_RECURSIVE_INITIALIZER;
1091 for (i = 0; i < VFIO_MAX_CONTAINERS; i++) {
1092 vfio_cfgs[i].vfio_container_fd = -1;
1093 vfio_cfgs[i].vfio_active_groups = 0;
1094 vfio_cfgs[i].vfio_iommu_type = NULL;
1095 vfio_cfgs[i].mem_maps.lock = lock;
1097 for (j = 0; j < VFIO_MAX_GROUPS; j++) {
1098 vfio_cfgs[i].vfio_groups[j].fd = -1;
1099 vfio_cfgs[i].vfio_groups[j].group_num = -1;
1100 vfio_cfgs[i].vfio_groups[j].devices = 0;
1104 RTE_LOG(DEBUG, EAL, "Probing VFIO support...\n");
1106 /* check if vfio module is loaded */
1107 vfio_available = rte_eal_check_module(modname);
1109 /* return error directly */
1110 if (vfio_available == -1) {
1111 RTE_LOG(INFO, EAL, "Could not get loaded module details!\n");
1115 /* return 0 if VFIO modules not loaded */
1116 if (vfio_available == 0) {
1118 "VFIO modules not loaded, skipping VFIO support...\n");
1122 if (internal_conf->process_type == RTE_PROC_PRIMARY) {
1123 /* open a new container */
1124 default_vfio_cfg->vfio_container_fd =
1125 rte_vfio_get_container_fd();
1127 /* get the default container from the primary process */
1128 default_vfio_cfg->vfio_container_fd =
1129 vfio_get_default_container_fd();
1132 /* check if we have VFIO driver enabled */
1133 if (default_vfio_cfg->vfio_container_fd != -1) {
1134 RTE_LOG(INFO, EAL, "VFIO support initialized\n");
1135 default_vfio_cfg->vfio_enabled = 1;
1137 RTE_LOG(NOTICE, EAL, "VFIO support could not be initialized\n");
1144 rte_vfio_is_enabled(const char *modname)
1146 const int mod_available = rte_eal_check_module(modname) > 0;
1147 return default_vfio_cfg->vfio_enabled && mod_available;
1151 vfio_get_default_container_fd(void)
1153 struct rte_mp_msg mp_req, *mp_rep;
1154 struct rte_mp_reply mp_reply = {0};
1155 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
1156 struct vfio_mp_param *p = (struct vfio_mp_param *)mp_req.param;
1158 const struct internal_config *internal_conf =
1159 eal_get_internal_configuration();
1161 if (default_vfio_cfg->vfio_enabled)
1162 return default_vfio_cfg->vfio_container_fd;
1164 if (internal_conf->process_type == RTE_PROC_PRIMARY) {
1165 /* if we were secondary process we would try requesting
1166 * container fd from the primary, but we're the primary
1167 * process so just exit here
1172 p->req = SOCKET_REQ_DEFAULT_CONTAINER;
1173 strcpy(mp_req.name, EAL_VFIO_MP);
1174 mp_req.len_param = sizeof(*p);
1177 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
1178 mp_reply.nb_received == 1) {
1179 mp_rep = &mp_reply.msgs[0];
1180 p = (struct vfio_mp_param *)mp_rep->param;
1181 if (p->result == SOCKET_OK && mp_rep->num_fds == 1) {
1182 container_fd = mp_rep->fds[0];
1183 free(mp_reply.msgs);
1184 return container_fd;
1188 free(mp_reply.msgs);
1189 RTE_LOG(ERR, EAL, "Cannot request default VFIO container fd\n");
1194 vfio_get_iommu_type(void)
1196 if (default_vfio_cfg->vfio_iommu_type == NULL)
1199 return default_vfio_cfg->vfio_iommu_type->type_id;
1202 const struct vfio_iommu_type *
1203 vfio_set_iommu_type(int vfio_container_fd)
1206 for (idx = 0; idx < RTE_DIM(iommu_types); idx++) {
1207 const struct vfio_iommu_type *t = &iommu_types[idx];
1209 int ret = ioctl(vfio_container_fd, VFIO_SET_IOMMU,
1212 RTE_LOG(INFO, EAL, "Using IOMMU type %d (%s)\n",
1213 t->type_id, t->name);
1216 /* not an error, there may be more supported IOMMU types */
1217 RTE_LOG(DEBUG, EAL, "Set IOMMU type %d (%s) failed, error "
1218 "%i (%s)\n", t->type_id, t->name, errno,
1221 /* if we didn't find a suitable IOMMU type, fail */
1226 vfio_has_supported_extensions(int vfio_container_fd)
1229 unsigned idx, n_extensions = 0;
1230 for (idx = 0; idx < RTE_DIM(iommu_types); idx++) {
1231 const struct vfio_iommu_type *t = &iommu_types[idx];
1233 ret = ioctl(vfio_container_fd, VFIO_CHECK_EXTENSION,
1236 RTE_LOG(ERR, EAL, "Could not get IOMMU type, error "
1237 "%i (%s)\n", errno, strerror(errno));
1238 close(vfio_container_fd);
1240 } else if (ret == 1) {
1241 /* we found a supported extension */
1244 RTE_LOG(DEBUG, EAL, "IOMMU type %d (%s) is %s\n",
1245 t->type_id, t->name,
1246 ret ? "supported" : "not supported");
1249 /* if we didn't find any supported IOMMU types, fail */
1250 if (!n_extensions) {
1251 close(vfio_container_fd);
1259 rte_vfio_get_container_fd(void)
1261 int ret, vfio_container_fd;
1262 struct rte_mp_msg mp_req, *mp_rep;
1263 struct rte_mp_reply mp_reply = {0};
1264 struct timespec ts = {.tv_sec = 5, .tv_nsec = 0};
1265 struct vfio_mp_param *p = (struct vfio_mp_param *)mp_req.param;
1266 const struct internal_config *internal_conf =
1267 eal_get_internal_configuration();
1270 /* if we're in a primary process, try to open the container */
1271 if (internal_conf->process_type == RTE_PROC_PRIMARY) {
1272 vfio_container_fd = open(VFIO_CONTAINER_PATH, O_RDWR);
1273 if (vfio_container_fd < 0) {
1275 "Cannot open VFIO container %s, error "
1276 "%i (%s)\n", VFIO_CONTAINER_PATH,
1277 errno, strerror(errno));
1281 /* check VFIO API version */
1282 ret = ioctl(vfio_container_fd, VFIO_GET_API_VERSION);
1283 if (ret != VFIO_API_VERSION) {
1286 "Could not get VFIO API version, error "
1287 "%i (%s)\n", errno, strerror(errno));
1289 RTE_LOG(ERR, EAL, "Unsupported VFIO API version!\n");
1290 close(vfio_container_fd);
1294 ret = vfio_has_supported_extensions(vfio_container_fd);
1297 "No supported IOMMU extensions found!\n");
1301 return vfio_container_fd;
1304 * if we're in a secondary process, request container fd from the
1305 * primary process via mp channel
1307 p->req = SOCKET_REQ_CONTAINER;
1308 strcpy(mp_req.name, EAL_VFIO_MP);
1309 mp_req.len_param = sizeof(*p);
1312 vfio_container_fd = -1;
1313 if (rte_mp_request_sync(&mp_req, &mp_reply, &ts) == 0 &&
1314 mp_reply.nb_received == 1) {
1315 mp_rep = &mp_reply.msgs[0];
1316 p = (struct vfio_mp_param *)mp_rep->param;
1317 if (p->result == SOCKET_OK && mp_rep->num_fds == 1) {
1318 vfio_container_fd = mp_rep->fds[0];
1319 free(mp_reply.msgs);
1320 return vfio_container_fd;
1324 free(mp_reply.msgs);
1325 RTE_LOG(ERR, EAL, "Cannot request VFIO container fd\n");
1330 rte_vfio_get_group_num(const char *sysfs_base,
1331 const char *dev_addr, int *iommu_group_num)
1333 char linkname[PATH_MAX];
1334 char filename[PATH_MAX];
1335 char *tok[16], *group_tok, *end;
1338 memset(linkname, 0, sizeof(linkname));
1339 memset(filename, 0, sizeof(filename));
1341 /* try to find out IOMMU group for this device */
1342 snprintf(linkname, sizeof(linkname),
1343 "%s/%s/iommu_group", sysfs_base, dev_addr);
1345 ret = readlink(linkname, filename, sizeof(filename));
1347 /* if the link doesn't exist, no VFIO for us */
1351 ret = rte_strsplit(filename, sizeof(filename),
1352 tok, RTE_DIM(tok), '/');
1355 RTE_LOG(ERR, EAL, "%s cannot get IOMMU group\n", dev_addr);
1359 /* IOMMU group is always the last token */
1361 group_tok = tok[ret - 1];
1363 *iommu_group_num = strtol(group_tok, &end, 10);
1364 if ((end != group_tok && *end != '\0') || errno != 0) {
1365 RTE_LOG(ERR, EAL, "%s error parsing IOMMU number!\n", dev_addr);
1373 type1_map_contig(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
1374 size_t len, void *arg)
1376 int *vfio_container_fd = arg;
1381 return vfio_type1_dma_mem_map(*vfio_container_fd, ms->addr_64, ms->iova,
1386 type1_map(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
1389 int *vfio_container_fd = arg;
1391 /* skip external memory that isn't a heap */
1392 if (msl->external && !msl->heap)
1395 /* skip any segments with invalid IOVA addresses */
1396 if (ms->iova == RTE_BAD_IOVA)
1399 /* if IOVA mode is VA, we've already mapped the internal segments */
1400 if (!msl->external && rte_eal_iova_mode() == RTE_IOVA_VA)
1403 return vfio_type1_dma_mem_map(*vfio_container_fd, ms->addr_64, ms->iova,
1408 vfio_type1_dma_mem_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,
1409 uint64_t len, int do_map)
1411 struct vfio_iommu_type1_dma_map dma_map;
1412 struct vfio_iommu_type1_dma_unmap dma_unmap;
1416 memset(&dma_map, 0, sizeof(dma_map));
1417 dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
1418 dma_map.vaddr = vaddr;
1420 dma_map.iova = iova;
1421 dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
1422 VFIO_DMA_MAP_FLAG_WRITE;
1424 ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
1427 * In case the mapping was already done EEXIST will be
1428 * returned from kernel.
1430 if (errno == EEXIST) {
1432 "Memory segment is already mapped, skipping");
1435 "Cannot set up DMA remapping, error "
1436 "%i (%s)\n", errno, strerror(errno));
1441 memset(&dma_unmap, 0, sizeof(dma_unmap));
1442 dma_unmap.argsz = sizeof(struct vfio_iommu_type1_dma_unmap);
1443 dma_unmap.size = len;
1444 dma_unmap.iova = iova;
1446 ret = ioctl(vfio_container_fd, VFIO_IOMMU_UNMAP_DMA,
1449 RTE_LOG(ERR, EAL, "Cannot clear DMA remapping, error "
1450 "%i (%s)\n", errno, strerror(errno));
1452 } else if (dma_unmap.size != len) {
1453 RTE_LOG(ERR, EAL, "Unexpected size %"PRIu64
1454 " of DMA remapping cleared instead of %"PRIu64"\n",
1455 (uint64_t)dma_unmap.size, len);
1465 vfio_type1_dma_map(int vfio_container_fd)
1467 if (rte_eal_iova_mode() == RTE_IOVA_VA) {
1468 /* with IOVA as VA mode, we can get away with mapping contiguous
1469 * chunks rather than going page-by-page.
1471 int ret = rte_memseg_contig_walk(type1_map_contig,
1472 &vfio_container_fd);
1475 /* we have to continue the walk because we've skipped the
1476 * external segments during the config walk.
1479 return rte_memseg_walk(type1_map, &vfio_container_fd);
1482 /* Track the size of the statically allocated DMA window for SPAPR */
1483 uint64_t spapr_dma_win_len;
1484 uint64_t spapr_dma_win_page_sz;
1487 vfio_spapr_dma_do_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,
1488 uint64_t len, int do_map)
1490 struct vfio_iommu_spapr_register_memory reg = {
1491 .argsz = sizeof(reg),
1492 .vaddr = (uintptr_t) vaddr,
1499 struct vfio_iommu_type1_dma_map dma_map;
1501 if (iova + len > spapr_dma_win_len) {
1502 RTE_LOG(ERR, EAL, "DMA map attempt outside DMA window\n");
1506 ret = ioctl(vfio_container_fd,
1507 VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®);
1510 "Cannot register vaddr for IOMMU, error "
1511 "%i (%s)\n", errno, strerror(errno));
1515 memset(&dma_map, 0, sizeof(dma_map));
1516 dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
1517 dma_map.vaddr = vaddr;
1519 dma_map.iova = iova;
1520 dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
1521 VFIO_DMA_MAP_FLAG_WRITE;
1523 ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
1525 RTE_LOG(ERR, EAL, "Cannot map vaddr for IOMMU, error "
1526 "%i (%s)\n", errno, strerror(errno));
1531 struct vfio_iommu_type1_dma_map dma_unmap;
1533 memset(&dma_unmap, 0, sizeof(dma_unmap));
1534 dma_unmap.argsz = sizeof(struct vfio_iommu_type1_dma_unmap);
1535 dma_unmap.size = len;
1536 dma_unmap.iova = iova;
1538 ret = ioctl(vfio_container_fd, VFIO_IOMMU_UNMAP_DMA,
1541 RTE_LOG(ERR, EAL, "Cannot unmap vaddr for IOMMU, error "
1542 "%i (%s)\n", errno, strerror(errno));
1546 ret = ioctl(vfio_container_fd,
1547 VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, ®);
1550 "Cannot unregister vaddr for IOMMU, error "
1551 "%i (%s)\n", errno, strerror(errno));
1560 vfio_spapr_map_walk(const struct rte_memseg_list *msl,
1561 const struct rte_memseg *ms, void *arg)
1563 int *vfio_container_fd = arg;
1565 /* skip external memory that isn't a heap */
1566 if (msl->external && !msl->heap)
1569 /* skip any segments with invalid IOVA addresses */
1570 if (ms->iova == RTE_BAD_IOVA)
1573 return vfio_spapr_dma_do_map(*vfio_container_fd,
1574 ms->addr_64, ms->iova, ms->len, 1);
1577 struct spapr_size_walk_param {
1580 bool is_user_managed;
1584 * In order to set the DMA window size required for the SPAPR IOMMU
1585 * we need to walk the existing virtual memory allocations as well as
1586 * find the hugepage size used.
1589 vfio_spapr_size_walk(const struct rte_memseg_list *msl, void *arg)
1591 struct spapr_size_walk_param *param = arg;
1592 uint64_t max = (uint64_t) msl->base_va + (uint64_t) msl->len;
1594 if (msl->external && !msl->heap) {
1595 /* ignore user managed external memory */
1596 param->is_user_managed = true;
1600 if (max > param->max_va) {
1601 param->page_sz = msl->page_sz;
1602 param->max_va = max;
1609 * Find the highest memory address used in physical or virtual address
1610 * space and use that as the top of the DMA window.
1613 find_highest_mem_addr(struct spapr_size_walk_param *param)
1615 /* find the maximum IOVA address for setting the DMA window size */
1616 if (rte_eal_iova_mode() == RTE_IOVA_PA) {
1617 static const char proc_iomem[] = "/proc/iomem";
1618 static const char str_sysram[] = "System RAM";
1619 uint64_t start, end, max = 0;
1625 * Example "System RAM" in /proc/iomem:
1626 * 00000000-1fffffffff : System RAM
1627 * 200000000000-201fffffffff : System RAM
1629 FILE *fd = fopen(proc_iomem, "r");
1631 RTE_LOG(ERR, EAL, "Cannot open %s\n", proc_iomem);
1634 /* Scan /proc/iomem for the highest PA in the system */
1635 while (getline(&line, &line_len, fd) != -1) {
1636 if (strstr(line, str_sysram) == NULL)
1639 space = strstr(line, " ");
1640 dash = strstr(line, "-");
1642 /* Validate the format of the memory string */
1643 if (space == NULL || dash == NULL || space < dash) {
1644 RTE_LOG(ERR, EAL, "Can't parse line \"%s\" in file %s\n",
1649 start = strtoull(line, NULL, 16);
1650 end = strtoull(dash + 1, NULL, 16);
1651 RTE_LOG(DEBUG, EAL, "Found system RAM from 0x%" PRIx64
1652 " to 0x%" PRIx64 "\n", start, end);
1660 RTE_LOG(ERR, EAL, "Failed to find valid \"System RAM\" "
1661 "entry in file %s\n", proc_iomem);
1665 spapr_dma_win_len = rte_align64pow2(max + 1);
1667 } else if (rte_eal_iova_mode() == RTE_IOVA_VA) {
1668 RTE_LOG(DEBUG, EAL, "Highest VA address in memseg list is 0x%"
1669 PRIx64 "\n", param->max_va);
1670 spapr_dma_win_len = rte_align64pow2(param->max_va);
1674 spapr_dma_win_len = 0;
1675 RTE_LOG(ERR, EAL, "Unsupported IOVA mode\n");
1681 * The SPAPRv2 IOMMU supports 2 DMA windows with starting
1682 * address at 0 or 1<<59. By default, a DMA window is set
1683 * at address 0, 2GB long, with a 4KB page. For DPDK we
1684 * must remove the default window and setup a new DMA window
1685 * based on the hugepage size and memory requirements of
1686 * the application before we can map memory for DMA.
1689 spapr_dma_win_size(void)
1691 struct spapr_size_walk_param param;
1693 /* only create DMA window once */
1694 if (spapr_dma_win_len > 0)
1697 /* walk the memseg list to find the page size/max VA address */
1698 memset(¶m, 0, sizeof(param));
1699 if (rte_memseg_list_walk(vfio_spapr_size_walk, ¶m) < 0) {
1700 RTE_LOG(ERR, EAL, "Failed to walk memseg list for DMA window size\n");
1704 /* we can't be sure if DMA window covers external memory */
1705 if (param.is_user_managed)
1706 RTE_LOG(WARNING, EAL, "Detected user managed external memory which may not be managed by the IOMMU\n");
1708 /* check physical/virtual memory size */
1709 if (find_highest_mem_addr(¶m) < 0)
1711 RTE_LOG(DEBUG, EAL, "Setting DMA window size to 0x%" PRIx64 "\n",
1713 spapr_dma_win_page_sz = param.page_sz;
1714 rte_mem_set_dma_mask(__builtin_ctzll(spapr_dma_win_len));
1719 vfio_spapr_create_dma_window(int vfio_container_fd)
1721 struct vfio_iommu_spapr_tce_create create = {
1722 .argsz = sizeof(create), };
1723 struct vfio_iommu_spapr_tce_remove remove = {
1724 .argsz = sizeof(remove), };
1725 struct vfio_iommu_spapr_tce_info info = {
1726 .argsz = sizeof(info), };
1729 ret = spapr_dma_win_size();
1733 ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
1735 RTE_LOG(ERR, EAL, "Cannot get IOMMU info, error %i (%s)\n",
1736 errno, strerror(errno));
1741 * sPAPR v1/v2 IOMMU always has a default 1G DMA window set. The window
1742 * can't be changed for v1 but it can be changed for v2. Since DPDK only
1743 * supports v2, remove the default DMA window so it can be resized.
1745 remove.start_addr = info.dma32_window_start;
1746 ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove);
1750 /* create a new DMA window (start address is not selectable) */
1751 create.window_size = spapr_dma_win_len;
1752 create.page_shift = __builtin_ctzll(spapr_dma_win_page_sz);
1754 ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
1755 #ifdef VFIO_IOMMU_SPAPR_INFO_DDW
1757 * The vfio_iommu_spapr_tce_info structure was modified in
1758 * Linux kernel 4.2.0 to add support for the
1759 * vfio_iommu_spapr_tce_ddw_info structure needed to try
1760 * multiple table levels. Skip the attempt if running with
1764 /* if at first we don't succeed, try more levels */
1767 for (levels = create.levels + 1;
1768 ret && levels <= info.ddw.levels; levels++) {
1769 create.levels = levels;
1770 ret = ioctl(vfio_container_fd,
1771 VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
1774 #endif /* VFIO_IOMMU_SPAPR_INFO_DDW */
1776 RTE_LOG(ERR, EAL, "Cannot create new DMA window, error "
1777 "%i (%s)\n", errno, strerror(errno));
1779 "Consider using a larger hugepage size if supported by the system\n");
1783 /* verify the start address */
1784 if (create.start_addr != 0) {
1785 RTE_LOG(ERR, EAL, "Received unsupported start address 0x%"
1786 PRIx64 "\n", (uint64_t)create.start_addr);
1793 vfio_spapr_dma_mem_map(int vfio_container_fd, uint64_t vaddr,
1794 uint64_t iova, uint64_t len, int do_map)
1799 if (vfio_spapr_dma_do_map(vfio_container_fd,
1800 vaddr, iova, len, 1)) {
1801 RTE_LOG(ERR, EAL, "Failed to map DMA\n");
1805 if (vfio_spapr_dma_do_map(vfio_container_fd,
1806 vaddr, iova, len, 0)) {
1807 RTE_LOG(ERR, EAL, "Failed to unmap DMA\n");
1816 vfio_spapr_dma_map(int vfio_container_fd)
1818 if (vfio_spapr_create_dma_window(vfio_container_fd) < 0) {
1819 RTE_LOG(ERR, EAL, "Could not create new DMA window!\n");
1823 /* map all existing DPDK segments for DMA */
1824 if (rte_memseg_walk(vfio_spapr_map_walk, &vfio_container_fd) < 0)
1831 vfio_noiommu_dma_map(int __rte_unused vfio_container_fd)
1833 /* No-IOMMU mode does not need DMA mapping */
1838 vfio_noiommu_dma_mem_map(int __rte_unused vfio_container_fd,
1839 uint64_t __rte_unused vaddr,
1840 uint64_t __rte_unused iova, uint64_t __rte_unused len,
1841 int __rte_unused do_map)
1843 /* No-IOMMU mode does not need DMA mapping */
1848 vfio_dma_mem_map(struct vfio_config *vfio_cfg, uint64_t vaddr, uint64_t iova,
1849 uint64_t len, int do_map)
1851 const struct vfio_iommu_type *t = vfio_cfg->vfio_iommu_type;
1854 RTE_LOG(ERR, EAL, "VFIO support not initialized\n");
1859 if (!t->dma_user_map_func) {
1861 "VFIO custom DMA region mapping not supported by IOMMU %s\n",
1863 rte_errno = ENOTSUP;
1867 return t->dma_user_map_func(vfio_cfg->vfio_container_fd, vaddr, iova,
1872 container_dma_map(struct vfio_config *vfio_cfg, uint64_t vaddr, uint64_t iova,
1875 struct user_mem_map *new_map;
1876 struct user_mem_maps *user_mem_maps;
1877 bool has_partial_unmap;
1880 user_mem_maps = &vfio_cfg->mem_maps;
1881 rte_spinlock_recursive_lock(&user_mem_maps->lock);
1882 if (user_mem_maps->n_maps == VFIO_MAX_USER_MEM_MAPS) {
1883 RTE_LOG(ERR, EAL, "No more space for user mem maps\n");
1889 if (vfio_dma_mem_map(vfio_cfg, vaddr, iova, len, 1)) {
1890 /* technically, this will fail if there are currently no devices
1891 * plugged in, even if a device were added later, this mapping
1892 * might have succeeded. however, since we cannot verify if this
1893 * is a valid mapping without having a device attached, consider
1894 * this to be unsupported, because we can't just store any old
1895 * mapping and pollute list of active mappings willy-nilly.
1897 RTE_LOG(ERR, EAL, "Couldn't map new region for DMA\n");
1901 /* do we have partial unmap support? */
1902 has_partial_unmap = vfio_cfg->vfio_iommu_type->partial_unmap;
1904 /* create new user mem map entry */
1905 new_map = &user_mem_maps->maps[user_mem_maps->n_maps++];
1906 new_map->addr = vaddr;
1907 new_map->iova = iova;
1909 /* for IOMMU types supporting partial unmap, we don't need chunking */
1910 new_map->chunk = has_partial_unmap ? 0 : len;
1912 compact_user_maps(user_mem_maps);
1914 rte_spinlock_recursive_unlock(&user_mem_maps->lock);
1919 container_dma_unmap(struct vfio_config *vfio_cfg, uint64_t vaddr, uint64_t iova,
1922 struct user_mem_map orig_maps[VFIO_MAX_USER_MEM_MAPS];
1923 struct user_mem_map new_maps[2]; /* can be at most 2 */
1924 struct user_mem_maps *user_mem_maps;
1925 int n_orig, n_new, newlen, ret = 0;
1926 bool has_partial_unmap;
1928 user_mem_maps = &vfio_cfg->mem_maps;
1929 rte_spinlock_recursive_lock(&user_mem_maps->lock);
1932 * Previously, we had adjacent mappings entirely contained within one
1933 * mapping entry. Since we now store original mapping length in some
1934 * cases, this is no longer the case, so unmapping can potentially go
1935 * over multiple segments and split them in any number of ways.
1937 * To complicate things further, some IOMMU types support arbitrary
1938 * partial unmapping, while others will only support unmapping along the
1939 * chunk size, so there are a lot of cases we need to handle. To make
1940 * things easier code wise, instead of trying to adjust existing
1941 * mappings, let's just rebuild them using information we have.
1945 * first thing to do is check if there exists a mapping that includes
1946 * the start and the end of our requested unmap. We need to collect all
1947 * maps that include our unmapped region.
1949 n_orig = find_user_mem_maps(user_mem_maps, vaddr, iova, len,
1950 orig_maps, RTE_DIM(orig_maps));
1951 /* did we find anything? */
1953 RTE_LOG(ERR, EAL, "Couldn't find previously mapped region\n");
1959 /* do we have partial unmap capability? */
1960 has_partial_unmap = vfio_cfg->vfio_iommu_type->partial_unmap;
1963 * if we don't support partial unmap, we must check if start and end of
1964 * current unmap region are chunk-aligned.
1966 if (!has_partial_unmap) {
1967 bool start_aligned, end_aligned;
1969 start_aligned = addr_is_chunk_aligned(orig_maps, n_orig,
1971 end_aligned = addr_is_chunk_aligned(orig_maps, n_orig,
1972 vaddr + len, iova + len);
1974 if (!start_aligned || !end_aligned) {
1975 RTE_LOG(DEBUG, EAL, "DMA partial unmap unsupported\n");
1976 rte_errno = ENOTSUP;
1983 * now we know we can potentially unmap the region, but we still have to
1984 * figure out if there is enough space in our list to store remaining
1985 * maps. for this, we will figure out how many segments we are going to
1986 * remove, and how many new segments we are going to create.
1988 n_new = process_maps(orig_maps, n_orig, new_maps, vaddr, len);
1990 /* can we store the new maps in our list? */
1991 newlen = (user_mem_maps->n_maps - n_orig) + n_new;
1992 if (newlen >= VFIO_MAX_USER_MEM_MAPS) {
1993 RTE_LOG(ERR, EAL, "Not enough space to store partial mapping\n");
1999 /* unmap the entry */
2000 if (vfio_dma_mem_map(vfio_cfg, vaddr, iova, len, 0)) {
2001 /* there may not be any devices plugged in, so unmapping will
2002 * fail with ENODEV/ENOTSUP rte_errno values, but that doesn't
2003 * stop us from removing the mapping, as the assumption is we
2004 * won't be needing this memory any more and thus will want to
2005 * prevent it from being remapped again on hotplug. so, only
2006 * fail if we indeed failed to unmap (e.g. if the mapping was
2007 * within our mapped range but had invalid alignment).
2009 if (rte_errno != ENODEV && rte_errno != ENOTSUP) {
2010 RTE_LOG(ERR, EAL, "Couldn't unmap region for DMA\n");
2014 RTE_LOG(DEBUG, EAL, "DMA unmapping failed, but removing mappings anyway\n");
2018 /* we have unmapped the region, so now update the maps */
2019 delete_maps(user_mem_maps, orig_maps, n_orig);
2020 copy_maps(user_mem_maps, new_maps, n_new);
2021 compact_user_maps(user_mem_maps);
2023 rte_spinlock_recursive_unlock(&user_mem_maps->lock);
2028 rte_vfio_noiommu_is_enabled(void)
2034 fd = open(VFIO_NOIOMMU_MODE, O_RDONLY);
2036 if (errno != ENOENT) {
2037 RTE_LOG(ERR, EAL, "Cannot open VFIO noiommu file "
2038 "%i (%s)\n", errno, strerror(errno));
2042 * else the file does not exists
2043 * i.e. noiommu is not enabled
2048 cnt = read(fd, &c, 1);
2051 RTE_LOG(ERR, EAL, "Unable to read from VFIO noiommu file "
2052 "%i (%s)\n", errno, strerror(errno));
2060 rte_vfio_container_create(void)
2064 /* Find an empty slot to store new vfio config */
2065 for (i = 1; i < VFIO_MAX_CONTAINERS; i++) {
2066 if (vfio_cfgs[i].vfio_container_fd == -1)
2070 if (i == VFIO_MAX_CONTAINERS) {
2071 RTE_LOG(ERR, EAL, "Exceed max VFIO container limit\n");
2075 vfio_cfgs[i].vfio_container_fd = rte_vfio_get_container_fd();
2076 if (vfio_cfgs[i].vfio_container_fd < 0) {
2077 RTE_LOG(NOTICE, EAL, "Fail to create a new VFIO container\n");
2081 return vfio_cfgs[i].vfio_container_fd;
2085 rte_vfio_container_destroy(int container_fd)
2087 struct vfio_config *vfio_cfg;
2090 vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
2091 if (vfio_cfg == NULL) {
2092 RTE_LOG(ERR, EAL, "Invalid VFIO container fd\n");
2096 for (i = 0; i < VFIO_MAX_GROUPS; i++)
2097 if (vfio_cfg->vfio_groups[i].group_num != -1)
2098 rte_vfio_container_group_unbind(container_fd,
2099 vfio_cfg->vfio_groups[i].group_num);
2101 close(container_fd);
2102 vfio_cfg->vfio_container_fd = -1;
2103 vfio_cfg->vfio_active_groups = 0;
2104 vfio_cfg->vfio_iommu_type = NULL;
2110 rte_vfio_container_group_bind(int container_fd, int iommu_group_num)
2112 struct vfio_config *vfio_cfg;
2114 vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
2115 if (vfio_cfg == NULL) {
2116 RTE_LOG(ERR, EAL, "Invalid VFIO container fd\n");
2120 return vfio_get_group_fd(vfio_cfg, iommu_group_num);
2124 rte_vfio_container_group_unbind(int container_fd, int iommu_group_num)
2126 struct vfio_config *vfio_cfg;
2127 struct vfio_group *cur_grp = NULL;
2130 vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
2131 if (vfio_cfg == NULL) {
2132 RTE_LOG(ERR, EAL, "Invalid VFIO container fd\n");
2136 for (i = 0; i < VFIO_MAX_GROUPS; i++) {
2137 if (vfio_cfg->vfio_groups[i].group_num == iommu_group_num) {
2138 cur_grp = &vfio_cfg->vfio_groups[i];
2143 /* This should not happen */
2144 if (i == VFIO_MAX_GROUPS || cur_grp == NULL) {
2145 RTE_LOG(ERR, EAL, "Specified VFIO group number not found\n");
2149 if (cur_grp->fd >= 0 && close(cur_grp->fd) < 0) {
2151 "Error when closing vfio_group_fd for iommu_group_num "
2152 "%d\n", iommu_group_num);
2155 cur_grp->group_num = -1;
2157 cur_grp->devices = 0;
2158 vfio_cfg->vfio_active_groups--;
2164 rte_vfio_container_dma_map(int container_fd, uint64_t vaddr, uint64_t iova,
2167 struct vfio_config *vfio_cfg;
2174 vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
2175 if (vfio_cfg == NULL) {
2176 RTE_LOG(ERR, EAL, "Invalid VFIO container fd\n");
2180 return container_dma_map(vfio_cfg, vaddr, iova, len);
2184 rte_vfio_container_dma_unmap(int container_fd, uint64_t vaddr, uint64_t iova,
2187 struct vfio_config *vfio_cfg;
2194 vfio_cfg = get_vfio_cfg_by_container_fd(container_fd);
2195 if (vfio_cfg == NULL) {
2196 RTE_LOG(ERR, EAL, "Invalid VFIO container fd\n");
2200 return container_dma_unmap(vfio_cfg, vaddr, iova, len);