4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #ifdef RTE_LIBRTE_IVSHMEM /* hide it from coverage */
42 #include <sys/queue.h>
46 #include <rte_memory.h>
48 #include <rte_eal_memconfig.h>
49 #include <rte_string_fns.h>
50 #include <rte_errno.h>
52 #include <rte_mempool.h>
53 #include <rte_malloc.h>
54 #include <rte_common.h>
55 #include <rte_ivshmem.h>
57 #include "eal_internal_cfg.h"
58 #include "eal_private.h"
60 #define PCI_VENDOR_ID_IVSHMEM 0x1Af4
61 #define PCI_DEVICE_ID_IVSHMEM 0x1110
63 #define IVSHMEM_MAGIC 0x0BADC0DE
65 #define IVSHMEM_RESOURCE_PATH "/sys/bus/pci/devices/%04x:%02x:%02x.%x/resource2"
66 #define IVSHMEM_CONFIG_PATH "/var/run/.%s_ivshmem_config"
71 #define FULL (PHYS|VIRT|IOREMAP)
73 #define METADATA_SIZE_ALIGNED \
74 (RTE_ALIGN_CEIL(sizeof(struct rte_ivshmem_metadata),pagesz))
76 #define CONTAINS(x,y)\
77 (((y).addr_64 >= (x).addr_64) && ((y).addr_64 < (x).addr_64 + (x).len))
79 #define DIM(x) (sizeof(x)/sizeof(x[0]))
81 struct ivshmem_pci_device {
83 phys_addr_t ioremap_addr;
86 /* data type to store in config */
87 struct ivshmem_segment {
88 struct rte_ivshmem_metadata_entry entry;
92 struct ivshmem_shared_config {
93 struct ivshmem_segment segment[RTE_MAX_MEMSEG];
95 struct ivshmem_pci_device pci_devs[RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS];
96 uint32_t pci_devs_idx;
98 static struct ivshmem_shared_config * ivshmem_config;
99 static int memseg_idx;
102 /* Tailq heads to add rings to */
103 TAILQ_HEAD(rte_ring_list, rte_tailq_entry);
110 is_ivshmem_device(struct rte_pci_device * dev)
112 return (dev->id.vendor_id == PCI_VENDOR_ID_IVSHMEM
113 && dev->id.device_id == PCI_DEVICE_ID_IVSHMEM);
117 map_metadata(int fd, uint64_t len)
119 size_t metadata_len = sizeof(struct rte_ivshmem_metadata);
120 size_t aligned_len = METADATA_SIZE_ALIGNED;
122 return mmap(NULL, metadata_len, PROT_READ | PROT_WRITE,
123 MAP_SHARED, fd, len - aligned_len);
127 unmap_metadata(void * ptr)
129 munmap(ptr, sizeof(struct rte_ivshmem_metadata));
133 has_ivshmem_metadata(int fd, uint64_t len)
135 struct rte_ivshmem_metadata metadata;
138 ptr = map_metadata(fd, len);
140 if (ptr == MAP_FAILED)
143 metadata = *(struct rte_ivshmem_metadata*) (ptr);
147 return metadata.magic_number == IVSHMEM_MAGIC;
151 remove_segment(struct ivshmem_segment * ms, int len, int idx)
155 for (i = idx; i < len - 1; i++)
156 memcpy(&ms[i], &ms[i+1], sizeof(struct ivshmem_segment));
157 memset(&ms[len-1], 0, sizeof(struct ivshmem_segment));
161 overlap(const struct rte_memzone * mz1, const struct rte_memzone * mz2)
163 uint64_t start1, end1, start2, end2;
164 uint64_t p_start1, p_end1, p_start2, p_end2;
165 uint64_t i_start1, i_end1, i_start2, i_end2;
168 /* gather virtual addresses */
169 start1 = mz1->addr_64;
170 end1 = mz1->addr_64 + mz1->len;
171 start2 = mz2->addr_64;
172 end2 = mz2->addr_64 + mz2->len;
174 /* gather physical addresses */
175 p_start1 = mz1->phys_addr;
176 p_end1 = mz1->phys_addr + mz1->len;
177 p_start2 = mz2->phys_addr;
178 p_end2 = mz2->phys_addr + mz2->len;
180 /* gather ioremap addresses */
181 i_start1 = mz1->ioremap_addr;
182 i_end1 = mz1->ioremap_addr + mz1->len;
183 i_start2 = mz2->ioremap_addr;
184 i_end2 = mz2->ioremap_addr + mz2->len;
186 /* check for overlap in virtual addresses */
187 if (start1 >= start2 && start1 < end2)
189 if (start2 >= start1 && start2 < end1)
192 /* check for overlap in physical addresses */
193 if (p_start1 >= p_start2 && p_start1 < p_end2)
195 if (p_start2 >= p_start1 && p_start2 < p_end1)
198 /* check for overlap in ioremap addresses */
199 if (i_start1 >= i_start2 && i_start1 < i_end2)
201 if (i_start2 >= i_start1 && i_start2 < i_end1)
208 adjacent(const struct rte_memzone * mz1, const struct rte_memzone * mz2)
210 uint64_t start1, end1, start2, end2;
211 uint64_t p_start1, p_end1, p_start2, p_end2;
212 uint64_t i_start1, i_end1, i_start2, i_end2;
215 /* gather virtual addresses */
216 start1 = mz1->addr_64;
217 end1 = mz1->addr_64 + mz1->len;
218 start2 = mz2->addr_64;
219 end2 = mz2->addr_64 + mz2->len;
221 /* gather physical addresses */
222 p_start1 = mz1->phys_addr;
223 p_end1 = mz1->phys_addr + mz1->len;
224 p_start2 = mz2->phys_addr;
225 p_end2 = mz2->phys_addr + mz2->len;
227 /* gather ioremap addresses */
228 i_start1 = mz1->ioremap_addr;
229 i_end1 = mz1->ioremap_addr + mz1->len;
230 i_start2 = mz2->ioremap_addr;
231 i_end2 = mz2->ioremap_addr + mz2->len;
233 /* check if segments are virtually adjacent */
239 /* check if segments are physically adjacent */
240 if (p_start1 == p_end2)
242 if (p_start2 == p_end1)
245 /* check if segments are ioremap-adjacent */
246 if (i_start1 == i_end2)
248 if (i_start2 == i_end1)
255 has_adjacent_segments(struct ivshmem_segment * ms, int len)
259 for (i = 0; i < len; i++)
260 for (j = i + 1; j < len; j++) {
261 a = adjacent(&ms[i].entry.mz, &ms[j].entry.mz);
263 /* check if segments are adjacent virtually and/or physically but
264 * not ioremap (since that would indicate that they are from
265 * different PCI devices and thus don't need to be concatenated.
267 if ((a & (VIRT|PHYS)) > 0 && (a & IOREMAP) == 0)
274 has_overlapping_segments(struct ivshmem_segment * ms, int len)
278 for (i = 0; i < len; i++)
279 for (j = i + 1; j < len; j++)
280 if (overlap(&ms[i].entry.mz, &ms[j].entry.mz))
286 seg_compare(const void * a, const void * b)
288 const struct ivshmem_segment * s1 = (const struct ivshmem_segment*) a;
289 const struct ivshmem_segment * s2 = (const struct ivshmem_segment*) b;
291 /* move unallocated zones to the end */
292 if (s1->entry.mz.addr == NULL && s2->entry.mz.addr == NULL)
294 if (s1->entry.mz.addr == 0)
296 if (s2->entry.mz.addr == 0)
299 return s1->entry.mz.phys_addr > s2->entry.mz.phys_addr;
302 #ifdef RTE_LIBRTE_IVSHMEM_DEBUG
304 entry_dump(struct rte_ivshmem_metadata_entry *e)
306 RTE_LOG(DEBUG, EAL, "\tvirt: %p-%p\n", e->mz.addr,
307 RTE_PTR_ADD(e->mz.addr, e->mz.len));
308 RTE_LOG(DEBUG, EAL, "\tphys: 0x%" PRIx64 "-0x%" PRIx64 "\n",
310 e->mz.phys_addr + e->mz.len);
311 RTE_LOG(DEBUG, EAL, "\tio: 0x%" PRIx64 "-0x%" PRIx64 "\n",
313 e->mz.ioremap_addr + e->mz.len);
314 RTE_LOG(DEBUG, EAL, "\tlen: 0x%" PRIx64 "\n", e->mz.len);
315 RTE_LOG(DEBUG, EAL, "\toff: 0x%" PRIx64 "\n", e->offset);
325 /* read through metadata mapped from the IVSHMEM device */
327 read_metadata(char * path, int path_len, int fd, uint64_t flen)
329 struct rte_ivshmem_metadata metadata;
330 struct rte_ivshmem_metadata_entry * entry;
334 ptr = map_metadata(fd, flen);
336 if (ptr == MAP_FAILED)
339 metadata = *(struct rte_ivshmem_metadata*) (ptr);
343 RTE_LOG(DEBUG, EAL, "Parsing metadata for \"%s\"\n", metadata.name);
345 idx = ivshmem_config->segment_idx;
347 for (i = 0; i < RTE_LIBRTE_IVSHMEM_MAX_ENTRIES &&
348 idx <= RTE_MAX_MEMSEG; i++) {
350 if (idx == RTE_MAX_MEMSEG) {
351 RTE_LOG(ERR, EAL, "Not enough memory segments!\n");
355 entry = &metadata.entry[i];
357 /* stop on uninitialized memzone */
358 if (entry->mz.len == 0)
361 /* copy metadata entry */
362 memcpy(&ivshmem_config->segment[idx].entry, entry,
363 sizeof(struct rte_ivshmem_metadata_entry));
366 snprintf(ivshmem_config->segment[idx].path, path_len, "%s", path);
370 ivshmem_config->segment_idx = idx;
375 /* check through each segment and look for adjacent or overlapping ones. */
377 cleanup_segments(struct ivshmem_segment * ms, int tbl_len)
379 struct ivshmem_segment * s, * tmp;
380 int i, j, concat, seg_adjacent, seg_overlapping;
381 uint64_t start1, start2, end1, end2, p_start1, p_start2, i_start1, i_start2;
383 qsort(ms, tbl_len, sizeof(struct ivshmem_segment),
386 while (has_overlapping_segments(ms, tbl_len) ||
387 has_adjacent_segments(ms, tbl_len)) {
389 for (i = 0; i < tbl_len; i++) {
394 for (j = i + 1; j < tbl_len; j++) {
397 /* check if this segment is overlapping with existing segment,
398 * or is adjacent to existing segment */
399 seg_overlapping = overlap(&s->entry.mz, &tmp->entry.mz);
400 seg_adjacent = adjacent(&s->entry.mz, &tmp->entry.mz);
402 /* check if segments fully overlap or are fully adjacent */
403 if ((seg_adjacent == FULL) || (seg_overlapping == FULL)) {
405 #ifdef RTE_LIBRTE_IVSHMEM_DEBUG
406 RTE_LOG(DEBUG, EAL, "Concatenating segments\n");
407 RTE_LOG(DEBUG, EAL, "Segment %i:\n", i);
408 entry_dump(&s->entry);
409 RTE_LOG(DEBUG, EAL, "Segment %i:\n", j);
410 entry_dump(&tmp->entry);
413 start1 = s->entry.mz.addr_64;
414 start2 = tmp->entry.mz.addr_64;
415 p_start1 = s->entry.mz.phys_addr;
416 p_start2 = tmp->entry.mz.phys_addr;
417 i_start1 = s->entry.mz.ioremap_addr;
418 i_start2 = tmp->entry.mz.ioremap_addr;
419 end1 = s->entry.mz.addr_64 + s->entry.mz.len;
420 end2 = tmp->entry.mz.addr_64 + tmp->entry.mz.len;
422 /* settle for minimum start address and maximum length */
423 s->entry.mz.addr_64 = RTE_MIN(start1, start2);
424 s->entry.mz.phys_addr = RTE_MIN(p_start1, p_start2);
425 s->entry.mz.ioremap_addr = RTE_MIN(i_start1, i_start2);
426 s->entry.offset = RTE_MIN(s->entry.offset, tmp->entry.offset);
427 s->entry.mz.len = RTE_MAX(end1, end2) - s->entry.mz.addr_64;
430 #ifdef RTE_LIBRTE_IVSHMEM_DEBUG
431 RTE_LOG(DEBUG, EAL, "Resulting segment:\n");
432 entry_dump(&s->entry);
436 /* if segments not fully overlap, we have an error condition.
437 * adjacent segments can coexist.
439 else if (seg_overlapping > 0) {
440 RTE_LOG(ERR, EAL, "Segments %i and %i overlap!\n", i, j);
441 #ifdef RTE_LIBRTE_IVSHMEM_DEBUG
442 RTE_LOG(DEBUG, EAL, "Segment %i:\n", i);
443 entry_dump(&s->entry);
444 RTE_LOG(DEBUG, EAL, "Segment %i:\n", j);
445 entry_dump(&tmp->entry);
452 /* if we concatenated, remove segment at j */
454 remove_segment(ms, tbl_len, j);
465 create_shared_config(void)
470 /* build ivshmem config file path */
471 snprintf(path, sizeof(path), IVSHMEM_CONFIG_PATH,
472 internal_config.hugefile_prefix);
474 fd = open(path, O_CREAT | O_RDWR, 0600);
477 RTE_LOG(ERR, EAL, "Could not open %s: %s\n", path, strerror(errno));
481 /* try ex-locking first - if the file is locked, we have a problem */
482 if (flock(fd, LOCK_EX | LOCK_NB) == -1) {
483 RTE_LOG(ERR, EAL, "Locking %s failed: %s\n", path, strerror(errno));
488 if (ftruncate(fd, sizeof(struct ivshmem_shared_config)) < 0) {
489 RTE_LOG(ERR, EAL, "ftruncate failed: %s\n", strerror(errno));
493 ivshmem_config = mmap(NULL, sizeof(struct ivshmem_shared_config),
494 PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
496 if (ivshmem_config == MAP_FAILED)
499 memset(ivshmem_config, 0, sizeof(struct ivshmem_shared_config));
501 /* change the exclusive lock we got earlier to a shared lock */
502 if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
503 RTE_LOG(ERR, EAL, "Locking %s failed: %s \n", path, strerror(errno));
512 /* open shared config file and, if present, map the config.
513 * having no config file is not an error condition, as we later check if
514 * ivshmem_config is NULL (if it is, that means nothing was mapped). */
516 open_shared_config(void)
521 /* build ivshmem config file path */
522 snprintf(path, sizeof(path), IVSHMEM_CONFIG_PATH,
523 internal_config.hugefile_prefix);
525 fd = open(path, O_RDONLY);
527 /* if the file doesn't exist, just return success */
528 if (fd < 0 && errno == ENOENT)
530 /* else we have an error condition */
532 RTE_LOG(ERR, EAL, "Could not open %s: %s\n",
533 path, strerror(errno));
537 /* try ex-locking first - if the lock *does* succeed, this means it's a
538 * stray config file, so it should be deleted.
540 if (flock(fd, LOCK_EX | LOCK_NB) != -1) {
542 /* if we can't remove the file, something is wrong */
543 if (unlink(path) < 0) {
544 RTE_LOG(ERR, EAL, "Could not remove %s: %s\n", path,
549 /* release the lock */
553 /* return success as having a stray config file is equivalent to not
554 * having config file at all.
559 ivshmem_config = mmap(NULL, sizeof(struct ivshmem_shared_config),
560 PROT_READ, MAP_SHARED, fd, 0);
562 if (ivshmem_config == MAP_FAILED)
565 /* place a shared lock on config file */
566 if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
567 RTE_LOG(ERR, EAL, "Locking %s failed: %s \n", path, strerror(errno));
577 * This function does the following:
579 * 1) Builds a table of ivshmem_segments with proper offset alignment
580 * 2) Cleans up that table so that we don't have any overlapping or adjacent
582 * 3) Creates memsegs from this table and maps them into memory.
585 map_all_segments(void)
587 struct ivshmem_segment ms_tbl[RTE_MAX_MEMSEG];
588 struct ivshmem_pci_device * pci_dev;
589 struct rte_mem_config * mcfg;
590 struct ivshmem_segment * seg;
593 struct rte_memzone mz;
594 struct rte_memseg ms;
597 phys_addr_t ioremap_addr;
601 memset(ms_tbl, 0, sizeof(ms_tbl));
602 memset(&mz, 0, sizeof(struct rte_memzone));
603 memset(&ms, 0, sizeof(struct rte_memseg));
605 /* first, build a table of memsegs to map, to avoid failed mmaps due to
608 for (i = 0; i < ivshmem_config->segment_idx && i <= RTE_MAX_MEMSEG; i++) {
609 if (i == RTE_MAX_MEMSEG) {
610 RTE_LOG(ERR, EAL, "Too many segments requested!\n");
614 seg = &ivshmem_config->segment[i];
616 /* copy segment to table */
617 memcpy(&ms_tbl[i], seg, sizeof(struct ivshmem_segment));
619 /* find ioremap addr */
620 for (j = 0; j < DIM(ivshmem_config->pci_devs); j++) {
621 pci_dev = &ivshmem_config->pci_devs[j];
622 if (!strncmp(pci_dev->path, seg->path, sizeof(pci_dev->path))) {
623 ioremap_addr = pci_dev->ioremap_addr;
627 if (ioremap_addr == 0) {
628 RTE_LOG(ERR, EAL, "Cannot find ioremap addr!\n");
632 /* work out alignments */
633 align = seg->entry.mz.addr_64 -
634 RTE_ALIGN_FLOOR(seg->entry.mz.addr_64, 0x1000);
635 len = RTE_ALIGN_CEIL(seg->entry.mz.len + align, 0x1000);
637 /* save original alignments */
638 ms_tbl[i].align = align;
640 /* create a memory zone */
641 mz.addr_64 = seg->entry.mz.addr_64 - align;
643 mz.hugepage_sz = seg->entry.mz.hugepage_sz;
644 mz.phys_addr = seg->entry.mz.phys_addr - align;
646 /* find true physical address */
647 mz.ioremap_addr = ioremap_addr + seg->entry.offset - align;
649 ms_tbl[i].entry.offset = seg->entry.offset - align;
651 memcpy(&ms_tbl[i].entry.mz, &mz, sizeof(struct rte_memzone));
654 /* clean up the segments */
655 memseg_idx = cleanup_segments(ms_tbl, ivshmem_config->segment_idx);
660 mcfg = rte_eal_get_configuration()->mem_config;
662 fd_zero = open("/dev/zero", O_RDWR);
665 RTE_LOG(ERR, EAL, "Cannot open /dev/zero: %s\n", strerror(errno));
669 /* create memsegs and put them into DPDK memory */
670 for (i = 0; i < (unsigned) memseg_idx; i++) {
674 ms.addr_64 = seg->entry.mz.addr_64;
675 ms.hugepage_sz = seg->entry.mz.hugepage_sz;
676 ms.len = seg->entry.mz.len;
677 ms.nchannel = rte_memory_get_nchannel();
678 ms.nrank = rte_memory_get_nrank();
679 ms.phys_addr = seg->entry.mz.phys_addr;
680 ms.ioremap_addr = seg->entry.mz.ioremap_addr;
681 ms.socket_id = seg->entry.mz.socket_id;
683 base_addr = mmap(ms.addr, ms.len,
684 PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_zero, 0);
686 if (base_addr == MAP_FAILED || base_addr != ms.addr) {
687 RTE_LOG(ERR, EAL, "Cannot map /dev/zero!\n");
691 fd = open(seg->path, O_RDWR);
694 RTE_LOG(ERR, EAL, "Cannot open %s: %s\n", seg->path,
699 munmap(ms.addr, ms.len);
701 base_addr = mmap(ms.addr, ms.len,
702 PROT_READ | PROT_WRITE, MAP_SHARED, fd,
706 if (base_addr == MAP_FAILED || base_addr != ms.addr) {
707 RTE_LOG(ERR, EAL, "Cannot map segment into memory: "
708 "expected %p got %p (%s)\n", ms.addr, base_addr,
713 RTE_LOG(DEBUG, EAL, "Memory segment mapped: %p (len %" PRIx64 ") at "
714 "offset 0x%" PRIx64 "\n",
715 ms.addr, ms.len, seg->entry.offset);
717 /* put the pointers back into their real positions using original
719 ms.addr_64 += seg->align;
720 ms.phys_addr += seg->align;
721 ms.ioremap_addr += seg->align;
722 ms.len -= seg->align;
724 /* at this point, the rest of DPDK memory is not initialized, so we
725 * expect memsegs to be empty */
726 memcpy(&mcfg->memseg[i], &ms,
727 sizeof(struct rte_memseg));
728 memcpy(&mcfg->free_memseg[i], &ms,
729 sizeof(struct rte_memseg));
732 /* adjust the free_memseg so that there's no free space left */
733 mcfg->free_memseg[i].ioremap_addr += mcfg->free_memseg[i].len;
734 mcfg->free_memseg[i].phys_addr += mcfg->free_memseg[i].len;
735 mcfg->free_memseg[i].addr_64 += mcfg->free_memseg[i].len;
736 mcfg->free_memseg[i].len = 0;
740 RTE_LOG(DEBUG, EAL, "IVSHMEM segment found, size: 0x%lx\n",
747 /* this happens at a later stage, after general EAL memory initialization */
749 rte_eal_ivshmem_obj_init(void)
751 struct rte_ring_list* ring_list = NULL;
752 struct rte_mem_config * mcfg;
753 struct ivshmem_segment * seg;
754 struct rte_memzone * mz;
756 struct rte_tailq_entry *te;
760 /* secondary process would not need any object discovery - it'll all
761 * already be in shared config */
762 if (rte_eal_process_type() != RTE_PROC_PRIMARY || ivshmem_config == NULL)
765 /* check that we have an initialised ring tail queue */
766 ring_list = RTE_TAILQ_LOOKUP(RTE_TAILQ_RING_NAME, rte_ring_list);
767 if (ring_list == NULL) {
768 RTE_LOG(ERR, EAL, "No rte_ring tailq found!\n");
772 mcfg = rte_eal_get_configuration()->mem_config;
774 /* create memzones */
775 for (i = 0; i < ivshmem_config->segment_idx && i <= RTE_MAX_MEMZONE; i++) {
777 seg = &ivshmem_config->segment[i];
780 if (mcfg->memzone_idx == RTE_MAX_MEMZONE) {
781 RTE_LOG(ERR, EAL, "No more memory zones available!\n");
785 idx = mcfg->memzone_idx;
787 RTE_LOG(DEBUG, EAL, "Found memzone: '%s' at %p (len 0x%" PRIx64 ")\n",
788 seg->entry.mz.name, seg->entry.mz.addr, seg->entry.mz.len);
790 memcpy(&mcfg->memzone[idx], &seg->entry.mz,
791 sizeof(struct rte_memzone));
793 /* find ioremap address */
794 for (ms = 0; ms <= RTE_MAX_MEMSEG; ms++) {
795 if (ms == RTE_MAX_MEMSEG) {
796 RTE_LOG(ERR, EAL, "Physical address of segment not found!\n");
799 if (CONTAINS(mcfg->memseg[ms], mcfg->memzone[idx])) {
800 offset = mcfg->memzone[idx].addr_64 -
801 mcfg->memseg[ms].addr_64;
802 mcfg->memzone[idx].ioremap_addr = mcfg->memseg[ms].ioremap_addr +
811 rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
814 for (i = 0; i < mcfg->memzone_idx; i++) {
815 mz = &mcfg->memzone[i];
817 /* check if memzone has a ring prefix */
818 if (strncmp(mz->name, RTE_RING_MZ_PREFIX,
819 sizeof(RTE_RING_MZ_PREFIX) - 1) != 0)
822 r = (struct rte_ring*) (mz->addr_64);
824 te = rte_zmalloc("RING_TAILQ_ENTRY", sizeof(*te), 0);
826 RTE_LOG(ERR, EAL, "Cannot allocate ring tailq entry!\n");
830 te->data = (void *) r;
832 TAILQ_INSERT_TAIL(ring_list, te, next);
834 RTE_LOG(DEBUG, EAL, "Found ring: '%s' at %p\n", r->name, mz->addr);
836 rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
838 #ifdef RTE_LIBRTE_IVSHMEM_DEBUG
839 rte_memzone_dump(stdout);
840 rte_ring_list_dump(stdout);
846 /* initialize ivshmem structures */
847 int rte_eal_ivshmem_init(void)
849 struct rte_pci_device * dev;
850 struct rte_pci_resource * res;
854 /* initialize everything to 0 */
855 memset(path, 0, sizeof(path));
856 ivshmem_config = NULL;
858 pagesz = getpagesize();
860 RTE_LOG(DEBUG, EAL, "Searching for IVSHMEM devices...\n");
862 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
864 if (open_shared_config() < 0) {
865 RTE_LOG(ERR, EAL, "Could not open IVSHMEM config!\n");
871 TAILQ_FOREACH(dev, &pci_device_list, next) {
873 if (is_ivshmem_device(dev)) {
875 /* IVSHMEM memory is always on BAR2 */
876 res = &dev->mem_resource[2];
878 /* if we don't have a BAR2 */
882 /* construct pci device path */
883 snprintf(path, sizeof(path), IVSHMEM_RESOURCE_PATH,
884 dev->addr.domain, dev->addr.bus, dev->addr.devid,
887 /* try to find memseg */
888 fd = open(path, O_RDWR);
890 RTE_LOG(ERR, EAL, "Could not open %s\n", path);
894 /* check if it's a DPDK IVSHMEM device */
895 ret = has_ivshmem_metadata(fd, res->len);
900 /* config file creation is deferred until the first
901 * DPDK device is found. then, it has to be created
903 if (ivshmem_config == NULL &&
904 create_shared_config() < 0) {
905 RTE_LOG(ERR, EAL, "Could not create IVSHMEM config!\n");
910 if (read_metadata(path, sizeof(path), fd, res->len) < 0) {
911 RTE_LOG(ERR, EAL, "Could not read metadata from"
912 " device %02x:%02x.%x!\n", dev->addr.bus,
913 dev->addr.devid, dev->addr.function);
918 if (ivshmem_config->pci_devs_idx == RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS) {
919 RTE_LOG(WARNING, EAL,
920 "IVSHMEM PCI device limit exceeded. Increase "
921 "CONFIG_RTE_LIBRTE_IVSHMEM_MAX_PCI_DEVS in "
922 "your config file.\n");
926 RTE_LOG(INFO, EAL, "Found IVSHMEM device %02x:%02x.%x\n",
927 dev->addr.bus, dev->addr.devid, dev->addr.function);
929 ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].ioremap_addr = res->phys_addr;
930 snprintf(ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].path,
931 sizeof(ivshmem_config->pci_devs[ivshmem_config->pci_devs_idx].path),
934 ivshmem_config->pci_devs_idx++;
938 RTE_LOG(ERR, EAL, "Could not read IVSHMEM device: %s\n",
943 /* not a DPDK device */
945 RTE_LOG(DEBUG, EAL, "Skipping non-DPDK IVSHMEM device\n");
947 /* close the BAR fd */
953 /* ivshmem_config is not NULL only if config was created and/or mapped */
954 if (ivshmem_config) {
955 if (map_all_segments() < 0) {
956 RTE_LOG(ERR, EAL, "Mapping IVSHMEM segments failed!\n");
961 RTE_LOG(DEBUG, EAL, "No IVSHMEM configuration found! \n");