lib/librte_eal/linuxapp/eal/eal_memory.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33 /*   BSD LICENSE
  34  *
  35  *   Copyright(c) 2013 6WIND.
  36  *
  37  *   Redistribution and use in source and binary forms, with or without
  38  *   modification, are permitted provided that the following conditions
  39  *   are met:
  40  *
  41  *     * Redistributions of source code must retain the above copyright
  42  *       notice, this list of conditions and the following disclaimer.
  43  *     * Redistributions in binary form must reproduce the above copyright
  44  *       notice, this list of conditions and the following disclaimer in
  45  *       the documentation and/or other materials provided with the
  46  *       distribution.
  47  *     * Neither the name of 6WIND S.A. nor the names of its
  48  *       contributors may be used to endorse or promote products derived
  49  *       from this software without specific prior written permission.
  50  *
  51  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  52  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  53  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  54  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  55  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  56  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  57  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  58  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  59  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  60  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  61  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  62  */
  63
  64 #define _FILE_OFFSET_BITS 64
  65 #include <errno.h>
  66 #include <stdarg.h>
  67 #include <stdlib.h>
  68 #include <stdio.h>
  69 #include <stdint.h>
  70 #include <inttypes.h>
  71 #include <string.h>
  72 #include <stdarg.h>
  73 #include <sys/mman.h>
  74 #include <sys/types.h>
  75 #include <sys/stat.h>
  76 #include <sys/queue.h>
  77 #include <sys/file.h>
  78 #include <unistd.h>
  79 #include <limits.h>
  80 #include <errno.h>
  81 #include <sys/ioctl.h>
  82 #include <sys/time.h>
  83
  84 #include <rte_log.h>
  85 #include <rte_memory.h>
  86 #include <rte_memzone.h>
  87 #include <rte_launch.h>
  88 #include <rte_tailq.h>
  89 #include <rte_eal.h>
  90 #include <rte_eal_memconfig.h>
  91 #include <rte_per_lcore.h>
  92 #include <rte_lcore.h>
  93 #include <rte_common.h>
  94 #include <rte_string_fns.h>
  95
  96 #include "eal_private.h"
  97 #include "eal_internal_cfg.h"
  98 #include "eal_filesystem.h"
  99 #include "eal_hugepages.h"
 100
 101 /**
 102  * @file
 103  * Huge page mapping under linux
 104  *
 105  * To reserve a big contiguous amount of memory, we use the hugepage
 106  * feature of linux. For that, we need to have hugetlbfs mounted. This
 107  * code will create many files in this directory (one per page) and
 108  * map them in virtual memory. For each page, we will retrieve its
 109  * physical address and remap it in order to have a virtual contiguous
 110  * zone as well as a physical contiguous zone.
 111  */
 112
 113 static uint64_t baseaddr_offset;
 114
 115 #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space"
 116
 117 /* Lock page in physical memory and prevent from swapping. */
 118 int
 119 rte_mem_lock_page(const void *virt)
 120 {
 121         unsigned long virtual = (unsigned long)virt;
 122         int page_size = getpagesize();
 123         unsigned long aligned = (virtual & ~ (page_size - 1));
 124         return mlock((void*)aligned, page_size);
 125 }
 126
 127 /*
 128  * Get physical address of any mapped virtual address in the current process.
 129  */
 130 phys_addr_t
 131 rte_mem_virt2phy(const void *virtaddr)
 132 {
 133         int fd;
 134         uint64_t page, physaddr;
 135         unsigned long virt_pfn;
 136         int page_size;
 137         off_t offset;
 138
 139         /* standard page size */
 140         page_size = getpagesize();
 141
 142         fd = open("/proc/self/pagemap", O_RDONLY);
 143         if (fd < 0) {
 144                 RTE_LOG(ERR, EAL, "%s(): cannot open /proc/self/pagemap: %s\n",
 145                         __func__, strerror(errno));
 146                 return RTE_BAD_PHYS_ADDR;
 147         }
 148
 149         virt_pfn = (unsigned long)virtaddr / page_size;
 150         offset = sizeof(uint64_t) * virt_pfn;
 151         if (lseek(fd, offset, SEEK_SET) == (off_t) -1) {
 152                 RTE_LOG(ERR, EAL, "%s(): seek error in /proc/self/pagemap: %s\n",
 153                                 __func__, strerror(errno));
 154                 close(fd);
 155                 return RTE_BAD_PHYS_ADDR;
 156         }
 157         if (read(fd, &page, sizeof(uint64_t)) < 0) {
 158                 RTE_LOG(ERR, EAL, "%s(): cannot read /proc/self/pagemap: %s\n",
 159                                 __func__, strerror(errno));
 160                 close(fd);
 161                 return RTE_BAD_PHYS_ADDR;
 162         }
 163
 164         /*
 165          * the pfn (page frame number) are bits 0-54 (see
 166          * pagemap.txt in linux Documentation)
 167          */
 168         physaddr = ((page & 0x7fffffffffffffULL) * page_size)
 169                 + ((unsigned long)virtaddr % page_size);
 170         close(fd);
 171         return physaddr;
 172 }
 173
 174 /*
 175  * For each hugepage in hugepg_tbl, fill the physaddr value. We find
 176  * it by browsing the /proc/self/pagemap special file.
 177  */
 178 static int
 179 find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
 180 {
 181         unsigned i;
 182         phys_addr_t addr;
 183
 184         for (i = 0; i < hpi->num_pages[0]; i++) {
 185                 addr = rte_mem_virt2phy(hugepg_tbl[i].orig_va);
 186                 if (addr == RTE_BAD_PHYS_ADDR)
 187                         return -1;
 188                 hugepg_tbl[i].physaddr = addr;
 189         }
 190         return 0;
 191 }
 192
 193 /*
 194  * Check whether address-space layout randomization is enabled in
 195  * the kernel. This is important for multi-process as it can prevent
 196  * two processes mapping data to the same virtual address
 197  * Returns:
 198  *    0 - address space randomization disabled
 199  *    1/2 - address space randomization enabled
 200  *    negative error code on error
 201  */
 202 static int
 203 aslr_enabled(void)
 204 {
 205         char c;
 206         int retval, fd = open(RANDOMIZE_VA_SPACE_FILE, O_RDONLY);
 207         if (fd < 0)
 208                 return -errno;
 209         retval = read(fd, &c, 1);
 210         close(fd);
 211         if (retval < 0)
 212                 return -errno;
 213         if (retval == 0)
 214                 return -EIO;
 215         switch (c) {
 216                 case '0' : return 0;
 217                 case '1' : return 1;
 218                 case '2' : return 2;
 219                 default: return -EINVAL;
 220         }
 221 }
 222
 223 /*
 224  * Try to mmap *size bytes in /dev/zero. If it is successful, return the
 225  * pointer to the mmap'd area and keep *size unmodified. Else, retry
 226  * with a smaller zone: decrease *size by hugepage_sz until it reaches
 227  * 0. In this case, return NULL. Note: this function returns an address
 228  * which is a multiple of hugepage size.
 229  */
 230 static void *
 231 get_virtual_area(size_t *size, size_t hugepage_sz)
 232 {
 233         void *addr;
 234         int fd;
 235         long aligned_addr;
 236
 237         if (internal_config.base_virtaddr != 0) {
 238                 addr = (void*) (uintptr_t) (internal_config.base_virtaddr +
 239                                 baseaddr_offset);
 240         }
 241         else addr = NULL;
 242
 243         RTE_LOG(INFO, EAL, "Ask a virtual area of 0x%zx bytes\n", *size);
 244
 245         fd = open("/dev/zero", O_RDONLY);
 246         if (fd < 0){
 247                 RTE_LOG(ERR, EAL, "Cannot open /dev/zero\n");
 248                 return NULL;
 249         }
 250         do {
 251                 addr = mmap(addr,
 252                                 (*size) + hugepage_sz, PROT_READ, MAP_PRIVATE, fd, 0);
 253                 if (addr == MAP_FAILED)
 254                         *size -= hugepage_sz;
 255         } while (addr == MAP_FAILED && *size > 0);
 256
 257         if (addr == MAP_FAILED) {
 258                 close(fd);
 259                 RTE_LOG(INFO, EAL, "Cannot get a virtual area\n");
 260                 return NULL;
 261         }
 262
 263         munmap(addr, (*size) + hugepage_sz);
 264         close(fd);
 265
 266         /* align addr to a huge page size boundary */
 267         aligned_addr = (long)addr;
 268         aligned_addr += (hugepage_sz - 1);
 269         aligned_addr &= (~(hugepage_sz - 1));
 270         addr = (void *)(aligned_addr);
 271
 272         RTE_LOG(INFO, EAL, "Virtual area found at %p (size = 0x%zx)\n",
 273                 addr, *size);
 274
 275         /* increment offset */
 276         baseaddr_offset += *size;
 277
 278         return addr;
 279 }
 280
 281 /*
 282  * Mmap all hugepages of hugepage table: it first open a file in
 283  * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the
 284  * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored
 285  * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to
 286  * map continguous physical blocks in contiguous virtual blocks.
 287  */
 288 static int
 289 map_all_hugepages(struct hugepage_file *hugepg_tbl,
 290                 struct hugepage_info *hpi, int orig)
 291 {
 292         int fd;
 293         unsigned i;
 294         void *virtaddr;
 295         void *vma_addr = NULL;
 296         size_t vma_len = 0;
 297
 298 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 299         RTE_SET_USED(vma_len);
 300 #endif
 301
 302         for (i = 0; i < hpi->num_pages[0]; i++) {
 303                 size_t hugepage_sz = hpi->hugepage_sz;
 304
 305                 if (orig) {
 306                         hugepg_tbl[i].file_id = i;
 307                         hugepg_tbl[i].size = hugepage_sz;
 308 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 309                         eal_get_hugefile_temp_path(hugepg_tbl[i].filepath,
 310                                         sizeof(hugepg_tbl[i].filepath), hpi->hugedir,
 311                                         hugepg_tbl[i].file_id);
 312 #else
 313                         eal_get_hugefile_path(hugepg_tbl[i].filepath,
 314                                         sizeof(hugepg_tbl[i].filepath), hpi->hugedir,
 315                                         hugepg_tbl[i].file_id);
 316 #endif
 317                         hugepg_tbl[i].filepath[sizeof(hugepg_tbl[i].filepath) - 1] = '\0';
 318                 }
 319 #ifndef RTE_ARCH_X86_64
 320                 /* for 32-bit systems, don't remap 1G pages, just reuse original
 321                  * map address as final map address.
 322                  */
 323                 else if (hugepage_sz == RTE_PGSIZE_1G){
 324                         hugepg_tbl[i].final_va = hugepg_tbl[i].orig_va;
 325                         hugepg_tbl[i].orig_va = NULL;
 326                         continue;
 327                 }
 328 #endif
 329
 330 #ifndef RTE_EAL_SINGLE_FILE_SEGMENTS
 331                 else if (vma_len == 0) {
 332                         unsigned j, num_pages;
 333
 334                         /* reserve a virtual area for next contiguous
 335                          * physical block: count the number of
 336                          * contiguous physical pages. */
 337                         for (j = i+1; j < hpi->num_pages[0] ; j++) {
 338                                 if (hugepg_tbl[j].physaddr !=
 339                                     hugepg_tbl[j-1].physaddr + hugepage_sz)
 340                                         break;
 341                         }
 342                         num_pages = j - i;
 343                         vma_len = num_pages * hugepage_sz;
 344
 345                         /* get the biggest virtual memory area up to
 346                          * vma_len. If it fails, vma_addr is NULL, so
 347                          * let the kernel provide the address. */
 348                         vma_addr = get_virtual_area(&vma_len, hpi->hugepage_sz);
 349                         if (vma_addr == NULL)
 350                                 vma_len = hugepage_sz;
 351                 }
 352 #endif
 353
 354                 /* try to create hugepage file */
 355                 fd = open(hugepg_tbl[i].filepath, O_CREAT | O_RDWR, 0755);
 356                 if (fd < 0) {
 357                         RTE_LOG(ERR, EAL, "%s(): open failed: %s\n", __func__,
 358                                         strerror(errno));
 359                         return -1;
 360                 }
 361
 362                 virtaddr = mmap(vma_addr, hugepage_sz, PROT_READ | PROT_WRITE,
 363                                 MAP_SHARED, fd, 0);
 364                 if (virtaddr == MAP_FAILED) {
 365                         RTE_LOG(ERR, EAL, "%s(): mmap failed: %s\n", __func__,
 366                                         strerror(errno));
 367                         close(fd);
 368                         return -1;
 369                 }
 370
 371                 if (orig) {
 372                         hugepg_tbl[i].orig_va = virtaddr;
 373                         memset(virtaddr, 0, hugepage_sz);
 374                 }
 375                 else {
 376                         hugepg_tbl[i].final_va = virtaddr;
 377                 }
 378
 379                 /* set shared flock on the file. */
 380                 if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
 381                         RTE_LOG(ERR, EAL, "%s(): Locking file failed:%s \n",
 382                                 __func__, strerror(errno));
 383                         close(fd);
 384                         return -1;
 385                 }
 386
 387                 close(fd);
 388
 389                 vma_addr = (char *)vma_addr + hugepage_sz;
 390                 vma_len -= hugepage_sz;
 391         }
 392         return 0;
 393 }
 394
 395 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 396
 397 /*
 398  * Remaps all hugepages into single file segments
 399  */
 400 static int
 401 remap_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
 402 {
 403         int fd;
 404         unsigned i = 0, j, num_pages, page_idx = 0;
 405         void *vma_addr = NULL, *old_addr = NULL, *page_addr = NULL;
 406         size_t vma_len = 0;
 407         size_t hugepage_sz = hpi->hugepage_sz;
 408         size_t total_size, offset;
 409         char filepath[MAX_HUGEPAGE_PATH];
 410         phys_addr_t physaddr;
 411         int socket;
 412
 413         while (i < hpi->num_pages[0]) {
 414
 415 #ifndef RTE_ARCH_X86_64
 416                 /* for 32-bit systems, don't remap 1G pages, just reuse original
 417                  * map address as final map address.
 418                  */
 419                 if (hugepage_sz == RTE_PGSIZE_1G){
 420                         hugepg_tbl[i].final_va = hugepg_tbl[i].orig_va;
 421                         hugepg_tbl[i].orig_va = NULL;
 422                         i++;
 423                         continue;
 424                 }
 425 #endif
 426
 427                 /* reserve a virtual area for next contiguous
 428                  * physical block: count the number of
 429                  * contiguous physical pages. */
 430                 for (j = i+1; j < hpi->num_pages[0] ; j++) {
 431                         if (hugepg_tbl[j].physaddr != hugepg_tbl[j-1].physaddr + hugepage_sz)
 432                                 break;
 433                 }
 434                 num_pages = j - i;
 435                 vma_len = num_pages * hugepage_sz;
 436
 437                 socket = hugepg_tbl[i].socket_id;
 438
 439                 /* get the biggest virtual memory area up to
 440                  * vma_len. If it fails, vma_addr is NULL, so
 441                  * let the kernel provide the address. */
 442                 vma_addr = get_virtual_area(&vma_len, hpi->hugepage_sz);
 443
 444                 /* If we can't find a big enough virtual area, work out how many pages
 445                  * we are going to get */
 446                 if (vma_addr == NULL)
 447                         j = i + 1;
 448                 else if (vma_len != num_pages * hugepage_sz) {
 449                         num_pages = vma_len / hugepage_sz;
 450                         j = i + num_pages;
 451
 452                 }
 453
 454                 hugepg_tbl[page_idx].file_id = page_idx;
 455                 eal_get_hugefile_path(filepath,
 456                                 sizeof(filepath),
 457                                 hpi->hugedir,
 458                                 hugepg_tbl[page_idx].file_id);
 459
 460                 /* try to create hugepage file */
 461                 fd = open(filepath, O_CREAT | O_RDWR, 0755);
 462                 if (fd < 0) {
 463                         RTE_LOG(ERR, EAL, "%s(): open failed: %s\n", __func__, strerror(errno));
 464                         return -1;
 465                 }
 466
 467                 total_size = 0;
 468                 for (;i < j; i++) {
 469
 470                         /* unmap current segment */
 471                         if (total_size > 0)
 472                                 munmap(vma_addr, total_size);
 473
 474                         /* unmap original page */
 475                         munmap(hugepg_tbl[i].orig_va, hugepage_sz);
 476                         unlink(hugepg_tbl[i].filepath);
 477
 478                         total_size += hugepage_sz;
 479
 480                         old_addr = vma_addr;
 481
 482                         /* map new, bigger segment */
 483                         vma_addr = mmap(vma_addr, total_size,
 484                                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
 485
 486                         if (vma_addr == MAP_FAILED || vma_addr != old_addr) {
 487                                 RTE_LOG(ERR, EAL, "%s(): mmap failed: %s\n", __func__, strerror(errno));
 488                                 close(fd);
 489                                 return -1;
 490                         }
 491
 492                         /* touch the page. this is needed because kernel postpones mapping
 493                          * creation until the first page fault. with this, we pin down
 494                          * the page and it is marked as used and gets into process' pagemap.
 495                          */
 496                         for (offset = 0; offset < total_size; offset += hugepage_sz)
 497                                 *((volatile uint8_t*) RTE_PTR_ADD(vma_addr, offset));
 498                 }
 499
 500                 /* set shared flock on the file. */
 501                 if (flock(fd, LOCK_SH | LOCK_NB) == -1) {
 502                         RTE_LOG(ERR, EAL, "%s(): Locking file failed:%s \n",
 503                                 __func__, strerror(errno));
 504                         close(fd);
 505                         return -1;
 506                 }
 507
 508                 rte_snprintf(hugepg_tbl[page_idx].filepath, MAX_HUGEPAGE_PATH, "%s",
 509                                 filepath);
 510
 511                 physaddr = rte_mem_virt2phy(vma_addr);
 512
 513                 if (physaddr == RTE_BAD_PHYS_ADDR)
 514                         return -1;
 515
 516                 hugepg_tbl[page_idx].final_va = vma_addr;
 517
 518                 hugepg_tbl[page_idx].physaddr = physaddr;
 519
 520                 hugepg_tbl[page_idx].repeated = num_pages;
 521
 522                 hugepg_tbl[page_idx].socket_id = socket;
 523
 524                 close(fd);
 525
 526                 /* verify the memory segment - that is, check that every VA corresponds
 527                  * to the physical address we expect to see
 528                  */
 529                 for (offset = 0; offset < vma_len; offset += hugepage_sz) {
 530                         uint64_t expected_physaddr;
 531
 532                         expected_physaddr = hugepg_tbl[page_idx].physaddr + offset;
 533                         page_addr = RTE_PTR_ADD(vma_addr, offset);
 534                         physaddr = rte_mem_virt2phy(page_addr);
 535
 536                         if (physaddr != expected_physaddr) {
 537                                 RTE_LOG(ERR, EAL, "Segment sanity check failed: wrong physaddr "
 538                                                 "at %p (offset 0x%" PRIx64 ": 0x%" PRIx64
 539                                                 " (expected 0x%" PRIx64 ")\n",
 540                                                 page_addr, offset, physaddr, expected_physaddr);
 541                                 return -1;
 542                         }
 543                 }
 544
 545                 /* zero out the whole segment */
 546                 memset(hugepg_tbl[page_idx].final_va, 0, total_size);
 547
 548                 page_idx++;
 549         }
 550
 551         /* zero out the rest */
 552         memset(&hugepg_tbl[page_idx], 0, (hpi->num_pages[0] - page_idx) * sizeof(struct hugepage_file));
 553         return page_idx;
 554 }
 555 #else/* RTE_EAL_SINGLE_FILE_SEGMENTS=n */
 556
 557 /* Unmap all hugepages from original mapping */
 558 static int
 559 unmap_all_hugepages_orig(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
 560 {
 561         unsigned i;
 562         for (i = 0; i < hpi->num_pages[0]; i++) {
 563                 if (hugepg_tbl[i].orig_va) {
 564                         munmap(hugepg_tbl[i].orig_va, hpi->hugepage_sz);
 565                         hugepg_tbl[i].orig_va = NULL;
 566                 }
 567         }
 568         return 0;
 569 }
 570 #endif /* RTE_EAL_SINGLE_FILE_SEGMENTS */
 571
 572 /*
 573  * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge
 574  * page.
 575  */
 576 static int
 577 find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
 578 {
 579         int socket_id;
 580         char *end, *nodestr;
 581         unsigned i, hp_count = 0;
 582         uint64_t virt_addr;
 583         char buf[BUFSIZ];
 584         char hugedir_str[PATH_MAX];
 585         FILE *f;
 586
 587         f = fopen("/proc/self/numa_maps", "r");
 588         if (f == NULL) {
 589                 RTE_LOG(INFO, EAL, "cannot open /proc/self/numa_maps,"
 590                                 " consider that all memory is in socket_id 0\n");
 591                 return 0;
 592         }
 593
 594         rte_snprintf(hugedir_str, sizeof(hugedir_str),
 595                         "%s/", hpi->hugedir);
 596
 597         /* parse numa map */
 598         while (fgets(buf, sizeof(buf), f) != NULL) {
 599
 600                 /* ignore non huge page */
 601                 if (strstr(buf, " huge ") == NULL &&
 602                                 strstr(buf, hugedir_str) == NULL)
 603                         continue;
 604
 605                 /* get zone addr */
 606                 virt_addr = strtoull(buf, &end, 16);
 607                 if (virt_addr == 0 || end == buf) {
 608                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
 609                         goto error;
 610                 }
 611
 612                 /* get node id (socket id) */
 613                 nodestr = strstr(buf, " N");
 614                 if (nodestr == NULL) {
 615                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
 616                         goto error;
 617                 }
 618                 nodestr += 2;
 619                 end = strstr(nodestr, "=");
 620                 if (end == NULL) {
 621                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
 622                         goto error;
 623                 }
 624                 end[0] = '\0';
 625                 end = NULL;
 626
 627                 socket_id = strtoul(nodestr, &end, 0);
 628                 if ((nodestr[0] == '\0') || (end == NULL) || (*end != '\0')) {
 629                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
 630                         goto error;
 631                 }
 632
 633                 /* if we find this page in our mappings, set socket_id */
 634                 for (i = 0; i < hpi->num_pages[0]; i++) {
 635                         void *va = (void *)(unsigned long)virt_addr;
 636                         if (hugepg_tbl[i].orig_va == va) {
 637                                 hugepg_tbl[i].socket_id = socket_id;
 638                                 hp_count++;
 639                         }
 640                 }
 641         }
 642
 643         if (hp_count < hpi->num_pages[0])
 644                 goto error;
 645
 646         fclose(f);
 647         return 0;
 648
 649 error:
 650         fclose(f);
 651         return -1;
 652 }
 653
 654 /*
 655  * Sort the hugepg_tbl by physical address (lower addresses first). We
 656  * use a slow algorithm, but we won't have millions of pages, and this
 657  * is only done at init time.
 658  */
 659 static int
 660 sort_by_physaddr(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
 661 {
 662         unsigned i, j;
 663         int smallest_idx;
 664         uint64_t smallest_addr;
 665         struct hugepage_file tmp;
 666
 667         for (i = 0; i < hpi->num_pages[0]; i++) {
 668                 smallest_addr = 0;
 669                 smallest_idx = -1;
 670
 671                 /*
 672                  * browse all entries starting at 'i', and find the
 673                  * entry with the smallest addr
 674                  */
 675                 for (j=i; j< hpi->num_pages[0]; j++) {
 676
 677                         if (smallest_addr == 0 ||
 678                             hugepg_tbl[j].physaddr < smallest_addr) {
 679                                 smallest_addr = hugepg_tbl[j].physaddr;
 680                                 smallest_idx = j;
 681                         }
 682                 }
 683
 684                 /* should not happen */
 685                 if (smallest_idx == -1) {
 686                         RTE_LOG(ERR, EAL, "%s(): error in physaddr sorting\n", __func__);
 687                         return -1;
 688                 }
 689
 690                 /* swap the 2 entries in the table */
 691                 memcpy(&tmp, &hugepg_tbl[smallest_idx], sizeof(struct hugepage_file));
 692                 memcpy(&hugepg_tbl[smallest_idx], &hugepg_tbl[i],
 693                                 sizeof(struct hugepage_file));
 694                 memcpy(&hugepg_tbl[i], &tmp, sizeof(struct hugepage_file));
 695         }
 696         return 0;
 697 }
 698
 699 /*
 700  * Uses mmap to create a shared memory area for storage of data
 701  * Used in this file to store the hugepage file map on disk
 702  */
 703 static void *
 704 create_shared_memory(const char *filename, const size_t mem_size)
 705 {
 706         void *retval;
 707         int fd = open(filename, O_CREAT | O_RDWR, 0666);
 708         if (fd < 0)
 709                 return NULL;
 710         if (ftruncate(fd, mem_size) < 0) {
 711                 close(fd);
 712                 return NULL;
 713         }
 714         retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
 715         close(fd);
 716         return retval;
 717 }
 718
 719 /*
 720  * this copies *active* hugepages from one hugepage table to another.
 721  * destination is typically the shared memory.
 722  */
 723 static int
 724 copy_hugepages_to_shared_mem(struct hugepage_file * dst, int dest_size,
 725                 const struct hugepage_file * src, int src_size)
 726 {
 727         int src_pos, dst_pos = 0;
 728
 729         for (src_pos = 0; src_pos < src_size; src_pos++) {
 730                 if (src[src_pos].final_va != NULL) {
 731                         /* error on overflow attempt */
 732                         if (dst_pos == dest_size)
 733                                 return -1;
 734                         memcpy(&dst[dst_pos], &src[src_pos], sizeof(struct hugepage_file));
 735                         dst_pos++;
 736                 }
 737         }
 738         return 0;
 739 }
 740
 741 /*
 742  * unmaps hugepages that are not going to be used. since we originally allocate
 743  * ALL hugepages (not just those we need), additional unmapping needs to be done.
 744  */
 745 static int
 746 unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl,
 747                 struct hugepage_info *hpi,
 748                 unsigned num_hp_info)
 749 {
 750         unsigned socket, size;
 751         int page, nrpages = 0;
 752
 753         /* get total number of hugepages */
 754         for (size = 0; size < num_hp_info; size++)
 755                 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++)
 756                         nrpages += internal_config.hugepage_info[size].num_pages[socket];
 757
 758         for (size = 0; size < num_hp_info; size++) {
 759                 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
 760                         unsigned pages_found = 0;
 761
 762                         /* traverse until we have unmapped all the unused pages */
 763                         for (page = 0; page < nrpages; page++) {
 764                                 struct hugepage_file *hp = &hugepg_tbl[page];
 765
 766 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 767                                 /* if this page was already cleared */
 768                                 if (hp->final_va == NULL)
 769                                         continue;
 770 #endif
 771
 772                                 /* find a page that matches the criteria */
 773                                 if ((hp->size == hpi[size].hugepage_sz) &&
 774                                                 (hp->socket_id == (int) socket)) {
 775
 776                                         /* if we skipped enough pages, unmap the rest */
 777                                         if (pages_found == hpi[size].num_pages[socket]) {
 778                                                 uint64_t unmap_len;
 779
 780 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 781                                                 unmap_len = hp->size * hp->repeated;
 782 #else
 783                                                 unmap_len = hp->size;
 784 #endif
 785
 786                                                 /* get start addr and len of the remaining segment */
 787                                                 munmap(hp->final_va, (size_t) unmap_len);
 788
 789                                                 hp->final_va = NULL;
 790                                                 if (unlink(hp->filepath) == -1) {
 791                                                         RTE_LOG(ERR, EAL, "%s(): Removing %s failed: %s\n",
 792                                                                         __func__, hp->filepath, strerror(errno));
 793                                                         return -1;
 794                                                 }
 795                                         }
 796 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
 797                                         /* else, check how much do we need to map */
 798                                         else {
 799                                                 int nr_pg_left =
 800                                                                 hpi[size].num_pages[socket] - pages_found;
 801
 802                                                 /* if we need enough memory to fit into the segment */
 803                                                 if (hp->repeated <= nr_pg_left) {
 804                                                         pages_found += hp->repeated;
 805                                                 }
 806                                                 /* truncate the segment */
 807                                                 else {
 808                                                         uint64_t final_size = nr_pg_left * hp->size;
 809                                                         uint64_t seg_size = hp->repeated * hp->size;
 810
 811                                                         void * unmap_va = RTE_PTR_ADD(hp->final_va,
 812                                                                         final_size);
 813                                                         int fd;
 814
 815                                                         munmap(unmap_va, seg_size - final_size);
 816
 817                                                         fd = open(hp->filepath, O_RDWR);
 818                                                         if (fd < 0) {
 819                                                                 RTE_LOG(ERR, EAL, "Cannot open %s: %s\n",
 820                                                                                 hp->filepath, strerror(errno));
 821                                                                 return -1;
 822                                                         }
 823                                                         if (ftruncate(fd, final_size) < 0) {
 824                                                                 RTE_LOG(ERR, EAL, "Cannot truncate %s: %s\n",
 825                                                                                 hp->filepath, strerror(errno));
 826                                                                 return -1;
 827                                                         }
 828                                                         close(fd);
 829
 830                                                         pages_found += nr_pg_left;
 831                                                         hp->repeated = nr_pg_left;
 832                                                 }
 833                                         }
 834 #else
 835                                         /* else, lock the page and skip */
 836                                         else
 837                                                 pages_found++;
 838 #endif
 839
 840                                 } /* match page */
 841                         } /* foreach page */
 842                 } /* foreach socket */
 843         } /* foreach pagesize */
 844
 845         return 0;
 846 }
 847
 848 static inline uint64_t
 849 get_socket_mem_size(int socket)
 850 {
 851         uint64_t size = 0;
 852         unsigned i;
 853
 854         for (i = 0; i < internal_config.num_hugepage_sizes; i++){
 855                 struct hugepage_info *hpi = &internal_config.hugepage_info[i];
 856                 if (hpi->hugedir != NULL)
 857                         size += hpi->hugepage_sz * hpi->num_pages[socket];
 858         }
 859
 860         return (size);
 861 }
 862
 863 /*
 864  * This function is a NUMA-aware equivalent of calc_num_pages.
 865  * It takes in the list of hugepage sizes and the
 866  * number of pages thereof, and calculates the best number of
 867  * pages of each size to fulfill the request for <memory> ram
 868  */
 869 static int
 870 calc_num_pages_per_socket(uint64_t * memory,
 871                 struct hugepage_info *hp_info,
 872                 struct hugepage_info *hp_used,
 873                 unsigned num_hp_info)
 874 {
 875         unsigned socket, j, i = 0;
 876         unsigned requested, available;
 877         int total_num_pages = 0;
 878         uint64_t remaining_mem, cur_mem;
 879         uint64_t total_mem = internal_config.memory;
 880
 881         if (num_hp_info == 0)
 882                 return -1;
 883
 884         /* if specific memory amounts per socket weren't requested */
 885         if (internal_config.force_sockets == 0) {
 886                 int cpu_per_socket[RTE_MAX_NUMA_NODES];
 887                 size_t default_size, total_size;
 888                 unsigned lcore_id;
 889
 890                 /* Compute number of cores per socket */
 891                 memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
 892                 RTE_LCORE_FOREACH(lcore_id) {
 893                         cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
 894                 }
 895
 896                 /*
 897                  * Automatically spread requested memory amongst detected sockets according
 898                  * to number of cores from cpu mask present on each socket
 899                  */
 900                 total_size = internal_config.memory;
 901                 for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
 902
 903                         /* Set memory amount per socket */
 904                         default_size = (internal_config.memory * cpu_per_socket[socket])
 905                                         / rte_lcore_count();
 906
 907                         /* Limit to maximum available memory on socket */
 908                         default_size = RTE_MIN(default_size, get_socket_mem_size(socket));
 909
 910                         /* Update sizes */
 911                         memory[socket] = default_size;
 912                         total_size -= default_size;
 913                 }
 914
 915                 /*
 916                  * If some memory is remaining, try to allocate it by getting all
 917                  * available memory from sockets, one after the other
 918                  */
 919                 for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
 920                         /* take whatever is available */
 921                         default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],
 922                                                total_size);
 923
 924                         /* Update sizes */
 925                         memory[socket] += default_size;
 926                         total_size -= default_size;
 927                 }
 928         }
 929
 930         for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem != 0; socket++) {
 931                 /* skips if the memory on specific socket wasn't requested */
 932                 for (i = 0; i < num_hp_info && memory[socket] != 0; i++){
 933                         hp_used[i].hugedir = hp_info[i].hugedir;
 934                         hp_used[i].num_pages[socket] = RTE_MIN(
 935                                         memory[socket] / hp_info[i].hugepage_sz,
 936                                         hp_info[i].num_pages[socket]);
 937
 938                         cur_mem = hp_used[i].num_pages[socket] *
 939                                         hp_used[i].hugepage_sz;
 940
 941                         memory[socket] -= cur_mem;
 942                         total_mem -= cur_mem;
 943
 944                         total_num_pages += hp_used[i].num_pages[socket];
 945
 946                         /* check if we have met all memory requests */
 947                         if (memory[socket] == 0)
 948                                 break;
 949
 950                         /* check if we have any more pages left at this size, if so
 951                          * move on to next size */
 952                         if (hp_used[i].num_pages[socket] == hp_info[i].num_pages[socket])
 953                                 continue;
 954                         /* At this point we know that there are more pages available that are
 955                          * bigger than the memory we want, so lets see if we can get enough
 956                          * from other page sizes.
 957                          */
 958                         remaining_mem = 0;
 959                         for (j = i+1; j < num_hp_info; j++)
 960                                 remaining_mem += hp_info[j].hugepage_sz *
 961                                 hp_info[j].num_pages[socket];
 962
 963                         /* is there enough other memory, if not allocate another page and quit */
 964                         if (remaining_mem < memory[socket]){
 965                                 cur_mem = RTE_MIN(memory[socket],
 966                                                 hp_info[i].hugepage_sz);
 967                                 memory[socket] -= cur_mem;
 968                                 total_mem -= cur_mem;
 969                                 hp_used[i].num_pages[socket]++;
 970                                 total_num_pages++;
 971                                 break; /* we are done with this socket*/
 972                         }
 973                 }
 974                 /* if we didn't satisfy all memory requirements per socket */
 975                 if (memory[socket] > 0) {
 976                         /* to prevent icc errors */
 977                         requested = (unsigned) (internal_config.socket_mem[socket] /
 978                                         0x100000);
 979                         available = requested -
 980                                         ((unsigned) (memory[socket] / 0x100000));
 981                         RTE_LOG(INFO, EAL, "Not enough memory available on socket %u! "
 982                                         "Requested: %uMB, available: %uMB\n", socket,
 983                                         requested, available);
 984                         return -1;
 985                 }
 986         }
 987
 988         /* if we didn't satisfy total memory requirements */
 989         if (total_mem > 0) {
 990                 requested = (unsigned) (internal_config.memory / 0x100000);
 991                 available = requested - (unsigned) (total_mem / 0x100000);
 992                 RTE_LOG(INFO, EAL, "Not enough memory available! Requested: %uMB,"
 993                                 " available: %uMB\n", requested, available);
 994                 return -1;
 995         }
 996         return total_num_pages;
 997 }
 998
 999 /*
1000  * Prepare physical memory mapping: fill configuration structure with
1001  * these infos, return 0 on success.
1002  *  1. map N huge pages in separate files in hugetlbfs
1003  *  2. find associated physical addr
1004  *  3. find associated NUMA socket ID
1005  *  4. sort all huge pages by physical address
1006  *  5. remap these N huge pages in the correct order
1007  *  6. unmap the first mapping
1008  *  7. fill memsegs in configuration with contiguous zones
1009  */
1010 static int
1011 rte_eal_hugepage_init(void)
1012 {
1013         struct rte_mem_config *mcfg;
1014         struct hugepage_file *hugepage, *tmp_hp = NULL;
1015         struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
1016
1017         uint64_t memory[RTE_MAX_NUMA_NODES];
1018
1019         unsigned hp_offset;
1020         int i, j, new_memseg;
1021         int nr_hugefiles, nr_hugepages = 0;
1022         void *addr;
1023 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1024         int new_pages_count[MAX_HUGEPAGE_SIZES];
1025 #endif
1026
1027         memset(used_hp, 0, sizeof(used_hp));
1028
1029         /* get pointer to global configuration */
1030         mcfg = rte_eal_get_configuration()->mem_config;
1031
1032         /* hugetlbfs can be disabled */
1033         if (internal_config.no_hugetlbfs) {
1034                 addr = malloc(internal_config.memory);
1035                 mcfg->memseg[0].phys_addr = (phys_addr_t)(uintptr_t)addr;
1036                 mcfg->memseg[0].addr = addr;
1037                 mcfg->memseg[0].len = internal_config.memory;
1038                 mcfg->memseg[0].socket_id = SOCKET_ID_ANY;
1039                 return 0;
1040         }
1041
1042 /* check if app runs on Xen Dom0 */
1043         if (internal_config.xen_dom0_support) {
1044 #ifdef RTE_LIBRTE_XEN_DOM0
1045                 /* use dom0_mm kernel driver to init memory */
1046                 if (rte_xen_dom0_memory_init() < 0)
1047                         return -1;
1048                 else
1049                         return 0;
1050 #endif
1051         }
1052
1053
1054         /* calculate total number of hugepages available. at this point we haven't
1055          * yet started sorting them so they all are on socket 0 */
1056         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1057                 /* meanwhile, also initialize used_hp hugepage sizes in used_hp */
1058                 used_hp[i].hugepage_sz = internal_config.hugepage_info[i].hugepage_sz;
1059
1060                 nr_hugepages += internal_config.hugepage_info[i].num_pages[0];
1061         }
1062
1063         /*
1064          * allocate a memory area for hugepage table.
1065          * this isn't shared memory yet. due to the fact that we need some
1066          * processing done on these pages, shared memory will be created
1067          * at a later stage.
1068          */
1069         tmp_hp = malloc(nr_hugepages * sizeof(struct hugepage_file));
1070         if (tmp_hp == NULL)
1071                 goto fail;
1072
1073         memset(tmp_hp, 0, nr_hugepages * sizeof(struct hugepage_file));
1074
1075         hp_offset = 0; /* where we start the current page size entries */
1076
1077         /* map all hugepages and sort them */
1078         for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){
1079                 struct hugepage_info *hpi;
1080
1081                 /*
1082                  * we don't yet mark hugepages as used at this stage, so
1083                  * we just map all hugepages available to the system
1084                  * all hugepages are still located on socket 0
1085                  */
1086                 hpi = &internal_config.hugepage_info[i];
1087
1088                 if (hpi->num_pages[0] == 0)
1089                         continue;
1090
1091                 /* map all hugepages available */
1092                 if (map_all_hugepages(&tmp_hp[hp_offset], hpi, 1) < 0){
1093                         RTE_LOG(DEBUG, EAL, "Failed to mmap %u MB hugepages\n",
1094                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1095                         goto fail;
1096                 }
1097
1098                 /* find physical addresses and sockets for each hugepage */
1099                 if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0){
1100                         RTE_LOG(DEBUG, EAL, "Failed to find phys addr for %u MB pages\n",
1101                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1102                         goto fail;
1103                 }
1104
1105                 if (find_numasocket(&tmp_hp[hp_offset], hpi) < 0){
1106                         RTE_LOG(DEBUG, EAL, "Failed to find NUMA socket for %u MB pages\n",
1107                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1108                         goto fail;
1109                 }
1110
1111                 if (sort_by_physaddr(&tmp_hp[hp_offset], hpi) < 0)
1112                         goto fail;
1113
1114 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1115                 /* remap all hugepages into single file segments */
1116                 new_pages_count[i] = remap_all_hugepages(&tmp_hp[hp_offset], hpi);
1117                 if (new_pages_count[i] < 0){
1118                         RTE_LOG(DEBUG, EAL, "Failed to remap %u MB pages\n",
1119                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1120                         goto fail;
1121                 }
1122
1123                 /* we have processed a num of hugepages of this size, so inc offset */
1124                 hp_offset += new_pages_count[i];
1125 #else
1126                 /* remap all hugepages */
1127                 if (map_all_hugepages(&tmp_hp[hp_offset], hpi, 0) < 0){
1128                         RTE_LOG(DEBUG, EAL, "Failed to remap %u MB pages\n",
1129                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1130                         goto fail;
1131                 }
1132
1133                 /* unmap original mappings */
1134                 if (unmap_all_hugepages_orig(&tmp_hp[hp_offset], hpi) < 0)
1135                         goto fail;
1136
1137                 /* we have processed a num of hugepages of this size, so inc offset */
1138                 hp_offset += hpi->num_pages[0];
1139 #endif
1140         }
1141
1142 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1143         nr_hugefiles = 0;
1144         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1145                 nr_hugefiles += new_pages_count[i];
1146         }
1147 #else
1148         nr_hugefiles = nr_hugepages;
1149 #endif
1150
1151
1152         /* clean out the numbers of pages */
1153         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++)
1154                 for (j = 0; j < RTE_MAX_NUMA_NODES; j++)
1155                         internal_config.hugepage_info[i].num_pages[j] = 0;
1156
1157         /* get hugepages for each socket */
1158         for (i = 0; i < nr_hugefiles; i++) {
1159                 int socket = tmp_hp[i].socket_id;
1160
1161                 /* find a hugepage info with right size and increment num_pages */
1162                 for (j = 0; j < (int) internal_config.num_hugepage_sizes; j++) {
1163                         if (tmp_hp[i].size ==
1164                                         internal_config.hugepage_info[j].hugepage_sz) {
1165 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1166                                         internal_config.hugepage_info[j].num_pages[socket] +=
1167                                                 tmp_hp[i].repeated;
1168 #else
1169                                 internal_config.hugepage_info[j].num_pages[socket]++;
1170 #endif
1171                         }
1172                 }
1173         }
1174
1175         /* make a copy of socket_mem, needed for number of pages calculation */
1176         for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
1177                 memory[i] = internal_config.socket_mem[i];
1178
1179         /* calculate final number of pages */
1180         nr_hugepages = calc_num_pages_per_socket(memory,
1181                         internal_config.hugepage_info, used_hp,
1182                         internal_config.num_hugepage_sizes);
1183
1184         /* error if not enough memory available */
1185         if (nr_hugepages < 0)
1186                 goto fail;
1187
1188         /* reporting in! */
1189         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1190                 for (j = 0; j < RTE_MAX_NUMA_NODES; j++) {
1191                         if (used_hp[i].num_pages[j] > 0) {
1192                                 RTE_LOG(INFO, EAL,
1193                                                 "Requesting %u pages of size %uMB"
1194                                                 " from socket %i\n",
1195                                                 used_hp[i].num_pages[j],
1196                                                 (unsigned)
1197                                                         (used_hp[i].hugepage_sz / 0x100000),
1198                                                 j);
1199                         }
1200                 }
1201         }
1202
1203         /* create shared memory */
1204         hugepage = create_shared_memory(eal_hugepage_info_path(),
1205                         nr_hugefiles * sizeof(struct hugepage_file));
1206
1207         if (hugepage == NULL) {
1208                 RTE_LOG(ERR, EAL, "Failed to create shared memory!\n");
1209                 goto fail;
1210         }
1211         memset(hugepage, 0, nr_hugefiles * sizeof(struct hugepage_file));
1212
1213         /*
1214          * unmap pages that we won't need (looks at used_hp).
1215          * also, sets final_va to NULL on pages that were unmapped.
1216          */
1217         if (unmap_unneeded_hugepages(tmp_hp, used_hp,
1218                         internal_config.num_hugepage_sizes) < 0) {
1219                 RTE_LOG(ERR, EAL, "Unmapping and locking hugepages failed!\n");
1220                 goto fail;
1221         }
1222
1223         /*
1224          * copy stuff from malloc'd hugepage* to the actual shared memory.
1225          * this procedure only copies those hugepages that have final_va
1226          * not NULL. has overflow protection.
1227          */
1228         if (copy_hugepages_to_shared_mem(hugepage, nr_hugefiles,
1229                         tmp_hp, nr_hugefiles) < 0) {
1230                 RTE_LOG(ERR, EAL, "Copying tables to shared memory failed!\n");
1231                 goto fail;
1232         }
1233
1234         /* free the temporary hugepage table */
1235         free(tmp_hp);
1236         tmp_hp = NULL;
1237
1238         /* find earliest free memseg - this is needed because in case of IVSHMEM,
1239          * segments might have already been initialized */
1240         for (j = 0; j < RTE_MAX_MEMSEG; j++)
1241                 if (mcfg->memseg[j].addr == NULL) {
1242                         /* move to previous segment and exit loop */
1243                         j--;
1244                         break;
1245                 }
1246
1247         for (i = 0; i < nr_hugefiles; i++) {
1248                 new_memseg = 0;
1249
1250                 /* if this is a new section, create a new memseg */
1251                 if (i == 0)
1252                         new_memseg = 1;
1253                 else if (hugepage[i].socket_id != hugepage[i-1].socket_id)
1254                         new_memseg = 1;
1255                 else if (hugepage[i].size != hugepage[i-1].size)
1256                         new_memseg = 1;
1257                 else if ((hugepage[i].physaddr - hugepage[i-1].physaddr) !=
1258                     hugepage[i].size)
1259                         new_memseg = 1;
1260                 else if (((unsigned long)hugepage[i].final_va -
1261                     (unsigned long)hugepage[i-1].final_va) != hugepage[i].size)
1262                         new_memseg = 1;
1263
1264                 if (new_memseg) {
1265                         j += 1;
1266                         if (j == RTE_MAX_MEMSEG)
1267                                 break;
1268
1269                         mcfg->memseg[j].phys_addr = hugepage[i].physaddr;
1270                         mcfg->memseg[j].addr = hugepage[i].final_va;
1271 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1272                         mcfg->memseg[j].len = hugepage[i].size * hugepage[i].repeated;
1273 #else
1274                         mcfg->memseg[j].len = hugepage[i].size;
1275 #endif
1276                         mcfg->memseg[j].socket_id = hugepage[i].socket_id;
1277                         mcfg->memseg[j].hugepage_sz = hugepage[i].size;
1278                 }
1279                 /* continuation of previous memseg */
1280                 else {
1281                         mcfg->memseg[j].len += mcfg->memseg[j].hugepage_sz;
1282                 }
1283                 hugepage[i].memseg_id = j;
1284         }
1285
1286         if (i < nr_hugefiles) {
1287                 RTE_LOG(ERR, EAL, "Can only reserve %d pages "
1288                         "from %d requested\n"
1289                         "Current %s=%d is not enough\n"
1290                         "Please either increase it or request less amount "
1291                         "of memory.\n",
1292                         i, nr_hugefiles, RTE_STR(CONFIG_RTE_MAX_MEMSEG),
1293                         RTE_MAX_MEMSEG);
1294                 return (-ENOMEM);
1295         }
1296
1297         return 0;
1298
1299 fail:
1300         if (tmp_hp)
1301                 free(tmp_hp);
1302         return -1;
1303 }
1304
1305 /*
1306  * uses fstat to report the size of a file on disk
1307  */
1308 static off_t
1309 getFileSize(int fd)
1310 {
1311         struct stat st;
1312         if (fstat(fd, &st) < 0)
1313                 return 0;
1314         return st.st_size;
1315 }
1316
1317 /*
1318  * This creates the memory mappings in the secondary process to match that of
1319  * the server process. It goes through each memory segment in the DPDK runtime
1320  * configuration and finds the hugepages which form that segment, mapping them
1321  * in order to form a contiguous block in the virtual memory space
1322  */
1323 static int
1324 rte_eal_hugepage_attach(void)
1325 {
1326         const struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
1327         const struct hugepage_file *hp = NULL;
1328         unsigned num_hp = 0;
1329         unsigned i, s = 0; /* s used to track the segment number */
1330         off_t size;
1331         int fd, fd_zero = -1, fd_hugepage = -1;
1332
1333         if (aslr_enabled() > 0) {
1334                 RTE_LOG(WARNING, EAL, "WARNING: Address Space Layout Randomization "
1335                                 "(ASLR) is enabled in the kernel.\n");
1336                 RTE_LOG(WARNING, EAL, "   This may cause issues with mapping memory "
1337                                 "into secondary processes\n");
1338         }
1339
1340         if (internal_config.xen_dom0_support) {
1341 #ifdef RTE_LIBRTE_XEN_DOM0
1342                 if (rte_xen_dom0_memory_attach() < 0) {
1343                         RTE_LOG(ERR, EAL,"Failed to attach memory setments of primay "
1344                                         "process\n");
1345                         return -1;
1346                 }
1347                 return 0;
1348 #endif
1349         }
1350
1351         fd_zero = open("/dev/zero", O_RDONLY);
1352         if (fd_zero < 0) {
1353                 RTE_LOG(ERR, EAL, "Could not open /dev/zero\n");
1354                 goto error;
1355         }
1356         fd_hugepage = open(eal_hugepage_info_path(), O_RDONLY);
1357         if (fd_hugepage < 0) {
1358                 RTE_LOG(ERR, EAL, "Could not open %s\n", eal_hugepage_info_path());
1359                 goto error;
1360         }
1361
1362         /* map all segments into memory to make sure we get the addrs */
1363         for (s = 0; s < RTE_MAX_MEMSEG; ++s) {
1364                 void *base_addr;
1365
1366                 /*
1367                  * the first memory segment with len==0 is the one that
1368                  * follows the last valid segment.
1369                  */
1370                 if (mcfg->memseg[s].len == 0)
1371                         break;
1372
1373 #ifdef RTE_LIBRTE_IVSHMEM
1374                 /*
1375                  * if segment has ioremap address set, it's an IVSHMEM segment and
1376                  * doesn't need mapping as it was already mapped earlier
1377                  */
1378                 if (mcfg->memseg[s].ioremap_addr != 0)
1379                         continue;
1380 #endif
1381
1382                 /*
1383                  * fdzero is mmapped to get a contiguous block of virtual
1384                  * addresses of the appropriate memseg size.
1385                  * use mmap to get identical addresses as the primary process.
1386                  */
1387                 base_addr = mmap(mcfg->memseg[s].addr, mcfg->memseg[s].len,
1388                                  PROT_READ, MAP_PRIVATE, fd_zero, 0);
1389                 if (base_addr == MAP_FAILED ||
1390                     base_addr != mcfg->memseg[s].addr) {
1391                         RTE_LOG(ERR, EAL, "Could not mmap %llu bytes "
1392                                 "in /dev/zero to requested address [%p]: '%s'\n",
1393                                 (unsigned long long)mcfg->memseg[s].len,
1394                                 mcfg->memseg[s].addr, strerror(errno));
1395                         if (aslr_enabled() > 0) {
1396                                 RTE_LOG(ERR, EAL, "It is recommended to "
1397                                         "disable ASLR in the kernel "
1398                                         "and retry running both primary "
1399                                         "and secondary processes\n");
1400                         }
1401                         goto error;
1402                 }
1403         }
1404
1405         size = getFileSize(fd_hugepage);
1406         hp = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd_hugepage, 0);
1407         if (hp == NULL) {
1408                 RTE_LOG(ERR, EAL, "Could not mmap %s\n", eal_hugepage_info_path());
1409                 goto error;
1410         }
1411
1412         num_hp = size / sizeof(struct hugepage_file);
1413         RTE_LOG(DEBUG, EAL, "Analysing %u files\n", num_hp);
1414
1415         s = 0;
1416         while (s < RTE_MAX_MEMSEG && mcfg->memseg[s].len > 0){
1417                 void *addr, *base_addr;
1418                 uintptr_t offset = 0;
1419                 size_t mapping_size;
1420 #ifdef RTE_LIBRTE_IVSHMEM
1421                 /*
1422                  * if segment has ioremap address set, it's an IVSHMEM segment and
1423                  * doesn't need mapping as it was already mapped earlier
1424                  */
1425                 if (mcfg->memseg[s].ioremap_addr != 0) {
1426                         s++;
1427                         continue;
1428                 }
1429 #endif
1430                 /*
1431                  * free previously mapped memory so we can map the
1432                  * hugepages into the space
1433                  */
1434                 base_addr = mcfg->memseg[s].addr;
1435                 munmap(base_addr, mcfg->memseg[s].len);
1436
1437                 /* find the hugepages for this segment and map them
1438                  * we don't need to worry about order, as the server sorted the
1439                  * entries before it did the second mmap of them */
1440                 for (i = 0; i < num_hp && offset < mcfg->memseg[s].len; i++){
1441                         if (hp[i].memseg_id == (int)s){
1442                                 fd = open(hp[i].filepath, O_RDWR);
1443                                 if (fd < 0) {
1444                                         RTE_LOG(ERR, EAL, "Could not open %s\n",
1445                                                 hp[i].filepath);
1446                                         goto error;
1447                                 }
1448 #ifdef RTE_EAL_SINGLE_FILE_SEGMENTS
1449                                 mapping_size = hp[i].size * hp[i].repeated;
1450 #else
1451                                 mapping_size = hp[i].size;
1452 #endif
1453                                 addr = mmap(RTE_PTR_ADD(base_addr, offset),
1454                                                 mapping_size, PROT_READ | PROT_WRITE,
1455                                                 MAP_SHARED, fd, 0);
1456                                 close(fd); /* close file both on success and on failure */
1457                                 if (addr == MAP_FAILED ||
1458                                                 addr != RTE_PTR_ADD(base_addr, offset)) {
1459                                         RTE_LOG(ERR, EAL, "Could not mmap %s\n",
1460                                                 hp[i].filepath);
1461                                         goto error;
1462                                 }
1463                                 offset+=mapping_size;
1464                         }
1465                 }
1466                 RTE_LOG(DEBUG, EAL, "Mapped segment %u of size 0x%llx\n", s,
1467                                 (unsigned long long)mcfg->memseg[s].len);
1468                 s++;
1469         }
1470         /* unmap the hugepage config file, since we are done using it */
1471         munmap((void *)(uintptr_t)hp, size);
1472         close(fd_zero);
1473         close(fd_hugepage);
1474         return 0;
1475
1476 error:
1477         if (fd_zero >= 0)
1478                 close(fd_zero);
1479         if (fd_hugepage >= 0)
1480                 close(fd_hugepage);
1481         return -1;
1482 }
1483
1484 static int
1485 rte_eal_memdevice_init(void)
1486 {
1487         struct rte_config *config;
1488
1489         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
1490                 return 0;
1491
1492         config = rte_eal_get_configuration();
1493         config->mem_config->nchannel = internal_config.force_nchannel;
1494         config->mem_config->nrank = internal_config.force_nrank;
1495
1496         return 0;
1497 }
1498
1499
1500 /* init memory subsystem */
1501 int
1502 rte_eal_memory_init(void)
1503 {
1504         RTE_LOG(INFO, EAL, "Setting up memory...\n");
1505         const int retval = rte_eal_process_type() == RTE_PROC_PRIMARY ?
1506                         rte_eal_hugepage_init() :
1507                         rte_eal_hugepage_attach();
1508         if (retval < 0)
1509                 return -1;
1510
1511         if (internal_config.no_shconf == 0 && rte_eal_memdevice_init() < 0)
1512                 return -1;
1513
1514         return 0;
1515 }