4913fb05e3c038f30bcfc9c1501690f9e9db14dd
[dpdk.git] / lib / librte_eal / linuxapp / eal / eal_memory.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2014 Intel Corporation.
3  * Copyright(c) 2013 6WIND S.A.
4  */
5
6 #define _FILE_OFFSET_BITS 64
7 #include <errno.h>
8 #include <stdarg.h>
9 #include <stdbool.h>
10 #include <stdlib.h>
11 #include <stdio.h>
12 #include <stdint.h>
13 #include <inttypes.h>
14 #include <string.h>
15 #include <sys/mman.h>
16 #include <sys/types.h>
17 #include <sys/stat.h>
18 #include <sys/queue.h>
19 #include <sys/file.h>
20 #include <sys/resource.h>
21 #include <unistd.h>
22 #include <limits.h>
23 #include <sys/ioctl.h>
24 #include <sys/time.h>
25 #include <signal.h>
26 #include <setjmp.h>
27 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
28 #include <numa.h>
29 #include <numaif.h>
30 #endif
31
32 #include <rte_errno.h>
33 #include <rte_log.h>
34 #include <rte_memory.h>
35 #include <rte_launch.h>
36 #include <rte_eal.h>
37 #include <rte_eal_memconfig.h>
38 #include <rte_per_lcore.h>
39 #include <rte_lcore.h>
40 #include <rte_common.h>
41 #include <rte_string_fns.h>
42
43 #include "eal_private.h"
44 #include "eal_memalloc.h"
45 #include "eal_internal_cfg.h"
46 #include "eal_filesystem.h"
47 #include "eal_hugepages.h"
48
49 #define PFN_MASK_SIZE   8
50
51 /**
52  * @file
53  * Huge page mapping under linux
54  *
55  * To reserve a big contiguous amount of memory, we use the hugepage
56  * feature of linux. For that, we need to have hugetlbfs mounted. This
57  * code will create many files in this directory (one per page) and
58  * map them in virtual memory. For each page, we will retrieve its
59  * physical address and remap it in order to have a virtual contiguous
60  * zone as well as a physical contiguous zone.
61  */
62
63 static bool phys_addrs_available = true;
64
65 #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space"
66
67 static void
68 test_phys_addrs_available(void)
69 {
70         uint64_t tmp = 0;
71         phys_addr_t physaddr;
72
73         if (!rte_eal_has_hugepages()) {
74                 RTE_LOG(ERR, EAL,
75                         "Started without hugepages support, physical addresses not available\n");
76                 phys_addrs_available = false;
77                 return;
78         }
79
80         physaddr = rte_mem_virt2phy(&tmp);
81         if (physaddr == RTE_BAD_PHYS_ADDR) {
82                 if (rte_eal_iova_mode() == RTE_IOVA_PA)
83                         RTE_LOG(ERR, EAL,
84                                 "Cannot obtain physical addresses: %s. "
85                                 "Only vfio will function.\n",
86                                 strerror(errno));
87                 phys_addrs_available = false;
88         }
89 }
90
91 /*
92  * Get physical address of any mapped virtual address in the current process.
93  */
94 phys_addr_t
95 rte_mem_virt2phy(const void *virtaddr)
96 {
97         int fd, retval;
98         uint64_t page, physaddr;
99         unsigned long virt_pfn;
100         int page_size;
101         off_t offset;
102
103         /* Cannot parse /proc/self/pagemap, no need to log errors everywhere */
104         if (!phys_addrs_available)
105                 return RTE_BAD_IOVA;
106
107         /* standard page size */
108         page_size = getpagesize();
109
110         fd = open("/proc/self/pagemap", O_RDONLY);
111         if (fd < 0) {
112                 RTE_LOG(ERR, EAL, "%s(): cannot open /proc/self/pagemap: %s\n",
113                         __func__, strerror(errno));
114                 return RTE_BAD_IOVA;
115         }
116
117         virt_pfn = (unsigned long)virtaddr / page_size;
118         offset = sizeof(uint64_t) * virt_pfn;
119         if (lseek(fd, offset, SEEK_SET) == (off_t) -1) {
120                 RTE_LOG(ERR, EAL, "%s(): seek error in /proc/self/pagemap: %s\n",
121                                 __func__, strerror(errno));
122                 close(fd);
123                 return RTE_BAD_IOVA;
124         }
125
126         retval = read(fd, &page, PFN_MASK_SIZE);
127         close(fd);
128         if (retval < 0) {
129                 RTE_LOG(ERR, EAL, "%s(): cannot read /proc/self/pagemap: %s\n",
130                                 __func__, strerror(errno));
131                 return RTE_BAD_IOVA;
132         } else if (retval != PFN_MASK_SIZE) {
133                 RTE_LOG(ERR, EAL, "%s(): read %d bytes from /proc/self/pagemap "
134                                 "but expected %d:\n",
135                                 __func__, retval, PFN_MASK_SIZE);
136                 return RTE_BAD_IOVA;
137         }
138
139         /*
140          * the pfn (page frame number) are bits 0-54 (see
141          * pagemap.txt in linux Documentation)
142          */
143         if ((page & 0x7fffffffffffffULL) == 0)
144                 return RTE_BAD_IOVA;
145
146         physaddr = ((page & 0x7fffffffffffffULL) * page_size)
147                 + ((unsigned long)virtaddr % page_size);
148
149         return physaddr;
150 }
151
152 rte_iova_t
153 rte_mem_virt2iova(const void *virtaddr)
154 {
155         if (rte_eal_iova_mode() == RTE_IOVA_VA)
156                 return (uintptr_t)virtaddr;
157         return rte_mem_virt2phy(virtaddr);
158 }
159
160 /*
161  * For each hugepage in hugepg_tbl, fill the physaddr value. We find
162  * it by browsing the /proc/self/pagemap special file.
163  */
164 static int
165 find_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
166 {
167         unsigned int i;
168         phys_addr_t addr;
169
170         for (i = 0; i < hpi->num_pages[0]; i++) {
171                 addr = rte_mem_virt2phy(hugepg_tbl[i].orig_va);
172                 if (addr == RTE_BAD_PHYS_ADDR)
173                         return -1;
174                 hugepg_tbl[i].physaddr = addr;
175         }
176         return 0;
177 }
178
179 /*
180  * For each hugepage in hugepg_tbl, fill the physaddr value sequentially.
181  */
182 static int
183 set_physaddrs(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
184 {
185         unsigned int i;
186         static phys_addr_t addr;
187
188         for (i = 0; i < hpi->num_pages[0]; i++) {
189                 hugepg_tbl[i].physaddr = addr;
190                 addr += hugepg_tbl[i].size;
191         }
192         return 0;
193 }
194
195 /*
196  * Check whether address-space layout randomization is enabled in
197  * the kernel. This is important for multi-process as it can prevent
198  * two processes mapping data to the same virtual address
199  * Returns:
200  *    0 - address space randomization disabled
201  *    1/2 - address space randomization enabled
202  *    negative error code on error
203  */
204 static int
205 aslr_enabled(void)
206 {
207         char c;
208         int retval, fd = open(RANDOMIZE_VA_SPACE_FILE, O_RDONLY);
209         if (fd < 0)
210                 return -errno;
211         retval = read(fd, &c, 1);
212         close(fd);
213         if (retval < 0)
214                 return -errno;
215         if (retval == 0)
216                 return -EIO;
217         switch (c) {
218                 case '0' : return 0;
219                 case '1' : return 1;
220                 case '2' : return 2;
221                 default: return -EINVAL;
222         }
223 }
224
225 static sigjmp_buf huge_jmpenv;
226
227 static void huge_sigbus_handler(int signo __rte_unused)
228 {
229         siglongjmp(huge_jmpenv, 1);
230 }
231
232 /* Put setjmp into a wrap method to avoid compiling error. Any non-volatile,
233  * non-static local variable in the stack frame calling sigsetjmp might be
234  * clobbered by a call to longjmp.
235  */
236 static int huge_wrap_sigsetjmp(void)
237 {
238         return sigsetjmp(huge_jmpenv, 1);
239 }
240
241 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
242 /* Callback for numa library. */
243 void numa_error(char *where)
244 {
245         RTE_LOG(ERR, EAL, "%s failed: %s\n", where, strerror(errno));
246 }
247 #endif
248
249 /*
250  * Mmap all hugepages of hugepage table: it first open a file in
251  * hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the
252  * virtual address is stored in hugepg_tbl[i].orig_va, else it is stored
253  * in hugepg_tbl[i].final_va. The second mapping (when orig is 0) tries to
254  * map contiguous physical blocks in contiguous virtual blocks.
255  */
256 static unsigned
257 map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi,
258                   uint64_t *essential_memory __rte_unused)
259 {
260         int fd;
261         unsigned i;
262         void *virtaddr;
263 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
264         int node_id = -1;
265         int essential_prev = 0;
266         int oldpolicy;
267         struct bitmask *oldmask = NULL;
268         bool have_numa = true;
269         unsigned long maxnode = 0;
270
271         /* Check if kernel supports NUMA. */
272         if (numa_available() != 0) {
273                 RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n");
274                 have_numa = false;
275         }
276
277         if (have_numa) {
278                 RTE_LOG(DEBUG, EAL, "Trying to obtain current memory policy.\n");
279                 oldmask = numa_allocate_nodemask();
280                 if (get_mempolicy(&oldpolicy, oldmask->maskp,
281                                   oldmask->size + 1, 0, 0) < 0) {
282                         RTE_LOG(ERR, EAL,
283                                 "Failed to get current mempolicy: %s. "
284                                 "Assuming MPOL_DEFAULT.\n", strerror(errno));
285                         oldpolicy = MPOL_DEFAULT;
286                 }
287                 for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
288                         if (internal_config.socket_mem[i])
289                                 maxnode = i + 1;
290         }
291 #endif
292
293         for (i = 0; i < hpi->num_pages[0]; i++) {
294                 struct hugepage_file *hf = &hugepg_tbl[i];
295                 uint64_t hugepage_sz = hpi->hugepage_sz;
296
297 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
298                 if (maxnode) {
299                         unsigned int j;
300
301                         for (j = 0; j < maxnode; j++)
302                                 if (essential_memory[j])
303                                         break;
304
305                         if (j == maxnode) {
306                                 node_id = (node_id + 1) % maxnode;
307                                 while (!internal_config.socket_mem[node_id]) {
308                                         node_id++;
309                                         node_id %= maxnode;
310                                 }
311                                 essential_prev = 0;
312                         } else {
313                                 node_id = j;
314                                 essential_prev = essential_memory[j];
315
316                                 if (essential_memory[j] < hugepage_sz)
317                                         essential_memory[j] = 0;
318                                 else
319                                         essential_memory[j] -= hugepage_sz;
320                         }
321
322                         RTE_LOG(DEBUG, EAL,
323                                 "Setting policy MPOL_PREFERRED for socket %d\n",
324                                 node_id);
325                         numa_set_preferred(node_id);
326                 }
327 #endif
328
329                 hf->file_id = i;
330                 hf->size = hugepage_sz;
331                 eal_get_hugefile_path(hf->filepath, sizeof(hf->filepath),
332                                 hpi->hugedir, hf->file_id);
333                 hf->filepath[sizeof(hf->filepath) - 1] = '\0';
334
335                 /* try to create hugepage file */
336                 fd = open(hf->filepath, O_CREAT | O_RDWR, 0600);
337                 if (fd < 0) {
338                         RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,
339                                         strerror(errno));
340                         goto out;
341                 }
342
343                 /* map the segment, and populate page tables,
344                  * the kernel fills this segment with zeros. we don't care where
345                  * this gets mapped - we already have contiguous memory areas
346                  * ready for us to map into.
347                  */
348                 virtaddr = mmap(NULL, hugepage_sz, PROT_READ | PROT_WRITE,
349                                 MAP_SHARED | MAP_POPULATE, fd, 0);
350                 if (virtaddr == MAP_FAILED) {
351                         RTE_LOG(DEBUG, EAL, "%s(): mmap failed: %s\n", __func__,
352                                         strerror(errno));
353                         close(fd);
354                         goto out;
355                 }
356
357                 hf->orig_va = virtaddr;
358
359                 /* In linux, hugetlb limitations, like cgroup, are
360                  * enforced at fault time instead of mmap(), even
361                  * with the option of MAP_POPULATE. Kernel will send
362                  * a SIGBUS signal. To avoid to be killed, save stack
363                  * environment here, if SIGBUS happens, we can jump
364                  * back here.
365                  */
366                 if (huge_wrap_sigsetjmp()) {
367                         RTE_LOG(DEBUG, EAL, "SIGBUS: Cannot mmap more "
368                                 "hugepages of size %u MB\n",
369                                 (unsigned int)(hugepage_sz / 0x100000));
370                         munmap(virtaddr, hugepage_sz);
371                         close(fd);
372                         unlink(hugepg_tbl[i].filepath);
373 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
374                         if (maxnode)
375                                 essential_memory[node_id] =
376                                         essential_prev;
377 #endif
378                         goto out;
379                 }
380                 *(int *)virtaddr = 0;
381
382                 /* set shared lock on the file. */
383                 if (flock(fd, LOCK_SH) < 0) {
384                         RTE_LOG(DEBUG, EAL, "%s(): Locking file failed:%s \n",
385                                 __func__, strerror(errno));
386                         close(fd);
387                         goto out;
388                 }
389
390                 close(fd);
391         }
392
393 out:
394 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
395         if (maxnode) {
396                 RTE_LOG(DEBUG, EAL,
397                         "Restoring previous memory policy: %d\n", oldpolicy);
398                 if (oldpolicy == MPOL_DEFAULT) {
399                         numa_set_localalloc();
400                 } else if (set_mempolicy(oldpolicy, oldmask->maskp,
401                                          oldmask->size + 1) < 0) {
402                         RTE_LOG(ERR, EAL, "Failed to restore mempolicy: %s\n",
403                                 strerror(errno));
404                         numa_set_localalloc();
405                 }
406         }
407         if (oldmask != NULL)
408                 numa_free_cpumask(oldmask);
409 #endif
410         return i;
411 }
412
413 /*
414  * Parse /proc/self/numa_maps to get the NUMA socket ID for each huge
415  * page.
416  */
417 static int
418 find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
419 {
420         int socket_id;
421         char *end, *nodestr;
422         unsigned i, hp_count = 0;
423         uint64_t virt_addr;
424         char buf[BUFSIZ];
425         char hugedir_str[PATH_MAX];
426         FILE *f;
427
428         f = fopen("/proc/self/numa_maps", "r");
429         if (f == NULL) {
430                 RTE_LOG(NOTICE, EAL, "NUMA support not available"
431                         " consider that all memory is in socket_id 0\n");
432                 return 0;
433         }
434
435         snprintf(hugedir_str, sizeof(hugedir_str),
436                         "%s/%s", hpi->hugedir, internal_config.hugefile_prefix);
437
438         /* parse numa map */
439         while (fgets(buf, sizeof(buf), f) != NULL) {
440
441                 /* ignore non huge page */
442                 if (strstr(buf, " huge ") == NULL &&
443                                 strstr(buf, hugedir_str) == NULL)
444                         continue;
445
446                 /* get zone addr */
447                 virt_addr = strtoull(buf, &end, 16);
448                 if (virt_addr == 0 || end == buf) {
449                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
450                         goto error;
451                 }
452
453                 /* get node id (socket id) */
454                 nodestr = strstr(buf, " N");
455                 if (nodestr == NULL) {
456                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
457                         goto error;
458                 }
459                 nodestr += 2;
460                 end = strstr(nodestr, "=");
461                 if (end == NULL) {
462                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
463                         goto error;
464                 }
465                 end[0] = '\0';
466                 end = NULL;
467
468                 socket_id = strtoul(nodestr, &end, 0);
469                 if ((nodestr[0] == '\0') || (end == NULL) || (*end != '\0')) {
470                         RTE_LOG(ERR, EAL, "%s(): error in numa_maps parsing\n", __func__);
471                         goto error;
472                 }
473
474                 /* if we find this page in our mappings, set socket_id */
475                 for (i = 0; i < hpi->num_pages[0]; i++) {
476                         void *va = (void *)(unsigned long)virt_addr;
477                         if (hugepg_tbl[i].orig_va == va) {
478                                 hugepg_tbl[i].socket_id = socket_id;
479                                 hp_count++;
480 #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
481                                 RTE_LOG(DEBUG, EAL,
482                                         "Hugepage %s is on socket %d\n",
483                                         hugepg_tbl[i].filepath, socket_id);
484 #endif
485                         }
486                 }
487         }
488
489         if (hp_count < hpi->num_pages[0])
490                 goto error;
491
492         fclose(f);
493         return 0;
494
495 error:
496         fclose(f);
497         return -1;
498 }
499
500 static int
501 cmp_physaddr(const void *a, const void *b)
502 {
503 #ifndef RTE_ARCH_PPC_64
504         const struct hugepage_file *p1 = a;
505         const struct hugepage_file *p2 = b;
506 #else
507         /* PowerPC needs memory sorted in reverse order from x86 */
508         const struct hugepage_file *p1 = b;
509         const struct hugepage_file *p2 = a;
510 #endif
511         if (p1->physaddr < p2->physaddr)
512                 return -1;
513         else if (p1->physaddr > p2->physaddr)
514                 return 1;
515         else
516                 return 0;
517 }
518
519 /*
520  * Uses mmap to create a shared memory area for storage of data
521  * Used in this file to store the hugepage file map on disk
522  */
523 static void *
524 create_shared_memory(const char *filename, const size_t mem_size)
525 {
526         void *retval;
527         int fd;
528
529         /* if no shared files mode is used, create anonymous memory instead */
530         if (internal_config.no_shconf) {
531                 retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE,
532                                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
533                 if (retval == MAP_FAILED)
534                         return NULL;
535                 return retval;
536         }
537
538         fd = open(filename, O_CREAT | O_RDWR, 0666);
539         if (fd < 0)
540                 return NULL;
541         if (ftruncate(fd, mem_size) < 0) {
542                 close(fd);
543                 return NULL;
544         }
545         retval = mmap(NULL, mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
546         close(fd);
547         if (retval == MAP_FAILED)
548                 return NULL;
549         return retval;
550 }
551
552 /*
553  * this copies *active* hugepages from one hugepage table to another.
554  * destination is typically the shared memory.
555  */
556 static int
557 copy_hugepages_to_shared_mem(struct hugepage_file * dst, int dest_size,
558                 const struct hugepage_file * src, int src_size)
559 {
560         int src_pos, dst_pos = 0;
561
562         for (src_pos = 0; src_pos < src_size; src_pos++) {
563                 if (src[src_pos].orig_va != NULL) {
564                         /* error on overflow attempt */
565                         if (dst_pos == dest_size)
566                                 return -1;
567                         memcpy(&dst[dst_pos], &src[src_pos], sizeof(struct hugepage_file));
568                         dst_pos++;
569                 }
570         }
571         return 0;
572 }
573
574 static int
575 unlink_hugepage_files(struct hugepage_file *hugepg_tbl,
576                 unsigned num_hp_info)
577 {
578         unsigned socket, size;
579         int page, nrpages = 0;
580
581         /* get total number of hugepages */
582         for (size = 0; size < num_hp_info; size++)
583                 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++)
584                         nrpages +=
585                         internal_config.hugepage_info[size].num_pages[socket];
586
587         for (page = 0; page < nrpages; page++) {
588                 struct hugepage_file *hp = &hugepg_tbl[page];
589
590                 if (hp->final_va != NULL && unlink(hp->filepath)) {
591                         RTE_LOG(WARNING, EAL, "%s(): Removing %s failed: %s\n",
592                                 __func__, hp->filepath, strerror(errno));
593                 }
594         }
595         return 0;
596 }
597
598 /*
599  * unmaps hugepages that are not going to be used. since we originally allocate
600  * ALL hugepages (not just those we need), additional unmapping needs to be done.
601  */
602 static int
603 unmap_unneeded_hugepages(struct hugepage_file *hugepg_tbl,
604                 struct hugepage_info *hpi,
605                 unsigned num_hp_info)
606 {
607         unsigned socket, size;
608         int page, nrpages = 0;
609
610         /* get total number of hugepages */
611         for (size = 0; size < num_hp_info; size++)
612                 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++)
613                         nrpages += internal_config.hugepage_info[size].num_pages[socket];
614
615         for (size = 0; size < num_hp_info; size++) {
616                 for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
617                         unsigned pages_found = 0;
618
619                         /* traverse until we have unmapped all the unused pages */
620                         for (page = 0; page < nrpages; page++) {
621                                 struct hugepage_file *hp = &hugepg_tbl[page];
622
623                                 /* find a page that matches the criteria */
624                                 if ((hp->size == hpi[size].hugepage_sz) &&
625                                                 (hp->socket_id == (int) socket)) {
626
627                                         /* if we skipped enough pages, unmap the rest */
628                                         if (pages_found == hpi[size].num_pages[socket]) {
629                                                 uint64_t unmap_len;
630
631                                                 unmap_len = hp->size;
632
633                                                 /* get start addr and len of the remaining segment */
634                                                 munmap(hp->orig_va,
635                                                         (size_t)unmap_len);
636
637                                                 hp->orig_va = NULL;
638                                                 if (unlink(hp->filepath) == -1) {
639                                                         RTE_LOG(ERR, EAL, "%s(): Removing %s failed: %s\n",
640                                                                         __func__, hp->filepath, strerror(errno));
641                                                         return -1;
642                                                 }
643                                         } else {
644                                                 /* lock the page and skip */
645                                                 pages_found++;
646                                         }
647
648                                 } /* match page */
649                         } /* foreach page */
650                 } /* foreach socket */
651         } /* foreach pagesize */
652
653         return 0;
654 }
655
656 static int
657 remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end)
658 {
659         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
660         struct rte_memseg_list *msl;
661         struct rte_fbarray *arr;
662         int cur_page, seg_len;
663         unsigned int msl_idx;
664         int ms_idx;
665         uint64_t page_sz;
666         size_t memseg_len;
667         int socket_id;
668
669         page_sz = hugepages[seg_start].size;
670         socket_id = hugepages[seg_start].socket_id;
671         seg_len = seg_end - seg_start;
672
673         RTE_LOG(DEBUG, EAL, "Attempting to map %" PRIu64 "M on socket %i\n",
674                         (seg_len * page_sz) >> 20ULL, socket_id);
675
676         /* find free space in memseg lists */
677         for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) {
678                 bool empty;
679                 msl = &mcfg->memsegs[msl_idx];
680                 arr = &msl->memseg_arr;
681
682                 if (msl->page_sz != page_sz)
683                         continue;
684                 if (msl->socket_id != socket_id)
685                         continue;
686
687                 /* leave space for a hole if array is not empty */
688                 empty = arr->count == 0;
689                 ms_idx = rte_fbarray_find_next_n_free(arr, 0,
690                                 seg_len + (empty ? 0 : 1));
691
692                 /* memseg list is full? */
693                 if (ms_idx < 0)
694                         continue;
695
696                 /* leave some space between memsegs, they are not IOVA
697                  * contiguous, so they shouldn't be VA contiguous either.
698                  */
699                 if (!empty)
700                         ms_idx++;
701                 break;
702         }
703         if (msl_idx == RTE_MAX_MEMSEG_LISTS) {
704                 RTE_LOG(ERR, EAL, "Could not find space for memseg. Please increase %s and/or %s in configuration.\n",
705                                 RTE_STR(CONFIG_RTE_MAX_MEMSEG_PER_TYPE),
706                                 RTE_STR(CONFIG_RTE_MAX_MEM_PER_TYPE));
707                 return -1;
708         }
709
710 #ifdef RTE_ARCH_PPC64
711         /* for PPC64 we go through the list backwards */
712         for (cur_page = seg_end - 1; cur_page >= seg_start;
713                         cur_page--, ms_idx++) {
714 #else
715         for (cur_page = seg_start; cur_page < seg_end; cur_page++, ms_idx++) {
716 #endif
717                 struct hugepage_file *hfile = &hugepages[cur_page];
718                 struct rte_memseg *ms = rte_fbarray_get(arr, ms_idx);
719                 void *addr;
720                 int fd;
721
722                 fd = open(hfile->filepath, O_RDWR);
723                 if (fd < 0) {
724                         RTE_LOG(ERR, EAL, "Could not open '%s': %s\n",
725                                         hfile->filepath, strerror(errno));
726                         return -1;
727                 }
728                 /* set shared lock on the file. */
729                 if (flock(fd, LOCK_SH) < 0) {
730                         RTE_LOG(DEBUG, EAL, "Could not lock '%s': %s\n",
731                                         hfile->filepath, strerror(errno));
732                         close(fd);
733                         return -1;
734                 }
735                 memseg_len = (size_t)page_sz;
736                 addr = RTE_PTR_ADD(msl->base_va, ms_idx * memseg_len);
737
738                 /* we know this address is already mmapped by memseg list, so
739                  * using MAP_FIXED here is safe
740                  */
741                 addr = mmap(addr, page_sz, PROT_READ | PROT_WRITE,
742                                 MAP_SHARED | MAP_POPULATE | MAP_FIXED, fd, 0);
743                 if (addr == MAP_FAILED) {
744                         RTE_LOG(ERR, EAL, "Couldn't remap '%s': %s\n",
745                                         hfile->filepath, strerror(errno));
746                         close(fd);
747                         return -1;
748                 }
749
750                 /* we have a new address, so unmap previous one */
751 #ifndef RTE_ARCH_64
752                 /* in 32-bit legacy mode, we have already unmapped the page */
753                 if (!internal_config.legacy_mem)
754                         munmap(hfile->orig_va, page_sz);
755 #else
756                 munmap(hfile->orig_va, page_sz);
757 #endif
758
759                 hfile->orig_va = NULL;
760                 hfile->final_va = addr;
761
762                 /* rewrite physical addresses in IOVA as VA mode */
763                 if (rte_eal_iova_mode() == RTE_IOVA_VA)
764                         hfile->physaddr = (uintptr_t)addr;
765
766                 /* set up memseg data */
767                 ms->addr = addr;
768                 ms->hugepage_sz = page_sz;
769                 ms->len = memseg_len;
770                 ms->iova = hfile->physaddr;
771                 ms->socket_id = hfile->socket_id;
772                 ms->nchannel = rte_memory_get_nchannel();
773                 ms->nrank = rte_memory_get_nrank();
774
775                 rte_fbarray_set_used(arr, ms_idx);
776
777                 /* store segment fd internally */
778                 if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0)
779                         RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",
780                                 rte_strerror(rte_errno));
781         }
782         RTE_LOG(DEBUG, EAL, "Allocated %" PRIu64 "M on socket %i\n",
783                         (seg_len * page_sz) >> 20, socket_id);
784         return 0;
785 }
786
787 static uint64_t
788 get_mem_amount(uint64_t page_sz, uint64_t max_mem)
789 {
790         uint64_t area_sz, max_pages;
791
792         /* limit to RTE_MAX_MEMSEG_PER_LIST pages or RTE_MAX_MEM_MB_PER_LIST */
793         max_pages = RTE_MAX_MEMSEG_PER_LIST;
794         max_mem = RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_LIST << 20, max_mem);
795
796         area_sz = RTE_MIN(page_sz * max_pages, max_mem);
797
798         /* make sure the list isn't smaller than the page size */
799         area_sz = RTE_MAX(area_sz, page_sz);
800
801         return RTE_ALIGN(area_sz, page_sz);
802 }
803
804 static int
805 free_memseg_list(struct rte_memseg_list *msl)
806 {
807         if (rte_fbarray_destroy(&msl->memseg_arr)) {
808                 RTE_LOG(ERR, EAL, "Cannot destroy memseg list\n");
809                 return -1;
810         }
811         memset(msl, 0, sizeof(*msl));
812         return 0;
813 }
814
815 #define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"
816 static int
817 alloc_memseg_list(struct rte_memseg_list *msl, uint64_t page_sz,
818                 int n_segs, int socket_id, int type_msl_idx)
819 {
820         char name[RTE_FBARRAY_NAME_LEN];
821
822         snprintf(name, sizeof(name), MEMSEG_LIST_FMT, page_sz >> 10, socket_id,
823                  type_msl_idx);
824         if (rte_fbarray_init(&msl->memseg_arr, name, n_segs,
825                         sizeof(struct rte_memseg))) {
826                 RTE_LOG(ERR, EAL, "Cannot allocate memseg list: %s\n",
827                         rte_strerror(rte_errno));
828                 return -1;
829         }
830
831         msl->page_sz = page_sz;
832         msl->socket_id = socket_id;
833         msl->base_va = NULL;
834
835         RTE_LOG(DEBUG, EAL, "Memseg list allocated: 0x%zxkB at socket %i\n",
836                         (size_t)page_sz >> 10, socket_id);
837
838         return 0;
839 }
840
841 static int
842 alloc_va_space(struct rte_memseg_list *msl)
843 {
844         uint64_t page_sz;
845         size_t mem_sz;
846         void *addr;
847         int flags = 0;
848
849 #ifdef RTE_ARCH_PPC_64
850         flags |= MAP_HUGETLB;
851 #endif
852
853         page_sz = msl->page_sz;
854         mem_sz = page_sz * msl->memseg_arr.len;
855
856         addr = eal_get_virtual_area(msl->base_va, &mem_sz, page_sz, 0, flags);
857         if (addr == NULL) {
858                 if (rte_errno == EADDRNOTAVAIL)
859                         RTE_LOG(ERR, EAL, "Could not mmap %llu bytes at [%p] - please use '--base-virtaddr' option\n",
860                                 (unsigned long long)mem_sz, msl->base_va);
861                 else
862                         RTE_LOG(ERR, EAL, "Cannot reserve memory\n");
863                 return -1;
864         }
865         msl->base_va = addr;
866         msl->len = mem_sz;
867
868         return 0;
869 }
870
871 /*
872  * Our VA space is not preallocated yet, so preallocate it here. We need to know
873  * how many segments there are in order to map all pages into one address space,
874  * and leave appropriate holes between segments so that rte_malloc does not
875  * concatenate them into one big segment.
876  *
877  * we also need to unmap original pages to free up address space.
878  */
879 static int __rte_unused
880 prealloc_segments(struct hugepage_file *hugepages, int n_pages)
881 {
882         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
883         int cur_page, seg_start_page, end_seg, new_memseg;
884         unsigned int hpi_idx, socket, i;
885         int n_contig_segs, n_segs;
886         int msl_idx;
887
888         /* before we preallocate segments, we need to free up our VA space.
889          * we're not removing files, and we already have information about
890          * PA-contiguousness, so it is safe to unmap everything.
891          */
892         for (cur_page = 0; cur_page < n_pages; cur_page++) {
893                 struct hugepage_file *hpi = &hugepages[cur_page];
894                 munmap(hpi->orig_va, hpi->size);
895                 hpi->orig_va = NULL;
896         }
897
898         /* we cannot know how many page sizes and sockets we have discovered, so
899          * loop over all of them
900          */
901         for (hpi_idx = 0; hpi_idx < internal_config.num_hugepage_sizes;
902                         hpi_idx++) {
903                 uint64_t page_sz =
904                         internal_config.hugepage_info[hpi_idx].hugepage_sz;
905
906                 for (i = 0; i < rte_socket_count(); i++) {
907                         struct rte_memseg_list *msl;
908
909                         socket = rte_socket_id_by_idx(i);
910                         n_contig_segs = 0;
911                         n_segs = 0;
912                         seg_start_page = -1;
913
914                         for (cur_page = 0; cur_page < n_pages; cur_page++) {
915                                 struct hugepage_file *prev, *cur;
916                                 int prev_seg_start_page = -1;
917
918                                 cur = &hugepages[cur_page];
919                                 prev = cur_page == 0 ? NULL :
920                                                 &hugepages[cur_page - 1];
921
922                                 new_memseg = 0;
923                                 end_seg = 0;
924
925                                 if (cur->size == 0)
926                                         end_seg = 1;
927                                 else if (cur->socket_id != (int) socket)
928                                         end_seg = 1;
929                                 else if (cur->size != page_sz)
930                                         end_seg = 1;
931                                 else if (cur_page == 0)
932                                         new_memseg = 1;
933 #ifdef RTE_ARCH_PPC_64
934                                 /* On PPC64 architecture, the mmap always start
935                                  * from higher address to lower address. Here,
936                                  * physical addresses are in descending order.
937                                  */
938                                 else if ((prev->physaddr - cur->physaddr) !=
939                                                 cur->size)
940                                         new_memseg = 1;
941 #else
942                                 else if ((cur->physaddr - prev->physaddr) !=
943                                                 cur->size)
944                                         new_memseg = 1;
945 #endif
946                                 if (new_memseg) {
947                                         /* if we're already inside a segment,
948                                          * new segment means end of current one
949                                          */
950                                         if (seg_start_page != -1) {
951                                                 end_seg = 1;
952                                                 prev_seg_start_page =
953                                                                 seg_start_page;
954                                         }
955                                         seg_start_page = cur_page;
956                                 }
957
958                                 if (end_seg) {
959                                         if (prev_seg_start_page != -1) {
960                                                 /* we've found a new segment */
961                                                 n_contig_segs++;
962                                                 n_segs += cur_page -
963                                                         prev_seg_start_page;
964                                         } else if (seg_start_page != -1) {
965                                                 /* we didn't find new segment,
966                                                  * but did end current one
967                                                  */
968                                                 n_contig_segs++;
969                                                 n_segs += cur_page -
970                                                                 seg_start_page;
971                                                 seg_start_page = -1;
972                                                 continue;
973                                         } else {
974                                                 /* we're skipping this page */
975                                                 continue;
976                                         }
977                                 }
978                                 /* segment continues */
979                         }
980                         /* check if we missed last segment */
981                         if (seg_start_page != -1) {
982                                 n_contig_segs++;
983                                 n_segs += cur_page - seg_start_page;
984                         }
985
986                         /* if no segments were found, do not preallocate */
987                         if (n_segs == 0)
988                                 continue;
989
990                         /* we now have total number of pages that we will
991                          * allocate for this segment list. add separator pages
992                          * to the total count, and preallocate VA space.
993                          */
994                         n_segs += n_contig_segs - 1;
995
996                         /* now, preallocate VA space for these segments */
997
998                         /* first, find suitable memseg list for this */
999                         for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS;
1000                                         msl_idx++) {
1001                                 msl = &mcfg->memsegs[msl_idx];
1002
1003                                 if (msl->base_va != NULL)
1004                                         continue;
1005                                 break;
1006                         }
1007                         if (msl_idx == RTE_MAX_MEMSEG_LISTS) {
1008                                 RTE_LOG(ERR, EAL, "Not enough space in memseg lists, please increase %s\n",
1009                                         RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
1010                                 return -1;
1011                         }
1012
1013                         /* now, allocate fbarray itself */
1014                         if (alloc_memseg_list(msl, page_sz, n_segs, socket,
1015                                                 msl_idx) < 0)
1016                                 return -1;
1017
1018                         /* finally, allocate VA space */
1019                         if (alloc_va_space(msl) < 0)
1020                                 return -1;
1021                 }
1022         }
1023         return 0;
1024 }
1025
1026 /*
1027  * We cannot reallocate memseg lists on the fly because PPC64 stores pages
1028  * backwards, therefore we have to process the entire memseg first before
1029  * remapping it into memseg list VA space.
1030  */
1031 static int
1032 remap_needed_hugepages(struct hugepage_file *hugepages, int n_pages)
1033 {
1034         int cur_page, seg_start_page, new_memseg, ret;
1035
1036         seg_start_page = 0;
1037         for (cur_page = 0; cur_page < n_pages; cur_page++) {
1038                 struct hugepage_file *prev, *cur;
1039
1040                 new_memseg = 0;
1041
1042                 cur = &hugepages[cur_page];
1043                 prev = cur_page == 0 ? NULL : &hugepages[cur_page - 1];
1044
1045                 /* if size is zero, no more pages left */
1046                 if (cur->size == 0)
1047                         break;
1048
1049                 if (cur_page == 0)
1050                         new_memseg = 1;
1051                 else if (cur->socket_id != prev->socket_id)
1052                         new_memseg = 1;
1053                 else if (cur->size != prev->size)
1054                         new_memseg = 1;
1055 #ifdef RTE_ARCH_PPC_64
1056                 /* On PPC64 architecture, the mmap always start from higher
1057                  * address to lower address. Here, physical addresses are in
1058                  * descending order.
1059                  */
1060                 else if ((prev->physaddr - cur->physaddr) != cur->size)
1061                         new_memseg = 1;
1062 #else
1063                 else if ((cur->physaddr - prev->physaddr) != cur->size)
1064                         new_memseg = 1;
1065 #endif
1066
1067                 if (new_memseg) {
1068                         /* if this isn't the first time, remap segment */
1069                         if (cur_page != 0) {
1070                                 ret = remap_segment(hugepages, seg_start_page,
1071                                                 cur_page);
1072                                 if (ret != 0)
1073                                         return -1;
1074                         }
1075                         /* remember where we started */
1076                         seg_start_page = cur_page;
1077                 }
1078                 /* continuation of previous memseg */
1079         }
1080         /* we were stopped, but we didn't remap the last segment, do it now */
1081         if (cur_page != 0) {
1082                 ret = remap_segment(hugepages, seg_start_page,
1083                                 cur_page);
1084                 if (ret != 0)
1085                         return -1;
1086         }
1087         return 0;
1088 }
1089
1090 static inline uint64_t
1091 get_socket_mem_size(int socket)
1092 {
1093         uint64_t size = 0;
1094         unsigned i;
1095
1096         for (i = 0; i < internal_config.num_hugepage_sizes; i++){
1097                 struct hugepage_info *hpi = &internal_config.hugepage_info[i];
1098                 size += hpi->hugepage_sz * hpi->num_pages[socket];
1099         }
1100
1101         return size;
1102 }
1103
1104 /*
1105  * This function is a NUMA-aware equivalent of calc_num_pages.
1106  * It takes in the list of hugepage sizes and the
1107  * number of pages thereof, and calculates the best number of
1108  * pages of each size to fulfill the request for <memory> ram
1109  */
1110 static int
1111 calc_num_pages_per_socket(uint64_t * memory,
1112                 struct hugepage_info *hp_info,
1113                 struct hugepage_info *hp_used,
1114                 unsigned num_hp_info)
1115 {
1116         unsigned socket, j, i = 0;
1117         unsigned requested, available;
1118         int total_num_pages = 0;
1119         uint64_t remaining_mem, cur_mem;
1120         uint64_t total_mem = internal_config.memory;
1121
1122         if (num_hp_info == 0)
1123                 return -1;
1124
1125         /* if specific memory amounts per socket weren't requested */
1126         if (internal_config.force_sockets == 0) {
1127                 size_t total_size;
1128 #ifdef RTE_ARCH_64
1129                 int cpu_per_socket[RTE_MAX_NUMA_NODES];
1130                 size_t default_size;
1131                 unsigned lcore_id;
1132
1133                 /* Compute number of cores per socket */
1134                 memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
1135                 RTE_LCORE_FOREACH(lcore_id) {
1136                         cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
1137                 }
1138
1139                 /*
1140                  * Automatically spread requested memory amongst detected sockets according
1141                  * to number of cores from cpu mask present on each socket
1142                  */
1143                 total_size = internal_config.memory;
1144                 for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
1145
1146                         /* Set memory amount per socket */
1147                         default_size = (internal_config.memory * cpu_per_socket[socket])
1148                                         / rte_lcore_count();
1149
1150                         /* Limit to maximum available memory on socket */
1151                         default_size = RTE_MIN(default_size, get_socket_mem_size(socket));
1152
1153                         /* Update sizes */
1154                         memory[socket] = default_size;
1155                         total_size -= default_size;
1156                 }
1157
1158                 /*
1159                  * If some memory is remaining, try to allocate it by getting all
1160                  * available memory from sockets, one after the other
1161                  */
1162                 for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
1163                         /* take whatever is available */
1164                         default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],
1165                                                total_size);
1166
1167                         /* Update sizes */
1168                         memory[socket] += default_size;
1169                         total_size -= default_size;
1170                 }
1171 #else
1172                 /* in 32-bit mode, allocate all of the memory only on master
1173                  * lcore socket
1174                  */
1175                 total_size = internal_config.memory;
1176                 for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0;
1177                                 socket++) {
1178                         struct rte_config *cfg = rte_eal_get_configuration();
1179                         unsigned int master_lcore_socket;
1180
1181                         master_lcore_socket =
1182                                 rte_lcore_to_socket_id(cfg->master_lcore);
1183
1184                         if (master_lcore_socket != socket)
1185                                 continue;
1186
1187                         /* Update sizes */
1188                         memory[socket] = total_size;
1189                         break;
1190                 }
1191 #endif
1192         }
1193
1194         for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem != 0; socket++) {
1195                 /* skips if the memory on specific socket wasn't requested */
1196                 for (i = 0; i < num_hp_info && memory[socket] != 0; i++){
1197                         strlcpy(hp_used[i].hugedir, hp_info[i].hugedir,
1198                                 sizeof(hp_used[i].hugedir));
1199                         hp_used[i].num_pages[socket] = RTE_MIN(
1200                                         memory[socket] / hp_info[i].hugepage_sz,
1201                                         hp_info[i].num_pages[socket]);
1202
1203                         cur_mem = hp_used[i].num_pages[socket] *
1204                                         hp_used[i].hugepage_sz;
1205
1206                         memory[socket] -= cur_mem;
1207                         total_mem -= cur_mem;
1208
1209                         total_num_pages += hp_used[i].num_pages[socket];
1210
1211                         /* check if we have met all memory requests */
1212                         if (memory[socket] == 0)
1213                                 break;
1214
1215                         /* check if we have any more pages left at this size, if so
1216                          * move on to next size */
1217                         if (hp_used[i].num_pages[socket] == hp_info[i].num_pages[socket])
1218                                 continue;
1219                         /* At this point we know that there are more pages available that are
1220                          * bigger than the memory we want, so lets see if we can get enough
1221                          * from other page sizes.
1222                          */
1223                         remaining_mem = 0;
1224                         for (j = i+1; j < num_hp_info; j++)
1225                                 remaining_mem += hp_info[j].hugepage_sz *
1226                                 hp_info[j].num_pages[socket];
1227
1228                         /* is there enough other memory, if not allocate another page and quit */
1229                         if (remaining_mem < memory[socket]){
1230                                 cur_mem = RTE_MIN(memory[socket],
1231                                                 hp_info[i].hugepage_sz);
1232                                 memory[socket] -= cur_mem;
1233                                 total_mem -= cur_mem;
1234                                 hp_used[i].num_pages[socket]++;
1235                                 total_num_pages++;
1236                                 break; /* we are done with this socket*/
1237                         }
1238                 }
1239                 /* if we didn't satisfy all memory requirements per socket */
1240                 if (memory[socket] > 0 &&
1241                                 internal_config.socket_mem[socket] != 0) {
1242                         /* to prevent icc errors */
1243                         requested = (unsigned) (internal_config.socket_mem[socket] /
1244                                         0x100000);
1245                         available = requested -
1246                                         ((unsigned) (memory[socket] / 0x100000));
1247                         RTE_LOG(ERR, EAL, "Not enough memory available on socket %u! "
1248                                         "Requested: %uMB, available: %uMB\n", socket,
1249                                         requested, available);
1250                         return -1;
1251                 }
1252         }
1253
1254         /* if we didn't satisfy total memory requirements */
1255         if (total_mem > 0) {
1256                 requested = (unsigned) (internal_config.memory / 0x100000);
1257                 available = requested - (unsigned) (total_mem / 0x100000);
1258                 RTE_LOG(ERR, EAL, "Not enough memory available! Requested: %uMB,"
1259                                 " available: %uMB\n", requested, available);
1260                 return -1;
1261         }
1262         return total_num_pages;
1263 }
1264
1265 static inline size_t
1266 eal_get_hugepage_mem_size(void)
1267 {
1268         uint64_t size = 0;
1269         unsigned i, j;
1270
1271         for (i = 0; i < internal_config.num_hugepage_sizes; i++) {
1272                 struct hugepage_info *hpi = &internal_config.hugepage_info[i];
1273                 if (strnlen(hpi->hugedir, sizeof(hpi->hugedir)) != 0) {
1274                         for (j = 0; j < RTE_MAX_NUMA_NODES; j++) {
1275                                 size += hpi->hugepage_sz * hpi->num_pages[j];
1276                         }
1277                 }
1278         }
1279
1280         return (size < SIZE_MAX) ? (size_t)(size) : SIZE_MAX;
1281 }
1282
1283 static struct sigaction huge_action_old;
1284 static int huge_need_recover;
1285
1286 static void
1287 huge_register_sigbus(void)
1288 {
1289         sigset_t mask;
1290         struct sigaction action;
1291
1292         sigemptyset(&mask);
1293         sigaddset(&mask, SIGBUS);
1294         action.sa_flags = 0;
1295         action.sa_mask = mask;
1296         action.sa_handler = huge_sigbus_handler;
1297
1298         huge_need_recover = !sigaction(SIGBUS, &action, &huge_action_old);
1299 }
1300
1301 static void
1302 huge_recover_sigbus(void)
1303 {
1304         if (huge_need_recover) {
1305                 sigaction(SIGBUS, &huge_action_old, NULL);
1306                 huge_need_recover = 0;
1307         }
1308 }
1309
1310 /*
1311  * Prepare physical memory mapping: fill configuration structure with
1312  * these infos, return 0 on success.
1313  *  1. map N huge pages in separate files in hugetlbfs
1314  *  2. find associated physical addr
1315  *  3. find associated NUMA socket ID
1316  *  4. sort all huge pages by physical address
1317  *  5. remap these N huge pages in the correct order
1318  *  6. unmap the first mapping
1319  *  7. fill memsegs in configuration with contiguous zones
1320  */
1321 static int
1322 eal_legacy_hugepage_init(void)
1323 {
1324         struct rte_mem_config *mcfg;
1325         struct hugepage_file *hugepage = NULL, *tmp_hp = NULL;
1326         struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
1327         struct rte_fbarray *arr;
1328         struct rte_memseg *ms;
1329
1330         uint64_t memory[RTE_MAX_NUMA_NODES];
1331
1332         unsigned hp_offset;
1333         int i, j;
1334         int nr_hugefiles, nr_hugepages = 0;
1335         void *addr;
1336
1337         test_phys_addrs_available();
1338
1339         memset(used_hp, 0, sizeof(used_hp));
1340
1341         /* get pointer to global configuration */
1342         mcfg = rte_eal_get_configuration()->mem_config;
1343
1344         /* hugetlbfs can be disabled */
1345         if (internal_config.no_hugetlbfs) {
1346                 struct rte_memseg_list *msl;
1347                 uint64_t page_sz;
1348                 int n_segs, cur_seg;
1349
1350                 /* nohuge mode is legacy mode */
1351                 internal_config.legacy_mem = 1;
1352
1353                 /* create a memseg list */
1354                 msl = &mcfg->memsegs[0];
1355
1356                 page_sz = RTE_PGSIZE_4K;
1357                 n_segs = internal_config.memory / page_sz;
1358
1359                 if (rte_fbarray_init(&msl->memseg_arr, "nohugemem", n_segs,
1360                                         sizeof(struct rte_memseg))) {
1361                         RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n");
1362                         return -1;
1363                 }
1364
1365                 addr = mmap(NULL, internal_config.memory, PROT_READ | PROT_WRITE,
1366                                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1367                 if (addr == MAP_FAILED) {
1368                         RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,
1369                                         strerror(errno));
1370                         return -1;
1371                 }
1372                 msl->base_va = addr;
1373                 msl->page_sz = page_sz;
1374                 msl->socket_id = 0;
1375                 msl->len = internal_config.memory;
1376
1377                 /* populate memsegs. each memseg is one page long */
1378                 for (cur_seg = 0; cur_seg < n_segs; cur_seg++) {
1379                         arr = &msl->memseg_arr;
1380
1381                         ms = rte_fbarray_get(arr, cur_seg);
1382                         if (rte_eal_iova_mode() == RTE_IOVA_VA)
1383                                 ms->iova = (uintptr_t)addr;
1384                         else
1385                                 ms->iova = RTE_BAD_IOVA;
1386                         ms->addr = addr;
1387                         ms->hugepage_sz = page_sz;
1388                         ms->socket_id = 0;
1389                         ms->len = page_sz;
1390
1391                         rte_fbarray_set_used(arr, cur_seg);
1392
1393                         addr = RTE_PTR_ADD(addr, (size_t)page_sz);
1394                 }
1395                 return 0;
1396         }
1397
1398         /* calculate total number of hugepages available. at this point we haven't
1399          * yet started sorting them so they all are on socket 0 */
1400         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1401                 /* meanwhile, also initialize used_hp hugepage sizes in used_hp */
1402                 used_hp[i].hugepage_sz = internal_config.hugepage_info[i].hugepage_sz;
1403
1404                 nr_hugepages += internal_config.hugepage_info[i].num_pages[0];
1405         }
1406
1407         /*
1408          * allocate a memory area for hugepage table.
1409          * this isn't shared memory yet. due to the fact that we need some
1410          * processing done on these pages, shared memory will be created
1411          * at a later stage.
1412          */
1413         tmp_hp = malloc(nr_hugepages * sizeof(struct hugepage_file));
1414         if (tmp_hp == NULL)
1415                 goto fail;
1416
1417         memset(tmp_hp, 0, nr_hugepages * sizeof(struct hugepage_file));
1418
1419         hp_offset = 0; /* where we start the current page size entries */
1420
1421         huge_register_sigbus();
1422
1423         /* make a copy of socket_mem, needed for balanced allocation. */
1424         for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
1425                 memory[i] = internal_config.socket_mem[i];
1426
1427         /* map all hugepages and sort them */
1428         for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){
1429                 unsigned pages_old, pages_new;
1430                 struct hugepage_info *hpi;
1431
1432                 /*
1433                  * we don't yet mark hugepages as used at this stage, so
1434                  * we just map all hugepages available to the system
1435                  * all hugepages are still located on socket 0
1436                  */
1437                 hpi = &internal_config.hugepage_info[i];
1438
1439                 if (hpi->num_pages[0] == 0)
1440                         continue;
1441
1442                 /* map all hugepages available */
1443                 pages_old = hpi->num_pages[0];
1444                 pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, memory);
1445                 if (pages_new < pages_old) {
1446                         RTE_LOG(DEBUG, EAL,
1447                                 "%d not %d hugepages of size %u MB allocated\n",
1448                                 pages_new, pages_old,
1449                                 (unsigned)(hpi->hugepage_sz / 0x100000));
1450
1451                         int pages = pages_old - pages_new;
1452
1453                         nr_hugepages -= pages;
1454                         hpi->num_pages[0] = pages_new;
1455                         if (pages_new == 0)
1456                                 continue;
1457                 }
1458
1459                 if (phys_addrs_available &&
1460                                 rte_eal_iova_mode() != RTE_IOVA_VA) {
1461                         /* find physical addresses for each hugepage */
1462                         if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) {
1463                                 RTE_LOG(DEBUG, EAL, "Failed to find phys addr "
1464                                         "for %u MB pages\n",
1465                                         (unsigned int)(hpi->hugepage_sz / 0x100000));
1466                                 goto fail;
1467                         }
1468                 } else {
1469                         /* set physical addresses for each hugepage */
1470                         if (set_physaddrs(&tmp_hp[hp_offset], hpi) < 0) {
1471                                 RTE_LOG(DEBUG, EAL, "Failed to set phys addr "
1472                                         "for %u MB pages\n",
1473                                         (unsigned int)(hpi->hugepage_sz / 0x100000));
1474                                 goto fail;
1475                         }
1476                 }
1477
1478                 if (find_numasocket(&tmp_hp[hp_offset], hpi) < 0){
1479                         RTE_LOG(DEBUG, EAL, "Failed to find NUMA socket for %u MB pages\n",
1480                                         (unsigned)(hpi->hugepage_sz / 0x100000));
1481                         goto fail;
1482                 }
1483
1484                 qsort(&tmp_hp[hp_offset], hpi->num_pages[0],
1485                       sizeof(struct hugepage_file), cmp_physaddr);
1486
1487                 /* we have processed a num of hugepages of this size, so inc offset */
1488                 hp_offset += hpi->num_pages[0];
1489         }
1490
1491         huge_recover_sigbus();
1492
1493         if (internal_config.memory == 0 && internal_config.force_sockets == 0)
1494                 internal_config.memory = eal_get_hugepage_mem_size();
1495
1496         nr_hugefiles = nr_hugepages;
1497
1498
1499         /* clean out the numbers of pages */
1500         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++)
1501                 for (j = 0; j < RTE_MAX_NUMA_NODES; j++)
1502                         internal_config.hugepage_info[i].num_pages[j] = 0;
1503
1504         /* get hugepages for each socket */
1505         for (i = 0; i < nr_hugefiles; i++) {
1506                 int socket = tmp_hp[i].socket_id;
1507
1508                 /* find a hugepage info with right size and increment num_pages */
1509                 const int nb_hpsizes = RTE_MIN(MAX_HUGEPAGE_SIZES,
1510                                 (int)internal_config.num_hugepage_sizes);
1511                 for (j = 0; j < nb_hpsizes; j++) {
1512                         if (tmp_hp[i].size ==
1513                                         internal_config.hugepage_info[j].hugepage_sz) {
1514                                 internal_config.hugepage_info[j].num_pages[socket]++;
1515                         }
1516                 }
1517         }
1518
1519         /* make a copy of socket_mem, needed for number of pages calculation */
1520         for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
1521                 memory[i] = internal_config.socket_mem[i];
1522
1523         /* calculate final number of pages */
1524         nr_hugepages = calc_num_pages_per_socket(memory,
1525                         internal_config.hugepage_info, used_hp,
1526                         internal_config.num_hugepage_sizes);
1527
1528         /* error if not enough memory available */
1529         if (nr_hugepages < 0)
1530                 goto fail;
1531
1532         /* reporting in! */
1533         for (i = 0; i < (int) internal_config.num_hugepage_sizes; i++) {
1534                 for (j = 0; j < RTE_MAX_NUMA_NODES; j++) {
1535                         if (used_hp[i].num_pages[j] > 0) {
1536                                 RTE_LOG(DEBUG, EAL,
1537                                         "Requesting %u pages of size %uMB"
1538                                         " from socket %i\n",
1539                                         used_hp[i].num_pages[j],
1540                                         (unsigned)
1541                                         (used_hp[i].hugepage_sz / 0x100000),
1542                                         j);
1543                         }
1544                 }
1545         }
1546
1547         /* create shared memory */
1548         hugepage = create_shared_memory(eal_hugepage_data_path(),
1549                         nr_hugefiles * sizeof(struct hugepage_file));
1550
1551         if (hugepage == NULL) {
1552                 RTE_LOG(ERR, EAL, "Failed to create shared memory!\n");
1553                 goto fail;
1554         }
1555         memset(hugepage, 0, nr_hugefiles * sizeof(struct hugepage_file));
1556
1557         /*
1558          * unmap pages that we won't need (looks at used_hp).
1559          * also, sets final_va to NULL on pages that were unmapped.
1560          */
1561         if (unmap_unneeded_hugepages(tmp_hp, used_hp,
1562                         internal_config.num_hugepage_sizes) < 0) {
1563                 RTE_LOG(ERR, EAL, "Unmapping and locking hugepages failed!\n");
1564                 goto fail;
1565         }
1566
1567         /*
1568          * copy stuff from malloc'd hugepage* to the actual shared memory.
1569          * this procedure only copies those hugepages that have orig_va
1570          * not NULL. has overflow protection.
1571          */
1572         if (copy_hugepages_to_shared_mem(hugepage, nr_hugefiles,
1573                         tmp_hp, nr_hugefiles) < 0) {
1574                 RTE_LOG(ERR, EAL, "Copying tables to shared memory failed!\n");
1575                 goto fail;
1576         }
1577
1578 #ifndef RTE_ARCH_64
1579         /* for legacy 32-bit mode, we did not preallocate VA space, so do it */
1580         if (internal_config.legacy_mem &&
1581                         prealloc_segments(hugepage, nr_hugefiles)) {
1582                 RTE_LOG(ERR, EAL, "Could not preallocate VA space for hugepages\n");
1583                 goto fail;
1584         }
1585 #endif
1586
1587         /* remap all pages we do need into memseg list VA space, so that those
1588          * pages become first-class citizens in DPDK memory subsystem
1589          */
1590         if (remap_needed_hugepages(hugepage, nr_hugefiles)) {
1591                 RTE_LOG(ERR, EAL, "Couldn't remap hugepage files into memseg lists\n");
1592                 goto fail;
1593         }
1594
1595         /* free the hugepage backing files */
1596         if (internal_config.hugepage_unlink &&
1597                 unlink_hugepage_files(tmp_hp, internal_config.num_hugepage_sizes) < 0) {
1598                 RTE_LOG(ERR, EAL, "Unlinking hugepage files failed!\n");
1599                 goto fail;
1600         }
1601
1602         /* free the temporary hugepage table */
1603         free(tmp_hp);
1604         tmp_hp = NULL;
1605
1606         munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file));
1607
1608         /* we're not going to allocate more pages, so release VA space for
1609          * unused memseg lists
1610          */
1611         for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
1612                 struct rte_memseg_list *msl = &mcfg->memsegs[i];
1613                 size_t mem_sz;
1614
1615                 /* skip inactive lists */
1616                 if (msl->base_va == NULL)
1617                         continue;
1618                 /* skip lists where there is at least one page allocated */
1619                 if (msl->memseg_arr.count > 0)
1620                         continue;
1621                 /* this is an unused list, deallocate it */
1622                 mem_sz = msl->len;
1623                 munmap(msl->base_va, mem_sz);
1624                 msl->base_va = NULL;
1625
1626                 /* destroy backing fbarray */
1627                 rte_fbarray_destroy(&msl->memseg_arr);
1628         }
1629
1630         return 0;
1631
1632 fail:
1633         huge_recover_sigbus();
1634         free(tmp_hp);
1635         if (hugepage != NULL)
1636                 munmap(hugepage, nr_hugefiles * sizeof(struct hugepage_file));
1637
1638         return -1;
1639 }
1640
1641 static int __rte_unused
1642 hugepage_count_walk(const struct rte_memseg_list *msl, void *arg)
1643 {
1644         struct hugepage_info *hpi = arg;
1645
1646         if (msl->page_sz != hpi->hugepage_sz)
1647                 return 0;
1648
1649         hpi->num_pages[msl->socket_id] += msl->memseg_arr.len;
1650         return 0;
1651 }
1652
1653 static int
1654 limits_callback(int socket_id, size_t cur_limit, size_t new_len)
1655 {
1656         RTE_SET_USED(socket_id);
1657         RTE_SET_USED(cur_limit);
1658         RTE_SET_USED(new_len);
1659         return -1;
1660 }
1661
1662 static int
1663 eal_hugepage_init(void)
1664 {
1665         struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES];
1666         uint64_t memory[RTE_MAX_NUMA_NODES];
1667         int hp_sz_idx, socket_id;
1668
1669         test_phys_addrs_available();
1670
1671         memset(used_hp, 0, sizeof(used_hp));
1672
1673         for (hp_sz_idx = 0;
1674                         hp_sz_idx < (int) internal_config.num_hugepage_sizes;
1675                         hp_sz_idx++) {
1676 #ifndef RTE_ARCH_64
1677                 struct hugepage_info dummy;
1678                 unsigned int i;
1679 #endif
1680                 /* also initialize used_hp hugepage sizes in used_hp */
1681                 struct hugepage_info *hpi;
1682                 hpi = &internal_config.hugepage_info[hp_sz_idx];
1683                 used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz;
1684
1685 #ifndef RTE_ARCH_64
1686                 /* for 32-bit, limit number of pages on socket to whatever we've
1687                  * preallocated, as we cannot allocate more.
1688                  */
1689                 memset(&dummy, 0, sizeof(dummy));
1690                 dummy.hugepage_sz = hpi->hugepage_sz;
1691                 if (rte_memseg_list_walk(hugepage_count_walk, &dummy) < 0)
1692                         return -1;
1693
1694                 for (i = 0; i < RTE_DIM(dummy.num_pages); i++) {
1695                         hpi->num_pages[i] = RTE_MIN(hpi->num_pages[i],
1696                                         dummy.num_pages[i]);
1697                 }
1698 #endif
1699         }
1700
1701         /* make a copy of socket_mem, needed for balanced allocation. */
1702         for (hp_sz_idx = 0; hp_sz_idx < RTE_MAX_NUMA_NODES; hp_sz_idx++)
1703                 memory[hp_sz_idx] = internal_config.socket_mem[hp_sz_idx];
1704
1705         /* calculate final number of pages */
1706         if (calc_num_pages_per_socket(memory,
1707                         internal_config.hugepage_info, used_hp,
1708                         internal_config.num_hugepage_sizes) < 0)
1709                 return -1;
1710
1711         for (hp_sz_idx = 0;
1712                         hp_sz_idx < (int)internal_config.num_hugepage_sizes;
1713                         hp_sz_idx++) {
1714                 for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES;
1715                                 socket_id++) {
1716                         struct rte_memseg **pages;
1717                         struct hugepage_info *hpi = &used_hp[hp_sz_idx];
1718                         unsigned int num_pages = hpi->num_pages[socket_id];
1719                         int num_pages_alloc, i;
1720
1721                         if (num_pages == 0)
1722                                 continue;
1723
1724                         pages = malloc(sizeof(*pages) * num_pages);
1725
1726                         RTE_LOG(DEBUG, EAL, "Allocating %u pages of size %" PRIu64 "M on socket %i\n",
1727                                 num_pages, hpi->hugepage_sz >> 20, socket_id);
1728
1729                         num_pages_alloc = eal_memalloc_alloc_seg_bulk(pages,
1730                                         num_pages, hpi->hugepage_sz,
1731                                         socket_id, true);
1732                         if (num_pages_alloc < 0) {
1733                                 free(pages);
1734                                 return -1;
1735                         }
1736
1737                         /* mark preallocated pages as unfreeable */
1738                         for (i = 0; i < num_pages_alloc; i++) {
1739                                 struct rte_memseg *ms = pages[i];
1740                                 ms->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE;
1741                         }
1742                         free(pages);
1743                 }
1744         }
1745         /* if socket limits were specified, set them */
1746         if (internal_config.force_socket_limits) {
1747                 unsigned int i;
1748                 for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
1749                         uint64_t limit = internal_config.socket_limit[i];
1750                         if (limit == 0)
1751                                 continue;
1752                         if (rte_mem_alloc_validator_register("socket-limit",
1753                                         limits_callback, i, limit))
1754                                 RTE_LOG(ERR, EAL, "Failed to register socket limits validator callback\n");
1755                 }
1756         }
1757         return 0;
1758 }
1759
1760 /*
1761  * uses fstat to report the size of a file on disk
1762  */
1763 static off_t
1764 getFileSize(int fd)
1765 {
1766         struct stat st;
1767         if (fstat(fd, &st) < 0)
1768                 return 0;
1769         return st.st_size;
1770 }
1771
1772 /*
1773  * This creates the memory mappings in the secondary process to match that of
1774  * the server process. It goes through each memory segment in the DPDK runtime
1775  * configuration and finds the hugepages which form that segment, mapping them
1776  * in order to form a contiguous block in the virtual memory space
1777  */
1778 static int
1779 eal_legacy_hugepage_attach(void)
1780 {
1781         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
1782         struct hugepage_file *hp = NULL;
1783         unsigned int num_hp = 0;
1784         unsigned int i = 0;
1785         unsigned int cur_seg;
1786         off_t size = 0;
1787         int fd, fd_hugepage = -1;
1788
1789         if (aslr_enabled() > 0) {
1790                 RTE_LOG(WARNING, EAL, "WARNING: Address Space Layout Randomization "
1791                                 "(ASLR) is enabled in the kernel.\n");
1792                 RTE_LOG(WARNING, EAL, "   This may cause issues with mapping memory "
1793                                 "into secondary processes\n");
1794         }
1795
1796         test_phys_addrs_available();
1797
1798         fd_hugepage = open(eal_hugepage_data_path(), O_RDONLY);
1799         if (fd_hugepage < 0) {
1800                 RTE_LOG(ERR, EAL, "Could not open %s\n",
1801                                 eal_hugepage_data_path());
1802                 goto error;
1803         }
1804
1805         size = getFileSize(fd_hugepage);
1806         hp = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd_hugepage, 0);
1807         if (hp == MAP_FAILED) {
1808                 RTE_LOG(ERR, EAL, "Could not mmap %s\n",
1809                                 eal_hugepage_data_path());
1810                 goto error;
1811         }
1812
1813         num_hp = size / sizeof(struct hugepage_file);
1814         RTE_LOG(DEBUG, EAL, "Analysing %u files\n", num_hp);
1815
1816         /* map all segments into memory to make sure we get the addrs. the
1817          * segments themselves are already in memseg list (which is shared and
1818          * has its VA space already preallocated), so we just need to map
1819          * everything into correct addresses.
1820          */
1821         for (i = 0; i < num_hp; i++) {
1822                 struct hugepage_file *hf = &hp[i];
1823                 size_t map_sz = hf->size;
1824                 void *map_addr = hf->final_va;
1825                 int msl_idx, ms_idx;
1826                 struct rte_memseg_list *msl;
1827                 struct rte_memseg *ms;
1828
1829                 /* if size is zero, no more pages left */
1830                 if (map_sz == 0)
1831                         break;
1832
1833                 fd = open(hf->filepath, O_RDWR);
1834                 if (fd < 0) {
1835                         RTE_LOG(ERR, EAL, "Could not open %s: %s\n",
1836                                 hf->filepath, strerror(errno));
1837                         goto error;
1838                 }
1839
1840                 map_addr = mmap(map_addr, map_sz, PROT_READ | PROT_WRITE,
1841                                 MAP_SHARED | MAP_FIXED, fd, 0);
1842                 if (map_addr == MAP_FAILED) {
1843                         RTE_LOG(ERR, EAL, "Could not map %s: %s\n",
1844                                 hf->filepath, strerror(errno));
1845                         goto fd_error;
1846                 }
1847
1848                 /* set shared lock on the file. */
1849                 if (flock(fd, LOCK_SH) < 0) {
1850                         RTE_LOG(DEBUG, EAL, "%s(): Locking file failed: %s\n",
1851                                 __func__, strerror(errno));
1852                         goto fd_error;
1853                 }
1854
1855                 /* find segment data */
1856                 msl = rte_mem_virt2memseg_list(map_addr);
1857                 if (msl == NULL) {
1858                         RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg list\n",
1859                                 __func__);
1860                         goto fd_error;
1861                 }
1862                 ms = rte_mem_virt2memseg(map_addr, msl);
1863                 if (ms == NULL) {
1864                         RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg\n",
1865                                 __func__);
1866                         goto fd_error;
1867                 }
1868
1869                 msl_idx = msl - mcfg->memsegs;
1870                 ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
1871                 if (ms_idx < 0) {
1872                         RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg idx\n",
1873                                 __func__);
1874                         goto fd_error;
1875                 }
1876
1877                 /* store segment fd internally */
1878                 if (eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd) < 0)
1879                         RTE_LOG(ERR, EAL, "Could not store segment fd: %s\n",
1880                                 rte_strerror(rte_errno));
1881         }
1882         /* unmap the hugepage config file, since we are done using it */
1883         munmap(hp, size);
1884         close(fd_hugepage);
1885         return 0;
1886
1887 fd_error:
1888         close(fd);
1889 error:
1890         /* map all segments into memory to make sure we get the addrs */
1891         cur_seg = 0;
1892         for (cur_seg = 0; cur_seg < i; cur_seg++) {
1893                 struct hugepage_file *hf = &hp[i];
1894                 size_t map_sz = hf->size;
1895                 void *map_addr = hf->final_va;
1896
1897                 munmap(map_addr, map_sz);
1898         }
1899         if (hp != NULL && hp != MAP_FAILED)
1900                 munmap(hp, size);
1901         if (fd_hugepage >= 0)
1902                 close(fd_hugepage);
1903         return -1;
1904 }
1905
1906 static int
1907 eal_hugepage_attach(void)
1908 {
1909         if (eal_memalloc_sync_with_primary()) {
1910                 RTE_LOG(ERR, EAL, "Could not map memory from primary process\n");
1911                 if (aslr_enabled() > 0)
1912                         RTE_LOG(ERR, EAL, "It is recommended to disable ASLR in the kernel and retry running both primary and secondary processes\n");
1913                 return -1;
1914         }
1915         return 0;
1916 }
1917
1918 int
1919 rte_eal_hugepage_init(void)
1920 {
1921         return internal_config.legacy_mem ?
1922                         eal_legacy_hugepage_init() :
1923                         eal_hugepage_init();
1924 }
1925
1926 int
1927 rte_eal_hugepage_attach(void)
1928 {
1929         return internal_config.legacy_mem ?
1930                         eal_legacy_hugepage_attach() :
1931                         eal_hugepage_attach();
1932 }
1933
1934 int
1935 rte_eal_using_phys_addrs(void)
1936 {
1937         return phys_addrs_available;
1938 }
1939
1940 static int __rte_unused
1941 memseg_primary_init_32(void)
1942 {
1943         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
1944         int active_sockets, hpi_idx, msl_idx = 0;
1945         unsigned int socket_id, i;
1946         struct rte_memseg_list *msl;
1947         uint64_t extra_mem_per_socket, total_extra_mem, total_requested_mem;
1948         uint64_t max_mem;
1949
1950         /* no-huge does not need this at all */
1951         if (internal_config.no_hugetlbfs)
1952                 return 0;
1953
1954         /* this is a giant hack, but desperate times call for desperate
1955          * measures. in legacy 32-bit mode, we cannot preallocate VA space,
1956          * because having upwards of 2 gigabytes of VA space already mapped will
1957          * interfere with our ability to map and sort hugepages.
1958          *
1959          * therefore, in legacy 32-bit mode, we will be initializing memseg
1960          * lists much later - in eal_memory.c, right after we unmap all the
1961          * unneeded pages. this will not affect secondary processes, as those
1962          * should be able to mmap the space without (too many) problems.
1963          */
1964         if (internal_config.legacy_mem)
1965                 return 0;
1966
1967         /* 32-bit mode is a very special case. we cannot know in advance where
1968          * the user will want to allocate their memory, so we have to do some
1969          * heuristics.
1970          */
1971         active_sockets = 0;
1972         total_requested_mem = 0;
1973         if (internal_config.force_sockets)
1974                 for (i = 0; i < rte_socket_count(); i++) {
1975                         uint64_t mem;
1976
1977                         socket_id = rte_socket_id_by_idx(i);
1978                         mem = internal_config.socket_mem[socket_id];
1979
1980                         if (mem == 0)
1981                                 continue;
1982
1983                         active_sockets++;
1984                         total_requested_mem += mem;
1985                 }
1986         else
1987                 total_requested_mem = internal_config.memory;
1988
1989         max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
1990         if (total_requested_mem > max_mem) {
1991                 RTE_LOG(ERR, EAL, "Invalid parameters: 32-bit process can at most use %uM of memory\n",
1992                                 (unsigned int)(max_mem >> 20));
1993                 return -1;
1994         }
1995         total_extra_mem = max_mem - total_requested_mem;
1996         extra_mem_per_socket = active_sockets == 0 ? total_extra_mem :
1997                         total_extra_mem / active_sockets;
1998
1999         /* the allocation logic is a little bit convoluted, but here's how it
2000          * works, in a nutshell:
2001          *  - if user hasn't specified on which sockets to allocate memory via
2002          *    --socket-mem, we allocate all of our memory on master core socket.
2003          *  - if user has specified sockets to allocate memory on, there may be
2004          *    some "unused" memory left (e.g. if user has specified --socket-mem
2005          *    such that not all memory adds up to 2 gigabytes), so add it to all
2006          *    sockets that are in use equally.
2007          *
2008          * page sizes are sorted by size in descending order, so we can safely
2009          * assume that we dispense with bigger page sizes first.
2010          */
2011
2012         /* create memseg lists */
2013         for (i = 0; i < rte_socket_count(); i++) {
2014                 int hp_sizes = (int) internal_config.num_hugepage_sizes;
2015                 uint64_t max_socket_mem, cur_socket_mem;
2016                 unsigned int master_lcore_socket;
2017                 struct rte_config *cfg = rte_eal_get_configuration();
2018                 bool skip;
2019
2020                 socket_id = rte_socket_id_by_idx(i);
2021
2022 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2023                 if (socket_id > 0)
2024                         break;
2025 #endif
2026
2027                 /* if we didn't specifically request memory on this socket */
2028                 skip = active_sockets != 0 &&
2029                                 internal_config.socket_mem[socket_id] == 0;
2030                 /* ...or if we didn't specifically request memory on *any*
2031                  * socket, and this is not master lcore
2032                  */
2033                 master_lcore_socket = rte_lcore_to_socket_id(cfg->master_lcore);
2034                 skip |= active_sockets == 0 && socket_id != master_lcore_socket;
2035
2036                 if (skip) {
2037                         RTE_LOG(DEBUG, EAL, "Will not preallocate memory on socket %u\n",
2038                                         socket_id);
2039                         continue;
2040                 }
2041
2042                 /* max amount of memory on this socket */
2043                 max_socket_mem = (active_sockets != 0 ?
2044                                         internal_config.socket_mem[socket_id] :
2045                                         internal_config.memory) +
2046                                         extra_mem_per_socket;
2047                 cur_socket_mem = 0;
2048
2049                 for (hpi_idx = 0; hpi_idx < hp_sizes; hpi_idx++) {
2050                         uint64_t max_pagesz_mem, cur_pagesz_mem = 0;
2051                         uint64_t hugepage_sz;
2052                         struct hugepage_info *hpi;
2053                         int type_msl_idx, max_segs, total_segs = 0;
2054
2055                         hpi = &internal_config.hugepage_info[hpi_idx];
2056                         hugepage_sz = hpi->hugepage_sz;
2057
2058                         /* check if pages are actually available */
2059                         if (hpi->num_pages[socket_id] == 0)
2060                                 continue;
2061
2062                         max_segs = RTE_MAX_MEMSEG_PER_TYPE;
2063                         max_pagesz_mem = max_socket_mem - cur_socket_mem;
2064
2065                         /* make it multiple of page size */
2066                         max_pagesz_mem = RTE_ALIGN_FLOOR(max_pagesz_mem,
2067                                         hugepage_sz);
2068
2069                         RTE_LOG(DEBUG, EAL, "Attempting to preallocate "
2070                                         "%" PRIu64 "M on socket %i\n",
2071                                         max_pagesz_mem >> 20, socket_id);
2072
2073                         type_msl_idx = 0;
2074                         while (cur_pagesz_mem < max_pagesz_mem &&
2075                                         total_segs < max_segs) {
2076                                 uint64_t cur_mem;
2077                                 unsigned int n_segs;
2078
2079                                 if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
2080                                         RTE_LOG(ERR, EAL,
2081                                                 "No more space in memseg lists, please increase %s\n",
2082                                                 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
2083                                         return -1;
2084                                 }
2085
2086                                 msl = &mcfg->memsegs[msl_idx];
2087
2088                                 cur_mem = get_mem_amount(hugepage_sz,
2089                                                 max_pagesz_mem);
2090                                 n_segs = cur_mem / hugepage_sz;
2091
2092                                 if (alloc_memseg_list(msl, hugepage_sz, n_segs,
2093                                                 socket_id, type_msl_idx)) {
2094                                         /* failing to allocate a memseg list is
2095                                          * a serious error.
2096                                          */
2097                                         RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n");
2098                                         return -1;
2099                                 }
2100
2101                                 if (alloc_va_space(msl)) {
2102                                         /* if we couldn't allocate VA space, we
2103                                          * can try with smaller page sizes.
2104                                          */
2105                                         RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list, retrying with different page size\n");
2106                                         /* deallocate memseg list */
2107                                         if (free_memseg_list(msl))
2108                                                 return -1;
2109                                         break;
2110                                 }
2111
2112                                 total_segs += msl->memseg_arr.len;
2113                                 cur_pagesz_mem = total_segs * hugepage_sz;
2114                                 type_msl_idx++;
2115                                 msl_idx++;
2116                         }
2117                         cur_socket_mem += cur_pagesz_mem;
2118                 }
2119                 if (cur_socket_mem == 0) {
2120                         RTE_LOG(ERR, EAL, "Cannot allocate VA space on socket %u\n",
2121                                 socket_id);
2122                         return -1;
2123                 }
2124         }
2125
2126         return 0;
2127 }
2128
2129 static int __rte_unused
2130 memseg_primary_init(void)
2131 {
2132         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
2133         int i, socket_id, hpi_idx, msl_idx = 0;
2134         struct rte_memseg_list *msl;
2135         uint64_t max_mem, total_mem;
2136
2137         /* no-huge does not need this at all */
2138         if (internal_config.no_hugetlbfs)
2139                 return 0;
2140
2141         max_mem = (uint64_t)RTE_MAX_MEM_MB << 20;
2142         total_mem = 0;
2143
2144         /* create memseg lists */
2145         for (hpi_idx = 0; hpi_idx < (int) internal_config.num_hugepage_sizes;
2146                         hpi_idx++) {
2147                 struct hugepage_info *hpi;
2148                 uint64_t hugepage_sz;
2149
2150                 hpi = &internal_config.hugepage_info[hpi_idx];
2151                 hugepage_sz = hpi->hugepage_sz;
2152
2153                 for (i = 0; i < (int) rte_socket_count(); i++) {
2154                         uint64_t max_type_mem, total_type_mem = 0;
2155                         int type_msl_idx, max_segs, total_segs = 0;
2156
2157                         socket_id = rte_socket_id_by_idx(i);
2158
2159 #ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES
2160                         if (socket_id > 0)
2161                                 break;
2162 #endif
2163
2164                         if (total_mem >= max_mem)
2165                                 break;
2166
2167                         max_type_mem = RTE_MIN(max_mem - total_mem,
2168                                 (uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20);
2169                         max_segs = RTE_MAX_MEMSEG_PER_TYPE;
2170
2171                         type_msl_idx = 0;
2172                         while (total_type_mem < max_type_mem &&
2173                                         total_segs < max_segs) {
2174                                 uint64_t cur_max_mem, cur_mem;
2175                                 unsigned int n_segs;
2176
2177                                 if (msl_idx >= RTE_MAX_MEMSEG_LISTS) {
2178                                         RTE_LOG(ERR, EAL,
2179                                                 "No more space in memseg lists, please increase %s\n",
2180                                                 RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));
2181                                         return -1;
2182                                 }
2183
2184                                 msl = &mcfg->memsegs[msl_idx++];
2185
2186                                 cur_max_mem = max_type_mem - total_type_mem;
2187
2188                                 cur_mem = get_mem_amount(hugepage_sz,
2189                                                 cur_max_mem);
2190                                 n_segs = cur_mem / hugepage_sz;
2191
2192                                 if (alloc_memseg_list(msl, hugepage_sz, n_segs,
2193                                                 socket_id, type_msl_idx))
2194                                         return -1;
2195
2196                                 total_segs += msl->memseg_arr.len;
2197                                 total_type_mem = total_segs * hugepage_sz;
2198                                 type_msl_idx++;
2199
2200                                 if (alloc_va_space(msl)) {
2201                                         RTE_LOG(ERR, EAL, "Cannot allocate VA space for memseg list\n");
2202                                         return -1;
2203                                 }
2204                         }
2205                         total_mem += total_type_mem;
2206                 }
2207         }
2208         return 0;
2209 }
2210
2211 static int
2212 memseg_secondary_init(void)
2213 {
2214         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
2215         int msl_idx = 0;
2216         struct rte_memseg_list *msl;
2217
2218         for (msl_idx = 0; msl_idx < RTE_MAX_MEMSEG_LISTS; msl_idx++) {
2219
2220                 msl = &mcfg->memsegs[msl_idx];
2221
2222                 /* skip empty memseg lists */
2223                 if (msl->memseg_arr.len == 0)
2224                         continue;
2225
2226                 if (rte_fbarray_attach(&msl->memseg_arr)) {
2227                         RTE_LOG(ERR, EAL, "Cannot attach to primary process memseg lists\n");
2228                         return -1;
2229                 }
2230
2231                 /* preallocate VA space */
2232                 if (alloc_va_space(msl)) {
2233                         RTE_LOG(ERR, EAL, "Cannot preallocate VA space for hugepage memory\n");
2234                         return -1;
2235                 }
2236         }
2237
2238         return 0;
2239 }
2240
2241 int
2242 rte_eal_memseg_init(void)
2243 {
2244         /* increase rlimit to maximum */
2245         struct rlimit lim;
2246
2247         if (getrlimit(RLIMIT_NOFILE, &lim) == 0) {
2248                 /* set limit to maximum */
2249                 lim.rlim_cur = lim.rlim_max;
2250
2251                 if (setrlimit(RLIMIT_NOFILE, &lim) < 0) {
2252                         RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",
2253                                         strerror(errno));
2254                 } else {
2255                         RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"
2256                                         PRIu64 "\n",
2257                                         (uint64_t)lim.rlim_cur);
2258                 }
2259         } else {
2260                 RTE_LOG(ERR, EAL, "Cannot get current resource limits\n");
2261         }
2262
2263         return rte_eal_process_type() == RTE_PROC_PRIMARY ?
2264 #ifndef RTE_ARCH_64
2265                         memseg_primary_init_32() :
2266 #else
2267                         memseg_primary_init() :
2268 #endif
2269                         memseg_secondary_init();
2270 }