lib/eal/linux/eal.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation.
   3  * Copyright(c) 2012-2014 6WIND S.A.
   4  */
   5
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <stdint.h>
   9 #include <string.h>
  10 #include <stdarg.h>
  11 #include <unistd.h>
  12 #include <pthread.h>
  13 #include <syslog.h>
  14 #include <getopt.h>
  15 #include <sys/file.h>
  16 #include <dirent.h>
  17 #include <fcntl.h>
  18 #include <fnmatch.h>
  19 #include <stddef.h>
  20 #include <errno.h>
  21 #include <limits.h>
  22 #include <sys/mman.h>
  23 #include <sys/queue.h>
  24 #include <sys/stat.h>
  25 #if defined(RTE_ARCH_X86)
  26 #include <sys/io.h>
  27 #endif
  28 #include <linux/version.h>
  29
  30 #include <rte_compat.h>
  31 #include <rte_common.h>
  32 #include <rte_debug.h>
  33 #include <rte_memory.h>
  34 #include <rte_launch.h>
  35 #include <rte_eal.h>
  36 #include <rte_errno.h>
  37 #include <rte_per_lcore.h>
  38 #include <rte_lcore.h>
  39 #include <rte_service_component.h>
  40 #include <rte_log.h>
  41 #include <rte_random.h>
  42 #include <rte_cycles.h>
  43 #include <rte_string_fns.h>
  44 #include <rte_cpuflags.h>
  45 #include <rte_interrupts.h>
  46 #include <rte_bus.h>
  47 #include <rte_dev.h>
  48 #include <rte_devargs.h>
  49 #include <rte_version.h>
  50 #include <malloc_heap.h>
  51 #include <rte_vfio.h>
  52
  53 #include <telemetry_internal.h>
  54 #include "eal_private.h"
  55 #include "eal_thread.h"
  56 #include "eal_internal_cfg.h"
  57 #include "eal_filesystem.h"
  58 #include "eal_hugepages.h"
  59 #include "eal_memcfg.h"
  60 #include "eal_trace.h"
  61 #include "eal_log.h"
  62 #include "eal_options.h"
  63 #include "eal_vfio.h"
  64 #include "hotplug_mp.h"
  65
  66 #define MEMSIZE_IF_NO_HUGE_PAGE (64ULL * 1024ULL * 1024ULL)
  67
  68 #define SOCKET_MEM_STRLEN (RTE_MAX_NUMA_NODES * 10)
  69
  70 #define KERNEL_IOMMU_GROUPS_PATH "/sys/kernel/iommu_groups"
  71
  72 /* define fd variable here, because file needs to be kept open for the
  73  * duration of the program, as we hold a write lock on it in the primary proc */
  74 static int mem_cfg_fd = -1;
  75
  76 static struct flock wr_lock = {
  77                 .l_type = F_WRLCK,
  78                 .l_whence = SEEK_SET,
  79                 .l_start = offsetof(struct rte_mem_config, memsegs),
  80                 .l_len = RTE_SIZEOF_FIELD(struct rte_mem_config, memsegs),
  81 };
  82
  83 /* internal configuration (per-core) */
  84 struct lcore_config lcore_config[RTE_MAX_LCORE];
  85
  86 /* used by rte_rdtsc() */
  87 int rte_cycles_vmware_tsc_map;
  88
  89 int
  90 eal_create_runtime_dir(void)
  91 {
  92         const char *directory;
  93         char run_dir[PATH_MAX];
  94         char tmp[PATH_MAX];
  95         int ret;
  96
  97         /* from RuntimeDirectory= see systemd.exec */
  98         directory = getenv("RUNTIME_DIRECTORY");
  99         if (directory == NULL) {
 100                 /*
 101                  * Used standard convention defined in
 102                  * XDG Base Directory Specification and
 103                  * Filesystem Hierarchy Standard.
 104                  */
 105                 if (getuid() == 0)
 106                         directory = "/var/run";
 107                 else
 108                         directory = getenv("XDG_RUNTIME_DIR") ? : "/tmp";
 109         }
 110
 111         /* create DPDK subdirectory under runtime dir */
 112         ret = snprintf(tmp, sizeof(tmp), "%s/dpdk", directory);
 113         if (ret < 0 || ret == sizeof(tmp)) {
 114                 RTE_LOG(ERR, EAL, "Error creating DPDK runtime path name\n");
 115                 return -1;
 116         }
 117
 118         /* create prefix-specific subdirectory under DPDK runtime dir */
 119         ret = snprintf(run_dir, sizeof(run_dir), "%s/%s",
 120                         tmp, eal_get_hugefile_prefix());
 121         if (ret < 0 || ret == sizeof(run_dir)) {
 122                 RTE_LOG(ERR, EAL, "Error creating prefix-specific runtime path name\n");
 123                 return -1;
 124         }
 125
 126         /* create the path if it doesn't exist. no "mkdir -p" here, so do it
 127          * step by step.
 128          */
 129         ret = mkdir(tmp, 0700);
 130         if (ret < 0 && errno != EEXIST) {
 131                 RTE_LOG(ERR, EAL, "Error creating '%s': %s\n",
 132                         tmp, strerror(errno));
 133                 return -1;
 134         }
 135
 136         ret = mkdir(run_dir, 0700);
 137         if (ret < 0 && errno != EEXIST) {
 138                 RTE_LOG(ERR, EAL, "Error creating '%s': %s\n",
 139                         run_dir, strerror(errno));
 140                 return -1;
 141         }
 142
 143         if (eal_set_runtime_dir(run_dir))
 144                 return -1;
 145
 146         return 0;
 147 }
 148
 149 int
 150 eal_clean_runtime_dir(void)
 151 {
 152         const char *runtime_dir = rte_eal_get_runtime_dir();
 153         DIR *dir;
 154         struct dirent *dirent;
 155         int dir_fd, fd, lck_result;
 156         static const char * const filters[] = {
 157                 "fbarray_*",
 158                 "mp_socket_*"
 159         };
 160
 161         /* open directory */
 162         dir = opendir(runtime_dir);
 163         if (!dir) {
 164                 RTE_LOG(ERR, EAL, "Unable to open runtime directory %s\n",
 165                                 runtime_dir);
 166                 goto error;
 167         }
 168         dir_fd = dirfd(dir);
 169
 170         /* lock the directory before doing anything, to avoid races */
 171         if (flock(dir_fd, LOCK_EX) < 0) {
 172                 RTE_LOG(ERR, EAL, "Unable to lock runtime directory %s\n",
 173                         runtime_dir);
 174                 goto error;
 175         }
 176
 177         dirent = readdir(dir);
 178         if (!dirent) {
 179                 RTE_LOG(ERR, EAL, "Unable to read runtime directory %s\n",
 180                                 runtime_dir);
 181                 goto error;
 182         }
 183
 184         while (dirent != NULL) {
 185                 unsigned int f_idx;
 186                 bool skip = true;
 187
 188                 /* skip files that don't match the patterns */
 189                 for (f_idx = 0; f_idx < RTE_DIM(filters); f_idx++) {
 190                         const char *filter = filters[f_idx];
 191
 192                         if (fnmatch(filter, dirent->d_name, 0) == 0) {
 193                                 skip = false;
 194                                 break;
 195                         }
 196                 }
 197                 if (skip) {
 198                         dirent = readdir(dir);
 199                         continue;
 200                 }
 201
 202                 /* try and lock the file */
 203                 fd = openat(dir_fd, dirent->d_name, O_RDONLY);
 204
 205                 /* skip to next file */
 206                 if (fd == -1) {
 207                         dirent = readdir(dir);
 208                         continue;
 209                 }
 210
 211                 /* non-blocking lock */
 212                 lck_result = flock(fd, LOCK_EX | LOCK_NB);
 213
 214                 /* if lock succeeds, remove the file */
 215                 if (lck_result != -1)
 216                         unlinkat(dir_fd, dirent->d_name, 0);
 217                 close(fd);
 218                 dirent = readdir(dir);
 219         }
 220
 221         /* closedir closes dir_fd and drops the lock */
 222         closedir(dir);
 223         return 0;
 224
 225 error:
 226         if (dir)
 227                 closedir(dir);
 228
 229         RTE_LOG(ERR, EAL, "Error while clearing runtime dir: %s\n",
 230                 strerror(errno));
 231
 232         return -1;
 233 }
 234
 235 /* parse a sysfs (or other) file containing one integer value */
 236 int
 237 eal_parse_sysfs_value(const char *filename, unsigned long *val)
 238 {
 239         FILE *f;
 240         char buf[BUFSIZ];
 241         char *end = NULL;
 242
 243         if ((f = fopen(filename, "r")) == NULL) {
 244                 RTE_LOG(ERR, EAL, "%s(): cannot open sysfs value %s\n",
 245                         __func__, filename);
 246                 return -1;
 247         }
 248
 249         if (fgets(buf, sizeof(buf), f) == NULL) {
 250                 RTE_LOG(ERR, EAL, "%s(): cannot read sysfs value %s\n",
 251                         __func__, filename);
 252                 fclose(f);
 253                 return -1;
 254         }
 255         *val = strtoul(buf, &end, 0);
 256         if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) {
 257                 RTE_LOG(ERR, EAL, "%s(): cannot parse sysfs value %s\n",
 258                                 __func__, filename);
 259                 fclose(f);
 260                 return -1;
 261         }
 262         fclose(f);
 263         return 0;
 264 }
 265
 266
 267 /* create memory configuration in shared/mmap memory. Take out
 268  * a write lock on the memsegs, so we can auto-detect primary/secondary.
 269  * This means we never close the file while running (auto-close on exit).
 270  * We also don't lock the whole file, so that in future we can use read-locks
 271  * on other parts, e.g. memzones, to detect if there are running secondary
 272  * processes. */
 273 static int
 274 rte_eal_config_create(void)
 275 {
 276         struct rte_config *config = rte_eal_get_configuration();
 277         size_t page_sz = sysconf(_SC_PAGE_SIZE);
 278         size_t cfg_len = sizeof(*config->mem_config);
 279         size_t cfg_len_aligned = RTE_ALIGN(cfg_len, page_sz);
 280         void *rte_mem_cfg_addr, *mapped_mem_cfg_addr;
 281         int retval;
 282         const struct internal_config *internal_conf =
 283                 eal_get_internal_configuration();
 284
 285         const char *pathname = eal_runtime_config_path();
 286
 287         if (internal_conf->no_shconf)
 288                 return 0;
 289
 290         /* map the config before hugepage address so that we don't waste a page */
 291         if (internal_conf->base_virtaddr != 0)
 292                 rte_mem_cfg_addr = (void *)
 293                         RTE_ALIGN_FLOOR(internal_conf->base_virtaddr -
 294                         sizeof(struct rte_mem_config), page_sz);
 295         else
 296                 rte_mem_cfg_addr = NULL;
 297
 298         if (mem_cfg_fd < 0){
 299                 mem_cfg_fd = open(pathname, O_RDWR | O_CREAT, 0600);
 300                 if (mem_cfg_fd < 0) {
 301                         RTE_LOG(ERR, EAL, "Cannot open '%s' for rte_mem_config\n",
 302                                 pathname);
 303                         return -1;
 304                 }
 305         }
 306
 307         retval = ftruncate(mem_cfg_fd, cfg_len);
 308         if (retval < 0){
 309                 close(mem_cfg_fd);
 310                 mem_cfg_fd = -1;
 311                 RTE_LOG(ERR, EAL, "Cannot resize '%s' for rte_mem_config\n",
 312                         pathname);
 313                 return -1;
 314         }
 315
 316         retval = fcntl(mem_cfg_fd, F_SETLK, &wr_lock);
 317         if (retval < 0){
 318                 close(mem_cfg_fd);
 319                 mem_cfg_fd = -1;
 320                 RTE_LOG(ERR, EAL, "Cannot create lock on '%s'. Is another primary "
 321                         "process running?\n", pathname);
 322                 return -1;
 323         }
 324
 325         /* reserve space for config */
 326         rte_mem_cfg_addr = eal_get_virtual_area(rte_mem_cfg_addr,
 327                         &cfg_len_aligned, page_sz, 0, 0);
 328         if (rte_mem_cfg_addr == NULL) {
 329                 RTE_LOG(ERR, EAL, "Cannot mmap memory for rte_config\n");
 330                 close(mem_cfg_fd);
 331                 mem_cfg_fd = -1;
 332                 return -1;
 333         }
 334
 335         /* remap the actual file into the space we've just reserved */
 336         mapped_mem_cfg_addr = mmap(rte_mem_cfg_addr,
 337                         cfg_len_aligned, PROT_READ | PROT_WRITE,
 338                         MAP_SHARED | MAP_FIXED, mem_cfg_fd, 0);
 339         if (mapped_mem_cfg_addr == MAP_FAILED) {
 340                 munmap(rte_mem_cfg_addr, cfg_len);
 341                 close(mem_cfg_fd);
 342                 mem_cfg_fd = -1;
 343                 RTE_LOG(ERR, EAL, "Cannot remap memory for rte_config\n");
 344                 return -1;
 345         }
 346
 347         memcpy(rte_mem_cfg_addr, config->mem_config, sizeof(struct rte_mem_config));
 348         config->mem_config = rte_mem_cfg_addr;
 349
 350         /* store address of the config in the config itself so that secondary
 351          * processes could later map the config into this exact location
 352          */
 353         config->mem_config->mem_cfg_addr = (uintptr_t) rte_mem_cfg_addr;
 354         config->mem_config->dma_maskbits = 0;
 355
 356         return 0;
 357 }
 358
 359 /* attach to an existing shared memory config */
 360 static int
 361 rte_eal_config_attach(void)
 362 {
 363         struct rte_config *config = rte_eal_get_configuration();
 364         struct rte_mem_config *mem_config;
 365         const struct internal_config *internal_conf =
 366                 eal_get_internal_configuration();
 367
 368         const char *pathname = eal_runtime_config_path();
 369
 370         if (internal_conf->no_shconf)
 371                 return 0;
 372
 373         if (mem_cfg_fd < 0){
 374                 mem_cfg_fd = open(pathname, O_RDWR);
 375                 if (mem_cfg_fd < 0) {
 376                         RTE_LOG(ERR, EAL, "Cannot open '%s' for rte_mem_config\n",
 377                                 pathname);
 378                         return -1;
 379                 }
 380         }
 381
 382         /* map it as read-only first */
 383         mem_config = (struct rte_mem_config *) mmap(NULL, sizeof(*mem_config),
 384                         PROT_READ, MAP_SHARED, mem_cfg_fd, 0);
 385         if (mem_config == MAP_FAILED) {
 386                 close(mem_cfg_fd);
 387                 mem_cfg_fd = -1;
 388                 RTE_LOG(ERR, EAL, "Cannot mmap memory for rte_config! error %i (%s)\n",
 389                         errno, strerror(errno));
 390                 return -1;
 391         }
 392
 393         config->mem_config = mem_config;
 394
 395         return 0;
 396 }
 397
 398 /* reattach the shared config at exact memory location primary process has it */
 399 static int
 400 rte_eal_config_reattach(void)
 401 {
 402         struct rte_config *config = rte_eal_get_configuration();
 403         struct rte_mem_config *mem_config;
 404         void *rte_mem_cfg_addr;
 405         const struct internal_config *internal_conf =
 406                 eal_get_internal_configuration();
 407
 408         if (internal_conf->no_shconf)
 409                 return 0;
 410
 411         /* save the address primary process has mapped shared config to */
 412         rte_mem_cfg_addr =
 413                 (void *) (uintptr_t) config->mem_config->mem_cfg_addr;
 414
 415         /* unmap original config */
 416         munmap(config->mem_config, sizeof(struct rte_mem_config));
 417
 418         /* remap the config at proper address */
 419         mem_config = (struct rte_mem_config *) mmap(rte_mem_cfg_addr,
 420                         sizeof(*mem_config), PROT_READ | PROT_WRITE, MAP_SHARED,
 421                         mem_cfg_fd, 0);
 422
 423         close(mem_cfg_fd);
 424         mem_cfg_fd = -1;
 425
 426         if (mem_config == MAP_FAILED || mem_config != rte_mem_cfg_addr) {
 427                 if (mem_config != MAP_FAILED) {
 428                         /* errno is stale, don't use */
 429                         RTE_LOG(ERR, EAL, "Cannot mmap memory for rte_config at [%p], got [%p]"
 430                                 " - please use '--" OPT_BASE_VIRTADDR
 431                                 "' option\n", rte_mem_cfg_addr, mem_config);
 432                         munmap(mem_config, sizeof(struct rte_mem_config));
 433                         return -1;
 434                 }
 435                 RTE_LOG(ERR, EAL, "Cannot mmap memory for rte_config! error %i (%s)\n",
 436                         errno, strerror(errno));
 437                 return -1;
 438         }
 439
 440         config->mem_config = mem_config;
 441
 442         return 0;
 443 }
 444
 445 /* Detect if we are a primary or a secondary process */
 446 enum rte_proc_type_t
 447 eal_proc_type_detect(void)
 448 {
 449         enum rte_proc_type_t ptype = RTE_PROC_PRIMARY;
 450         const char *pathname = eal_runtime_config_path();
 451         const struct internal_config *internal_conf =
 452                 eal_get_internal_configuration();
 453
 454         /* if there no shared config, there can be no secondary processes */
 455         if (!internal_conf->no_shconf) {
 456                 /* if we can open the file but not get a write-lock we are a
 457                  * secondary process. NOTE: if we get a file handle back, we
 458                  * keep that open and don't close it to prevent a race condition
 459                  * between multiple opens.
 460                  */
 461                 if (((mem_cfg_fd = open(pathname, O_RDWR)) >= 0) &&
 462                                 (fcntl(mem_cfg_fd, F_SETLK, &wr_lock) < 0))
 463                         ptype = RTE_PROC_SECONDARY;
 464         }
 465
 466         RTE_LOG(INFO, EAL, "Auto-detected process type: %s\n",
 467                         ptype == RTE_PROC_PRIMARY ? "PRIMARY" : "SECONDARY");
 468
 469         return ptype;
 470 }
 471
 472 /* Sets up rte_config structure with the pointer to shared memory config.*/
 473 static int
 474 rte_config_init(void)
 475 {
 476         struct rte_config *config = rte_eal_get_configuration();
 477         const struct internal_config *internal_conf =
 478                 eal_get_internal_configuration();
 479
 480         config->process_type = internal_conf->process_type;
 481
 482         switch (config->process_type) {
 483         case RTE_PROC_PRIMARY:
 484                 if (rte_eal_config_create() < 0)
 485                         return -1;
 486                 eal_mcfg_update_from_internal();
 487                 break;
 488         case RTE_PROC_SECONDARY:
 489                 if (rte_eal_config_attach() < 0)
 490                         return -1;
 491                 eal_mcfg_wait_complete();
 492                 if (eal_mcfg_check_version() < 0) {
 493                         RTE_LOG(ERR, EAL, "Primary and secondary process DPDK version mismatch\n");
 494                         return -1;
 495                 }
 496                 if (rte_eal_config_reattach() < 0)
 497                         return -1;
 498                 if (!__rte_mp_enable()) {
 499                         RTE_LOG(ERR, EAL, "Primary process refused secondary attachment\n");
 500                         return -1;
 501                 }
 502                 eal_mcfg_update_internal();
 503                 break;
 504         case RTE_PROC_AUTO:
 505         case RTE_PROC_INVALID:
 506                 RTE_LOG(ERR, EAL, "Invalid process type %d\n",
 507                         config->process_type);
 508                 return -1;
 509         }
 510
 511         return 0;
 512 }
 513
 514 /* Unlocks hugepage directories that were locked by eal_hugepage_info_init */
 515 static void
 516 eal_hugedirs_unlock(void)
 517 {
 518         int i;
 519         struct internal_config *internal_conf =
 520                 eal_get_internal_configuration();
 521
 522         for (i = 0; i < MAX_HUGEPAGE_SIZES; i++)
 523         {
 524                 /* skip uninitialized */
 525                 if (internal_conf->hugepage_info[i].lock_descriptor < 0)
 526                         continue;
 527                 /* unlock hugepage file */
 528                 flock(internal_conf->hugepage_info[i].lock_descriptor, LOCK_UN);
 529                 close(internal_conf->hugepage_info[i].lock_descriptor);
 530                 /* reset the field */
 531                 internal_conf->hugepage_info[i].lock_descriptor = -1;
 532         }
 533 }
 534
 535 /* display usage */
 536 static void
 537 eal_usage(const char *prgname)
 538 {
 539         rte_usage_hook_t hook = eal_get_application_usage_hook();
 540
 541         printf("\nUsage: %s ", prgname);
 542         eal_common_usage();
 543         printf("EAL Linux options:\n"
 544                "  --"OPT_SOCKET_MEM"        Memory to allocate on sockets (comma separated values)\n"
 545                "  --"OPT_SOCKET_LIMIT"      Limit memory allocation on sockets (comma separated values)\n"
 546                "  --"OPT_HUGE_DIR"          Directory where hugetlbfs is mounted\n"
 547                "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 548                "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 549                "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
 550                "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 551                "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 552                "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 553                "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
 554                "\n");
 555         /* Allow the application to print its usage message too if hook is set */
 556         if (hook) {
 557                 printf("===== Application Usage =====\n\n");
 558                 (hook)(prgname);
 559         }
 560 }
 561
 562 static int
 563 eal_parse_socket_arg(char *strval, volatile uint64_t *socket_arg)
 564 {
 565         char * arg[RTE_MAX_NUMA_NODES];
 566         char *end;
 567         int arg_num, i, len;
 568
 569         len = strnlen(strval, SOCKET_MEM_STRLEN);
 570         if (len == SOCKET_MEM_STRLEN) {
 571                 RTE_LOG(ERR, EAL, "--socket-mem is too long\n");
 572                 return -1;
 573         }
 574
 575         /* all other error cases will be caught later */
 576         if (!isdigit(strval[len-1]))
 577                 return -1;
 578
 579         /* split the optarg into separate socket values */
 580         arg_num = rte_strsplit(strval, len,
 581                         arg, RTE_MAX_NUMA_NODES, ',');
 582
 583         /* if split failed, or 0 arguments */
 584         if (arg_num <= 0)
 585                 return -1;
 586
 587         /* parse each defined socket option */
 588         errno = 0;
 589         for (i = 0; i < arg_num; i++) {
 590                 uint64_t val;
 591                 end = NULL;
 592                 val = strtoull(arg[i], &end, 10);
 593
 594                 /* check for invalid input */
 595                 if ((errno != 0)  ||
 596                                 (arg[i][0] == '\0') || (end == NULL) || (*end != '\0'))
 597                         return -1;
 598                 val <<= 20;
 599                 socket_arg[i] = val;
 600         }
 601
 602         return 0;
 603 }
 604
 605 static int
 606 eal_parse_vfio_intr(const char *mode)
 607 {
 608         struct internal_config *internal_conf =
 609                 eal_get_internal_configuration();
 610         unsigned i;
 611         static struct {
 612                 const char *name;
 613                 enum rte_intr_mode value;
 614         } map[] = {
 615                 { "legacy", RTE_INTR_MODE_LEGACY },
 616                 { "msi", RTE_INTR_MODE_MSI },
 617                 { "msix", RTE_INTR_MODE_MSIX },
 618         };
 619
 620         for (i = 0; i < RTE_DIM(map); i++) {
 621                 if (!strcmp(mode, map[i].name)) {
 622                         internal_conf->vfio_intr_mode = map[i].value;
 623                         return 0;
 624                 }
 625         }
 626         return -1;
 627 }
 628
 629 static int
 630 eal_parse_vfio_vf_token(const char *vf_token)
 631 {
 632         struct internal_config *cfg = eal_get_internal_configuration();
 633         rte_uuid_t uuid;
 634
 635         if (!rte_uuid_parse(vf_token, uuid)) {
 636                 rte_uuid_copy(cfg->vfio_vf_token, uuid);
 637                 return 0;
 638         }
 639
 640         return -1;
 641 }
 642
 643 /* Parse the arguments for --log-level only */
 644 static void
 645 eal_log_level_parse(int argc, char **argv)
 646 {
 647         int opt;
 648         char **argvopt;
 649         int option_index;
 650         const int old_optind = optind;
 651         const int old_optopt = optopt;
 652         char * const old_optarg = optarg;
 653         struct internal_config *internal_conf =
 654                 eal_get_internal_configuration();
 655
 656         argvopt = argv;
 657         optind = 1;
 658
 659         while ((opt = getopt_long(argc, argvopt, eal_short_options,
 660                                   eal_long_options, &option_index)) != EOF) {
 661
 662                 int ret;
 663
 664                 /* getopt is not happy, stop right now */
 665                 if (opt == '?')
 666                         break;
 667
 668                 ret = (opt == OPT_LOG_LEVEL_NUM) ?
 669                         eal_parse_common_option(opt, optarg, internal_conf) : 0;
 670
 671                 /* common parser is not happy */
 672                 if (ret < 0)
 673                         break;
 674         }
 675
 676         /* restore getopt lib */
 677         optind = old_optind;
 678         optopt = old_optopt;
 679         optarg = old_optarg;
 680 }
 681
 682 /* Parse the argument given in the command line of the application */
 683 static int
 684 eal_parse_args(int argc, char **argv)
 685 {
 686         int opt, ret;
 687         char **argvopt;
 688         int option_index;
 689         char *prgname = argv[0];
 690         const int old_optind = optind;
 691         const int old_optopt = optopt;
 692         char * const old_optarg = optarg;
 693         struct internal_config *internal_conf =
 694                 eal_get_internal_configuration();
 695
 696         argvopt = argv;
 697         optind = 1;
 698
 699         while ((opt = getopt_long(argc, argvopt, eal_short_options,
 700                                   eal_long_options, &option_index)) != EOF) {
 701
 702                 /* getopt didn't recognise the option */
 703                 if (opt == '?') {
 704                         eal_usage(prgname);
 705                         ret = -1;
 706                         goto out;
 707                 }
 708
 709                 /* eal_log_level_parse() already handled this option */
 710                 if (opt == OPT_LOG_LEVEL_NUM)
 711                         continue;
 712
 713                 ret = eal_parse_common_option(opt, optarg, internal_conf);
 714                 /* common parser is not happy */
 715                 if (ret < 0) {
 716                         eal_usage(prgname);
 717                         ret = -1;
 718                         goto out;
 719                 }
 720                 /* common parser handled this option */
 721                 if (ret == 0)
 722                         continue;
 723
 724                 switch (opt) {
 725                 case 'h':
 726                         eal_usage(prgname);
 727                         exit(EXIT_SUCCESS);
 728
 729                 case OPT_HUGE_DIR_NUM:
 730                 {
 731                         char *hdir = strdup(optarg);
 732                         if (hdir == NULL)
 733                                 RTE_LOG(ERR, EAL, "Could not store hugepage directory\n");
 734                         else {
 735                                 /* free old hugepage dir */
 736                                 if (internal_conf->hugepage_dir != NULL)
 737                                         free(internal_conf->hugepage_dir);
 738                                 internal_conf->hugepage_dir = hdir;
 739                         }
 740                         break;
 741                 }
 742                 case OPT_FILE_PREFIX_NUM:
 743                 {
 744                         char *prefix = strdup(optarg);
 745                         if (prefix == NULL)
 746                                 RTE_LOG(ERR, EAL, "Could not store file prefix\n");
 747                         else {
 748                                 /* free old prefix */
 749                                 if (internal_conf->hugefile_prefix != NULL)
 750                                         free(internal_conf->hugefile_prefix);
 751                                 internal_conf->hugefile_prefix = prefix;
 752                         }
 753                         break;
 754                 }
 755                 case OPT_SOCKET_MEM_NUM:
 756                         if (eal_parse_socket_arg(optarg,
 757                                         internal_conf->socket_mem) < 0) {
 758                                 RTE_LOG(ERR, EAL, "invalid parameters for --"
 759                                                 OPT_SOCKET_MEM "\n");
 760                                 eal_usage(prgname);
 761                                 ret = -1;
 762                                 goto out;
 763                         }
 764                         internal_conf->force_sockets = 1;
 765                         break;
 766
 767                 case OPT_SOCKET_LIMIT_NUM:
 768                         if (eal_parse_socket_arg(optarg,
 769                                         internal_conf->socket_limit) < 0) {
 770                                 RTE_LOG(ERR, EAL, "invalid parameters for --"
 771                                                 OPT_SOCKET_LIMIT "\n");
 772                                 eal_usage(prgname);
 773                                 ret = -1;
 774                                 goto out;
 775                         }
 776                         internal_conf->force_socket_limits = 1;
 777                         break;
 778
 779                 case OPT_VFIO_INTR_NUM:
 780                         if (eal_parse_vfio_intr(optarg) < 0) {
 781                                 RTE_LOG(ERR, EAL, "invalid parameters for --"
 782                                                 OPT_VFIO_INTR "\n");
 783                                 eal_usage(prgname);
 784                                 ret = -1;
 785                                 goto out;
 786                         }
 787                         break;
 788
 789                 case OPT_VFIO_VF_TOKEN_NUM:
 790                         if (eal_parse_vfio_vf_token(optarg) < 0) {
 791                                 RTE_LOG(ERR, EAL, "invalid parameters for --"
 792                                                 OPT_VFIO_VF_TOKEN "\n");
 793                                 eal_usage(prgname);
 794                                 ret = -1;
 795                                 goto out;
 796                         }
 797                         break;
 798
 799                 case OPT_CREATE_UIO_DEV_NUM:
 800                         internal_conf->create_uio_dev = 1;
 801                         break;
 802
 803                 case OPT_MBUF_POOL_OPS_NAME_NUM:
 804                 {
 805                         char *ops_name = strdup(optarg);
 806                         if (ops_name == NULL)
 807                                 RTE_LOG(ERR, EAL, "Could not store mbuf pool ops name\n");
 808                         else {
 809                                 /* free old ops name */
 810                                 if (internal_conf->user_mbuf_pool_ops_name !=
 811                                                 NULL)
 812                                         free(internal_conf->user_mbuf_pool_ops_name);
 813
 814                                 internal_conf->user_mbuf_pool_ops_name =
 815                                                 ops_name;
 816                         }
 817                         break;
 818                 }
 819                 case OPT_MATCH_ALLOCATIONS_NUM:
 820                         internal_conf->match_allocations = 1;
 821                         break;
 822
 823                 default:
 824                         if (opt < OPT_LONG_MIN_NUM && isprint(opt)) {
 825                                 RTE_LOG(ERR, EAL, "Option %c is not supported "
 826                                         "on Linux\n", opt);
 827                         } else if (opt >= OPT_LONG_MIN_NUM &&
 828                                    opt < OPT_LONG_MAX_NUM) {
 829                                 RTE_LOG(ERR, EAL, "Option %s is not supported "
 830                                         "on Linux\n",
 831                                         eal_long_options[option_index].name);
 832                         } else {
 833                                 RTE_LOG(ERR, EAL, "Option %d is not supported "
 834                                         "on Linux\n", opt);
 835                         }
 836                         eal_usage(prgname);
 837                         ret = -1;
 838                         goto out;
 839                 }
 840         }
 841
 842         /* create runtime data directory. In no_shconf mode, skip any errors */
 843         if (eal_create_runtime_dir() < 0) {
 844                 if (internal_conf->no_shconf == 0) {
 845                         RTE_LOG(ERR, EAL, "Cannot create runtime directory\n");
 846                         ret = -1;
 847                         goto out;
 848                 } else
 849                         RTE_LOG(WARNING, EAL, "No DPDK runtime directory created\n");
 850         }
 851
 852         if (eal_adjust_config(internal_conf) != 0) {
 853                 ret = -1;
 854                 goto out;
 855         }
 856
 857         /* sanity checks */
 858         if (eal_check_common_options(internal_conf) != 0) {
 859                 eal_usage(prgname);
 860                 ret = -1;
 861                 goto out;
 862         }
 863
 864         if (optind >= 0)
 865                 argv[optind-1] = prgname;
 866         ret = optind-1;
 867
 868 out:
 869         /* restore getopt lib */
 870         optind = old_optind;
 871         optopt = old_optopt;
 872         optarg = old_optarg;
 873
 874         return ret;
 875 }
 876
 877 static int
 878 check_socket(const struct rte_memseg_list *msl, void *arg)
 879 {
 880         int *socket_id = arg;
 881
 882         if (msl->external)
 883                 return 0;
 884
 885         return *socket_id == msl->socket_id;
 886 }
 887
 888 static void
 889 eal_check_mem_on_local_socket(void)
 890 {
 891         int socket_id;
 892         const struct rte_config *config = rte_eal_get_configuration();
 893
 894         socket_id = rte_lcore_to_socket_id(config->main_lcore);
 895
 896         if (rte_memseg_list_walk(check_socket, &socket_id) == 0)
 897                 RTE_LOG(WARNING, EAL, "WARNING: Main core has no memory on local socket!\n");
 898 }
 899
 900 static int
 901 sync_func(__rte_unused void *arg)
 902 {
 903         return 0;
 904 }
 905
 906 /*
 907  * Request iopl privilege for all RPL, returns 0 on success
 908  * iopl() call is mostly for the i386 architecture. For other architectures,
 909  * return -1 to indicate IO privilege can't be changed in this way.
 910  */
 911 int
 912 rte_eal_iopl_init(void)
 913 {
 914 #if defined(RTE_ARCH_X86)
 915         if (iopl(3) != 0)
 916                 return -1;
 917 #endif
 918         return 0;
 919 }
 920
 921 #ifdef VFIO_PRESENT
 922 static int rte_eal_vfio_setup(void)
 923 {
 924         if (rte_vfio_enable("vfio"))
 925                 return -1;
 926
 927         return 0;
 928 }
 929 #endif
 930
 931 static void rte_eal_init_alert(const char *msg)
 932 {
 933         fprintf(stderr, "EAL: FATAL: %s\n", msg);
 934         RTE_LOG(ERR, EAL, "%s\n", msg);
 935 }
 936
 937 /*
 938  * On Linux 3.6+, even if VFIO is not loaded, whenever IOMMU is enabled in the
 939  * BIOS and in the kernel, /sys/kernel/iommu_groups path will contain kernel
 940  * IOMMU groups. If IOMMU is not enabled, that path would be empty.
 941  * Therefore, checking if the path is empty will tell us if IOMMU is enabled.
 942  */
 943 static bool
 944 is_iommu_enabled(void)
 945 {
 946         DIR *dir = opendir(KERNEL_IOMMU_GROUPS_PATH);
 947         struct dirent *d;
 948         int n = 0;
 949
 950         /* if directory doesn't exist, assume IOMMU is not enabled */
 951         if (dir == NULL)
 952                 return false;
 953
 954         while ((d = readdir(dir)) != NULL) {
 955                 /* skip dot and dot-dot */
 956                 if (++n > 2)
 957                         break;
 958         }
 959         closedir(dir);
 960
 961         return n > 2;
 962 }
 963
 964 /* Launch threads, called at application init(). */
 965 int
 966 rte_eal_init(int argc, char **argv)
 967 {
 968         int i, fctret, ret;
 969         pthread_t thread_id;
 970         static uint32_t run_once;
 971         uint32_t has_run = 0;
 972         const char *p;
 973         static char logid[PATH_MAX];
 974         char cpuset[RTE_CPU_AFFINITY_STR_LEN];
 975         char thread_name[RTE_MAX_THREAD_NAME_LEN];
 976         bool phys_addrs;
 977         const struct rte_config *config = rte_eal_get_configuration();
 978         struct internal_config *internal_conf =
 979                 eal_get_internal_configuration();
 980
 981         /* checks if the machine is adequate */
 982         if (!rte_cpu_is_supported()) {
 983                 rte_eal_init_alert("unsupported cpu type.");
 984                 rte_errno = ENOTSUP;
 985                 return -1;
 986         }
 987
 988         if (!__atomic_compare_exchange_n(&run_once, &has_run, 1, 0,
 989                                         __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
 990                 rte_eal_init_alert("already called initialization.");
 991                 rte_errno = EALREADY;
 992                 return -1;
 993         }
 994
 995         p = strrchr(argv[0], '/');
 996         strlcpy(logid, p ? p + 1 : argv[0], sizeof(logid));
 997         thread_id = pthread_self();
 998
 999         eal_reset_internal_config(internal_conf);
1000
1001         /* set log level as early as possible */
1002         eal_log_level_parse(argc, argv);
1003
1004         /* clone argv to report out later in telemetry */
1005         eal_save_args(argc, argv);
1006
1007         if (rte_eal_cpu_init() < 0) {
1008                 rte_eal_init_alert("Cannot detect lcores.");
1009                 rte_errno = ENOTSUP;
1010                 return -1;
1011         }
1012
1013         fctret = eal_parse_args(argc, argv);
1014         if (fctret < 0) {
1015                 rte_eal_init_alert("Invalid 'command line' arguments.");
1016                 rte_errno = EINVAL;
1017                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1018                 return -1;
1019         }
1020
1021         if (eal_plugins_init() < 0) {
1022                 rte_eal_init_alert("Cannot init plugins");
1023                 rte_errno = EINVAL;
1024                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1025                 return -1;
1026         }
1027
1028         if (eal_trace_init() < 0) {
1029                 rte_eal_init_alert("Cannot init trace");
1030                 rte_errno = EFAULT;
1031                 return -1;
1032         }
1033
1034         if (eal_option_device_parse()) {
1035                 rte_errno = ENODEV;
1036                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1037                 return -1;
1038         }
1039
1040         if (rte_config_init() < 0) {
1041                 rte_eal_init_alert("Cannot init config");
1042                 return -1;
1043         }
1044
1045         if (rte_eal_intr_init() < 0) {
1046                 rte_eal_init_alert("Cannot init interrupt-handling thread");
1047                 return -1;
1048         }
1049
1050         if (rte_eal_alarm_init() < 0) {
1051                 rte_eal_init_alert("Cannot init alarm");
1052                 /* rte_eal_alarm_init sets rte_errno on failure. */
1053                 return -1;
1054         }
1055
1056         /* Put mp channel init before bus scan so that we can init the vdev
1057          * bus through mp channel in the secondary process before the bus scan.
1058          */
1059         if (rte_mp_channel_init() < 0 && rte_errno != ENOTSUP) {
1060                 rte_eal_init_alert("failed to init mp channel");
1061                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
1062                         rte_errno = EFAULT;
1063                         return -1;
1064                 }
1065         }
1066
1067         /* register multi-process action callbacks for hotplug */
1068         if (eal_mp_dev_hotplug_init() < 0) {
1069                 rte_eal_init_alert("failed to register mp callback for hotplug");
1070                 return -1;
1071         }
1072
1073         if (rte_bus_scan()) {
1074                 rte_eal_init_alert("Cannot scan the buses for devices");
1075                 rte_errno = ENODEV;
1076                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1077                 return -1;
1078         }
1079
1080         phys_addrs = rte_eal_using_phys_addrs() != 0;
1081
1082         /* if no EAL option "--iova-mode=<pa|va>", use bus IOVA scheme */
1083         if (internal_conf->iova_mode == RTE_IOVA_DC) {
1084                 /* autodetect the IOVA mapping mode */
1085                 enum rte_iova_mode iova_mode = rte_bus_get_iommu_class();
1086
1087                 if (iova_mode == RTE_IOVA_DC) {
1088                         RTE_LOG(DEBUG, EAL, "Buses did not request a specific IOVA mode.\n");
1089
1090                         if (!phys_addrs) {
1091                                 /* if we have no access to physical addresses,
1092                                  * pick IOVA as VA mode.
1093                                  */
1094                                 iova_mode = RTE_IOVA_VA;
1095                                 RTE_LOG(DEBUG, EAL, "Physical addresses are unavailable, selecting IOVA as VA mode.\n");
1096 #if defined(RTE_LIB_KNI) && LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
1097                         } else if (rte_eal_check_module("rte_kni") == 1) {
1098                                 iova_mode = RTE_IOVA_PA;
1099                                 RTE_LOG(DEBUG, EAL, "KNI is loaded, selecting IOVA as PA mode for better KNI performance.\n");
1100 #endif
1101                         } else if (is_iommu_enabled()) {
1102                                 /* we have an IOMMU, pick IOVA as VA mode */
1103                                 iova_mode = RTE_IOVA_VA;
1104                                 RTE_LOG(DEBUG, EAL, "IOMMU is available, selecting IOVA as VA mode.\n");
1105                         } else {
1106                                 /* physical addresses available, and no IOMMU
1107                                  * found, so pick IOVA as PA.
1108                                  */
1109                                 iova_mode = RTE_IOVA_PA;
1110                                 RTE_LOG(DEBUG, EAL, "IOMMU is not available, selecting IOVA as PA mode.\n");
1111                         }
1112                 }
1113 #if defined(RTE_LIB_KNI) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)
1114                 /* Workaround for KNI which requires physical address to work
1115                  * in kernels < 4.10
1116                  */
1117                 if (iova_mode == RTE_IOVA_VA &&
1118                                 rte_eal_check_module("rte_kni") == 1) {
1119                         if (phys_addrs) {
1120                                 iova_mode = RTE_IOVA_PA;
1121                                 RTE_LOG(WARNING, EAL, "Forcing IOVA as 'PA' because KNI module is loaded\n");
1122                         } else {
1123                                 RTE_LOG(DEBUG, EAL, "KNI can not work since physical addresses are unavailable\n");
1124                         }
1125                 }
1126 #endif
1127                 rte_eal_get_configuration()->iova_mode = iova_mode;
1128         } else {
1129                 rte_eal_get_configuration()->iova_mode =
1130                         internal_conf->iova_mode;
1131         }
1132
1133         if (rte_eal_iova_mode() == RTE_IOVA_PA && !phys_addrs) {
1134                 rte_eal_init_alert("Cannot use IOVA as 'PA' since physical addresses are not available");
1135                 rte_errno = EINVAL;
1136                 return -1;
1137         }
1138
1139         RTE_LOG(INFO, EAL, "Selected IOVA mode '%s'\n",
1140                 rte_eal_iova_mode() == RTE_IOVA_PA ? "PA" : "VA");
1141
1142         if (internal_conf->no_hugetlbfs == 0) {
1143                 /* rte_config isn't initialized yet */
1144                 ret = internal_conf->process_type == RTE_PROC_PRIMARY ?
1145                                 eal_hugepage_info_init() :
1146                                 eal_hugepage_info_read();
1147                 if (ret < 0) {
1148                         rte_eal_init_alert("Cannot get hugepage information.");
1149                         rte_errno = EACCES;
1150                         __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1151                         return -1;
1152                 }
1153         }
1154
1155         if (internal_conf->memory == 0 && internal_conf->force_sockets == 0) {
1156                 if (internal_conf->no_hugetlbfs)
1157                         internal_conf->memory = MEMSIZE_IF_NO_HUGE_PAGE;
1158         }
1159
1160         if (internal_conf->vmware_tsc_map == 1) {
1161 #ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
1162                 rte_cycles_vmware_tsc_map = 1;
1163                 RTE_LOG (DEBUG, EAL, "Using VMWARE TSC MAP, "
1164                                 "you must have monitor_control.pseudo_perfctr = TRUE\n");
1165 #else
1166                 RTE_LOG (WARNING, EAL, "Ignoring --vmware-tsc-map because "
1167                                 "RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT is not set\n");
1168 #endif
1169         }
1170
1171         if (eal_log_init(logid, internal_conf->syslog_facility) < 0) {
1172                 rte_eal_init_alert("Cannot init logging.");
1173                 rte_errno = ENOMEM;
1174                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1175                 return -1;
1176         }
1177
1178 #ifdef VFIO_PRESENT
1179         if (rte_eal_vfio_setup() < 0) {
1180                 rte_eal_init_alert("Cannot init VFIO");
1181                 rte_errno = EAGAIN;
1182                 __atomic_store_n(&run_once, 0, __ATOMIC_RELAXED);
1183                 return -1;
1184         }
1185 #endif
1186         /* in secondary processes, memory init may allocate additional fbarrays
1187          * not present in primary processes, so to avoid any potential issues,
1188          * initialize memzones first.
1189          */
1190         if (rte_eal_memzone_init() < 0) {
1191                 rte_eal_init_alert("Cannot init memzone");
1192                 rte_errno = ENODEV;
1193                 return -1;
1194         }
1195
1196         if (rte_eal_memory_init() < 0) {
1197                 rte_eal_init_alert("Cannot init memory");
1198                 rte_errno = ENOMEM;
1199                 return -1;
1200         }
1201
1202         /* the directories are locked during eal_hugepage_info_init */
1203         eal_hugedirs_unlock();
1204
1205         if (rte_eal_malloc_heap_init() < 0) {
1206                 rte_eal_init_alert("Cannot init malloc heap");
1207                 rte_errno = ENODEV;
1208                 return -1;
1209         }
1210
1211         if (rte_eal_tailqs_init() < 0) {
1212                 rte_eal_init_alert("Cannot init tail queues for objects");
1213                 rte_errno = EFAULT;
1214                 return -1;
1215         }
1216
1217         if (rte_eal_timer_init() < 0) {
1218                 rte_eal_init_alert("Cannot init HPET or TSC timers");
1219                 rte_errno = ENOTSUP;
1220                 return -1;
1221         }
1222
1223         eal_check_mem_on_local_socket();
1224
1225         if (pthread_setaffinity_np(pthread_self(), sizeof(rte_cpuset_t),
1226                         &lcore_config[config->main_lcore].cpuset) != 0) {
1227                 rte_eal_init_alert("Cannot set affinity");
1228                 rte_errno = EINVAL;
1229                 return -1;
1230         }
1231         __rte_thread_init(config->main_lcore,
1232                 &lcore_config[config->main_lcore].cpuset);
1233
1234         ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
1235         RTE_LOG(DEBUG, EAL, "Main lcore %u is ready (tid=%zx;cpuset=[%s%s])\n",
1236                 config->main_lcore, (uintptr_t)thread_id, cpuset,
1237                 ret == 0 ? "" : "...");
1238
1239         RTE_LCORE_FOREACH_WORKER(i) {
1240
1241                 /*
1242                  * create communication pipes between main thread
1243                  * and children
1244                  */
1245                 if (pipe(lcore_config[i].pipe_main2worker) < 0)
1246                         rte_panic("Cannot create pipe\n");
1247                 if (pipe(lcore_config[i].pipe_worker2main) < 0)
1248                         rte_panic("Cannot create pipe\n");
1249
1250                 lcore_config[i].state = WAIT;
1251
1252                 /* create a thread for each lcore */
1253                 ret = pthread_create(&lcore_config[i].thread_id, NULL,
1254                                      eal_thread_loop, NULL);
1255                 if (ret != 0)
1256                         rte_panic("Cannot create thread\n");
1257
1258                 /* Set thread_name for aid in debugging. */
1259                 snprintf(thread_name, sizeof(thread_name),
1260                         "lcore-worker-%d", i);
1261                 ret = rte_thread_setname(lcore_config[i].thread_id,
1262                                                 thread_name);
1263                 if (ret != 0)
1264                         RTE_LOG(DEBUG, EAL,
1265                                 "Cannot set name for lcore thread\n");
1266
1267                 ret = pthread_setaffinity_np(lcore_config[i].thread_id,
1268                         sizeof(rte_cpuset_t), &lcore_config[i].cpuset);
1269                 if (ret != 0)
1270                         rte_panic("Cannot set affinity\n");
1271         }
1272
1273         /*
1274          * Launch a dummy function on all worker lcores, so that main lcore
1275          * knows they are all ready when this function returns.
1276          */
1277         rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MAIN);
1278         rte_eal_mp_wait_lcore();
1279
1280         /* initialize services so vdevs register service during bus_probe. */
1281         ret = rte_service_init();
1282         if (ret) {
1283                 rte_eal_init_alert("rte_service_init() failed");
1284                 rte_errno = -ret;
1285                 return -1;
1286         }
1287
1288         /* Probe all the buses and devices/drivers on them */
1289         if (rte_bus_probe()) {
1290                 rte_eal_init_alert("Cannot probe devices");
1291                 rte_errno = ENOTSUP;
1292                 return -1;
1293         }
1294
1295 #ifdef VFIO_PRESENT
1296         /* Register mp action after probe() so that we got enough info */
1297         if (rte_vfio_is_enabled("vfio") && vfio_mp_sync_setup() < 0)
1298                 return -1;
1299 #endif
1300
1301         /* initialize default service/lcore mappings and start running. Ignore
1302          * -ENOTSUP, as it indicates no service coremask passed to EAL.
1303          */
1304         ret = rte_service_start_with_defaults();
1305         if (ret < 0 && ret != -ENOTSUP) {
1306                 rte_errno = -ret;
1307                 return -1;
1308         }
1309
1310         /*
1311          * Clean up unused files in runtime directory. We do this at the end of
1312          * init and not at the beginning because we want to clean stuff up
1313          * whether we are primary or secondary process, but we cannot remove
1314          * primary process' files because secondary should be able to run even
1315          * if primary process is dead.
1316          *
1317          * In no_shconf mode, no runtime directory is created in the first
1318          * place, so no cleanup needed.
1319          */
1320         if (!internal_conf->no_shconf && eal_clean_runtime_dir() < 0) {
1321                 rte_eal_init_alert("Cannot clear runtime directory");
1322                 return -1;
1323         }
1324         if (rte_eal_process_type() == RTE_PROC_PRIMARY && !internal_conf->no_telemetry) {
1325                 int tlog = rte_log_register_type_and_pick_level(
1326                                 "lib.telemetry", RTE_LOG_WARNING);
1327                 if (tlog < 0)
1328                         tlog = RTE_LOGTYPE_EAL;
1329                 if (rte_telemetry_init(rte_eal_get_runtime_dir(),
1330                                 rte_version(),
1331                                 &internal_conf->ctrl_cpuset, rte_log, tlog) != 0)
1332                         return -1;
1333         }
1334
1335         eal_mcfg_complete();
1336
1337         return fctret;
1338 }
1339
1340 static int
1341 mark_freeable(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
1342                 void *arg __rte_unused)
1343 {
1344         /* ms is const, so find this memseg */
1345         struct rte_memseg *found;
1346
1347         if (msl->external)
1348                 return 0;
1349
1350         found = rte_mem_virt2memseg(ms->addr, msl);
1351
1352         found->flags &= ~RTE_MEMSEG_FLAG_DO_NOT_FREE;
1353
1354         return 0;
1355 }
1356
1357 int
1358 rte_eal_cleanup(void)
1359 {
1360         /* if we're in a primary process, we need to mark hugepages as freeable
1361          * so that finalization can release them back to the system.
1362          */
1363         struct internal_config *internal_conf =
1364                 eal_get_internal_configuration();
1365
1366         if (rte_eal_process_type() == RTE_PROC_PRIMARY &&
1367                         internal_conf->hugepage_file.unlink_existing)
1368                 rte_memseg_walk(mark_freeable, NULL);
1369         rte_service_finalize();
1370         rte_mp_channel_cleanup();
1371         /* after this point, any DPDK pointers will become dangling */
1372         rte_eal_memory_detach();
1373         rte_eal_alarm_cleanup();
1374         rte_trace_save();
1375         eal_trace_fini();
1376         eal_cleanup_config(internal_conf);
1377         return 0;
1378 }
1379
1380 int rte_eal_create_uio_dev(void)
1381 {
1382         const struct internal_config *internal_conf =
1383                 eal_get_internal_configuration();
1384
1385         return internal_conf->create_uio_dev;
1386 }
1387
1388 enum rte_intr_mode
1389 rte_eal_vfio_intr_mode(void)
1390 {
1391         const struct internal_config *internal_conf =
1392                 eal_get_internal_configuration();
1393
1394         return internal_conf->vfio_intr_mode;
1395 }
1396
1397 void
1398 rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
1399 {
1400         struct internal_config *cfg = eal_get_internal_configuration();
1401
1402         rte_uuid_copy(vf_token, cfg->vfio_vf_token);
1403 }
1404
1405 int
1406 rte_eal_check_module(const char *module_name)
1407 {
1408         char sysfs_mod_name[PATH_MAX];
1409         struct stat st;
1410         int n;
1411
1412         if (NULL == module_name)
1413                 return -1;
1414
1415         /* Check if there is sysfs mounted */
1416         if (stat("/sys/module", &st) != 0) {
1417                 RTE_LOG(DEBUG, EAL, "sysfs is not mounted! error %i (%s)\n",
1418                         errno, strerror(errno));
1419                 return -1;
1420         }
1421
1422         /* A module might be built-in, therefore try sysfs */
1423         n = snprintf(sysfs_mod_name, PATH_MAX, "/sys/module/%s", module_name);
1424         if (n < 0 || n > PATH_MAX) {
1425                 RTE_LOG(DEBUG, EAL, "Could not format module path\n");
1426                 return -1;
1427         }
1428
1429         if (stat(sysfs_mod_name, &st) != 0) {
1430                 RTE_LOG(DEBUG, EAL, "Module %s not found! error %i (%s)\n",
1431                         sysfs_mod_name, errno, strerror(errno));
1432                 return 0;
1433         }
1434
1435         /* Module has been found */
1436         return 1;
1437 }