lib/librte_power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24
  25 #ifdef RTE_LIBRTE_POWER_DEBUG
  26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  27                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  28 } while (0)
  29 #else
  30 #define POWER_DEBUG_TRACE(fmt, args...)
  31 #endif
  32
  33 #define FOPEN_OR_ERR_RET(f, retval) do { \
  34                 if ((f) == NULL) { \
  35                         RTE_LOG(ERR, POWER, "File not opened\n"); \
  36                         return retval; \
  37                 } \
  38 } while (0)
  39
  40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  41                 if ((ret) == NULL) { \
  42                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  43                         goto label; \
  44                 } \
  45 } while (0)
  46
  47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  48                 if ((ret) < 0) { \
  49                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  50                         goto label; \
  51                 } \
  52 } while (0)
  53
  54
  55 #define POWER_CONVERT_TO_DECIMAL 10
  56 #define BUS_FREQ     100000
  57
  58 #define POWER_GOVERNOR_PERF "performance"
  59 #define POWER_SYSFILE_GOVERNOR  \
  60                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  61 #define POWER_SYSFILE_MAX_FREQ \
  62                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  63 #define POWER_SYSFILE_MIN_FREQ  \
  64                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  65 #define POWER_SYSFILE_CUR_FREQ  \
  66                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  67 #define POWER_SYSFILE_BASE_MAX_FREQ \
  68                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  69 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  70                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  71 #define POWER_SYSFILE_BASE_FREQ  \
  72                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  73 #define POWER_PSTATE_DRIVER "intel_pstate"
  74 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  75
  76 /*
  77  * MSR related
  78  */
  79 #define PLATFORM_INFO     0x0CE
  80 #define NON_TURBO_MASK    0xFF00
  81 #define NON_TURBO_OFFSET  0x8
  82
  83
  84 enum power_state {
  85         POWER_IDLE = 0,
  86         POWER_ONGOING,
  87         POWER_USED,
  88         POWER_UNKNOWN
  89 };
  90
  91 struct pstate_power_info {
  92         unsigned int lcore_id;               /**< Logical core id */
  93         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  94         uint32_t nb_freqs;                   /**< number of available freqs */
  95         FILE *f_cur_min;                     /**< FD of scaling_min */
  96         FILE *f_cur_max;                     /**< FD of scaling_max */
  97         char governor_ori[32];               /**< Original governor name */
  98         uint32_t curr_idx;                   /**< Freq index in freqs array */
  99         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 100         uint32_t sys_max_freq;               /**< system wide max freq  */
 101         uint32_t core_base_freq;             /**< core base freq  */
 102         uint32_t state;                      /**< Power in use state */
 103         uint16_t turbo_available;            /**< Turbo Boost available */
 104         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 105         uint16_t priority_core;              /**< High Performance core */
 106 } __rte_cache_aligned;
 107
 108
 109 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 110
 111 /**
 112  * It is to read the specific MSR.
 113  */
 114
 115 static int32_t
 116 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 117 {
 118         int fd, ret;
 119         char fullpath[PATH_MAX];
 120
 121         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 122
 123         fd = open(fullpath, O_RDONLY);
 124
 125         if (fd < 0) {
 126                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 127                                  strerror(errno));
 128                 return fd;
 129         }
 130
 131         ret = pread(fd, val, sizeof(uint64_t), msr);
 132
 133         if (ret < 0) {
 134                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 135                                  strerror(errno));
 136                 goto out;
 137         }
 138
 139         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 140                         fullpath, msr, lcore_id);
 141
 142         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 143
 144 out:    close(fd);
 145         return ret;
 146 }
 147
 148 /**
 149  * It is to fopen the sys file for the future setting the lcore frequency.
 150  */
 151 static int
 152 power_init_for_setting_freq(struct pstate_power_info *pi)
 153 {
 154         FILE *f_min, *f_max, *f_base;
 155         char fullpath_min[PATH_MAX];
 156         char fullpath_max[PATH_MAX];
 157         char fullpath_base[PATH_MAX];
 158         char buf_base[BUFSIZ];
 159         char *s_base;
 160         uint32_t base_ratio = 0;
 161         uint64_t max_non_turbo = 0;
 162         int  ret_val = 0;
 163
 164         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 165                         pi->lcore_id);
 166
 167         f_min = fopen(fullpath_min, "rw+");
 168         FOPEN_OR_ERR_RET(f_min, -1);
 169
 170         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 171                         pi->lcore_id);
 172
 173         f_max = fopen(fullpath_max, "rw+");
 174         if (f_max == NULL)
 175                 fclose(f_min);
 176
 177         FOPEN_OR_ERR_RET(f_max, -1);
 178
 179         pi->f_cur_min = f_min;
 180         pi->f_cur_max = f_max;
 181
 182         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 183                         pi->lcore_id);
 184
 185         f_base = fopen(fullpath_base, "r");
 186         if (f_base == NULL) {
 187                 /* No sysfs base_frequency, that's OK, continue without */
 188                 base_ratio = 0;
 189         } else {
 190                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 191                 FOPS_OR_NULL_GOTO(s_base, out);
 192
 193                 buf_base[BUFSIZ-1] = '\0';
 194                 if (strlen(buf_base))
 195                         /* Strip off terminating '\n' */
 196                         strtok(buf_base, "\n");
 197
 198                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 199                                 / BUS_FREQ;
 200         }
 201
 202         /* Add MSR read to detect turbo status */
 203
 204         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 205                 ret_val = -1;
 206                 goto out;
 207         }
 208
 209         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 210
 211         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 212
 213         pi->non_turbo_max_ratio = max_non_turbo;
 214
 215         /*
 216          * If base_frequency is reported as greater than the maximum
 217          * non-turbo frequency, then mark it as a high priority core.
 218          */
 219         if (base_ratio > max_non_turbo)
 220                 pi->priority_core = 1;
 221         else
 222                 pi->priority_core = 0;
 223         pi->core_base_freq = base_ratio * BUS_FREQ;
 224
 225 out:
 226         if (f_base != NULL)
 227                 fclose(f_base);
 228         return ret_val;
 229 }
 230
 231 static int
 232 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 233 {
 234         uint32_t target_freq = 0;
 235
 236         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 237                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 238                                 "should be less than %u\n", idx, pi->nb_freqs);
 239                 return -1;
 240         }
 241
 242         /* Check if it is the same as current */
 243         if (idx == pi->curr_idx)
 244                 return 0;
 245
 246         /* Because Intel Pstate Driver only allow user change min/max hint
 247          * User need change the min/max as same value.
 248          */
 249         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 250                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 251                                 "for setting frequency for lcore %u\n",
 252                                 pi->lcore_id);
 253                 return -1;
 254         }
 255
 256         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 257                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 258                                 "for setting frequency for lcore %u\n",
 259                                 pi->lcore_id);
 260                 return -1;
 261         }
 262
 263         /* Turbo is available and enabled, first freq bucket is sys max freq */
 264         if (pi->turbo_available && idx == 0) {
 265                 if (pi->turbo_enable)
 266                         target_freq = pi->sys_max_freq;
 267                 else {
 268                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 269                                         pi->lcore_id);
 270                         return -1;
 271                 }
 272         } else
 273                 target_freq = pi->freqs[idx];
 274
 275         /* Decrease freq, the min freq should be updated first */
 276         if (idx  >  pi->curr_idx) {
 277
 278                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 279                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 280                                         "lcore %u\n", pi->lcore_id);
 281                         return -1;
 282                 }
 283
 284                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 285                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 286                                         "lcore %u\n", pi->lcore_id);
 287                         return -1;
 288                 }
 289
 290                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 291                                   target_freq, pi->lcore_id);
 292
 293                 fflush(pi->f_cur_min);
 294                 fflush(pi->f_cur_max);
 295
 296         }
 297
 298         /* Increase freq, the max freq should be updated first */
 299         if (idx  <  pi->curr_idx) {
 300
 301                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 302                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 303                                         "lcore %u\n", pi->lcore_id);
 304                         return -1;
 305                 }
 306
 307                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 308                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 309                                         "lcore %u\n", pi->lcore_id);
 310                         return -1;
 311                 }
 312
 313                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 314                                   target_freq, pi->lcore_id);
 315
 316                 fflush(pi->f_cur_max);
 317                 fflush(pi->f_cur_min);
 318         }
 319
 320         pi->curr_idx = idx;
 321
 322         return 1;
 323 }
 324
 325 /**
 326  * It is to check the current scaling governor by reading sys file, and then
 327  * set it into 'performance' if it is not by writing the sys file. The original
 328  * governor will be saved for rolling back.
 329  */
 330 static int
 331 power_set_governor_performance(struct pstate_power_info *pi)
 332 {
 333         FILE *f;
 334         int ret = -1;
 335         char buf[BUFSIZ];
 336         char fullpath[PATH_MAX];
 337         char *s;
 338         int val;
 339
 340         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 341                         pi->lcore_id);
 342         f = fopen(fullpath, "rw+");
 343         FOPEN_OR_ERR_RET(f, ret);
 344
 345         s = fgets(buf, sizeof(buf), f);
 346         FOPS_OR_NULL_GOTO(s, out);
 347         /* Strip off terminating '\n' */
 348         strtok(buf, "\n");
 349
 350         /* Check if current governor is performance */
 351         if (strncmp(buf, POWER_GOVERNOR_PERF,
 352                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 353                 ret = 0;
 354                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 355                                 "already performance\n", pi->lcore_id);
 356                 goto out;
 357         }
 358         /* Save the original governor */
 359         strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 360
 361         /* Write 'performance' to the governor */
 362         val = fseek(f, 0, SEEK_SET);
 363         FOPS_OR_ERR_GOTO(val, out);
 364
 365         val = fputs(POWER_GOVERNOR_PERF, f);
 366         FOPS_OR_ERR_GOTO(val, out);
 367
 368         /* We need to flush to see if the fputs succeeds */
 369         val = fflush(f);
 370         FOPS_OR_ERR_GOTO(val, out);
 371
 372         ret = 0;
 373         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 374                         "set to performance successfully\n", pi->lcore_id);
 375 out:
 376         fclose(f);
 377
 378         return ret;
 379 }
 380
 381 /**
 382  * It is to check the governor and then set the original governor back if
 383  * needed by writing the sys file.
 384  */
 385 static int
 386 power_set_governor_original(struct pstate_power_info *pi)
 387 {
 388         FILE *f;
 389         int ret = -1;
 390         char buf[BUFSIZ];
 391         char fullpath[PATH_MAX];
 392         char *s;
 393         int val;
 394
 395         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 396                         pi->lcore_id);
 397         f = fopen(fullpath, "rw+");
 398         FOPEN_OR_ERR_RET(f, ret);
 399
 400         s = fgets(buf, sizeof(buf), f);
 401         FOPS_OR_NULL_GOTO(s, out);
 402
 403         /* Check if the governor to be set is the same as current */
 404         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 405                 ret = 0;
 406                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 407                                 "has already been set to %s\n",
 408                                 pi->lcore_id, pi->governor_ori);
 409                 goto out;
 410         }
 411
 412         /* Write back the original governor */
 413         val = fseek(f, 0, SEEK_SET);
 414         FOPS_OR_ERR_GOTO(val, out);
 415
 416         val = fputs(pi->governor_ori, f);
 417         FOPS_OR_ERR_GOTO(val, out);
 418
 419         ret = 0;
 420         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 421                         "has been set back to %s successfully\n",
 422                         pi->lcore_id, pi->governor_ori);
 423 out:
 424         fclose(f);
 425
 426         return ret;
 427 }
 428
 429 /**
 430  * It is to get the available frequencies of the specific lcore by reading the
 431  * sys file.
 432  */
 433 static int
 434 power_get_available_freqs(struct pstate_power_info *pi)
 435 {
 436         FILE *f_min, *f_max;
 437         int ret = -1;
 438         char *p_min, *p_max;
 439         char buf_min[BUFSIZ];
 440         char buf_max[BUFSIZ];
 441         char fullpath_min[PATH_MAX];
 442         char fullpath_max[PATH_MAX];
 443         char *s_min, *s_max;
 444         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 445         uint32_t i, num_freqs = 0;
 446
 447         snprintf(fullpath_max, sizeof(fullpath_max),
 448                         POWER_SYSFILE_BASE_MAX_FREQ,
 449                         pi->lcore_id);
 450         snprintf(fullpath_min, sizeof(fullpath_min),
 451                         POWER_SYSFILE_BASE_MIN_FREQ,
 452                         pi->lcore_id);
 453
 454         f_min = fopen(fullpath_min, "r");
 455         FOPEN_OR_ERR_RET(f_min, ret);
 456
 457         f_max = fopen(fullpath_max, "r");
 458         if (f_max == NULL)
 459                 fclose(f_min);
 460
 461         FOPEN_OR_ERR_RET(f_max, ret);
 462
 463         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 464         FOPS_OR_NULL_GOTO(s_min, out);
 465
 466         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 467         FOPS_OR_NULL_GOTO(s_max, out);
 468
 469
 470         /* Strip the line break if there is */
 471         p_min = strchr(buf_min, '\n');
 472         if (p_min != NULL)
 473                 *p_min = 0;
 474
 475         p_max = strchr(buf_max, '\n');
 476         if (p_max != NULL)
 477                 *p_max = 0;
 478
 479         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 480         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 481
 482         if (sys_max_freq < sys_min_freq)
 483                 goto out;
 484
 485         pi->sys_max_freq = sys_max_freq;
 486
 487         if (pi->priority_core == 1)
 488                 base_max_freq = pi->core_base_freq;
 489         else
 490                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 491
 492         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 493                         sys_min_freq,
 494                         sys_max_freq,
 495                         base_max_freq);
 496
 497         if (base_max_freq < sys_max_freq)
 498                 pi->turbo_available = 1;
 499         else
 500                 pi->turbo_available = 0;
 501
 502         /* If turbo is available then there is one extra freq bucket
 503          * to store the sys max freq which value is base_max +1
 504          */
 505         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 506                 pi->turbo_available;
 507
 508         /* Generate the freq bucket array.
 509          * If turbo is available the freq bucket[0] value is base_max +1
 510          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 511          * and so on.
 512          * If turbo is not available bucket[0] is base_max and so on
 513          */
 514         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 515                 if ((i == 0) && pi->turbo_available)
 516                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 517                 else
 518                         pi->freqs[pi->nb_freqs++] =
 519                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 520         }
 521
 522         ret = 0;
 523
 524         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 525                         num_freqs, pi->lcore_id);
 526
 527 out:
 528         fclose(f_min);
 529         fclose(f_max);
 530
 531         return ret;
 532 }
 533
 534 int
 535 power_pstate_cpufreq_check_supported(void)
 536 {
 537         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 538 }
 539
 540 int
 541 power_pstate_cpufreq_init(unsigned int lcore_id)
 542 {
 543         struct pstate_power_info *pi;
 544         uint32_t exp_state;
 545
 546         if (lcore_id >= RTE_MAX_LCORE) {
 547                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 548                                 lcore_id, RTE_MAX_LCORE - 1U);
 549                 return -1;
 550         }
 551
 552         pi = &lcore_power_info[lcore_id];
 553         exp_state = POWER_IDLE;
 554         /* The power in use state works as a guard variable between
 555          * the CPU frequency control initialization and exit process.
 556          * The ACQUIRE memory ordering here pairs with the RELEASE
 557          * ordering below as lock to make sure the frequency operations
 558          * in the critical section are done under the correct state.
 559          */
 560         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 561                                         POWER_ONGOING, 0,
 562                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 563                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 564                                 "in use\n", lcore_id);
 565                 return -1;
 566         }
 567
 568         pi->lcore_id = lcore_id;
 569         /* Check and set the governor */
 570         if (power_set_governor_performance(pi) < 0) {
 571                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 572                                 "performance\n", lcore_id);
 573                 goto fail;
 574         }
 575         /* Init for setting lcore frequency */
 576         if (power_init_for_setting_freq(pi) < 0) {
 577                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 578                                 "lcore %u\n", lcore_id);
 579                 goto fail;
 580         }
 581
 582         /* Get the available frequencies */
 583         if (power_get_available_freqs(pi) < 0) {
 584                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 585                                 "lcore %u\n", lcore_id);
 586                 goto fail;
 587         }
 588
 589
 590         /* Set freq to max by default */
 591         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 592                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 593                                 "to max\n", lcore_id);
 594                 goto fail;
 595         }
 596
 597         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 598                         "power management\n", lcore_id);
 599         exp_state = POWER_ONGOING;
 600         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 601                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 602
 603         return 0;
 604
 605 fail:
 606         exp_state = POWER_ONGOING;
 607         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 608                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 609
 610         return -1;
 611 }
 612
 613 int
 614 power_pstate_cpufreq_exit(unsigned int lcore_id)
 615 {
 616         struct pstate_power_info *pi;
 617         uint32_t exp_state;
 618
 619         if (lcore_id >= RTE_MAX_LCORE) {
 620                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 621                                 lcore_id, RTE_MAX_LCORE - 1U);
 622                 return -1;
 623         }
 624         pi = &lcore_power_info[lcore_id];
 625
 626         exp_state = POWER_USED;
 627         /* The power in use state works as a guard variable between
 628          * the CPU frequency control initialization and exit process.
 629          * The ACQUIRE memory ordering here pairs with the RELEASE
 630          * ordering below as lock to make sure the frequency operations
 631          * in the critical section are under done the correct state.
 632          */
 633         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 634                                         POWER_ONGOING, 0,
 635                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 636                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 637                                 "not used\n", lcore_id);
 638                 return -1;
 639         }
 640
 641         /* Close FD of setting freq */
 642         fclose(pi->f_cur_min);
 643         fclose(pi->f_cur_max);
 644         pi->f_cur_min = NULL;
 645         pi->f_cur_max = NULL;
 646
 647         /* Set the governor back to the original */
 648         if (power_set_governor_original(pi) < 0) {
 649                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 650                                 "to the original\n", lcore_id);
 651                 goto fail;
 652         }
 653
 654         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 655                         "'performance' mode and been set back to the "
 656                         "original\n", lcore_id);
 657         exp_state = POWER_ONGOING;
 658         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 659                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 660
 661         return 0;
 662
 663 fail:
 664         exp_state = POWER_ONGOING;
 665         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 666                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 667
 668         return -1;
 669 }
 670
 671
 672 uint32_t
 673 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 674 {
 675         struct pstate_power_info *pi;
 676
 677         if (lcore_id >= RTE_MAX_LCORE) {
 678                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 679                 return 0;
 680         }
 681
 682         if (freqs == NULL) {
 683                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 684                 return 0;
 685         }
 686
 687         pi = &lcore_power_info[lcore_id];
 688         if (num < pi->nb_freqs) {
 689                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 690                 return 0;
 691         }
 692         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 693
 694         return pi->nb_freqs;
 695 }
 696
 697 uint32_t
 698 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 699 {
 700         if (lcore_id >= RTE_MAX_LCORE) {
 701                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 702                 return RTE_POWER_INVALID_FREQ_INDEX;
 703         }
 704
 705         return lcore_power_info[lcore_id].curr_idx;
 706 }
 707
 708
 709 int
 710 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 711 {
 712         if (lcore_id >= RTE_MAX_LCORE) {
 713                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 714                 return -1;
 715         }
 716
 717         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 718 }
 719
 720 int
 721 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 722 {
 723         struct pstate_power_info *pi;
 724
 725         if (lcore_id >= RTE_MAX_LCORE) {
 726                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 727                 return -1;
 728         }
 729
 730         pi = &lcore_power_info[lcore_id];
 731         if (pi->curr_idx == 0 ||
 732             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 733                 return 0;
 734
 735         /* Frequencies in the array are from high to low. */
 736         return set_freq_internal(pi, pi->curr_idx - 1);
 737 }
 738
 739 int
 740 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 741 {
 742         struct pstate_power_info *pi;
 743
 744         if (lcore_id >= RTE_MAX_LCORE) {
 745                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 746                 return -1;
 747         }
 748
 749         pi = &lcore_power_info[lcore_id];
 750         if (pi->curr_idx + 1 == pi->nb_freqs)
 751                 return 0;
 752
 753         /* Frequencies in the array are from high to low. */
 754         return set_freq_internal(pi, pi->curr_idx + 1);
 755 }
 756
 757 int
 758 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 759 {
 760         if (lcore_id >= RTE_MAX_LCORE) {
 761                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 762                 return -1;
 763         }
 764
 765         /* Frequencies in the array are from high to low. */
 766         if (lcore_power_info[lcore_id].turbo_available) {
 767                 if (lcore_power_info[lcore_id].turbo_enable)
 768                         /* Set to Turbo */
 769                         return set_freq_internal(
 770                                         &lcore_power_info[lcore_id], 0);
 771                 else
 772                         /* Set to max non-turbo */
 773                         return set_freq_internal(
 774                                         &lcore_power_info[lcore_id], 1);
 775         } else
 776                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 777 }
 778
 779
 780 int
 781 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 782 {
 783         struct pstate_power_info *pi;
 784
 785         if (lcore_id >= RTE_MAX_LCORE) {
 786                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 787                 return -1;
 788         }
 789
 790         pi = &lcore_power_info[lcore_id];
 791
 792         /* Frequencies in the array are from high to low. */
 793         return set_freq_internal(pi, pi->nb_freqs - 1);
 794 }
 795
 796
 797 int
 798 power_pstate_turbo_status(unsigned int lcore_id)
 799 {
 800         struct pstate_power_info *pi;
 801
 802         if (lcore_id >= RTE_MAX_LCORE) {
 803                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 804                 return -1;
 805         }
 806
 807         pi = &lcore_power_info[lcore_id];
 808
 809         return pi->turbo_enable;
 810 }
 811
 812 int
 813 power_pstate_enable_turbo(unsigned int lcore_id)
 814 {
 815         struct pstate_power_info *pi;
 816
 817         if (lcore_id >= RTE_MAX_LCORE) {
 818                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 819                 return -1;
 820         }
 821
 822         pi = &lcore_power_info[lcore_id];
 823
 824         if (pi->turbo_available)
 825                 pi->turbo_enable = 1;
 826         else {
 827                 pi->turbo_enable = 0;
 828                 RTE_LOG(ERR, POWER,
 829                         "Failed to enable turbo on lcore %u\n",
 830                         lcore_id);
 831                         return -1;
 832         }
 833
 834         return 0;
 835 }
 836
 837
 838 int
 839 power_pstate_disable_turbo(unsigned int lcore_id)
 840 {
 841         struct pstate_power_info *pi;
 842
 843         if (lcore_id >= RTE_MAX_LCORE) {
 844                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 845                 return -1;
 846         }
 847
 848         pi = &lcore_power_info[lcore_id];
 849
 850         pi->turbo_enable = 0;
 851
 852         if (pi->turbo_available && pi->curr_idx <= 1) {
 853                 /* Try to set freq to max by default coming out of turbo */
 854                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 855                         RTE_LOG(ERR, POWER,
 856                                 "Failed to set frequency of lcore %u to max\n",
 857                                 lcore_id);
 858                         return -1;
 859                 }
 860         }
 861
 862         return 0;
 863 }
 864
 865
 866 int power_pstate_get_capabilities(unsigned int lcore_id,
 867                 struct rte_power_core_capabilities *caps)
 868 {
 869         struct pstate_power_info *pi;
 870
 871         if (lcore_id >= RTE_MAX_LCORE) {
 872                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 873                 return -1;
 874         }
 875         if (caps == NULL) {
 876                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 877                 return -1;
 878         }
 879
 880         pi = &lcore_power_info[lcore_id];
 881         caps->capabilities = 0;
 882         caps->turbo = !!(pi->turbo_available);
 883         caps->priority = pi->priority_core;
 884
 885         return 0;
 886 }