lib/power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24 /* macros used for rounding frequency to nearest 100000 */
  25 #define FREQ_ROUNDING_DELTA 50000
  26 #define ROUND_FREQ_TO_N_100000 100000
  27
  28 #define BUS_FREQ     100000
  29
  30 #define POWER_GOVERNOR_PERF "performance"
  31 #define POWER_SYSFILE_MAX_FREQ \
  32                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  33 #define POWER_SYSFILE_MIN_FREQ  \
  34                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  35 #define POWER_SYSFILE_CUR_FREQ  \
  36                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  37 #define POWER_SYSFILE_BASE_MAX_FREQ \
  38                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  39 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  40                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  41 #define POWER_SYSFILE_BASE_FREQ  \
  42                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  43 #define POWER_PSTATE_DRIVER "intel_pstate"
  44 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  45
  46 /*
  47  * MSR related
  48  */
  49 #define PLATFORM_INFO     0x0CE
  50 #define NON_TURBO_MASK    0xFF00
  51 #define NON_TURBO_OFFSET  0x8
  52
  53
  54 enum power_state {
  55         POWER_IDLE = 0,
  56         POWER_ONGOING,
  57         POWER_USED,
  58         POWER_UNKNOWN
  59 };
  60
  61 struct pstate_power_info {
  62         unsigned int lcore_id;               /**< Logical core id */
  63         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  64         uint32_t nb_freqs;                   /**< number of available freqs */
  65         FILE *f_cur_min;                     /**< FD of scaling_min */
  66         FILE *f_cur_max;                     /**< FD of scaling_max */
  67         char governor_ori[32];               /**< Original governor name */
  68         uint32_t curr_idx;                   /**< Freq index in freqs array */
  69         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
  70         uint32_t sys_max_freq;               /**< system wide max freq  */
  71         uint32_t core_base_freq;             /**< core base freq  */
  72         uint32_t state;                      /**< Power in use state */
  73         uint16_t turbo_available;            /**< Turbo Boost available */
  74         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
  75         uint16_t priority_core;              /**< High Performance core */
  76 } __rte_cache_aligned;
  77
  78
  79 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
  80
  81 /**
  82  * It is to read the specific MSR.
  83  */
  84
  85 static int32_t
  86 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
  87 {
  88         int fd, ret;
  89         char fullpath[PATH_MAX];
  90
  91         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
  92
  93         fd = open(fullpath, O_RDONLY);
  94
  95         if (fd < 0) {
  96                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
  97                                  strerror(errno));
  98                 return fd;
  99         }
 100
 101         ret = pread(fd, val, sizeof(uint64_t), msr);
 102
 103         if (ret < 0) {
 104                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 105                                  strerror(errno));
 106                 goto out;
 107         }
 108
 109         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 110                         fullpath, msr, lcore_id);
 111
 112         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 113
 114 out:    close(fd);
 115         return ret;
 116 }
 117
 118 /**
 119  * It is to fopen the sys file for the future setting the lcore frequency.
 120  */
 121 static int
 122 power_init_for_setting_freq(struct pstate_power_info *pi)
 123 {
 124         FILE *f_base = NULL, *f_base_max = NULL, *f_min = NULL, *f_max = NULL;
 125         uint32_t base_ratio, base_max_ratio;
 126         uint64_t max_non_turbo;
 127         int ret;
 128
 129         /* open all files we expect to have open */
 130         open_core_sysfs_file(&f_base_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 131                         pi->lcore_id);
 132         if (f_base_max == NULL) {
 133                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 134                                 POWER_SYSFILE_BASE_MAX_FREQ);
 135                 goto err;
 136         }
 137
 138         open_core_sysfs_file(&f_min, "rw+", POWER_SYSFILE_MIN_FREQ,
 139                         pi->lcore_id);
 140         if (f_min == NULL) {
 141                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 142                                 POWER_SYSFILE_MIN_FREQ);
 143                 goto err;
 144         }
 145
 146         open_core_sysfs_file(&f_max, "rw+", POWER_SYSFILE_MAX_FREQ,
 147                         pi->lcore_id);
 148         if (f_max == NULL) {
 149                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 150                                 POWER_SYSFILE_MAX_FREQ);
 151                 goto err;
 152         }
 153
 154         open_core_sysfs_file(&f_base, "r", POWER_SYSFILE_BASE_FREQ,
 155                         pi->lcore_id);
 156         /* base ratio file may not exist in some kernels, so no error check */
 157
 158         /* read base max ratio */
 159         ret = read_core_sysfs_u32(f_base_max, &base_max_ratio);
 160         if (ret < 0) {
 161                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 162                                 POWER_SYSFILE_BASE_MAX_FREQ);
 163                 goto err;
 164         }
 165
 166         /* base ratio may not exist */
 167         if (f_base != NULL) {
 168                 ret = read_core_sysfs_u32(f_base, &base_ratio);
 169                 if (ret < 0) {
 170                         RTE_LOG(ERR, POWER, "Failed to read %s\n",
 171                                         POWER_SYSFILE_BASE_FREQ);
 172                         goto err;
 173                 }
 174         } else {
 175                 base_ratio = 0;
 176         }
 177
 178         /* Add MSR read to detect turbo status */
 179         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0)
 180                 goto err;
 181         /* no errors after this point */
 182
 183         /* convert ratios to bins */
 184         base_max_ratio /= BUS_FREQ;
 185         base_ratio /= BUS_FREQ;
 186
 187         /* assign file handles */
 188         pi->f_cur_min = f_min;
 189         pi->f_cur_max = f_max;
 190
 191         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 192
 193         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 194
 195         pi->non_turbo_max_ratio = (uint32_t)max_non_turbo;
 196
 197         /*
 198          * If base_frequency is reported as greater than the maximum
 199          * turbo frequency, that's a known issue with some kernels.
 200          * Set base_frequency to max_non_turbo as a workaround.
 201          */
 202         if (base_ratio > base_max_ratio) {
 203                 /* base_ratio is greater than max turbo. Kernel bug. */
 204                 pi->priority_core = 0;
 205                 goto out;
 206         }
 207
 208         /*
 209          * If base_frequency is reported as greater than the maximum
 210          * non-turbo frequency, then mark it as a high priority core.
 211          */
 212         if (base_ratio > max_non_turbo)
 213                 pi->priority_core = 1;
 214         else
 215                 pi->priority_core = 0;
 216         pi->core_base_freq = base_ratio * BUS_FREQ;
 217
 218 out:
 219         if (f_base != NULL)
 220                 fclose(f_base);
 221         fclose(f_base_max);
 222         /* f_min and f_max are stored, no need to close */
 223         return 0;
 224
 225 err:
 226         if (f_base != NULL)
 227                 fclose(f_base);
 228         if (f_base_max != NULL)
 229                 fclose(f_base_max);
 230         if (f_min != NULL)
 231                 fclose(f_min);
 232         if (f_max != NULL)
 233                 fclose(f_max);
 234         return -1;
 235 }
 236
 237 static int
 238 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 239 {
 240         uint32_t target_freq = 0;
 241
 242         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 243                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 244                                 "should be less than %u\n", idx, pi->nb_freqs);
 245                 return -1;
 246         }
 247
 248         /* Check if it is the same as current */
 249         if (idx == pi->curr_idx)
 250                 return 0;
 251
 252         /* Because Intel Pstate Driver only allow user change min/max hint
 253          * User need change the min/max as same value.
 254          */
 255         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 256                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 257                                 "for setting frequency for lcore %u\n",
 258                                 pi->lcore_id);
 259                 return -1;
 260         }
 261
 262         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 263                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 264                                 "for setting frequency for lcore %u\n",
 265                                 pi->lcore_id);
 266                 return -1;
 267         }
 268
 269         /* Turbo is available and enabled, first freq bucket is sys max freq */
 270         if (pi->turbo_available && idx == 0) {
 271                 if (pi->turbo_enable)
 272                         target_freq = pi->sys_max_freq;
 273                 else {
 274                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 275                                         pi->lcore_id);
 276                         return -1;
 277                 }
 278         } else
 279                 target_freq = pi->freqs[idx];
 280
 281         /* Decrease freq, the min freq should be updated first */
 282         if (idx  >  pi->curr_idx) {
 283
 284                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 285                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 286                                         "lcore %u\n", pi->lcore_id);
 287                         return -1;
 288                 }
 289
 290                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 291                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 292                                         "lcore %u\n", pi->lcore_id);
 293                         return -1;
 294                 }
 295
 296                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 297                                   target_freq, pi->lcore_id);
 298
 299                 fflush(pi->f_cur_min);
 300                 fflush(pi->f_cur_max);
 301
 302         }
 303
 304         /* Increase freq, the max freq should be updated first */
 305         if (idx  <  pi->curr_idx) {
 306
 307                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 308                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 309                                         "lcore %u\n", pi->lcore_id);
 310                         return -1;
 311                 }
 312
 313                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 314                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 315                                         "lcore %u\n", pi->lcore_id);
 316                         return -1;
 317                 }
 318
 319                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 320                                   target_freq, pi->lcore_id);
 321
 322                 fflush(pi->f_cur_max);
 323                 fflush(pi->f_cur_min);
 324         }
 325
 326         pi->curr_idx = idx;
 327
 328         return 1;
 329 }
 330
 331 /**
 332  * It is to check the current scaling governor by reading sys file, and then
 333  * set it into 'performance' if it is not by writing the sys file. The original
 334  * governor will be saved for rolling back.
 335  */
 336 static int
 337 power_set_governor_performance(struct pstate_power_info *pi)
 338 {
 339         return power_set_governor(pi->lcore_id, POWER_GOVERNOR_PERF,
 340                         pi->governor_ori, sizeof(pi->governor_ori));
 341 }
 342
 343 /**
 344  * It is to check the governor and then set the original governor back if
 345  * needed by writing the sys file.
 346  */
 347 static int
 348 power_set_governor_original(struct pstate_power_info *pi)
 349 {
 350         return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
 351 }
 352
 353 /**
 354  * It is to get the available frequencies of the specific lcore by reading the
 355  * sys file.
 356  */
 357 static int
 358 power_get_available_freqs(struct pstate_power_info *pi)
 359 {
 360         FILE *f_min = NULL, *f_max = NULL;
 361         int ret = -1;
 362         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 363         uint32_t i, num_freqs = 0;
 364
 365         /* open all files */
 366         open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 367                         pi->lcore_id);
 368         if (f_max == NULL) {
 369                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 370                                 POWER_SYSFILE_BASE_MAX_FREQ);
 371                 goto out;
 372         }
 373
 374         open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_BASE_MIN_FREQ,
 375                         pi->lcore_id);
 376         if (f_min == NULL) {
 377                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 378                                 POWER_SYSFILE_BASE_MIN_FREQ);
 379                 goto out;
 380         }
 381
 382         /* read base ratios */
 383         ret = read_core_sysfs_u32(f_max, &sys_max_freq);
 384         if (ret < 0) {
 385                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 386                                 POWER_SYSFILE_BASE_MAX_FREQ);
 387                 goto out;
 388         }
 389
 390         ret = read_core_sysfs_u32(f_min, &sys_min_freq);
 391         if (ret < 0) {
 392                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 393                                 POWER_SYSFILE_BASE_MIN_FREQ);
 394                 goto out;
 395         }
 396
 397         if (sys_max_freq < sys_min_freq)
 398                 goto out;
 399
 400         pi->sys_max_freq = sys_max_freq;
 401
 402         if (pi->priority_core == 1)
 403                 base_max_freq = pi->core_base_freq;
 404         else
 405                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 406
 407         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 408                         sys_min_freq,
 409                         sys_max_freq,
 410                         base_max_freq);
 411
 412         if (base_max_freq < sys_max_freq)
 413                 pi->turbo_available = 1;
 414         else
 415                 pi->turbo_available = 0;
 416
 417         /* If turbo is available then there is one extra freq bucket
 418          * to store the sys max freq which value is base_max +1
 419          */
 420         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 421                 pi->turbo_available;
 422
 423         /* Generate the freq bucket array.
 424          * If turbo is available the freq bucket[0] value is base_max +1
 425          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 426          * and so on.
 427          * If turbo is not available bucket[0] is base_max and so on
 428          */
 429         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 430                 if ((i == 0) && pi->turbo_available)
 431                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 432                 else
 433                         pi->freqs[pi->nb_freqs++] =
 434                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 435         }
 436
 437         ret = 0;
 438
 439         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 440                         num_freqs, pi->lcore_id);
 441
 442 out:
 443         if (f_min != NULL)
 444                 fclose(f_min);
 445         if (f_max != NULL)
 446                 fclose(f_max);
 447
 448         return ret;
 449 }
 450
 451 static int
 452 power_get_cur_idx(struct pstate_power_info *pi)
 453 {
 454         FILE *f_cur;
 455         int ret = -1;
 456         uint32_t sys_cur_freq = 0;
 457         unsigned int i;
 458
 459         open_core_sysfs_file(&f_cur, "r", POWER_SYSFILE_CUR_FREQ,
 460                         pi->lcore_id);
 461         if (f_cur == NULL) {
 462                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 463                                 POWER_SYSFILE_CUR_FREQ);
 464                 goto fail;
 465         }
 466
 467         ret = read_core_sysfs_u32(f_cur, &sys_cur_freq);
 468         if (ret < 0) {
 469                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 470                                 POWER_SYSFILE_CUR_FREQ);
 471                 goto fail;
 472         }
 473
 474         /* convert the frequency to nearest 100000 value
 475          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 476          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 477          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 478          */
 479         unsigned int freq_conv = 0;
 480         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 481                                 / ROUND_FREQ_TO_N_100000;
 482         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 483
 484         for (i = 0; i < pi->nb_freqs; i++) {
 485                 if (freq_conv == pi->freqs[i]) {
 486                         pi->curr_idx = i;
 487                         break;
 488                 }
 489         }
 490
 491         ret = 0;
 492 fail:
 493         if (f_cur != NULL)
 494                 fclose(f_cur);
 495         return ret;
 496 }
 497
 498 int
 499 power_pstate_cpufreq_check_supported(void)
 500 {
 501         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 502 }
 503
 504 int
 505 power_pstate_cpufreq_init(unsigned int lcore_id)
 506 {
 507         struct pstate_power_info *pi;
 508         uint32_t exp_state;
 509
 510         if (lcore_id >= RTE_MAX_LCORE) {
 511                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 512                                 lcore_id, RTE_MAX_LCORE - 1U);
 513                 return -1;
 514         }
 515
 516         pi = &lcore_power_info[lcore_id];
 517         exp_state = POWER_IDLE;
 518         /* The power in use state works as a guard variable between
 519          * the CPU frequency control initialization and exit process.
 520          * The ACQUIRE memory ordering here pairs with the RELEASE
 521          * ordering below as lock to make sure the frequency operations
 522          * in the critical section are done under the correct state.
 523          */
 524         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 525                                         POWER_ONGOING, 0,
 526                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 527                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 528                                 "in use\n", lcore_id);
 529                 return -1;
 530         }
 531
 532         pi->lcore_id = lcore_id;
 533         /* Check and set the governor */
 534         if (power_set_governor_performance(pi) < 0) {
 535                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 536                                 "performance\n", lcore_id);
 537                 goto fail;
 538         }
 539         /* Init for setting lcore frequency */
 540         if (power_init_for_setting_freq(pi) < 0) {
 541                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 542                                 "lcore %u\n", lcore_id);
 543                 goto fail;
 544         }
 545
 546         /* Get the available frequencies */
 547         if (power_get_available_freqs(pi) < 0) {
 548                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 549                                 "lcore %u\n", lcore_id);
 550                 goto fail;
 551         }
 552
 553         if (power_get_cur_idx(pi) < 0) {
 554                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 555                                 "index of lcore %u\n", lcore_id);
 556                 goto fail;
 557         }
 558
 559         /* Set freq to max by default */
 560         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 561                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 562                                 "to max\n", lcore_id);
 563                 goto fail;
 564         }
 565
 566         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 567                         "power management\n", lcore_id);
 568         exp_state = POWER_ONGOING;
 569         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 570                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 571
 572         return 0;
 573
 574 fail:
 575         exp_state = POWER_ONGOING;
 576         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 577                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 578
 579         return -1;
 580 }
 581
 582 int
 583 power_pstate_cpufreq_exit(unsigned int lcore_id)
 584 {
 585         struct pstate_power_info *pi;
 586         uint32_t exp_state;
 587
 588         if (lcore_id >= RTE_MAX_LCORE) {
 589                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 590                                 lcore_id, RTE_MAX_LCORE - 1U);
 591                 return -1;
 592         }
 593         pi = &lcore_power_info[lcore_id];
 594
 595         exp_state = POWER_USED;
 596         /* The power in use state works as a guard variable between
 597          * the CPU frequency control initialization and exit process.
 598          * The ACQUIRE memory ordering here pairs with the RELEASE
 599          * ordering below as lock to make sure the frequency operations
 600          * in the critical section are under done the correct state.
 601          */
 602         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 603                                         POWER_ONGOING, 0,
 604                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 605                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 606                                 "not used\n", lcore_id);
 607                 return -1;
 608         }
 609
 610         /* Close FD of setting freq */
 611         fclose(pi->f_cur_min);
 612         fclose(pi->f_cur_max);
 613         pi->f_cur_min = NULL;
 614         pi->f_cur_max = NULL;
 615
 616         /* Set the governor back to the original */
 617         if (power_set_governor_original(pi) < 0) {
 618                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 619                                 "to the original\n", lcore_id);
 620                 goto fail;
 621         }
 622
 623         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 624                         "'performance' mode and been set back to the "
 625                         "original\n", lcore_id);
 626         exp_state = POWER_ONGOING;
 627         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 628                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 629
 630         return 0;
 631
 632 fail:
 633         exp_state = POWER_ONGOING;
 634         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 635                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 636
 637         return -1;
 638 }
 639
 640
 641 uint32_t
 642 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 643 {
 644         struct pstate_power_info *pi;
 645
 646         if (lcore_id >= RTE_MAX_LCORE) {
 647                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 648                 return 0;
 649         }
 650
 651         if (freqs == NULL) {
 652                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 653                 return 0;
 654         }
 655
 656         pi = &lcore_power_info[lcore_id];
 657         if (num < pi->nb_freqs) {
 658                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 659                 return 0;
 660         }
 661         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 662
 663         return pi->nb_freqs;
 664 }
 665
 666 uint32_t
 667 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 668 {
 669         if (lcore_id >= RTE_MAX_LCORE) {
 670                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 671                 return RTE_POWER_INVALID_FREQ_INDEX;
 672         }
 673
 674         return lcore_power_info[lcore_id].curr_idx;
 675 }
 676
 677
 678 int
 679 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 680 {
 681         if (lcore_id >= RTE_MAX_LCORE) {
 682                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 683                 return -1;
 684         }
 685
 686         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 687 }
 688
 689 int
 690 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 691 {
 692         struct pstate_power_info *pi;
 693
 694         if (lcore_id >= RTE_MAX_LCORE) {
 695                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 696                 return -1;
 697         }
 698
 699         pi = &lcore_power_info[lcore_id];
 700         if (pi->curr_idx == 0 ||
 701             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 702                 return 0;
 703
 704         /* Frequencies in the array are from high to low. */
 705         return set_freq_internal(pi, pi->curr_idx - 1);
 706 }
 707
 708 int
 709 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 710 {
 711         struct pstate_power_info *pi;
 712
 713         if (lcore_id >= RTE_MAX_LCORE) {
 714                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 715                 return -1;
 716         }
 717
 718         pi = &lcore_power_info[lcore_id];
 719         if (pi->curr_idx + 1 == pi->nb_freqs)
 720                 return 0;
 721
 722         /* Frequencies in the array are from high to low. */
 723         return set_freq_internal(pi, pi->curr_idx + 1);
 724 }
 725
 726 int
 727 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 728 {
 729         if (lcore_id >= RTE_MAX_LCORE) {
 730                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 731                 return -1;
 732         }
 733
 734         /* Frequencies in the array are from high to low. */
 735         if (lcore_power_info[lcore_id].turbo_available) {
 736                 if (lcore_power_info[lcore_id].turbo_enable)
 737                         /* Set to Turbo */
 738                         return set_freq_internal(
 739                                         &lcore_power_info[lcore_id], 0);
 740                 else
 741                         /* Set to max non-turbo */
 742                         return set_freq_internal(
 743                                         &lcore_power_info[lcore_id], 1);
 744         } else
 745                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 746 }
 747
 748
 749 int
 750 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 751 {
 752         struct pstate_power_info *pi;
 753
 754         if (lcore_id >= RTE_MAX_LCORE) {
 755                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 756                 return -1;
 757         }
 758
 759         pi = &lcore_power_info[lcore_id];
 760
 761         /* Frequencies in the array are from high to low. */
 762         return set_freq_internal(pi, pi->nb_freqs - 1);
 763 }
 764
 765
 766 int
 767 power_pstate_turbo_status(unsigned int lcore_id)
 768 {
 769         struct pstate_power_info *pi;
 770
 771         if (lcore_id >= RTE_MAX_LCORE) {
 772                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 773                 return -1;
 774         }
 775
 776         pi = &lcore_power_info[lcore_id];
 777
 778         return pi->turbo_enable;
 779 }
 780
 781 int
 782 power_pstate_enable_turbo(unsigned int lcore_id)
 783 {
 784         struct pstate_power_info *pi;
 785
 786         if (lcore_id >= RTE_MAX_LCORE) {
 787                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 788                 return -1;
 789         }
 790
 791         pi = &lcore_power_info[lcore_id];
 792
 793         if (pi->turbo_available)
 794                 pi->turbo_enable = 1;
 795         else {
 796                 pi->turbo_enable = 0;
 797                 RTE_LOG(ERR, POWER,
 798                         "Failed to enable turbo on lcore %u\n",
 799                         lcore_id);
 800                         return -1;
 801         }
 802
 803         return 0;
 804 }
 805
 806
 807 int
 808 power_pstate_disable_turbo(unsigned int lcore_id)
 809 {
 810         struct pstate_power_info *pi;
 811
 812         if (lcore_id >= RTE_MAX_LCORE) {
 813                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 814                 return -1;
 815         }
 816
 817         pi = &lcore_power_info[lcore_id];
 818
 819         pi->turbo_enable = 0;
 820
 821         if (pi->turbo_available && pi->curr_idx <= 1) {
 822                 /* Try to set freq to max by default coming out of turbo */
 823                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 824                         RTE_LOG(ERR, POWER,
 825                                 "Failed to set frequency of lcore %u to max\n",
 826                                 lcore_id);
 827                         return -1;
 828                 }
 829         }
 830
 831         return 0;
 832 }
 833
 834
 835 int power_pstate_get_capabilities(unsigned int lcore_id,
 836                 struct rte_power_core_capabilities *caps)
 837 {
 838         struct pstate_power_info *pi;
 839
 840         if (lcore_id >= RTE_MAX_LCORE) {
 841                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 842                 return -1;
 843         }
 844         if (caps == NULL) {
 845                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 846                 return -1;
 847         }
 848
 849         pi = &lcore_power_info[lcore_id];
 850         caps->capabilities = 0;
 851         caps->turbo = !!(pi->turbo_available);
 852         caps->priority = pi->priority_core;
 853
 854         return 0;
 855 }