lib/power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24 /* macros used for rounding frequency to nearest 100000 */
  25 #define FREQ_ROUNDING_DELTA 50000
  26 #define ROUND_FREQ_TO_N_100000 100000
  27
  28 #define BUS_FREQ     100000
  29
  30 #define POWER_GOVERNOR_PERF "performance"
  31 #define POWER_SYSFILE_MAX_FREQ \
  32                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  33 #define POWER_SYSFILE_MIN_FREQ  \
  34                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  35 #define POWER_SYSFILE_CUR_FREQ  \
  36                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  37 #define POWER_SYSFILE_BASE_MAX_FREQ \
  38                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  39 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  40                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  41 #define POWER_SYSFILE_BASE_FREQ  \
  42                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  43 #define POWER_PSTATE_DRIVER "intel_pstate"
  44 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  45
  46 /*
  47  * MSR related
  48  */
  49 #define PLATFORM_INFO     0x0CE
  50 #define NON_TURBO_MASK    0xFF00
  51 #define NON_TURBO_OFFSET  0x8
  52
  53
  54 enum power_state {
  55         POWER_IDLE = 0,
  56         POWER_ONGOING,
  57         POWER_USED,
  58         POWER_UNKNOWN
  59 };
  60
  61 struct pstate_power_info {
  62         unsigned int lcore_id;               /**< Logical core id */
  63         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  64         uint32_t nb_freqs;                   /**< number of available freqs */
  65         FILE *f_cur_min;                     /**< FD of scaling_min */
  66         FILE *f_cur_max;                     /**< FD of scaling_max */
  67         char governor_ori[32];               /**< Original governor name */
  68         uint32_t curr_idx;                   /**< Freq index in freqs array */
  69         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
  70         uint32_t sys_max_freq;               /**< system wide max freq  */
  71         uint32_t core_base_freq;             /**< core base freq  */
  72         uint32_t state;                      /**< Power in use state */
  73         uint16_t turbo_available;            /**< Turbo Boost available */
  74         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
  75         uint16_t priority_core;              /**< High Performance core */
  76 } __rte_cache_aligned;
  77
  78
  79 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
  80
  81 /**
  82  * It is to read the specific MSR.
  83  */
  84
  85 static int32_t
  86 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
  87 {
  88         int fd, ret;
  89         char fullpath[PATH_MAX];
  90
  91         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
  92
  93         fd = open(fullpath, O_RDONLY);
  94
  95         if (fd < 0) {
  96                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
  97                                  strerror(errno));
  98                 return fd;
  99         }
 100
 101         ret = pread(fd, val, sizeof(uint64_t), msr);
 102
 103         if (ret < 0) {
 104                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 105                                  strerror(errno));
 106                 goto out;
 107         }
 108
 109         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 110                         fullpath, msr, lcore_id);
 111
 112         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 113
 114 out:    close(fd);
 115         return ret;
 116 }
 117
 118 /**
 119  * It is to fopen the sys file for the future setting the lcore frequency.
 120  */
 121 static int
 122 power_init_for_setting_freq(struct pstate_power_info *pi)
 123 {
 124         FILE *f_base = NULL, *f_base_max = NULL, *f_min = NULL, *f_max = NULL;
 125         uint32_t base_ratio, base_max_ratio;
 126         uint64_t max_non_turbo;
 127         int ret;
 128
 129         /* open all files we expect to have open */
 130         open_core_sysfs_file(&f_base_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 131                         pi->lcore_id);
 132         if (f_base_max == NULL) {
 133                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 134                                 POWER_SYSFILE_BASE_MAX_FREQ);
 135                 goto err;
 136         }
 137
 138         open_core_sysfs_file(&f_min, "rw+", POWER_SYSFILE_MIN_FREQ,
 139                         pi->lcore_id);
 140         if (f_min == NULL) {
 141                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 142                                 POWER_SYSFILE_MIN_FREQ);
 143                 goto err;
 144         }
 145
 146         open_core_sysfs_file(&f_max, "rw+", POWER_SYSFILE_MAX_FREQ,
 147                         pi->lcore_id);
 148         if (f_max == NULL) {
 149                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 150                                 POWER_SYSFILE_MAX_FREQ);
 151                 goto err;
 152         }
 153
 154         open_core_sysfs_file(&f_base, "r", POWER_SYSFILE_BASE_FREQ,
 155                         pi->lcore_id);
 156         /* base ratio file may not exist in some kernels, so no error check */
 157
 158         /* read base max ratio */
 159         ret = read_core_sysfs_u32(f_base_max, &base_max_ratio);
 160         if (ret < 0) {
 161                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 162                                 POWER_SYSFILE_BASE_MAX_FREQ);
 163                 goto err;
 164         }
 165
 166         /* base ratio may not exist */
 167         if (f_base != NULL) {
 168                 ret = read_core_sysfs_u32(f_base, &base_ratio);
 169                 if (ret < 0) {
 170                         RTE_LOG(ERR, POWER, "Failed to read %s\n",
 171                                         POWER_SYSFILE_BASE_FREQ);
 172                         goto err;
 173                 }
 174         } else {
 175                 base_ratio = 0;
 176         }
 177
 178         /* Add MSR read to detect turbo status */
 179         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0)
 180                 goto err;
 181         /* no errors after this point */
 182
 183         /* convert ratios to bins */
 184         base_max_ratio /= BUS_FREQ;
 185         base_ratio /= BUS_FREQ;
 186
 187         /* assign file handles */
 188         pi->f_cur_min = f_min;
 189         pi->f_cur_max = f_max;
 190
 191         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 192
 193         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 194
 195         pi->non_turbo_max_ratio = (uint32_t)max_non_turbo;
 196
 197         /*
 198          * If base_frequency is reported as greater than the maximum
 199          * turbo frequency, that's a known issue with some kernels.
 200          * Set base_frequency to max_non_turbo as a workaround.
 201          */
 202         if (base_ratio > base_max_ratio) {
 203                 /* base_ratio is greater than max turbo. Kernel bug. */
 204                 pi->priority_core = 0;
 205                 goto out;
 206         }
 207
 208         /*
 209          * If base_frequency is reported as greater than the maximum
 210          * non-turbo frequency, then mark it as a high priority core.
 211          */
 212         if (base_ratio > max_non_turbo)
 213                 pi->priority_core = 1;
 214         else
 215                 pi->priority_core = 0;
 216         pi->core_base_freq = base_ratio * BUS_FREQ;
 217
 218 out:
 219         if (f_base != NULL)
 220                 fclose(f_base);
 221         fclose(f_base_max);
 222         /* f_min and f_max are stored, no need to close */
 223         return 0;
 224
 225 err:
 226         if (f_base != NULL)
 227                 fclose(f_base);
 228         if (f_base_max != NULL)
 229                 fclose(f_base_max);
 230         if (f_min != NULL)
 231                 fclose(f_min);
 232         if (f_max != NULL)
 233                 fclose(f_max);
 234         return -1;
 235 }
 236
 237 static int
 238 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 239 {
 240         uint32_t target_freq = 0;
 241
 242         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 243                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 244                                 "should be less than %u\n", idx, pi->nb_freqs);
 245                 return -1;
 246         }
 247
 248         /* Check if it is the same as current */
 249         if (idx == pi->curr_idx)
 250                 return 0;
 251
 252         /* Because Intel Pstate Driver only allow user change min/max hint
 253          * User need change the min/max as same value.
 254          */
 255         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 256                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 257                                 "for setting frequency for lcore %u\n",
 258                                 pi->lcore_id);
 259                 return -1;
 260         }
 261
 262         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 263                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 264                                 "for setting frequency for lcore %u\n",
 265                                 pi->lcore_id);
 266                 return -1;
 267         }
 268
 269         /* Turbo is available and enabled, first freq bucket is sys max freq */
 270         if (pi->turbo_available && idx == 0) {
 271                 if (pi->turbo_enable)
 272                         target_freq = pi->sys_max_freq;
 273                 else {
 274                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 275                                         pi->lcore_id);
 276                         return -1;
 277                 }
 278         } else
 279                 target_freq = pi->freqs[idx];
 280
 281         /* Decrease freq, the min freq should be updated first */
 282         if (idx  >  pi->curr_idx) {
 283
 284                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 285                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 286                                         "lcore %u\n", pi->lcore_id);
 287                         return -1;
 288                 }
 289
 290                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 291                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 292                                         "lcore %u\n", pi->lcore_id);
 293                         return -1;
 294                 }
 295
 296                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 297                                   target_freq, pi->lcore_id);
 298
 299                 fflush(pi->f_cur_min);
 300                 fflush(pi->f_cur_max);
 301
 302         }
 303
 304         /* Increase freq, the max freq should be updated first */
 305         if (idx  <  pi->curr_idx) {
 306
 307                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 308                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 309                                         "lcore %u\n", pi->lcore_id);
 310                         return -1;
 311                 }
 312
 313                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 314                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 315                                         "lcore %u\n", pi->lcore_id);
 316                         return -1;
 317                 }
 318
 319                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 320                                   target_freq, pi->lcore_id);
 321
 322                 fflush(pi->f_cur_max);
 323                 fflush(pi->f_cur_min);
 324         }
 325
 326         pi->curr_idx = idx;
 327
 328         return 1;
 329 }
 330
 331 /**
 332  * It is to check the current scaling governor by reading sys file, and then
 333  * set it into 'performance' if it is not by writing the sys file. The original
 334  * governor will be saved for rolling back.
 335  */
 336 static int
 337 power_set_governor_performance(struct pstate_power_info *pi)
 338 {
 339         return power_set_governor(pi->lcore_id, POWER_GOVERNOR_PERF,
 340                         pi->governor_ori, sizeof(pi->governor_ori));
 341 }
 342
 343 /**
 344  * It is to check the governor and then set the original governor back if
 345  * needed by writing the sys file.
 346  */
 347 static int
 348 power_set_governor_original(struct pstate_power_info *pi)
 349 {
 350         return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
 351 }
 352
 353 /**
 354  * It is to get the available frequencies of the specific lcore by reading the
 355  * sys file.
 356  */
 357 static int
 358 power_get_available_freqs(struct pstate_power_info *pi)
 359 {
 360         FILE *f_min = NULL, *f_max = NULL;
 361         int ret = -1;
 362         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 363         uint32_t i, num_freqs = 0;
 364
 365         /* open all files */
 366         open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 367                         pi->lcore_id);
 368         if (f_max == NULL) {
 369                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 370                                 POWER_SYSFILE_BASE_MAX_FREQ);
 371                 goto out;
 372         }
 373
 374         open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_BASE_MIN_FREQ,
 375                         pi->lcore_id);
 376         if (f_min == NULL) {
 377                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 378                                 POWER_SYSFILE_BASE_MIN_FREQ);
 379                 goto out;
 380         }
 381
 382         /* read base ratios */
 383         ret = read_core_sysfs_u32(f_max, &sys_max_freq);
 384         if (ret < 0) {
 385                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 386                                 POWER_SYSFILE_BASE_MAX_FREQ);
 387                 goto out;
 388         }
 389
 390         ret = read_core_sysfs_u32(f_min, &sys_min_freq);
 391         if (ret < 0) {
 392                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 393                                 POWER_SYSFILE_BASE_MIN_FREQ);
 394                 goto out;
 395         }
 396
 397         if (sys_max_freq < sys_min_freq)
 398                 goto out;
 399
 400         pi->sys_max_freq = sys_max_freq;
 401
 402         if (pi->priority_core == 1)
 403                 base_max_freq = pi->core_base_freq;
 404         else
 405                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 406
 407         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 408                         sys_min_freq,
 409                         sys_max_freq,
 410                         base_max_freq);
 411
 412         if (base_max_freq < sys_max_freq)
 413                 pi->turbo_available = 1;
 414         else
 415                 pi->turbo_available = 0;
 416
 417         /* If turbo is available then there is one extra freq bucket
 418          * to store the sys max freq which value is base_max +1
 419          */
 420         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 421                 pi->turbo_available;
 422         if (num_freqs >= RTE_MAX_LCORE_FREQS) {
 423                 RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
 424                                 num_freqs);
 425                 goto out;
 426         }
 427
 428         /* Generate the freq bucket array.
 429          * If turbo is available the freq bucket[0] value is base_max +1
 430          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 431          * and so on.
 432          * If turbo is not available bucket[0] is base_max and so on
 433          */
 434         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 435                 if ((i == 0) && pi->turbo_available)
 436                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 437                 else
 438                         pi->freqs[pi->nb_freqs++] =
 439                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 440         }
 441
 442         ret = 0;
 443
 444         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 445                         num_freqs, pi->lcore_id);
 446
 447 out:
 448         if (f_min != NULL)
 449                 fclose(f_min);
 450         if (f_max != NULL)
 451                 fclose(f_max);
 452
 453         return ret;
 454 }
 455
 456 static int
 457 power_get_cur_idx(struct pstate_power_info *pi)
 458 {
 459         FILE *f_cur;
 460         int ret = -1;
 461         uint32_t sys_cur_freq = 0;
 462         unsigned int i;
 463
 464         open_core_sysfs_file(&f_cur, "r", POWER_SYSFILE_CUR_FREQ,
 465                         pi->lcore_id);
 466         if (f_cur == NULL) {
 467                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 468                                 POWER_SYSFILE_CUR_FREQ);
 469                 goto fail;
 470         }
 471
 472         ret = read_core_sysfs_u32(f_cur, &sys_cur_freq);
 473         if (ret < 0) {
 474                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 475                                 POWER_SYSFILE_CUR_FREQ);
 476                 goto fail;
 477         }
 478
 479         /* convert the frequency to nearest 100000 value
 480          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 481          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 482          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 483          */
 484         unsigned int freq_conv = 0;
 485         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 486                                 / ROUND_FREQ_TO_N_100000;
 487         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 488
 489         for (i = 0; i < pi->nb_freqs; i++) {
 490                 if (freq_conv == pi->freqs[i]) {
 491                         pi->curr_idx = i;
 492                         break;
 493                 }
 494         }
 495
 496         ret = 0;
 497 fail:
 498         if (f_cur != NULL)
 499                 fclose(f_cur);
 500         return ret;
 501 }
 502
 503 int
 504 power_pstate_cpufreq_check_supported(void)
 505 {
 506         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 507 }
 508
 509 int
 510 power_pstate_cpufreq_init(unsigned int lcore_id)
 511 {
 512         struct pstate_power_info *pi;
 513         uint32_t exp_state;
 514
 515         if (lcore_id >= RTE_MAX_LCORE) {
 516                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 517                                 lcore_id, RTE_MAX_LCORE - 1U);
 518                 return -1;
 519         }
 520
 521         pi = &lcore_power_info[lcore_id];
 522         exp_state = POWER_IDLE;
 523         /* The power in use state works as a guard variable between
 524          * the CPU frequency control initialization and exit process.
 525          * The ACQUIRE memory ordering here pairs with the RELEASE
 526          * ordering below as lock to make sure the frequency operations
 527          * in the critical section are done under the correct state.
 528          */
 529         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 530                                         POWER_ONGOING, 0,
 531                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 532                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 533                                 "in use\n", lcore_id);
 534                 return -1;
 535         }
 536
 537         pi->lcore_id = lcore_id;
 538         /* Check and set the governor */
 539         if (power_set_governor_performance(pi) < 0) {
 540                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 541                                 "performance\n", lcore_id);
 542                 goto fail;
 543         }
 544         /* Init for setting lcore frequency */
 545         if (power_init_for_setting_freq(pi) < 0) {
 546                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 547                                 "lcore %u\n", lcore_id);
 548                 goto fail;
 549         }
 550
 551         /* Get the available frequencies */
 552         if (power_get_available_freqs(pi) < 0) {
 553                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 554                                 "lcore %u\n", lcore_id);
 555                 goto fail;
 556         }
 557
 558         if (power_get_cur_idx(pi) < 0) {
 559                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 560                                 "index of lcore %u\n", lcore_id);
 561                 goto fail;
 562         }
 563
 564         /* Set freq to max by default */
 565         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 566                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 567                                 "to max\n", lcore_id);
 568                 goto fail;
 569         }
 570
 571         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 572                         "power management\n", lcore_id);
 573         exp_state = POWER_ONGOING;
 574         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 575                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 576
 577         return 0;
 578
 579 fail:
 580         exp_state = POWER_ONGOING;
 581         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 582                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 583
 584         return -1;
 585 }
 586
 587 int
 588 power_pstate_cpufreq_exit(unsigned int lcore_id)
 589 {
 590         struct pstate_power_info *pi;
 591         uint32_t exp_state;
 592
 593         if (lcore_id >= RTE_MAX_LCORE) {
 594                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 595                                 lcore_id, RTE_MAX_LCORE - 1U);
 596                 return -1;
 597         }
 598         pi = &lcore_power_info[lcore_id];
 599
 600         exp_state = POWER_USED;
 601         /* The power in use state works as a guard variable between
 602          * the CPU frequency control initialization and exit process.
 603          * The ACQUIRE memory ordering here pairs with the RELEASE
 604          * ordering below as lock to make sure the frequency operations
 605          * in the critical section are under done the correct state.
 606          */
 607         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 608                                         POWER_ONGOING, 0,
 609                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 610                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 611                                 "not used\n", lcore_id);
 612                 return -1;
 613         }
 614
 615         /* Close FD of setting freq */
 616         fclose(pi->f_cur_min);
 617         fclose(pi->f_cur_max);
 618         pi->f_cur_min = NULL;
 619         pi->f_cur_max = NULL;
 620
 621         /* Set the governor back to the original */
 622         if (power_set_governor_original(pi) < 0) {
 623                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 624                                 "to the original\n", lcore_id);
 625                 goto fail;
 626         }
 627
 628         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 629                         "'performance' mode and been set back to the "
 630                         "original\n", lcore_id);
 631         exp_state = POWER_ONGOING;
 632         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 633                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 634
 635         return 0;
 636
 637 fail:
 638         exp_state = POWER_ONGOING;
 639         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 640                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 641
 642         return -1;
 643 }
 644
 645
 646 uint32_t
 647 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 648 {
 649         struct pstate_power_info *pi;
 650
 651         if (lcore_id >= RTE_MAX_LCORE) {
 652                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 653                 return 0;
 654         }
 655
 656         if (freqs == NULL) {
 657                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 658                 return 0;
 659         }
 660
 661         pi = &lcore_power_info[lcore_id];
 662         if (num < pi->nb_freqs) {
 663                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 664                 return 0;
 665         }
 666         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 667
 668         return pi->nb_freqs;
 669 }
 670
 671 uint32_t
 672 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 673 {
 674         if (lcore_id >= RTE_MAX_LCORE) {
 675                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 676                 return RTE_POWER_INVALID_FREQ_INDEX;
 677         }
 678
 679         return lcore_power_info[lcore_id].curr_idx;
 680 }
 681
 682
 683 int
 684 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 685 {
 686         if (lcore_id >= RTE_MAX_LCORE) {
 687                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 688                 return -1;
 689         }
 690
 691         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 692 }
 693
 694 int
 695 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 696 {
 697         struct pstate_power_info *pi;
 698
 699         if (lcore_id >= RTE_MAX_LCORE) {
 700                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 701                 return -1;
 702         }
 703
 704         pi = &lcore_power_info[lcore_id];
 705         if (pi->curr_idx == 0 ||
 706             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 707                 return 0;
 708
 709         /* Frequencies in the array are from high to low. */
 710         return set_freq_internal(pi, pi->curr_idx - 1);
 711 }
 712
 713 int
 714 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 715 {
 716         struct pstate_power_info *pi;
 717
 718         if (lcore_id >= RTE_MAX_LCORE) {
 719                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 720                 return -1;
 721         }
 722
 723         pi = &lcore_power_info[lcore_id];
 724         if (pi->curr_idx + 1 == pi->nb_freqs)
 725                 return 0;
 726
 727         /* Frequencies in the array are from high to low. */
 728         return set_freq_internal(pi, pi->curr_idx + 1);
 729 }
 730
 731 int
 732 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 733 {
 734         if (lcore_id >= RTE_MAX_LCORE) {
 735                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 736                 return -1;
 737         }
 738
 739         /* Frequencies in the array are from high to low. */
 740         if (lcore_power_info[lcore_id].turbo_available) {
 741                 if (lcore_power_info[lcore_id].turbo_enable)
 742                         /* Set to Turbo */
 743                         return set_freq_internal(
 744                                         &lcore_power_info[lcore_id], 0);
 745                 else
 746                         /* Set to max non-turbo */
 747                         return set_freq_internal(
 748                                         &lcore_power_info[lcore_id], 1);
 749         } else
 750                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 751 }
 752
 753
 754 int
 755 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 756 {
 757         struct pstate_power_info *pi;
 758
 759         if (lcore_id >= RTE_MAX_LCORE) {
 760                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 761                 return -1;
 762         }
 763
 764         pi = &lcore_power_info[lcore_id];
 765
 766         /* Frequencies in the array are from high to low. */
 767         return set_freq_internal(pi, pi->nb_freqs - 1);
 768 }
 769
 770
 771 int
 772 power_pstate_turbo_status(unsigned int lcore_id)
 773 {
 774         struct pstate_power_info *pi;
 775
 776         if (lcore_id >= RTE_MAX_LCORE) {
 777                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 778                 return -1;
 779         }
 780
 781         pi = &lcore_power_info[lcore_id];
 782
 783         return pi->turbo_enable;
 784 }
 785
 786 int
 787 power_pstate_enable_turbo(unsigned int lcore_id)
 788 {
 789         struct pstate_power_info *pi;
 790
 791         if (lcore_id >= RTE_MAX_LCORE) {
 792                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 793                 return -1;
 794         }
 795
 796         pi = &lcore_power_info[lcore_id];
 797
 798         if (pi->turbo_available)
 799                 pi->turbo_enable = 1;
 800         else {
 801                 pi->turbo_enable = 0;
 802                 RTE_LOG(ERR, POWER,
 803                         "Failed to enable turbo on lcore %u\n",
 804                         lcore_id);
 805                         return -1;
 806         }
 807
 808         return 0;
 809 }
 810
 811
 812 int
 813 power_pstate_disable_turbo(unsigned int lcore_id)
 814 {
 815         struct pstate_power_info *pi;
 816
 817         if (lcore_id >= RTE_MAX_LCORE) {
 818                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 819                 return -1;
 820         }
 821
 822         pi = &lcore_power_info[lcore_id];
 823
 824         pi->turbo_enable = 0;
 825
 826         if (pi->turbo_available && pi->curr_idx <= 1) {
 827                 /* Try to set freq to max by default coming out of turbo */
 828                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 829                         RTE_LOG(ERR, POWER,
 830                                 "Failed to set frequency of lcore %u to max\n",
 831                                 lcore_id);
 832                         return -1;
 833                 }
 834         }
 835
 836         return 0;
 837 }
 838
 839
 840 int power_pstate_get_capabilities(unsigned int lcore_id,
 841                 struct rte_power_core_capabilities *caps)
 842 {
 843         struct pstate_power_info *pi;
 844
 845         if (lcore_id >= RTE_MAX_LCORE) {
 846                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 847                 return -1;
 848         }
 849         if (caps == NULL) {
 850                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 851                 return -1;
 852         }
 853
 854         pi = &lcore_power_info[lcore_id];
 855         caps->capabilities = 0;
 856         caps->turbo = !!(pi->turbo_available);
 857         caps->priority = pi->priority_core;
 858
 859         return 0;
 860 }