lib/librte_power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_atomic.h>
  18 #include <rte_memcpy.h>
  19 #include <rte_memory.h>
  20 #include <rte_string_fns.h>
  21
  22 #include "power_pstate_cpufreq.h"
  23 #include "power_common.h"
  24
  25
  26 #ifdef RTE_LIBRTE_POWER_DEBUG
  27 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  28                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  29 } while (0)
  30 #else
  31 #define POWER_DEBUG_TRACE(fmt, args...)
  32 #endif
  33
  34 #define FOPEN_OR_ERR_RET(f, retval) do { \
  35                 if ((f) == NULL) { \
  36                         RTE_LOG(ERR, POWER, "File not opened\n"); \
  37                         return retval; \
  38                 } \
  39 } while (0)
  40
  41 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  42                 if ((ret) == NULL) { \
  43                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  44                         goto label; \
  45                 } \
  46 } while (0)
  47
  48 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  49                 if ((ret) < 0) { \
  50                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  51                         goto label; \
  52                 } \
  53 } while (0)
  54
  55
  56 #define POWER_CONVERT_TO_DECIMAL 10
  57 #define BUS_FREQ     100000
  58
  59 #define POWER_GOVERNOR_PERF "performance"
  60 #define POWER_SYSFILE_GOVERNOR  \
  61                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  62 #define POWER_SYSFILE_MAX_FREQ \
  63                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  64 #define POWER_SYSFILE_MIN_FREQ  \
  65                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  66 #define POWER_SYSFILE_CUR_FREQ  \
  67                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  68 #define POWER_SYSFILE_BASE_MAX_FREQ \
  69                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  70 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  71                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  72 #define POWER_SYSFILE_BASE_FREQ  \
  73                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  74 #define POWER_PSTATE_DRIVER "intel_pstate"
  75 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  76
  77 /*
  78  * MSR related
  79  */
  80 #define PLATFORM_INFO     0x0CE
  81 #define NON_TURBO_MASK    0xFF00
  82 #define NON_TURBO_OFFSET  0x8
  83
  84
  85 enum power_state {
  86         POWER_IDLE = 0,
  87         POWER_ONGOING,
  88         POWER_USED,
  89         POWER_UNKNOWN
  90 };
  91
  92 struct pstate_power_info {
  93         unsigned int lcore_id;               /**< Logical core id */
  94         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  95         uint32_t nb_freqs;                   /**< number of available freqs */
  96         FILE *f_cur_min;                     /**< FD of scaling_min */
  97         FILE *f_cur_max;                     /**< FD of scaling_max */
  98         char governor_ori[32];               /**< Original governor name */
  99         uint32_t curr_idx;                   /**< Freq index in freqs array */
 100         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 101         uint32_t sys_max_freq;               /**< system wide max freq  */
 102         uint32_t core_base_freq;             /**< core base freq  */
 103         volatile uint32_t state;             /**< Power in use state */
 104         uint16_t turbo_available;            /**< Turbo Boost available */
 105         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 106         uint16_t priority_core;              /**< High Performance core */
 107 } __rte_cache_aligned;
 108
 109
 110 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 111
 112 /**
 113  * It is to read the specific MSR.
 114  */
 115
 116 static int32_t
 117 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 118 {
 119         int fd, ret;
 120         char fullpath[PATH_MAX];
 121
 122         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 123
 124         fd = open(fullpath, O_RDONLY);
 125
 126         if (fd < 0) {
 127                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 128                                  strerror(errno));
 129                 return fd;
 130         }
 131
 132         ret = pread(fd, val, sizeof(uint64_t), msr);
 133
 134         if (ret < 0) {
 135                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 136                                  strerror(errno));
 137                 goto out;
 138         }
 139
 140         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 141                         fullpath, msr, lcore_id);
 142
 143         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 144
 145 out:    close(fd);
 146         return ret;
 147 }
 148
 149 /**
 150  * It is to fopen the sys file for the future setting the lcore frequency.
 151  */
 152 static int
 153 power_init_for_setting_freq(struct pstate_power_info *pi)
 154 {
 155         FILE *f_min, *f_max, *f_base;
 156         char fullpath_min[PATH_MAX];
 157         char fullpath_max[PATH_MAX];
 158         char fullpath_base[PATH_MAX];
 159         char buf_base[BUFSIZ];
 160         char *s_base;
 161         uint32_t base_ratio = 0;
 162         uint64_t max_non_turbo = 0;
 163         int  ret_val = 0;
 164
 165         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 166                         pi->lcore_id);
 167
 168         f_min = fopen(fullpath_min, "rw+");
 169         FOPEN_OR_ERR_RET(f_min, -1);
 170
 171         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 172                         pi->lcore_id);
 173
 174         f_max = fopen(fullpath_max, "rw+");
 175         if (f_max == NULL)
 176                 fclose(f_min);
 177
 178         FOPEN_OR_ERR_RET(f_max, -1);
 179
 180         pi->f_cur_min = f_min;
 181         pi->f_cur_max = f_max;
 182
 183         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 184                         pi->lcore_id);
 185
 186         f_base = fopen(fullpath_base, "r");
 187         if (f_base == NULL) {
 188                 /* No sysfs base_frequency, that's OK, continue without */
 189                 base_ratio = 0;
 190         } else {
 191                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 192                 FOPS_OR_NULL_GOTO(s_base, out);
 193
 194                 buf_base[BUFSIZ-1] = '\0';
 195                 if (strlen(buf_base))
 196                         /* Strip off terminating '\n' */
 197                         strtok(buf_base, "\n");
 198
 199                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 200                                 / BUS_FREQ;
 201         }
 202
 203         /* Add MSR read to detect turbo status */
 204
 205         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 206                 ret_val = -1;
 207                 goto out;
 208         }
 209
 210         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 211
 212         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 213
 214         pi->non_turbo_max_ratio = max_non_turbo;
 215
 216         /*
 217          * If base_frequency is reported as greater than the maximum
 218          * non-turbo frequency, then mark it as a high priority core.
 219          */
 220         if (base_ratio > max_non_turbo)
 221                 pi->priority_core = 1;
 222         else
 223                 pi->priority_core = 0;
 224         pi->core_base_freq = base_ratio * BUS_FREQ;
 225
 226 out:
 227         if (f_base != NULL)
 228                 fclose(f_base);
 229         return ret_val;
 230 }
 231
 232 static int
 233 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 234 {
 235         uint32_t target_freq = 0;
 236
 237         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 238                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 239                                 "should be less than %u\n", idx, pi->nb_freqs);
 240                 return -1;
 241         }
 242
 243         /* Check if it is the same as current */
 244         if (idx == pi->curr_idx)
 245                 return 0;
 246
 247         /* Because Intel Pstate Driver only allow user change min/max hint
 248          * User need change the min/max as same value.
 249          */
 250         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 251                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 252                                 "for setting frequency for lcore %u\n",
 253                                 pi->lcore_id);
 254                 return -1;
 255         }
 256
 257         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 258                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 259                                 "for setting frequency for lcore %u\n",
 260                                 pi->lcore_id);
 261                 return -1;
 262         }
 263
 264         /* Turbo is available and enabled, first freq bucket is sys max freq */
 265         if (pi->turbo_available && idx == 0) {
 266                 if (pi->turbo_enable)
 267                         target_freq = pi->sys_max_freq;
 268                 else {
 269                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 270                                         pi->lcore_id);
 271                         return -1;
 272                 }
 273         } else
 274                 target_freq = pi->freqs[idx];
 275
 276         /* Decrease freq, the min freq should be updated first */
 277         if (idx  >  pi->curr_idx) {
 278
 279                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 280                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 281                                         "lcore %u\n", pi->lcore_id);
 282                         return -1;
 283                 }
 284
 285                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 286                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 287                                         "lcore %u\n", pi->lcore_id);
 288                         return -1;
 289                 }
 290
 291                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 292                                   target_freq, pi->lcore_id);
 293
 294                 fflush(pi->f_cur_min);
 295                 fflush(pi->f_cur_max);
 296
 297         }
 298
 299         /* Increase freq, the max freq should be updated first */
 300         if (idx  <  pi->curr_idx) {
 301
 302                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 303                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 304                                         "lcore %u\n", pi->lcore_id);
 305                         return -1;
 306                 }
 307
 308                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 309                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 310                                         "lcore %u\n", pi->lcore_id);
 311                         return -1;
 312                 }
 313
 314                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 315                                   target_freq, pi->lcore_id);
 316
 317                 fflush(pi->f_cur_max);
 318                 fflush(pi->f_cur_min);
 319         }
 320
 321         pi->curr_idx = idx;
 322
 323         return 1;
 324 }
 325
 326 /**
 327  * It is to check the current scaling governor by reading sys file, and then
 328  * set it into 'performance' if it is not by writing the sys file. The original
 329  * governor will be saved for rolling back.
 330  */
 331 static int
 332 power_set_governor_performance(struct pstate_power_info *pi)
 333 {
 334         FILE *f;
 335         int ret = -1;
 336         char buf[BUFSIZ];
 337         char fullpath[PATH_MAX];
 338         char *s;
 339         int val;
 340
 341         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 342                         pi->lcore_id);
 343         f = fopen(fullpath, "rw+");
 344         FOPEN_OR_ERR_RET(f, ret);
 345
 346         s = fgets(buf, sizeof(buf), f);
 347         FOPS_OR_NULL_GOTO(s, out);
 348         /* Strip off terminating '\n' */
 349         strtok(buf, "\n");
 350
 351         /* Check if current governor is performance */
 352         if (strncmp(buf, POWER_GOVERNOR_PERF,
 353                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 354                 ret = 0;
 355                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 356                                 "already performance\n", pi->lcore_id);
 357                 goto out;
 358         }
 359         /* Save the original governor */
 360         strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 361
 362         /* Write 'performance' to the governor */
 363         val = fseek(f, 0, SEEK_SET);
 364         FOPS_OR_ERR_GOTO(val, out);
 365
 366         val = fputs(POWER_GOVERNOR_PERF, f);
 367         FOPS_OR_ERR_GOTO(val, out);
 368
 369         /* We need to flush to see if the fputs succeeds */
 370         val = fflush(f);
 371         FOPS_OR_ERR_GOTO(val, out);
 372
 373         ret = 0;
 374         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 375                         "set to performance successfully\n", pi->lcore_id);
 376 out:
 377         fclose(f);
 378
 379         return ret;
 380 }
 381
 382 /**
 383  * It is to check the governor and then set the original governor back if
 384  * needed by writing the sys file.
 385  */
 386 static int
 387 power_set_governor_original(struct pstate_power_info *pi)
 388 {
 389         FILE *f;
 390         int ret = -1;
 391         char buf[BUFSIZ];
 392         char fullpath[PATH_MAX];
 393         char *s;
 394         int val;
 395
 396         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 397                         pi->lcore_id);
 398         f = fopen(fullpath, "rw+");
 399         FOPEN_OR_ERR_RET(f, ret);
 400
 401         s = fgets(buf, sizeof(buf), f);
 402         FOPS_OR_NULL_GOTO(s, out);
 403
 404         /* Check if the governor to be set is the same as current */
 405         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 406                 ret = 0;
 407                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 408                                 "has already been set to %s\n",
 409                                 pi->lcore_id, pi->governor_ori);
 410                 goto out;
 411         }
 412
 413         /* Write back the original governor */
 414         val = fseek(f, 0, SEEK_SET);
 415         FOPS_OR_ERR_GOTO(val, out);
 416
 417         val = fputs(pi->governor_ori, f);
 418         FOPS_OR_ERR_GOTO(val, out);
 419
 420         ret = 0;
 421         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 422                         "has been set back to %s successfully\n",
 423                         pi->lcore_id, pi->governor_ori);
 424 out:
 425         fclose(f);
 426
 427         return ret;
 428 }
 429
 430 /**
 431  * It is to get the available frequencies of the specific lcore by reading the
 432  * sys file.
 433  */
 434 static int
 435 power_get_available_freqs(struct pstate_power_info *pi)
 436 {
 437         FILE *f_min, *f_max;
 438         int ret = -1;
 439         char *p_min, *p_max;
 440         char buf_min[BUFSIZ];
 441         char buf_max[BUFSIZ];
 442         char fullpath_min[PATH_MAX];
 443         char fullpath_max[PATH_MAX];
 444         char *s_min, *s_max;
 445         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 446         uint32_t i, num_freqs = 0;
 447
 448         snprintf(fullpath_max, sizeof(fullpath_max),
 449                         POWER_SYSFILE_BASE_MAX_FREQ,
 450                         pi->lcore_id);
 451         snprintf(fullpath_min, sizeof(fullpath_min),
 452                         POWER_SYSFILE_BASE_MIN_FREQ,
 453                         pi->lcore_id);
 454
 455         f_min = fopen(fullpath_min, "r");
 456         FOPEN_OR_ERR_RET(f_min, ret);
 457
 458         f_max = fopen(fullpath_max, "r");
 459         if (f_max == NULL)
 460                 fclose(f_min);
 461
 462         FOPEN_OR_ERR_RET(f_max, ret);
 463
 464         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 465         FOPS_OR_NULL_GOTO(s_min, out);
 466
 467         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 468         FOPS_OR_NULL_GOTO(s_max, out);
 469
 470
 471         /* Strip the line break if there is */
 472         p_min = strchr(buf_min, '\n');
 473         if (p_min != NULL)
 474                 *p_min = 0;
 475
 476         p_max = strchr(buf_max, '\n');
 477         if (p_max != NULL)
 478                 *p_max = 0;
 479
 480         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 481         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 482
 483         if (sys_max_freq < sys_min_freq)
 484                 goto out;
 485
 486         pi->sys_max_freq = sys_max_freq;
 487
 488         if (pi->priority_core == 1)
 489                 base_max_freq = pi->core_base_freq;
 490         else
 491                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 492
 493         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 494                         sys_min_freq,
 495                         sys_max_freq,
 496                         base_max_freq);
 497
 498         if (base_max_freq < sys_max_freq)
 499                 pi->turbo_available = 1;
 500         else
 501                 pi->turbo_available = 0;
 502
 503         /* If turbo is available then there is one extra freq bucket
 504          * to store the sys max freq which value is base_max +1
 505          */
 506         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 507                 pi->turbo_available;
 508
 509         /* Generate the freq bucket array.
 510          * If turbo is available the freq bucket[0] value is base_max +1
 511          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 512          * and so on.
 513          * If turbo is not available bucket[0] is base_max and so on
 514          */
 515         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 516                 if ((i == 0) && pi->turbo_available)
 517                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 518                 else
 519                         pi->freqs[pi->nb_freqs++] =
 520                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 521         }
 522
 523         ret = 0;
 524
 525         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 526                         num_freqs, pi->lcore_id);
 527
 528 out:
 529         fclose(f_min);
 530         fclose(f_max);
 531
 532         return ret;
 533 }
 534
 535 int
 536 power_pstate_cpufreq_check_supported(void)
 537 {
 538         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 539 }
 540
 541 int
 542 power_pstate_cpufreq_init(unsigned int lcore_id)
 543 {
 544         struct pstate_power_info *pi;
 545
 546         if (lcore_id >= RTE_MAX_LCORE) {
 547                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 548                                 lcore_id, RTE_MAX_LCORE - 1U);
 549                 return -1;
 550         }
 551
 552         pi = &lcore_power_info[lcore_id];
 553         if (rte_atomic32_cmpset(&(pi->state), POWER_IDLE, POWER_ONGOING)
 554                         == 0) {
 555                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 556                                 "in use\n", lcore_id);
 557                 return -1;
 558         }
 559
 560         pi->lcore_id = lcore_id;
 561         /* Check and set the governor */
 562         if (power_set_governor_performance(pi) < 0) {
 563                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 564                                 "performance\n", lcore_id);
 565                 goto fail;
 566         }
 567         /* Init for setting lcore frequency */
 568         if (power_init_for_setting_freq(pi) < 0) {
 569                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 570                                 "lcore %u\n", lcore_id);
 571                 goto fail;
 572         }
 573
 574         /* Get the available frequencies */
 575         if (power_get_available_freqs(pi) < 0) {
 576                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 577                                 "lcore %u\n", lcore_id);
 578                 goto fail;
 579         }
 580
 581
 582         /* Set freq to max by default */
 583         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 584                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 585                                 "to max\n", lcore_id);
 586                 goto fail;
 587         }
 588
 589         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 590                         "power management\n", lcore_id);
 591         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_USED);
 592
 593         return 0;
 594
 595 fail:
 596         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
 597
 598         return -1;
 599 }
 600
 601 int
 602 power_pstate_cpufreq_exit(unsigned int lcore_id)
 603 {
 604         struct pstate_power_info *pi;
 605
 606         if (lcore_id >= RTE_MAX_LCORE) {
 607                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 608                                 lcore_id, RTE_MAX_LCORE - 1U);
 609                 return -1;
 610         }
 611         pi = &lcore_power_info[lcore_id];
 612
 613         if (rte_atomic32_cmpset(&(pi->state), POWER_USED, POWER_ONGOING)
 614                         == 0) {
 615                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 616                                 "not used\n", lcore_id);
 617                 return -1;
 618         }
 619
 620         /* Close FD of setting freq */
 621         fclose(pi->f_cur_min);
 622         fclose(pi->f_cur_max);
 623         pi->f_cur_min = NULL;
 624         pi->f_cur_max = NULL;
 625
 626         /* Set the governor back to the original */
 627         if (power_set_governor_original(pi) < 0) {
 628                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 629                                 "to the original\n", lcore_id);
 630                 goto fail;
 631         }
 632
 633         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 634                         "'performance' mode and been set back to the "
 635                         "original\n", lcore_id);
 636         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_IDLE);
 637
 638         return 0;
 639
 640 fail:
 641         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
 642
 643         return -1;
 644 }
 645
 646
 647 uint32_t
 648 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 649 {
 650         struct pstate_power_info *pi;
 651
 652         if (lcore_id >= RTE_MAX_LCORE) {
 653                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 654                 return 0;
 655         }
 656
 657         if (freqs == NULL) {
 658                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 659                 return 0;
 660         }
 661
 662         pi = &lcore_power_info[lcore_id];
 663         if (num < pi->nb_freqs) {
 664                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 665                 return 0;
 666         }
 667         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 668
 669         return pi->nb_freqs;
 670 }
 671
 672 uint32_t
 673 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 674 {
 675         if (lcore_id >= RTE_MAX_LCORE) {
 676                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 677                 return RTE_POWER_INVALID_FREQ_INDEX;
 678         }
 679
 680         return lcore_power_info[lcore_id].curr_idx;
 681 }
 682
 683
 684 int
 685 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 686 {
 687         if (lcore_id >= RTE_MAX_LCORE) {
 688                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 689                 return -1;
 690         }
 691
 692         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 693 }
 694
 695 int
 696 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 697 {
 698         struct pstate_power_info *pi;
 699
 700         if (lcore_id >= RTE_MAX_LCORE) {
 701                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 702                 return -1;
 703         }
 704
 705         pi = &lcore_power_info[lcore_id];
 706         if (pi->curr_idx == 0 ||
 707             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 708                 return 0;
 709
 710         /* Frequencies in the array are from high to low. */
 711         return set_freq_internal(pi, pi->curr_idx - 1);
 712 }
 713
 714 int
 715 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 716 {
 717         struct pstate_power_info *pi;
 718
 719         if (lcore_id >= RTE_MAX_LCORE) {
 720                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 721                 return -1;
 722         }
 723
 724         pi = &lcore_power_info[lcore_id];
 725         if (pi->curr_idx + 1 == pi->nb_freqs)
 726                 return 0;
 727
 728         /* Frequencies in the array are from high to low. */
 729         return set_freq_internal(pi, pi->curr_idx + 1);
 730 }
 731
 732 int
 733 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 734 {
 735         if (lcore_id >= RTE_MAX_LCORE) {
 736                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 737                 return -1;
 738         }
 739
 740         /* Frequencies in the array are from high to low. */
 741         if (lcore_power_info[lcore_id].turbo_available) {
 742                 if (lcore_power_info[lcore_id].turbo_enable)
 743                         /* Set to Turbo */
 744                         return set_freq_internal(
 745                                         &lcore_power_info[lcore_id], 0);
 746                 else
 747                         /* Set to max non-turbo */
 748                         return set_freq_internal(
 749                                         &lcore_power_info[lcore_id], 1);
 750         } else
 751                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 752 }
 753
 754
 755 int
 756 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 757 {
 758         struct pstate_power_info *pi;
 759
 760         if (lcore_id >= RTE_MAX_LCORE) {
 761                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 762                 return -1;
 763         }
 764
 765         pi = &lcore_power_info[lcore_id];
 766
 767         /* Frequencies in the array are from high to low. */
 768         return set_freq_internal(pi, pi->nb_freqs - 1);
 769 }
 770
 771
 772 int
 773 power_pstate_turbo_status(unsigned int lcore_id)
 774 {
 775         struct pstate_power_info *pi;
 776
 777         if (lcore_id >= RTE_MAX_LCORE) {
 778                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 779                 return -1;
 780         }
 781
 782         pi = &lcore_power_info[lcore_id];
 783
 784         return pi->turbo_enable;
 785 }
 786
 787 int
 788 power_pstate_enable_turbo(unsigned int lcore_id)
 789 {
 790         struct pstate_power_info *pi;
 791
 792         if (lcore_id >= RTE_MAX_LCORE) {
 793                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 794                 return -1;
 795         }
 796
 797         pi = &lcore_power_info[lcore_id];
 798
 799         if (pi->turbo_available)
 800                 pi->turbo_enable = 1;
 801         else {
 802                 pi->turbo_enable = 0;
 803                 RTE_LOG(ERR, POWER,
 804                         "Failed to enable turbo on lcore %u\n",
 805                         lcore_id);
 806                         return -1;
 807         }
 808
 809         return 0;
 810 }
 811
 812
 813 int
 814 power_pstate_disable_turbo(unsigned int lcore_id)
 815 {
 816         struct pstate_power_info *pi;
 817
 818         if (lcore_id >= RTE_MAX_LCORE) {
 819                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 820                 return -1;
 821         }
 822
 823         pi = &lcore_power_info[lcore_id];
 824
 825         pi->turbo_enable = 0;
 826
 827         if (pi->turbo_available && pi->curr_idx <= 1) {
 828                 /* Try to set freq to max by default coming out of turbo */
 829                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 830                         RTE_LOG(ERR, POWER,
 831                                 "Failed to set frequency of lcore %u to max\n",
 832                                 lcore_id);
 833                         return -1;
 834                 }
 835         }
 836
 837         return 0;
 838 }
 839
 840
 841 int power_pstate_get_capabilities(unsigned int lcore_id,
 842                 struct rte_power_core_capabilities *caps)
 843 {
 844         struct pstate_power_info *pi;
 845
 846         if (lcore_id >= RTE_MAX_LCORE) {
 847                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 848                 return -1;
 849         }
 850         if (caps == NULL) {
 851                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 852                 return -1;
 853         }
 854
 855         pi = &lcore_power_info[lcore_id];
 856         caps->capabilities = 0;
 857         caps->turbo = !!(pi->turbo_available);
 858         caps->priority = pi->priority_core;
 859
 860         return 0;
 861 }