lib/power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24
  25 #ifdef RTE_LIBRTE_POWER_DEBUG
  26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  27                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  28 } while (0)
  29 #else
  30 #define POWER_DEBUG_TRACE(fmt, args...)
  31 #endif
  32
  33 #define FOPEN_OR_ERR_RET(f, retval) do { \
  34                 if ((f) == NULL) { \
  35                         RTE_LOG(ERR, POWER, "File not opened\n"); \
  36                         return retval; \
  37                 } \
  38 } while (0)
  39
  40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  41                 if ((ret) == NULL) { \
  42                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  43                         goto label; \
  44                 } \
  45 } while (0)
  46
  47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  48                 if ((ret) < 0) { \
  49                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  50                         goto label; \
  51                 } \
  52 } while (0)
  53
  54 /* macros used for rounding frequency to nearest 100000 */
  55 #define FREQ_ROUNDING_DELTA 50000
  56 #define ROUND_FREQ_TO_N_100000 100000
  57
  58 #define POWER_CONVERT_TO_DECIMAL 10
  59 #define BUS_FREQ     100000
  60
  61 #define POWER_GOVERNOR_PERF "performance"
  62 #define POWER_SYSFILE_GOVERNOR  \
  63                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  64 #define POWER_SYSFILE_MAX_FREQ \
  65                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  66 #define POWER_SYSFILE_MIN_FREQ  \
  67                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  68 #define POWER_SYSFILE_CUR_FREQ  \
  69                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  70 #define POWER_SYSFILE_BASE_MAX_FREQ \
  71                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  72 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  73                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  74 #define POWER_SYSFILE_BASE_FREQ  \
  75                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  76 #define POWER_PSTATE_DRIVER "intel_pstate"
  77 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  78
  79 /*
  80  * MSR related
  81  */
  82 #define PLATFORM_INFO     0x0CE
  83 #define NON_TURBO_MASK    0xFF00
  84 #define NON_TURBO_OFFSET  0x8
  85
  86
  87 enum power_state {
  88         POWER_IDLE = 0,
  89         POWER_ONGOING,
  90         POWER_USED,
  91         POWER_UNKNOWN
  92 };
  93
  94 struct pstate_power_info {
  95         unsigned int lcore_id;               /**< Logical core id */
  96         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  97         uint32_t nb_freqs;                   /**< number of available freqs */
  98         FILE *f_cur_min;                     /**< FD of scaling_min */
  99         FILE *f_cur_max;                     /**< FD of scaling_max */
 100         char governor_ori[32];               /**< Original governor name */
 101         uint32_t curr_idx;                   /**< Freq index in freqs array */
 102         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 103         uint32_t sys_max_freq;               /**< system wide max freq  */
 104         uint32_t core_base_freq;             /**< core base freq  */
 105         uint32_t state;                      /**< Power in use state */
 106         uint16_t turbo_available;            /**< Turbo Boost available */
 107         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 108         uint16_t priority_core;              /**< High Performance core */
 109 } __rte_cache_aligned;
 110
 111
 112 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 113
 114 /**
 115  * It is to read the specific MSR.
 116  */
 117
 118 static int32_t
 119 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 120 {
 121         int fd, ret;
 122         char fullpath[PATH_MAX];
 123
 124         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 125
 126         fd = open(fullpath, O_RDONLY);
 127
 128         if (fd < 0) {
 129                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 130                                  strerror(errno));
 131                 return fd;
 132         }
 133
 134         ret = pread(fd, val, sizeof(uint64_t), msr);
 135
 136         if (ret < 0) {
 137                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 138                                  strerror(errno));
 139                 goto out;
 140         }
 141
 142         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 143                         fullpath, msr, lcore_id);
 144
 145         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 146
 147 out:    close(fd);
 148         return ret;
 149 }
 150
 151 /**
 152  * It is to fopen the sys file for the future setting the lcore frequency.
 153  */
 154 static int
 155 power_init_for_setting_freq(struct pstate_power_info *pi)
 156 {
 157         FILE *f_min, *f_max, *f_base = NULL, *f_base_max;
 158         char fullpath_min[PATH_MAX];
 159         char fullpath_max[PATH_MAX];
 160         char fullpath_base[PATH_MAX];
 161         char fullpath_base_max[PATH_MAX];
 162         char buf_base[BUFSIZ];
 163         char *s_base;
 164         char *s_base_max;
 165         uint32_t base_ratio = 0;
 166         uint32_t base_max_ratio = 0;
 167         uint64_t max_non_turbo = 0;
 168         int  ret_val = 0;
 169
 170         snprintf(fullpath_base_max,
 171                         sizeof(fullpath_base_max),
 172                         POWER_SYSFILE_BASE_MAX_FREQ,
 173                         pi->lcore_id);
 174         f_base_max = fopen(fullpath_base_max, "r");
 175         FOPEN_OR_ERR_RET(f_base_max, -1);
 176         if (f_base_max != NULL) {
 177                 s_base_max = fgets(buf_base, sizeof(buf_base), f_base_max);
 178
 179                 /* close the file unconditionally */
 180                 fclose(f_base_max);
 181                 f_base_max = NULL;
 182
 183                 FOPS_OR_NULL_GOTO(s_base_max, out);
 184
 185                 buf_base[BUFSIZ-1] = '\0';
 186                 if (strlen(buf_base))
 187                         /* Strip off terminating '\n' */
 188                         strtok(buf_base, "\n");
 189
 190                 base_max_ratio =
 191                         strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 192                                 / BUS_FREQ;
 193         }
 194
 195         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 196                         pi->lcore_id);
 197         f_min = fopen(fullpath_min, "rw+");
 198         FOPEN_OR_ERR_RET(f_min, -1);
 199
 200         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 201                         pi->lcore_id);
 202         f_max = fopen(fullpath_max, "rw+");
 203         if (f_max == NULL)
 204                 fclose(f_min);
 205         FOPEN_OR_ERR_RET(f_max, -1);
 206
 207         pi->f_cur_min = f_min;
 208         pi->f_cur_max = f_max;
 209
 210         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 211                         pi->lcore_id);
 212
 213         f_base = fopen(fullpath_base, "r");
 214         if (f_base == NULL) {
 215                 /* No sysfs base_frequency, that's OK, continue without */
 216                 base_ratio = 0;
 217         } else {
 218                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 219                 FOPS_OR_NULL_GOTO(s_base, out);
 220
 221                 buf_base[BUFSIZ-1] = '\0';
 222                 if (strlen(buf_base))
 223                         /* Strip off terminating '\n' */
 224                         strtok(buf_base, "\n");
 225
 226                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 227                                 / BUS_FREQ;
 228         }
 229
 230         /* Add MSR read to detect turbo status */
 231
 232         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 233                 ret_val = -1;
 234                 goto out;
 235         }
 236
 237         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 238
 239         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 240
 241         pi->non_turbo_max_ratio = max_non_turbo;
 242
 243         /*
 244          * If base_frequency is reported as greater than the maximum
 245          * turbo frequency, that's a known issue with some kernels.
 246          * Set base_frequency to max_non_turbo as a workaround.
 247          */
 248         if (base_ratio > base_max_ratio) {
 249                 /* base_ratio is greater than max turbo. Kernel bug. */
 250                 pi->priority_core = 0;
 251                 goto out;
 252         }
 253
 254         /*
 255          * If base_frequency is reported as greater than the maximum
 256          * non-turbo frequency, then mark it as a high priority core.
 257          */
 258         if (base_ratio > max_non_turbo)
 259                 pi->priority_core = 1;
 260         else
 261                 pi->priority_core = 0;
 262         pi->core_base_freq = base_ratio * BUS_FREQ;
 263
 264 out:
 265         if (f_base != NULL)
 266                 fclose(f_base);
 267         return ret_val;
 268 }
 269
 270 static int
 271 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 272 {
 273         uint32_t target_freq = 0;
 274
 275         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 276                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 277                                 "should be less than %u\n", idx, pi->nb_freqs);
 278                 return -1;
 279         }
 280
 281         /* Check if it is the same as current */
 282         if (idx == pi->curr_idx)
 283                 return 0;
 284
 285         /* Because Intel Pstate Driver only allow user change min/max hint
 286          * User need change the min/max as same value.
 287          */
 288         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 289                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 290                                 "for setting frequency for lcore %u\n",
 291                                 pi->lcore_id);
 292                 return -1;
 293         }
 294
 295         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 296                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 297                                 "for setting frequency for lcore %u\n",
 298                                 pi->lcore_id);
 299                 return -1;
 300         }
 301
 302         /* Turbo is available and enabled, first freq bucket is sys max freq */
 303         if (pi->turbo_available && idx == 0) {
 304                 if (pi->turbo_enable)
 305                         target_freq = pi->sys_max_freq;
 306                 else {
 307                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 308                                         pi->lcore_id);
 309                         return -1;
 310                 }
 311         } else
 312                 target_freq = pi->freqs[idx];
 313
 314         /* Decrease freq, the min freq should be updated first */
 315         if (idx  >  pi->curr_idx) {
 316
 317                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 318                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 319                                         "lcore %u\n", pi->lcore_id);
 320                         return -1;
 321                 }
 322
 323                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 324                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 325                                         "lcore %u\n", pi->lcore_id);
 326                         return -1;
 327                 }
 328
 329                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 330                                   target_freq, pi->lcore_id);
 331
 332                 fflush(pi->f_cur_min);
 333                 fflush(pi->f_cur_max);
 334
 335         }
 336
 337         /* Increase freq, the max freq should be updated first */
 338         if (idx  <  pi->curr_idx) {
 339
 340                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 341                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 342                                         "lcore %u\n", pi->lcore_id);
 343                         return -1;
 344                 }
 345
 346                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 347                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 348                                         "lcore %u\n", pi->lcore_id);
 349                         return -1;
 350                 }
 351
 352                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 353                                   target_freq, pi->lcore_id);
 354
 355                 fflush(pi->f_cur_max);
 356                 fflush(pi->f_cur_min);
 357         }
 358
 359         pi->curr_idx = idx;
 360
 361         return 1;
 362 }
 363
 364 /**
 365  * It is to check the current scaling governor by reading sys file, and then
 366  * set it into 'performance' if it is not by writing the sys file. The original
 367  * governor will be saved for rolling back.
 368  */
 369 static int
 370 power_set_governor_performance(struct pstate_power_info *pi)
 371 {
 372         FILE *f;
 373         int ret = -1;
 374         char buf[BUFSIZ];
 375         char fullpath[PATH_MAX];
 376         char *s;
 377         int val;
 378
 379         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 380                         pi->lcore_id);
 381         f = fopen(fullpath, "rw+");
 382         FOPEN_OR_ERR_RET(f, ret);
 383
 384         s = fgets(buf, sizeof(buf), f);
 385         FOPS_OR_NULL_GOTO(s, out);
 386         /* Strip off terminating '\n' */
 387         strtok(buf, "\n");
 388
 389         /* Save the original governor */
 390         rte_strscpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 391
 392         /* Check if current governor is performance */
 393         if (strncmp(buf, POWER_GOVERNOR_PERF,
 394                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 395                 ret = 0;
 396                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 397                                 "already performance\n", pi->lcore_id);
 398                 goto out;
 399         }
 400
 401         /* Write 'performance' to the governor */
 402         val = fseek(f, 0, SEEK_SET);
 403         FOPS_OR_ERR_GOTO(val, out);
 404
 405         val = fputs(POWER_GOVERNOR_PERF, f);
 406         FOPS_OR_ERR_GOTO(val, out);
 407
 408         /* We need to flush to see if the fputs succeeds */
 409         val = fflush(f);
 410         FOPS_OR_ERR_GOTO(val, out);
 411
 412         ret = 0;
 413         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 414                         "set to performance successfully\n", pi->lcore_id);
 415 out:
 416         fclose(f);
 417
 418         return ret;
 419 }
 420
 421 /**
 422  * It is to check the governor and then set the original governor back if
 423  * needed by writing the sys file.
 424  */
 425 static int
 426 power_set_governor_original(struct pstate_power_info *pi)
 427 {
 428         FILE *f;
 429         int ret = -1;
 430         char buf[BUFSIZ];
 431         char fullpath[PATH_MAX];
 432         char *s;
 433         int val;
 434
 435         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 436                         pi->lcore_id);
 437         f = fopen(fullpath, "rw+");
 438         FOPEN_OR_ERR_RET(f, ret);
 439
 440         s = fgets(buf, sizeof(buf), f);
 441         FOPS_OR_NULL_GOTO(s, out);
 442
 443         /* Check if the governor to be set is the same as current */
 444         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 445                 ret = 0;
 446                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 447                                 "has already been set to %s\n",
 448                                 pi->lcore_id, pi->governor_ori);
 449                 goto out;
 450         }
 451
 452         /* Write back the original governor */
 453         val = fseek(f, 0, SEEK_SET);
 454         FOPS_OR_ERR_GOTO(val, out);
 455
 456         val = fputs(pi->governor_ori, f);
 457         FOPS_OR_ERR_GOTO(val, out);
 458
 459         ret = 0;
 460         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 461                         "has been set back to %s successfully\n",
 462                         pi->lcore_id, pi->governor_ori);
 463 out:
 464         fclose(f);
 465
 466         return ret;
 467 }
 468
 469 /**
 470  * It is to get the available frequencies of the specific lcore by reading the
 471  * sys file.
 472  */
 473 static int
 474 power_get_available_freqs(struct pstate_power_info *pi)
 475 {
 476         FILE *f_min, *f_max;
 477         int ret = -1;
 478         char *p_min, *p_max;
 479         char buf_min[BUFSIZ];
 480         char buf_max[BUFSIZ];
 481         char fullpath_min[PATH_MAX];
 482         char fullpath_max[PATH_MAX];
 483         char *s_min, *s_max;
 484         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 485         uint32_t i, num_freqs = 0;
 486
 487         snprintf(fullpath_max, sizeof(fullpath_max),
 488                         POWER_SYSFILE_BASE_MAX_FREQ,
 489                         pi->lcore_id);
 490         snprintf(fullpath_min, sizeof(fullpath_min),
 491                         POWER_SYSFILE_BASE_MIN_FREQ,
 492                         pi->lcore_id);
 493
 494         f_min = fopen(fullpath_min, "r");
 495         FOPEN_OR_ERR_RET(f_min, ret);
 496
 497         f_max = fopen(fullpath_max, "r");
 498         if (f_max == NULL)
 499                 fclose(f_min);
 500
 501         FOPEN_OR_ERR_RET(f_max, ret);
 502
 503         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 504         FOPS_OR_NULL_GOTO(s_min, out);
 505
 506         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 507         FOPS_OR_NULL_GOTO(s_max, out);
 508
 509
 510         /* Strip the line break if there is */
 511         p_min = strchr(buf_min, '\n');
 512         if (p_min != NULL)
 513                 *p_min = 0;
 514
 515         p_max = strchr(buf_max, '\n');
 516         if (p_max != NULL)
 517                 *p_max = 0;
 518
 519         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 520         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 521
 522         if (sys_max_freq < sys_min_freq)
 523                 goto out;
 524
 525         pi->sys_max_freq = sys_max_freq;
 526
 527         if (pi->priority_core == 1)
 528                 base_max_freq = pi->core_base_freq;
 529         else
 530                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 531
 532         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 533                         sys_min_freq,
 534                         sys_max_freq,
 535                         base_max_freq);
 536
 537         if (base_max_freq < sys_max_freq)
 538                 pi->turbo_available = 1;
 539         else
 540                 pi->turbo_available = 0;
 541
 542         /* If turbo is available then there is one extra freq bucket
 543          * to store the sys max freq which value is base_max +1
 544          */
 545         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 546                 pi->turbo_available;
 547
 548         /* Generate the freq bucket array.
 549          * If turbo is available the freq bucket[0] value is base_max +1
 550          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 551          * and so on.
 552          * If turbo is not available bucket[0] is base_max and so on
 553          */
 554         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 555                 if ((i == 0) && pi->turbo_available)
 556                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 557                 else
 558                         pi->freqs[pi->nb_freqs++] =
 559                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 560         }
 561
 562         ret = 0;
 563
 564         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 565                         num_freqs, pi->lcore_id);
 566
 567 out:
 568         fclose(f_min);
 569         fclose(f_max);
 570
 571         return ret;
 572 }
 573
 574 static int
 575 power_get_cur_idx(struct pstate_power_info *pi)
 576 {
 577         FILE *f_cur;
 578         int ret = -1;
 579         char *p_cur;
 580         char buf_cur[BUFSIZ];
 581         char fullpath_cur[PATH_MAX];
 582         char *s_cur;
 583         uint32_t sys_cur_freq = 0;
 584         unsigned int i;
 585
 586         snprintf(fullpath_cur, sizeof(fullpath_cur),
 587                         POWER_SYSFILE_CUR_FREQ,
 588                         pi->lcore_id);
 589         f_cur = fopen(fullpath_cur, "r");
 590         FOPEN_OR_ERR_RET(f_cur, ret);
 591
 592         /* initialize the cur_idx to matching current frequency freq index */
 593         s_cur = fgets(buf_cur, sizeof(buf_cur), f_cur);
 594         FOPS_OR_NULL_GOTO(s_cur, fail);
 595
 596         p_cur = strchr(buf_cur, '\n');
 597         if (p_cur != NULL)
 598                 *p_cur = 0;
 599         sys_cur_freq = strtoul(buf_cur, &p_cur, POWER_CONVERT_TO_DECIMAL);
 600
 601         /* convert the frequency to nearest 100000 value
 602          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 603          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 604          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 605          */
 606         unsigned int freq_conv = 0;
 607         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 608                                 / ROUND_FREQ_TO_N_100000;
 609         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 610
 611         for (i = 0; i < pi->nb_freqs; i++) {
 612                 if (freq_conv == pi->freqs[i]) {
 613                         pi->curr_idx = i;
 614                         break;
 615                 }
 616         }
 617
 618         fclose(f_cur);
 619         return 0;
 620 fail:
 621         fclose(f_cur);
 622         return ret;
 623 }
 624
 625 int
 626 power_pstate_cpufreq_check_supported(void)
 627 {
 628         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 629 }
 630
 631 int
 632 power_pstate_cpufreq_init(unsigned int lcore_id)
 633 {
 634         struct pstate_power_info *pi;
 635         uint32_t exp_state;
 636
 637         if (lcore_id >= RTE_MAX_LCORE) {
 638                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 639                                 lcore_id, RTE_MAX_LCORE - 1U);
 640                 return -1;
 641         }
 642
 643         pi = &lcore_power_info[lcore_id];
 644         exp_state = POWER_IDLE;
 645         /* The power in use state works as a guard variable between
 646          * the CPU frequency control initialization and exit process.
 647          * The ACQUIRE memory ordering here pairs with the RELEASE
 648          * ordering below as lock to make sure the frequency operations
 649          * in the critical section are done under the correct state.
 650          */
 651         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 652                                         POWER_ONGOING, 0,
 653                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 654                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 655                                 "in use\n", lcore_id);
 656                 return -1;
 657         }
 658
 659         pi->lcore_id = lcore_id;
 660         /* Check and set the governor */
 661         if (power_set_governor_performance(pi) < 0) {
 662                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 663                                 "performance\n", lcore_id);
 664                 goto fail;
 665         }
 666         /* Init for setting lcore frequency */
 667         if (power_init_for_setting_freq(pi) < 0) {
 668                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 669                                 "lcore %u\n", lcore_id);
 670                 goto fail;
 671         }
 672
 673         /* Get the available frequencies */
 674         if (power_get_available_freqs(pi) < 0) {
 675                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 676                                 "lcore %u\n", lcore_id);
 677                 goto fail;
 678         }
 679
 680         if (power_get_cur_idx(pi) < 0) {
 681                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 682                                 "index of lcore %u\n", lcore_id);
 683                 goto fail;
 684         }
 685
 686         /* Set freq to max by default */
 687         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 688                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 689                                 "to max\n", lcore_id);
 690                 goto fail;
 691         }
 692
 693         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 694                         "power management\n", lcore_id);
 695         exp_state = POWER_ONGOING;
 696         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 697                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 698
 699         return 0;
 700
 701 fail:
 702         exp_state = POWER_ONGOING;
 703         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 704                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 705
 706         return -1;
 707 }
 708
 709 int
 710 power_pstate_cpufreq_exit(unsigned int lcore_id)
 711 {
 712         struct pstate_power_info *pi;
 713         uint32_t exp_state;
 714
 715         if (lcore_id >= RTE_MAX_LCORE) {
 716                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 717                                 lcore_id, RTE_MAX_LCORE - 1U);
 718                 return -1;
 719         }
 720         pi = &lcore_power_info[lcore_id];
 721
 722         exp_state = POWER_USED;
 723         /* The power in use state works as a guard variable between
 724          * the CPU frequency control initialization and exit process.
 725          * The ACQUIRE memory ordering here pairs with the RELEASE
 726          * ordering below as lock to make sure the frequency operations
 727          * in the critical section are under done the correct state.
 728          */
 729         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 730                                         POWER_ONGOING, 0,
 731                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 732                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 733                                 "not used\n", lcore_id);
 734                 return -1;
 735         }
 736
 737         /* Close FD of setting freq */
 738         fclose(pi->f_cur_min);
 739         fclose(pi->f_cur_max);
 740         pi->f_cur_min = NULL;
 741         pi->f_cur_max = NULL;
 742
 743         /* Set the governor back to the original */
 744         if (power_set_governor_original(pi) < 0) {
 745                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 746                                 "to the original\n", lcore_id);
 747                 goto fail;
 748         }
 749
 750         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 751                         "'performance' mode and been set back to the "
 752                         "original\n", lcore_id);
 753         exp_state = POWER_ONGOING;
 754         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 755                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 756
 757         return 0;
 758
 759 fail:
 760         exp_state = POWER_ONGOING;
 761         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 762                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 763
 764         return -1;
 765 }
 766
 767
 768 uint32_t
 769 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 770 {
 771         struct pstate_power_info *pi;
 772
 773         if (lcore_id >= RTE_MAX_LCORE) {
 774                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 775                 return 0;
 776         }
 777
 778         if (freqs == NULL) {
 779                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 780                 return 0;
 781         }
 782
 783         pi = &lcore_power_info[lcore_id];
 784         if (num < pi->nb_freqs) {
 785                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 786                 return 0;
 787         }
 788         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 789
 790         return pi->nb_freqs;
 791 }
 792
 793 uint32_t
 794 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 795 {
 796         if (lcore_id >= RTE_MAX_LCORE) {
 797                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 798                 return RTE_POWER_INVALID_FREQ_INDEX;
 799         }
 800
 801         return lcore_power_info[lcore_id].curr_idx;
 802 }
 803
 804
 805 int
 806 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 807 {
 808         if (lcore_id >= RTE_MAX_LCORE) {
 809                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 810                 return -1;
 811         }
 812
 813         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 814 }
 815
 816 int
 817 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 818 {
 819         struct pstate_power_info *pi;
 820
 821         if (lcore_id >= RTE_MAX_LCORE) {
 822                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 823                 return -1;
 824         }
 825
 826         pi = &lcore_power_info[lcore_id];
 827         if (pi->curr_idx == 0 ||
 828             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 829                 return 0;
 830
 831         /* Frequencies in the array are from high to low. */
 832         return set_freq_internal(pi, pi->curr_idx - 1);
 833 }
 834
 835 int
 836 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 837 {
 838         struct pstate_power_info *pi;
 839
 840         if (lcore_id >= RTE_MAX_LCORE) {
 841                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 842                 return -1;
 843         }
 844
 845         pi = &lcore_power_info[lcore_id];
 846         if (pi->curr_idx + 1 == pi->nb_freqs)
 847                 return 0;
 848
 849         /* Frequencies in the array are from high to low. */
 850         return set_freq_internal(pi, pi->curr_idx + 1);
 851 }
 852
 853 int
 854 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 855 {
 856         if (lcore_id >= RTE_MAX_LCORE) {
 857                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 858                 return -1;
 859         }
 860
 861         /* Frequencies in the array are from high to low. */
 862         if (lcore_power_info[lcore_id].turbo_available) {
 863                 if (lcore_power_info[lcore_id].turbo_enable)
 864                         /* Set to Turbo */
 865                         return set_freq_internal(
 866                                         &lcore_power_info[lcore_id], 0);
 867                 else
 868                         /* Set to max non-turbo */
 869                         return set_freq_internal(
 870                                         &lcore_power_info[lcore_id], 1);
 871         } else
 872                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 873 }
 874
 875
 876 int
 877 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 878 {
 879         struct pstate_power_info *pi;
 880
 881         if (lcore_id >= RTE_MAX_LCORE) {
 882                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 883                 return -1;
 884         }
 885
 886         pi = &lcore_power_info[lcore_id];
 887
 888         /* Frequencies in the array are from high to low. */
 889         return set_freq_internal(pi, pi->nb_freqs - 1);
 890 }
 891
 892
 893 int
 894 power_pstate_turbo_status(unsigned int lcore_id)
 895 {
 896         struct pstate_power_info *pi;
 897
 898         if (lcore_id >= RTE_MAX_LCORE) {
 899                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 900                 return -1;
 901         }
 902
 903         pi = &lcore_power_info[lcore_id];
 904
 905         return pi->turbo_enable;
 906 }
 907
 908 int
 909 power_pstate_enable_turbo(unsigned int lcore_id)
 910 {
 911         struct pstate_power_info *pi;
 912
 913         if (lcore_id >= RTE_MAX_LCORE) {
 914                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 915                 return -1;
 916         }
 917
 918         pi = &lcore_power_info[lcore_id];
 919
 920         if (pi->turbo_available)
 921                 pi->turbo_enable = 1;
 922         else {
 923                 pi->turbo_enable = 0;
 924                 RTE_LOG(ERR, POWER,
 925                         "Failed to enable turbo on lcore %u\n",
 926                         lcore_id);
 927                         return -1;
 928         }
 929
 930         return 0;
 931 }
 932
 933
 934 int
 935 power_pstate_disable_turbo(unsigned int lcore_id)
 936 {
 937         struct pstate_power_info *pi;
 938
 939         if (lcore_id >= RTE_MAX_LCORE) {
 940                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 941                 return -1;
 942         }
 943
 944         pi = &lcore_power_info[lcore_id];
 945
 946         pi->turbo_enable = 0;
 947
 948         if (pi->turbo_available && pi->curr_idx <= 1) {
 949                 /* Try to set freq to max by default coming out of turbo */
 950                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 951                         RTE_LOG(ERR, POWER,
 952                                 "Failed to set frequency of lcore %u to max\n",
 953                                 lcore_id);
 954                         return -1;
 955                 }
 956         }
 957
 958         return 0;
 959 }
 960
 961
 962 int power_pstate_get_capabilities(unsigned int lcore_id,
 963                 struct rte_power_core_capabilities *caps)
 964 {
 965         struct pstate_power_info *pi;
 966
 967         if (lcore_id >= RTE_MAX_LCORE) {
 968                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 969                 return -1;
 970         }
 971         if (caps == NULL) {
 972                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 973                 return -1;
 974         }
 975
 976         pi = &lcore_power_info[lcore_id];
 977         caps->capabilities = 0;
 978         caps->turbo = !!(pi->turbo_available);
 979         caps->priority = pi->priority_core;
 980
 981         return 0;
 982 }