lib/librte_power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24
  25 #ifdef RTE_LIBRTE_POWER_DEBUG
  26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  27                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  28 } while (0)
  29 #else
  30 #define POWER_DEBUG_TRACE(fmt, args...)
  31 #endif
  32
  33 #define FOPEN_OR_ERR_RET(f, retval) do { \
  34                 if ((f) == NULL) { \
  35                         RTE_LOG(ERR, POWER, "File not opened\n"); \
  36                         return retval; \
  37                 } \
  38 } while (0)
  39
  40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  41                 if ((ret) == NULL) { \
  42                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  43                         goto label; \
  44                 } \
  45 } while (0)
  46
  47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  48                 if ((ret) < 0) { \
  49                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  50                         goto label; \
  51                 } \
  52 } while (0)
  53
  54 /* macros used for rounding frequency to nearest 100000 */
  55 #define FREQ_ROUNDING_DELTA 50000
  56 #define ROUND_FREQ_TO_N_100000 100000
  57
  58 #define POWER_CONVERT_TO_DECIMAL 10
  59 #define BUS_FREQ     100000
  60
  61 #define POWER_GOVERNOR_PERF "performance"
  62 #define POWER_SYSFILE_GOVERNOR  \
  63                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  64 #define POWER_SYSFILE_MAX_FREQ \
  65                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  66 #define POWER_SYSFILE_MIN_FREQ  \
  67                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  68 #define POWER_SYSFILE_CUR_FREQ  \
  69                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  70 #define POWER_SYSFILE_BASE_MAX_FREQ \
  71                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  72 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  73                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  74 #define POWER_SYSFILE_BASE_FREQ  \
  75                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  76 #define POWER_PSTATE_DRIVER "intel_pstate"
  77 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  78
  79 /*
  80  * MSR related
  81  */
  82 #define PLATFORM_INFO     0x0CE
  83 #define NON_TURBO_MASK    0xFF00
  84 #define NON_TURBO_OFFSET  0x8
  85
  86
  87 enum power_state {
  88         POWER_IDLE = 0,
  89         POWER_ONGOING,
  90         POWER_USED,
  91         POWER_UNKNOWN
  92 };
  93
  94 struct pstate_power_info {
  95         unsigned int lcore_id;               /**< Logical core id */
  96         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  97         uint32_t nb_freqs;                   /**< number of available freqs */
  98         FILE *f_cur_min;                     /**< FD of scaling_min */
  99         FILE *f_cur_max;                     /**< FD of scaling_max */
 100         char governor_ori[32];               /**< Original governor name */
 101         uint32_t curr_idx;                   /**< Freq index in freqs array */
 102         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 103         uint32_t sys_max_freq;               /**< system wide max freq  */
 104         uint32_t core_base_freq;             /**< core base freq  */
 105         uint32_t state;                      /**< Power in use state */
 106         uint16_t turbo_available;            /**< Turbo Boost available */
 107         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 108         uint16_t priority_core;              /**< High Performance core */
 109 } __rte_cache_aligned;
 110
 111
 112 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 113
 114 /**
 115  * It is to read the specific MSR.
 116  */
 117
 118 static int32_t
 119 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 120 {
 121         int fd, ret;
 122         char fullpath[PATH_MAX];
 123
 124         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 125
 126         fd = open(fullpath, O_RDONLY);
 127
 128         if (fd < 0) {
 129                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 130                                  strerror(errno));
 131                 return fd;
 132         }
 133
 134         ret = pread(fd, val, sizeof(uint64_t), msr);
 135
 136         if (ret < 0) {
 137                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 138                                  strerror(errno));
 139                 goto out;
 140         }
 141
 142         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 143                         fullpath, msr, lcore_id);
 144
 145         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 146
 147 out:    close(fd);
 148         return ret;
 149 }
 150
 151 /**
 152  * It is to fopen the sys file for the future setting the lcore frequency.
 153  */
 154 static int
 155 power_init_for_setting_freq(struct pstate_power_info *pi)
 156 {
 157         FILE *f_min, *f_max, *f_base;
 158         char fullpath_min[PATH_MAX];
 159         char fullpath_max[PATH_MAX];
 160         char fullpath_base[PATH_MAX];
 161         char buf_base[BUFSIZ];
 162         char *s_base;
 163         uint32_t base_ratio = 0;
 164         uint64_t max_non_turbo = 0;
 165         int  ret_val = 0;
 166
 167         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 168                         pi->lcore_id);
 169
 170         f_min = fopen(fullpath_min, "rw+");
 171         FOPEN_OR_ERR_RET(f_min, -1);
 172
 173         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 174                         pi->lcore_id);
 175
 176         f_max = fopen(fullpath_max, "rw+");
 177         if (f_max == NULL)
 178                 fclose(f_min);
 179
 180         FOPEN_OR_ERR_RET(f_max, -1);
 181
 182         pi->f_cur_min = f_min;
 183         pi->f_cur_max = f_max;
 184
 185         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 186                         pi->lcore_id);
 187
 188         f_base = fopen(fullpath_base, "r");
 189         if (f_base == NULL) {
 190                 /* No sysfs base_frequency, that's OK, continue without */
 191                 base_ratio = 0;
 192         } else {
 193                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 194                 FOPS_OR_NULL_GOTO(s_base, out);
 195
 196                 buf_base[BUFSIZ-1] = '\0';
 197                 if (strlen(buf_base))
 198                         /* Strip off terminating '\n' */
 199                         strtok(buf_base, "\n");
 200
 201                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 202                                 / BUS_FREQ;
 203         }
 204
 205         /* Add MSR read to detect turbo status */
 206
 207         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 208                 ret_val = -1;
 209                 goto out;
 210         }
 211
 212         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 213
 214         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 215
 216         pi->non_turbo_max_ratio = max_non_turbo;
 217
 218         /*
 219          * If base_frequency is reported as greater than the maximum
 220          * non-turbo frequency, then mark it as a high priority core.
 221          */
 222         if (base_ratio > max_non_turbo)
 223                 pi->priority_core = 1;
 224         else
 225                 pi->priority_core = 0;
 226         pi->core_base_freq = base_ratio * BUS_FREQ;
 227
 228 out:
 229         if (f_base != NULL)
 230                 fclose(f_base);
 231         return ret_val;
 232 }
 233
 234 static int
 235 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 236 {
 237         uint32_t target_freq = 0;
 238
 239         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 240                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 241                                 "should be less than %u\n", idx, pi->nb_freqs);
 242                 return -1;
 243         }
 244
 245         /* Check if it is the same as current */
 246         if (idx == pi->curr_idx)
 247                 return 0;
 248
 249         /* Because Intel Pstate Driver only allow user change min/max hint
 250          * User need change the min/max as same value.
 251          */
 252         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 253                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 254                                 "for setting frequency for lcore %u\n",
 255                                 pi->lcore_id);
 256                 return -1;
 257         }
 258
 259         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 260                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 261                                 "for setting frequency for lcore %u\n",
 262                                 pi->lcore_id);
 263                 return -1;
 264         }
 265
 266         /* Turbo is available and enabled, first freq bucket is sys max freq */
 267         if (pi->turbo_available && idx == 0) {
 268                 if (pi->turbo_enable)
 269                         target_freq = pi->sys_max_freq;
 270                 else {
 271                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 272                                         pi->lcore_id);
 273                         return -1;
 274                 }
 275         } else
 276                 target_freq = pi->freqs[idx];
 277
 278         /* Decrease freq, the min freq should be updated first */
 279         if (idx  >  pi->curr_idx) {
 280
 281                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 282                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 283                                         "lcore %u\n", pi->lcore_id);
 284                         return -1;
 285                 }
 286
 287                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 288                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 289                                         "lcore %u\n", pi->lcore_id);
 290                         return -1;
 291                 }
 292
 293                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 294                                   target_freq, pi->lcore_id);
 295
 296                 fflush(pi->f_cur_min);
 297                 fflush(pi->f_cur_max);
 298
 299         }
 300
 301         /* Increase freq, the max freq should be updated first */
 302         if (idx  <  pi->curr_idx) {
 303
 304                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 305                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 306                                         "lcore %u\n", pi->lcore_id);
 307                         return -1;
 308                 }
 309
 310                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 311                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 312                                         "lcore %u\n", pi->lcore_id);
 313                         return -1;
 314                 }
 315
 316                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 317                                   target_freq, pi->lcore_id);
 318
 319                 fflush(pi->f_cur_max);
 320                 fflush(pi->f_cur_min);
 321         }
 322
 323         pi->curr_idx = idx;
 324
 325         return 1;
 326 }
 327
 328 /**
 329  * It is to check the current scaling governor by reading sys file, and then
 330  * set it into 'performance' if it is not by writing the sys file. The original
 331  * governor will be saved for rolling back.
 332  */
 333 static int
 334 power_set_governor_performance(struct pstate_power_info *pi)
 335 {
 336         FILE *f;
 337         int ret = -1;
 338         char buf[BUFSIZ];
 339         char fullpath[PATH_MAX];
 340         char *s;
 341         int val;
 342
 343         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 344                         pi->lcore_id);
 345         f = fopen(fullpath, "rw+");
 346         FOPEN_OR_ERR_RET(f, ret);
 347
 348         s = fgets(buf, sizeof(buf), f);
 349         FOPS_OR_NULL_GOTO(s, out);
 350         /* Strip off terminating '\n' */
 351         strtok(buf, "\n");
 352
 353         /* Check if current governor is performance */
 354         if (strncmp(buf, POWER_GOVERNOR_PERF,
 355                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 356                 ret = 0;
 357                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 358                                 "already performance\n", pi->lcore_id);
 359                 goto out;
 360         }
 361         /* Save the original governor */
 362         strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 363
 364         /* Write 'performance' to the governor */
 365         val = fseek(f, 0, SEEK_SET);
 366         FOPS_OR_ERR_GOTO(val, out);
 367
 368         val = fputs(POWER_GOVERNOR_PERF, f);
 369         FOPS_OR_ERR_GOTO(val, out);
 370
 371         /* We need to flush to see if the fputs succeeds */
 372         val = fflush(f);
 373         FOPS_OR_ERR_GOTO(val, out);
 374
 375         ret = 0;
 376         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 377                         "set to performance successfully\n", pi->lcore_id);
 378 out:
 379         fclose(f);
 380
 381         return ret;
 382 }
 383
 384 /**
 385  * It is to check the governor and then set the original governor back if
 386  * needed by writing the sys file.
 387  */
 388 static int
 389 power_set_governor_original(struct pstate_power_info *pi)
 390 {
 391         FILE *f;
 392         int ret = -1;
 393         char buf[BUFSIZ];
 394         char fullpath[PATH_MAX];
 395         char *s;
 396         int val;
 397
 398         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 399                         pi->lcore_id);
 400         f = fopen(fullpath, "rw+");
 401         FOPEN_OR_ERR_RET(f, ret);
 402
 403         s = fgets(buf, sizeof(buf), f);
 404         FOPS_OR_NULL_GOTO(s, out);
 405
 406         /* Check if the governor to be set is the same as current */
 407         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 408                 ret = 0;
 409                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 410                                 "has already been set to %s\n",
 411                                 pi->lcore_id, pi->governor_ori);
 412                 goto out;
 413         }
 414
 415         /* Write back the original governor */
 416         val = fseek(f, 0, SEEK_SET);
 417         FOPS_OR_ERR_GOTO(val, out);
 418
 419         val = fputs(pi->governor_ori, f);
 420         FOPS_OR_ERR_GOTO(val, out);
 421
 422         ret = 0;
 423         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 424                         "has been set back to %s successfully\n",
 425                         pi->lcore_id, pi->governor_ori);
 426 out:
 427         fclose(f);
 428
 429         return ret;
 430 }
 431
 432 /**
 433  * It is to get the available frequencies of the specific lcore by reading the
 434  * sys file.
 435  */
 436 static int
 437 power_get_available_freqs(struct pstate_power_info *pi)
 438 {
 439         FILE *f_min, *f_max;
 440         int ret = -1;
 441         char *p_min, *p_max;
 442         char buf_min[BUFSIZ];
 443         char buf_max[BUFSIZ];
 444         char fullpath_min[PATH_MAX];
 445         char fullpath_max[PATH_MAX];
 446         char *s_min, *s_max;
 447         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 448         uint32_t i, num_freqs = 0;
 449
 450         snprintf(fullpath_max, sizeof(fullpath_max),
 451                         POWER_SYSFILE_BASE_MAX_FREQ,
 452                         pi->lcore_id);
 453         snprintf(fullpath_min, sizeof(fullpath_min),
 454                         POWER_SYSFILE_BASE_MIN_FREQ,
 455                         pi->lcore_id);
 456
 457         f_min = fopen(fullpath_min, "r");
 458         FOPEN_OR_ERR_RET(f_min, ret);
 459
 460         f_max = fopen(fullpath_max, "r");
 461         if (f_max == NULL)
 462                 fclose(f_min);
 463
 464         FOPEN_OR_ERR_RET(f_max, ret);
 465
 466         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 467         FOPS_OR_NULL_GOTO(s_min, out);
 468
 469         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 470         FOPS_OR_NULL_GOTO(s_max, out);
 471
 472
 473         /* Strip the line break if there is */
 474         p_min = strchr(buf_min, '\n');
 475         if (p_min != NULL)
 476                 *p_min = 0;
 477
 478         p_max = strchr(buf_max, '\n');
 479         if (p_max != NULL)
 480                 *p_max = 0;
 481
 482         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 483         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 484
 485         if (sys_max_freq < sys_min_freq)
 486                 goto out;
 487
 488         pi->sys_max_freq = sys_max_freq;
 489
 490         if (pi->priority_core == 1)
 491                 base_max_freq = pi->core_base_freq;
 492         else
 493                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 494
 495         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 496                         sys_min_freq,
 497                         sys_max_freq,
 498                         base_max_freq);
 499
 500         if (base_max_freq < sys_max_freq)
 501                 pi->turbo_available = 1;
 502         else
 503                 pi->turbo_available = 0;
 504
 505         /* If turbo is available then there is one extra freq bucket
 506          * to store the sys max freq which value is base_max +1
 507          */
 508         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 509                 pi->turbo_available;
 510
 511         /* Generate the freq bucket array.
 512          * If turbo is available the freq bucket[0] value is base_max +1
 513          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 514          * and so on.
 515          * If turbo is not available bucket[0] is base_max and so on
 516          */
 517         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 518                 if ((i == 0) && pi->turbo_available)
 519                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 520                 else
 521                         pi->freqs[pi->nb_freqs++] =
 522                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 523         }
 524
 525         ret = 0;
 526
 527         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 528                         num_freqs, pi->lcore_id);
 529
 530 out:
 531         fclose(f_min);
 532         fclose(f_max);
 533
 534         return ret;
 535 }
 536
 537 static int
 538 power_get_cur_idx(struct pstate_power_info *pi)
 539 {
 540         FILE *f_cur;
 541         int ret = -1;
 542         char *p_cur;
 543         char buf_cur[BUFSIZ];
 544         char fullpath_cur[PATH_MAX];
 545         char *s_cur;
 546         uint32_t sys_cur_freq = 0;
 547         unsigned int i;
 548
 549         snprintf(fullpath_cur, sizeof(fullpath_cur),
 550                         POWER_SYSFILE_CUR_FREQ,
 551                         pi->lcore_id);
 552         f_cur = fopen(fullpath_cur, "r");
 553         FOPEN_OR_ERR_RET(f_cur, ret);
 554
 555         /* initialize the cur_idx to matching current frequency freq index */
 556         s_cur = fgets(buf_cur, sizeof(buf_cur), f_cur);
 557         FOPS_OR_NULL_GOTO(s_cur, fail);
 558
 559         p_cur = strchr(buf_cur, '\n');
 560         if (p_cur != NULL)
 561                 *p_cur = 0;
 562         sys_cur_freq = strtoul(buf_cur, &p_cur, POWER_CONVERT_TO_DECIMAL);
 563
 564         /* convert the frequency to nearest 100000 value
 565          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 566          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 567          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 568          */
 569         unsigned int freq_conv = 0;
 570         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 571                                 / ROUND_FREQ_TO_N_100000;
 572         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 573
 574         for (i = 0; i < pi->nb_freqs; i++) {
 575                 if (freq_conv == pi->freqs[i]) {
 576                         pi->curr_idx = i;
 577                         break;
 578                 }
 579         }
 580
 581         fclose(f_cur);
 582         return 0;
 583 fail:
 584         fclose(f_cur);
 585         return ret;
 586 }
 587
 588 int
 589 power_pstate_cpufreq_check_supported(void)
 590 {
 591         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 592 }
 593
 594 int
 595 power_pstate_cpufreq_init(unsigned int lcore_id)
 596 {
 597         struct pstate_power_info *pi;
 598         uint32_t exp_state;
 599
 600         if (lcore_id >= RTE_MAX_LCORE) {
 601                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 602                                 lcore_id, RTE_MAX_LCORE - 1U);
 603                 return -1;
 604         }
 605
 606         pi = &lcore_power_info[lcore_id];
 607         exp_state = POWER_IDLE;
 608         /* The power in use state works as a guard variable between
 609          * the CPU frequency control initialization and exit process.
 610          * The ACQUIRE memory ordering here pairs with the RELEASE
 611          * ordering below as lock to make sure the frequency operations
 612          * in the critical section are done under the correct state.
 613          */
 614         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 615                                         POWER_ONGOING, 0,
 616                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 617                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 618                                 "in use\n", lcore_id);
 619                 return -1;
 620         }
 621
 622         pi->lcore_id = lcore_id;
 623         /* Check and set the governor */
 624         if (power_set_governor_performance(pi) < 0) {
 625                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 626                                 "performance\n", lcore_id);
 627                 goto fail;
 628         }
 629         /* Init for setting lcore frequency */
 630         if (power_init_for_setting_freq(pi) < 0) {
 631                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 632                                 "lcore %u\n", lcore_id);
 633                 goto fail;
 634         }
 635
 636         /* Get the available frequencies */
 637         if (power_get_available_freqs(pi) < 0) {
 638                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 639                                 "lcore %u\n", lcore_id);
 640                 goto fail;
 641         }
 642
 643         if (power_get_cur_idx(pi) < 0) {
 644                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 645                                 "index of lcore %u\n", lcore_id);
 646                 goto fail;
 647         }
 648
 649         /* Set freq to max by default */
 650         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 651                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 652                                 "to max\n", lcore_id);
 653                 goto fail;
 654         }
 655
 656         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 657                         "power management\n", lcore_id);
 658         exp_state = POWER_ONGOING;
 659         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 660                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 661
 662         return 0;
 663
 664 fail:
 665         exp_state = POWER_ONGOING;
 666         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 667                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 668
 669         return -1;
 670 }
 671
 672 int
 673 power_pstate_cpufreq_exit(unsigned int lcore_id)
 674 {
 675         struct pstate_power_info *pi;
 676         uint32_t exp_state;
 677
 678         if (lcore_id >= RTE_MAX_LCORE) {
 679                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 680                                 lcore_id, RTE_MAX_LCORE - 1U);
 681                 return -1;
 682         }
 683         pi = &lcore_power_info[lcore_id];
 684
 685         exp_state = POWER_USED;
 686         /* The power in use state works as a guard variable between
 687          * the CPU frequency control initialization and exit process.
 688          * The ACQUIRE memory ordering here pairs with the RELEASE
 689          * ordering below as lock to make sure the frequency operations
 690          * in the critical section are under done the correct state.
 691          */
 692         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 693                                         POWER_ONGOING, 0,
 694                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 695                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 696                                 "not used\n", lcore_id);
 697                 return -1;
 698         }
 699
 700         /* Close FD of setting freq */
 701         fclose(pi->f_cur_min);
 702         fclose(pi->f_cur_max);
 703         pi->f_cur_min = NULL;
 704         pi->f_cur_max = NULL;
 705
 706         /* Set the governor back to the original */
 707         if (power_set_governor_original(pi) < 0) {
 708                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 709                                 "to the original\n", lcore_id);
 710                 goto fail;
 711         }
 712
 713         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 714                         "'performance' mode and been set back to the "
 715                         "original\n", lcore_id);
 716         exp_state = POWER_ONGOING;
 717         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 718                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 719
 720         return 0;
 721
 722 fail:
 723         exp_state = POWER_ONGOING;
 724         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 725                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 726
 727         return -1;
 728 }
 729
 730
 731 uint32_t
 732 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 733 {
 734         struct pstate_power_info *pi;
 735
 736         if (lcore_id >= RTE_MAX_LCORE) {
 737                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 738                 return 0;
 739         }
 740
 741         if (freqs == NULL) {
 742                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 743                 return 0;
 744         }
 745
 746         pi = &lcore_power_info[lcore_id];
 747         if (num < pi->nb_freqs) {
 748                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 749                 return 0;
 750         }
 751         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 752
 753         return pi->nb_freqs;
 754 }
 755
 756 uint32_t
 757 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 758 {
 759         if (lcore_id >= RTE_MAX_LCORE) {
 760                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 761                 return RTE_POWER_INVALID_FREQ_INDEX;
 762         }
 763
 764         return lcore_power_info[lcore_id].curr_idx;
 765 }
 766
 767
 768 int
 769 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 770 {
 771         if (lcore_id >= RTE_MAX_LCORE) {
 772                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 773                 return -1;
 774         }
 775
 776         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 777 }
 778
 779 int
 780 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 781 {
 782         struct pstate_power_info *pi;
 783
 784         if (lcore_id >= RTE_MAX_LCORE) {
 785                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 786                 return -1;
 787         }
 788
 789         pi = &lcore_power_info[lcore_id];
 790         if (pi->curr_idx == 0 ||
 791             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 792                 return 0;
 793
 794         /* Frequencies in the array are from high to low. */
 795         return set_freq_internal(pi, pi->curr_idx - 1);
 796 }
 797
 798 int
 799 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 800 {
 801         struct pstate_power_info *pi;
 802
 803         if (lcore_id >= RTE_MAX_LCORE) {
 804                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 805                 return -1;
 806         }
 807
 808         pi = &lcore_power_info[lcore_id];
 809         if (pi->curr_idx + 1 == pi->nb_freqs)
 810                 return 0;
 811
 812         /* Frequencies in the array are from high to low. */
 813         return set_freq_internal(pi, pi->curr_idx + 1);
 814 }
 815
 816 int
 817 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 818 {
 819         if (lcore_id >= RTE_MAX_LCORE) {
 820                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 821                 return -1;
 822         }
 823
 824         /* Frequencies in the array are from high to low. */
 825         if (lcore_power_info[lcore_id].turbo_available) {
 826                 if (lcore_power_info[lcore_id].turbo_enable)
 827                         /* Set to Turbo */
 828                         return set_freq_internal(
 829                                         &lcore_power_info[lcore_id], 0);
 830                 else
 831                         /* Set to max non-turbo */
 832                         return set_freq_internal(
 833                                         &lcore_power_info[lcore_id], 1);
 834         } else
 835                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 836 }
 837
 838
 839 int
 840 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 841 {
 842         struct pstate_power_info *pi;
 843
 844         if (lcore_id >= RTE_MAX_LCORE) {
 845                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 846                 return -1;
 847         }
 848
 849         pi = &lcore_power_info[lcore_id];
 850
 851         /* Frequencies in the array are from high to low. */
 852         return set_freq_internal(pi, pi->nb_freqs - 1);
 853 }
 854
 855
 856 int
 857 power_pstate_turbo_status(unsigned int lcore_id)
 858 {
 859         struct pstate_power_info *pi;
 860
 861         if (lcore_id >= RTE_MAX_LCORE) {
 862                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 863                 return -1;
 864         }
 865
 866         pi = &lcore_power_info[lcore_id];
 867
 868         return pi->turbo_enable;
 869 }
 870
 871 int
 872 power_pstate_enable_turbo(unsigned int lcore_id)
 873 {
 874         struct pstate_power_info *pi;
 875
 876         if (lcore_id >= RTE_MAX_LCORE) {
 877                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 878                 return -1;
 879         }
 880
 881         pi = &lcore_power_info[lcore_id];
 882
 883         if (pi->turbo_available)
 884                 pi->turbo_enable = 1;
 885         else {
 886                 pi->turbo_enable = 0;
 887                 RTE_LOG(ERR, POWER,
 888                         "Failed to enable turbo on lcore %u\n",
 889                         lcore_id);
 890                         return -1;
 891         }
 892
 893         return 0;
 894 }
 895
 896
 897 int
 898 power_pstate_disable_turbo(unsigned int lcore_id)
 899 {
 900         struct pstate_power_info *pi;
 901
 902         if (lcore_id >= RTE_MAX_LCORE) {
 903                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 904                 return -1;
 905         }
 906
 907         pi = &lcore_power_info[lcore_id];
 908
 909         pi->turbo_enable = 0;
 910
 911         if (pi->turbo_available && pi->curr_idx <= 1) {
 912                 /* Try to set freq to max by default coming out of turbo */
 913                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 914                         RTE_LOG(ERR, POWER,
 915                                 "Failed to set frequency of lcore %u to max\n",
 916                                 lcore_id);
 917                         return -1;
 918                 }
 919         }
 920
 921         return 0;
 922 }
 923
 924
 925 int power_pstate_get_capabilities(unsigned int lcore_id,
 926                 struct rte_power_core_capabilities *caps)
 927 {
 928         struct pstate_power_info *pi;
 929
 930         if (lcore_id >= RTE_MAX_LCORE) {
 931                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 932                 return -1;
 933         }
 934         if (caps == NULL) {
 935                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 936                 return -1;
 937         }
 938
 939         pi = &lcore_power_info[lcore_id];
 940         caps->capabilities = 0;
 941         caps->turbo = !!(pi->turbo_available);
 942         caps->priority = pi->priority_core;
 943
 944         return 0;
 945 }