lib/librte_power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24
  25 #ifdef RTE_LIBRTE_POWER_DEBUG
  26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  27                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  28 } while (0)
  29 #else
  30 #define POWER_DEBUG_TRACE(fmt, args...)
  31 #endif
  32
  33 #define FOPEN_OR_ERR_RET(f, retval) do { \
  34                 if ((f) == NULL) { \
  35                         RTE_LOG(ERR, POWER, "File not opened\n"); \
  36                         return retval; \
  37                 } \
  38 } while (0)
  39
  40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  41                 if ((ret) == NULL) { \
  42                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  43                         goto label; \
  44                 } \
  45 } while (0)
  46
  47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  48                 if ((ret) < 0) { \
  49                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  50                         goto label; \
  51                 } \
  52 } while (0)
  53
  54 /* macros used for rounding frequency to nearest 100000 */
  55 #define FREQ_ROUNDING_DELTA 50000
  56 #define ROUND_FREQ_TO_N_100000 100000
  57
  58 #define POWER_CONVERT_TO_DECIMAL 10
  59 #define BUS_FREQ     100000
  60
  61 #define POWER_GOVERNOR_PERF "performance"
  62 #define POWER_SYSFILE_GOVERNOR  \
  63                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  64 #define POWER_SYSFILE_MAX_FREQ \
  65                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  66 #define POWER_SYSFILE_MIN_FREQ  \
  67                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  68 #define POWER_SYSFILE_CUR_FREQ  \
  69                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  70 #define POWER_SYSFILE_BASE_MAX_FREQ \
  71                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  72 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  73                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  74 #define POWER_SYSFILE_BASE_FREQ  \
  75                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  76 #define POWER_PSTATE_DRIVER "intel_pstate"
  77 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  78
  79 /*
  80  * MSR related
  81  */
  82 #define PLATFORM_INFO     0x0CE
  83 #define NON_TURBO_MASK    0xFF00
  84 #define NON_TURBO_OFFSET  0x8
  85
  86
  87 enum power_state {
  88         POWER_IDLE = 0,
  89         POWER_ONGOING,
  90         POWER_USED,
  91         POWER_UNKNOWN
  92 };
  93
  94 struct pstate_power_info {
  95         unsigned int lcore_id;               /**< Logical core id */
  96         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  97         uint32_t nb_freqs;                   /**< number of available freqs */
  98         FILE *f_cur_min;                     /**< FD of scaling_min */
  99         FILE *f_cur_max;                     /**< FD of scaling_max */
 100         char governor_ori[32];               /**< Original governor name */
 101         uint32_t curr_idx;                   /**< Freq index in freqs array */
 102         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
 103         uint32_t sys_max_freq;               /**< system wide max freq  */
 104         uint32_t core_base_freq;             /**< core base freq  */
 105         uint32_t state;                      /**< Power in use state */
 106         uint16_t turbo_available;            /**< Turbo Boost available */
 107         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 108         uint16_t priority_core;              /**< High Performance core */
 109 } __rte_cache_aligned;
 110
 111
 112 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 113
 114 /**
 115  * It is to read the specific MSR.
 116  */
 117
 118 static int32_t
 119 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 120 {
 121         int fd, ret;
 122         char fullpath[PATH_MAX];
 123
 124         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 125
 126         fd = open(fullpath, O_RDONLY);
 127
 128         if (fd < 0) {
 129                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 130                                  strerror(errno));
 131                 return fd;
 132         }
 133
 134         ret = pread(fd, val, sizeof(uint64_t), msr);
 135
 136         if (ret < 0) {
 137                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 138                                  strerror(errno));
 139                 goto out;
 140         }
 141
 142         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 143                         fullpath, msr, lcore_id);
 144
 145         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 146
 147 out:    close(fd);
 148         return ret;
 149 }
 150
 151 /**
 152  * It is to fopen the sys file for the future setting the lcore frequency.
 153  */
 154 static int
 155 power_init_for_setting_freq(struct pstate_power_info *pi)
 156 {
 157         FILE *f_min, *f_max, *f_base = NULL, *f_base_max;
 158         char fullpath_min[PATH_MAX];
 159         char fullpath_max[PATH_MAX];
 160         char fullpath_base[PATH_MAX];
 161         char fullpath_base_max[PATH_MAX];
 162         char buf_base[BUFSIZ];
 163         char *s_base;
 164         char *s_base_max;
 165         uint32_t base_ratio = 0;
 166         uint32_t base_max_ratio = 0;
 167         uint64_t max_non_turbo = 0;
 168         int  ret_val = 0;
 169
 170         snprintf(fullpath_base_max,
 171                         sizeof(fullpath_base_max),
 172                         POWER_SYSFILE_BASE_MAX_FREQ,
 173                         pi->lcore_id);
 174         f_base_max = fopen(fullpath_base_max, "r");
 175         FOPEN_OR_ERR_RET(f_base_max, -1);
 176         if (f_base_max != NULL) {
 177                 s_base_max = fgets(buf_base, sizeof(buf_base), f_base_max);
 178                 FOPS_OR_NULL_GOTO(s_base_max, out);
 179
 180                 buf_base[BUFSIZ-1] = '\0';
 181                 if (strlen(buf_base))
 182                         /* Strip off terminating '\n' */
 183                         strtok(buf_base, "\n");
 184
 185                 base_max_ratio =
 186                         strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 187                                 / BUS_FREQ;
 188         }
 189
 190         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 191                         pi->lcore_id);
 192         f_min = fopen(fullpath_min, "rw+");
 193         FOPEN_OR_ERR_RET(f_min, -1);
 194
 195         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 196                         pi->lcore_id);
 197         f_max = fopen(fullpath_max, "rw+");
 198         if (f_max == NULL)
 199                 fclose(f_min);
 200         FOPEN_OR_ERR_RET(f_max, -1);
 201
 202         pi->f_cur_min = f_min;
 203         pi->f_cur_max = f_max;
 204
 205         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 206                         pi->lcore_id);
 207
 208         f_base = fopen(fullpath_base, "r");
 209         if (f_base == NULL) {
 210                 /* No sysfs base_frequency, that's OK, continue without */
 211                 base_ratio = 0;
 212         } else {
 213                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 214                 FOPS_OR_NULL_GOTO(s_base, out);
 215
 216                 buf_base[BUFSIZ-1] = '\0';
 217                 if (strlen(buf_base))
 218                         /* Strip off terminating '\n' */
 219                         strtok(buf_base, "\n");
 220
 221                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 222                                 / BUS_FREQ;
 223                 fclose(f_base);
 224         }
 225
 226         /* Add MSR read to detect turbo status */
 227
 228         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 229                 ret_val = -1;
 230                 goto out;
 231         }
 232
 233         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 234
 235         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 236
 237         pi->non_turbo_max_ratio = max_non_turbo;
 238
 239         /*
 240          * If base_frequency is reported as greater than the maximum
 241          * turbo frequency, that's a known issue with some kernels.
 242          * Set base_frequency to max_non_turbo as a workaround.
 243          */
 244         if (base_ratio > base_max_ratio) {
 245                 /* base_ratio is greater than max turbo. Kernel bug. */
 246                 pi->priority_core = 0;
 247                 goto out;
 248         }
 249
 250         /*
 251          * If base_frequency is reported as greater than the maximum
 252          * non-turbo frequency, then mark it as a high priority core.
 253          */
 254         if (base_ratio > max_non_turbo)
 255                 pi->priority_core = 1;
 256         else
 257                 pi->priority_core = 0;
 258         pi->core_base_freq = base_ratio * BUS_FREQ;
 259
 260 out:
 261         if (f_base != NULL)
 262                 fclose(f_base);
 263         return ret_val;
 264 }
 265
 266 static int
 267 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 268 {
 269         uint32_t target_freq = 0;
 270
 271         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 272                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 273                                 "should be less than %u\n", idx, pi->nb_freqs);
 274                 return -1;
 275         }
 276
 277         /* Check if it is the same as current */
 278         if (idx == pi->curr_idx)
 279                 return 0;
 280
 281         /* Because Intel Pstate Driver only allow user change min/max hint
 282          * User need change the min/max as same value.
 283          */
 284         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 285                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 286                                 "for setting frequency for lcore %u\n",
 287                                 pi->lcore_id);
 288                 return -1;
 289         }
 290
 291         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 292                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 293                                 "for setting frequency for lcore %u\n",
 294                                 pi->lcore_id);
 295                 return -1;
 296         }
 297
 298         /* Turbo is available and enabled, first freq bucket is sys max freq */
 299         if (pi->turbo_available && idx == 0) {
 300                 if (pi->turbo_enable)
 301                         target_freq = pi->sys_max_freq;
 302                 else {
 303                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 304                                         pi->lcore_id);
 305                         return -1;
 306                 }
 307         } else
 308                 target_freq = pi->freqs[idx];
 309
 310         /* Decrease freq, the min freq should be updated first */
 311         if (idx  >  pi->curr_idx) {
 312
 313                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 314                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 315                                         "lcore %u\n", pi->lcore_id);
 316                         return -1;
 317                 }
 318
 319                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 320                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 321                                         "lcore %u\n", pi->lcore_id);
 322                         return -1;
 323                 }
 324
 325                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 326                                   target_freq, pi->lcore_id);
 327
 328                 fflush(pi->f_cur_min);
 329                 fflush(pi->f_cur_max);
 330
 331         }
 332
 333         /* Increase freq, the max freq should be updated first */
 334         if (idx  <  pi->curr_idx) {
 335
 336                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 337                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 338                                         "lcore %u\n", pi->lcore_id);
 339                         return -1;
 340                 }
 341
 342                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 343                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 344                                         "lcore %u\n", pi->lcore_id);
 345                         return -1;
 346                 }
 347
 348                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 349                                   target_freq, pi->lcore_id);
 350
 351                 fflush(pi->f_cur_max);
 352                 fflush(pi->f_cur_min);
 353         }
 354
 355         pi->curr_idx = idx;
 356
 357         return 1;
 358 }
 359
 360 /**
 361  * It is to check the current scaling governor by reading sys file, and then
 362  * set it into 'performance' if it is not by writing the sys file. The original
 363  * governor will be saved for rolling back.
 364  */
 365 static int
 366 power_set_governor_performance(struct pstate_power_info *pi)
 367 {
 368         FILE *f;
 369         int ret = -1;
 370         char buf[BUFSIZ];
 371         char fullpath[PATH_MAX];
 372         char *s;
 373         int val;
 374
 375         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 376                         pi->lcore_id);
 377         f = fopen(fullpath, "rw+");
 378         FOPEN_OR_ERR_RET(f, ret);
 379
 380         s = fgets(buf, sizeof(buf), f);
 381         FOPS_OR_NULL_GOTO(s, out);
 382         /* Strip off terminating '\n' */
 383         strtok(buf, "\n");
 384
 385         /* Save the original governor */
 386         rte_strscpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 387
 388         /* Check if current governor is performance */
 389         if (strncmp(buf, POWER_GOVERNOR_PERF,
 390                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 391                 ret = 0;
 392                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 393                                 "already performance\n", pi->lcore_id);
 394                 goto out;
 395         }
 396
 397         /* Write 'performance' to the governor */
 398         val = fseek(f, 0, SEEK_SET);
 399         FOPS_OR_ERR_GOTO(val, out);
 400
 401         val = fputs(POWER_GOVERNOR_PERF, f);
 402         FOPS_OR_ERR_GOTO(val, out);
 403
 404         /* We need to flush to see if the fputs succeeds */
 405         val = fflush(f);
 406         FOPS_OR_ERR_GOTO(val, out);
 407
 408         ret = 0;
 409         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 410                         "set to performance successfully\n", pi->lcore_id);
 411 out:
 412         fclose(f);
 413
 414         return ret;
 415 }
 416
 417 /**
 418  * It is to check the governor and then set the original governor back if
 419  * needed by writing the sys file.
 420  */
 421 static int
 422 power_set_governor_original(struct pstate_power_info *pi)
 423 {
 424         FILE *f;
 425         int ret = -1;
 426         char buf[BUFSIZ];
 427         char fullpath[PATH_MAX];
 428         char *s;
 429         int val;
 430
 431         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 432                         pi->lcore_id);
 433         f = fopen(fullpath, "rw+");
 434         FOPEN_OR_ERR_RET(f, ret);
 435
 436         s = fgets(buf, sizeof(buf), f);
 437         FOPS_OR_NULL_GOTO(s, out);
 438
 439         /* Check if the governor to be set is the same as current */
 440         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 441                 ret = 0;
 442                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 443                                 "has already been set to %s\n",
 444                                 pi->lcore_id, pi->governor_ori);
 445                 goto out;
 446         }
 447
 448         /* Write back the original governor */
 449         val = fseek(f, 0, SEEK_SET);
 450         FOPS_OR_ERR_GOTO(val, out);
 451
 452         val = fputs(pi->governor_ori, f);
 453         FOPS_OR_ERR_GOTO(val, out);
 454
 455         ret = 0;
 456         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 457                         "has been set back to %s successfully\n",
 458                         pi->lcore_id, pi->governor_ori);
 459 out:
 460         fclose(f);
 461
 462         return ret;
 463 }
 464
 465 /**
 466  * It is to get the available frequencies of the specific lcore by reading the
 467  * sys file.
 468  */
 469 static int
 470 power_get_available_freqs(struct pstate_power_info *pi)
 471 {
 472         FILE *f_min, *f_max;
 473         int ret = -1;
 474         char *p_min, *p_max;
 475         char buf_min[BUFSIZ];
 476         char buf_max[BUFSIZ];
 477         char fullpath_min[PATH_MAX];
 478         char fullpath_max[PATH_MAX];
 479         char *s_min, *s_max;
 480         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 481         uint32_t i, num_freqs = 0;
 482
 483         snprintf(fullpath_max, sizeof(fullpath_max),
 484                         POWER_SYSFILE_BASE_MAX_FREQ,
 485                         pi->lcore_id);
 486         snprintf(fullpath_min, sizeof(fullpath_min),
 487                         POWER_SYSFILE_BASE_MIN_FREQ,
 488                         pi->lcore_id);
 489
 490         f_min = fopen(fullpath_min, "r");
 491         FOPEN_OR_ERR_RET(f_min, ret);
 492
 493         f_max = fopen(fullpath_max, "r");
 494         if (f_max == NULL)
 495                 fclose(f_min);
 496
 497         FOPEN_OR_ERR_RET(f_max, ret);
 498
 499         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 500         FOPS_OR_NULL_GOTO(s_min, out);
 501
 502         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 503         FOPS_OR_NULL_GOTO(s_max, out);
 504
 505
 506         /* Strip the line break if there is */
 507         p_min = strchr(buf_min, '\n');
 508         if (p_min != NULL)
 509                 *p_min = 0;
 510
 511         p_max = strchr(buf_max, '\n');
 512         if (p_max != NULL)
 513                 *p_max = 0;
 514
 515         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 516         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 517
 518         if (sys_max_freq < sys_min_freq)
 519                 goto out;
 520
 521         pi->sys_max_freq = sys_max_freq;
 522
 523         if (pi->priority_core == 1)
 524                 base_max_freq = pi->core_base_freq;
 525         else
 526                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 527
 528         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 529                         sys_min_freq,
 530                         sys_max_freq,
 531                         base_max_freq);
 532
 533         if (base_max_freq < sys_max_freq)
 534                 pi->turbo_available = 1;
 535         else
 536                 pi->turbo_available = 0;
 537
 538         /* If turbo is available then there is one extra freq bucket
 539          * to store the sys max freq which value is base_max +1
 540          */
 541         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 542                 pi->turbo_available;
 543
 544         /* Generate the freq bucket array.
 545          * If turbo is available the freq bucket[0] value is base_max +1
 546          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 547          * and so on.
 548          * If turbo is not available bucket[0] is base_max and so on
 549          */
 550         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 551                 if ((i == 0) && pi->turbo_available)
 552                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 553                 else
 554                         pi->freqs[pi->nb_freqs++] =
 555                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 556         }
 557
 558         ret = 0;
 559
 560         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 561                         num_freqs, pi->lcore_id);
 562
 563 out:
 564         fclose(f_min);
 565         fclose(f_max);
 566
 567         return ret;
 568 }
 569
 570 static int
 571 power_get_cur_idx(struct pstate_power_info *pi)
 572 {
 573         FILE *f_cur;
 574         int ret = -1;
 575         char *p_cur;
 576         char buf_cur[BUFSIZ];
 577         char fullpath_cur[PATH_MAX];
 578         char *s_cur;
 579         uint32_t sys_cur_freq = 0;
 580         unsigned int i;
 581
 582         snprintf(fullpath_cur, sizeof(fullpath_cur),
 583                         POWER_SYSFILE_CUR_FREQ,
 584                         pi->lcore_id);
 585         f_cur = fopen(fullpath_cur, "r");
 586         FOPEN_OR_ERR_RET(f_cur, ret);
 587
 588         /* initialize the cur_idx to matching current frequency freq index */
 589         s_cur = fgets(buf_cur, sizeof(buf_cur), f_cur);
 590         FOPS_OR_NULL_GOTO(s_cur, fail);
 591
 592         p_cur = strchr(buf_cur, '\n');
 593         if (p_cur != NULL)
 594                 *p_cur = 0;
 595         sys_cur_freq = strtoul(buf_cur, &p_cur, POWER_CONVERT_TO_DECIMAL);
 596
 597         /* convert the frequency to nearest 100000 value
 598          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 599          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 600          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 601          */
 602         unsigned int freq_conv = 0;
 603         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 604                                 / ROUND_FREQ_TO_N_100000;
 605         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 606
 607         for (i = 0; i < pi->nb_freqs; i++) {
 608                 if (freq_conv == pi->freqs[i]) {
 609                         pi->curr_idx = i;
 610                         break;
 611                 }
 612         }
 613
 614         fclose(f_cur);
 615         return 0;
 616 fail:
 617         fclose(f_cur);
 618         return ret;
 619 }
 620
 621 int
 622 power_pstate_cpufreq_check_supported(void)
 623 {
 624         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 625 }
 626
 627 int
 628 power_pstate_cpufreq_init(unsigned int lcore_id)
 629 {
 630         struct pstate_power_info *pi;
 631         uint32_t exp_state;
 632
 633         if (lcore_id >= RTE_MAX_LCORE) {
 634                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 635                                 lcore_id, RTE_MAX_LCORE - 1U);
 636                 return -1;
 637         }
 638
 639         pi = &lcore_power_info[lcore_id];
 640         exp_state = POWER_IDLE;
 641         /* The power in use state works as a guard variable between
 642          * the CPU frequency control initialization and exit process.
 643          * The ACQUIRE memory ordering here pairs with the RELEASE
 644          * ordering below as lock to make sure the frequency operations
 645          * in the critical section are done under the correct state.
 646          */
 647         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 648                                         POWER_ONGOING, 0,
 649                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 650                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 651                                 "in use\n", lcore_id);
 652                 return -1;
 653         }
 654
 655         pi->lcore_id = lcore_id;
 656         /* Check and set the governor */
 657         if (power_set_governor_performance(pi) < 0) {
 658                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 659                                 "performance\n", lcore_id);
 660                 goto fail;
 661         }
 662         /* Init for setting lcore frequency */
 663         if (power_init_for_setting_freq(pi) < 0) {
 664                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 665                                 "lcore %u\n", lcore_id);
 666                 goto fail;
 667         }
 668
 669         /* Get the available frequencies */
 670         if (power_get_available_freqs(pi) < 0) {
 671                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 672                                 "lcore %u\n", lcore_id);
 673                 goto fail;
 674         }
 675
 676         if (power_get_cur_idx(pi) < 0) {
 677                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 678                                 "index of lcore %u\n", lcore_id);
 679                 goto fail;
 680         }
 681
 682         /* Set freq to max by default */
 683         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 684                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 685                                 "to max\n", lcore_id);
 686                 goto fail;
 687         }
 688
 689         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 690                         "power management\n", lcore_id);
 691         exp_state = POWER_ONGOING;
 692         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 693                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 694
 695         return 0;
 696
 697 fail:
 698         exp_state = POWER_ONGOING;
 699         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 700                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 701
 702         return -1;
 703 }
 704
 705 int
 706 power_pstate_cpufreq_exit(unsigned int lcore_id)
 707 {
 708         struct pstate_power_info *pi;
 709         uint32_t exp_state;
 710
 711         if (lcore_id >= RTE_MAX_LCORE) {
 712                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 713                                 lcore_id, RTE_MAX_LCORE - 1U);
 714                 return -1;
 715         }
 716         pi = &lcore_power_info[lcore_id];
 717
 718         exp_state = POWER_USED;
 719         /* The power in use state works as a guard variable between
 720          * the CPU frequency control initialization and exit process.
 721          * The ACQUIRE memory ordering here pairs with the RELEASE
 722          * ordering below as lock to make sure the frequency operations
 723          * in the critical section are under done the correct state.
 724          */
 725         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 726                                         POWER_ONGOING, 0,
 727                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 728                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 729                                 "not used\n", lcore_id);
 730                 return -1;
 731         }
 732
 733         /* Close FD of setting freq */
 734         fclose(pi->f_cur_min);
 735         fclose(pi->f_cur_max);
 736         pi->f_cur_min = NULL;
 737         pi->f_cur_max = NULL;
 738
 739         /* Set the governor back to the original */
 740         if (power_set_governor_original(pi) < 0) {
 741                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 742                                 "to the original\n", lcore_id);
 743                 goto fail;
 744         }
 745
 746         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 747                         "'performance' mode and been set back to the "
 748                         "original\n", lcore_id);
 749         exp_state = POWER_ONGOING;
 750         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 751                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 752
 753         return 0;
 754
 755 fail:
 756         exp_state = POWER_ONGOING;
 757         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 758                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 759
 760         return -1;
 761 }
 762
 763
 764 uint32_t
 765 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 766 {
 767         struct pstate_power_info *pi;
 768
 769         if (lcore_id >= RTE_MAX_LCORE) {
 770                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 771                 return 0;
 772         }
 773
 774         if (freqs == NULL) {
 775                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 776                 return 0;
 777         }
 778
 779         pi = &lcore_power_info[lcore_id];
 780         if (num < pi->nb_freqs) {
 781                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 782                 return 0;
 783         }
 784         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 785
 786         return pi->nb_freqs;
 787 }
 788
 789 uint32_t
 790 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 791 {
 792         if (lcore_id >= RTE_MAX_LCORE) {
 793                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 794                 return RTE_POWER_INVALID_FREQ_INDEX;
 795         }
 796
 797         return lcore_power_info[lcore_id].curr_idx;
 798 }
 799
 800
 801 int
 802 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 803 {
 804         if (lcore_id >= RTE_MAX_LCORE) {
 805                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 806                 return -1;
 807         }
 808
 809         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 810 }
 811
 812 int
 813 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 814 {
 815         struct pstate_power_info *pi;
 816
 817         if (lcore_id >= RTE_MAX_LCORE) {
 818                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 819                 return -1;
 820         }
 821
 822         pi = &lcore_power_info[lcore_id];
 823         if (pi->curr_idx == 0 ||
 824             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 825                 return 0;
 826
 827         /* Frequencies in the array are from high to low. */
 828         return set_freq_internal(pi, pi->curr_idx - 1);
 829 }
 830
 831 int
 832 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 833 {
 834         struct pstate_power_info *pi;
 835
 836         if (lcore_id >= RTE_MAX_LCORE) {
 837                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 838                 return -1;
 839         }
 840
 841         pi = &lcore_power_info[lcore_id];
 842         if (pi->curr_idx + 1 == pi->nb_freqs)
 843                 return 0;
 844
 845         /* Frequencies in the array are from high to low. */
 846         return set_freq_internal(pi, pi->curr_idx + 1);
 847 }
 848
 849 int
 850 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 851 {
 852         if (lcore_id >= RTE_MAX_LCORE) {
 853                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 854                 return -1;
 855         }
 856
 857         /* Frequencies in the array are from high to low. */
 858         if (lcore_power_info[lcore_id].turbo_available) {
 859                 if (lcore_power_info[lcore_id].turbo_enable)
 860                         /* Set to Turbo */
 861                         return set_freq_internal(
 862                                         &lcore_power_info[lcore_id], 0);
 863                 else
 864                         /* Set to max non-turbo */
 865                         return set_freq_internal(
 866                                         &lcore_power_info[lcore_id], 1);
 867         } else
 868                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 869 }
 870
 871
 872 int
 873 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 874 {
 875         struct pstate_power_info *pi;
 876
 877         if (lcore_id >= RTE_MAX_LCORE) {
 878                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 879                 return -1;
 880         }
 881
 882         pi = &lcore_power_info[lcore_id];
 883
 884         /* Frequencies in the array are from high to low. */
 885         return set_freq_internal(pi, pi->nb_freqs - 1);
 886 }
 887
 888
 889 int
 890 power_pstate_turbo_status(unsigned int lcore_id)
 891 {
 892         struct pstate_power_info *pi;
 893
 894         if (lcore_id >= RTE_MAX_LCORE) {
 895                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 896                 return -1;
 897         }
 898
 899         pi = &lcore_power_info[lcore_id];
 900
 901         return pi->turbo_enable;
 902 }
 903
 904 int
 905 power_pstate_enable_turbo(unsigned int lcore_id)
 906 {
 907         struct pstate_power_info *pi;
 908
 909         if (lcore_id >= RTE_MAX_LCORE) {
 910                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 911                 return -1;
 912         }
 913
 914         pi = &lcore_power_info[lcore_id];
 915
 916         if (pi->turbo_available)
 917                 pi->turbo_enable = 1;
 918         else {
 919                 pi->turbo_enable = 0;
 920                 RTE_LOG(ERR, POWER,
 921                         "Failed to enable turbo on lcore %u\n",
 922                         lcore_id);
 923                         return -1;
 924         }
 925
 926         return 0;
 927 }
 928
 929
 930 int
 931 power_pstate_disable_turbo(unsigned int lcore_id)
 932 {
 933         struct pstate_power_info *pi;
 934
 935         if (lcore_id >= RTE_MAX_LCORE) {
 936                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 937                 return -1;
 938         }
 939
 940         pi = &lcore_power_info[lcore_id];
 941
 942         pi->turbo_enable = 0;
 943
 944         if (pi->turbo_available && pi->curr_idx <= 1) {
 945                 /* Try to set freq to max by default coming out of turbo */
 946                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 947                         RTE_LOG(ERR, POWER,
 948                                 "Failed to set frequency of lcore %u to max\n",
 949                                 lcore_id);
 950                         return -1;
 951                 }
 952         }
 953
 954         return 0;
 955 }
 956
 957
 958 int power_pstate_get_capabilities(unsigned int lcore_id,
 959                 struct rte_power_core_capabilities *caps)
 960 {
 961         struct pstate_power_info *pi;
 962
 963         if (lcore_id >= RTE_MAX_LCORE) {
 964                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 965                 return -1;
 966         }
 967         if (caps == NULL) {
 968                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 969                 return -1;
 970         }
 971
 972         pi = &lcore_power_info[lcore_id];
 973         caps->capabilities = 0;
 974         caps->turbo = !!(pi->turbo_available);
 975         caps->priority = pi->priority_core;
 976
 977         return 0;
 978 }