lib/power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_memcpy.h>
  18 #include <rte_memory.h>
  19 #include <rte_string_fns.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24 /* macros used for rounding frequency to nearest 100000 */
  25 #define FREQ_ROUNDING_DELTA 50000
  26 #define ROUND_FREQ_TO_N_100000 100000
  27
  28 #define BUS_FREQ     100000
  29
  30 #define POWER_GOVERNOR_PERF "performance"
  31 #define POWER_SYSFILE_MAX_FREQ \
  32                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  33 #define POWER_SYSFILE_MIN_FREQ  \
  34                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  35 #define POWER_SYSFILE_CUR_FREQ  \
  36                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  37 #define POWER_SYSFILE_BASE_MAX_FREQ \
  38                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  39 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  40                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  41 #define POWER_SYSFILE_BASE_FREQ  \
  42                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  43 #define POWER_PSTATE_DRIVER "intel_pstate"
  44 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  45
  46 /*
  47  * MSR related
  48  */
  49 #define PLATFORM_INFO     0x0CE
  50 #define NON_TURBO_MASK    0xFF00
  51 #define NON_TURBO_OFFSET  0x8
  52
  53
  54 enum power_state {
  55         POWER_IDLE = 0,
  56         POWER_ONGOING,
  57         POWER_USED,
  58         POWER_UNKNOWN
  59 };
  60
  61 struct pstate_power_info {
  62         unsigned int lcore_id;               /**< Logical core id */
  63         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  64         uint32_t nb_freqs;                   /**< number of available freqs */
  65         FILE *f_cur_min;                     /**< FD of scaling_min */
  66         FILE *f_cur_max;                     /**< FD of scaling_max */
  67         char governor_ori[32];               /**< Original governor name */
  68         uint32_t curr_idx;                   /**< Freq index in freqs array */
  69         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
  70         uint32_t sys_max_freq;               /**< system wide max freq  */
  71         uint32_t core_base_freq;             /**< core base freq  */
  72         uint32_t state;                      /**< Power in use state */
  73         uint16_t turbo_available;            /**< Turbo Boost available */
  74         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
  75         uint16_t priority_core;              /**< High Performance core */
  76 } __rte_cache_aligned;
  77
  78
  79 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
  80
  81 /**
  82  * It is to read the specific MSR.
  83  */
  84
  85 static int32_t
  86 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
  87 {
  88         int fd, ret;
  89         char fullpath[PATH_MAX];
  90
  91         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
  92
  93         fd = open(fullpath, O_RDONLY);
  94
  95         if (fd < 0) {
  96                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
  97                                  strerror(errno));
  98                 return fd;
  99         }
 100
 101         ret = pread(fd, val, sizeof(uint64_t), msr);
 102
 103         if (ret < 0) {
 104                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 105                                  strerror(errno));
 106                 goto out;
 107         }
 108
 109         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 110                         fullpath, msr, lcore_id);
 111
 112         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 113
 114 out:    close(fd);
 115         return ret;
 116 }
 117
 118 /**
 119  * It is to fopen the sys file for the future setting the lcore frequency.
 120  */
 121 static int
 122 power_init_for_setting_freq(struct pstate_power_info *pi)
 123 {
 124         FILE *f_base = NULL, *f_base_max = NULL, *f_min = NULL, *f_max = NULL;
 125         uint32_t base_ratio, base_max_ratio;
 126         uint64_t max_non_turbo;
 127         int ret;
 128
 129         /* open all files we expect to have open */
 130         open_core_sysfs_file(&f_base_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 131                         pi->lcore_id);
 132         if (f_base_max == NULL) {
 133                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 134                                 POWER_SYSFILE_BASE_MAX_FREQ);
 135                 goto err;
 136         }
 137
 138         open_core_sysfs_file(&f_min, "rw+", POWER_SYSFILE_MIN_FREQ,
 139                         pi->lcore_id);
 140         if (f_min == NULL) {
 141                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 142                                 POWER_SYSFILE_MIN_FREQ);
 143                 goto err;
 144         }
 145
 146         open_core_sysfs_file(&f_max, "rw+", POWER_SYSFILE_MAX_FREQ,
 147                         pi->lcore_id);
 148         if (f_max == NULL) {
 149                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 150                                 POWER_SYSFILE_MAX_FREQ);
 151                 goto err;
 152         }
 153
 154         open_core_sysfs_file(&f_base, "r", POWER_SYSFILE_BASE_FREQ,
 155                         pi->lcore_id);
 156         /* base ratio file may not exist in some kernels, so no error check */
 157
 158         /* read base max ratio */
 159         ret = read_core_sysfs_u32(f_base_max, &base_max_ratio);
 160         if (ret < 0) {
 161                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 162                                 POWER_SYSFILE_BASE_MAX_FREQ);
 163                 goto err;
 164         }
 165
 166         /* base ratio may not exist */
 167         if (f_base != NULL) {
 168                 ret = read_core_sysfs_u32(f_base, &base_ratio);
 169                 if (ret < 0) {
 170                         RTE_LOG(ERR, POWER, "Failed to read %s\n",
 171                                         POWER_SYSFILE_BASE_FREQ);
 172                         goto err;
 173                 }
 174         } else {
 175                 base_ratio = 0;
 176         }
 177
 178         /* Add MSR read to detect turbo status */
 179         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0)
 180                 goto err;
 181         /* no errors after this point */
 182
 183         /* convert ratios to bins */
 184         base_max_ratio /= BUS_FREQ;
 185         base_ratio /= BUS_FREQ;
 186
 187         /* assign file handles */
 188         pi->f_cur_min = f_min;
 189         pi->f_cur_max = f_max;
 190
 191         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 192
 193         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 194
 195         pi->non_turbo_max_ratio = (uint32_t)max_non_turbo;
 196
 197         /*
 198          * If base_frequency is reported as greater than the maximum
 199          * turbo frequency, that's a known issue with some kernels.
 200          * Set base_frequency to max_non_turbo as a workaround.
 201          */
 202         if (base_ratio > base_max_ratio) {
 203                 /* base_ratio is greater than max turbo. Kernel bug. */
 204                 pi->priority_core = 0;
 205                 goto out;
 206         }
 207
 208         /*
 209          * If base_frequency is reported as greater than the maximum
 210          * non-turbo frequency, then mark it as a high priority core.
 211          */
 212         if (base_ratio > max_non_turbo)
 213                 pi->priority_core = 1;
 214         else
 215                 pi->priority_core = 0;
 216         pi->core_base_freq = base_ratio * BUS_FREQ;
 217
 218 out:
 219         if (f_base != NULL)
 220                 fclose(f_base);
 221         fclose(f_base_max);
 222         /* f_min and f_max are stored, no need to close */
 223         return 0;
 224
 225 err:
 226         if (f_base != NULL)
 227                 fclose(f_base);
 228         if (f_base_max != NULL)
 229                 fclose(f_base_max);
 230         if (f_min != NULL)
 231                 fclose(f_min);
 232         if (f_max != NULL)
 233                 fclose(f_max);
 234         return -1;
 235 }
 236
 237 static int
 238 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 239 {
 240         uint32_t target_freq = 0;
 241
 242         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 243                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 244                                 "should be less than %u\n", idx, pi->nb_freqs);
 245                 return -1;
 246         }
 247
 248         /* Check if it is the same as current */
 249         if (idx == pi->curr_idx)
 250                 return 0;
 251
 252         /* Because Intel Pstate Driver only allow user change min/max hint
 253          * User need change the min/max as same value.
 254          */
 255         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 256                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 257                                 "for setting frequency for lcore %u\n",
 258                                 pi->lcore_id);
 259                 return -1;
 260         }
 261
 262         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 263                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 264                                 "for setting frequency for lcore %u\n",
 265                                 pi->lcore_id);
 266                 return -1;
 267         }
 268
 269         /* Turbo is available and enabled, first freq bucket is sys max freq */
 270         if (pi->turbo_available && idx == 0) {
 271                 if (pi->turbo_enable)
 272                         target_freq = pi->sys_max_freq;
 273                 else {
 274                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 275                                         pi->lcore_id);
 276                         return -1;
 277                 }
 278         } else
 279                 target_freq = pi->freqs[idx];
 280
 281         /* Decrease freq, the min freq should be updated first */
 282         if (idx  >  pi->curr_idx) {
 283
 284                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 285                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 286                                         "lcore %u\n", pi->lcore_id);
 287                         return -1;
 288                 }
 289
 290                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 291                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 292                                         "lcore %u\n", pi->lcore_id);
 293                         return -1;
 294                 }
 295
 296                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 297                                   target_freq, pi->lcore_id);
 298
 299                 fflush(pi->f_cur_min);
 300                 fflush(pi->f_cur_max);
 301
 302         }
 303
 304         /* Increase freq, the max freq should be updated first */
 305         if (idx  <  pi->curr_idx) {
 306
 307                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 308                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 309                                         "lcore %u\n", pi->lcore_id);
 310                         return -1;
 311                 }
 312
 313                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 314                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 315                                         "lcore %u\n", pi->lcore_id);
 316                         return -1;
 317                 }
 318
 319                 POWER_DEBUG_TRACE("Frequency '%u' to be set for lcore %u\n",
 320                                   target_freq, pi->lcore_id);
 321
 322                 fflush(pi->f_cur_max);
 323                 fflush(pi->f_cur_min);
 324         }
 325
 326         pi->curr_idx = idx;
 327
 328         return 1;
 329 }
 330
 331 /**
 332  * It is to check the current scaling governor by reading sys file, and then
 333  * set it into 'performance' if it is not by writing the sys file. The original
 334  * governor will be saved for rolling back.
 335  */
 336 static int
 337 power_set_governor_performance(struct pstate_power_info *pi)
 338 {
 339         return power_set_governor(pi->lcore_id, POWER_GOVERNOR_PERF,
 340                         pi->governor_ori, sizeof(pi->governor_ori));
 341 }
 342
 343 /**
 344  * It is to check the governor and then set the original governor back if
 345  * needed by writing the sys file.
 346  */
 347 static int
 348 power_set_governor_original(struct pstate_power_info *pi)
 349 {
 350         return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
 351 }
 352
 353 /**
 354  * It is to get the available frequencies of the specific lcore by reading the
 355  * sys file.
 356  */
 357 static int
 358 power_get_available_freqs(struct pstate_power_info *pi)
 359 {
 360         FILE *f_min = NULL, *f_max = NULL;
 361         int ret = -1;
 362         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 363         uint32_t i, num_freqs = 0;
 364
 365         /* open all files */
 366         open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_BASE_MAX_FREQ,
 367                         pi->lcore_id);
 368         if (f_max == NULL) {
 369                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 370                                 POWER_SYSFILE_BASE_MAX_FREQ);
 371                 goto out;
 372         }
 373
 374         open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_BASE_MIN_FREQ,
 375                         pi->lcore_id);
 376         if (f_min == NULL) {
 377                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 378                                 POWER_SYSFILE_BASE_MIN_FREQ);
 379                 goto out;
 380         }
 381
 382         /* read base ratios */
 383         ret = read_core_sysfs_u32(f_max, &sys_max_freq);
 384         if (ret < 0) {
 385                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 386                                 POWER_SYSFILE_BASE_MAX_FREQ);
 387                 goto out;
 388         }
 389
 390         ret = read_core_sysfs_u32(f_min, &sys_min_freq);
 391         if (ret < 0) {
 392                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 393                                 POWER_SYSFILE_BASE_MIN_FREQ);
 394                 goto out;
 395         }
 396
 397         if (sys_max_freq < sys_min_freq)
 398                 goto out;
 399
 400         pi->sys_max_freq = sys_max_freq;
 401
 402         if (pi->priority_core == 1)
 403                 base_max_freq = pi->core_base_freq;
 404         else
 405                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 406
 407         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 408                         sys_min_freq,
 409                         sys_max_freq,
 410                         base_max_freq);
 411
 412         if (base_max_freq < sys_max_freq)
 413                 pi->turbo_available = 1;
 414         else
 415                 pi->turbo_available = 0;
 416
 417         /* If turbo is available then there is one extra freq bucket
 418          * to store the sys max freq which value is base_max +1
 419          */
 420         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 421                 pi->turbo_available;
 422
 423         /* Generate the freq bucket array.
 424          * If turbo is available the freq bucket[0] value is base_max +1
 425          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 426          * and so on.
 427          * If turbo is not available bucket[0] is base_max and so on
 428          */
 429         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 430                 if ((i == 0) && pi->turbo_available)
 431                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 432                 else
 433                         pi->freqs[pi->nb_freqs++] =
 434                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 435         }
 436
 437         ret = 0;
 438
 439         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 440                         num_freqs, pi->lcore_id);
 441
 442 out:
 443         fclose(f_min);
 444         fclose(f_max);
 445
 446         return ret;
 447 }
 448
 449 static int
 450 power_get_cur_idx(struct pstate_power_info *pi)
 451 {
 452         FILE *f_cur;
 453         int ret = -1;
 454         uint32_t sys_cur_freq = 0;
 455         unsigned int i;
 456
 457         open_core_sysfs_file(&f_cur, "r", POWER_SYSFILE_CUR_FREQ,
 458                         pi->lcore_id);
 459         if (f_cur == NULL) {
 460                 RTE_LOG(ERR, POWER, "failed to open %s\n",
 461                                 POWER_SYSFILE_CUR_FREQ);
 462                 goto fail;
 463         }
 464
 465         ret = read_core_sysfs_u32(f_cur, &sys_cur_freq);
 466         if (ret < 0) {
 467                 RTE_LOG(ERR, POWER, "Failed to read %s\n",
 468                                 POWER_SYSFILE_CUR_FREQ);
 469                 goto fail;
 470         }
 471
 472         /* convert the frequency to nearest 100000 value
 473          * Ex: if sys_cur_freq=1396789 then freq_conv=1400000
 474          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 475          * Ex: if sys_cur_freq=800030 then freq_conv=800000
 476          */
 477         unsigned int freq_conv = 0;
 478         freq_conv = (sys_cur_freq + FREQ_ROUNDING_DELTA)
 479                                 / ROUND_FREQ_TO_N_100000;
 480         freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
 481
 482         for (i = 0; i < pi->nb_freqs; i++) {
 483                 if (freq_conv == pi->freqs[i]) {
 484                         pi->curr_idx = i;
 485                         break;
 486                 }
 487         }
 488
 489         ret = 0;
 490 fail:
 491         if (f_cur != NULL)
 492                 fclose(f_cur);
 493         return ret;
 494 }
 495
 496 int
 497 power_pstate_cpufreq_check_supported(void)
 498 {
 499         return cpufreq_check_scaling_driver(POWER_PSTATE_DRIVER);
 500 }
 501
 502 int
 503 power_pstate_cpufreq_init(unsigned int lcore_id)
 504 {
 505         struct pstate_power_info *pi;
 506         uint32_t exp_state;
 507
 508         if (lcore_id >= RTE_MAX_LCORE) {
 509                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 510                                 lcore_id, RTE_MAX_LCORE - 1U);
 511                 return -1;
 512         }
 513
 514         pi = &lcore_power_info[lcore_id];
 515         exp_state = POWER_IDLE;
 516         /* The power in use state works as a guard variable between
 517          * the CPU frequency control initialization and exit process.
 518          * The ACQUIRE memory ordering here pairs with the RELEASE
 519          * ordering below as lock to make sure the frequency operations
 520          * in the critical section are done under the correct state.
 521          */
 522         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 523                                         POWER_ONGOING, 0,
 524                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 525                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 526                                 "in use\n", lcore_id);
 527                 return -1;
 528         }
 529
 530         pi->lcore_id = lcore_id;
 531         /* Check and set the governor */
 532         if (power_set_governor_performance(pi) < 0) {
 533                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 534                                 "performance\n", lcore_id);
 535                 goto fail;
 536         }
 537         /* Init for setting lcore frequency */
 538         if (power_init_for_setting_freq(pi) < 0) {
 539                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 540                                 "lcore %u\n", lcore_id);
 541                 goto fail;
 542         }
 543
 544         /* Get the available frequencies */
 545         if (power_get_available_freqs(pi) < 0) {
 546                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 547                                 "lcore %u\n", lcore_id);
 548                 goto fail;
 549         }
 550
 551         if (power_get_cur_idx(pi) < 0) {
 552                 RTE_LOG(ERR, POWER, "Cannot get current frequency "
 553                                 "index of lcore %u\n", lcore_id);
 554                 goto fail;
 555         }
 556
 557         /* Set freq to max by default */
 558         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 559                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 560                                 "to max\n", lcore_id);
 561                 goto fail;
 562         }
 563
 564         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 565                         "power management\n", lcore_id);
 566         exp_state = POWER_ONGOING;
 567         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_USED,
 568                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 569
 570         return 0;
 571
 572 fail:
 573         exp_state = POWER_ONGOING;
 574         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 575                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 576
 577         return -1;
 578 }
 579
 580 int
 581 power_pstate_cpufreq_exit(unsigned int lcore_id)
 582 {
 583         struct pstate_power_info *pi;
 584         uint32_t exp_state;
 585
 586         if (lcore_id >= RTE_MAX_LCORE) {
 587                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 588                                 lcore_id, RTE_MAX_LCORE - 1U);
 589                 return -1;
 590         }
 591         pi = &lcore_power_info[lcore_id];
 592
 593         exp_state = POWER_USED;
 594         /* The power in use state works as a guard variable between
 595          * the CPU frequency control initialization and exit process.
 596          * The ACQUIRE memory ordering here pairs with the RELEASE
 597          * ordering below as lock to make sure the frequency operations
 598          * in the critical section are under done the correct state.
 599          */
 600         if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
 601                                         POWER_ONGOING, 0,
 602                                         __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
 603                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 604                                 "not used\n", lcore_id);
 605                 return -1;
 606         }
 607
 608         /* Close FD of setting freq */
 609         fclose(pi->f_cur_min);
 610         fclose(pi->f_cur_max);
 611         pi->f_cur_min = NULL;
 612         pi->f_cur_max = NULL;
 613
 614         /* Set the governor back to the original */
 615         if (power_set_governor_original(pi) < 0) {
 616                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 617                                 "to the original\n", lcore_id);
 618                 goto fail;
 619         }
 620
 621         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 622                         "'performance' mode and been set back to the "
 623                         "original\n", lcore_id);
 624         exp_state = POWER_ONGOING;
 625         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_IDLE,
 626                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 627
 628         return 0;
 629
 630 fail:
 631         exp_state = POWER_ONGOING;
 632         __atomic_compare_exchange_n(&(pi->state), &exp_state, POWER_UNKNOWN,
 633                                     0, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 634
 635         return -1;
 636 }
 637
 638
 639 uint32_t
 640 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 641 {
 642         struct pstate_power_info *pi;
 643
 644         if (lcore_id >= RTE_MAX_LCORE) {
 645                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 646                 return 0;
 647         }
 648
 649         if (freqs == NULL) {
 650                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 651                 return 0;
 652         }
 653
 654         pi = &lcore_power_info[lcore_id];
 655         if (num < pi->nb_freqs) {
 656                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 657                 return 0;
 658         }
 659         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 660
 661         return pi->nb_freqs;
 662 }
 663
 664 uint32_t
 665 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 666 {
 667         if (lcore_id >= RTE_MAX_LCORE) {
 668                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 669                 return RTE_POWER_INVALID_FREQ_INDEX;
 670         }
 671
 672         return lcore_power_info[lcore_id].curr_idx;
 673 }
 674
 675
 676 int
 677 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 678 {
 679         if (lcore_id >= RTE_MAX_LCORE) {
 680                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 681                 return -1;
 682         }
 683
 684         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 685 }
 686
 687 int
 688 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 689 {
 690         struct pstate_power_info *pi;
 691
 692         if (lcore_id >= RTE_MAX_LCORE) {
 693                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 694                 return -1;
 695         }
 696
 697         pi = &lcore_power_info[lcore_id];
 698         if (pi->curr_idx == 0 ||
 699             (pi->curr_idx == 1 && pi->turbo_available && !pi->turbo_enable))
 700                 return 0;
 701
 702         /* Frequencies in the array are from high to low. */
 703         return set_freq_internal(pi, pi->curr_idx - 1);
 704 }
 705
 706 int
 707 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 708 {
 709         struct pstate_power_info *pi;
 710
 711         if (lcore_id >= RTE_MAX_LCORE) {
 712                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 713                 return -1;
 714         }
 715
 716         pi = &lcore_power_info[lcore_id];
 717         if (pi->curr_idx + 1 == pi->nb_freqs)
 718                 return 0;
 719
 720         /* Frequencies in the array are from high to low. */
 721         return set_freq_internal(pi, pi->curr_idx + 1);
 722 }
 723
 724 int
 725 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 726 {
 727         if (lcore_id >= RTE_MAX_LCORE) {
 728                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 729                 return -1;
 730         }
 731
 732         /* Frequencies in the array are from high to low. */
 733         if (lcore_power_info[lcore_id].turbo_available) {
 734                 if (lcore_power_info[lcore_id].turbo_enable)
 735                         /* Set to Turbo */
 736                         return set_freq_internal(
 737                                         &lcore_power_info[lcore_id], 0);
 738                 else
 739                         /* Set to max non-turbo */
 740                         return set_freq_internal(
 741                                         &lcore_power_info[lcore_id], 1);
 742         } else
 743                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 744 }
 745
 746
 747 int
 748 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 749 {
 750         struct pstate_power_info *pi;
 751
 752         if (lcore_id >= RTE_MAX_LCORE) {
 753                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 754                 return -1;
 755         }
 756
 757         pi = &lcore_power_info[lcore_id];
 758
 759         /* Frequencies in the array are from high to low. */
 760         return set_freq_internal(pi, pi->nb_freqs - 1);
 761 }
 762
 763
 764 int
 765 power_pstate_turbo_status(unsigned int lcore_id)
 766 {
 767         struct pstate_power_info *pi;
 768
 769         if (lcore_id >= RTE_MAX_LCORE) {
 770                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 771                 return -1;
 772         }
 773
 774         pi = &lcore_power_info[lcore_id];
 775
 776         return pi->turbo_enable;
 777 }
 778
 779 int
 780 power_pstate_enable_turbo(unsigned int lcore_id)
 781 {
 782         struct pstate_power_info *pi;
 783
 784         if (lcore_id >= RTE_MAX_LCORE) {
 785                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 786                 return -1;
 787         }
 788
 789         pi = &lcore_power_info[lcore_id];
 790
 791         if (pi->turbo_available)
 792                 pi->turbo_enable = 1;
 793         else {
 794                 pi->turbo_enable = 0;
 795                 RTE_LOG(ERR, POWER,
 796                         "Failed to enable turbo on lcore %u\n",
 797                         lcore_id);
 798                         return -1;
 799         }
 800
 801         return 0;
 802 }
 803
 804
 805 int
 806 power_pstate_disable_turbo(unsigned int lcore_id)
 807 {
 808         struct pstate_power_info *pi;
 809
 810         if (lcore_id >= RTE_MAX_LCORE) {
 811                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 812                 return -1;
 813         }
 814
 815         pi = &lcore_power_info[lcore_id];
 816
 817         pi->turbo_enable = 0;
 818
 819         if (pi->turbo_available && pi->curr_idx <= 1) {
 820                 /* Try to set freq to max by default coming out of turbo */
 821                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 822                         RTE_LOG(ERR, POWER,
 823                                 "Failed to set frequency of lcore %u to max\n",
 824                                 lcore_id);
 825                         return -1;
 826                 }
 827         }
 828
 829         return 0;
 830 }
 831
 832
 833 int power_pstate_get_capabilities(unsigned int lcore_id,
 834                 struct rte_power_core_capabilities *caps)
 835 {
 836         struct pstate_power_info *pi;
 837
 838         if (lcore_id >= RTE_MAX_LCORE) {
 839                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 840                 return -1;
 841         }
 842         if (caps == NULL) {
 843                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 844                 return -1;
 845         }
 846
 847         pi = &lcore_power_info[lcore_id];
 848         caps->capabilities = 0;
 849         caps->turbo = !!(pi->turbo_available);
 850         caps->priority = pi->priority_core;
 851
 852         return 0;
 853 }