lib/librte_power/power_pstate_cpufreq.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <sys/types.h>
   7 #include <sys/stat.h>
   8 #include <fcntl.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <unistd.h>
  12 #include <signal.h>
  13 #include <limits.h>
  14 #include <errno.h>
  15 #include <inttypes.h>
  16
  17 #include <rte_string_fns.h>
  18 #include <rte_memcpy.h>
  19 #include <rte_atomic.h>
  20
  21 #include "power_pstate_cpufreq.h"
  22 #include "power_common.h"
  23
  24
  25 #ifdef RTE_LIBRTE_POWER_DEBUG
  26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
  27                 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
  28 } while (0)
  29 #else
  30 #define POWER_DEBUG_TRACE(fmt, args...)
  31 #endif
  32
  33 #define FOPEN_OR_ERR_RET(f, retval) do { \
  34                 if ((f) == NULL) { \
  35                         RTE_LOG(ERR, POWER, "File not openned\n"); \
  36                         return retval; \
  37                 } \
  38 } while (0)
  39
  40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
  41                 if ((ret) == NULL) { \
  42                         RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
  43                         goto label; \
  44                 } \
  45 } while (0)
  46
  47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
  48                 if ((ret) < 0) { \
  49                         RTE_LOG(ERR, POWER, "File operations failed\n"); \
  50                         goto label; \
  51                 } \
  52 } while (0)
  53
  54
  55 #define POWER_CONVERT_TO_DECIMAL 10
  56 #define BUS_FREQ     100000
  57
  58 #define POWER_GOVERNOR_PERF "performance"
  59 #define POWER_SYSFILE_GOVERNOR  \
  60                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
  61 #define POWER_SYSFILE_MAX_FREQ \
  62                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
  63 #define POWER_SYSFILE_MIN_FREQ  \
  64                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
  65 #define POWER_SYSFILE_CUR_FREQ  \
  66                 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
  67 #define POWER_SYSFILE_BASE_MAX_FREQ \
  68                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
  69 #define POWER_SYSFILE_BASE_MIN_FREQ  \
  70                 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
  71 #define POWER_SYSFILE_BASE_FREQ  \
  72                 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
  73 #define POWER_MSR_PATH  "/dev/cpu/%u/msr"
  74
  75 /*
  76  * MSR related
  77  */
  78 #define PLATFORM_INFO     0x0CE
  79 #define NON_TURBO_MASK    0xFF00
  80 #define NON_TURBO_OFFSET  0x8
  81
  82
  83 enum power_state {
  84         POWER_IDLE = 0,
  85         POWER_ONGOING,
  86         POWER_USED,
  87         POWER_UNKNOWN
  88 };
  89
  90 struct pstate_power_info {
  91         unsigned int lcore_id;               /**< Logical core id */
  92         uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
  93         uint32_t nb_freqs;                   /**< number of available freqs */
  94         FILE *f_cur_min;                     /**< FD of scaling_min */
  95         FILE *f_cur_max;                     /**< FD of scaling_max */
  96         char governor_ori[32];               /**< Original governor name */
  97         uint32_t curr_idx;                   /**< Freq index in freqs array */
  98         uint32_t non_turbo_max_ratio;        /**< Non Turbo Max ratio  */
  99         uint32_t sys_max_freq;               /**< system wide max freq  */
 100         uint32_t core_base_freq;             /**< core base freq  */
 101         volatile uint32_t state;             /**< Power in use state */
 102         uint16_t turbo_available;            /**< Turbo Boost available */
 103         uint16_t turbo_enable;               /**< Turbo Boost enable/disable */
 104         uint16_t priority_core;              /**< High Performance core */
 105 } __rte_cache_aligned;
 106
 107
 108 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
 109
 110 /**
 111  * It is to read the specific MSR.
 112  */
 113
 114 static int32_t
 115 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
 116 {
 117         int fd, ret;
 118         char fullpath[PATH_MAX];
 119
 120         snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
 121
 122         fd = open(fullpath, O_RDONLY);
 123
 124         if (fd < 0) {
 125                 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
 126                                  strerror(errno));
 127                 return fd;
 128         }
 129
 130         ret = pread(fd, val, sizeof(uint64_t), msr);
 131
 132         if (ret < 0) {
 133                 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
 134                                  strerror(errno));
 135                 goto out;
 136         }
 137
 138         POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
 139                         fullpath, msr, lcore_id);
 140
 141         POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
 142
 143 out:    close(fd);
 144         return ret;
 145 }
 146
 147 /**
 148  * It is to fopen the sys file for the future setting the lcore frequency.
 149  */
 150 static int
 151 power_init_for_setting_freq(struct pstate_power_info *pi)
 152 {
 153         FILE *f_min, *f_max, *f_base;
 154         char fullpath_min[PATH_MAX];
 155         char fullpath_max[PATH_MAX];
 156         char fullpath_base[PATH_MAX];
 157         char buf_base[BUFSIZ];
 158         char *s_base;
 159         uint32_t base_ratio = 0;
 160         uint64_t max_non_turbo = 0;
 161         int  ret_val = 0;
 162
 163         snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
 164                         pi->lcore_id);
 165
 166         f_min = fopen(fullpath_min, "rw+");
 167         FOPEN_OR_ERR_RET(f_min, -1);
 168
 169         snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
 170                         pi->lcore_id);
 171
 172         f_max = fopen(fullpath_max, "rw+");
 173         if (f_max == NULL)
 174                 fclose(f_min);
 175
 176         FOPEN_OR_ERR_RET(f_max, -1);
 177
 178         pi->f_cur_min = f_min;
 179         pi->f_cur_max = f_max;
 180
 181         snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
 182                         pi->lcore_id);
 183
 184         f_base = fopen(fullpath_base, "r");
 185         if (f_base == NULL) {
 186                 /* No sysfs base_frequency, that's OK, continue without */
 187                 base_ratio = 0;
 188         } else {
 189                 s_base = fgets(buf_base, sizeof(buf_base), f_base);
 190                 FOPS_OR_NULL_GOTO(s_base, out);
 191
 192                 buf_base[BUFSIZ-1] = '\0';
 193                 if (strlen(buf_base))
 194                         /* Strip off terminating '\n' */
 195                         strtok(buf_base, "\n");
 196
 197                 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
 198                                 / BUS_FREQ;
 199         }
 200
 201         /* Add MSR read to detect turbo status */
 202
 203         if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
 204                 ret_val = -1;
 205                 goto out;
 206         }
 207
 208         max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
 209
 210         POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
 211
 212         pi->non_turbo_max_ratio = max_non_turbo;
 213
 214         /*
 215          * If base_frequency is reported as greater than the maximum
 216          * non-turbo frequency, then mark it as a high priority core.
 217          */
 218         if (base_ratio > max_non_turbo)
 219                 pi->priority_core = 1;
 220         else
 221                 pi->priority_core = 0;
 222         pi->core_base_freq = base_ratio * BUS_FREQ;
 223
 224 out:
 225         if (f_base != NULL)
 226                 fclose(f_base);
 227         return ret_val;
 228 }
 229
 230 static int
 231 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
 232 {
 233         uint32_t target_freq = 0;
 234
 235         if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
 236                 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
 237                                 "should be less than %u\n", idx, pi->nb_freqs);
 238                 return -1;
 239         }
 240
 241         /* Check if it is the same as current */
 242         if (idx == pi->curr_idx)
 243                 return 0;
 244
 245         /* Because Intel Pstate Driver only allow user change min/max hint
 246          * User need change the min/max as same value.
 247          */
 248         if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
 249                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 250                                 "for setting frequency for lcore %u\n",
 251                                 pi->lcore_id);
 252                 return -1;
 253         }
 254
 255         if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
 256                 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
 257                                 "for setting frequency for lcore %u\n",
 258                                 pi->lcore_id);
 259                 return -1;
 260         }
 261
 262         /* Turbo is available and enabled, first freq bucket is sys max freq */
 263         if (pi->turbo_available && idx == 0) {
 264                 if (pi->turbo_enable)
 265                         target_freq = pi->sys_max_freq;
 266                 else {
 267                         RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
 268                                         pi->lcore_id);
 269                         return -1;
 270                 }
 271         } else
 272                 target_freq = pi->freqs[idx];
 273
 274         /* Decrease freq, the min freq should be updated first */
 275         if (idx  >  pi->curr_idx) {
 276
 277                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 278                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 279                                         "lcore %u\n", pi->lcore_id);
 280                         return -1;
 281                 }
 282
 283                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 284                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 285                                         "lcore %u\n", pi->lcore_id);
 286                         return -1;
 287                 }
 288
 289                 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
 290                                   target_freq, pi->lcore_id);
 291
 292                 fflush(pi->f_cur_min);
 293                 fflush(pi->f_cur_max);
 294
 295         }
 296
 297         /* Increase freq, the max freq should be updated first */
 298         if (idx  <  pi->curr_idx) {
 299
 300                 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
 301                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 302                                         "lcore %u\n", pi->lcore_id);
 303                         return -1;
 304                 }
 305
 306                 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
 307                         RTE_LOG(ERR, POWER, "Fail to write new frequency for "
 308                                         "lcore %u\n", pi->lcore_id);
 309                         return -1;
 310                 }
 311
 312                 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
 313                                   target_freq, pi->lcore_id);
 314
 315                 fflush(pi->f_cur_max);
 316                 fflush(pi->f_cur_min);
 317         }
 318
 319         pi->curr_idx = idx;
 320
 321         return 1;
 322 }
 323
 324 /**
 325  * It is to check the current scaling governor by reading sys file, and then
 326  * set it into 'performance' if it is not by writing the sys file. The original
 327  * governor will be saved for rolling back.
 328  */
 329 static int
 330 power_set_governor_performance(struct pstate_power_info *pi)
 331 {
 332         FILE *f;
 333         int ret = -1;
 334         char buf[BUFSIZ];
 335         char fullpath[PATH_MAX];
 336         char *s;
 337         int val;
 338
 339         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 340                         pi->lcore_id);
 341         f = fopen(fullpath, "rw+");
 342         FOPEN_OR_ERR_RET(f, ret);
 343
 344         s = fgets(buf, sizeof(buf), f);
 345         FOPS_OR_NULL_GOTO(s, out);
 346         /* Strip off terminating '\n' */
 347         strtok(buf, "\n");
 348
 349         /* Check if current governor is performance */
 350         if (strncmp(buf, POWER_GOVERNOR_PERF,
 351                         sizeof(POWER_GOVERNOR_PERF)) == 0) {
 352                 ret = 0;
 353                 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
 354                                 "already performance\n", pi->lcore_id);
 355                 goto out;
 356         }
 357         /* Save the original governor */
 358         strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
 359
 360         /* Write 'performance' to the governor */
 361         val = fseek(f, 0, SEEK_SET);
 362         FOPS_OR_ERR_GOTO(val, out);
 363
 364         val = fputs(POWER_GOVERNOR_PERF, f);
 365         FOPS_OR_ERR_GOTO(val, out);
 366
 367         /* We need to flush to see if the fputs succeeds */
 368         val = fflush(f);
 369         FOPS_OR_ERR_GOTO(val, out);
 370
 371         ret = 0;
 372         RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
 373                         "set to performance successfully\n", pi->lcore_id);
 374 out:
 375         fclose(f);
 376
 377         return ret;
 378 }
 379
 380 /**
 381  * It is to check the governor and then set the original governor back if
 382  * needed by writing the sys file.
 383  */
 384 static int
 385 power_set_governor_original(struct pstate_power_info *pi)
 386 {
 387         FILE *f;
 388         int ret = -1;
 389         char buf[BUFSIZ];
 390         char fullpath[PATH_MAX];
 391         char *s;
 392         int val;
 393
 394         snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
 395                         pi->lcore_id);
 396         f = fopen(fullpath, "rw+");
 397         FOPEN_OR_ERR_RET(f, ret);
 398
 399         s = fgets(buf, sizeof(buf), f);
 400         FOPS_OR_NULL_GOTO(s, out);
 401
 402         /* Check if the governor to be set is the same as current */
 403         if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
 404                 ret = 0;
 405                 POWER_DEBUG_TRACE("Power management governor of lcore %u "
 406                                 "has already been set to %s\n",
 407                                 pi->lcore_id, pi->governor_ori);
 408                 goto out;
 409         }
 410
 411         /* Write back the original governor */
 412         val = fseek(f, 0, SEEK_SET);
 413         FOPS_OR_ERR_GOTO(val, out);
 414
 415         val = fputs(pi->governor_ori, f);
 416         FOPS_OR_ERR_GOTO(val, out);
 417
 418         ret = 0;
 419         RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
 420                         "has been set back to %s successfully\n",
 421                         pi->lcore_id, pi->governor_ori);
 422 out:
 423         fclose(f);
 424
 425         return ret;
 426 }
 427
 428 /**
 429  * It is to get the available frequencies of the specific lcore by reading the
 430  * sys file.
 431  */
 432 static int
 433 power_get_available_freqs(struct pstate_power_info *pi)
 434 {
 435         FILE *f_min, *f_max;
 436         int ret = -1;
 437         char *p_min, *p_max;
 438         char buf_min[BUFSIZ];
 439         char buf_max[BUFSIZ];
 440         char fullpath_min[PATH_MAX];
 441         char fullpath_max[PATH_MAX];
 442         char *s_min, *s_max;
 443         uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
 444         uint32_t i, num_freqs = 0;
 445
 446         snprintf(fullpath_max, sizeof(fullpath_max),
 447                         POWER_SYSFILE_BASE_MAX_FREQ,
 448                         pi->lcore_id);
 449         snprintf(fullpath_min, sizeof(fullpath_min),
 450                         POWER_SYSFILE_BASE_MIN_FREQ,
 451                         pi->lcore_id);
 452
 453         f_min = fopen(fullpath_min, "r");
 454         FOPEN_OR_ERR_RET(f_min, ret);
 455
 456         f_max = fopen(fullpath_max, "r");
 457         if (f_max == NULL)
 458                 fclose(f_min);
 459
 460         FOPEN_OR_ERR_RET(f_max, ret);
 461
 462         s_min = fgets(buf_min, sizeof(buf_min), f_min);
 463         FOPS_OR_NULL_GOTO(s_min, out);
 464
 465         s_max = fgets(buf_max, sizeof(buf_max), f_max);
 466         FOPS_OR_NULL_GOTO(s_max, out);
 467
 468
 469         /* Strip the line break if there is */
 470         p_min = strchr(buf_min, '\n');
 471         if (p_min != NULL)
 472                 *p_min = 0;
 473
 474         p_max = strchr(buf_max, '\n');
 475         if (p_max != NULL)
 476                 *p_max = 0;
 477
 478         sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
 479         sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
 480
 481         if (sys_max_freq < sys_min_freq)
 482                 goto out;
 483
 484         pi->sys_max_freq = sys_max_freq;
 485
 486         if (pi->priority_core == 1)
 487                 base_max_freq = pi->core_base_freq;
 488         else
 489                 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
 490
 491         POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
 492                         sys_min_freq,
 493                         sys_max_freq,
 494                         base_max_freq);
 495
 496         if (base_max_freq < sys_max_freq)
 497                 pi->turbo_available = 1;
 498         else
 499                 pi->turbo_available = 0;
 500
 501         /* If turbo is available then there is one extra freq bucket
 502          * to store the sys max freq which value is base_max +1
 503          */
 504         num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
 505                 pi->turbo_available;
 506
 507         /* Generate the freq bucket array.
 508          * If turbo is available the freq bucket[0] value is base_max +1
 509          * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
 510          * and so on.
 511          * If turbo is not available bucket[0] is base_max and so on
 512          */
 513         for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
 514                 if ((i == 0) && pi->turbo_available)
 515                         pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
 516                 else
 517                         pi->freqs[pi->nb_freqs++] =
 518                         base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
 519         }
 520
 521         ret = 0;
 522
 523         POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
 524                         num_freqs, pi->lcore_id);
 525
 526 out:
 527         fclose(f_min);
 528         fclose(f_max);
 529
 530         return ret;
 531 }
 532
 533 int
 534 power_pstate_cpufreq_init(unsigned int lcore_id)
 535 {
 536         struct pstate_power_info *pi;
 537
 538         if (lcore_id >= RTE_MAX_LCORE) {
 539                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
 540                                 lcore_id, RTE_MAX_LCORE - 1U);
 541                 return -1;
 542         }
 543
 544         pi = &lcore_power_info[lcore_id];
 545         if (rte_atomic32_cmpset(&(pi->state), POWER_IDLE, POWER_ONGOING)
 546                         == 0) {
 547                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 548                                 "in use\n", lcore_id);
 549                 return -1;
 550         }
 551
 552         pi->lcore_id = lcore_id;
 553         /* Check and set the governor */
 554         if (power_set_governor_performance(pi) < 0) {
 555                 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
 556                                 "performance\n", lcore_id);
 557                 goto fail;
 558         }
 559         /* Init for setting lcore frequency */
 560         if (power_init_for_setting_freq(pi) < 0) {
 561                 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
 562                                 "lcore %u\n", lcore_id);
 563                 goto fail;
 564         }
 565
 566         /* Get the available frequencies */
 567         if (power_get_available_freqs(pi) < 0) {
 568                 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
 569                                 "lcore %u\n", lcore_id);
 570                 goto fail;
 571         }
 572
 573
 574         /* Set freq to max by default */
 575         if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 576                 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
 577                                 "to max\n", lcore_id);
 578                 goto fail;
 579         }
 580
 581         RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
 582                         "power management\n", lcore_id);
 583         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_USED);
 584
 585         return 0;
 586
 587 fail:
 588         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
 589
 590         return -1;
 591 }
 592
 593 int
 594 power_pstate_cpufreq_exit(unsigned int lcore_id)
 595 {
 596         struct pstate_power_info *pi;
 597
 598         if (lcore_id >= RTE_MAX_LCORE) {
 599                 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
 600                                 lcore_id, RTE_MAX_LCORE - 1U);
 601                 return -1;
 602         }
 603         pi = &lcore_power_info[lcore_id];
 604
 605         if (rte_atomic32_cmpset(&(pi->state), POWER_USED, POWER_ONGOING)
 606                         == 0) {
 607                 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
 608                                 "not used\n", lcore_id);
 609                 return -1;
 610         }
 611
 612         /* Close FD of setting freq */
 613         fclose(pi->f_cur_min);
 614         fclose(pi->f_cur_max);
 615         pi->f_cur_min = NULL;
 616         pi->f_cur_max = NULL;
 617
 618         /* Set the governor back to the original */
 619         if (power_set_governor_original(pi) < 0) {
 620                 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
 621                                 "to the original\n", lcore_id);
 622                 goto fail;
 623         }
 624
 625         RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
 626                         "'performance' mode and been set back to the "
 627                         "original\n", lcore_id);
 628         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_IDLE);
 629
 630         return 0;
 631
 632 fail:
 633         rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
 634
 635         return -1;
 636 }
 637
 638
 639 uint32_t
 640 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
 641 {
 642         struct pstate_power_info *pi;
 643
 644         if (lcore_id >= RTE_MAX_LCORE) {
 645                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 646                 return 0;
 647         }
 648
 649         if (freqs == NULL) {
 650                 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
 651                 return 0;
 652         }
 653
 654         pi = &lcore_power_info[lcore_id];
 655         if (num < pi->nb_freqs) {
 656                 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
 657                 return 0;
 658         }
 659         rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
 660
 661         return pi->nb_freqs;
 662 }
 663
 664 uint32_t
 665 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
 666 {
 667         if (lcore_id >= RTE_MAX_LCORE) {
 668                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 669                 return RTE_POWER_INVALID_FREQ_INDEX;
 670         }
 671
 672         return lcore_power_info[lcore_id].curr_idx;
 673 }
 674
 675
 676 int
 677 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
 678 {
 679         if (lcore_id >= RTE_MAX_LCORE) {
 680                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 681                 return -1;
 682         }
 683
 684         return set_freq_internal(&(lcore_power_info[lcore_id]), index);
 685 }
 686
 687 int
 688 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
 689 {
 690         struct pstate_power_info *pi;
 691
 692         if (lcore_id >= RTE_MAX_LCORE) {
 693                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 694                 return -1;
 695         }
 696
 697         pi = &lcore_power_info[lcore_id];
 698         if (pi->curr_idx == 0)
 699                 return 0;
 700
 701         /* Frequencies in the array are from high to low. */
 702         return set_freq_internal(pi, pi->curr_idx - 1);
 703 }
 704
 705 int
 706 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
 707 {
 708         struct pstate_power_info *pi;
 709
 710         if (lcore_id >= RTE_MAX_LCORE) {
 711                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 712                 return -1;
 713         }
 714
 715         pi = &lcore_power_info[lcore_id];
 716         if (pi->curr_idx + 1 == pi->nb_freqs)
 717                 return 0;
 718
 719         /* Frequencies in the array are from high to low. */
 720         return set_freq_internal(pi, pi->curr_idx + 1);
 721 }
 722
 723 int
 724 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
 725 {
 726         if (lcore_id >= RTE_MAX_LCORE) {
 727                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 728                 return -1;
 729         }
 730
 731         /* Frequencies in the array are from high to low. */
 732         if (lcore_power_info[lcore_id].turbo_available) {
 733                 if (lcore_power_info[lcore_id].turbo_enable)
 734                         /* Set to Turbo */
 735                         return set_freq_internal(
 736                                         &lcore_power_info[lcore_id], 0);
 737                 else
 738                         /* Set to max non-turbo */
 739                         return set_freq_internal(
 740                                         &lcore_power_info[lcore_id], 1);
 741         } else
 742                 return set_freq_internal(&lcore_power_info[lcore_id], 0);
 743 }
 744
 745
 746 int
 747 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
 748 {
 749         struct pstate_power_info *pi;
 750
 751         if (lcore_id >= RTE_MAX_LCORE) {
 752                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 753                 return -1;
 754         }
 755
 756         pi = &lcore_power_info[lcore_id];
 757
 758         /* Frequencies in the array are from high to low. */
 759         return set_freq_internal(pi, pi->nb_freqs - 1);
 760 }
 761
 762
 763 int
 764 power_pstate_turbo_status(unsigned int lcore_id)
 765 {
 766         struct pstate_power_info *pi;
 767
 768         if (lcore_id >= RTE_MAX_LCORE) {
 769                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 770                 return -1;
 771         }
 772
 773         pi = &lcore_power_info[lcore_id];
 774
 775         return pi->turbo_enable;
 776 }
 777
 778 int
 779 power_pstate_enable_turbo(unsigned int lcore_id)
 780 {
 781         struct pstate_power_info *pi;
 782
 783         if (lcore_id >= RTE_MAX_LCORE) {
 784                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 785                 return -1;
 786         }
 787
 788         pi = &lcore_power_info[lcore_id];
 789
 790         if (pi->turbo_available)
 791                 pi->turbo_enable = 1;
 792         else {
 793                 pi->turbo_enable = 0;
 794                 RTE_LOG(ERR, POWER,
 795                         "Failed to enable turbo on lcore %u\n",
 796                         lcore_id);
 797                         return -1;
 798         }
 799
 800         return 0;
 801 }
 802
 803
 804 int
 805 power_pstate_disable_turbo(unsigned int lcore_id)
 806 {
 807         struct pstate_power_info *pi;
 808
 809         if (lcore_id >= RTE_MAX_LCORE) {
 810                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 811                 return -1;
 812         }
 813
 814         pi = &lcore_power_info[lcore_id];
 815
 816         pi->turbo_enable = 0;
 817
 818         if (pi->turbo_available && pi->curr_idx <= 1) {
 819                 /* Try to set freq to max by default coming out of turbo */
 820                 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
 821                         RTE_LOG(ERR, POWER,
 822                                 "Failed to set frequency of lcore %u to max\n",
 823                                 lcore_id);
 824                         return -1;
 825                 }
 826         }
 827
 828         return 0;
 829 }
 830
 831
 832 int power_pstate_get_capabilities(unsigned int lcore_id,
 833                 struct rte_power_core_capabilities *caps)
 834 {
 835         struct pstate_power_info *pi;
 836
 837         if (lcore_id >= RTE_MAX_LCORE) {
 838                 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
 839                 return -1;
 840         }
 841         if (caps == NULL) {
 842                 RTE_LOG(ERR, POWER, "Invalid argument\n");
 843                 return -1;
 844         }
 845
 846         pi = &lcore_power_info[lcore_id];
 847         caps->capabilities = 0;
 848         caps->turbo = !!(pi->turbo_available);
 849         caps->priority = pi->priority_core;
 850
 851         return 0;
 852 }