1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
17 #include <rte_string_fns.h>
18 #include <rte_memcpy.h>
19 #include <rte_atomic.h>
21 #include "power_pstate_cpufreq.h"
22 #include "power_common.h"
25 #ifdef RTE_LIBRTE_POWER_DEBUG
26 #define POWER_DEBUG_TRACE(fmt, args...) do { \
27 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
30 #define POWER_DEBUG_TRACE(fmt, args...)
33 #define FOPEN_OR_ERR_RET(f, retval) do { \
35 RTE_LOG(ERR, POWER, "File not openned\n"); \
40 #define FOPS_OR_NULL_GOTO(ret, label) do { \
41 if ((ret) == NULL) { \
42 RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
47 #define FOPS_OR_ERR_GOTO(ret, label) do { \
49 RTE_LOG(ERR, POWER, "File operations failed\n"); \
55 #define POWER_CONVERT_TO_DECIMAL 10
56 #define BUS_FREQ 100000
58 #define POWER_GOVERNOR_PERF "performance"
59 #define POWER_SYSFILE_GOVERNOR \
60 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
61 #define POWER_SYSFILE_MAX_FREQ \
62 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
63 #define POWER_SYSFILE_MIN_FREQ \
64 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
65 #define POWER_SYSFILE_CUR_FREQ \
66 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
67 #define POWER_SYSFILE_BASE_MAX_FREQ \
68 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
69 #define POWER_SYSFILE_BASE_MIN_FREQ \
70 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
71 #define POWER_SYSFILE_BASE_FREQ \
72 "/sys/devices/system/cpu/cpu%u/cpufreq/base_frequency"
73 #define POWER_MSR_PATH "/dev/cpu/%u/msr"
78 #define PLATFORM_INFO 0x0CE
79 #define NON_TURBO_MASK 0xFF00
80 #define NON_TURBO_OFFSET 0x8
90 struct pstate_power_info {
91 unsigned int lcore_id; /**< Logical core id */
92 uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
93 uint32_t nb_freqs; /**< number of available freqs */
94 FILE *f_cur_min; /**< FD of scaling_min */
95 FILE *f_cur_max; /**< FD of scaling_max */
96 char governor_ori[32]; /**< Original governor name */
97 uint32_t curr_idx; /**< Freq index in freqs array */
98 uint32_t non_turbo_max_ratio; /**< Non Turbo Max ratio */
99 uint32_t sys_max_freq; /**< system wide max freq */
100 uint32_t core_base_freq; /**< core base freq */
101 volatile uint32_t state; /**< Power in use state */
102 uint16_t turbo_available; /**< Turbo Boost available */
103 uint16_t turbo_enable; /**< Turbo Boost enable/disable */
104 uint16_t priority_core; /**< High Performance core */
105 } __rte_cache_aligned;
108 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
111 * It is to read the specific MSR.
115 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
118 char fullpath[PATH_MAX];
120 snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
122 fd = open(fullpath, O_RDONLY);
125 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
130 ret = pread(fd, val, sizeof(uint64_t), msr);
133 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
138 POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
139 fullpath, msr, lcore_id);
141 POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
148 * It is to fopen the sys file for the future setting the lcore frequency.
151 power_init_for_setting_freq(struct pstate_power_info *pi)
153 FILE *f_min, *f_max, *f_base;
154 char fullpath_min[PATH_MAX];
155 char fullpath_max[PATH_MAX];
156 char fullpath_base[PATH_MAX];
157 char buf_base[BUFSIZ];
159 uint32_t base_ratio = 0;
160 uint64_t max_non_turbo = 0;
163 snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
166 f_min = fopen(fullpath_min, "rw+");
167 FOPEN_OR_ERR_RET(f_min, -1);
169 snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
172 f_max = fopen(fullpath_max, "rw+");
176 FOPEN_OR_ERR_RET(f_max, -1);
178 pi->f_cur_min = f_min;
179 pi->f_cur_max = f_max;
181 snprintf(fullpath_base, sizeof(fullpath_base), POWER_SYSFILE_BASE_FREQ,
184 f_base = fopen(fullpath_base, "r");
185 if (f_base == NULL) {
186 /* No sysfs base_frequency, that's OK, continue without */
189 s_base = fgets(buf_base, sizeof(buf_base), f_base);
190 FOPS_OR_NULL_GOTO(s_base, out);
192 buf_base[BUFSIZ-1] = '\0';
193 if (strlen(buf_base))
194 /* Strip off terminating '\n' */
195 strtok(buf_base, "\n");
197 base_ratio = strtoul(buf_base, NULL, POWER_CONVERT_TO_DECIMAL)
201 /* Add MSR read to detect turbo status */
203 if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0) {
208 max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
210 POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
212 pi->non_turbo_max_ratio = max_non_turbo;
215 * If base_frequency is reported as greater than the maximum
216 * non-turbo frequency, then mark it as a high priority core.
218 if (base_ratio > max_non_turbo)
219 pi->priority_core = 1;
221 pi->priority_core = 0;
222 pi->core_base_freq = base_ratio * BUS_FREQ;
231 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
233 uint32_t target_freq = 0;
235 if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
236 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
237 "should be less than %u\n", idx, pi->nb_freqs);
241 /* Check if it is the same as current */
242 if (idx == pi->curr_idx)
245 /* Because Intel Pstate Driver only allow user change min/max hint
246 * User need change the min/max as same value.
248 if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
249 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
250 "for setting frequency for lcore %u\n",
255 if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
256 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
257 "for setting frequency for lcore %u\n",
262 /* Turbo is available and enabled, first freq bucket is sys max freq */
263 if (pi->turbo_available && idx == 0) {
264 if (pi->turbo_enable)
265 target_freq = pi->sys_max_freq;
267 RTE_LOG(ERR, POWER, "Turbo is off, frequency can't be scaled up more %u\n",
272 target_freq = pi->freqs[idx];
274 /* Decrease freq, the min freq should be updated first */
275 if (idx > pi->curr_idx) {
277 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
278 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
279 "lcore %u\n", pi->lcore_id);
283 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
284 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
285 "lcore %u\n", pi->lcore_id);
289 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
290 target_freq, pi->lcore_id);
292 fflush(pi->f_cur_min);
293 fflush(pi->f_cur_max);
297 /* Increase freq, the max freq should be updated first */
298 if (idx < pi->curr_idx) {
300 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
301 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
302 "lcore %u\n", pi->lcore_id);
306 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
307 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
308 "lcore %u\n", pi->lcore_id);
312 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
313 target_freq, pi->lcore_id);
315 fflush(pi->f_cur_max);
316 fflush(pi->f_cur_min);
325 * It is to check the current scaling governor by reading sys file, and then
326 * set it into 'performance' if it is not by writing the sys file. The original
327 * governor will be saved for rolling back.
330 power_set_governor_performance(struct pstate_power_info *pi)
335 char fullpath[PATH_MAX];
339 snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
341 f = fopen(fullpath, "rw+");
342 FOPEN_OR_ERR_RET(f, ret);
344 s = fgets(buf, sizeof(buf), f);
345 FOPS_OR_NULL_GOTO(s, out);
346 /* Strip off terminating '\n' */
349 /* Check if current governor is performance */
350 if (strncmp(buf, POWER_GOVERNOR_PERF,
351 sizeof(POWER_GOVERNOR_PERF)) == 0) {
353 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
354 "already performance\n", pi->lcore_id);
357 /* Save the original governor */
358 strlcpy(pi->governor_ori, buf, sizeof(pi->governor_ori));
360 /* Write 'performance' to the governor */
361 val = fseek(f, 0, SEEK_SET);
362 FOPS_OR_ERR_GOTO(val, out);
364 val = fputs(POWER_GOVERNOR_PERF, f);
365 FOPS_OR_ERR_GOTO(val, out);
367 /* We need to flush to see if the fputs succeeds */
369 FOPS_OR_ERR_GOTO(val, out);
372 RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
373 "set to performance successfully\n", pi->lcore_id);
381 * It is to check the governor and then set the original governor back if
382 * needed by writing the sys file.
385 power_set_governor_original(struct pstate_power_info *pi)
390 char fullpath[PATH_MAX];
394 snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
396 f = fopen(fullpath, "rw+");
397 FOPEN_OR_ERR_RET(f, ret);
399 s = fgets(buf, sizeof(buf), f);
400 FOPS_OR_NULL_GOTO(s, out);
402 /* Check if the governor to be set is the same as current */
403 if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
405 POWER_DEBUG_TRACE("Power management governor of lcore %u "
406 "has already been set to %s\n",
407 pi->lcore_id, pi->governor_ori);
411 /* Write back the original governor */
412 val = fseek(f, 0, SEEK_SET);
413 FOPS_OR_ERR_GOTO(val, out);
415 val = fputs(pi->governor_ori, f);
416 FOPS_OR_ERR_GOTO(val, out);
419 RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
420 "has been set back to %s successfully\n",
421 pi->lcore_id, pi->governor_ori);
429 * It is to get the available frequencies of the specific lcore by reading the
433 power_get_available_freqs(struct pstate_power_info *pi)
438 char buf_min[BUFSIZ];
439 char buf_max[BUFSIZ];
440 char fullpath_min[PATH_MAX];
441 char fullpath_max[PATH_MAX];
443 uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
444 uint32_t i, num_freqs = 0;
446 snprintf(fullpath_max, sizeof(fullpath_max),
447 POWER_SYSFILE_BASE_MAX_FREQ,
449 snprintf(fullpath_min, sizeof(fullpath_min),
450 POWER_SYSFILE_BASE_MIN_FREQ,
453 f_min = fopen(fullpath_min, "r");
454 FOPEN_OR_ERR_RET(f_min, ret);
456 f_max = fopen(fullpath_max, "r");
460 FOPEN_OR_ERR_RET(f_max, ret);
462 s_min = fgets(buf_min, sizeof(buf_min), f_min);
463 FOPS_OR_NULL_GOTO(s_min, out);
465 s_max = fgets(buf_max, sizeof(buf_max), f_max);
466 FOPS_OR_NULL_GOTO(s_max, out);
469 /* Strip the line break if there is */
470 p_min = strchr(buf_min, '\n');
474 p_max = strchr(buf_max, '\n');
478 sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
479 sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
481 if (sys_max_freq < sys_min_freq)
484 pi->sys_max_freq = sys_max_freq;
486 if (pi->priority_core == 1)
487 base_max_freq = pi->core_base_freq;
489 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
491 POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
496 if (base_max_freq < sys_max_freq)
497 pi->turbo_available = 1;
499 pi->turbo_available = 0;
501 /* If turbo is available then there is one extra freq bucket
502 * to store the sys max freq which value is base_max +1
504 num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
507 /* Generate the freq bucket array.
508 * If turbo is available the freq bucket[0] value is base_max +1
509 * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
511 * If turbo is not available bucket[0] is base_max and so on
513 for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
514 if ((i == 0) && pi->turbo_available)
515 pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
517 pi->freqs[pi->nb_freqs++] =
518 base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
523 POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
524 num_freqs, pi->lcore_id);
534 power_pstate_cpufreq_init(unsigned int lcore_id)
536 struct pstate_power_info *pi;
538 if (lcore_id >= RTE_MAX_LCORE) {
539 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
540 lcore_id, RTE_MAX_LCORE - 1U);
544 pi = &lcore_power_info[lcore_id];
545 if (rte_atomic32_cmpset(&(pi->state), POWER_IDLE, POWER_ONGOING)
547 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
548 "in use\n", lcore_id);
552 pi->lcore_id = lcore_id;
553 /* Check and set the governor */
554 if (power_set_governor_performance(pi) < 0) {
555 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
556 "performance\n", lcore_id);
559 /* Init for setting lcore frequency */
560 if (power_init_for_setting_freq(pi) < 0) {
561 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
562 "lcore %u\n", lcore_id);
566 /* Get the available frequencies */
567 if (power_get_available_freqs(pi) < 0) {
568 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
569 "lcore %u\n", lcore_id);
574 /* Set freq to max by default */
575 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
576 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
577 "to max\n", lcore_id);
581 RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
582 "power management\n", lcore_id);
583 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_USED);
588 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
594 power_pstate_cpufreq_exit(unsigned int lcore_id)
596 struct pstate_power_info *pi;
598 if (lcore_id >= RTE_MAX_LCORE) {
599 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
600 lcore_id, RTE_MAX_LCORE - 1U);
603 pi = &lcore_power_info[lcore_id];
605 if (rte_atomic32_cmpset(&(pi->state), POWER_USED, POWER_ONGOING)
607 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
608 "not used\n", lcore_id);
612 /* Close FD of setting freq */
613 fclose(pi->f_cur_min);
614 fclose(pi->f_cur_max);
615 pi->f_cur_min = NULL;
616 pi->f_cur_max = NULL;
618 /* Set the governor back to the original */
619 if (power_set_governor_original(pi) < 0) {
620 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
621 "to the original\n", lcore_id);
625 RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
626 "'performance' mode and been set back to the "
627 "original\n", lcore_id);
628 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_IDLE);
633 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
640 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
642 struct pstate_power_info *pi;
644 if (lcore_id >= RTE_MAX_LCORE) {
645 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
650 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
654 pi = &lcore_power_info[lcore_id];
655 if (num < pi->nb_freqs) {
656 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
659 rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
665 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
667 if (lcore_id >= RTE_MAX_LCORE) {
668 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
669 return RTE_POWER_INVALID_FREQ_INDEX;
672 return lcore_power_info[lcore_id].curr_idx;
677 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
679 if (lcore_id >= RTE_MAX_LCORE) {
680 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
684 return set_freq_internal(&(lcore_power_info[lcore_id]), index);
688 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
690 struct pstate_power_info *pi;
692 if (lcore_id >= RTE_MAX_LCORE) {
693 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
697 pi = &lcore_power_info[lcore_id];
698 if (pi->curr_idx == 0)
701 /* Frequencies in the array are from high to low. */
702 return set_freq_internal(pi, pi->curr_idx - 1);
706 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
708 struct pstate_power_info *pi;
710 if (lcore_id >= RTE_MAX_LCORE) {
711 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
715 pi = &lcore_power_info[lcore_id];
716 if (pi->curr_idx + 1 == pi->nb_freqs)
719 /* Frequencies in the array are from high to low. */
720 return set_freq_internal(pi, pi->curr_idx + 1);
724 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
726 if (lcore_id >= RTE_MAX_LCORE) {
727 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
731 /* Frequencies in the array are from high to low. */
732 if (lcore_power_info[lcore_id].turbo_available) {
733 if (lcore_power_info[lcore_id].turbo_enable)
735 return set_freq_internal(
736 &lcore_power_info[lcore_id], 0);
738 /* Set to max non-turbo */
739 return set_freq_internal(
740 &lcore_power_info[lcore_id], 1);
742 return set_freq_internal(&lcore_power_info[lcore_id], 0);
747 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
749 struct pstate_power_info *pi;
751 if (lcore_id >= RTE_MAX_LCORE) {
752 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
756 pi = &lcore_power_info[lcore_id];
758 /* Frequencies in the array are from high to low. */
759 return set_freq_internal(pi, pi->nb_freqs - 1);
764 power_pstate_turbo_status(unsigned int lcore_id)
766 struct pstate_power_info *pi;
768 if (lcore_id >= RTE_MAX_LCORE) {
769 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
773 pi = &lcore_power_info[lcore_id];
775 return pi->turbo_enable;
779 power_pstate_enable_turbo(unsigned int lcore_id)
781 struct pstate_power_info *pi;
783 if (lcore_id >= RTE_MAX_LCORE) {
784 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
788 pi = &lcore_power_info[lcore_id];
790 if (pi->turbo_available)
791 pi->turbo_enable = 1;
793 pi->turbo_enable = 0;
795 "Failed to enable turbo on lcore %u\n",
805 power_pstate_disable_turbo(unsigned int lcore_id)
807 struct pstate_power_info *pi;
809 if (lcore_id >= RTE_MAX_LCORE) {
810 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
814 pi = &lcore_power_info[lcore_id];
816 pi->turbo_enable = 0;
818 if (pi->turbo_available && pi->curr_idx <= 1) {
819 /* Try to set freq to max by default coming out of turbo */
820 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
822 "Failed to set frequency of lcore %u to max\n",
832 int power_pstate_get_capabilities(unsigned int lcore_id,
833 struct rte_power_core_capabilities *caps)
835 struct pstate_power_info *pi;
837 if (lcore_id >= RTE_MAX_LCORE) {
838 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
842 RTE_LOG(ERR, POWER, "Invalid argument\n");
846 pi = &lcore_power_info[lcore_id];
847 caps->capabilities = 0;
848 caps->turbo = !!(pi->turbo_available);
849 caps->priority = pi->priority_core;