1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
17 #include <rte_memcpy.h>
18 #include <rte_atomic.h>
20 #include "power_pstate_cpufreq.h"
21 #include "power_common.h"
24 #ifdef RTE_LIBRTE_POWER_DEBUG
25 #define POWER_DEBUG_TRACE(fmt, args...) do { \
26 RTE_LOG(ERR, POWER, "%s: " fmt, __func__, ## args); \
29 #define POWER_DEBUG_TRACE(fmt, args...)
32 #define FOPEN_OR_ERR_RET(f, retval) do { \
34 RTE_LOG(ERR, POWER, "File not openned\n"); \
39 #define FOPS_OR_NULL_GOTO(ret, label) do { \
40 if ((ret) == NULL) { \
41 RTE_LOG(ERR, POWER, "fgets returns nothing\n"); \
46 #define FOPS_OR_ERR_GOTO(ret, label) do { \
48 RTE_LOG(ERR, POWER, "File operations failed\n"); \
54 #define POWER_CONVERT_TO_DECIMAL 10
55 #define BUS_FREQ 100000
57 #define POWER_GOVERNOR_PERF "performance"
58 #define POWER_SYSFILE_GOVERNOR \
59 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_governor"
60 #define POWER_SYSFILE_MAX_FREQ \
61 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
62 #define POWER_SYSFILE_MIN_FREQ \
63 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
64 #define POWER_SYSFILE_CUR_FREQ \
65 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_cur_freq"
66 #define POWER_SYSFILE_BASE_MAX_FREQ \
67 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
68 #define POWER_SYSFILE_BASE_MIN_FREQ \
69 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_min_freq"
70 #define POWER_MSR_PATH "/dev/cpu/%u/msr"
75 #define PLATFORM_INFO 0x0CE
76 #define NON_TURBO_MASK 0xFF00
77 #define NON_TURBO_OFFSET 0x8
87 struct pstate_power_info {
88 unsigned int lcore_id; /**< Logical core id */
89 uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
90 uint32_t nb_freqs; /**< number of available freqs */
91 FILE *f_cur_min; /**< FD of scaling_min */
92 FILE *f_cur_max; /**< FD of scaling_max */
93 char governor_ori[32]; /**< Original governor name */
94 uint32_t curr_idx; /**< Freq index in freqs array */
95 uint32_t non_turbo_max_ratio; /**< Non Turbo Max ratio */
96 uint32_t sys_max_freq; /**< system wide max freq */
97 volatile uint32_t state; /**< Power in use state */
98 uint16_t turbo_available; /**< Turbo Boost available */
99 uint16_t turbo_enable; /**< Turbo Boost enable/disable */
100 } __rte_cache_aligned;
103 static struct pstate_power_info lcore_power_info[RTE_MAX_LCORE];
106 * It is to read the specific MSR.
110 power_rdmsr(int msr, uint64_t *val, unsigned int lcore_id)
113 char fullpath[PATH_MAX];
115 snprintf(fullpath, sizeof(fullpath), POWER_MSR_PATH, lcore_id);
117 fd = open(fullpath, O_RDONLY);
120 RTE_LOG(ERR, POWER, "Error opening '%s': %s\n", fullpath,
125 ret = pread(fd, val, sizeof(uint64_t), msr);
128 RTE_LOG(ERR, POWER, "Error reading '%s': %s\n", fullpath,
133 POWER_DEBUG_TRACE("MSR Path %s, offset 0x%X for lcore %u\n",
134 fullpath, msr, lcore_id);
136 POWER_DEBUG_TRACE("Ret value %d, content is 0x%"PRIx64"\n", ret, *val);
143 * It is to fopen the sys file for the future setting the lcore frequency.
146 power_init_for_setting_freq(struct pstate_power_info *pi)
149 char fullpath_min[PATH_MAX];
150 char fullpath_max[PATH_MAX];
151 uint64_t max_non_turbo = 0;
153 snprintf(fullpath_min, sizeof(fullpath_min), POWER_SYSFILE_MIN_FREQ,
156 f_min = fopen(fullpath_min, "rw+");
157 FOPEN_OR_ERR_RET(f_min, -1);
159 snprintf(fullpath_max, sizeof(fullpath_max), POWER_SYSFILE_MAX_FREQ,
162 f_max = fopen(fullpath_max, "rw+");
166 FOPEN_OR_ERR_RET(f_max, -1);
168 pi->f_cur_min = f_min;
169 pi->f_cur_max = f_max;
171 /* Add MSR read to detect turbo status */
173 if (power_rdmsr(PLATFORM_INFO, &max_non_turbo, pi->lcore_id) < 0)
176 max_non_turbo = (max_non_turbo&NON_TURBO_MASK)>>NON_TURBO_OFFSET;
178 POWER_DEBUG_TRACE("no turbo perf %"PRIu64"\n", max_non_turbo);
180 pi->non_turbo_max_ratio = max_non_turbo;
186 set_freq_internal(struct pstate_power_info *pi, uint32_t idx)
188 uint32_t target_freq = 0;
190 if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
191 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
192 "should be less than %u\n", idx, pi->nb_freqs);
196 /* Check if it is the same as current */
197 if (idx == pi->curr_idx)
200 /* Because Intel Pstate Driver only allow user change min/max hint
201 * User need change the min/max as same value.
203 if (fseek(pi->f_cur_min, 0, SEEK_SET) < 0) {
204 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
205 "for setting frequency for lcore %u\n",
210 if (fseek(pi->f_cur_max, 0, SEEK_SET) < 0) {
211 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
212 "for setting frequency for lcore %u\n",
217 /* Turbo is available and enabled, first freq bucket is sys max freq */
218 if (pi->turbo_available && pi->turbo_enable && (idx == 0))
219 target_freq = pi->sys_max_freq;
221 target_freq = pi->freqs[idx];
223 /* Decrease freq, the min freq should be updated first */
224 if (idx > pi->curr_idx) {
226 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
227 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
228 "lcore %u\n", pi->lcore_id);
232 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
233 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
234 "lcore %u\n", pi->lcore_id);
238 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
239 target_freq, pi->lcore_id);
241 fflush(pi->f_cur_min);
242 fflush(pi->f_cur_max);
246 /* Increase freq, the max freq should be updated first */
247 if (idx < pi->curr_idx) {
249 if (fprintf(pi->f_cur_max, "%u", target_freq) < 0) {
250 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
251 "lcore %u\n", pi->lcore_id);
255 if (fprintf(pi->f_cur_min, "%u", target_freq) < 0) {
256 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
257 "lcore %u\n", pi->lcore_id);
261 POWER_DEBUG_TRACE("Freqency '%u' to be set for lcore %u\n",
262 target_freq, pi->lcore_id);
264 fflush(pi->f_cur_max);
265 fflush(pi->f_cur_min);
274 * It is to check the current scaling governor by reading sys file, and then
275 * set it into 'performance' if it is not by writing the sys file. The original
276 * governor will be saved for rolling back.
279 power_set_governor_performance(struct pstate_power_info *pi)
284 char fullpath[PATH_MAX];
288 snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
290 f = fopen(fullpath, "rw+");
291 FOPEN_OR_ERR_RET(f, ret);
293 s = fgets(buf, sizeof(buf), f);
294 FOPS_OR_NULL_GOTO(s, out);
295 /* Strip off terminating '\n' */
298 /* Check if current governor is performance */
299 if (strncmp(buf, POWER_GOVERNOR_PERF,
300 sizeof(POWER_GOVERNOR_PERF)) == 0) {
302 POWER_DEBUG_TRACE("Power management governor of lcore %u is "
303 "already performance\n", pi->lcore_id);
306 /* Save the original governor */
307 snprintf(pi->governor_ori, sizeof(pi->governor_ori), "%s", buf);
309 /* Write 'performance' to the governor */
310 val = fseek(f, 0, SEEK_SET);
311 FOPS_OR_ERR_GOTO(val, out);
313 val = fputs(POWER_GOVERNOR_PERF, f);
314 FOPS_OR_ERR_GOTO(val, out);
316 /* We need to flush to see if the fputs succeeds */
318 FOPS_OR_ERR_GOTO(val, out);
321 RTE_LOG(INFO, POWER, "Power management governor of lcore %u has been "
322 "set to performance successfully\n", pi->lcore_id);
330 * It is to check the governor and then set the original governor back if
331 * needed by writing the sys file.
334 power_set_governor_original(struct pstate_power_info *pi)
339 char fullpath[PATH_MAX];
343 snprintf(fullpath, sizeof(fullpath), POWER_SYSFILE_GOVERNOR,
345 f = fopen(fullpath, "rw+");
346 FOPEN_OR_ERR_RET(f, ret);
348 s = fgets(buf, sizeof(buf), f);
349 FOPS_OR_NULL_GOTO(s, out);
351 /* Check if the governor to be set is the same as current */
352 if (strncmp(buf, pi->governor_ori, sizeof(pi->governor_ori)) == 0) {
354 POWER_DEBUG_TRACE("Power management governor of lcore %u "
355 "has already been set to %s\n",
356 pi->lcore_id, pi->governor_ori);
360 /* Write back the original governor */
361 val = fseek(f, 0, SEEK_SET);
362 FOPS_OR_ERR_GOTO(val, out);
364 val = fputs(pi->governor_ori, f);
365 FOPS_OR_ERR_GOTO(val, out);
368 RTE_LOG(INFO, POWER, "Power management governor of lcore %u "
369 "has been set back to %s successfully\n",
370 pi->lcore_id, pi->governor_ori);
378 * It is to get the available frequencies of the specific lcore by reading the
382 power_get_available_freqs(struct pstate_power_info *pi)
387 char buf_min[BUFSIZ];
388 char buf_max[BUFSIZ];
389 char fullpath_min[PATH_MAX];
390 char fullpath_max[PATH_MAX];
392 uint32_t sys_min_freq = 0, sys_max_freq = 0, base_max_freq = 0;
393 uint32_t i, num_freqs = 0;
395 snprintf(fullpath_max, sizeof(fullpath_max),
396 POWER_SYSFILE_BASE_MAX_FREQ,
398 snprintf(fullpath_min, sizeof(fullpath_min),
399 POWER_SYSFILE_BASE_MIN_FREQ,
402 f_min = fopen(fullpath_min, "r");
403 FOPEN_OR_ERR_RET(f_min, ret);
405 f_max = fopen(fullpath_max, "r");
409 FOPEN_OR_ERR_RET(f_max, ret);
411 s_min = fgets(buf_min, sizeof(buf_min), f_min);
412 FOPS_OR_NULL_GOTO(s_min, out);
414 s_max = fgets(buf_max, sizeof(buf_max), f_max);
415 FOPS_OR_NULL_GOTO(s_max, out);
418 /* Strip the line break if there is */
419 p_min = strchr(buf_min, '\n');
423 p_max = strchr(buf_max, '\n');
427 sys_min_freq = strtoul(buf_min, &p_min, POWER_CONVERT_TO_DECIMAL);
428 sys_max_freq = strtoul(buf_max, &p_max, POWER_CONVERT_TO_DECIMAL);
430 if (sys_max_freq < sys_min_freq)
433 pi->sys_max_freq = sys_max_freq;
435 base_max_freq = pi->non_turbo_max_ratio * BUS_FREQ;
437 POWER_DEBUG_TRACE("sys min %u, sys max %u, base_max %u\n",
442 if (base_max_freq < sys_max_freq)
443 pi->turbo_available = 1;
445 pi->turbo_available = 0;
447 /* If turbo is available then there is one extra freq bucket
448 * to store the sys max freq which value is base_max +1
450 num_freqs = (base_max_freq - sys_min_freq) / BUS_FREQ + 1 +
453 /* Generate the freq bucket array.
454 * If turbo is available the freq bucket[0] value is base_max +1
455 * the bucket[1] is base_max, bucket[2] is base_max - BUS_FREQ
457 * If turbo is not available bucket[0] is base_max and so on
459 for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
460 if ((i == 0) && pi->turbo_available)
461 pi->freqs[pi->nb_freqs++] = base_max_freq + 1;
463 pi->freqs[pi->nb_freqs++] =
464 base_max_freq - (i - pi->turbo_available) * BUS_FREQ;
469 POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
470 num_freqs, pi->lcore_id);
480 power_pstate_cpufreq_init(unsigned int lcore_id)
482 struct pstate_power_info *pi;
484 if (lcore_id >= RTE_MAX_LCORE) {
485 RTE_LOG(ERR, POWER, "Lcore id %u can not exceed %u\n",
486 lcore_id, RTE_MAX_LCORE - 1U);
490 pi = &lcore_power_info[lcore_id];
491 if (rte_atomic32_cmpset(&(pi->state), POWER_IDLE, POWER_ONGOING)
493 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
494 "in use\n", lcore_id);
498 pi->lcore_id = lcore_id;
499 /* Check and set the governor */
500 if (power_set_governor_performance(pi) < 0) {
501 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
502 "performance\n", lcore_id);
505 /* Init for setting lcore frequency */
506 if (power_init_for_setting_freq(pi) < 0) {
507 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
508 "lcore %u\n", lcore_id);
512 /* Get the available frequencies */
513 if (power_get_available_freqs(pi) < 0) {
514 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
515 "lcore %u\n", lcore_id);
520 /* Set freq to max by default */
521 if (power_pstate_cpufreq_freq_max(lcore_id) < 0) {
522 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
523 "to max\n", lcore_id);
527 RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
528 "power management\n", lcore_id);
529 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_USED);
534 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
540 power_pstate_cpufreq_exit(unsigned int lcore_id)
542 struct pstate_power_info *pi;
544 if (lcore_id >= RTE_MAX_LCORE) {
545 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
546 lcore_id, RTE_MAX_LCORE - 1U);
549 pi = &lcore_power_info[lcore_id];
551 if (rte_atomic32_cmpset(&(pi->state), POWER_USED, POWER_ONGOING)
553 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
554 "not used\n", lcore_id);
558 /* Close FD of setting freq */
559 fclose(pi->f_cur_min);
560 fclose(pi->f_cur_max);
561 pi->f_cur_min = NULL;
562 pi->f_cur_max = NULL;
564 /* Set the governor back to the original */
565 if (power_set_governor_original(pi) < 0) {
566 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
567 "to the original\n", lcore_id);
571 RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
572 "'performance' mode and been set back to the "
573 "original\n", lcore_id);
574 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_IDLE);
579 rte_atomic32_cmpset(&(pi->state), POWER_ONGOING, POWER_UNKNOWN);
586 power_pstate_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
588 struct pstate_power_info *pi;
590 if (lcore_id >= RTE_MAX_LCORE) {
591 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
596 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
600 pi = &lcore_power_info[lcore_id];
601 if (num < pi->nb_freqs) {
602 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
605 rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
611 power_pstate_cpufreq_get_freq(unsigned int lcore_id)
613 if (lcore_id >= RTE_MAX_LCORE) {
614 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
615 return RTE_POWER_INVALID_FREQ_INDEX;
618 return lcore_power_info[lcore_id].curr_idx;
623 power_pstate_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
625 if (lcore_id >= RTE_MAX_LCORE) {
626 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
630 return set_freq_internal(&(lcore_power_info[lcore_id]), index);
634 power_pstate_cpufreq_freq_up(unsigned int lcore_id)
636 struct pstate_power_info *pi;
638 if (lcore_id >= RTE_MAX_LCORE) {
639 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
643 pi = &lcore_power_info[lcore_id];
644 if (pi->curr_idx == 0)
647 /* Frequencies in the array are from high to low. */
648 return set_freq_internal(pi, pi->curr_idx - 1);
652 power_pstate_cpufreq_freq_down(unsigned int lcore_id)
654 struct pstate_power_info *pi;
656 if (lcore_id >= RTE_MAX_LCORE) {
657 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
661 pi = &lcore_power_info[lcore_id];
662 if (pi->curr_idx + 1 == pi->nb_freqs)
665 /* Frequencies in the array are from high to low. */
666 return set_freq_internal(pi, pi->curr_idx + 1);
670 power_pstate_cpufreq_freq_max(unsigned int lcore_id)
672 if (lcore_id >= RTE_MAX_LCORE) {
673 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
677 /* Frequencies in the array are from high to low. */
678 if (lcore_power_info[lcore_id].turbo_available) {
679 if (lcore_power_info[lcore_id].turbo_enable)
681 return set_freq_internal(
682 &lcore_power_info[lcore_id], 0);
684 /* Set to max non-turbo */
685 return set_freq_internal(
686 &lcore_power_info[lcore_id], 1);
688 return set_freq_internal(&lcore_power_info[lcore_id], 0);
693 power_pstate_cpufreq_freq_min(unsigned int lcore_id)
695 struct pstate_power_info *pi;
697 if (lcore_id >= RTE_MAX_LCORE) {
698 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
702 pi = &lcore_power_info[lcore_id];
704 /* Frequencies in the array are from high to low. */
705 return set_freq_internal(pi, pi->nb_freqs - 1);
710 power_pstate_turbo_status(unsigned int lcore_id)
712 struct pstate_power_info *pi;
714 if (lcore_id >= RTE_MAX_LCORE) {
715 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
719 pi = &lcore_power_info[lcore_id];
721 return pi->turbo_enable;
725 power_pstate_enable_turbo(unsigned int lcore_id)
727 struct pstate_power_info *pi;
729 if (lcore_id >= RTE_MAX_LCORE) {
730 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
734 pi = &lcore_power_info[lcore_id];
736 if (pi->turbo_available)
737 pi->turbo_enable = 1;
739 pi->turbo_enable = 0;
741 "Failed to enable turbo on lcore %u\n",
751 power_pstate_disable_turbo(unsigned int lcore_id)
753 struct pstate_power_info *pi;
755 if (lcore_id >= RTE_MAX_LCORE) {
756 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
760 pi = &lcore_power_info[lcore_id];
762 pi->turbo_enable = 0;
769 int power_pstate_get_capabilities(unsigned int lcore_id,
770 struct rte_power_core_capabilities *caps)
772 struct pstate_power_info *pi;
774 if (lcore_id >= RTE_MAX_LCORE) {
775 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
779 RTE_LOG(ERR, POWER, "Invalid argument\n");
783 pi = &lcore_power_info[lcore_id];
784 caps->capabilities = 0;
785 caps->turbo = !!(pi->turbo_available);