1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2021 Intel Corporation
3 * Copyright(c) 2021 Arm Limited
6 #include <rte_memcpy.h>
7 #include <rte_memory.h>
9 #include "power_cppc_cpufreq.h"
10 #include "power_common.h"
12 /* macros used for rounding frequency to nearest 100000 */
13 #define FREQ_ROUNDING_DELTA 50000
14 #define ROUND_FREQ_TO_N_100000 100000
16 /* the unit of highest_perf and nominal_perf differs on different arm platforms.
17 * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
19 #define UNIT_DIFF 10000
21 #define POWER_CONVERT_TO_DECIMAL 10
23 #define POWER_GOVERNOR_USERSPACE "userspace"
24 #define POWER_SYSFILE_SETSPEED \
25 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
26 #define POWER_SYSFILE_SCALING_MAX_FREQ \
27 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
28 #define POWER_SYSFILE_SCALING_MIN_FREQ \
29 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
30 #define POWER_SYSFILE_HIGHEST_PERF \
31 "/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
32 #define POWER_SYSFILE_NOMINAL_PERF \
33 "/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
34 #define POWER_SYSFILE_SYS_MAX \
35 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
37 #define POWER_CPPC_DRIVER "cppc-cpufreq"
38 #define BUS_FREQ 100000
48 * Power info per lcore.
50 struct cppc_power_info {
51 unsigned int lcore_id; /**< Logical core id */
52 uint32_t state; /**< Power in use state */
53 FILE *f; /**< FD of scaling_setspeed */
54 char governor_ori[32]; /**< Original governor name */
55 uint32_t curr_idx; /**< Freq index in freqs array */
56 uint32_t highest_perf; /**< system wide max freq */
57 uint32_t nominal_perf; /**< system wide nominal freq */
58 uint16_t turbo_available; /**< Turbo Boost available */
59 uint16_t turbo_enable; /**< Turbo Boost enable/disable */
60 uint32_t nb_freqs; /**< number of available freqs */
61 uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
62 } __rte_cache_aligned;
64 static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
67 * It is to set specific freq for specific logical core, according to the index
68 * of supported frequencies.
71 set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
73 if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
74 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
75 "should be less than %u\n", idx, pi->nb_freqs);
79 /* Check if it is the same as current */
80 if (idx == pi->curr_idx)
83 POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
84 idx, pi->freqs[idx], pi->lcore_id);
85 if (fseek(pi->f, 0, SEEK_SET) < 0) {
86 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
87 "for setting frequency for lcore %u\n", pi->lcore_id);
90 if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
91 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
92 "lcore %u\n", pi->lcore_id);
102 * It is to check the current scaling governor by reading sys file, and then
103 * set it into 'userspace' if it is not by writing the sys file. The original
104 * governor will be saved for rolling back.
107 power_set_governor_userspace(struct cppc_power_info *pi)
109 return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
110 pi->governor_ori, sizeof(pi->governor_ori));
114 power_check_turbo(struct cppc_power_info *pi)
116 FILE *f_nom = NULL, *f_max = NULL, *f_cmax = NULL;
118 uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
120 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_HIGHEST_PERF,
123 RTE_LOG(ERR, POWER, "failed to open %s\n",
124 POWER_SYSFILE_HIGHEST_PERF);
128 open_core_sysfs_file(&f_nom, "r", POWER_SYSFILE_NOMINAL_PERF,
131 RTE_LOG(ERR, POWER, "failed to open %s\n",
132 POWER_SYSFILE_NOMINAL_PERF);
136 open_core_sysfs_file(&f_cmax, "r", POWER_SYSFILE_SYS_MAX,
138 if (f_cmax == NULL) {
139 RTE_LOG(ERR, POWER, "failed to open %s\n",
140 POWER_SYSFILE_SYS_MAX);
144 ret = read_core_sysfs_u32(f_max, &highest_perf);
146 RTE_LOG(ERR, POWER, "Failed to read %s\n",
147 POWER_SYSFILE_HIGHEST_PERF);
151 ret = read_core_sysfs_u32(f_nom, &nominal_perf);
153 RTE_LOG(ERR, POWER, "Failed to read %s\n",
154 POWER_SYSFILE_NOMINAL_PERF);
158 ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
160 RTE_LOG(ERR, POWER, "Failed to read %s\n",
161 POWER_SYSFILE_SYS_MAX);
165 pi->highest_perf = highest_perf;
166 pi->nominal_perf = nominal_perf;
168 if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
169 || cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
170 pi->turbo_available = 1;
171 pi->turbo_enable = 1;
173 POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
175 pi->lcore_id, highest_perf, nominal_perf);
177 pi->turbo_available = 0;
178 pi->turbo_enable = 0;
179 POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
181 pi->lcore_id, highest_perf, nominal_perf);
196 * It is to get the available frequencies of the specific lcore by reading the
200 power_get_available_freqs(struct cppc_power_info *pi)
202 FILE *f_min = NULL, *f_max = NULL;
204 uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
205 uint32_t i, num_freqs = 0;
207 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_SCALING_MAX_FREQ,
210 RTE_LOG(ERR, POWER, "failed to open %s\n",
211 POWER_SYSFILE_SCALING_MAX_FREQ);
215 open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_SCALING_MIN_FREQ,
218 RTE_LOG(ERR, POWER, "failed to open %s\n",
219 POWER_SYSFILE_SCALING_MIN_FREQ);
223 ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
225 RTE_LOG(ERR, POWER, "Failed to read %s\n",
226 POWER_SYSFILE_SCALING_MAX_FREQ);
230 ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
232 RTE_LOG(ERR, POWER, "Failed to read %s\n",
233 POWER_SYSFILE_SCALING_MIN_FREQ);
237 power_check_turbo(pi);
239 if (scaling_max_freq < scaling_min_freq)
242 /* If turbo is available then there is one extra freq bucket
243 * to store the sys max freq which value is scaling_max_freq
245 nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
246 pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
247 num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
249 if (num_freqs >= RTE_MAX_LCORE_FREQS) {
250 RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
255 /* Generate the freq bucket array. */
256 for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
257 if ((i == 0) && pi->turbo_available)
258 pi->freqs[pi->nb_freqs++] = scaling_max_freq;
260 pi->freqs[pi->nb_freqs++] =
261 nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
266 POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
267 num_freqs, pi->lcore_id);
279 * It is to fopen the sys file for the future setting the lcore frequency.
282 power_init_for_setting_freq(struct cppc_power_info *pi)
289 open_core_sysfs_file(&f, "rw+", POWER_SYSFILE_SETSPEED, pi->lcore_id);
291 RTE_LOG(ERR, POWER, "failed to open %s\n",
292 POWER_SYSFILE_SETSPEED);
296 ret = read_core_sysfs_s(f, buf, sizeof(buf));
298 RTE_LOG(ERR, POWER, "Failed to read %s\n",
299 POWER_SYSFILE_SETSPEED);
303 freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
305 /* convert the frequency to nearest 100000 value
306 * Ex: if freq=1396789 then freq_conv=1400000
307 * Ex: if freq=800030 then freq_conv=800000
309 unsigned int freq_conv = 0;
310 freq_conv = (freq + FREQ_ROUNDING_DELTA)
311 / ROUND_FREQ_TO_N_100000;
312 freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
314 for (i = 0; i < pi->nb_freqs; i++) {
315 if (freq_conv == pi->freqs[i]) {
330 power_cppc_cpufreq_check_supported(void)
332 return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
336 power_cppc_cpufreq_init(unsigned int lcore_id)
338 struct cppc_power_info *pi;
341 if (lcore_id >= RTE_MAX_LCORE) {
342 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
343 lcore_id, RTE_MAX_LCORE - 1U);
347 pi = &lcore_power_info[lcore_id];
348 exp_state = POWER_IDLE;
349 /* The power in use state works as a guard variable between
350 * the CPU frequency control initialization and exit process.
351 * The ACQUIRE memory ordering here pairs with the RELEASE
352 * ordering below as lock to make sure the frequency operations
353 * in the critical section are done under the correct state.
355 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
357 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
358 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
359 "in use\n", lcore_id);
363 pi->lcore_id = lcore_id;
364 /* Check and set the governor */
365 if (power_set_governor_userspace(pi) < 0) {
366 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
367 "userspace\n", lcore_id);
371 /* Get the available frequencies */
372 if (power_get_available_freqs(pi) < 0) {
373 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
374 "lcore %u\n", lcore_id);
378 /* Init for setting lcore frequency */
379 if (power_init_for_setting_freq(pi) < 0) {
380 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
381 "lcore %u\n", lcore_id);
385 /* Set freq to max by default */
386 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
387 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
388 "to max\n", lcore_id);
392 RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
393 "power management\n", lcore_id);
395 __atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
400 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
405 * It is to check the governor and then set the original governor back if
406 * needed by writing the sys file.
409 power_set_governor_original(struct cppc_power_info *pi)
411 return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
415 power_cppc_cpufreq_exit(unsigned int lcore_id)
417 struct cppc_power_info *pi;
420 if (lcore_id >= RTE_MAX_LCORE) {
421 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
422 lcore_id, RTE_MAX_LCORE - 1U);
425 pi = &lcore_power_info[lcore_id];
426 exp_state = POWER_USED;
427 /* The power in use state works as a guard variable between
428 * the CPU frequency control initialization and exit process.
429 * The ACQUIRE memory ordering here pairs with the RELEASE
430 * ordering below as lock to make sure the frequency operations
431 * in the critical section are done under the correct state.
433 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
435 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
436 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
437 "not used\n", lcore_id);
441 /* Close FD of setting freq */
445 /* Set the governor back to the original */
446 if (power_set_governor_original(pi) < 0) {
447 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
448 "to the original\n", lcore_id);
452 RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
453 "'userspace' mode and been set back to the "
454 "original\n", lcore_id);
455 __atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
460 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
466 power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
468 struct cppc_power_info *pi;
470 if (lcore_id >= RTE_MAX_LCORE) {
471 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
476 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
480 pi = &lcore_power_info[lcore_id];
481 if (num < pi->nb_freqs) {
482 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
485 rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
491 power_cppc_cpufreq_get_freq(unsigned int lcore_id)
493 if (lcore_id >= RTE_MAX_LCORE) {
494 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
495 return RTE_POWER_INVALID_FREQ_INDEX;
498 return lcore_power_info[lcore_id].curr_idx;
502 power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
504 if (lcore_id >= RTE_MAX_LCORE) {
505 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
509 return set_freq_internal(&(lcore_power_info[lcore_id]), index);
513 power_cppc_cpufreq_freq_down(unsigned int lcore_id)
515 struct cppc_power_info *pi;
517 if (lcore_id >= RTE_MAX_LCORE) {
518 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
522 pi = &lcore_power_info[lcore_id];
523 if (pi->curr_idx + 1 == pi->nb_freqs)
526 /* Frequencies in the array are from high to low. */
527 return set_freq_internal(pi, pi->curr_idx + 1);
531 power_cppc_cpufreq_freq_up(unsigned int lcore_id)
533 struct cppc_power_info *pi;
535 if (lcore_id >= RTE_MAX_LCORE) {
536 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
540 pi = &lcore_power_info[lcore_id];
541 if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
542 pi->turbo_available && !pi->turbo_enable))
545 /* Frequencies in the array are from high to low. */
546 return set_freq_internal(pi, pi->curr_idx - 1);
550 power_cppc_cpufreq_freq_max(unsigned int lcore_id)
552 if (lcore_id >= RTE_MAX_LCORE) {
553 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
557 /* Frequencies in the array are from high to low. */
558 if (lcore_power_info[lcore_id].turbo_available) {
559 if (lcore_power_info[lcore_id].turbo_enable)
561 return set_freq_internal(
562 &lcore_power_info[lcore_id], 0);
564 /* Set to max non-turbo */
565 return set_freq_internal(
566 &lcore_power_info[lcore_id], 1);
568 return set_freq_internal(&lcore_power_info[lcore_id], 0);
572 power_cppc_cpufreq_freq_min(unsigned int lcore_id)
574 struct cppc_power_info *pi;
576 if (lcore_id >= RTE_MAX_LCORE) {
577 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
581 pi = &lcore_power_info[lcore_id];
583 /* Frequencies in the array are from high to low. */
584 return set_freq_internal(pi, pi->nb_freqs - 1);
588 power_cppc_turbo_status(unsigned int lcore_id)
590 struct cppc_power_info *pi;
592 if (lcore_id >= RTE_MAX_LCORE) {
593 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
597 pi = &lcore_power_info[lcore_id];
599 return pi->turbo_enable;
603 power_cppc_enable_turbo(unsigned int lcore_id)
605 struct cppc_power_info *pi;
607 if (lcore_id >= RTE_MAX_LCORE) {
608 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
612 pi = &lcore_power_info[lcore_id];
614 if (pi->turbo_available)
615 pi->turbo_enable = 1;
617 pi->turbo_enable = 0;
619 "Failed to enable turbo on lcore %u\n",
624 /* TODO: must set to max once enbling Turbo? Considering add condition:
625 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
627 /* Max may have changed, so call to max function */
628 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
630 "Failed to set frequency of lcore %u to max\n",
639 power_cppc_disable_turbo(unsigned int lcore_id)
641 struct cppc_power_info *pi;
643 if (lcore_id >= RTE_MAX_LCORE) {
644 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
648 pi = &lcore_power_info[lcore_id];
650 pi->turbo_enable = 0;
652 if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
653 /* Try to set freq to max by default coming out of turbo */
654 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
656 "Failed to set frequency of lcore %u to max\n",
666 power_cppc_get_capabilities(unsigned int lcore_id,
667 struct rte_power_core_capabilities *caps)
669 struct cppc_power_info *pi;
671 if (lcore_id >= RTE_MAX_LCORE) {
672 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
676 RTE_LOG(ERR, POWER, "Invalid argument\n");
680 pi = &lcore_power_info[lcore_id];
681 caps->capabilities = 0;
682 caps->turbo = !!(pi->turbo_available);