1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2021 Intel Corporation
3 * Copyright(c) 2021 Arm Limited
6 #include <rte_memcpy.h>
8 #include "power_cppc_cpufreq.h"
9 #include "power_common.h"
11 /* macros used for rounding frequency to nearest 100000 */
12 #define FREQ_ROUNDING_DELTA 50000
13 #define ROUND_FREQ_TO_N_100000 100000
15 /* the unit of highest_perf and nominal_perf differs on different arm platforms.
16 * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
18 #define UNIT_DIFF 10000
20 #define POWER_CONVERT_TO_DECIMAL 10
22 #define POWER_GOVERNOR_USERSPACE "userspace"
23 #define POWER_SYSFILE_SETSPEED \
24 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
25 #define POWER_SYSFILE_SCALING_MAX_FREQ \
26 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
27 #define POWER_SYSFILE_SCALING_MIN_FREQ \
28 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
29 #define POWER_SYSFILE_HIGHEST_PERF \
30 "/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
31 #define POWER_SYSFILE_NOMINAL_PERF \
32 "/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
33 #define POWER_SYSFILE_SYS_MAX \
34 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
36 #define POWER_CPPC_DRIVER "cppc-cpufreq"
37 #define BUS_FREQ 100000
47 * Power info per lcore.
49 struct cppc_power_info {
50 unsigned int lcore_id; /**< Logical core id */
51 uint32_t state; /**< Power in use state */
52 FILE *f; /**< FD of scaling_setspeed */
53 char governor_ori[32]; /**< Original governor name */
54 uint32_t curr_idx; /**< Freq index in freqs array */
55 uint32_t highest_perf; /**< system wide max freq */
56 uint32_t nominal_perf; /**< system wide nominal freq */
57 uint16_t turbo_available; /**< Turbo Boost available */
58 uint16_t turbo_enable; /**< Turbo Boost enable/disable */
59 uint32_t nb_freqs; /**< number of available freqs */
60 uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
61 } __rte_cache_aligned;
63 static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
66 * It is to set specific freq for specific logical core, according to the index
67 * of supported frequencies.
70 set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
72 if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
73 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
74 "should be less than %u\n", idx, pi->nb_freqs);
78 /* Check if it is the same as current */
79 if (idx == pi->curr_idx)
82 POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
83 idx, pi->freqs[idx], pi->lcore_id);
84 if (fseek(pi->f, 0, SEEK_SET) < 0) {
85 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
86 "for setting frequency for lcore %u\n", pi->lcore_id);
89 if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
90 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
91 "lcore %u\n", pi->lcore_id);
101 * It is to check the current scaling governor by reading sys file, and then
102 * set it into 'userspace' if it is not by writing the sys file. The original
103 * governor will be saved for rolling back.
106 power_set_governor_userspace(struct cppc_power_info *pi)
108 return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
109 pi->governor_ori, sizeof(pi->governor_ori));
113 power_check_turbo(struct cppc_power_info *pi)
115 FILE *f_nom = NULL, *f_max = NULL, *f_cmax = NULL;
117 uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
119 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_HIGHEST_PERF,
122 RTE_LOG(ERR, POWER, "failed to open %s\n",
123 POWER_SYSFILE_HIGHEST_PERF);
127 open_core_sysfs_file(&f_nom, "r", POWER_SYSFILE_NOMINAL_PERF,
130 RTE_LOG(ERR, POWER, "failed to open %s\n",
131 POWER_SYSFILE_NOMINAL_PERF);
135 open_core_sysfs_file(&f_cmax, "r", POWER_SYSFILE_SYS_MAX,
137 if (f_cmax == NULL) {
138 RTE_LOG(ERR, POWER, "failed to open %s\n",
139 POWER_SYSFILE_SYS_MAX);
143 ret = read_core_sysfs_u32(f_max, &highest_perf);
145 RTE_LOG(ERR, POWER, "Failed to read %s\n",
146 POWER_SYSFILE_HIGHEST_PERF);
150 ret = read_core_sysfs_u32(f_nom, &nominal_perf);
152 RTE_LOG(ERR, POWER, "Failed to read %s\n",
153 POWER_SYSFILE_NOMINAL_PERF);
157 ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
159 RTE_LOG(ERR, POWER, "Failed to read %s\n",
160 POWER_SYSFILE_SYS_MAX);
164 pi->highest_perf = highest_perf;
165 pi->nominal_perf = nominal_perf;
167 if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
168 || cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
169 pi->turbo_available = 1;
170 pi->turbo_enable = 1;
172 POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
174 pi->lcore_id, highest_perf, nominal_perf);
176 pi->turbo_available = 0;
177 pi->turbo_enable = 0;
178 POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
180 pi->lcore_id, highest_perf, nominal_perf);
195 * It is to get the available frequencies of the specific lcore by reading the
199 power_get_available_freqs(struct cppc_power_info *pi)
201 FILE *f_min = NULL, *f_max = NULL;
203 uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
204 uint32_t i, num_freqs = 0;
206 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_SCALING_MAX_FREQ,
209 RTE_LOG(ERR, POWER, "failed to open %s\n",
210 POWER_SYSFILE_SCALING_MAX_FREQ);
214 open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_SCALING_MIN_FREQ,
217 RTE_LOG(ERR, POWER, "failed to open %s\n",
218 POWER_SYSFILE_SCALING_MIN_FREQ);
222 ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
224 RTE_LOG(ERR, POWER, "Failed to read %s\n",
225 POWER_SYSFILE_SCALING_MAX_FREQ);
229 ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
231 RTE_LOG(ERR, POWER, "Failed to read %s\n",
232 POWER_SYSFILE_SCALING_MIN_FREQ);
236 power_check_turbo(pi);
238 if (scaling_max_freq < scaling_min_freq)
241 /* If turbo is available then there is one extra freq bucket
242 * to store the sys max freq which value is scaling_max_freq
244 nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
245 pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
246 num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
248 if (num_freqs >= RTE_MAX_LCORE_FREQS) {
249 RTE_LOG(ERR, POWER, "Too many available frequencies: %d\n",
254 /* Generate the freq bucket array. */
255 for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
256 if ((i == 0) && pi->turbo_available)
257 pi->freqs[pi->nb_freqs++] = scaling_max_freq;
259 pi->freqs[pi->nb_freqs++] =
260 nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
265 POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
266 num_freqs, pi->lcore_id);
278 * It is to fopen the sys file for the future setting the lcore frequency.
281 power_init_for_setting_freq(struct cppc_power_info *pi)
288 open_core_sysfs_file(&f, "rw+", POWER_SYSFILE_SETSPEED, pi->lcore_id);
290 RTE_LOG(ERR, POWER, "failed to open %s\n",
291 POWER_SYSFILE_SETSPEED);
295 ret = read_core_sysfs_s(f, buf, sizeof(buf));
297 RTE_LOG(ERR, POWER, "Failed to read %s\n",
298 POWER_SYSFILE_SETSPEED);
302 freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
304 /* convert the frequency to nearest 100000 value
305 * Ex: if freq=1396789 then freq_conv=1400000
306 * Ex: if freq=800030 then freq_conv=800000
308 unsigned int freq_conv = 0;
309 freq_conv = (freq + FREQ_ROUNDING_DELTA)
310 / ROUND_FREQ_TO_N_100000;
311 freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
313 for (i = 0; i < pi->nb_freqs; i++) {
314 if (freq_conv == pi->freqs[i]) {
329 power_cppc_cpufreq_check_supported(void)
331 return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
335 power_cppc_cpufreq_init(unsigned int lcore_id)
337 struct cppc_power_info *pi;
340 if (lcore_id >= RTE_MAX_LCORE) {
341 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
342 lcore_id, RTE_MAX_LCORE - 1U);
346 pi = &lcore_power_info[lcore_id];
347 exp_state = POWER_IDLE;
348 /* The power in use state works as a guard variable between
349 * the CPU frequency control initialization and exit process.
350 * The ACQUIRE memory ordering here pairs with the RELEASE
351 * ordering below as lock to make sure the frequency operations
352 * in the critical section are done under the correct state.
354 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
356 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
357 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
358 "in use\n", lcore_id);
362 pi->lcore_id = lcore_id;
363 /* Check and set the governor */
364 if (power_set_governor_userspace(pi) < 0) {
365 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
366 "userspace\n", lcore_id);
370 /* Get the available frequencies */
371 if (power_get_available_freqs(pi) < 0) {
372 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
373 "lcore %u\n", lcore_id);
377 /* Init for setting lcore frequency */
378 if (power_init_for_setting_freq(pi) < 0) {
379 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
380 "lcore %u\n", lcore_id);
384 /* Set freq to max by default */
385 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
386 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
387 "to max\n", lcore_id);
391 RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
392 "power management\n", lcore_id);
394 __atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
399 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
404 * It is to check the governor and then set the original governor back if
405 * needed by writing the sys file.
408 power_set_governor_original(struct cppc_power_info *pi)
410 return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
414 power_cppc_cpufreq_exit(unsigned int lcore_id)
416 struct cppc_power_info *pi;
419 if (lcore_id >= RTE_MAX_LCORE) {
420 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
421 lcore_id, RTE_MAX_LCORE - 1U);
424 pi = &lcore_power_info[lcore_id];
425 exp_state = POWER_USED;
426 /* The power in use state works as a guard variable between
427 * the CPU frequency control initialization and exit process.
428 * The ACQUIRE memory ordering here pairs with the RELEASE
429 * ordering below as lock to make sure the frequency operations
430 * in the critical section are done under the correct state.
432 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
434 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
435 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
436 "not used\n", lcore_id);
440 /* Close FD of setting freq */
444 /* Set the governor back to the original */
445 if (power_set_governor_original(pi) < 0) {
446 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
447 "to the original\n", lcore_id);
451 RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
452 "'userspace' mode and been set back to the "
453 "original\n", lcore_id);
454 __atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
459 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
465 power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
467 struct cppc_power_info *pi;
469 if (lcore_id >= RTE_MAX_LCORE) {
470 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
475 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
479 pi = &lcore_power_info[lcore_id];
480 if (num < pi->nb_freqs) {
481 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
484 rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
490 power_cppc_cpufreq_get_freq(unsigned int lcore_id)
492 if (lcore_id >= RTE_MAX_LCORE) {
493 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
494 return RTE_POWER_INVALID_FREQ_INDEX;
497 return lcore_power_info[lcore_id].curr_idx;
501 power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
503 if (lcore_id >= RTE_MAX_LCORE) {
504 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
508 return set_freq_internal(&(lcore_power_info[lcore_id]), index);
512 power_cppc_cpufreq_freq_down(unsigned int lcore_id)
514 struct cppc_power_info *pi;
516 if (lcore_id >= RTE_MAX_LCORE) {
517 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
521 pi = &lcore_power_info[lcore_id];
522 if (pi->curr_idx + 1 == pi->nb_freqs)
525 /* Frequencies in the array are from high to low. */
526 return set_freq_internal(pi, pi->curr_idx + 1);
530 power_cppc_cpufreq_freq_up(unsigned int lcore_id)
532 struct cppc_power_info *pi;
534 if (lcore_id >= RTE_MAX_LCORE) {
535 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
539 pi = &lcore_power_info[lcore_id];
540 if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
541 pi->turbo_available && !pi->turbo_enable))
544 /* Frequencies in the array are from high to low. */
545 return set_freq_internal(pi, pi->curr_idx - 1);
549 power_cppc_cpufreq_freq_max(unsigned int lcore_id)
551 if (lcore_id >= RTE_MAX_LCORE) {
552 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
556 /* Frequencies in the array are from high to low. */
557 if (lcore_power_info[lcore_id].turbo_available) {
558 if (lcore_power_info[lcore_id].turbo_enable)
560 return set_freq_internal(
561 &lcore_power_info[lcore_id], 0);
563 /* Set to max non-turbo */
564 return set_freq_internal(
565 &lcore_power_info[lcore_id], 1);
567 return set_freq_internal(&lcore_power_info[lcore_id], 0);
571 power_cppc_cpufreq_freq_min(unsigned int lcore_id)
573 struct cppc_power_info *pi;
575 if (lcore_id >= RTE_MAX_LCORE) {
576 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
580 pi = &lcore_power_info[lcore_id];
582 /* Frequencies in the array are from high to low. */
583 return set_freq_internal(pi, pi->nb_freqs - 1);
587 power_cppc_turbo_status(unsigned int lcore_id)
589 struct cppc_power_info *pi;
591 if (lcore_id >= RTE_MAX_LCORE) {
592 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
596 pi = &lcore_power_info[lcore_id];
598 return pi->turbo_enable;
602 power_cppc_enable_turbo(unsigned int lcore_id)
604 struct cppc_power_info *pi;
606 if (lcore_id >= RTE_MAX_LCORE) {
607 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
611 pi = &lcore_power_info[lcore_id];
613 if (pi->turbo_available)
614 pi->turbo_enable = 1;
616 pi->turbo_enable = 0;
618 "Failed to enable turbo on lcore %u\n",
623 /* TODO: must set to max once enabling Turbo? Considering add condition:
624 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
626 /* Max may have changed, so call to max function */
627 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
629 "Failed to set frequency of lcore %u to max\n",
638 power_cppc_disable_turbo(unsigned int lcore_id)
640 struct cppc_power_info *pi;
642 if (lcore_id >= RTE_MAX_LCORE) {
643 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
647 pi = &lcore_power_info[lcore_id];
649 pi->turbo_enable = 0;
651 if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
652 /* Try to set freq to max by default coming out of turbo */
653 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
655 "Failed to set frequency of lcore %u to max\n",
665 power_cppc_get_capabilities(unsigned int lcore_id,
666 struct rte_power_core_capabilities *caps)
668 struct cppc_power_info *pi;
670 if (lcore_id >= RTE_MAX_LCORE) {
671 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
675 RTE_LOG(ERR, POWER, "Invalid argument\n");
679 pi = &lcore_power_info[lcore_id];
680 caps->capabilities = 0;
681 caps->turbo = !!(pi->turbo_available);