1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2021 Intel Corporation
3 * Copyright(c) 2021 Arm Limited
6 #include <rte_memcpy.h>
7 #include <rte_memory.h>
9 #include "power_cppc_cpufreq.h"
10 #include "power_common.h"
12 /* macros used for rounding frequency to nearest 100000 */
13 #define FREQ_ROUNDING_DELTA 50000
14 #define ROUND_FREQ_TO_N_100000 100000
16 /* the unit of highest_perf and nominal_perf differs on different arm platforms.
17 * For highest_perf, it maybe 300 or 3000000, both means 3.0GHz.
19 #define UNIT_DIFF 10000
21 #define POWER_CONVERT_TO_DECIMAL 10
23 #define POWER_GOVERNOR_USERSPACE "userspace"
24 #define POWER_SYSFILE_SETSPEED \
25 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_setspeed"
26 #define POWER_SYSFILE_SCALING_MAX_FREQ \
27 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_max_freq"
28 #define POWER_SYSFILE_SCALING_MIN_FREQ \
29 "/sys/devices/system/cpu/cpu%u/cpufreq/scaling_min_freq"
30 #define POWER_SYSFILE_HIGHEST_PERF \
31 "/sys/devices/system/cpu/cpu%u/acpi_cppc/highest_perf"
32 #define POWER_SYSFILE_NOMINAL_PERF \
33 "/sys/devices/system/cpu/cpu%u/acpi_cppc/nominal_perf"
34 #define POWER_SYSFILE_SYS_MAX \
35 "/sys/devices/system/cpu/cpu%u/cpufreq/cpuinfo_max_freq"
37 #define POWER_CPPC_DRIVER "cppc-cpufreq"
38 #define BUS_FREQ 100000
48 * Power info per lcore.
50 struct cppc_power_info {
51 unsigned int lcore_id; /**< Logical core id */
52 uint32_t state; /**< Power in use state */
53 FILE *f; /**< FD of scaling_setspeed */
54 char governor_ori[32]; /**< Original governor name */
55 uint32_t curr_idx; /**< Freq index in freqs array */
56 uint32_t highest_perf; /**< system wide max freq */
57 uint32_t nominal_perf; /**< system wide nominal freq */
58 uint16_t turbo_available; /**< Turbo Boost available */
59 uint16_t turbo_enable; /**< Turbo Boost enable/disable */
60 uint32_t nb_freqs; /**< number of available freqs */
61 uint32_t freqs[RTE_MAX_LCORE_FREQS]; /**< Frequency array */
62 } __rte_cache_aligned;
64 static struct cppc_power_info lcore_power_info[RTE_MAX_LCORE];
67 * It is to set specific freq for specific logical core, according to the index
68 * of supported frequencies.
71 set_freq_internal(struct cppc_power_info *pi, uint32_t idx)
73 if (idx >= RTE_MAX_LCORE_FREQS || idx >= pi->nb_freqs) {
74 RTE_LOG(ERR, POWER, "Invalid frequency index %u, which "
75 "should be less than %u\n", idx, pi->nb_freqs);
79 /* Check if it is the same as current */
80 if (idx == pi->curr_idx)
83 POWER_DEBUG_TRACE("Frequency[%u] %u to be set for lcore %u\n",
84 idx, pi->freqs[idx], pi->lcore_id);
85 if (fseek(pi->f, 0, SEEK_SET) < 0) {
86 RTE_LOG(ERR, POWER, "Fail to set file position indicator to 0 "
87 "for setting frequency for lcore %u\n", pi->lcore_id);
90 if (fprintf(pi->f, "%u", pi->freqs[idx]) < 0) {
91 RTE_LOG(ERR, POWER, "Fail to write new frequency for "
92 "lcore %u\n", pi->lcore_id);
102 * It is to check the current scaling governor by reading sys file, and then
103 * set it into 'userspace' if it is not by writing the sys file. The original
104 * governor will be saved for rolling back.
107 power_set_governor_userspace(struct cppc_power_info *pi)
109 return power_set_governor(pi->lcore_id, POWER_GOVERNOR_USERSPACE,
110 pi->governor_ori, sizeof(pi->governor_ori));
114 power_check_turbo(struct cppc_power_info *pi)
116 FILE *f_nom = NULL, *f_max = NULL, *f_cmax = NULL;
118 uint32_t nominal_perf = 0, highest_perf = 0, cpuinfo_max_freq = 0;
120 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_HIGHEST_PERF,
123 RTE_LOG(ERR, POWER, "failed to open %s\n",
124 POWER_SYSFILE_HIGHEST_PERF);
128 open_core_sysfs_file(&f_nom, "r", POWER_SYSFILE_NOMINAL_PERF,
131 RTE_LOG(ERR, POWER, "failed to open %s\n",
132 POWER_SYSFILE_NOMINAL_PERF);
136 open_core_sysfs_file(&f_cmax, "r", POWER_SYSFILE_SYS_MAX,
138 if (f_cmax == NULL) {
139 RTE_LOG(ERR, POWER, "failed to open %s\n",
140 POWER_SYSFILE_SYS_MAX);
144 ret = read_core_sysfs_u32(f_max, &highest_perf);
146 RTE_LOG(ERR, POWER, "Failed to read %s\n",
147 POWER_SYSFILE_HIGHEST_PERF);
151 ret = read_core_sysfs_u32(f_nom, &nominal_perf);
153 RTE_LOG(ERR, POWER, "Failed to read %s\n",
154 POWER_SYSFILE_NOMINAL_PERF);
158 ret = read_core_sysfs_u32(f_cmax, &cpuinfo_max_freq);
160 RTE_LOG(ERR, POWER, "Failed to read %s\n",
161 POWER_SYSFILE_SYS_MAX);
165 pi->highest_perf = highest_perf;
166 pi->nominal_perf = nominal_perf;
168 if ((highest_perf > nominal_perf) && ((cpuinfo_max_freq == highest_perf)
169 || cpuinfo_max_freq == highest_perf * UNIT_DIFF)) {
170 pi->turbo_available = 1;
171 pi->turbo_enable = 1;
173 POWER_DEBUG_TRACE("Lcore %u can do Turbo Boost! highest perf %u, "
175 pi->lcore_id, highest_perf, nominal_perf);
177 pi->turbo_available = 0;
178 pi->turbo_enable = 0;
179 POWER_DEBUG_TRACE("Lcore %u Turbo not available! highest perf %u, "
181 pi->lcore_id, highest_perf, nominal_perf);
196 * It is to get the available frequencies of the specific lcore by reading the
200 power_get_available_freqs(struct cppc_power_info *pi)
202 FILE *f_min = NULL, *f_max = NULL;
204 uint32_t scaling_min_freq = 0, scaling_max_freq = 0, nominal_perf = 0;
205 uint32_t i, num_freqs = 0;
207 open_core_sysfs_file(&f_max, "r", POWER_SYSFILE_SCALING_MAX_FREQ,
210 RTE_LOG(ERR, POWER, "failed to open %s\n",
211 POWER_SYSFILE_SCALING_MAX_FREQ);
215 open_core_sysfs_file(&f_min, "r", POWER_SYSFILE_SCALING_MIN_FREQ,
218 RTE_LOG(ERR, POWER, "failed to open %s\n",
219 POWER_SYSFILE_SCALING_MIN_FREQ);
223 ret = read_core_sysfs_u32(f_max, &scaling_max_freq);
225 RTE_LOG(ERR, POWER, "Failed to read %s\n",
226 POWER_SYSFILE_SCALING_MAX_FREQ);
230 ret = read_core_sysfs_u32(f_min, &scaling_min_freq);
232 RTE_LOG(ERR, POWER, "Failed to read %s\n",
233 POWER_SYSFILE_SCALING_MIN_FREQ);
237 power_check_turbo(pi);
239 if (scaling_max_freq < scaling_min_freq)
242 /* If turbo is available then there is one extra freq bucket
243 * to store the sys max freq which value is scaling_max_freq
245 nominal_perf = (pi->nominal_perf < UNIT_DIFF) ?
246 pi->nominal_perf * UNIT_DIFF : pi->nominal_perf;
247 num_freqs = (nominal_perf - scaling_min_freq) / BUS_FREQ + 1 +
250 /* Generate the freq bucket array. */
251 for (i = 0, pi->nb_freqs = 0; i < num_freqs; i++) {
252 if ((i == 0) && pi->turbo_available)
253 pi->freqs[pi->nb_freqs++] = scaling_max_freq;
255 pi->freqs[pi->nb_freqs++] =
256 nominal_perf - (i - pi->turbo_available) * BUS_FREQ;
261 POWER_DEBUG_TRACE("%d frequency(s) of lcore %u are available\n",
262 num_freqs, pi->lcore_id);
274 * It is to fopen the sys file for the future setting the lcore frequency.
277 power_init_for_setting_freq(struct cppc_power_info *pi)
284 open_core_sysfs_file(&f, "rw+", POWER_SYSFILE_SETSPEED, pi->lcore_id);
286 RTE_LOG(ERR, POWER, "failed to open %s\n",
287 POWER_SYSFILE_SETSPEED);
291 ret = read_core_sysfs_s(f, buf, sizeof(buf));
293 RTE_LOG(ERR, POWER, "Failed to read %s\n",
294 POWER_SYSFILE_SETSPEED);
298 freq = strtoul(buf, NULL, POWER_CONVERT_TO_DECIMAL);
300 /* convert the frequency to nearest 100000 value
301 * Ex: if freq=1396789 then freq_conv=1400000
302 * Ex: if freq=800030 then freq_conv=800000
304 unsigned int freq_conv = 0;
305 freq_conv = (freq + FREQ_ROUNDING_DELTA)
306 / ROUND_FREQ_TO_N_100000;
307 freq_conv = freq_conv * ROUND_FREQ_TO_N_100000;
309 for (i = 0; i < pi->nb_freqs; i++) {
310 if (freq_conv == pi->freqs[i]) {
325 power_cppc_cpufreq_check_supported(void)
327 return cpufreq_check_scaling_driver(POWER_CPPC_DRIVER);
331 power_cppc_cpufreq_init(unsigned int lcore_id)
333 struct cppc_power_info *pi;
336 if (lcore_id >= RTE_MAX_LCORE) {
337 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
338 lcore_id, RTE_MAX_LCORE - 1U);
342 pi = &lcore_power_info[lcore_id];
343 exp_state = POWER_IDLE;
344 /* The power in use state works as a guard variable between
345 * the CPU frequency control initialization and exit process.
346 * The ACQUIRE memory ordering here pairs with the RELEASE
347 * ordering below as lock to make sure the frequency operations
348 * in the critical section are done under the correct state.
350 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
352 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
353 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
354 "in use\n", lcore_id);
358 pi->lcore_id = lcore_id;
359 /* Check and set the governor */
360 if (power_set_governor_userspace(pi) < 0) {
361 RTE_LOG(ERR, POWER, "Cannot set governor of lcore %u to "
362 "userspace\n", lcore_id);
366 /* Get the available frequencies */
367 if (power_get_available_freqs(pi) < 0) {
368 RTE_LOG(ERR, POWER, "Cannot get available frequencies of "
369 "lcore %u\n", lcore_id);
373 /* Init for setting lcore frequency */
374 if (power_init_for_setting_freq(pi) < 0) {
375 RTE_LOG(ERR, POWER, "Cannot init for setting frequency for "
376 "lcore %u\n", lcore_id);
380 /* Set freq to max by default */
381 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
382 RTE_LOG(ERR, POWER, "Cannot set frequency of lcore %u "
383 "to max\n", lcore_id);
387 RTE_LOG(INFO, POWER, "Initialized successfully for lcore %u "
388 "power management\n", lcore_id);
390 __atomic_store_n(&(pi->state), POWER_USED, __ATOMIC_RELEASE);
395 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
400 * It is to check the governor and then set the original governor back if
401 * needed by writing the sys file.
404 power_set_governor_original(struct cppc_power_info *pi)
406 return power_set_governor(pi->lcore_id, pi->governor_ori, NULL, 0);
410 power_cppc_cpufreq_exit(unsigned int lcore_id)
412 struct cppc_power_info *pi;
415 if (lcore_id >= RTE_MAX_LCORE) {
416 RTE_LOG(ERR, POWER, "Lcore id %u can not exceeds %u\n",
417 lcore_id, RTE_MAX_LCORE - 1U);
420 pi = &lcore_power_info[lcore_id];
421 exp_state = POWER_USED;
422 /* The power in use state works as a guard variable between
423 * the CPU frequency control initialization and exit process.
424 * The ACQUIRE memory ordering here pairs with the RELEASE
425 * ordering below as lock to make sure the frequency operations
426 * in the critical section are done under the correct state.
428 if (!__atomic_compare_exchange_n(&(pi->state), &exp_state,
430 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
431 RTE_LOG(INFO, POWER, "Power management of lcore %u is "
432 "not used\n", lcore_id);
436 /* Close FD of setting freq */
440 /* Set the governor back to the original */
441 if (power_set_governor_original(pi) < 0) {
442 RTE_LOG(ERR, POWER, "Cannot set the governor of %u back "
443 "to the original\n", lcore_id);
447 RTE_LOG(INFO, POWER, "Power management of lcore %u has exited from "
448 "'userspace' mode and been set back to the "
449 "original\n", lcore_id);
450 __atomic_store_n(&(pi->state), POWER_IDLE, __ATOMIC_RELEASE);
455 __atomic_store_n(&(pi->state), POWER_UNKNOWN, __ATOMIC_RELEASE);
461 power_cppc_cpufreq_freqs(unsigned int lcore_id, uint32_t *freqs, uint32_t num)
463 struct cppc_power_info *pi;
465 if (lcore_id >= RTE_MAX_LCORE) {
466 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
471 RTE_LOG(ERR, POWER, "NULL buffer supplied\n");
475 pi = &lcore_power_info[lcore_id];
476 if (num < pi->nb_freqs) {
477 RTE_LOG(ERR, POWER, "Buffer size is not enough\n");
480 rte_memcpy(freqs, pi->freqs, pi->nb_freqs * sizeof(uint32_t));
486 power_cppc_cpufreq_get_freq(unsigned int lcore_id)
488 if (lcore_id >= RTE_MAX_LCORE) {
489 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
490 return RTE_POWER_INVALID_FREQ_INDEX;
493 return lcore_power_info[lcore_id].curr_idx;
497 power_cppc_cpufreq_set_freq(unsigned int lcore_id, uint32_t index)
499 if (lcore_id >= RTE_MAX_LCORE) {
500 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
504 return set_freq_internal(&(lcore_power_info[lcore_id]), index);
508 power_cppc_cpufreq_freq_down(unsigned int lcore_id)
510 struct cppc_power_info *pi;
512 if (lcore_id >= RTE_MAX_LCORE) {
513 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
517 pi = &lcore_power_info[lcore_id];
518 if (pi->curr_idx + 1 == pi->nb_freqs)
521 /* Frequencies in the array are from high to low. */
522 return set_freq_internal(pi, pi->curr_idx + 1);
526 power_cppc_cpufreq_freq_up(unsigned int lcore_id)
528 struct cppc_power_info *pi;
530 if (lcore_id >= RTE_MAX_LCORE) {
531 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
535 pi = &lcore_power_info[lcore_id];
536 if (pi->curr_idx == 0 || (pi->curr_idx == 1 &&
537 pi->turbo_available && !pi->turbo_enable))
540 /* Frequencies in the array are from high to low. */
541 return set_freq_internal(pi, pi->curr_idx - 1);
545 power_cppc_cpufreq_freq_max(unsigned int lcore_id)
547 if (lcore_id >= RTE_MAX_LCORE) {
548 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
552 /* Frequencies in the array are from high to low. */
553 if (lcore_power_info[lcore_id].turbo_available) {
554 if (lcore_power_info[lcore_id].turbo_enable)
556 return set_freq_internal(
557 &lcore_power_info[lcore_id], 0);
559 /* Set to max non-turbo */
560 return set_freq_internal(
561 &lcore_power_info[lcore_id], 1);
563 return set_freq_internal(&lcore_power_info[lcore_id], 0);
567 power_cppc_cpufreq_freq_min(unsigned int lcore_id)
569 struct cppc_power_info *pi;
571 if (lcore_id >= RTE_MAX_LCORE) {
572 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
576 pi = &lcore_power_info[lcore_id];
578 /* Frequencies in the array are from high to low. */
579 return set_freq_internal(pi, pi->nb_freqs - 1);
583 power_cppc_turbo_status(unsigned int lcore_id)
585 struct cppc_power_info *pi;
587 if (lcore_id >= RTE_MAX_LCORE) {
588 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
592 pi = &lcore_power_info[lcore_id];
594 return pi->turbo_enable;
598 power_cppc_enable_turbo(unsigned int lcore_id)
600 struct cppc_power_info *pi;
602 if (lcore_id >= RTE_MAX_LCORE) {
603 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
607 pi = &lcore_power_info[lcore_id];
609 if (pi->turbo_available)
610 pi->turbo_enable = 1;
612 pi->turbo_enable = 0;
614 "Failed to enable turbo on lcore %u\n",
619 /* TODO: must set to max once enbling Turbo? Considering add condition:
620 * if ((pi->turbo_available) && (pi->curr_idx <= 1))
622 /* Max may have changed, so call to max function */
623 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
625 "Failed to set frequency of lcore %u to max\n",
634 power_cppc_disable_turbo(unsigned int lcore_id)
636 struct cppc_power_info *pi;
638 if (lcore_id >= RTE_MAX_LCORE) {
639 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
643 pi = &lcore_power_info[lcore_id];
645 pi->turbo_enable = 0;
647 if ((pi->turbo_available) && (pi->curr_idx <= 1)) {
648 /* Try to set freq to max by default coming out of turbo */
649 if (power_cppc_cpufreq_freq_max(lcore_id) < 0) {
651 "Failed to set frequency of lcore %u to max\n",
661 power_cppc_get_capabilities(unsigned int lcore_id,
662 struct rte_power_core_capabilities *caps)
664 struct cppc_power_info *pi;
666 if (lcore_id >= RTE_MAX_LCORE) {
667 RTE_LOG(ERR, POWER, "Invalid lcore ID\n");
671 RTE_LOG(ERR, POWER, "Invalid argument\n");
675 pi = &lcore_power_info[lcore_id];
676 caps->capabilities = 0;
677 caps->turbo = !!(pi->turbo_available);