1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
5 #include "rte_power_intrinsics.h"
7 static bool wait_supported;
10 __get_umwait_val(const volatile void *p, const uint8_t sz)
14 return *(const volatile uint8_t *)p;
15 case sizeof(uint16_t):
16 return *(const volatile uint16_t *)p;
17 case sizeof(uint32_t):
18 return *(const volatile uint32_t *)p;
19 case sizeof(uint64_t):
20 return *(const volatile uint64_t *)p;
22 /* shouldn't happen */
29 __check_val_size(const uint8_t sz)
32 case sizeof(uint8_t): /* fall-through */
33 case sizeof(uint16_t): /* fall-through */
34 case sizeof(uint32_t): /* fall-through */
35 case sizeof(uint64_t): /* fall-through */
44 * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
45 * For more information about usage of these instructions, please refer to
46 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
49 rte_power_monitor(const volatile void *p, const uint64_t expected_value,
50 const uint64_t value_mask, const uint64_t tsc_timestamp,
51 const uint8_t data_sz)
53 const uint32_t tsc_l = (uint32_t)tsc_timestamp;
54 const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
56 /* prevent user from running this instruction if it's not supported */
60 if (__check_val_size(data_sz) < 0)
64 * we're using raw byte codes for now as only the newest compiler
65 * versions support this instruction natively.
68 /* set address for UMONITOR */
69 asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
74 const uint64_t cur_value = __get_umwait_val(p, data_sz);
75 const uint64_t masked = cur_value & value_mask;
77 /* if the masked value is already matching, abort */
78 if (masked == expected_value)
82 asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
84 : "D"(0), /* enter C0.2 */
85 "a"(tsc_l), "d"(tsc_h));
91 * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
92 * For more information about usage of these instructions, please refer to
93 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
96 rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
97 const uint64_t value_mask, const uint64_t tsc_timestamp,
98 const uint8_t data_sz, rte_spinlock_t *lck)
100 const uint32_t tsc_l = (uint32_t)tsc_timestamp;
101 const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
103 /* prevent user from running this instruction if it's not supported */
107 if (__check_val_size(data_sz) < 0)
111 * we're using raw byte codes for now as only the newest compiler
112 * versions support this instruction natively.
115 /* set address for UMONITOR */
116 asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
121 const uint64_t cur_value = __get_umwait_val(p, data_sz);
122 const uint64_t masked = cur_value & value_mask;
124 /* if the masked value is already matching, abort */
125 if (masked == expected_value)
128 rte_spinlock_unlock(lck);
131 asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
132 : /* ignore rflags */
133 : "D"(0), /* enter C0.2 */
134 "a"(tsc_l), "d"(tsc_h));
136 rte_spinlock_lock(lck);
142 * This function uses TPAUSE instruction and will enter C0.2 state. For more
143 * information about usage of this instruction, please refer to Intel(R) 64 and
144 * IA-32 Architectures Software Developer's Manual.
147 rte_power_pause(const uint64_t tsc_timestamp)
149 const uint32_t tsc_l = (uint32_t)tsc_timestamp;
150 const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
152 /* prevent user from running this instruction if it's not supported */
157 asm volatile(".byte 0x66, 0x0f, 0xae, 0xf7;"
158 : /* ignore rflags */
159 : "D"(0), /* enter C0.2 */
160 "a"(tsc_l), "d"(tsc_h));
165 RTE_INIT(rte_power_intrinsics_init) {
166 struct rte_cpu_intrinsics i;
168 rte_cpu_get_intrinsics_support(&i);
170 if (i.power_monitor && i.power_pause)