eal: uninline power intrinsics

Currently, power intrinsics are inline functions. Make them part of the
ABI so that we can have various internal data associated with them
without exposing said data to the outside world.

Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

diff --git a/lib/librte_eal/arm/include/rte_power_intrinsics.h b/lib/librte_eal/arm/include/rte_power_intrinsics.h
index a4a1bc11598fea0355385debda752acdea68cb0b..9e498e9ebf9d3308e187b8e585a2bb6b4c8d3b2d 100644 (file)
--- a/lib/librte_eal/arm/include/rte_power_intrinsics.h
+++ b/lib/librte_eal/arm/include/rte_power_intrinsics.h
@@ -13,46 +13,6 @@ extern "C" {
 
 #include "generic/rte_power_intrinsics.h"
 
-/**
- * This function is not supported on ARM.
- */
-static inline void
-rte_power_monitor(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz)
-{
-       RTE_SET_USED(p);
-       RTE_SET_USED(expected_value);
-       RTE_SET_USED(value_mask);
-       RTE_SET_USED(tsc_timestamp);
-       RTE_SET_USED(data_sz);
-}
-
-/**
- * This function is not supported on ARM.
- */
-static inline void
-rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz, rte_spinlock_t *lck)
-{
-       RTE_SET_USED(p);
-       RTE_SET_USED(expected_value);
-       RTE_SET_USED(value_mask);
-       RTE_SET_USED(tsc_timestamp);
-       RTE_SET_USED(lck);
-       RTE_SET_USED(data_sz);
-}
-
-/**
- * This function is not supported on ARM.
- */
-static inline void
-rte_power_pause(const uint64_t tsc_timestamp)
-{
-       RTE_SET_USED(tsc_timestamp);
-}
-
 #ifdef __cplusplus
 }
 #endif

diff --git a/lib/librte_eal/arm/meson.build b/lib/librte_eal/arm/meson.build
index d62875ebaed9b83d169e989bebd44cd9383d6fac..6ec53ea03a405215c729e7ca8dd23f7223941a6e 100644 (file)
--- a/lib/librte_eal/arm/meson.build
+++ b/lib/librte_eal/arm/meson.build
@@ -7,4 +7,5 @@ sources += files(
        'rte_cpuflags.c',
        'rte_cycles.c',
        'rte_hypervisor.c',
+       'rte_power_intrinsics.c',
 )

diff --git a/lib/librte_eal/arm/rte_power_intrinsics.c b/lib/librte_eal/arm/rte_power_intrinsics.c
new file mode 100644 (file)
index 0000000..ab1f44f
--- /dev/null
+++ b/lib/librte_eal/arm/rte_power_intrinsics.c
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+#include "rte_power_intrinsics.h"
+
+/**
+ * This function is not supported on ARM.
+ */
+void
+rte_power_monitor(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz)
+{
+       RTE_SET_USED(p);
+       RTE_SET_USED(expected_value);
+       RTE_SET_USED(value_mask);
+       RTE_SET_USED(tsc_timestamp);
+       RTE_SET_USED(data_sz);
+}
+
+/**
+ * This function is not supported on ARM.
+ */
+void
+rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz, rte_spinlock_t *lck)
+{
+       RTE_SET_USED(p);
+       RTE_SET_USED(expected_value);
+       RTE_SET_USED(value_mask);
+       RTE_SET_USED(tsc_timestamp);
+       RTE_SET_USED(lck);
+       RTE_SET_USED(data_sz);
+}
+
+/**
+ * This function is not supported on ARM.
+ */
+void
+rte_power_pause(const uint64_t tsc_timestamp)
+{
+       RTE_SET_USED(tsc_timestamp);
+}

diff --git a/lib/librte_eal/include/generic/rte_power_intrinsics.h b/lib/librte_eal/include/generic/rte_power_intrinsics.h
index dd520d90fa2db4b5c01ffaf413240cfe261da1f7..67977bd511df96761ccf3a35ce91a17dea424c91 100644 (file)
--- a/lib/librte_eal/include/generic/rte_power_intrinsics.h
+++ b/lib/librte_eal/include/generic/rte_power_intrinsics.h
@@ -52,7 +52,7 @@
  *   to undefined result.
  */
 __rte_experimental
-static inline void rte_power_monitor(const volatile void *p,
+void rte_power_monitor(const volatile void *p,
                const uint64_t expected_value, const uint64_t value_mask,
                const uint64_t tsc_timestamp, const uint8_t data_sz);
 
@@ -97,7 +97,7 @@ static inline void rte_power_monitor(const volatile void *p,
  *   wakes up.
  */
 __rte_experimental
-static inline void rte_power_monitor_sync(const volatile void *p,
+void rte_power_monitor_sync(const volatile void *p,
                const uint64_t expected_value, const uint64_t value_mask,
                const uint64_t tsc_timestamp, const uint8_t data_sz,
                rte_spinlock_t *lck);
@@ -118,6 +118,6 @@ static inline void rte_power_monitor_sync(const volatile void *p,
  *   architecture-dependent.
  */
 __rte_experimental
-static inline void rte_power_pause(const uint64_t tsc_timestamp);
+void rte_power_pause(const uint64_t tsc_timestamp);
 
 #endif /* _RTE_POWER_INTRINSIC_H_ */

diff --git a/lib/librte_eal/ppc/include/rte_power_intrinsics.h b/lib/librte_eal/ppc/include/rte_power_intrinsics.h
index 4ed03d521f0907441b7522f41989946da38ab505..c0e9ac279fbbc4b8fd92a433690e02adec15a023 100644 (file)
--- a/lib/librte_eal/ppc/include/rte_power_intrinsics.h
+++ b/lib/librte_eal/ppc/include/rte_power_intrinsics.h
@@ -13,46 +13,6 @@ extern "C" {
 
 #include "generic/rte_power_intrinsics.h"
 
-/**
- * This function is not supported on PPC64.
- */
-static inline void
-rte_power_monitor(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz)
-{
-       RTE_SET_USED(p);
-       RTE_SET_USED(expected_value);
-       RTE_SET_USED(value_mask);
-       RTE_SET_USED(tsc_timestamp);
-       RTE_SET_USED(data_sz);
-}
-
-/**
- * This function is not supported on PPC64.
- */
-static inline void
-rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz, rte_spinlock_t *lck)
-{
-       RTE_SET_USED(p);
-       RTE_SET_USED(expected_value);
-       RTE_SET_USED(value_mask);
-       RTE_SET_USED(tsc_timestamp);
-       RTE_SET_USED(lck);
-       RTE_SET_USED(data_sz);
-}
-
-/**
- * This function is not supported on PPC64.
- */
-static inline void
-rte_power_pause(const uint64_t tsc_timestamp)
-{
-       RTE_SET_USED(tsc_timestamp);
-}
-
 #ifdef __cplusplus
 }
 #endif

diff --git a/lib/librte_eal/ppc/meson.build b/lib/librte_eal/ppc/meson.build
index f4b6d95c4208240daee18e68075c71fca60cdbcf..43c46542fb48ca613563d388e7a0d94c9c8ef4c6 100644 (file)
--- a/lib/librte_eal/ppc/meson.build
+++ b/lib/librte_eal/ppc/meson.build
@@ -7,4 +7,5 @@ sources += files(
        'rte_cpuflags.c',
        'rte_cycles.c',
        'rte_hypervisor.c',
+       'rte_power_intrinsics.c',
 )

diff --git a/lib/librte_eal/ppc/rte_power_intrinsics.c b/lib/librte_eal/ppc/rte_power_intrinsics.c
new file mode 100644 (file)
index 0000000..84340ca
--- /dev/null
+++ b/lib/librte_eal/ppc/rte_power_intrinsics.c
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Intel Corporation
+ */
+
+#include "rte_power_intrinsics.h"
+
+/**
+ * This function is not supported on PPC64.
+ */
+void
+rte_power_monitor(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz)
+{
+       RTE_SET_USED(p);
+       RTE_SET_USED(expected_value);
+       RTE_SET_USED(value_mask);
+       RTE_SET_USED(tsc_timestamp);
+       RTE_SET_USED(data_sz);
+}
+
+/**
+ * This function is not supported on PPC64.
+ */
+void
+rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz, rte_spinlock_t *lck)
+{
+       RTE_SET_USED(p);
+       RTE_SET_USED(expected_value);
+       RTE_SET_USED(value_mask);
+       RTE_SET_USED(tsc_timestamp);
+       RTE_SET_USED(lck);
+       RTE_SET_USED(data_sz);
+}
+
+/**
+ * This function is not supported on PPC64.
+ */
+void
+rte_power_pause(const uint64_t tsc_timestamp)
+{
+       RTE_SET_USED(tsc_timestamp);
+}

diff --git a/lib/librte_eal/version.map b/lib/librte_eal/version.map
index b1db7ec79505feca751af3c0b5ccd2d5a06ff1be..32eceb886993f743a9789b8c2d94344118f5b98b 100644 (file)
--- a/lib/librte_eal/version.map
+++ b/lib/librte_eal/version.map
@@ -405,6 +405,9 @@ EXPERIMENTAL {
        rte_vect_set_max_simd_bitwidth;
 
        # added in 21.02
+       rte_power_monitor;
+       rte_power_monitor_sync;
+       rte_power_pause;
        rte_thread_tls_key_create;
        rte_thread_tls_key_delete;
        rte_thread_tls_value_get;

diff --git a/lib/librte_eal/x86/include/rte_power_intrinsics.h b/lib/librte_eal/x86/include/rte_power_intrinsics.h
index c7d790c85403c93ba5dcb6ee5890eb3c9558d8e3..e4c2b87f7341078c67ecb64f8c15abfc985db3a2 100644 (file)
--- a/lib/librte_eal/x86/include/rte_power_intrinsics.h
+++ b/lib/librte_eal/x86/include/rte_power_intrinsics.h
@@ -13,121 +13,6 @@ extern "C" {
 
 #include "generic/rte_power_intrinsics.h"
 
-static inline uint64_t
-__rte_power_get_umwait_val(const volatile void *p, const uint8_t sz)
-{
-       switch (sz) {
-       case sizeof(uint8_t):
-               return *(const volatile uint8_t *)p;
-       case sizeof(uint16_t):
-               return *(const volatile uint16_t *)p;
-       case sizeof(uint32_t):
-               return *(const volatile uint32_t *)p;
-       case sizeof(uint64_t):
-               return *(const volatile uint64_t *)p;
-       default:
-               /* this is an intrinsic, so we can't have any error handling */
-               RTE_ASSERT(0);
-               return 0;
-       }
-}
-
-/**
- * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
- * For more information about usage of these instructions, please refer to
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
- */
-static inline void
-rte_power_monitor(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz)
-{
-       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
-       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
-       /*
-        * we're using raw byte codes for now as only the newest compiler
-        * versions support this instruction natively.
-        */
-
-       /* set address for UMONITOR */
-       asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
-                       :
-                       : "D"(p));
-
-       if (value_mask) {
-               const uint64_t cur_value = __rte_power_get_umwait_val(p, data_sz);
-               const uint64_t masked = cur_value & value_mask;
-
-               /* if the masked value is already matching, abort */
-               if (masked == expected_value)
-                       return;
-       }
-       /* execute UMWAIT */
-       asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
-                       : /* ignore rflags */
-                       : "D"(0), /* enter C0.2 */
-                         "a"(tsc_l), "d"(tsc_h));
-}
-
-/**
- * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
- * For more information about usage of these instructions, please refer to
- * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
- */
-static inline void
-rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
-               const uint64_t value_mask, const uint64_t tsc_timestamp,
-               const uint8_t data_sz, rte_spinlock_t *lck)
-{
-       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
-       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
-       /*
-        * we're using raw byte codes for now as only the newest compiler
-        * versions support this instruction natively.
-        */
-
-       /* set address for UMONITOR */
-       asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
-                       :
-                       : "D"(p));
-
-       if (value_mask) {
-               const uint64_t cur_value = __rte_power_get_umwait_val(p, data_sz);
-               const uint64_t masked = cur_value & value_mask;
-
-               /* if the masked value is already matching, abort */
-               if (masked == expected_value)
-                       return;
-       }
-       rte_spinlock_unlock(lck);
-
-       /* execute UMWAIT */
-       asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
-                       : /* ignore rflags */
-                       : "D"(0), /* enter C0.2 */
-                         "a"(tsc_l), "d"(tsc_h));
-
-       rte_spinlock_lock(lck);
-}
-
-/**
- * This function uses TPAUSE instruction  and will enter C0.2 state. For more
- * information about usage of this instruction, please refer to Intel(R) 64 and
- * IA-32 Architectures Software Developer's Manual.
- */
-static inline void
-rte_power_pause(const uint64_t tsc_timestamp)
-{
-       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
-       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
-
-       /* execute TPAUSE */
-       asm volatile(".byte 0x66, 0x0f, 0xae, 0xf7;"
-               : /* ignore rflags */
-               : "D"(0), /* enter C0.2 */
-                 "a"(tsc_l), "d"(tsc_h));
-}
-
 #ifdef __cplusplus
 }
 #endif

diff --git a/lib/librte_eal/x86/meson.build b/lib/librte_eal/x86/meson.build
index e78f29002e115c70923e3065cba8ee43b6c5fb8f..dfd42dee0c0dbb848b104961ff76149d12c39d96 100644 (file)
--- a/lib/librte_eal/x86/meson.build
+++ b/lib/librte_eal/x86/meson.build
@@ -8,4 +8,5 @@ sources += files(
        'rte_cycles.c',
        'rte_hypervisor.c',
        'rte_spinlock.c',
+       'rte_power_intrinsics.c',
 )

diff --git a/lib/librte_eal/x86/rte_power_intrinsics.c b/lib/librte_eal/x86/rte_power_intrinsics.c
new file mode 100644 (file)
index 0000000..34c5fd9
--- /dev/null
+++ b/lib/librte_eal/x86/rte_power_intrinsics.c
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_power_intrinsics.h"
+
+static inline uint64_t
+__get_umwait_val(const volatile void *p, const uint8_t sz)
+{
+       switch (sz) {
+       case sizeof(uint8_t):
+               return *(const volatile uint8_t *)p;
+       case sizeof(uint16_t):
+               return *(const volatile uint16_t *)p;
+       case sizeof(uint32_t):
+               return *(const volatile uint32_t *)p;
+       case sizeof(uint64_t):
+               return *(const volatile uint64_t *)p;
+       default:
+               /* this is an intrinsic, so we can't have any error handling */
+               RTE_ASSERT(0);
+               return 0;
+       }
+}
+
+/**
+ * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
+ * For more information about usage of these instructions, please refer to
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
+ */
+void
+rte_power_monitor(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz)
+{
+       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
+       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
+       /*
+        * we're using raw byte codes for now as only the newest compiler
+        * versions support this instruction natively.
+        */
+
+       /* set address for UMONITOR */
+       asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
+                       :
+                       : "D"(p));
+
+       if (value_mask) {
+               const uint64_t cur_value = __get_umwait_val(p, data_sz);
+               const uint64_t masked = cur_value & value_mask;
+
+               /* if the masked value is already matching, abort */
+               if (masked == expected_value)
+                       return;
+       }
+       /* execute UMWAIT */
+       asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
+                       : /* ignore rflags */
+                       : "D"(0), /* enter C0.2 */
+                         "a"(tsc_l), "d"(tsc_h));
+}
+
+/**
+ * This function uses UMONITOR/UMWAIT instructions and will enter C0.2 state.
+ * For more information about usage of these instructions, please refer to
+ * Intel(R) 64 and IA-32 Architectures Software Developer's Manual.
+ */
+void
+rte_power_monitor_sync(const volatile void *p, const uint64_t expected_value,
+               const uint64_t value_mask, const uint64_t tsc_timestamp,
+               const uint8_t data_sz, rte_spinlock_t *lck)
+{
+       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
+       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
+       /*
+        * we're using raw byte codes for now as only the newest compiler
+        * versions support this instruction natively.
+        */
+
+       /* set address for UMONITOR */
+       asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7;"
+                       :
+                       : "D"(p));
+
+       if (value_mask) {
+               const uint64_t cur_value = __get_umwait_val(p, data_sz);
+               const uint64_t masked = cur_value & value_mask;
+
+               /* if the masked value is already matching, abort */
+               if (masked == expected_value)
+                       return;
+       }
+       rte_spinlock_unlock(lck);
+
+       /* execute UMWAIT */
+       asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7;"
+                       : /* ignore rflags */
+                       : "D"(0), /* enter C0.2 */
+                         "a"(tsc_l), "d"(tsc_h));
+
+       rte_spinlock_lock(lck);
+}
+
+/**
+ * This function uses TPAUSE instruction  and will enter C0.2 state. For more
+ * information about usage of this instruction, please refer to Intel(R) 64 and
+ * IA-32 Architectures Software Developer's Manual.
+ */
+void
+rte_power_pause(const uint64_t tsc_timestamp)
+{
+       const uint32_t tsc_l = (uint32_t)tsc_timestamp;
+       const uint32_t tsc_h = (uint32_t)(tsc_timestamp >> 32);
+
+       /* execute TPAUSE */
+       asm volatile(".byte 0x66, 0x0f, 0xae, 0xf7;"
+                       : /* ignore rflags */
+                       : "D"(0), /* enter C0.2 */
+                       "a"(tsc_l), "d"(tsc_h));
+}