eal: add wait until equal API

[dpdk.git] / lib / librte_eal / common / include / arch / arm / rte_pause_64.h

diff --git a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
index 93895d3..e87d10b 100644 (file)
--- a/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
+++ b/lib/librte_eal/common/include/arch/arm/rte_pause_64.h
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2017 Cavium, Inc
+ * Copyright(c) 2019 Arm Limited
  */
 
 #ifndef _RTE_PAUSE_ARM64_H_
@@ -10,6 +11,11 @@ extern "C" {
 #endif
 
 #include <rte_common.h>
+
+#ifdef RTE_ARM_USE_WFE
+#define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+#endif
+
 #include "generic/rte_pause.h"
 
 static inline void rte_pause(void)
@@ -17,6 +23,133 @@ static inline void rte_pause(void)
        asm volatile("yield" ::: "memory");
 }
 
+#ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
+
+/* Send an event to quit WFE. */
+#define __SEVL() { asm volatile("sevl" : : : "memory"); }
+
+/* Put processor into low power WFE(Wait For Event) state. */
+#define __WFE() { asm volatile("wfe" : : : "memory"); }
+
+static __rte_always_inline void
+rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
+               int memorder)
+{
+       uint16_t value;
+
+       assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+
+       /*
+        * Atomic exclusive load from addr, it returns the 16-bit content of
+        * *addr while making it 'monitored',when it is written by someone
+        * else, the 'monitored' state is cleared and a event is generated
+        * implicitly to exit WFE.
+        */
+#define __LOAD_EXC_16(src, dst, memorder) {               \
+       if (memorder == __ATOMIC_RELAXED) {               \
+               asm volatile("ldxrh %w[tmp], [%x[addr]]"  \
+                       : [tmp] "=&r" (dst)               \
+                       : [addr] "r"(src)                 \
+                       : "memory");                      \
+       } else {                                          \
+               asm volatile("ldaxrh %w[tmp], [%x[addr]]" \
+                       : [tmp] "=&r" (dst)               \
+                       : [addr] "r"(src)                 \
+                       : "memory");                      \
+       } }
+
+       __LOAD_EXC_16(addr, value, memorder)
+       if (value != expected) {
+               __SEVL()
+               do {
+                       __WFE()
+                       __LOAD_EXC_16(addr, value, memorder)
+               } while (value != expected);
+       }
+#undef __LOAD_EXC_16
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
+               int memorder)
+{
+       uint32_t value;
+
+       assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+
+       /*
+        * Atomic exclusive load from addr, it returns the 32-bit content of
+        * *addr while making it 'monitored',when it is written by someone
+        * else, the 'monitored' state is cleared and a event is generated
+        * implicitly to exit WFE.
+        */
+#define __LOAD_EXC_32(src, dst, memorder) {              \
+       if (memorder == __ATOMIC_RELAXED) {              \
+               asm volatile("ldxr %w[tmp], [%x[addr]]"  \
+                       : [tmp] "=&r" (dst)              \
+                       : [addr] "r"(src)                \
+                       : "memory");                     \
+       } else {                                         \
+               asm volatile("ldaxr %w[tmp], [%x[addr]]" \
+                       : [tmp] "=&r" (dst)              \
+                       : [addr] "r"(src)                \
+                       : "memory");                     \
+       } }
+
+       __LOAD_EXC_32(addr, value, memorder)
+       if (value != expected) {
+               __SEVL()
+               do {
+                       __WFE()
+                       __LOAD_EXC_32(addr, value, memorder)
+               } while (value != expected);
+       }
+#undef __LOAD_EXC_32
+}
+
+static __rte_always_inline void
+rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
+               int memorder)
+{
+       uint64_t value;
+
+       assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
+
+       /*
+        * Atomic exclusive load from addr, it returns the 64-bit content of
+        * *addr while making it 'monitored',when it is written by someone
+        * else, the 'monitored' state is cleared and a event is generated
+        * implicitly to exit WFE.
+        */
+#define __LOAD_EXC_64(src, dst, memorder) {              \
+       if (memorder == __ATOMIC_RELAXED) {              \
+               asm volatile("ldxr %x[tmp], [%x[addr]]"  \
+                       : [tmp] "=&r" (dst)              \
+                       : [addr] "r"(src)                \
+                       : "memory");                     \
+       } else {                                         \
+               asm volatile("ldaxr %x[tmp], [%x[addr]]" \
+                       : [tmp] "=&r" (dst)              \
+                       : [addr] "r"(src)                \
+                       : "memory");                     \
+       } }
+
+       __LOAD_EXC_64(addr, value, memorder)
+       if (value != expected) {
+               __SEVL()
+               do {
+                       __WFE()
+                       __LOAD_EXC_64(addr, value, memorder)
+               } while (value != expected);
+       }
+}
+#undef __LOAD_EXC_64
+
+#undef __SEVL
+#undef __WFE
+
+#endif
+
 #ifdef __cplusplus
 }
 #endif