lib/eal/arm/include/rte_atomic_64.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015 Cavium, Inc
   3  * Copyright(c) 2020 Arm Limited
   4  */
   5
   6 #ifndef _RTE_ATOMIC_ARM64_H_
   7 #define _RTE_ATOMIC_ARM64_H_
   8
   9 #ifndef RTE_FORCE_INTRINSICS
  10 #  error Platform must be built with RTE_FORCE_INTRINSICS
  11 #endif
  12
  13 #ifdef __cplusplus
  14 extern "C" {
  15 #endif
  16
  17 #include "generic/rte_atomic.h"
  18 #include <rte_branch_prediction.h>
  19 #include <rte_compat.h>
  20 #include <rte_debug.h>
  21
  22 #define rte_mb() asm volatile("dmb osh" : : : "memory")
  23
  24 #define rte_wmb() asm volatile("dmb oshst" : : : "memory")
  25
  26 #define rte_rmb() asm volatile("dmb oshld" : : : "memory")
  27
  28 #define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
  29
  30 #define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
  31
  32 #define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
  33
  34 #define rte_io_mb() rte_mb()
  35
  36 #define rte_io_wmb() rte_wmb()
  37
  38 #define rte_io_rmb() rte_rmb()
  39
  40 static __rte_always_inline void
  41 rte_atomic_thread_fence(int memorder)
  42 {
  43         __atomic_thread_fence(memorder);
  44 }
  45
  46 /*------------------------ 128 bit atomic operations -------------------------*/
  47
  48 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
  49
  50 #if defined(RTE_CC_CLANG)
  51 #define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
  52 static __rte_noinline void                                                  \
  53 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated)     \
  54 {                                                                           \
  55         /* caspX instructions register pair must start from even-numbered
  56          * register at operand 1.
  57          * So, specify registers for local variables here.
  58          */                                                                 \
  59         register uint64_t x0 __asm("x0") = (uint64_t)old->val[0];           \
  60         register uint64_t x1 __asm("x1") = (uint64_t)old->val[1];           \
  61         register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0];        \
  62         register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1];        \
  63         asm volatile(                                                       \
  64                 ".arch armv8-a+lse\n"                                       \
  65                 op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]"   \
  66                 : [old0] "+r" (x0),                                         \
  67                 [old1] "+r" (x1)                                            \
  68                 : [upd0] "r" (x2),                                          \
  69                 [upd1] "r" (x3),                                            \
  70                 [dst] "r" (dst)                                             \
  71                 : "memory");                                                \
  72         old->val[0] = x0;                                                   \
  73         old->val[1] = x1;                                                   \
  74 }
  75 #else
  76 #define __ATOMIC128_CAS_OP(cas_op_name, op_string)                          \
  77 static __rte_always_inline void                                             \
  78 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated)     \
  79 {                                                                           \
  80         asm volatile(                                                       \
  81                 op_string " %[old], %H[old], %[upd], %H[upd], [%[dst]]"     \
  82                 : [old] "+r"(old->int128)                                   \
  83                 : [upd] "r"(updated.int128), [dst] "r"(dst)                 \
  84                 : "memory");                                                \
  85 }
  86 #endif
  87
  88 __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp")
  89 __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa")
  90 __ATOMIC128_CAS_OP(__cas_128_release, "caspl")
  91 __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal")
  92
  93 #undef __ATOMIC128_CAS_OP
  94
  95 #endif
  96
  97 __rte_experimental
  98 static inline int
  99 rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp,
 100                 const rte_int128_t *src, unsigned int weak, int success,
 101                 int failure)
 102 {
 103         /* Always do strong CAS */
 104         RTE_SET_USED(weak);
 105         /* Ignore memory ordering for failure, memory order for
 106          * success must be stronger or equal
 107          */
 108         RTE_SET_USED(failure);
 109         /* Find invalid memory order */
 110         RTE_ASSERT(success == __ATOMIC_RELAXED ||
 111                 success == __ATOMIC_ACQUIRE ||
 112                 success == __ATOMIC_RELEASE ||
 113                 success == __ATOMIC_ACQ_REL ||
 114                 success == __ATOMIC_SEQ_CST);
 115
 116         rte_int128_t expected = *exp;
 117         rte_int128_t desired = *src;
 118         rte_int128_t old;
 119
 120 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
 121         if (success == __ATOMIC_RELAXED)
 122                 __cas_128_relaxed(dst, exp, desired);
 123         else if (success == __ATOMIC_ACQUIRE)
 124                 __cas_128_acquire(dst, exp, desired);
 125         else if (success == __ATOMIC_RELEASE)
 126                 __cas_128_release(dst, exp, desired);
 127         else
 128                 __cas_128_acq_rel(dst, exp, desired);
 129         old = *exp;
 130 #else
 131 #define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
 132 #define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
 133                 (mo) == __ATOMIC_SEQ_CST)
 134
 135         int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
 136         int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED;
 137
 138 #undef __HAS_ACQ
 139 #undef __HAS_RLS
 140
 141         uint32_t ret = 1;
 142
 143         /* ldx128 can not guarantee atomic,
 144          * Must write back src or old to verify atomicity of ldx128;
 145          */
 146         do {
 147
 148 #define __LOAD_128(op_string, src, dst) { \
 149         asm volatile(                     \
 150                 op_string " %0, %1, %2"   \
 151                 : "=&r" (dst.val[0]),     \
 152                   "=&r" (dst.val[1])      \
 153                 : "Q" (src->val[0])       \
 154                 : "memory"); }
 155
 156                 if (ldx_mo == __ATOMIC_RELAXED)
 157                         __LOAD_128("ldxp", dst, old)
 158                 else
 159                         __LOAD_128("ldaxp", dst, old)
 160
 161 #undef __LOAD_128
 162
 163 #define __STORE_128(op_string, dst, src, ret) { \
 164         asm volatile(                           \
 165                 op_string " %w0, %1, %2, %3"    \
 166                 : "=&r" (ret)                   \
 167                 : "r" (src.val[0]),             \
 168                   "r" (src.val[1]),             \
 169                   "Q" (dst->val[0])             \
 170                 : "memory"); }
 171
 172                 if (likely(old.int128 == expected.int128)) {
 173                         if (stx_mo == __ATOMIC_RELAXED)
 174                                 __STORE_128("stxp", dst, desired, ret)
 175                         else
 176                                 __STORE_128("stlxp", dst, desired, ret)
 177                 } else {
 178                         /* In the failure case (since 'weak' is ignored and only
 179                          * weak == 0 is implemented), expected should contain
 180                          * the atomically read value of dst. This means, 'old'
 181                          * needs to be stored back to ensure it was read
 182                          * atomically.
 183                          */
 184                         if (stx_mo == __ATOMIC_RELAXED)
 185                                 __STORE_128("stxp", dst, old, ret)
 186                         else
 187                                 __STORE_128("stlxp", dst, old, ret)
 188                 }
 189
 190 #undef __STORE_128
 191
 192         } while (unlikely(ret));
 193
 194         /* Unconditionally updating the value of exp removes an 'if' statement.
 195          * The value of exp should already be in register if not in the cache.
 196          */
 197         *exp = old;
 198 #endif
 199
 200         return (old.int128 == expected.int128);
 201 }
 202
 203 #ifdef __cplusplus
 204 }
 205 #endif
 206
 207 #endif /* _RTE_ATOMIC_ARM64_H_ */