From 264ff3f250d51c99084495bea3ca329d2c995b79 Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Fri, 5 Nov 2021 14:27:12 +0530 Subject: [PATCH] eal/arm: inline 128-bit atomic compare exchange with GCC GCC [1] now assigns even register pairs for CASP, the fix has also been backported to all stable releases of older GCC versions. Removing the manual register allocation allows GCC to inline the functions and pick optimal registers for performing CASP. 1: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=563cc649beaf Signed-off-by: Pavan Nikhilesh Acked-by: Ruifeng Wang --- lib/eal/arm/include/rte_atomic_64.h | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/eal/arm/include/rte_atomic_64.h b/lib/eal/arm/include/rte_atomic_64.h index fa6f334c0d..6047911507 100644 --- a/lib/eal/arm/include/rte_atomic_64.h +++ b/lib/eal/arm/include/rte_atomic_64.h @@ -46,12 +46,8 @@ rte_atomic_thread_fence(int memorder) /*------------------------ 128 bit atomic operations -------------------------*/ #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) -#if defined(RTE_CC_CLANG) -#define __LSE_PREAMBLE ".arch armv8-a+lse\n" -#else -#define __LSE_PREAMBLE "" -#endif +#if defined(RTE_CC_CLANG) #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ static __rte_noinline void \ cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ @@ -65,7 +61,7 @@ cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ asm volatile( \ - __LSE_PREAMBLE \ + ".arch armv8-a+lse\n" \ op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ : [old0] "+r" (x0), \ [old1] "+r" (x1) \ @@ -76,13 +72,24 @@ cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ old->val[0] = x0; \ old->val[1] = x1; \ } +#else +#define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ +static __rte_always_inline void \ +cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ +{ \ + asm volatile( \ + op_string " %[old], %H[old], %[upd], %H[upd], [%[dst]]" \ + : [old] "+r"(old->int128) \ + : [upd] "r"(updated.int128), [dst] "r"(dst) \ + : "memory"); \ +} +#endif __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp") __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa") __ATOMIC128_CAS_OP(__cas_128_release, "caspl") __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal") -#undef __LSE_PREAMBLE #undef __ATOMIC128_CAS_OP #endif -- 2.20.1