1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
6 * Inspired from FreeBSD src/sys/amd64/include/atomic.h
7 * Copyright (c) 1998 Doug Rabson
8 * Copyright (c) 2019 Intel Corporation
12 #ifndef _RTE_ATOMIC_X86_H_
13 #error do not include this file directly, use <rte_atomic.h> instead
16 #ifndef _RTE_ATOMIC_X86_64_H_
17 #define _RTE_ATOMIC_X86_64_H_
20 #include <rte_common.h>
21 #include <rte_compat.h>
22 #include <rte_atomic.h>
24 /*------------------------- 64 bit atomic operations -------------------------*/
26 #ifndef RTE_FORCE_INTRINSICS
28 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
35 "cmpxchgq %[src], %[dst];"
37 : [res] "=a" (res), /* output */
39 : [src] "r" (src), /* input */
42 : "memory"); /* no-clobber list */
47 static inline uint64_t
48 rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
53 : "=r" (val), "=m" (*dst)
54 : "0" (val), "m" (*dst)
55 : "memory"); /* no-clobber list */
60 rte_atomic64_init(rte_atomic64_t *v)
66 rte_atomic64_read(rte_atomic64_t *v)
72 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
78 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
83 : [cnt] "=m" (v->cnt) /* output */
84 : [inc] "ir" (inc), /* input */
90 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
95 : [cnt] "=m" (v->cnt) /* output */
96 : [dec] "ir" (dec), /* input */
102 rte_atomic64_inc(rte_atomic64_t *v)
107 : [cnt] "=m" (v->cnt) /* output */
108 : "m" (v->cnt) /* input */
113 rte_atomic64_dec(rte_atomic64_t *v)
118 : [cnt] "=m" (v->cnt) /* output */
119 : "m" (v->cnt) /* input */
123 static inline int64_t
124 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
130 "xaddq %[prev], %[cnt]"
131 : [prev] "+r" (prev), /* output */
133 : "m" (v->cnt) /* input */
138 static inline int64_t
139 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
141 return rte_atomic64_add_return(v, -dec);
144 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
152 : [cnt] "+m" (v->cnt), /* output */
159 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
167 : [cnt] "+m" (v->cnt), /* output */
173 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
175 return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
178 static inline void rte_atomic64_clear(rte_atomic64_t *v)
184 /*------------------------ 128 bit atomic operations -------------------------*/
188 rte_atomic128_cmp_exchange(rte_int128_t *dst,
190 const rte_int128_t *src,
196 RTE_SET_USED(success);
197 RTE_SET_USED(failure);
204 : [dst] "=m" (dst->val[0]),
218 #endif /* _RTE_ATOMIC_X86_64_H_ */