X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_eal%2Fcommon%2Finclude%2Fx86_64%2Farch%2Frte_atomic.h;h=8772f80404813a9c736387cb1de295547abdb1dc;hb=1c1d4d7a923d4804f1926fc5264f9ecdd8977b04;hp=dc15acd433f0df9ff571e57c88352b990b33dc96;hpb=b6df9fc8715f9a925136006b18fdd65f9c621757;p=dpdk.git diff --git a/lib/librte_eal/common/include/x86_64/arch/rte_atomic.h b/lib/librte_eal/common/include/x86_64/arch/rte_atomic.h index dc15acd433..8772f80404 100644 --- a/lib/librte_eal/common/include/x86_64/arch/rte_atomic.h +++ b/lib/librte_eal/common/include/x86_64/arch/rte_atomic.h @@ -4,32 +4,31 @@ * Copyright(c) 2010-2013 Intel Corporation. All rights reserved. * All rights reserved. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions * are met: * - * * Redistributions of source code must retain the above copyright + * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * */ /* @@ -45,612 +44,6 @@ #ifndef _RTE_X86_64_ATOMIC_H_ #define _RTE_X86_64_ATOMIC_H_ -/** - * @file - * Atomic Operations on x86_64 - */ - -#if RTE_MAX_LCORE == 1 -#define MPLOCKED /**< No need to insert MP lock prefix. */ -#else -#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */ -#endif - -/** - * General memory barrier. - * - * Guarantees that the LOAD and STORE operations generated before the - * barrier occur before the LOAD and STORE operations generated after. - */ -#define rte_mb() asm volatile("mfence;" : : : "memory") - -/** - * Write memory barrier. - * - * Guarantees that the STORE operations generated before the barrier - * occur before the STORE operations generated after. - */ -#define rte_wmb() asm volatile("sfence;" : : : "memory") - -/** - * Read memory barrier. - * - * Guarantees that the LOAD operations generated before the barrier - * occur before the LOAD operations generated after. - */ -#define rte_rmb() asm volatile("lfence;" : : : "memory") - -/*------------------------- 16 bit atomic operations -------------------------*/ - -/** - * Atomic compare and set. - * - * (atomic) equivalent to: - * if (*dst == exp) - * *dst = src (all 16-bit words) - * - * @param dst - * The destination location into which the value will be written. - * @param exp - * The expected value. - * @param src - * The new value. - * @return - * Non-zero on success; 0 on failure. - */ -static inline int -rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgw %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -/** - * The atomic counter structure. - */ -typedef struct { - volatile int16_t cnt; /**< An internal counter value. */ -} rte_atomic16_t; - -/** - * Static initializer for an atomic counter. - */ -#define RTE_ATOMIC16_INIT(val) { (val) } - -/** - * Initialize an atomic counter. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic16_init(rte_atomic16_t *v) -{ - v->cnt = 0; -} - -/** - * Atomically read a 16-bit value from a counter. - * - * @param v - * A pointer to the atomic counter. - * @return - * The value of the counter. - */ -static inline int16_t -rte_atomic16_read(const rte_atomic16_t *v) -{ - return v->cnt; -} - -/** - * Atomically set a counter to a 16-bit value. - * - * @param v - * A pointer to the atomic counter. - * @param new_value - * The new value for the counter. - */ -static inline void -rte_atomic16_set(rte_atomic16_t *v, int16_t new_value) -{ - v->cnt = new_value; -} - -/** - * Atomically add a 16-bit value to an atomic counter. - * - * @param v - * A pointer to the atomic counter. - * @param inc - * The value to be added to the counter. - */ -static inline void -rte_atomic16_add(rte_atomic16_t *v, int16_t inc) -{ - asm volatile( - MPLOCKED - "addw %[inc], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [inc] "ir" (inc), /* input */ - "m" (v->cnt) - ); -} - -/** - * Atomically subtract a 16-bit value from an atomic counter. - * - * @param v - * A pointer to the atomic counter. - * @param dec - * The value to be subtracted from the counter. - */ -static inline void -rte_atomic16_sub(rte_atomic16_t *v, int16_t dec) -{ - asm volatile( - MPLOCKED - "subw %[dec], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [dec] "ir" (dec), /* input */ - "m" (v->cnt) - ); -} - -/** - * Atomically increment a counter by one. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic16_inc(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "incw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -/** - * Atomically decrement a counter by one. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic16_dec(rte_atomic16_t *v) -{ - asm volatile( - MPLOCKED - "decw %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -/** - * Atomically add a 16-bit value to a counter and return the result. - * - * Atomically adds the 16-bits value (inc) to the atomic counter (v) and - * returns the value of v after addition. - * - * @param v - * A pointer to the atomic counter. - * @param inc - * The value to be added to the counter. - * @return - * The value of v after the addition. - */ -static inline int16_t -rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc) -{ - int16_t prev = inc; - - asm volatile( - MPLOCKED - "xaddw %[prev], %[cnt]" - : [prev] "+r" (prev), /* output */ - [cnt] "=m" (v->cnt) - : "m" (v->cnt) /* input */ - ); - return (int16_t)(prev + inc); -} - -/** - * Atomically subtract a 16-bit value from a counter and return - * the result. - * - * Atomically subtracts the 16-bit value (inc) from the atomic counter - * (v) and returns the value of v after the subtraction. - * - * @param v - * A pointer to the atomic counter. - * @param dec - * The value to be subtracted from the counter. - * @return - * The value of v after the subtraction. - */ -static inline int16_t -rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec) -{ - return rte_atomic16_add_return(v, (int16_t)-dec); -} - -/** - * Atomically increment a 16-bit counter by one and test. - * - * Atomically increments the atomic counter (v) by one and returns true if - * the result is 0, or false in all other cases. - * - * @param v - * A pointer to the atomic counter. - * @return - * True if the result after the increment operation is 0; false otherwise. - */ -static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/** - * Atomically decrement a 16-bit counter by one and test. - * - * Atomically decrements the atomic counter (v) by one and returns true if - * the result is 0, or false in all other cases. - * - * @param v - * A pointer to the atomic counter. - * @return - * True if the result after the decrement operation is 0; false otherwise. - */ -static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decw %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/** - * Atomically test and set a 16-bit atomic counter. - * - * If the counter value is already set, return 0 (failed). Otherwise, set - * the counter value to 1 and return 1 (success). - * - * @param v - * A pointer to the atomic counter. - * @return - * 0 if failed; else 1, success. - */ -static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) -{ - return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1); -} - -/** - * Atomically set a 16-bit counter to 0. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void rte_atomic16_clear(rte_atomic16_t *v) -{ - v->cnt = 0; -} - -/*------------------------- 32 bit atomic operations -------------------------*/ - -/** - * Atomic compare and set. - * - * (atomic) equivalent to: - * if (*dst == exp) - * *dst = src (all 32-bit words) - * - * @param dst - * The destination location into which the value will be written. - * @param exp - * The expected value. - * @param src - * The new value. - * @return - * Non-zero on success; 0 on failure. - */ -static inline int -rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) -{ - uint8_t res; - - asm volatile( - MPLOCKED - "cmpxchgl %[src], %[dst];" - "sete %[res];" - : [res] "=a" (res), /* output */ - [dst] "=m" (*dst) - : [src] "r" (src), /* input */ - "a" (exp), - "m" (*dst) - : "memory"); /* no-clobber list */ - return res; -} - -/** - * The atomic counter structure. - */ -typedef struct { - volatile int32_t cnt; /**< An internal counter value. */ -} rte_atomic32_t; - -/** - * Static initializer for an atomic counter. - */ -#define RTE_ATOMIC32_INIT(val) { (val) } - -/** - * Initialize an atomic counter. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic32_init(rte_atomic32_t *v) -{ - v->cnt = 0; -} - -/** - * Atomically read a 32-bit value from a counter. - * - * @param v - * A pointer to the atomic counter. - * @return - * The value of the counter. - */ -static inline int32_t -rte_atomic32_read(const rte_atomic32_t *v) -{ - return v->cnt; -} - -/** - * Atomically set a counter to a 32-bit value. - * - * @param v - * A pointer to the atomic counter. - * @param new_value - * The new value for the counter. - */ -static inline void -rte_atomic32_set(rte_atomic32_t *v, int32_t new_value) -{ - v->cnt = new_value; -} - -/** - * Atomically add a 32-bit value to an atomic counter. - * - * @param v - * A pointer to the atomic counter. - * @param inc - * The value to be added to the counter. - */ -static inline void -rte_atomic32_add(rte_atomic32_t *v, int32_t inc) -{ - asm volatile( - MPLOCKED - "addl %[inc], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [inc] "ir" (inc), /* input */ - "m" (v->cnt) - ); -} - -/** - * Atomically subtract a 32-bit value from an atomic counter. - * - * @param v - * A pointer to the atomic counter. - * @param dec - * The value to be subtracted from the counter. - */ -static inline void -rte_atomic32_sub(rte_atomic32_t *v, int32_t dec) -{ - asm volatile( - MPLOCKED - "subl %[dec], %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : [dec] "ir" (dec), /* input */ - "m" (v->cnt) - ); -} - -/** - * Atomically increment a counter by one. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic32_inc(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "incl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -/** - * Atomically decrement a counter by one. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void -rte_atomic32_dec(rte_atomic32_t *v) -{ - asm volatile( - MPLOCKED - "decl %[cnt]" - : [cnt] "=m" (v->cnt) /* output */ - : "m" (v->cnt) /* input */ - ); -} - -/** - * Atomically add a 32-bit value to a counter and return the result. - * - * Atomically adds the 32-bits value (inc) to the atomic counter (v) and - * returns the value of v after addition. - * - * @param v - * A pointer to the atomic counter. - * @param inc - * The value to be added to the counter. - * @return - * The value of v after the addition. - */ -static inline int32_t -rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc) -{ - int32_t prev = inc; - - asm volatile( - MPLOCKED - "xaddl %[prev], %[cnt]" - : [prev] "+r" (prev), /* output */ - [cnt] "=m" (v->cnt) - : "m" (v->cnt) /* input */ - ); - return (int32_t)(prev + inc); -} - -/** - * Atomically subtract a 32-bit value from a counter and return - * the result. - * - * Atomically subtracts the 32-bit value (inc) from the atomic counter - * (v) and returns the value of v after the subtraction. - * - * @param v - * A pointer to the atomic counter. - * @param dec - * The value to be subtracted from the counter. - * @return - * The value of v after the subtraction. - */ -static inline int32_t -rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec) -{ - return rte_atomic32_add_return(v, -dec); -} - -/** - * Atomically increment a 32-bit counter by one and test. - * - * Atomically increments the atomic counter (v) by one and returns true if - * the result is 0, or false in all other cases. - * - * @param v - * A pointer to the atomic counter. - * @return - * True if the result after the increment operation is 0; false otherwise. - */ -static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile( - MPLOCKED - "incl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/** - * Atomically decrement a 32-bit counter by one and test. - * - * Atomically decrements the atomic counter (v) by one and returns true if - * the result is 0, or false in all other cases. - * - * @param v - * A pointer to the atomic counter. - * @return - * True if the result after the decrement operation is 0; false otherwise. - */ -static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) -{ - uint8_t ret; - - asm volatile(MPLOCKED - "decl %[cnt] ; " - "sete %[ret]" - : [cnt] "+m" (v->cnt), /* output */ - [ret] "=qm" (ret) - ); - return (ret != 0); -} - -/** - * Atomically test and set a 32-bit atomic counter. - * - * If the counter value is already set, return 0 (failed). Otherwise, set - * the counter value to 1 and return 1 (success). - * - * @param v - * A pointer to the atomic counter. - * @return - * 0 if failed; else 1, success. - */ -static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) -{ - return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1); -} - -/** - * Atomically set a 32-bit counter to 0. - * - * @param v - * A pointer to the atomic counter. - */ -static inline void rte_atomic32_clear(rte_atomic32_t *v) -{ - v->cnt = 0; -} /*------------------------- 64 bit atomic operations -------------------------*/