4 * Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #ifndef _RTE_ATOMIC_H_
36 #define _RTE_ATOMIC_H_
42 * This file defines a generic API for atomic
43 * operations. The implementation is architecture-specific.
45 * See lib/librte_eal/common/include/i686/arch/rte_atomic.h
46 * See lib/librte_eal/common/include/x86_64/arch/rte_atomic.h
54 #include "arch/rte_atomic.h"
60 * General memory barrier.
62 * Guarantees that the LOAD and STORE operations generated before the
63 * barrier occur before the LOAD and STORE operations generated after.
65 #define rte_mb() asm volatile("mfence;" : : : "memory")
68 * Write memory barrier.
70 * Guarantees that the STORE operations generated before the barrier
71 * occur before the STORE operations generated after.
73 #define rte_wmb() asm volatile("sfence;" : : : "memory")
76 * Read memory barrier.
78 * Guarantees that the LOAD operations generated before the barrier
79 * occur before the LOAD operations generated after.
81 #define rte_rmb() asm volatile("lfence;" : : : "memory")
83 /*------------------------- 16 bit atomic operations -------------------------*/
86 * Atomic compare and set.
88 * (atomic) equivalent to:
90 * *dst = src (all 16-bit words)
93 * The destination location into which the value will be written.
99 * Non-zero on success; 0 on failure.
102 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src);
105 * The atomic counter structure.
108 volatile int16_t cnt; /**< An internal counter value. */
112 * Static initializer for an atomic counter.
114 #define RTE_ATOMIC16_INIT(val) { (val) }
117 * Initialize an atomic counter.
120 * A pointer to the atomic counter.
123 rte_atomic16_init(rte_atomic16_t *v)
129 * Atomically read a 16-bit value from a counter.
132 * A pointer to the atomic counter.
134 * The value of the counter.
136 static inline int16_t
137 rte_atomic16_read(const rte_atomic16_t *v);
140 * Atomically set a counter to a 16-bit value.
143 * A pointer to the atomic counter.
145 * The new value for the counter.
148 rte_atomic16_set(rte_atomic16_t *v, int16_t new_value);
151 * Atomically add a 16-bit value to an atomic counter.
154 * A pointer to the atomic counter.
156 * The value to be added to the counter.
159 rte_atomic16_add(rte_atomic16_t *v, int16_t inc);
162 * Atomically subtract a 16-bit value from an atomic counter.
165 * A pointer to the atomic counter.
167 * The value to be subtracted from the counter.
170 rte_atomic16_sub(rte_atomic16_t *v, int16_t dec);
173 * Atomically increment a counter by one.
176 * A pointer to the atomic counter.
179 rte_atomic16_inc(rte_atomic16_t *v);
182 * Atomically decrement a counter by one.
185 * A pointer to the atomic counter.
188 rte_atomic16_dec(rte_atomic16_t *v);
191 * Atomically add a 16-bit value to a counter and return the result.
193 * Atomically adds the 16-bits value (inc) to the atomic counter (v) and
194 * returns the value of v after addition.
197 * A pointer to the atomic counter.
199 * The value to be added to the counter.
201 * The value of v after the addition.
203 static inline int16_t
204 rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc);
207 * Atomically subtract a 16-bit value from a counter and return
210 * Atomically subtracts the 16-bit value (inc) from the atomic counter
211 * (v) and returns the value of v after the subtraction.
214 * A pointer to the atomic counter.
216 * The value to be subtracted from the counter.
218 * The value of v after the subtraction.
220 static inline int16_t
221 rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec);
224 * Atomically increment a 16-bit counter by one and test.
226 * Atomically increments the atomic counter (v) by one and returns true if
227 * the result is 0, or false in all other cases.
230 * A pointer to the atomic counter.
232 * True if the result after the increment operation is 0; false otherwise.
235 rte_atomic16_inc_and_test(rte_atomic16_t *v);
238 * Atomically decrement a 16-bit counter by one and test.
240 * Atomically decrements the atomic counter (v) by one and returns true if
241 * the result is 0, or false in all other cases.
244 * A pointer to the atomic counter.
246 * True if the result after the decrement operation is 0; false otherwise.
249 rte_atomic16_dec_and_test(rte_atomic16_t *v);
252 * Atomically test and set a 16-bit atomic counter.
254 * If the counter value is already set, return 0 (failed). Otherwise, set
255 * the counter value to 1 and return 1 (success).
258 * A pointer to the atomic counter.
260 * 0 if failed; else 1, success.
263 rte_atomic16_test_and_set(rte_atomic16_t *v);
266 * Atomically set a 16-bit counter to 0.
269 * A pointer to the atomic counter.
272 rte_atomic16_clear(rte_atomic16_t *v);
274 /*------------------------- 32 bit atomic operations -------------------------*/
277 * Atomic compare and set.
279 * (atomic) equivalent to:
281 * *dst = src (all 32-bit words)
284 * The destination location into which the value will be written.
286 * The expected value.
290 * Non-zero on success; 0 on failure.
293 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src);
296 * The atomic counter structure.
299 volatile int32_t cnt; /**< An internal counter value. */
303 * Static initializer for an atomic counter.
305 #define RTE_ATOMIC32_INIT(val) { (val) }
308 * Initialize an atomic counter.
311 * A pointer to the atomic counter.
314 rte_atomic32_init(rte_atomic32_t *v);
317 * Atomically read a 32-bit value from a counter.
320 * A pointer to the atomic counter.
322 * The value of the counter.
324 static inline int32_t
325 rte_atomic32_read(const rte_atomic32_t *v);
328 * Atomically set a counter to a 32-bit value.
331 * A pointer to the atomic counter.
333 * The new value for the counter.
336 rte_atomic32_set(rte_atomic32_t *v, int32_t new_value);
339 * Atomically add a 32-bit value to an atomic counter.
342 * A pointer to the atomic counter.
344 * The value to be added to the counter.
347 rte_atomic32_add(rte_atomic32_t *v, int32_t inc);
350 * Atomically subtract a 32-bit value from an atomic counter.
353 * A pointer to the atomic counter.
355 * The value to be subtracted from the counter.
358 rte_atomic32_sub(rte_atomic32_t *v, int32_t dec);
361 * Atomically increment a counter by one.
364 * A pointer to the atomic counter.
367 rte_atomic32_inc(rte_atomic32_t *v);
370 * Atomically decrement a counter by one.
373 * A pointer to the atomic counter.
376 rte_atomic32_dec(rte_atomic32_t *v);
379 * Atomically add a 32-bit value to a counter and return the result.
381 * Atomically adds the 32-bits value (inc) to the atomic counter (v) and
382 * returns the value of v after addition.
385 * A pointer to the atomic counter.
387 * The value to be added to the counter.
389 * The value of v after the addition.
391 static inline int32_t
392 rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc);
395 * Atomically subtract a 32-bit value from a counter and return
398 * Atomically subtracts the 32-bit value (inc) from the atomic counter
399 * (v) and returns the value of v after the subtraction.
402 * A pointer to the atomic counter.
404 * The value to be subtracted from the counter.
406 * The value of v after the subtraction.
408 static inline int32_t
409 rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec);
412 * Atomically increment a 32-bit counter by one and test.
414 * Atomically increments the atomic counter (v) by one and returns true if
415 * the result is 0, or false in all other cases.
418 * A pointer to the atomic counter.
420 * True if the result after the increment operation is 0; false otherwise.
423 rte_atomic32_inc_and_test(rte_atomic32_t *v);
426 * Atomically decrement a 32-bit counter by one and test.
428 * Atomically decrements the atomic counter (v) by one and returns true if
429 * the result is 0, or false in all other cases.
432 * A pointer to the atomic counter.
434 * True if the result after the decrement operation is 0; false otherwise.
437 rte_atomic32_dec_and_test(rte_atomic32_t *v);
440 * Atomically test and set a 32-bit atomic counter.
442 * If the counter value is already set, return 0 (failed). Otherwise, set
443 * the counter value to 1 and return 1 (success).
446 * A pointer to the atomic counter.
448 * 0 if failed; else 1, success.
451 rte_atomic32_test_and_set(rte_atomic32_t *v);
454 * Atomically set a 32-bit counter to 0.
457 * A pointer to the atomic counter.
460 rte_atomic32_clear(rte_atomic32_t *v);
462 /*------------------------- 64 bit atomic operations -------------------------*/
465 * An atomic compare and set function used by the mutex functions.
466 * (atomic) equivalent to:
468 * *dst = src (all 64-bit words)
471 * The destination into which the value will be written.
473 * The expected value.
477 * Non-zero on success; 0 on failure.
480 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src);
483 * The atomic counter structure.
486 volatile int64_t cnt; /**< Internal counter value. */
490 * Static initializer for an atomic counter.
492 #define RTE_ATOMIC64_INIT(val) { (val) }
495 * Initialize the atomic counter.
498 * A pointer to the atomic counter.
501 rte_atomic64_init(rte_atomic64_t *v);
504 * Atomically read a 64-bit counter.
507 * A pointer to the atomic counter.
509 * The value of the counter.
511 static inline int64_t
512 rte_atomic64_read(rte_atomic64_t *v);
515 * Atomically set a 64-bit counter.
518 * A pointer to the atomic counter.
520 * The new value of the counter.
523 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value);
526 * Atomically add a 64-bit value to a counter.
529 * A pointer to the atomic counter.
531 * The value to be added to the counter.
534 rte_atomic64_add(rte_atomic64_t *v, int64_t inc);
537 * Atomically subtract a 64-bit value from a counter.
540 * A pointer to the atomic counter.
542 * The value to be subtracted from the counter.
545 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec);
548 * Atomically increment a 64-bit counter by one and test.
551 * A pointer to the atomic counter.
554 rte_atomic64_inc(rte_atomic64_t *v);
557 * Atomically decrement a 64-bit counter by one and test.
560 * A pointer to the atomic counter.
563 rte_atomic64_dec(rte_atomic64_t *v);
566 * Add a 64-bit value to an atomic counter and return the result.
568 * Atomically adds the 64-bit value (inc) to the atomic counter (v) and
569 * returns the value of v after the addition.
572 * A pointer to the atomic counter.
574 * The value to be added to the counter.
576 * The value of v after the addition.
578 static inline int64_t
579 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc);
582 * Subtract a 64-bit value from an atomic counter and return the result.
584 * Atomically subtracts the 64-bit value (dec) from the atomic counter (v)
585 * and returns the value of v after the subtraction.
588 * A pointer to the atomic counter.
590 * The value to be subtracted from the counter.
592 * The value of v after the subtraction.
594 static inline int64_t
595 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec);
598 * Atomically increment a 64-bit counter by one and test.
600 * Atomically increments the atomic counter (v) by one and returns
601 * true if the result is 0, or false in all other cases.
604 * A pointer to the atomic counter.
606 * True if the result after the addition is 0; false otherwise.
609 rte_atomic64_inc_and_test(rte_atomic64_t *v);
612 * Atomically decrement a 64-bit counter by one and test.
614 * Atomically decrements the atomic counter (v) by one and returns true if
615 * the result is 0, or false in all other cases.
618 * A pointer to the atomic counter.
620 * True if the result after subtraction is 0; false otherwise.
623 rte_atomic64_dec_and_test(rte_atomic64_t *v);
626 * Atomically test and set a 64-bit atomic counter.
628 * If the counter value is already set, return 0 (failed). Otherwise, set
629 * the counter value to 1 and return 1 (success).
632 * A pointer to the atomic counter.
634 * 0 if failed; else 1, success.
637 rte_atomic64_test_and_set(rte_atomic64_t *v);
640 * Atomically set a 64-bit counter to 0.
643 * A pointer to the atomic counter.
646 rte_atomic64_clear(rte_atomic64_t *v);
648 #endif /* __DOXYGEN__ */
655 #endif /* _RTE_ATOMIC_H_ */