lib/eal/x86/include/rte_atomic_32.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2014 Intel Corporation.
   3  */
   4
   5 /*
   6  * Inspired from FreeBSD src/sys/i386/include/atomic.h
   7  * Copyright (c) 1998 Doug Rabson
   8  * All rights reserved.
   9  */
  10
  11 #ifndef _RTE_ATOMIC_X86_H_
  12 #error do not include this file directly, use <rte_atomic.h> instead
  13 #endif
  14
  15 #ifndef _RTE_ATOMIC_I686_H_
  16 #define _RTE_ATOMIC_I686_H_
  17
  18 #include <stdint.h>
  19 #include <rte_common.h>
  20
  21 /*------------------------- 64 bit atomic operations -------------------------*/
  22
  23 #ifndef RTE_FORCE_INTRINSICS
  24 static inline int
  25 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
  26 {
  27         uint8_t res;
  28         RTE_STD_C11
  29         union {
  30                 struct {
  31                         uint32_t l32;
  32                         uint32_t h32;
  33                 };
  34                 uint64_t u64;
  35         } _exp, _src;
  36
  37         _exp.u64 = exp;
  38         _src.u64 = src;
  39
  40 #ifndef __PIC__
  41     asm volatile (
  42             MPLOCKED
  43             "cmpxchg8b (%[dst]);"
  44             "setz %[res];"
  45             : [res] "=a" (res)      /* result in eax */
  46             : [dst] "S" (dst),      /* esi */
  47              "b" (_src.l32),       /* ebx */
  48              "c" (_src.h32),       /* ecx */
  49              "a" (_exp.l32),       /* eax */
  50              "d" (_exp.h32)        /* edx */
  51                         : "memory" );           /* no-clobber list */
  52 #else
  53         asm volatile (
  54             "xchgl %%ebx, %%edi;\n"
  55                         MPLOCKED
  56                         "cmpxchg8b (%[dst]);"
  57                         "setz %[res];"
  58             "xchgl %%ebx, %%edi;\n"
  59                         : [res] "=a" (res)      /* result in eax */
  60                         : [dst] "S" (dst),      /* esi */
  61                           "D" (_src.l32),       /* ebx */
  62                           "c" (_src.h32),       /* ecx */
  63                           "a" (_exp.l32),       /* eax */
  64                           "d" (_exp.h32)        /* edx */
  65                         : "memory" );           /* no-clobber list */
  66 #endif
  67
  68         return res;
  69 }
  70
  71 static inline uint64_t
  72 rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
  73 {
  74         uint64_t old;
  75
  76         do {
  77                 old = *dest;
  78         } while (rte_atomic64_cmpset(dest, old, val) == 0);
  79
  80         return old;
  81 }
  82
  83 static inline void
  84 rte_atomic64_init(rte_atomic64_t *v)
  85 {
  86         int success = 0;
  87         uint64_t tmp;
  88
  89         while (success == 0) {
  90                 tmp = v->cnt;
  91                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
  92                                               tmp, 0);
  93         }
  94 }
  95
  96 static inline int64_t
  97 rte_atomic64_read(rte_atomic64_t *v)
  98 {
  99         int success = 0;
 100         uint64_t tmp;
 101
 102         while (success == 0) {
 103                 tmp = v->cnt;
 104                 /* replace the value by itself */
 105                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 106                                               tmp, tmp);
 107         }
 108         return tmp;
 109 }
 110
 111 static inline void
 112 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
 113 {
 114         int success = 0;
 115         uint64_t tmp;
 116
 117         while (success == 0) {
 118                 tmp = v->cnt;
 119                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 120                                               tmp, new_value);
 121         }
 122 }
 123
 124 static inline void
 125 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
 126 {
 127         int success = 0;
 128         uint64_t tmp;
 129
 130         while (success == 0) {
 131                 tmp = v->cnt;
 132                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 133                                               tmp, tmp + inc);
 134         }
 135 }
 136
 137 static inline void
 138 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
 139 {
 140         int success = 0;
 141         uint64_t tmp;
 142
 143         while (success == 0) {
 144                 tmp = v->cnt;
 145                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 146                                               tmp, tmp - dec);
 147         }
 148 }
 149
 150 static inline void
 151 rte_atomic64_inc(rte_atomic64_t *v)
 152 {
 153         rte_atomic64_add(v, 1);
 154 }
 155
 156 static inline void
 157 rte_atomic64_dec(rte_atomic64_t *v)
 158 {
 159         rte_atomic64_sub(v, 1);
 160 }
 161
 162 static inline int64_t
 163 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
 164 {
 165         int success = 0;
 166         uint64_t tmp;
 167
 168         while (success == 0) {
 169                 tmp = v->cnt;
 170                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 171                                               tmp, tmp + inc);
 172         }
 173
 174         return tmp + inc;
 175 }
 176
 177 static inline int64_t
 178 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
 179 {
 180         int success = 0;
 181         uint64_t tmp;
 182
 183         while (success == 0) {
 184                 tmp = v->cnt;
 185                 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
 186                                               tmp, tmp - dec);
 187         }
 188
 189         return tmp - dec;
 190 }
 191
 192 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
 193 {
 194         return rte_atomic64_add_return(v, 1) == 0;
 195 }
 196
 197 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
 198 {
 199         return rte_atomic64_sub_return(v, 1) == 0;
 200 }
 201
 202 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
 203 {
 204         return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
 205 }
 206
 207 static inline void rte_atomic64_clear(rte_atomic64_t *v)
 208 {
 209         rte_atomic64_set(v, 0);
 210 }
 211 #endif
 212
 213 #endif /* _RTE_ATOMIC_I686_H_ */