1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ASM_X86_ATOMIC64_32_H 3#define _ASM_X86_ATOMIC64_32_H 4 5#include <linux/compiler.h> 6#include <linux/types.h> 7//#include <asm/cmpxchg.h> 8 9/* An 64bit atomic type */ 10 11typedef struct { 12 s64 __aligned(8) counter; 13} atomic64_t; 14 15#define ATOMIC64_INIT(val) { (val) } 16 17#define __ATOMIC64_DECL(sym) void atomic64_##sym(atomic64_t *, ...) 18#ifndef ATOMIC64_EXPORT 19#define ATOMIC64_DECL_ONE __ATOMIC64_DECL 20#else 21#define ATOMIC64_DECL_ONE(sym) __ATOMIC64_DECL(sym); \ 22 ATOMIC64_EXPORT(atomic64_##sym) 23#endif 24 25#ifdef CONFIG_X86_CMPXCHG64 26#define __alternative_atomic64(f, g, out, in...) \ 27 asm volatile("call %P[func]" \ 28 : out : [func] "i" (atomic64_##g##_cx8), ## in) 29 30#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8) 31#else 32#define __alternative_atomic64(f, g, out, in...) \ 33 alternative_call(atomic64_##f##_386, atomic64_##g##_cx8, \ 34 X86_FEATURE_CX8, ASM_OUTPUT2(out), ## in) 35 36#define ATOMIC64_DECL(sym) ATOMIC64_DECL_ONE(sym##_cx8); \ 37 ATOMIC64_DECL_ONE(sym##_386) 38 39ATOMIC64_DECL_ONE(add_386); 40ATOMIC64_DECL_ONE(sub_386); 41ATOMIC64_DECL_ONE(inc_386); 42ATOMIC64_DECL_ONE(dec_386); 43#endif 44 45#define alternative_atomic64(f, out, in...) \ 46 __alternative_atomic64(f, f, ASM_OUTPUT2(out), ## in) 47 48ATOMIC64_DECL(read); 49ATOMIC64_DECL(set); 50ATOMIC64_DECL(xchg); 51ATOMIC64_DECL(add_return); 52ATOMIC64_DECL(sub_return); 53ATOMIC64_DECL(inc_return); 54ATOMIC64_DECL(dec_return); 55ATOMIC64_DECL(dec_if_positive); 56ATOMIC64_DECL(inc_not_zero); 57ATOMIC64_DECL(add_unless); 58 59#undef ATOMIC64_DECL 60#undef ATOMIC64_DECL_ONE 61#undef __ATOMIC64_DECL 62#undef ATOMIC64_EXPORT 63 64static __always_inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) 65{ 66 return arch_cmpxchg64(&v->counter, o, n); 67} 68#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 69 70static __always_inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) 71{ 72 s64 o; 73 unsigned high = (unsigned)(n >> 32); 74 unsigned low = (unsigned)n; 75 alternative_atomic64(xchg, "=&A" (o), 76 "S" (v), "b" (low), "c" (high) 77 : "memory"); 78 return o; 79} 80#define arch_atomic64_xchg arch_atomic64_xchg 81 82static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i) 83{ 84 unsigned high = (unsigned)(i >> 32); 85 unsigned low = (unsigned)i; 86 alternative_atomic64(set, /* no output */, 87 "S" (v), "b" (low), "c" (high) 88 : "eax", "edx", "memory"); 89} 90 91static __always_inline s64 arch_atomic64_read(const atomic64_t *v) 92{ 93 s64 r; 94 alternative_atomic64(read, "=&A" (r), "c" (v) : "memory"); 95 return r; 96} 97 98static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) 99{ 100 alternative_atomic64(add_return, 101 ASM_OUTPUT2("+A" (i), "+c" (v)), 102 ASM_NO_INPUT_CLOBBER("memory")); 103 return i; 104} 105#define arch_atomic64_add_return arch_atomic64_add_return 106 107static __always_inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) 108{ 109 alternative_atomic64(sub_return, 110 ASM_OUTPUT2("+A" (i), "+c" (v)), 111 ASM_NO_INPUT_CLOBBER("memory")); 112 return i; 113} 114#define arch_atomic64_sub_return arch_atomic64_sub_return 115 116static __always_inline s64 arch_atomic64_inc_return(atomic64_t *v) 117{ 118 s64 a; 119 alternative_atomic64(inc_return, "=&A" (a), 120 "S" (v) : "memory", "ecx"); 121 return a; 122} 123#define arch_atomic64_inc_return arch_atomic64_inc_return 124 125static __always_inline s64 arch_atomic64_dec_return(atomic64_t *v) 126{ 127 s64 a; 128 alternative_atomic64(dec_return, "=&A" (a), 129 "S" (v) : "memory", "ecx"); 130 return a; 131} 132#define arch_atomic64_dec_return arch_atomic64_dec_return 133 134static __always_inline s64 arch_atomic64_add(s64 i, atomic64_t *v) 135{ 136 __alternative_atomic64(add, add_return, 137 ASM_OUTPUT2("+A" (i), "+c" (v)), 138 ASM_NO_INPUT_CLOBBER("memory")); 139 return i; 140} 141 142static __always_inline s64 arch_atomic64_sub(s64 i, atomic64_t *v) 143{ 144 __alternative_atomic64(sub, sub_return, 145 ASM_OUTPUT2("+A" (i), "+c" (v)), 146 ASM_NO_INPUT_CLOBBER("memory")); 147 return i; 148} 149 150static __always_inline void arch_atomic64_inc(atomic64_t *v) 151{ 152 __alternative_atomic64(inc, inc_return, /* no output */, 153 "S" (v) : "memory", "eax", "ecx", "edx"); 154} 155#define arch_atomic64_inc arch_atomic64_inc 156 157static __always_inline void arch_atomic64_dec(atomic64_t *v) 158{ 159 __alternative_atomic64(dec, dec_return, /* no output */, 160 "S" (v) : "memory", "eax", "ecx", "edx"); 161} 162#define arch_atomic64_dec arch_atomic64_dec 163 164static __always_inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 165{ 166 unsigned low = (unsigned)u; 167 unsigned high = (unsigned)(u >> 32); 168 alternative_atomic64(add_unless, 169 ASM_OUTPUT2("+A" (a), "+c" (low), "+D" (high)), 170 "S" (v) : "memory"); 171 return (int)a; 172} 173#define arch_atomic64_add_unless arch_atomic64_add_unless 174 175static __always_inline int arch_atomic64_inc_not_zero(atomic64_t *v) 176{ 177 int r; 178 alternative_atomic64(inc_not_zero, "=&a" (r), 179 "S" (v) : "ecx", "edx", "memory"); 180 return r; 181} 182#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero 183 184static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 185{ 186 s64 r; 187 alternative_atomic64(dec_if_positive, "=&A" (r), 188 "S" (v) : "ecx", "memory"); 189 return r; 190} 191#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 192 193#undef alternative_atomic64 194#undef __alternative_atomic64 195 196static __always_inline void arch_atomic64_and(s64 i, atomic64_t *v) 197{ 198 s64 old, c = 0; 199 200 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c) 201 c = old; 202} 203 204static __always_inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) 205{ 206 s64 old, c = 0; 207 208 while ((old = arch_atomic64_cmpxchg(v, c, c & i)) != c) 209 c = old; 210 211 return old; 212} 213#define arch_atomic64_fetch_and arch_atomic64_fetch_and 214 215static __always_inline void arch_atomic64_or(s64 i, atomic64_t *v) 216{ 217 s64 old, c = 0; 218 219 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c) 220 c = old; 221} 222 223static __always_inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) 224{ 225 s64 old, c = 0; 226 227 while ((old = arch_atomic64_cmpxchg(v, c, c | i)) != c) 228 c = old; 229 230 return old; 231} 232#define arch_atomic64_fetch_or arch_atomic64_fetch_or 233 234static __always_inline void arch_atomic64_xor(s64 i, atomic64_t *v) 235{ 236 s64 old, c = 0; 237 238 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c) 239 c = old; 240} 241 242static __always_inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 243{ 244 s64 old, c = 0; 245 246 while ((old = arch_atomic64_cmpxchg(v, c, c ^ i)) != c) 247 c = old; 248 249 return old; 250} 251#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 252 253static __always_inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) 254{ 255 s64 old, c = 0; 256 257 while ((old = arch_atomic64_cmpxchg(v, c, c + i)) != c) 258 c = old; 259 260 return old; 261} 262#define arch_atomic64_fetch_add arch_atomic64_fetch_add 263 264#define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v)) 265 266#endif /* _ASM_X86_ATOMIC64_32_H */ 267