1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Based on arch/arm/include/asm/atomic.h 4 * 5 * Copyright (C) 1996 Russell King. 6 * Copyright (C) 2002 Deep Blue Solutions Ltd. 7 * Copyright (C) 2012 ARM Ltd. 8 */ 9 10#ifndef __ASM_ATOMIC_LL_SC_H 11#define __ASM_ATOMIC_LL_SC_H 12 13#include <linux/stringify.h> 14 15#ifndef CONFIG_CC_HAS_K_CONSTRAINT 16#define K 17#endif 18 19/* 20 * AArch64 UP and SMP safe atomic ops. We use load exclusive and 21 * store exclusive to ensure that these are atomic. We may loop 22 * to ensure that the update happens. 23 */ 24 25#define ATOMIC_OP(op, asm_op, constraint) \ 26static __always_inline void \ 27__ll_sc_atomic_##op(int i, atomic_t *v) \ 28{ \ 29 unsigned long tmp; \ 30 int result; \ 31 \ 32 asm volatile("// atomic_" #op "\n" \ 33 " prfm pstl1strm, %2\n" \ 34 "1: ldxr %w0, %2\n" \ 35 " " #asm_op " %w0, %w0, %w3\n" \ 36 " stxr %w1, %w0, %2\n" \ 37 " cbnz %w1, 1b\n" \ 38 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 39 : __stringify(constraint) "r" (i)); \ 40} 41 42#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 43static __always_inline int \ 44__ll_sc_atomic_##op##_return##name(int i, atomic_t *v) \ 45{ \ 46 unsigned long tmp; \ 47 int result; \ 48 \ 49 asm volatile("// atomic_" #op "_return" #name "\n" \ 50 " prfm pstl1strm, %2\n" \ 51 "1: ld" #acq "xr %w0, %2\n" \ 52 " " #asm_op " %w0, %w0, %w3\n" \ 53 " st" #rel "xr %w1, %w0, %2\n" \ 54 " cbnz %w1, 1b\n" \ 55 " " #mb \ 56 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 57 : __stringify(constraint) "r" (i) \ 58 : cl); \ 59 \ 60 return result; \ 61} 62 63#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \ 64static __always_inline int \ 65__ll_sc_atomic_fetch_##op##name(int i, atomic_t *v) \ 66{ \ 67 unsigned long tmp; \ 68 int val, result; \ 69 \ 70 asm volatile("// atomic_fetch_" #op #name "\n" \ 71 " prfm pstl1strm, %3\n" \ 72 "1: ld" #acq "xr %w0, %3\n" \ 73 " " #asm_op " %w1, %w0, %w4\n" \ 74 " st" #rel "xr %w2, %w1, %3\n" \ 75 " cbnz %w2, 1b\n" \ 76 " " #mb \ 77 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 78 : __stringify(constraint) "r" (i) \ 79 : cl); \ 80 \ 81 return result; \ 82} 83 84#define ATOMIC_OPS(...) \ 85 ATOMIC_OP(__VA_ARGS__) \ 86 ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)\ 87 ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\ 88 ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\ 89 ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)\ 90 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 91 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 92 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 93 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 94 95ATOMIC_OPS(add, add, I) 96ATOMIC_OPS(sub, sub, J) 97 98#undef ATOMIC_OPS 99#define ATOMIC_OPS(...) \ 100 ATOMIC_OP(__VA_ARGS__) \ 101 ATOMIC_FETCH_OP ( , dmb ish, , l, "memory", __VA_ARGS__)\ 102 ATOMIC_FETCH_OP (_relaxed, , , , , __VA_ARGS__)\ 103 ATOMIC_FETCH_OP (_acquire, , a, , "memory", __VA_ARGS__)\ 104 ATOMIC_FETCH_OP (_release, , , l, "memory", __VA_ARGS__) 105 106ATOMIC_OPS(and, and, K) 107ATOMIC_OPS(or, orr, K) 108ATOMIC_OPS(xor, eor, K) 109/* 110 * GAS converts the mysterious and undocumented BIC (immediate) alias to 111 * an AND (immediate) instruction with the immediate inverted. We don't 112 * have a constraint for this, so fall back to register. 113 */ 114ATOMIC_OPS(andnot, bic, ) 115 116#undef ATOMIC_OPS 117#undef ATOMIC_FETCH_OP 118#undef ATOMIC_OP_RETURN 119#undef ATOMIC_OP 120 121#define ATOMIC64_OP(op, asm_op, constraint) \ 122static __always_inline void \ 123__ll_sc_atomic64_##op(s64 i, atomic64_t *v) \ 124{ \ 125 s64 result; \ 126 unsigned long tmp; \ 127 \ 128 asm volatile("// atomic64_" #op "\n" \ 129 " prfm pstl1strm, %2\n" \ 130 "1: ldxr %0, %2\n" \ 131 " " #asm_op " %0, %0, %3\n" \ 132 " stxr %w1, %0, %2\n" \ 133 " cbnz %w1, 1b" \ 134 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 135 : __stringify(constraint) "r" (i)); \ 136} 137 138#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\ 139static __always_inline long \ 140__ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v) \ 141{ \ 142 s64 result; \ 143 unsigned long tmp; \ 144 \ 145 asm volatile("// atomic64_" #op "_return" #name "\n" \ 146 " prfm pstl1strm, %2\n" \ 147 "1: ld" #acq "xr %0, %2\n" \ 148 " " #asm_op " %0, %0, %3\n" \ 149 " st" #rel "xr %w1, %0, %2\n" \ 150 " cbnz %w1, 1b\n" \ 151 " " #mb \ 152 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ 153 : __stringify(constraint) "r" (i) \ 154 : cl); \ 155 \ 156 return result; \ 157} 158 159#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\ 160static __always_inline long \ 161__ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v) \ 162{ \ 163 s64 result, val; \ 164 unsigned long tmp; \ 165 \ 166 asm volatile("// atomic64_fetch_" #op #name "\n" \ 167 " prfm pstl1strm, %3\n" \ 168 "1: ld" #acq "xr %0, %3\n" \ 169 " " #asm_op " %1, %0, %4\n" \ 170 " st" #rel "xr %w2, %1, %3\n" \ 171 " cbnz %w2, 1b\n" \ 172 " " #mb \ 173 : "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter) \ 174 : __stringify(constraint) "r" (i) \ 175 : cl); \ 176 \ 177 return result; \ 178} 179 180#define ATOMIC64_OPS(...) \ 181 ATOMIC64_OP(__VA_ARGS__) \ 182 ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__) \ 183 ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \ 184 ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \ 185 ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__) \ 186 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 187 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 188 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 189 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 190 191ATOMIC64_OPS(add, add, I) 192ATOMIC64_OPS(sub, sub, J) 193 194#undef ATOMIC64_OPS 195#define ATOMIC64_OPS(...) \ 196 ATOMIC64_OP(__VA_ARGS__) \ 197 ATOMIC64_FETCH_OP (, dmb ish, , l, "memory", __VA_ARGS__) \ 198 ATOMIC64_FETCH_OP (_relaxed,, , , , __VA_ARGS__) \ 199 ATOMIC64_FETCH_OP (_acquire,, a, , "memory", __VA_ARGS__) \ 200 ATOMIC64_FETCH_OP (_release,, , l, "memory", __VA_ARGS__) 201 202ATOMIC64_OPS(and, and, L) 203ATOMIC64_OPS(or, orr, L) 204ATOMIC64_OPS(xor, eor, L) 205/* 206 * GAS converts the mysterious and undocumented BIC (immediate) alias to 207 * an AND (immediate) instruction with the immediate inverted. We don't 208 * have a constraint for this, so fall back to register. 209 */ 210ATOMIC64_OPS(andnot, bic, ) 211 212#undef ATOMIC64_OPS 213#undef ATOMIC64_FETCH_OP 214#undef ATOMIC64_OP_RETURN 215#undef ATOMIC64_OP 216 217static __always_inline s64 218__ll_sc_atomic64_dec_if_positive(atomic64_t *v) 219{ 220 s64 result; 221 unsigned long tmp; 222 223 asm volatile("// atomic64_dec_if_positive\n" 224 " prfm pstl1strm, %2\n" 225 "1: ldxr %0, %2\n" 226 " subs %0, %0, #1\n" 227 " b.lt 2f\n" 228 " stlxr %w1, %0, %2\n" 229 " cbnz %w1, 1b\n" 230 " dmb ish\n" 231 "2:" 232 : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) 233 : 234 : "cc", "memory"); 235 236 return result; 237} 238 239#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint) \ 240static __always_inline u##sz \ 241__ll_sc__cmpxchg_case_##name##sz(volatile void *ptr, \ 242 unsigned long old, \ 243 u##sz new) \ 244{ \ 245 unsigned long tmp; \ 246 u##sz oldval; \ 247 \ 248 /* \ 249 * Sub-word sizes require explicit casting so that the compare \ 250 * part of the cmpxchg doesn't end up interpreting non-zero \ 251 * upper bits of the register containing "old". \ 252 */ \ 253 if (sz < 32) \ 254 old = (u##sz)old; \ 255 \ 256 asm volatile( \ 257 " prfm pstl1strm, %[v]\n" \ 258 "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \ 259 " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ 260 " cbnz %" #w "[tmp], 2f\n" \ 261 " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \ 262 " cbnz %w[tmp], 1b\n" \ 263 " " #mb "\n" \ 264 "2:" \ 265 : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ 266 [v] "+Q" (*(u##sz *)ptr) \ 267 : [old] __stringify(constraint) "r" (old), [new] "r" (new) \ 268 : cl); \ 269 \ 270 return oldval; \ 271} 272 273/* 274 * Earlier versions of GCC (no later than 8.1.0) appear to incorrectly 275 * handle the 'K' constraint for the value 4294967295 - thus we use no 276 * constraint for 32 bit operations. 277 */ 278__CMPXCHG_CASE(w, b, , 8, , , , , K) 279__CMPXCHG_CASE(w, h, , 16, , , , , K) 280__CMPXCHG_CASE(w, , , 32, , , , , K) 281__CMPXCHG_CASE( , , , 64, , , , , L) 282__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory", K) 283__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory", K) 284__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory", K) 285__CMPXCHG_CASE( , , acq_, 64, , a, , "memory", L) 286__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory", K) 287__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory", K) 288__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory", K) 289__CMPXCHG_CASE( , , rel_, 64, , , l, "memory", L) 290__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory", K) 291__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory", K) 292__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory", K) 293__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory", L) 294 295#undef __CMPXCHG_CASE 296 297union __u128_halves { 298 u128 full; 299 struct { 300 u64 low, high; 301 }; 302}; 303 304#define __CMPXCHG128(name, mb, rel, cl...) \ 305static __always_inline u128 \ 306__ll_sc__cmpxchg128##name(volatile u128 *ptr, u128 old, u128 new) \ 307{ \ 308 union __u128_halves r, o = { .full = (old) }, \ 309 n = { .full = (new) }; \ 310 unsigned int tmp; \ 311 \ 312 asm volatile("// __cmpxchg128" #name "\n" \ 313 " prfm pstl1strm, %[v]\n" \ 314 "1: ldxp %[rl], %[rh], %[v]\n" \ 315 " cmp %[rl], %[ol]\n" \ 316 " ccmp %[rh], %[oh], 0, eq\n" \ 317 " b.ne 2f\n" \ 318 " st" #rel "xp %w[tmp], %[nl], %[nh], %[v]\n" \ 319 " cbnz %w[tmp], 1b\n" \ 320 " " #mb "\n" \ 321 "2:" \ 322 : [v] "+Q" (*(u128 *)ptr), \ 323 [rl] "=&r" (r.low), [rh] "=&r" (r.high), \ 324 [tmp] "=&r" (tmp) \ 325 : [ol] "r" (o.low), [oh] "r" (o.high), \ 326 [nl] "r" (n.low), [nh] "r" (n.high) \ 327 : "cc", ##cl); \ 328 \ 329 return r.full; \ 330} 331 332__CMPXCHG128( , , ) 333__CMPXCHG128(_mb, dmb ish, l, "memory") 334 335#undef __CMPXCHG128 336 337#undef K 338 339#endif /* __ASM_ATOMIC_LL_SC_H */ 340