1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ALPHA_ATOMIC_H 3#define _ALPHA_ATOMIC_H 4 5#include <linux/types.h> 6#include <asm/barrier.h> 7#include <asm/cmpxchg.h> 8 9/* 10 * Atomic operations that C can't guarantee us. Useful for 11 * resource counting etc... 12 * 13 * But use these as seldom as possible since they are much slower 14 * than regular operations. 15 */ 16 17/* 18 * To ensure dependency ordering is preserved for the _relaxed and 19 * _release atomics, an smp_mb() is unconditionally inserted into the 20 * _relaxed variants, which are used to build the barriered versions. 21 * Avoid redundant back-to-back fences in the _acquire and _fence 22 * versions. 23 */ 24#define __atomic_acquire_fence() 25#define __atomic_post_full_fence() 26 27#define ATOMIC64_INIT(i) { (i) } 28 29#define arch_atomic_read(v) READ_ONCE((v)->counter) 30#define arch_atomic64_read(v) READ_ONCE((v)->counter) 31 32#define arch_atomic_set(v,i) WRITE_ONCE((v)->counter, (i)) 33#define arch_atomic64_set(v,i) WRITE_ONCE((v)->counter, (i)) 34 35/* 36 * To get proper branch prediction for the main line, we must branch 37 * forward to code at the end of this object's .text section, then 38 * branch back to restart the operation. 39 */ 40 41#define ATOMIC_OP(op, asm_op) \ 42static __inline__ void arch_atomic_##op(int i, atomic_t * v) \ 43{ \ 44 unsigned long temp; \ 45 __asm__ __volatile__( \ 46 "1: ldl_l %0,%1\n" \ 47 " " #asm_op " %0,%2,%0\n" \ 48 " stl_c %0,%1\n" \ 49 " beq %0,2f\n" \ 50 ".subsection 2\n" \ 51 "2: br 1b\n" \ 52 ".previous" \ 53 :"=&r" (temp), "=m" (v->counter) \ 54 :"Ir" (i), "m" (v->counter)); \ 55} \ 56 57#define ATOMIC_OP_RETURN(op, asm_op) \ 58static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \ 59{ \ 60 long temp, result; \ 61 __asm__ __volatile__( \ 62 "1: ldl_l %0,%1\n" \ 63 " " #asm_op " %0,%3,%2\n" \ 64 " " #asm_op " %0,%3,%0\n" \ 65 " stl_c %0,%1\n" \ 66 " beq %0,2f\n" \ 67 ".subsection 2\n" \ 68 "2: br 1b\n" \ 69 ".previous" \ 70 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 71 :"Ir" (i), "m" (v->counter) : "memory"); \ 72 smp_mb(); \ 73 return result; \ 74} 75 76#define ATOMIC_FETCH_OP(op, asm_op) \ 77static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \ 78{ \ 79 long temp, result; \ 80 __asm__ __volatile__( \ 81 "1: ldl_l %2,%1\n" \ 82 " " #asm_op " %2,%3,%0\n" \ 83 " stl_c %0,%1\n" \ 84 " beq %0,2f\n" \ 85 ".subsection 2\n" \ 86 "2: br 1b\n" \ 87 ".previous" \ 88 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 89 :"Ir" (i), "m" (v->counter) : "memory"); \ 90 smp_mb(); \ 91 return result; \ 92} 93 94#define ATOMIC64_OP(op, asm_op) \ 95static __inline__ void arch_atomic64_##op(s64 i, atomic64_t * v) \ 96{ \ 97 s64 temp; \ 98 __asm__ __volatile__( \ 99 "1: ldq_l %0,%1\n" \ 100 " " #asm_op " %0,%2,%0\n" \ 101 " stq_c %0,%1\n" \ 102 " beq %0,2f\n" \ 103 ".subsection 2\n" \ 104 "2: br 1b\n" \ 105 ".previous" \ 106 :"=&r" (temp), "=m" (v->counter) \ 107 :"Ir" (i), "m" (v->counter)); \ 108} \ 109 110#define ATOMIC64_OP_RETURN(op, asm_op) \ 111static __inline__ s64 \ 112arch_atomic64_##op##_return_relaxed(s64 i, atomic64_t * v) \ 113{ \ 114 s64 temp, result; \ 115 __asm__ __volatile__( \ 116 "1: ldq_l %0,%1\n" \ 117 " " #asm_op " %0,%3,%2\n" \ 118 " " #asm_op " %0,%3,%0\n" \ 119 " stq_c %0,%1\n" \ 120 " beq %0,2f\n" \ 121 ".subsection 2\n" \ 122 "2: br 1b\n" \ 123 ".previous" \ 124 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 125 :"Ir" (i), "m" (v->counter) : "memory"); \ 126 smp_mb(); \ 127 return result; \ 128} 129 130#define ATOMIC64_FETCH_OP(op, asm_op) \ 131static __inline__ s64 \ 132arch_atomic64_fetch_##op##_relaxed(s64 i, atomic64_t * v) \ 133{ \ 134 s64 temp, result; \ 135 __asm__ __volatile__( \ 136 "1: ldq_l %2,%1\n" \ 137 " " #asm_op " %2,%3,%0\n" \ 138 " stq_c %0,%1\n" \ 139 " beq %0,2f\n" \ 140 ".subsection 2\n" \ 141 "2: br 1b\n" \ 142 ".previous" \ 143 :"=&r" (temp), "=m" (v->counter), "=&r" (result) \ 144 :"Ir" (i), "m" (v->counter) : "memory"); \ 145 smp_mb(); \ 146 return result; \ 147} 148 149#define ATOMIC_OPS(op) \ 150 ATOMIC_OP(op, op##l) \ 151 ATOMIC_OP_RETURN(op, op##l) \ 152 ATOMIC_FETCH_OP(op, op##l) \ 153 ATOMIC64_OP(op, op##q) \ 154 ATOMIC64_OP_RETURN(op, op##q) \ 155 ATOMIC64_FETCH_OP(op, op##q) 156 157ATOMIC_OPS(add) 158ATOMIC_OPS(sub) 159 160#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed 161#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed 162#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed 163#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed 164 165#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 166#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 167#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 168#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 169 170#define arch_atomic_andnot arch_atomic_andnot 171#define arch_atomic64_andnot arch_atomic64_andnot 172 173#undef ATOMIC_OPS 174#define ATOMIC_OPS(op, asm) \ 175 ATOMIC_OP(op, asm) \ 176 ATOMIC_FETCH_OP(op, asm) \ 177 ATOMIC64_OP(op, asm) \ 178 ATOMIC64_FETCH_OP(op, asm) 179 180ATOMIC_OPS(and, and) 181ATOMIC_OPS(andnot, bic) 182ATOMIC_OPS(or, bis) 183ATOMIC_OPS(xor, xor) 184 185#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed 186#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 187#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed 188#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed 189 190#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 191#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 192#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 193#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 194 195#undef ATOMIC_OPS 196#undef ATOMIC64_FETCH_OP 197#undef ATOMIC64_OP_RETURN 198#undef ATOMIC64_OP 199#undef ATOMIC_FETCH_OP 200#undef ATOMIC_OP_RETURN 201#undef ATOMIC_OP 202 203static __inline__ int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 204{ 205 int c, new, old; 206 smp_mb(); 207 __asm__ __volatile__( 208 "1: ldl_l %[old],%[mem]\n" 209 " cmpeq %[old],%[u],%[c]\n" 210 " addl %[old],%[a],%[new]\n" 211 " bne %[c],2f\n" 212 " stl_c %[new],%[mem]\n" 213 " beq %[new],3f\n" 214 "2:\n" 215 ".subsection 2\n" 216 "3: br 1b\n" 217 ".previous" 218 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 219 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"((long)u) 220 : "memory"); 221 smp_mb(); 222 return old; 223} 224#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 225 226static __inline__ s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 227{ 228 s64 c, new, old; 229 smp_mb(); 230 __asm__ __volatile__( 231 "1: ldq_l %[old],%[mem]\n" 232 " cmpeq %[old],%[u],%[c]\n" 233 " addq %[old],%[a],%[new]\n" 234 " bne %[c],2f\n" 235 " stq_c %[new],%[mem]\n" 236 " beq %[new],3f\n" 237 "2:\n" 238 ".subsection 2\n" 239 "3: br 1b\n" 240 ".previous" 241 : [old] "=&r"(old), [new] "=&r"(new), [c] "=&r"(c) 242 : [mem] "m"(*v), [a] "rI"(a), [u] "rI"(u) 243 : "memory"); 244 smp_mb(); 245 return old; 246} 247#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 248 249static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 250{ 251 s64 old, tmp; 252 smp_mb(); 253 __asm__ __volatile__( 254 "1: ldq_l %[old],%[mem]\n" 255 " subq %[old],1,%[tmp]\n" 256 " ble %[old],2f\n" 257 " stq_c %[tmp],%[mem]\n" 258 " beq %[tmp],3f\n" 259 "2:\n" 260 ".subsection 2\n" 261 "3: br 1b\n" 262 ".previous" 263 : [old] "=&r"(old), [tmp] "=&r"(tmp) 264 : [mem] "m"(*v) 265 : "memory"); 266 smp_mb(); 267 return old - 1; 268} 269#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 270 271#endif /* _ALPHA_ATOMIC_H */ 272