1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2014 Regents of the University of California 4 */ 5 6#ifndef _ASM_RISCV_CMPXCHG_H 7#define _ASM_RISCV_CMPXCHG_H 8 9#include <linux/bug.h> 10 11#include <asm/fence.h> 12 13#define __xchg_relaxed(ptr, new, size) \ 14({ \ 15 __typeof__(ptr) __ptr = (ptr); \ 16 __typeof__(new) __new = (new); \ 17 __typeof__(*(ptr)) __ret; \ 18 switch (size) { \ 19 case 4: \ 20 __asm__ __volatile__ ( \ 21 " amoswap.w %0, %2, %1\n" \ 22 : "=r" (__ret), "+A" (*__ptr) \ 23 : "r" (__new) \ 24 : "memory"); \ 25 break; \ 26 case 8: \ 27 __asm__ __volatile__ ( \ 28 " amoswap.d %0, %2, %1\n" \ 29 : "=r" (__ret), "+A" (*__ptr) \ 30 : "r" (__new) \ 31 : "memory"); \ 32 break; \ 33 default: \ 34 BUILD_BUG(); \ 35 } \ 36 __ret; \ 37}) 38 39#define arch_xchg_relaxed(ptr, x) \ 40({ \ 41 __typeof__(*(ptr)) _x_ = (x); \ 42 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ 43 _x_, sizeof(*(ptr))); \ 44}) 45 46#define __xchg_acquire(ptr, new, size) \ 47({ \ 48 __typeof__(ptr) __ptr = (ptr); \ 49 __typeof__(new) __new = (new); \ 50 __typeof__(*(ptr)) __ret; \ 51 switch (size) { \ 52 case 4: \ 53 __asm__ __volatile__ ( \ 54 " amoswap.w %0, %2, %1\n" \ 55 RISCV_ACQUIRE_BARRIER \ 56 : "=r" (__ret), "+A" (*__ptr) \ 57 : "r" (__new) \ 58 : "memory"); \ 59 break; \ 60 case 8: \ 61 __asm__ __volatile__ ( \ 62 " amoswap.d %0, %2, %1\n" \ 63 RISCV_ACQUIRE_BARRIER \ 64 : "=r" (__ret), "+A" (*__ptr) \ 65 : "r" (__new) \ 66 : "memory"); \ 67 break; \ 68 default: \ 69 BUILD_BUG(); \ 70 } \ 71 __ret; \ 72}) 73 74#define arch_xchg_acquire(ptr, x) \ 75({ \ 76 __typeof__(*(ptr)) _x_ = (x); \ 77 (__typeof__(*(ptr))) __xchg_acquire((ptr), \ 78 _x_, sizeof(*(ptr))); \ 79}) 80 81#define __xchg_release(ptr, new, size) \ 82({ \ 83 __typeof__(ptr) __ptr = (ptr); \ 84 __typeof__(new) __new = (new); \ 85 __typeof__(*(ptr)) __ret; \ 86 switch (size) { \ 87 case 4: \ 88 __asm__ __volatile__ ( \ 89 RISCV_RELEASE_BARRIER \ 90 " amoswap.w %0, %2, %1\n" \ 91 : "=r" (__ret), "+A" (*__ptr) \ 92 : "r" (__new) \ 93 : "memory"); \ 94 break; \ 95 case 8: \ 96 __asm__ __volatile__ ( \ 97 RISCV_RELEASE_BARRIER \ 98 " amoswap.d %0, %2, %1\n" \ 99 : "=r" (__ret), "+A" (*__ptr) \ 100 : "r" (__new) \ 101 : "memory"); \ 102 break; \ 103 default: \ 104 BUILD_BUG(); \ 105 } \ 106 __ret; \ 107}) 108 109#define arch_xchg_release(ptr, x) \ 110({ \ 111 __typeof__(*(ptr)) _x_ = (x); \ 112 (__typeof__(*(ptr))) __xchg_release((ptr), \ 113 _x_, sizeof(*(ptr))); \ 114}) 115 116#define __arch_xchg(ptr, new, size) \ 117({ \ 118 __typeof__(ptr) __ptr = (ptr); \ 119 __typeof__(new) __new = (new); \ 120 __typeof__(*(ptr)) __ret; \ 121 switch (size) { \ 122 case 4: \ 123 __asm__ __volatile__ ( \ 124 " amoswap.w.aqrl %0, %2, %1\n" \ 125 : "=r" (__ret), "+A" (*__ptr) \ 126 : "r" (__new) \ 127 : "memory"); \ 128 break; \ 129 case 8: \ 130 __asm__ __volatile__ ( \ 131 " amoswap.d.aqrl %0, %2, %1\n" \ 132 : "=r" (__ret), "+A" (*__ptr) \ 133 : "r" (__new) \ 134 : "memory"); \ 135 break; \ 136 default: \ 137 BUILD_BUG(); \ 138 } \ 139 __ret; \ 140}) 141 142#define arch_xchg(ptr, x) \ 143({ \ 144 __typeof__(*(ptr)) _x_ = (x); \ 145 (__typeof__(*(ptr))) __arch_xchg((ptr), _x_, sizeof(*(ptr))); \ 146}) 147 148#define xchg32(ptr, x) \ 149({ \ 150 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 151 arch_xchg((ptr), (x)); \ 152}) 153 154#define xchg64(ptr, x) \ 155({ \ 156 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 157 arch_xchg((ptr), (x)); \ 158}) 159 160/* 161 * Atomic compare and exchange. Compare OLD with MEM, if identical, 162 * store NEW in MEM. Return the initial value in MEM. Success is 163 * indicated by comparing RETURN with OLD. 164 */ 165#define __cmpxchg_relaxed(ptr, old, new, size) \ 166({ \ 167 __typeof__(ptr) __ptr = (ptr); \ 168 __typeof__(*(ptr)) __old = (old); \ 169 __typeof__(*(ptr)) __new = (new); \ 170 __typeof__(*(ptr)) __ret; \ 171 register unsigned int __rc; \ 172 switch (size) { \ 173 case 4: \ 174 __asm__ __volatile__ ( \ 175 "0: lr.w %0, %2\n" \ 176 " bne %0, %z3, 1f\n" \ 177 " sc.w %1, %z4, %2\n" \ 178 " bnez %1, 0b\n" \ 179 "1:\n" \ 180 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 181 : "rJ" ((long)__old), "rJ" (__new) \ 182 : "memory"); \ 183 break; \ 184 case 8: \ 185 __asm__ __volatile__ ( \ 186 "0: lr.d %0, %2\n" \ 187 " bne %0, %z3, 1f\n" \ 188 " sc.d %1, %z4, %2\n" \ 189 " bnez %1, 0b\n" \ 190 "1:\n" \ 191 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 192 : "rJ" (__old), "rJ" (__new) \ 193 : "memory"); \ 194 break; \ 195 default: \ 196 BUILD_BUG(); \ 197 } \ 198 __ret; \ 199}) 200 201#define arch_cmpxchg_relaxed(ptr, o, n) \ 202({ \ 203 __typeof__(*(ptr)) _o_ = (o); \ 204 __typeof__(*(ptr)) _n_ = (n); \ 205 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ 206 _o_, _n_, sizeof(*(ptr))); \ 207}) 208 209#define __cmpxchg_acquire(ptr, old, new, size) \ 210({ \ 211 __typeof__(ptr) __ptr = (ptr); \ 212 __typeof__(*(ptr)) __old = (old); \ 213 __typeof__(*(ptr)) __new = (new); \ 214 __typeof__(*(ptr)) __ret; \ 215 register unsigned int __rc; \ 216 switch (size) { \ 217 case 4: \ 218 __asm__ __volatile__ ( \ 219 "0: lr.w %0, %2\n" \ 220 " bne %0, %z3, 1f\n" \ 221 " sc.w %1, %z4, %2\n" \ 222 " bnez %1, 0b\n" \ 223 RISCV_ACQUIRE_BARRIER \ 224 "1:\n" \ 225 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 226 : "rJ" ((long)__old), "rJ" (__new) \ 227 : "memory"); \ 228 break; \ 229 case 8: \ 230 __asm__ __volatile__ ( \ 231 "0: lr.d %0, %2\n" \ 232 " bne %0, %z3, 1f\n" \ 233 " sc.d %1, %z4, %2\n" \ 234 " bnez %1, 0b\n" \ 235 RISCV_ACQUIRE_BARRIER \ 236 "1:\n" \ 237 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 238 : "rJ" (__old), "rJ" (__new) \ 239 : "memory"); \ 240 break; \ 241 default: \ 242 BUILD_BUG(); \ 243 } \ 244 __ret; \ 245}) 246 247#define arch_cmpxchg_acquire(ptr, o, n) \ 248({ \ 249 __typeof__(*(ptr)) _o_ = (o); \ 250 __typeof__(*(ptr)) _n_ = (n); \ 251 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ 252 _o_, _n_, sizeof(*(ptr))); \ 253}) 254 255#define __cmpxchg_release(ptr, old, new, size) \ 256({ \ 257 __typeof__(ptr) __ptr = (ptr); \ 258 __typeof__(*(ptr)) __old = (old); \ 259 __typeof__(*(ptr)) __new = (new); \ 260 __typeof__(*(ptr)) __ret; \ 261 register unsigned int __rc; \ 262 switch (size) { \ 263 case 4: \ 264 __asm__ __volatile__ ( \ 265 RISCV_RELEASE_BARRIER \ 266 "0: lr.w %0, %2\n" \ 267 " bne %0, %z3, 1f\n" \ 268 " sc.w %1, %z4, %2\n" \ 269 " bnez %1, 0b\n" \ 270 "1:\n" \ 271 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 272 : "rJ" ((long)__old), "rJ" (__new) \ 273 : "memory"); \ 274 break; \ 275 case 8: \ 276 __asm__ __volatile__ ( \ 277 RISCV_RELEASE_BARRIER \ 278 "0: lr.d %0, %2\n" \ 279 " bne %0, %z3, 1f\n" \ 280 " sc.d %1, %z4, %2\n" \ 281 " bnez %1, 0b\n" \ 282 "1:\n" \ 283 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 284 : "rJ" (__old), "rJ" (__new) \ 285 : "memory"); \ 286 break; \ 287 default: \ 288 BUILD_BUG(); \ 289 } \ 290 __ret; \ 291}) 292 293#define arch_cmpxchg_release(ptr, o, n) \ 294({ \ 295 __typeof__(*(ptr)) _o_ = (o); \ 296 __typeof__(*(ptr)) _n_ = (n); \ 297 (__typeof__(*(ptr))) __cmpxchg_release((ptr), \ 298 _o_, _n_, sizeof(*(ptr))); \ 299}) 300 301#define __cmpxchg(ptr, old, new, size) \ 302({ \ 303 __typeof__(ptr) __ptr = (ptr); \ 304 __typeof__(*(ptr)) __old = (old); \ 305 __typeof__(*(ptr)) __new = (new); \ 306 __typeof__(*(ptr)) __ret; \ 307 register unsigned int __rc; \ 308 switch (size) { \ 309 case 4: \ 310 __asm__ __volatile__ ( \ 311 "0: lr.w %0, %2\n" \ 312 " bne %0, %z3, 1f\n" \ 313 " sc.w.rl %1, %z4, %2\n" \ 314 " bnez %1, 0b\n" \ 315 RISCV_FULL_BARRIER \ 316 "1:\n" \ 317 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 318 : "rJ" ((long)__old), "rJ" (__new) \ 319 : "memory"); \ 320 break; \ 321 case 8: \ 322 __asm__ __volatile__ ( \ 323 "0: lr.d %0, %2\n" \ 324 " bne %0, %z3, 1f\n" \ 325 " sc.d.rl %1, %z4, %2\n" \ 326 " bnez %1, 0b\n" \ 327 RISCV_FULL_BARRIER \ 328 "1:\n" \ 329 : "=&r" (__ret), "=&r" (__rc), "+A" (*__ptr) \ 330 : "rJ" (__old), "rJ" (__new) \ 331 : "memory"); \ 332 break; \ 333 default: \ 334 BUILD_BUG(); \ 335 } \ 336 __ret; \ 337}) 338 339#define arch_cmpxchg(ptr, o, n) \ 340({ \ 341 __typeof__(*(ptr)) _o_ = (o); \ 342 __typeof__(*(ptr)) _n_ = (n); \ 343 (__typeof__(*(ptr))) __cmpxchg((ptr), \ 344 _o_, _n_, sizeof(*(ptr))); \ 345}) 346 347#define arch_cmpxchg_local(ptr, o, n) \ 348 (__cmpxchg_relaxed((ptr), (o), (n), sizeof(*(ptr)))) 349 350#define arch_cmpxchg64(ptr, o, n) \ 351({ \ 352 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 353 arch_cmpxchg((ptr), (o), (n)); \ 354}) 355 356#define arch_cmpxchg64_local(ptr, o, n) \ 357({ \ 358 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 359 arch_cmpxchg_relaxed((ptr), (o), (n)); \ 360}) 361 362#endif /* _ASM_RISCV_CMPXCHG_H */ 363