1278277Sgonzo/* 2278277Sgonzo * This file is subject to the terms and conditions of the GNU General Public 3278277Sgonzo * License. See the file "COPYING" in the main directory of this archive 4278277Sgonzo * for more details. 5278277Sgonzo * 6278277Sgonzo * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7278277Sgonzo */ 8278277Sgonzo#ifndef __ASM_CMPXCHG_H 9278277Sgonzo#define __ASM_CMPXCHG_H 10278277Sgonzo 11278277Sgonzo#include <linux/bug.h> 12278277Sgonzo#include <linux/irqflags.h> 13278277Sgonzo#include <asm/asm.h> 14278277Sgonzo#include <asm/compiler.h> 15278277Sgonzo#include <asm/sync.h> 16278277Sgonzo 17278277Sgonzo/* 18278277Sgonzo * These functions doesn't exist, so if they are called you'll either: 19278277Sgonzo * 20278277Sgonzo * - Get an error at compile-time due to __compiletime_error, if supported by 21278277Sgonzo * your compiler. 22278277Sgonzo * 23278277Sgonzo * or: 24278277Sgonzo * 25278277Sgonzo * - Get an error at link-time due to the call to the missing function. 26278277Sgonzo */ 27278277Sgonzoextern unsigned long __cmpxchg_called_with_bad_pointer(void) 28278277Sgonzo __compiletime_error("Bad argument size for cmpxchg"); 29278277Sgonzoextern unsigned long __cmpxchg64_unsupported(void) 30278277Sgonzo __compiletime_error("cmpxchg64 not available; cpu_has_64bits may be false"); 31278277Sgonzoextern unsigned long __xchg_called_with_bad_pointer(void) 32278277Sgonzo __compiletime_error("Bad argument size for xchg"); 33278277Sgonzo 34278277Sgonzo#define __xchg_asm(ld, st, m, val) \ 35278277Sgonzo({ \ 36278277Sgonzo __typeof(*(m)) __ret; \ 37278277Sgonzo \ 38278277Sgonzo if (kernel_uses_llsc) { \ 39278277Sgonzo __asm__ __volatile__( \ 40278277Sgonzo " .set push \n" \ 41278277Sgonzo " .set noat \n" \ 42278277Sgonzo " .set push \n" \ 43278277Sgonzo " .set " MIPS_ISA_ARCH_LEVEL " \n" \ 44278277Sgonzo " " __SYNC(full, loongson3_war) " \n" \ 45278277Sgonzo "1: " ld " %0, %2 # __xchg_asm \n" \ 46278277Sgonzo " .set pop \n" \ 47278277Sgonzo " move $1, %z3 \n" \ 48278277Sgonzo " .set " MIPS_ISA_ARCH_LEVEL " \n" \ 49278277Sgonzo " " st " $1, %1 \n" \ 50278277Sgonzo "\t" __stringify(SC_BEQZ) " $1, 1b \n" \ 51278277Sgonzo " .set pop \n" \ 52278277Sgonzo : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 53278277Sgonzo : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) \ 54278277Sgonzo : __LLSC_CLOBBER); \ 55278277Sgonzo } else { \ 56278277Sgonzo unsigned long __flags; \ 57278277Sgonzo \ 58278277Sgonzo raw_local_irq_save(__flags); \ 59278277Sgonzo __ret = *m; \ 60278277Sgonzo *m = val; \ 61278277Sgonzo raw_local_irq_restore(__flags); \ 62278277Sgonzo } \ 63278277Sgonzo \ 64278277Sgonzo __ret; \ 65278277Sgonzo}) 66278277Sgonzo 67278277Sgonzoextern unsigned long __xchg_small(volatile void *ptr, unsigned long val, 68278277Sgonzo unsigned int size); 69278277Sgonzo 70278277Sgonzostatic __always_inline 71278277Sgonzounsigned long __arch_xchg(volatile void *ptr, unsigned long x, int size) 72278277Sgonzo{ 73278277Sgonzo switch (size) { 74278277Sgonzo case 1: 75278277Sgonzo case 2: 76278277Sgonzo return __xchg_small(ptr, x, size); 77278277Sgonzo 78278277Sgonzo case 4: 79278277Sgonzo return __xchg_asm("ll", "sc", (volatile u32 *)ptr, x); 80278277Sgonzo 81278277Sgonzo case 8: 82278277Sgonzo if (!IS_ENABLED(CONFIG_64BIT)) 83278277Sgonzo return __xchg_called_with_bad_pointer(); 84278277Sgonzo 85278277Sgonzo return __xchg_asm("lld", "scd", (volatile u64 *)ptr, x); 86278277Sgonzo 87278277Sgonzo default: 88278277Sgonzo return __xchg_called_with_bad_pointer(); 89278277Sgonzo } 90278277Sgonzo} 91278277Sgonzo 92278277Sgonzo#define arch_xchg(ptr, x) \ 93278277Sgonzo({ \ 94278277Sgonzo __typeof__(*(ptr)) __res; \ 95278277Sgonzo \ 96278277Sgonzo /* \ 97278277Sgonzo * In the Loongson3 workaround case __xchg_asm() already \ 98278277Sgonzo * contains a completion barrier prior to the LL, so we don't \ 99278277Sgonzo * need to emit an extra one here. \ 100278277Sgonzo */ \ 101278277Sgonzo if (__SYNC_loongson3_war == 0) \ 102278277Sgonzo smp_mb__before_llsc(); \ 103278277Sgonzo \ 104278277Sgonzo __res = (__typeof__(*(ptr))) \ 105278277Sgonzo __arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \ 106278277Sgonzo \ 107278277Sgonzo smp_llsc_mb(); \ 108278277Sgonzo \ 109278277Sgonzo __res; \ 110278277Sgonzo}) 111278277Sgonzo 112278277Sgonzo#define __cmpxchg_asm(ld, st, m, old, new) \ 113278277Sgonzo({ \ 114278277Sgonzo __typeof(*(m)) __ret; \ 115278277Sgonzo \ 116278277Sgonzo if (kernel_uses_llsc) { \ 117278277Sgonzo __asm__ __volatile__( \ 118278277Sgonzo " .set push \n" \ 119278277Sgonzo " .set noat \n" \ 120278277Sgonzo " .set push \n" \ 121278277Sgonzo " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 122278277Sgonzo " " __SYNC(full, loongson3_war) " \n" \ 123278277Sgonzo "1: " ld " %0, %2 # __cmpxchg_asm \n" \ 124278277Sgonzo " bne %0, %z3, 2f \n" \ 125278277Sgonzo " .set pop \n" \ 126278277Sgonzo " move $1, %z4 \n" \ 127278277Sgonzo " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 128278277Sgonzo " " st " $1, %1 \n" \ 129278277Sgonzo "\t" __stringify(SC_BEQZ) " $1, 1b \n" \ 130278277Sgonzo " .set pop \n" \ 131278277Sgonzo "2: " __SYNC(full, loongson3_war) " \n" \ 132278277Sgonzo : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 133278277Sgonzo : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ 134278277Sgonzo : __LLSC_CLOBBER); \ 135278277Sgonzo } else { \ 136278277Sgonzo unsigned long __flags; \ 137278277Sgonzo \ 138278277Sgonzo raw_local_irq_save(__flags); \ 139278277Sgonzo __ret = *m; \ 140278277Sgonzo if (__ret == old) \ 141278277Sgonzo *m = new; \ 142278277Sgonzo raw_local_irq_restore(__flags); \ 143278277Sgonzo } \ 144278277Sgonzo \ 145278277Sgonzo __ret; \ 146278277Sgonzo}) 147278277Sgonzo 148278277Sgonzoextern unsigned long __cmpxchg_small(volatile void *ptr, unsigned long old, 149278277Sgonzo unsigned long new, unsigned int size); 150278277Sgonzo 151278277Sgonzostatic __always_inline 152278277Sgonzounsigned long __cmpxchg(volatile void *ptr, unsigned long old, 153278277Sgonzo unsigned long new, unsigned int size) 154278277Sgonzo{ 155278277Sgonzo switch (size) { 156278277Sgonzo case 1: 157278277Sgonzo case 2: 158278277Sgonzo return __cmpxchg_small(ptr, old, new, size); 159278277Sgonzo 160278277Sgonzo case 4: 161278277Sgonzo return __cmpxchg_asm("ll", "sc", (volatile u32 *)ptr, 162278277Sgonzo (u32)old, new); 163278277Sgonzo 164278277Sgonzo case 8: 165278277Sgonzo /* lld/scd are only available for MIPS64 */ 166278277Sgonzo if (!IS_ENABLED(CONFIG_64BIT)) 167278277Sgonzo return __cmpxchg_called_with_bad_pointer(); 168278277Sgonzo 169278277Sgonzo return __cmpxchg_asm("lld", "scd", (volatile u64 *)ptr, 170278277Sgonzo (u64)old, new); 171278277Sgonzo 172278277Sgonzo default: 173278277Sgonzo return __cmpxchg_called_with_bad_pointer(); 174278277Sgonzo } 175278277Sgonzo} 176278277Sgonzo 177278277Sgonzo#define arch_cmpxchg_local(ptr, old, new) \ 178278277Sgonzo ((__typeof__(*(ptr))) \ 179278277Sgonzo __cmpxchg((ptr), \ 180278277Sgonzo (unsigned long)(__typeof__(*(ptr)))(old), \ 181278277Sgonzo (unsigned long)(__typeof__(*(ptr)))(new), \ 182278277Sgonzo sizeof(*(ptr)))) 183278277Sgonzo 184278277Sgonzo#define arch_cmpxchg(ptr, old, new) \ 185278277Sgonzo({ \ 186278277Sgonzo __typeof__(*(ptr)) __res; \ 187278277Sgonzo \ 188278277Sgonzo /* \ 189278277Sgonzo * In the Loongson3 workaround case __cmpxchg_asm() already \ 190278277Sgonzo * contains a completion barrier prior to the LL, so we don't \ 191278277Sgonzo * need to emit an extra one here. \ 192278277Sgonzo */ \ 193278277Sgonzo if (__SYNC_loongson3_war == 0) \ 194278277Sgonzo smp_mb__before_llsc(); \ 195278277Sgonzo \ 196278277Sgonzo __res = arch_cmpxchg_local((ptr), (old), (new)); \ 197278277Sgonzo \ 198278277Sgonzo /* \ 199278277Sgonzo * In the Loongson3 workaround case __cmpxchg_asm() already \ 200278277Sgonzo * contains a completion barrier after the SC, so we don't \ 201278277Sgonzo * need to emit an extra one here. \ 202278277Sgonzo */ \ 203278277Sgonzo if (__SYNC_loongson3_war == 0) \ 204278277Sgonzo smp_llsc_mb(); \ 205278277Sgonzo \ 206278277Sgonzo __res; \ 207278277Sgonzo}) 208278277Sgonzo 209278277Sgonzo#ifdef CONFIG_64BIT 210278277Sgonzo#define arch_cmpxchg64_local(ptr, o, n) \ 211278277Sgonzo ({ \ 212278277Sgonzo BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 213278277Sgonzo arch_cmpxchg_local((ptr), (o), (n)); \ 214278277Sgonzo }) 215278277Sgonzo 216278277Sgonzo#define arch_cmpxchg64(ptr, o, n) \ 217278277Sgonzo ({ \ 218278277Sgonzo BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 219278277Sgonzo arch_cmpxchg((ptr), (o), (n)); \ 220278277Sgonzo }) 221278277Sgonzo#else 222278277Sgonzo 223278277Sgonzo# include <asm-generic/cmpxchg-local.h> 224278277Sgonzo# define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) 225278277Sgonzo 226278277Sgonzo# ifdef CONFIG_SMP 227278277Sgonzo 228278277Sgonzostatic inline unsigned long __cmpxchg64(volatile void *ptr, 229278277Sgonzo unsigned long long old, 230278277Sgonzo unsigned long long new) 231278277Sgonzo{ 232278277Sgonzo unsigned long long tmp, ret; 233278277Sgonzo unsigned long flags; 234278277Sgonzo 235278277Sgonzo /* 236278277Sgonzo * The assembly below has to combine 32 bit values into a 64 bit 237278277Sgonzo * register, and split 64 bit values from one register into two. If we 238278277Sgonzo * were to take an interrupt in the middle of this we'd only save the 239278277Sgonzo * least significant 32 bits of each register & probably clobber the 240278277Sgonzo * most significant 32 bits of the 64 bit values we're using. In order 241278277Sgonzo * to avoid this we must disable interrupts. 242278277Sgonzo */ 243278277Sgonzo local_irq_save(flags); 244278277Sgonzo 245278277Sgonzo asm volatile( 246278277Sgonzo " .set push \n" 247278277Sgonzo " .set " MIPS_ISA_ARCH_LEVEL " \n" 248278277Sgonzo /* Load 64 bits from ptr */ 249278277Sgonzo " " __SYNC(full, loongson3_war) " \n" 250278277Sgonzo "1: lld %L0, %3 # __cmpxchg64 \n" 251278277Sgonzo " .set pop \n" 252278277Sgonzo /* 253278277Sgonzo * Split the 64 bit value we loaded into the 2 registers that hold the 254278277Sgonzo * ret variable. 255278277Sgonzo */ 256278277Sgonzo " dsra %M0, %L0, 32 \n" 257278277Sgonzo " sll %L0, %L0, 0 \n" 258278277Sgonzo /* 259278277Sgonzo * Compare ret against old, breaking out of the loop if they don't 260278277Sgonzo * match. 261278277Sgonzo */ 262278277Sgonzo " bne %M0, %M4, 2f \n" 263278277Sgonzo " bne %L0, %L4, 2f \n" 264278277Sgonzo /* 265278277Sgonzo * Combine the 32 bit halves from the 2 registers that hold the new 266278277Sgonzo * variable into a single 64 bit register. 267278277Sgonzo */ 268278277Sgonzo# if MIPS_ISA_REV >= 2 269278277Sgonzo " move %L1, %L5 \n" 270278277Sgonzo " dins %L1, %M5, 32, 32 \n" 271278277Sgonzo# else 272278277Sgonzo " dsll %L1, %L5, 32 \n" 273278277Sgonzo " dsrl %L1, %L1, 32 \n" 274278277Sgonzo " .set noat \n" 275278277Sgonzo " dsll $at, %M5, 32 \n" 276278277Sgonzo " or %L1, %L1, $at \n" 277278277Sgonzo " .set at \n" 278278277Sgonzo# endif 279278277Sgonzo " .set push \n" 280278277Sgonzo " .set " MIPS_ISA_ARCH_LEVEL " \n" 281278277Sgonzo /* Attempt to store new at ptr */ 282278277Sgonzo " scd %L1, %2 \n" 283278277Sgonzo /* If we failed, loop! */ 284278277Sgonzo "\t" __stringify(SC_BEQZ) " %L1, 1b \n" 285278277Sgonzo "2: " __SYNC(full, loongson3_war) " \n" 286278277Sgonzo " .set pop \n" 287278277Sgonzo : "=&r"(ret), 288278277Sgonzo "=&r"(tmp), 289278277Sgonzo "=" GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr) 290278277Sgonzo : GCC_OFF_SMALL_ASM() (*(unsigned long long *)ptr), 291278277Sgonzo "r" (old), 292278277Sgonzo "r" (new) 293278277Sgonzo : "memory"); 294278277Sgonzo 295278277Sgonzo local_irq_restore(flags); 296278277Sgonzo return ret; 297278277Sgonzo} 298278277Sgonzo 299278277Sgonzo# define arch_cmpxchg64(ptr, o, n) ({ \ 300278277Sgonzo unsigned long long __old = (__typeof__(*(ptr)))(o); \ 301278277Sgonzo unsigned long long __new = (__typeof__(*(ptr)))(n); \ 302278277Sgonzo __typeof__(*(ptr)) __res; \ 303278277Sgonzo \ 304278277Sgonzo /* \ 305278277Sgonzo * We can only use cmpxchg64 if we know that the CPU supports \ 306278277Sgonzo * 64-bits, ie. lld & scd. Our call to __cmpxchg64_unsupported \ 307278277Sgonzo * will cause a build error unless cpu_has_64bits is a \ 308278277Sgonzo * compile-time constant 1. \ 309278277Sgonzo */ \ 310278277Sgonzo if (cpu_has_64bits && kernel_uses_llsc) { \ 311278277Sgonzo smp_mb__before_llsc(); \ 312278277Sgonzo __res = __cmpxchg64((ptr), __old, __new); \ 313278277Sgonzo smp_llsc_mb(); \ 314278277Sgonzo } else { \ 315278277Sgonzo __res = __cmpxchg64_unsupported(); \ 316278277Sgonzo } \ 317278277Sgonzo \ 318278277Sgonzo __res; \ 319278277Sgonzo}) 320278277Sgonzo 321278277Sgonzo# else /* !CONFIG_SMP */ 322278277Sgonzo# define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) 323278277Sgonzo# endif /* !CONFIG_SMP */ 324278277Sgonzo#endif /* !CONFIG_64BIT */ 325 326#endif /* __ASM_CMPXCHG_H */ 327