1#ifndef _ASM_X86_CMPXCHG_32_H 2#define _ASM_X86_CMPXCHG_32_H 3 4#include <linux/bitops.h> /* for LOCK_PREFIX */ 5 6/* 7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you 8 * you need to test for the feature in boot_cpu_data. 9 */ 10 11extern void __xchg_wrong_size(void); 12 13/* 14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway. 15 * Since this is generally used to protect other memory information, we 16 * use "asm volatile" and "memory" clobbers to prevent gcc from moving 17 * information around. 18 */ 19#define __xchg(x, ptr, size) \ 20({ \ 21 __typeof(*(ptr)) __x = (x); \ 22 switch (size) { \ 23 case 1: \ 24 { \ 25 volatile u8 *__ptr = (volatile u8 *)(ptr); \ 26 asm volatile("xchgb %0,%1" \ 27 : "=q" (__x), "+m" (*__ptr) \ 28 : "0" (__x) \ 29 : "memory"); \ 30 break; \ 31 } \ 32 case 2: \ 33 { \ 34 volatile u16 *__ptr = (volatile u16 *)(ptr); \ 35 asm volatile("xchgw %0,%1" \ 36 : "=r" (__x), "+m" (*__ptr) \ 37 : "0" (__x) \ 38 : "memory"); \ 39 break; \ 40 } \ 41 case 4: \ 42 { \ 43 volatile u32 *__ptr = (volatile u32 *)(ptr); \ 44 asm volatile("xchgl %0,%1" \ 45 : "=r" (__x), "+m" (*__ptr) \ 46 : "0" (__x) \ 47 : "memory"); \ 48 break; \ 49 } \ 50 default: \ 51 __xchg_wrong_size(); \ 52 } \ 53 __x; \ 54}) 55 56#define xchg(ptr, v) \ 57 __xchg((v), (ptr), sizeof(*ptr)) 58 59/* 60 * CMPXCHG8B only writes to the target if we had the previous 61 * value in registers, otherwise it acts as a read and gives us the 62 * "new previous" value. That is why there is a loop. Preloading 63 * EDX:EAX is a performance optimization: in the common case it means 64 * we need only one locked operation. 65 * 66 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very 67 * least an FPU save and/or %cr0.ts manipulation. 68 * 69 * cmpxchg8b must be used with the lock prefix here to allow the 70 * instruction to be executed atomically. We need to have the reader 71 * side to see the coherent 64bit value. 72 */ 73static inline void set_64bit(volatile u64 *ptr, u64 value) 74{ 75 u32 low = value; 76 u32 high = value >> 32; 77 u64 prev = *ptr; 78 79 asm volatile("\n1:\t" 80 LOCK_PREFIX "cmpxchg8b %0\n\t" 81 "jnz 1b" 82 : "=m" (*ptr), "+A" (prev) 83 : "b" (low), "c" (high) 84 : "memory"); 85} 86 87extern void __cmpxchg_wrong_size(void); 88 89/* 90 * Atomic compare and exchange. Compare OLD with MEM, if identical, 91 * store NEW in MEM. Return the initial value in MEM. Success is 92 * indicated by comparing RETURN with OLD. 93 */ 94#define __raw_cmpxchg(ptr, old, new, size, lock) \ 95({ \ 96 __typeof__(*(ptr)) __ret; \ 97 __typeof__(*(ptr)) __old = (old); \ 98 __typeof__(*(ptr)) __new = (new); \ 99 switch (size) { \ 100 case 1: \ 101 { \ 102 volatile u8 *__ptr = (volatile u8 *)(ptr); \ 103 asm volatile(lock "cmpxchgb %2,%1" \ 104 : "=a" (__ret), "+m" (*__ptr) \ 105 : "q" (__new), "0" (__old) \ 106 : "memory"); \ 107 break; \ 108 } \ 109 case 2: \ 110 { \ 111 volatile u16 *__ptr = (volatile u16 *)(ptr); \ 112 asm volatile(lock "cmpxchgw %2,%1" \ 113 : "=a" (__ret), "+m" (*__ptr) \ 114 : "r" (__new), "0" (__old) \ 115 : "memory"); \ 116 break; \ 117 } \ 118 case 4: \ 119 { \ 120 volatile u32 *__ptr = (volatile u32 *)(ptr); \ 121 asm volatile(lock "cmpxchgl %2,%1" \ 122 : "=a" (__ret), "+m" (*__ptr) \ 123 : "r" (__new), "0" (__old) \ 124 : "memory"); \ 125 break; \ 126 } \ 127 default: \ 128 __cmpxchg_wrong_size(); \ 129 } \ 130 __ret; \ 131}) 132 133#define __cmpxchg(ptr, old, new, size) \ 134 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) 135 136#define __sync_cmpxchg(ptr, old, new, size) \ 137 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ") 138 139#define __cmpxchg_local(ptr, old, new, size) \ 140 __raw_cmpxchg((ptr), (old), (new), (size), "") 141 142#ifdef CONFIG_X86_CMPXCHG 143#define __HAVE_ARCH_CMPXCHG 1 144 145#define cmpxchg(ptr, old, new) \ 146 __cmpxchg((ptr), (old), (new), sizeof(*ptr)) 147 148#define sync_cmpxchg(ptr, old, new) \ 149 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr)) 150 151#define cmpxchg_local(ptr, old, new) \ 152 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr)) 153#endif 154 155#ifdef CONFIG_X86_CMPXCHG64 156#define cmpxchg64(ptr, o, n) \ 157 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ 158 (unsigned long long)(n))) 159#define cmpxchg64_local(ptr, o, n) \ 160 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ 161 (unsigned long long)(n))) 162#endif 163 164static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 165{ 166 u64 prev; 167 asm volatile(LOCK_PREFIX "cmpxchg8b %1" 168 : "=A" (prev), 169 "+m" (*ptr) 170 : "b" ((u32)new), 171 "c" ((u32)(new >> 32)), 172 "0" (old) 173 : "memory"); 174 return prev; 175} 176 177static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 178{ 179 u64 prev; 180 asm volatile("cmpxchg8b %1" 181 : "=A" (prev), 182 "+m" (*ptr) 183 : "b" ((u32)new), 184 "c" ((u32)(new >> 32)), 185 "0" (old) 186 : "memory"); 187 return prev; 188} 189 190#ifndef CONFIG_X86_CMPXCHG 191/* 192 * Building a kernel capable running on 80386. It may be necessary to 193 * simulate the cmpxchg on the 80386 CPU. For that purpose we define 194 * a function for each of the sizes we support. 195 */ 196 197extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8); 198extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); 199extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); 200 201static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, 202 unsigned long new, int size) 203{ 204 switch (size) { 205 case 1: 206 return cmpxchg_386_u8(ptr, old, new); 207 case 2: 208 return cmpxchg_386_u16(ptr, old, new); 209 case 4: 210 return cmpxchg_386_u32(ptr, old, new); 211 } 212 return old; 213} 214 215#define cmpxchg(ptr, o, n) \ 216({ \ 217 __typeof__(*(ptr)) __ret; \ 218 if (likely(boot_cpu_data.x86 > 3)) \ 219 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \ 220 (unsigned long)(o), (unsigned long)(n), \ 221 sizeof(*(ptr))); \ 222 else \ 223 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 224 (unsigned long)(o), (unsigned long)(n), \ 225 sizeof(*(ptr))); \ 226 __ret; \ 227}) 228#define cmpxchg_local(ptr, o, n) \ 229({ \ 230 __typeof__(*(ptr)) __ret; \ 231 if (likely(boot_cpu_data.x86 > 3)) \ 232 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \ 233 (unsigned long)(o), (unsigned long)(n), \ 234 sizeof(*(ptr))); \ 235 else \ 236 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 237 (unsigned long)(o), (unsigned long)(n), \ 238 sizeof(*(ptr))); \ 239 __ret; \ 240}) 241#endif 242 243#ifndef CONFIG_X86_CMPXCHG64 244/* 245 * Building a kernel capable running on 80386 and 80486. It may be necessary 246 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 247 */ 248 249#define cmpxchg64(ptr, o, n) \ 250({ \ 251 __typeof__(*(ptr)) __ret; \ 252 __typeof__(*(ptr)) __old = (o); \ 253 __typeof__(*(ptr)) __new = (n); \ 254 alternative_io(LOCK_PREFIX_HERE \ 255 "call cmpxchg8b_emu", \ 256 "lock; cmpxchg8b (%%esi)" , \ 257 X86_FEATURE_CX8, \ 258 "=A" (__ret), \ 259 "S" ((ptr)), "0" (__old), \ 260 "b" ((unsigned int)__new), \ 261 "c" ((unsigned int)(__new>>32)) \ 262 : "memory"); \ 263 __ret; }) 264 265 266#define cmpxchg64_local(ptr, o, n) \ 267({ \ 268 __typeof__(*(ptr)) __ret; \ 269 __typeof__(*(ptr)) __old = (o); \ 270 __typeof__(*(ptr)) __new = (n); \ 271 alternative_io("call cmpxchg8b_emu", \ 272 "cmpxchg8b (%%esi)" , \ 273 X86_FEATURE_CX8, \ 274 "=A" (__ret), \ 275 "S" ((ptr)), "0" (__old), \ 276 "b" ((unsigned int)__new), \ 277 "c" ((unsigned int)(__new>>32)) \ 278 : "memory"); \ 279 __ret; }) 280 281#endif 282 283#endif /* _ASM_X86_CMPXCHG_32_H */ 284