1/* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for x86_64+ 2 * 3 * Written by David Howells (dhowells@redhat.com). 4 * Ported by Andi Kleen <ak@suse.de> to x86-64. 5 * 6 * Derived from asm-i386/semaphore.h and asm-i386/rwsem.h 7 * 8 * Trylock by Brian Watson (Brian.J.Watson@compaq.com). 9 * 10 * The MSW of the count is the negated number of active writers and waiting 11 * lockers, and the LSW is the total number of active locks 12 * 13 * The lock count is initialized to 0 (no active and no waiting lockers). 14 * 15 * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an 16 * uncontended lock. This can be determined because XADD returns the old value. 17 * Readers increment by 1 and see a positive value when uncontended, negative 18 * if there are writers (and maybe) readers waiting (in which case it goes to 19 * sleep). 20 * 21 * The value of WAITING_BIAS supports up to 32766 waiting processes. This can 22 * be extended to 65534 by manually checking the whole MSW rather than relying 23 * on the S flag. 24 * 25 * The value of ACTIVE_BIAS supports up to 65535 active processes. 26 * 27 * This should be totally fair - if anything is waiting, a process that wants a 28 * lock will go to the back of the queue. When the currently active lock is 29 * released, if there's a writer at the front of the queue, then that and only 30 * that will be woken up; if there's a bunch of consequtive readers at the 31 * front, then they'll all be woken up, but no other readers will be. 32 */ 33 34#ifndef _X8664_RWSEM_H 35#define _X8664_RWSEM_H 36 37#ifndef _LINUX_RWSEM_H 38#error please dont include asm/rwsem.h directly, use linux/rwsem.h instead 39#endif 40 41#ifdef __KERNEL__ 42 43#include <linux/list.h> 44#include <linux/spinlock.h> 45 46struct rwsem_waiter; 47 48extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); 49extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); 50extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); 51 52/* 53 * the semaphore definition 54 */ 55struct rw_semaphore { 56 signed int count; 57#define RWSEM_UNLOCKED_VALUE 0x00000000 58#define RWSEM_ACTIVE_BIAS 0x00000001 59#define RWSEM_ACTIVE_MASK 0x0000ffff 60#define RWSEM_WAITING_BIAS (-0x00010000) 61#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS 62#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) 63 spinlock_t wait_lock; 64 struct list_head wait_list; 65#if RWSEM_DEBUG 66 int debug; 67#endif 68}; 69 70/* 71 * initialisation 72 */ 73#if RWSEM_DEBUG 74#define __RWSEM_DEBUG_INIT , 0 75#else 76#define __RWSEM_DEBUG_INIT /* */ 77#endif 78 79#define __RWSEM_INITIALIZER(name) \ 80{ RWSEM_UNLOCKED_VALUE, SPIN_LOCK_UNLOCKED, LIST_HEAD_INIT((name).wait_list) \ 81 __RWSEM_DEBUG_INIT } 82 83#define DECLARE_RWSEM(name) \ 84 struct rw_semaphore name = __RWSEM_INITIALIZER(name) 85 86static inline void init_rwsem(struct rw_semaphore *sem) 87{ 88 sem->count = RWSEM_UNLOCKED_VALUE; 89 spin_lock_init(&sem->wait_lock); 90 INIT_LIST_HEAD(&sem->wait_list); 91#if RWSEM_DEBUG 92 sem->debug = 0; 93#endif 94} 95 96/* 97 * lock for reading 98 */ 99static inline void __down_read(struct rw_semaphore *sem) 100{ 101 __asm__ __volatile__( 102 "# beginning down_read\n\t" 103LOCK_PREFIX " incl (%%rdi)\n\t" /* adds 0x00000001, returns the old value */ 104 " js 2f\n\t" /* jump if we weren't granted the lock */ 105 "1:\n\t" 106 LOCK_SECTION_START("") \ 107 "2:\n\t" 108 " call rwsem_down_read_failed_thunk\n\t" 109 " jmp 1b\n" 110 LOCK_SECTION_END \ 111 "# ending down_read\n\t" 112 : "+m"(sem->count) 113 : "D"(sem) 114 : "memory", "cc"); 115} 116 117 118/* 119 * trylock for reading -- returns 1 if successful, 0 if contention 120 */ 121static inline int __down_read_trylock(struct rw_semaphore *sem) 122{ 123 __s32 result, tmp; 124 __asm__ __volatile__( 125 "# beginning __down_read_trylock\n\t" 126 " movl %0,%1\n\t" 127 "1:\n\t" 128 " movl %1,%2\n\t" 129 " addl %3,%2\n\t" 130 " jle 2f\n\t" 131LOCK_PREFIX " cmpxchgl %2,%0\n\t" 132 " jnz 1b\n\t" 133 "2:\n\t" 134 "# ending __down_read_trylock\n\t" 135 : "+m"(sem->count), "=&a"(result), "=&r"(tmp) 136 : "i"(RWSEM_ACTIVE_READ_BIAS) 137 : "memory", "cc"); 138 return result>=0 ? 1 : 0; 139} 140 141/* 142 * lock for writing 143 */ 144static inline void __down_write(struct rw_semaphore *sem) 145{ 146 int tmp; 147 148 tmp = RWSEM_ACTIVE_WRITE_BIAS; 149 __asm__ __volatile__( 150 "# beginning down_write\n\t" 151LOCK_PREFIX " xaddl %0,(%%rdi)\n\t" /* subtract 0x0000ffff, returns the old value */ 152 " testl %0,%0\n\t" /* was the count 0 before? */ 153 " jnz 2f\n\t" /* jump if we weren't granted the lock */ 154 "1:\n\t" 155 LOCK_SECTION_START("") 156 "2:\n\t" 157 " call rwsem_down_write_failed_thunk\n\t" 158 " jmp 1b\n" 159 LOCK_SECTION_END 160 "# ending down_write" 161 : "=&r" (tmp) 162 : "0"(tmp), "D"(sem) 163 : "memory", "cc"); 164} 165 166/* 167 * trylock for writing -- returns 1 if successful, 0 if contention 168 */ 169static inline int __down_write_trylock(struct rw_semaphore *sem) 170{ 171 signed long ret = cmpxchg(&sem->count, 172 RWSEM_UNLOCKED_VALUE, 173 RWSEM_ACTIVE_WRITE_BIAS); 174 if (ret == RWSEM_UNLOCKED_VALUE) 175 return 1; 176 return 0; 177} 178 179/* 180 * unlock after reading 181 */ 182static inline void __up_read(struct rw_semaphore *sem) 183{ 184 __s32 tmp = -RWSEM_ACTIVE_READ_BIAS; 185 __asm__ __volatile__( 186 "# beginning __up_read\n\t" 187LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* subtracts 1, returns the old value */ 188 " js 2f\n\t" /* jump if the lock is being waited upon */ 189 "1:\n\t" 190 LOCK_SECTION_START("") 191 "2:\n\t" 192 " decw %%dx\n\t" /* do nothing if still outstanding active readers */ 193 " jnz 1b\n\t" 194 " call rwsem_wake_thunk\n\t" 195 " jmp 1b\n" 196 LOCK_SECTION_END 197 "# ending __up_read\n" 198 : "+m"(sem->count), "+d"(tmp) 199 : "D"(sem) 200 : "memory", "cc"); 201} 202 203/* 204 * unlock after writing 205 */ 206static inline void __up_write(struct rw_semaphore *sem) 207{ 208 __asm__ __volatile__( 209 "# beginning __up_write\n\t" 210 " movl %2,%%edx\n\t" 211LOCK_PREFIX " xaddl %%edx,(%%rdi)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ 212 " jnz 2f\n\t" /* jump if the lock is being waited upon */ 213 "1:\n\t" 214 LOCK_SECTION_START("") 215 "2:\n\t" 216 " decw %%dx\n\t" /* did the active count reduce to 0? */ 217 " jnz 1b\n\t" /* jump back if not */ 218 " call rwsem_wake_thunk\n\t" 219 " jmp 1b\n" 220 LOCK_SECTION_END 221 "# ending __up_write\n" 222 : "+m"(sem->count) 223 : "D"(sem), "i"(-RWSEM_ACTIVE_WRITE_BIAS) 224 : "memory", "cc", "rdx"); 225} 226 227/* 228 * implement atomic add functionality 229 */ 230static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem) 231{ 232 __asm__ __volatile__( 233LOCK_PREFIX "addl %1,%0" 234 :"=m"(sem->count) 235 :"ir"(delta), "m"(sem->count)); 236} 237 238/* 239 * implement exchange and add functionality 240 */ 241static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem) 242{ 243 int tmp = delta; 244 245 __asm__ __volatile__( 246LOCK_PREFIX "xaddl %0,(%2)" 247 : "=r"(tmp), "=m"(sem->count) 248 : "r"(sem), "m"(sem->count), "0" (tmp) 249 : "memory"); 250 251 return tmp+delta; 252} 253 254#endif /* __KERNEL__ */ 255#endif /* _X8664_RWSEM_H */ 256