1/* 2 * include/asm-arm/mutex.h 3 * 4 * ARM optimized mutex locking primitives 5 * 6 * Please look into asm-generic/mutex-xchg.h for a formal definition. 7 */ 8#ifndef _ASM_MUTEX_H 9#define _ASM_MUTEX_H 10 11#if __LINUX_ARM_ARCH__ < 6 12/* On pre-ARMv6 hardware the swp based implementation is the most efficient. */ 13# include <asm-generic/mutex-xchg.h> 14#else 15 16/* 17 * Attempting to lock a mutex on ARMv6+ can be done with a bastardized 18 * atomic decrement (it is not a reliable atomic decrement but it satisfies 19 * the defined semantics for our purpose, while being smaller and faster 20 * than a real atomic decrement or atomic swap. The idea is to attempt 21 * decrementing the lock value only once. If once decremented it isn't zero, 22 * or if its store-back fails due to a dispute on the exclusive store, we 23 * simply bail out immediately through the slow path where the lock will be 24 * reattempted until it succeeds. 25 */ 26static inline void 27__mutex_fastpath_lock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *)) 28{ 29 int __ex_flag, __res; 30 31 __asm__ ( 32 33 "ldrex %0, [%2] \n\t" 34 "sub %0, %0, #1 \n\t" 35 "strex %1, %0, [%2] " 36 37 : "=&r" (__res), "=&r" (__ex_flag) 38 : "r" (&(count)->counter) 39 : "cc","memory" ); 40 41 __res |= __ex_flag; 42 if (unlikely(__res != 0)) 43 fail_fn(count); 44} 45 46static inline int 47__mutex_fastpath_lock_retval(atomic_t *count, fastcall int (*fail_fn)(atomic_t *)) 48{ 49 int __ex_flag, __res; 50 51 __asm__ ( 52 53 "ldrex %0, [%2] \n\t" 54 "sub %0, %0, #1 \n\t" 55 "strex %1, %0, [%2] " 56 57 : "=&r" (__res), "=&r" (__ex_flag) 58 : "r" (&(count)->counter) 59 : "cc","memory" ); 60 61 __res |= __ex_flag; 62 if (unlikely(__res != 0)) 63 __res = fail_fn(count); 64 return __res; 65} 66 67/* 68 * Same trick is used for the unlock fast path. However the original value, 69 * rather than the result, is used to test for success in order to have 70 * better generated assembly. 71 */ 72static inline void 73__mutex_fastpath_unlock(atomic_t *count, fastcall void (*fail_fn)(atomic_t *)) 74{ 75 int __ex_flag, __res, __orig; 76 77 __asm__ ( 78 79 "ldrex %0, [%3] \n\t" 80 "add %1, %0, #1 \n\t" 81 "strex %2, %1, [%3] " 82 83 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag) 84 : "r" (&(count)->counter) 85 : "cc","memory" ); 86 87 __orig |= __ex_flag; 88 if (unlikely(__orig != 0)) 89 fail_fn(count); 90} 91 92/* 93 * If the unlock was done on a contended lock, or if the unlock simply fails 94 * then the mutex remains locked. 95 */ 96#define __mutex_slowpath_needs_to_unlock() 1 97 98/* 99 * For __mutex_fastpath_trylock we use another construct which could be 100 * described as a "single value cmpxchg". 101 * 102 * This provides the needed trylock semantics like cmpxchg would, but it is 103 * lighter and less generic than a true cmpxchg implementation. 104 */ 105static inline int 106__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *)) 107{ 108 int __ex_flag, __res, __orig; 109 110 __asm__ ( 111 112 "1: ldrex %0, [%3] \n\t" 113 "subs %1, %0, #1 \n\t" 114 "strexeq %2, %1, [%3] \n\t" 115 "movlt %0, #0 \n\t" 116 "cmpeq %2, #0 \n\t" 117 "bgt 1b " 118 119 : "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag) 120 : "r" (&count->counter) 121 : "cc", "memory" ); 122 123 return __orig; 124} 125 126#endif 127#endif 128