powerpc/lib/qspinlock.c

105 static inline int decode_tail_cpu(u32 val)
107 	return (val >> _Q_TAIL_CPU_OFFSET) - 1;
110 static inline int get_owner_cpu(u32 val)
112 	return (val & _Q_OWNER_CPU_MASK) >> _Q_OWNER_CPU_OFFSET;
145 	: "r" (&lock->val), "r"(tail), "r" (newval),
175 	: "r" (&lock->val), "r" (tail), "r"(_Q_TAIL_CPU_MASK)
191 	: "r" (&lock->val), "r" (_Q_MUST_Q_VAL)
207 	: "r" (&lock->val), "r" (_Q_MUST_Q_VAL)
229 	: "r" (&lock->val), "r"(old), "r" (new)
235 static __always_inline void seen_sleepy_owner(struct qspinlock *lock, u32 val)
240 		if (!(val & _Q_SLEEPY_VAL))
241 			try_set_sleepy(lock, val);
256 		/* Don't set sleepy because we likely have a stale val */
267 	 * previous val (which is the control dependency), this barrier
284 static __always_inline bool __yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt, bool mustq)
290 	BUG_ON(!(val & _Q_LOCKED_VAL));
298 	owner = get_owner_cpu(val);
306 	seen_sleepy_owner(lock, val);
318 	if (READ_ONCE(lock->val) == val) {
337 static __always_inline bool yield_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt)
339 	return __yield_to_locked_owner(lock, val, paravirt, false);
343 static __always_inline bool yield_head_to_locked_owner(struct qspinlock *lock, u32 val, bool paravirt)
347 	if ((val & _Q_MUST_Q_VAL) && pv_yield_allow_steal)
350 	return __yield_to_locked_owner(lock, val, paravirt, mustq);
353 static __always_inline void propagate_sleepy(struct qnode *node, u32 val, bool paravirt)
370 	owner = get_owner_cpu(val);
392 		u32 val = READ_ONCE(lock->val);
394 		if (val & _Q_LOCKED_VAL) {
403 				if (vcpu_is_preempted(get_owner_cpu(val)))
407 			preempted = yield_to_locked_owner(lock, val, paravirt);
442 static __always_inline bool steal_break(u32 val, int iters, bool paravirt, bool sleepy)
449 		int cpu = get_owner_cpu(val);
461 	u32 val;
473 		val = READ_ONCE(lock->val);
474 		if (val & _Q_MUST_Q_VAL)
478 		if (unlikely(!(val & _Q_LOCKED_VAL))) {
484 			preempted = yield_to_locked_owner(lock, val, paravirt);
489 				if (val & _Q_SLEEPY_VAL) {
497 			    !(val & _Q_SLEEPY_VAL)) {
498 				if (try_set_sleepy(lock, val))
499 					val |= _Q_SLEEPY_VAL;
519 	} while (!steal_break(val, iters, paravirt, sleepy));
530 	u32 val, old, tail;
612 		val = READ_ONCE(lock->val);
613 		if (!(val & _Q_LOCKED_VAL))
619 				if (val & _Q_SLEEPY_VAL) {
627 			    !(val & _Q_SLEEPY_VAL)) {
628 				if (try_set_sleepy(lock, val))
629 					val |= _Q_SLEEPY_VAL;
633 		propagate_sleepy(node, val, paravirt);
634 		preempted = yield_head_to_locked_owner(lock, val, paravirt);
653 			val |= _Q_MUST_Q_VAL;
733 static int steal_spins_set(void *data, u64 val)
737 	steal_spins = val;
749 	if (val && !steal_spins) {
753 		steal_spins = val;
754 	} else if (!val && steal_spins) {
755 		steal_spins = val;
760 		steal_spins = val;
768 static int steal_spins_get(void *data, u64 *val)
770 	*val = steal_spins;
777 static int remote_steal_spins_set(void *data, u64 val)
779 	remote_steal_spins = val;
784 static int remote_steal_spins_get(void *data, u64 *val)
786 	*val = remote_steal_spins;
793 static int head_spins_set(void *data, u64 val)
795 	head_spins = val;
800 static int head_spins_get(void *data, u64 *val)
802 	*val = head_spins;
809 static int pv_yield_owner_set(void *data, u64 val)
811 	pv_yield_owner = !!val;
816 static int pv_yield_owner_get(void *data, u64 *val)
818 	*val = pv_yield_owner;
825 static int pv_yield_allow_steal_set(void *data, u64 val)
827 	pv_yield_allow_steal = !!val;
832 static int pv_yield_allow_steal_get(void *data, u64 *val)
834 	*val = pv_yield_allow_steal;
841 static int pv_spin_on_preempted_owner_set(void *data, u64 val)
843 	pv_spin_on_preempted_owner = !!val;
848 static int pv_spin_on_preempted_owner_get(void *data, u64 *val)
850 	*val = pv_spin_on_preempted_owner;
857 static int pv_sleepy_lock_set(void *data, u64 val)
859 	pv_sleepy_lock = !!val;
864 static int pv_sleepy_lock_get(void *data, u64 *val)
866 	*val = pv_sleepy_lock;
873 static int pv_sleepy_lock_sticky_set(void *data, u64 val)
875 	pv_sleepy_lock_sticky = !!val;
880 static int pv_sleepy_lock_sticky_get(void *data, u64 *val)
882 	*val = pv_sleepy_lock_sticky;
889 static int pv_sleepy_lock_interval_ns_set(void *data, u64 val)
891 	pv_sleepy_lock_interval_ns = val;
896 static int pv_sleepy_lock_interval_ns_get(void *data, u64 *val)
898 	*val = pv_sleepy_lock_interval_ns;
905 static int pv_sleepy_lock_factor_set(void *data, u64 val)
907 	pv_sleepy_lock_factor = val;
912 static int pv_sleepy_lock_factor_get(void *data, u64 *val)
914 	*val = pv_sleepy_lock_factor;
921 static int pv_yield_prev_set(void *data, u64 val)
923 	pv_yield_prev = !!val;
928 static int pv_yield_prev_get(void *data, u64 *val)
930 	*val = pv_yield_prev;
937 static int pv_yield_sleepy_owner_set(void *data, u64 val)
939 	pv_yield_sleepy_owner = !!val;
944 static int pv_yield_sleepy_owner_get(void *data, u64 *val)
946 	*val = pv_yield_sleepy_owner;
953 static int pv_prod_head_set(void *data, u64 val)
955 	pv_prod_head = !!val;
960 static int pv_prod_head_get(void *data, u64 *val)
962 	*val = pv_prod_head;