1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
4 */
5
6#ifndef __ASM_SPINLOCK_H
7#define __ASM_SPINLOCK_H
8
9#include <asm/spinlock_types.h>
10#include <asm/processor.h>
11#include <asm/barrier.h>
12
13#define arch_spin_is_locked(x)	((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
14
15#ifdef CONFIG_ARC_HAS_LLSC
16
17static inline void arch_spin_lock(arch_spinlock_t *lock)
18{
19	unsigned int val;
20
21	__asm__ __volatile__(
22	"1:	llock	%[val], [%[slock]]	\n"
23	"	breq	%[val], %[LOCKED], 1b	\n"	/* spin while LOCKED */
24	"	scond	%[LOCKED], [%[slock]]	\n"	/* acquire */
25	"	bnz	1b			\n"
26	"					\n"
27	: [val]		"=&r"	(val)
28	: [slock]	"r"	(&(lock->slock)),
29	  [LOCKED]	"r"	(__ARCH_SPIN_LOCK_LOCKED__)
30	: "memory", "cc");
31
32	/*
33	 * ACQUIRE barrier to ensure load/store after taking the lock
34	 * don't "bleed-up" out of the critical section (leak-in is allowed)
35	 * http://www.spinics.net/lists/kernel/msg2010409.html
36	 *
37	 * ARCv2 only has load-load, store-store and all-all barrier
38	 * thus need the full all-all barrier
39	 */
40	smp_mb();
41}
42
43/* 1 - lock taken successfully */
44static inline int arch_spin_trylock(arch_spinlock_t *lock)
45{
46	unsigned int val, got_it = 0;
47
48	__asm__ __volatile__(
49	"1:	llock	%[val], [%[slock]]	\n"
50	"	breq	%[val], %[LOCKED], 4f	\n"	/* already LOCKED, just bail */
51	"	scond	%[LOCKED], [%[slock]]	\n"	/* acquire */
52	"	bnz	1b			\n"
53	"	mov	%[got_it], 1		\n"
54	"4:					\n"
55	"					\n"
56	: [val]		"=&r"	(val),
57	  [got_it]	"+&r"	(got_it)
58	: [slock]	"r"	(&(lock->slock)),
59	  [LOCKED]	"r"	(__ARCH_SPIN_LOCK_LOCKED__)
60	: "memory", "cc");
61
62	smp_mb();
63
64	return got_it;
65}
66
67static inline void arch_spin_unlock(arch_spinlock_t *lock)
68{
69	smp_mb();
70
71	WRITE_ONCE(lock->slock, __ARCH_SPIN_LOCK_UNLOCKED__);
72}
73
74/*
75 * Read-write spinlocks, allowing multiple readers but only one writer.
76 * Unfair locking as Writers could be starved indefinitely by Reader(s)
77 */
78
79static inline void arch_read_lock(arch_rwlock_t *rw)
80{
81	unsigned int val;
82
83	/*
84	 * zero means writer holds the lock exclusively, deny Reader.
85	 * Otherwise grant lock to first/subseq reader
86	 *
87	 * 	if (rw->counter > 0) {
88	 *		rw->counter--;
89	 *		ret = 1;
90	 *	}
91	 */
92
93	__asm__ __volatile__(
94	"1:	llock	%[val], [%[rwlock]]	\n"
95	"	brls	%[val], %[WR_LOCKED], 1b\n"	/* <= 0: spin while write locked */
96	"	sub	%[val], %[val], 1	\n"	/* reader lock */
97	"	scond	%[val], [%[rwlock]]	\n"
98	"	bnz	1b			\n"
99	"					\n"
100	: [val]		"=&r"	(val)
101	: [rwlock]	"r"	(&(rw->counter)),
102	  [WR_LOCKED]	"ir"	(0)
103	: "memory", "cc");
104
105	smp_mb();
106}
107
108/* 1 - lock taken successfully */
109static inline int arch_read_trylock(arch_rwlock_t *rw)
110{
111	unsigned int val, got_it = 0;
112
113	__asm__ __volatile__(
114	"1:	llock	%[val], [%[rwlock]]	\n"
115	"	brls	%[val], %[WR_LOCKED], 4f\n"	/* <= 0: already write locked, bail */
116	"	sub	%[val], %[val], 1	\n"	/* counter-- */
117	"	scond	%[val], [%[rwlock]]	\n"
118	"	bnz	1b			\n"	/* retry if collided with someone */
119	"	mov	%[got_it], 1		\n"
120	"					\n"
121	"4: ; --- done ---			\n"
122
123	: [val]		"=&r"	(val),
124	  [got_it]	"+&r"	(got_it)
125	: [rwlock]	"r"	(&(rw->counter)),
126	  [WR_LOCKED]	"ir"	(0)
127	: "memory", "cc");
128
129	smp_mb();
130
131	return got_it;
132}
133
134static inline void arch_write_lock(arch_rwlock_t *rw)
135{
136	unsigned int val;
137
138	/*
139	 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
140	 * deny writer. Otherwise if unlocked grant to writer
141	 * Hence the claim that Linux rwlocks are unfair to writers.
142	 * (can be starved for an indefinite time by readers).
143	 *
144	 *	if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
145	 *		rw->counter = 0;
146	 *		ret = 1;
147	 *	}
148	 */
149
150	__asm__ __volatile__(
151	"1:	llock	%[val], [%[rwlock]]	\n"
152	"	brne	%[val], %[UNLOCKED], 1b	\n"	/* while !UNLOCKED spin */
153	"	mov	%[val], %[WR_LOCKED]	\n"
154	"	scond	%[val], [%[rwlock]]	\n"
155	"	bnz	1b			\n"
156	"					\n"
157	: [val]		"=&r"	(val)
158	: [rwlock]	"r"	(&(rw->counter)),
159	  [UNLOCKED]	"ir"	(__ARCH_RW_LOCK_UNLOCKED__),
160	  [WR_LOCKED]	"ir"	(0)
161	: "memory", "cc");
162
163	smp_mb();
164}
165
166/* 1 - lock taken successfully */
167static inline int arch_write_trylock(arch_rwlock_t *rw)
168{
169	unsigned int val, got_it = 0;
170
171	__asm__ __volatile__(
172	"1:	llock	%[val], [%[rwlock]]	\n"
173	"	brne	%[val], %[UNLOCKED], 4f	\n"	/* !UNLOCKED, bail */
174	"	mov	%[val], %[WR_LOCKED]	\n"
175	"	scond	%[val], [%[rwlock]]	\n"
176	"	bnz	1b			\n"	/* retry if collided with someone */
177	"	mov	%[got_it], 1		\n"
178	"					\n"
179	"4: ; --- done ---			\n"
180
181	: [val]		"=&r"	(val),
182	  [got_it]	"+&r"	(got_it)
183	: [rwlock]	"r"	(&(rw->counter)),
184	  [UNLOCKED]	"ir"	(__ARCH_RW_LOCK_UNLOCKED__),
185	  [WR_LOCKED]	"ir"	(0)
186	: "memory", "cc");
187
188	smp_mb();
189
190	return got_it;
191}
192
193static inline void arch_read_unlock(arch_rwlock_t *rw)
194{
195	unsigned int val;
196
197	smp_mb();
198
199	/*
200	 * rw->counter++;
201	 */
202	__asm__ __volatile__(
203	"1:	llock	%[val], [%[rwlock]]	\n"
204	"	add	%[val], %[val], 1	\n"
205	"	scond	%[val], [%[rwlock]]	\n"
206	"	bnz	1b			\n"
207	"					\n"
208	: [val]		"=&r"	(val)
209	: [rwlock]	"r"	(&(rw->counter))
210	: "memory", "cc");
211}
212
213static inline void arch_write_unlock(arch_rwlock_t *rw)
214{
215	smp_mb();
216
217	WRITE_ONCE(rw->counter, __ARCH_RW_LOCK_UNLOCKED__);
218}
219
220#else	/* !CONFIG_ARC_HAS_LLSC */
221
222static inline void arch_spin_lock(arch_spinlock_t *lock)
223{
224	unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
225
226	/*
227	 * Per lkmm, smp_mb() is only required after _lock (and before_unlock)
228	 * for ACQ and REL semantics respectively. However EX based spinlocks
229	 * need the extra smp_mb to workaround a hardware quirk.
230	 */
231	smp_mb();
232
233	__asm__ __volatile__(
234	"1:	ex  %0, [%1]		\n"
235	"	breq  %0, %2, 1b	\n"
236	: "+&r" (val)
237	: "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
238	: "memory");
239
240	smp_mb();
241}
242
243/* 1 - lock taken successfully */
244static inline int arch_spin_trylock(arch_spinlock_t *lock)
245{
246	unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
247
248	smp_mb();
249
250	__asm__ __volatile__(
251	"1:	ex  %0, [%1]		\n"
252	: "+r" (val)
253	: "r"(&(lock->slock))
254	: "memory");
255
256	smp_mb();
257
258	return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
259}
260
261static inline void arch_spin_unlock(arch_spinlock_t *lock)
262{
263	unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
264
265	/*
266	 * RELEASE barrier: given the instructions avail on ARCv2, full barrier
267	 * is the only option
268	 */
269	smp_mb();
270
271	/*
272	 * EX is not really required here, a simple STore of 0 suffices.
273	 * However this causes tasklist livelocks in SystemC based SMP virtual
274	 * platforms where the systemc core scheduler uses EX as a cue for
275	 * moving to next core. Do a git log of this file for details
276	 */
277	__asm__ __volatile__(
278	"	ex  %0, [%1]		\n"
279	: "+r" (val)
280	: "r"(&(lock->slock))
281	: "memory");
282
283	/*
284	 * see pairing version/comment in arch_spin_lock above
285	 */
286	smp_mb();
287}
288
289/*
290 * Read-write spinlocks, allowing multiple readers but only one writer.
291 * Unfair locking as Writers could be starved indefinitely by Reader(s)
292 *
293 * The spinlock itself is contained in @counter and access to it is
294 * serialized with @lock_mutex.
295 */
296
297/* 1 - lock taken successfully */
298static inline int arch_read_trylock(arch_rwlock_t *rw)
299{
300	int ret = 0;
301	unsigned long flags;
302
303	local_irq_save(flags);
304	arch_spin_lock(&(rw->lock_mutex));
305
306	/*
307	 * zero means writer holds the lock exclusively, deny Reader.
308	 * Otherwise grant lock to first/subseq reader
309	 */
310	if (rw->counter > 0) {
311		rw->counter--;
312		ret = 1;
313	}
314
315	arch_spin_unlock(&(rw->lock_mutex));
316	local_irq_restore(flags);
317
318	return ret;
319}
320
321/* 1 - lock taken successfully */
322static inline int arch_write_trylock(arch_rwlock_t *rw)
323{
324	int ret = 0;
325	unsigned long flags;
326
327	local_irq_save(flags);
328	arch_spin_lock(&(rw->lock_mutex));
329
330	/*
331	 * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
332	 * deny writer. Otherwise if unlocked grant to writer
333	 * Hence the claim that Linux rwlocks are unfair to writers.
334	 * (can be starved for an indefinite time by readers).
335	 */
336	if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
337		rw->counter = 0;
338		ret = 1;
339	}
340	arch_spin_unlock(&(rw->lock_mutex));
341	local_irq_restore(flags);
342
343	return ret;
344}
345
346static inline void arch_read_lock(arch_rwlock_t *rw)
347{
348	while (!arch_read_trylock(rw))
349		cpu_relax();
350}
351
352static inline void arch_write_lock(arch_rwlock_t *rw)
353{
354	while (!arch_write_trylock(rw))
355		cpu_relax();
356}
357
358static inline void arch_read_unlock(arch_rwlock_t *rw)
359{
360	unsigned long flags;
361
362	local_irq_save(flags);
363	arch_spin_lock(&(rw->lock_mutex));
364	rw->counter++;
365	arch_spin_unlock(&(rw->lock_mutex));
366	local_irq_restore(flags);
367}
368
369static inline void arch_write_unlock(arch_rwlock_t *rw)
370{
371	unsigned long flags;
372
373	local_irq_save(flags);
374	arch_spin_lock(&(rw->lock_mutex));
375	rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
376	arch_spin_unlock(&(rw->lock_mutex));
377	local_irq_restore(flags);
378}
379
380#endif
381
382#endif /* __ASM_SPINLOCK_H */
383