1// SPDX-License-Identifier: GPL-2.0
2/*
3 * atomic32.c: 32-bit atomic_t implementation
4 *
5 * Copyright (C) 2004 Keith M Wesolowski
6 * Copyright (C) 2007 Kyle McMartin
7 *
8 * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf
9 */
10
11#include <linux/atomic.h>
12#include <linux/spinlock.h>
13#include <linux/module.h>
14
15#ifdef CONFIG_SMP
16#define ATOMIC_HASH_SIZE	4
17#define ATOMIC_HASH(a)	(&__atomic_hash[(((unsigned long)a)>>8) & (ATOMIC_HASH_SIZE-1)])
18
19spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] = {
20	[0 ... (ATOMIC_HASH_SIZE-1)] = __SPIN_LOCK_UNLOCKED(__atomic_hash)
21};
22
23#else /* SMP */
24
25static DEFINE_SPINLOCK(dummy);
26#define ATOMIC_HASH_SIZE	1
27#define ATOMIC_HASH(a)		(&dummy)
28
29#endif /* SMP */
30
31#define ATOMIC_FETCH_OP(op, c_op)					\
32int arch_atomic_fetch_##op(int i, atomic_t *v)				\
33{									\
34	int ret;							\
35	unsigned long flags;						\
36	spin_lock_irqsave(ATOMIC_HASH(v), flags);			\
37									\
38	ret = v->counter;						\
39	v->counter c_op i;						\
40									\
41	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);			\
42	return ret;							\
43}									\
44EXPORT_SYMBOL(arch_atomic_fetch_##op);
45
46#define ATOMIC_OP_RETURN(op, c_op)					\
47int arch_atomic_##op##_return(int i, atomic_t *v)			\
48{									\
49	int ret;							\
50	unsigned long flags;						\
51	spin_lock_irqsave(ATOMIC_HASH(v), flags);			\
52									\
53	ret = (v->counter c_op i);					\
54									\
55	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);			\
56	return ret;							\
57}									\
58EXPORT_SYMBOL(arch_atomic_##op##_return);
59
60ATOMIC_OP_RETURN(add, +=)
61
62ATOMIC_FETCH_OP(add, +=)
63ATOMIC_FETCH_OP(and, &=)
64ATOMIC_FETCH_OP(or, |=)
65ATOMIC_FETCH_OP(xor, ^=)
66
67#undef ATOMIC_FETCH_OP
68#undef ATOMIC_OP_RETURN
69
70int arch_atomic_xchg(atomic_t *v, int new)
71{
72	int ret;
73	unsigned long flags;
74
75	spin_lock_irqsave(ATOMIC_HASH(v), flags);
76	ret = v->counter;
77	v->counter = new;
78	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
79	return ret;
80}
81EXPORT_SYMBOL(arch_atomic_xchg);
82
83int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
84{
85	int ret;
86	unsigned long flags;
87
88	spin_lock_irqsave(ATOMIC_HASH(v), flags);
89	ret = v->counter;
90	if (likely(ret == old))
91		v->counter = new;
92
93	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
94	return ret;
95}
96EXPORT_SYMBOL(arch_atomic_cmpxchg);
97
98int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
99{
100	int ret;
101	unsigned long flags;
102
103	spin_lock_irqsave(ATOMIC_HASH(v), flags);
104	ret = v->counter;
105	if (ret != u)
106		v->counter += a;
107	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
108	return ret;
109}
110EXPORT_SYMBOL(arch_atomic_fetch_add_unless);
111
112/* Atomic operations are already serializing */
113void arch_atomic_set(atomic_t *v, int i)
114{
115	unsigned long flags;
116
117	spin_lock_irqsave(ATOMIC_HASH(v), flags);
118	v->counter = i;
119	spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
120}
121EXPORT_SYMBOL(arch_atomic_set);
122
123unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask)
124{
125	unsigned long old, flags;
126
127	spin_lock_irqsave(ATOMIC_HASH(addr), flags);
128	old = *addr;
129	*addr = old | mask;
130	spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
131
132	return old & mask;
133}
134EXPORT_SYMBOL(sp32___set_bit);
135
136unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask)
137{
138	unsigned long old, flags;
139
140	spin_lock_irqsave(ATOMIC_HASH(addr), flags);
141	old = *addr;
142	*addr = old & ~mask;
143	spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
144
145	return old & mask;
146}
147EXPORT_SYMBOL(sp32___clear_bit);
148
149unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask)
150{
151	unsigned long old, flags;
152
153	spin_lock_irqsave(ATOMIC_HASH(addr), flags);
154	old = *addr;
155	*addr = old ^ mask;
156	spin_unlock_irqrestore(ATOMIC_HASH(addr), flags);
157
158	return old & mask;
159}
160EXPORT_SYMBOL(sp32___change_bit);
161
162unsigned long __cmpxchg_u32(volatile u32 *ptr, u32 old, u32 new)
163{
164	unsigned long flags;
165	u32 prev;
166
167	spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
168	if ((prev = *ptr) == old)
169		*ptr = new;
170	spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
171
172	return (unsigned long)prev;
173}
174EXPORT_SYMBOL(__cmpxchg_u32);
175
176u64 __cmpxchg_u64(u64 *ptr, u64 old, u64 new)
177{
178	unsigned long flags;
179	u64 prev;
180
181	spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
182	if ((prev = *ptr) == old)
183		*ptr = new;
184	spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
185
186	return prev;
187}
188EXPORT_SYMBOL(__cmpxchg_u64);
189
190unsigned long __xchg_u32(volatile u32 *ptr, u32 new)
191{
192	unsigned long flags;
193	u32 prev;
194
195	spin_lock_irqsave(ATOMIC_HASH(ptr), flags);
196	prev = *ptr;
197	*ptr = new;
198	spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);
199
200	return (unsigned long)prev;
201}
202EXPORT_SYMBOL(__xchg_u32);
203