1/*
2 * Copyright 2002-2005, Axel D��rfler, axeld@pinc-software.de.
3 * Distributed under the terms of the MIT License.
4 *
5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
6 * Distributed under the terms of the NewOS License.
7 */
8#ifndef KERNEL_SMP_H
9#define KERNEL_SMP_H
10
11
12#include <arch/atomic.h>
13#include <boot/kernel_args.h>
14#include <kernel.h>
15
16#include <KernelExport.h>
17
18#include <string.h>
19
20
21struct kernel_args;
22
23
24// intercpu messages
25enum {
26	SMP_MSG_INVALIDATE_PAGE_RANGE = 0,
27	SMP_MSG_INVALIDATE_PAGE_LIST,
28	SMP_MSG_USER_INVALIDATE_PAGES,
29	SMP_MSG_GLOBAL_INVALIDATE_PAGES,
30	SMP_MSG_CPU_HALT,
31	SMP_MSG_CALL_FUNCTION,
32	SMP_MSG_RESCHEDULE
33};
34
35enum {
36	SMP_MSG_FLAG_ASYNC		= 0x0,
37	SMP_MSG_FLAG_SYNC		= 0x1,
38	SMP_MSG_FLAG_FREE_ARG	= 0x2,
39};
40
41typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
42
43class CPUSet {
44public:
45	inline				CPUSet();
46
47	inline	void		ClearAll();
48	inline	void		SetAll();
49
50	inline	void		SetBit(int32 cpu);
51	inline	void		ClearBit(int32 cpu);
52
53	inline	void		SetBitAtomic(int32 cpu);
54	inline	void		ClearBitAtomic(int32 cpu);
55
56	inline	bool		GetBit(int32 cpu) const;
57
58	inline	bool		IsEmpty() const;
59
60private:
61	static	const int	kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
62
63			uint32		fBitmap[kArraySize];
64};
65
66
67#ifdef __cplusplus
68extern "C" {
69#endif
70
71bool try_acquire_spinlock(spinlock* lock);
72
73status_t smp_init(struct kernel_args *args);
74status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
75status_t smp_init_post_generic_syscalls(void);
76bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
77void smp_wake_up_non_boot_cpus(void);
78void smp_cpu_rendezvous(uint32* var);
79void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
80		void *data_ptr, uint32 flags);
81void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
82		addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
83void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
84		void *data_ptr, uint32 flags);
85void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
86		addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
87
88int32 smp_get_num_cpus(void);
89void smp_set_num_cpus(int32 numCPUs);
90int32 smp_get_current_cpu(void);
91
92int smp_intercpu_int_handler(int32 cpu);
93
94void call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie);
95void call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int),
96	void* cookie);
97
98
99#ifdef __cplusplus
100}
101#endif
102
103
104inline
105CPUSet::CPUSet()
106{
107	memset(fBitmap, 0, sizeof(fBitmap));
108}
109
110
111inline void
112CPUSet::ClearAll()
113{
114	memset(fBitmap, 0, sizeof(fBitmap));
115}
116
117
118inline void
119CPUSet::SetAll()
120{
121	memset(fBitmap, ~uint8(0), sizeof(fBitmap));
122}
123
124
125inline void
126CPUSet::SetBit(int32 cpu)
127{
128	int32* element = (int32*)&fBitmap[cpu % kArraySize];
129	*element |= 1u << (cpu / kArraySize);
130}
131
132
133inline void
134CPUSet::ClearBit(int32 cpu)
135{
136	int32* element = (int32*)&fBitmap[cpu % kArraySize];
137	*element &= ~uint32(1u << (cpu / kArraySize));
138}
139
140
141inline void
142CPUSet::SetBitAtomic(int32 cpu)
143{
144	int32* element = (int32*)&fBitmap[cpu % kArraySize];
145	atomic_or(element, 1u << (cpu / kArraySize));
146}
147
148
149inline void
150CPUSet::ClearBitAtomic(int32 cpu)
151{
152	int32* element = (int32*)&fBitmap[cpu % kArraySize];
153	atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
154}
155
156
157inline bool
158CPUSet::GetBit(int32 cpu) const
159{
160	int32* element = (int32*)&fBitmap[cpu % kArraySize];
161	return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
162}
163
164
165inline bool
166CPUSet::IsEmpty() const
167{
168	for (int i = 0; i < kArraySize; i++) {
169		if (fBitmap[i] != 0)
170			return false;
171	}
172
173	return true;
174}
175
176
177// Unless spinlock debug features are enabled, try to inline
178// {acquire,release}_spinlock().
179#if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
180
181
182static inline bool
183try_acquire_spinlock_inline(spinlock* lock)
184{
185	return atomic_get_and_set(&lock->lock, 1) == 0;
186}
187
188
189static inline void
190acquire_spinlock_inline(spinlock* lock)
191{
192	if (try_acquire_spinlock_inline(lock))
193		return;
194	acquire_spinlock(lock);
195}
196
197
198static inline void
199release_spinlock_inline(spinlock* lock)
200{
201	atomic_set(&lock->lock, 0);
202}
203
204
205#define try_acquire_spinlock(lock)	try_acquire_spinlock_inline(lock)
206#define acquire_spinlock(lock)		acquire_spinlock_inline(lock)
207#define release_spinlock(lock)		release_spinlock_inline(lock)
208
209
210static inline bool
211try_acquire_write_spinlock_inline(rw_spinlock* lock)
212{
213	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
214}
215
216
217static inline void
218acquire_write_spinlock_inline(rw_spinlock* lock)
219{
220	if (try_acquire_write_spinlock(lock))
221		return;
222	acquire_write_spinlock(lock);
223}
224
225
226static inline void
227release_write_spinlock_inline(rw_spinlock* lock)
228{
229	atomic_set(&lock->lock, 0);
230}
231
232
233static inline bool
234try_acquire_read_spinlock_inline(rw_spinlock* lock)
235{
236	uint32 previous = atomic_add(&lock->lock, 1);
237	return (previous & (1u << 31)) == 0;
238}
239
240
241static inline void
242acquire_read_spinlock_inline(rw_spinlock* lock)
243{
244	if (try_acquire_read_spinlock(lock))
245		return;
246	acquire_read_spinlock(lock);
247}
248
249
250static inline void
251release_read_spinlock_inline(rw_spinlock* lock)
252{
253	atomic_add(&lock->lock, -1);
254}
255
256
257#define try_acquire_read_spinlock(lock)	try_acquire_read_spinlock_inline(lock)
258#define acquire_read_spinlock(lock)		acquire_read_spinlock_inline(lock)
259#define release_read_spinlock(lock)		release_read_spinlock_inline(lock)
260#define try_acquire_write_spinlock(lock) \
261	try_acquire_write_spinlock(lock)
262#define acquire_write_spinlock(lock)	acquire_write_spinlock_inline(lock)
263#define release_write_spinlock(lock)	release_write_spinlock_inline(lock)
264
265
266static inline bool
267try_acquire_write_seqlock_inline(seqlock* lock) {
268	bool succeed = try_acquire_spinlock(&lock->lock);
269	if (succeed)
270		atomic_add((int32*)&lock->count, 1);
271	return succeed;
272}
273
274
275static inline void
276acquire_write_seqlock_inline(seqlock* lock) {
277	acquire_spinlock(&lock->lock);
278	atomic_add((int32*)&lock->count, 1);
279}
280
281
282static inline void
283release_write_seqlock_inline(seqlock* lock) {
284	atomic_add((int32*)&lock->count, 1);
285	release_spinlock(&lock->lock);
286}
287
288
289static inline uint32
290acquire_read_seqlock_inline(seqlock* lock)
291{
292	return (uint32)atomic_get((int32*)&lock->count);
293}
294
295
296static inline bool
297release_read_seqlock_inline(seqlock* lock, uint32 count)
298{
299	uint32 current = (uint32)atomic_get((int32*)&lock->count);
300
301	return count % 2 == 0 && current == count;
302}
303
304
305#define try_acquire_write_seqlock(lock)	try_acquire_write_seqlock_inline(lock)
306#define acquire_write_seqlock(lock)		acquire_write_seqlock_inline(lock)
307#define release_write_seqlock(lock)		release_write_seqlock_inline(lock)
308#define acquire_read_seqlock(lock)		acquire_read_seqlock_inline(lock)
309#define release_read_seqlock(lock, count)	\
310	release_read_seqlock_inline(lock, count)
311
312
313#endif	// !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
314
315
316#endif	/* KERNEL_SMP_H */
317