1/*
2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <libkern/OSAtomic.h>
30
31enum {
32	false	= 0,
33	true	= 1
34};
35
36#ifndef NULL
37#define NULL ((void *)0)
38#endif
39
40/*
41 * atomic operations
42 *	these are _the_ atomic operations, currently cast atop CompareAndSwap,
43 *	which is implemented in assembler.	if we are worried about the cost of
44 *	this layering (we shouldn't be), then all this stuff could be
45 *	implemented in assembler, as it is in MacOS8/9
46 *	(derived from SuperMario/NativeLibs/IO/DriverServices/Synchronization.s,
47 *	which I wrote for NuKernel in a previous life with a different last name...)
48 *
49 * native Boolean	CompareAndSwap(UInt32 oldValue, UInt32 newValue, UInt32 * oldValuePtr);
50 *
51 * We've since implemented a few more of these -- OSAddAtomic, OSDequeueAtomic,
52 * OSEnqueueAtomic etc -- in assembler, either for speed or correctness.  See also the
53 * commpage atomic operations, and the platform specific versions.
54 * Like standards, there are a lot of atomic ops to choose from!
55 */
56
57#if defined(__i386__) || defined(__x86_64__)
58/* Implemented in assembly for i386 and x86_64 */
59#else
60#error Unsupported arch
61#endif
62
63#undef OSIncrementAtomic
64SInt32	OSIncrementAtomic(volatile SInt32 * value)
65{
66	return OSAddAtomic(1, value);
67}
68
69#undef OSDecrementAtomic
70SInt32	OSDecrementAtomic(volatile SInt32 * value)
71{
72	return OSAddAtomic(-1, value);
73}
74
75static UInt32	OSBitwiseAtomic(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt32 * value)
76{
77	UInt32	oldValue;
78	UInt32	newValue;
79
80	do {
81		oldValue = *value;
82		newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
83	} while (! OSCompareAndSwap(oldValue, newValue, value));
84
85	return oldValue;
86}
87
88#undef OSBitAndAtomic
89UInt32	OSBitAndAtomic(UInt32 mask, volatile UInt32 * value)
90{
91	return OSBitwiseAtomic(mask, 0, 0, value);
92}
93
94#undef OSBitOrAtomic
95UInt32	OSBitOrAtomic(UInt32 mask, volatile UInt32 * value)
96{
97	return OSBitwiseAtomic((UInt32) -1, mask, 0, value);
98}
99
100#undef OSBitXorAtomic
101UInt32	OSBitXorAtomic(UInt32 mask, volatile UInt32 * value)
102{
103	return OSBitwiseAtomic((UInt32) -1, 0, mask, value);
104}
105
106#if defined(__i386__) || defined(__x86_64__)
107static Boolean OSCompareAndSwap8(UInt8 oldValue8, UInt8 newValue8, volatile UInt8 * value8)
108{
109	UInt32				mask		= 0x000000ff;
110	UInt32				alignment	= (UInt32)((unsigned long) value8) & (sizeof(UInt32) - 1);
111	UInt32				shiftValues = (24 << 24) | (16 << 16) | (8 << 8);
112	int					shift		= (UInt32) *(((UInt8 *) &shiftValues) + alignment);
113	volatile UInt32 *	value32		= (volatile UInt32 *) ((uintptr_t)value8 - alignment);
114	UInt32				oldValue;
115	UInt32				newValue;
116
117	mask <<= shift;
118
119	oldValue = *value32;
120	oldValue = (oldValue & ~mask) | (oldValue8 << shift);
121	newValue = (oldValue & ~mask) | (newValue8 << shift);
122
123	return OSCompareAndSwap(oldValue, newValue, value32);
124}
125#endif
126
127static Boolean	OSTestAndSetClear(UInt32 bit, Boolean wantSet, volatile UInt8 * startAddress)
128{
129	UInt8		mask = 1;
130	UInt8		oldValue;
131	UInt8		wantValue;
132
133	startAddress += (bit / 8);
134	mask <<= (7 - (bit % 8));
135	wantValue = wantSet ? mask : 0;
136
137	do {
138		oldValue = *startAddress;
139		if ((oldValue & mask) == wantValue) {
140			break;
141		}
142	} while (! OSCompareAndSwap8(oldValue, (oldValue & ~mask) | wantValue, startAddress));
143
144	return (oldValue & mask) == wantValue;
145}
146
147Boolean OSTestAndSet(UInt32 bit, volatile UInt8 * startAddress)
148{
149	return OSTestAndSetClear(bit, true, startAddress);
150}
151
152Boolean OSTestAndClear(UInt32 bit, volatile UInt8 * startAddress)
153{
154	return OSTestAndSetClear(bit, false, startAddress);
155}
156
157/*
158 * silly unaligned versions
159 */
160
161SInt8	OSIncrementAtomic8(volatile SInt8 * value)
162{
163	return OSAddAtomic8(1, value);
164}
165
166SInt8	OSDecrementAtomic8(volatile SInt8 * value)
167{
168	return OSAddAtomic8(-1, value);
169}
170
171#if defined(__i386__) || defined(__x86_64__)
172SInt8	OSAddAtomic8(SInt32 amount, volatile SInt8 * value)
173{
174	SInt8	oldValue;
175	SInt8	newValue;
176
177	do {
178		oldValue = *value;
179		newValue = oldValue + amount;
180	} while (! OSCompareAndSwap8((UInt8) oldValue, (UInt8) newValue, (volatile UInt8 *) value));
181
182	return oldValue;
183}
184#endif
185
186static UInt8	OSBitwiseAtomic8(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt8 * value)
187{
188	UInt8	oldValue;
189	UInt8	newValue;
190
191	do {
192		oldValue = *value;
193		newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
194	} while (! OSCompareAndSwap8(oldValue, newValue, value));
195
196	return oldValue;
197}
198
199UInt8	OSBitAndAtomic8(UInt32 mask, volatile UInt8 * value)
200{
201	return OSBitwiseAtomic8(mask, 0, 0, value);
202}
203
204UInt8	OSBitOrAtomic8(UInt32 mask, volatile UInt8 * value)
205{
206	return OSBitwiseAtomic8((UInt32) -1, mask, 0, value);
207}
208
209UInt8	OSBitXorAtomic8(UInt32 mask, volatile UInt8 * value)
210{
211	return OSBitwiseAtomic8((UInt32) -1, 0, mask, value);
212}
213
214#if defined(__i386__) || defined(__x86_64__)
215static Boolean OSCompareAndSwap16(UInt16 oldValue16, UInt16 newValue16, volatile UInt16 * value16)
216{
217	UInt32				mask		= 0x0000ffff;
218	UInt32				alignment	= (UInt32)((unsigned long) value16) & (sizeof(UInt32) - 1);
219	UInt32				shiftValues = (16 << 24) | (16 << 16);
220	UInt32				shift		= (UInt32) *(((UInt8 *) &shiftValues) + alignment);
221	volatile UInt32 *	value32		= (volatile UInt32 *) (((unsigned long) value16) - alignment);
222	UInt32				oldValue;
223	UInt32				newValue;
224
225	mask <<= shift;
226
227	oldValue = *value32;
228	oldValue = (oldValue & ~mask) | (oldValue16 << shift);
229	newValue = (oldValue & ~mask) | (newValue16 << shift);
230
231	return OSCompareAndSwap(oldValue, newValue, value32);
232}
233#endif
234
235SInt16	OSIncrementAtomic16(volatile SInt16 * value)
236{
237	return OSAddAtomic16(1, value);
238}
239
240SInt16	OSDecrementAtomic16(volatile SInt16 * value)
241{
242	return OSAddAtomic16(-1, value);
243}
244
245#if defined(__i386__) || defined(__x86_64__)
246SInt16	OSAddAtomic16(SInt32 amount, volatile SInt16 * value)
247{
248	SInt16	oldValue;
249	SInt16	newValue;
250
251	do {
252		oldValue = *value;
253		newValue = oldValue + amount;
254	} while (! OSCompareAndSwap16((UInt16) oldValue, (UInt16) newValue, (volatile UInt16 *) value));
255
256	return oldValue;
257}
258#endif
259
260static UInt16	OSBitwiseAtomic16(UInt32 and_mask, UInt32 or_mask, UInt32 xor_mask, volatile UInt16 * value)
261{
262	UInt16	oldValue;
263	UInt16	newValue;
264
265	do {
266		oldValue = *value;
267		newValue = ((oldValue & and_mask) | or_mask) ^ xor_mask;
268	} while (! OSCompareAndSwap16(oldValue, newValue, value));
269
270	return oldValue;
271}
272
273UInt16	OSBitAndAtomic16(UInt32 mask, volatile UInt16 * value)
274{
275	return OSBitwiseAtomic16(mask, 0, 0, value);
276}
277
278UInt16	OSBitOrAtomic16(UInt32 mask, volatile UInt16 * value)
279{
280	return OSBitwiseAtomic16((UInt32) -1, mask, 0, value);
281}
282
283UInt16	OSBitXorAtomic16(UInt32 mask, volatile UInt16 * value)
284{
285	return OSBitwiseAtomic16((UInt32) -1, 0, mask, value);
286}
287
288