opensolaris_atomic.S revision 168482 
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27	.ident	"%Z%%M%	%I%	%E% SMI"
28
29	.file	"%M%"
30
31#define	_ASM
32#include <sys/asm_linkage.h>
33
34#if defined(_KERNEL)
35	/*
36	 * Legacy kernel interfaces; they will go away (eventually).
37	 */
38	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
39	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
40	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
41	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
42	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
43	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
44	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
45#endif
46
47	ENTRY(atomic_inc_8)
48	ALTENTRY(atomic_inc_uchar)
49	movl	4(%esp), %eax
50	lock
51	incb	(%eax)
52	ret
53	SET_SIZE(atomic_inc_uchar)
54	SET_SIZE(atomic_inc_8)
55
56	ENTRY(atomic_inc_16)
57	ALTENTRY(atomic_inc_ushort)
58	movl	4(%esp), %eax
59	lock
60	incw	(%eax)
61	ret
62	SET_SIZE(atomic_inc_ushort)
63	SET_SIZE(atomic_inc_16)
64
65	ENTRY(atomic_inc_32)
66	ALTENTRY(atomic_inc_uint)
67	ALTENTRY(atomic_inc_ulong)
68	movl	4(%esp), %eax
69	lock
70	incl	(%eax)
71	ret
72	SET_SIZE(atomic_inc_ulong)
73	SET_SIZE(atomic_inc_uint)
74	SET_SIZE(atomic_inc_32)
75
76	ENTRY(atomic_inc_8_nv)
77	ALTENTRY(atomic_inc_uchar_nv)
78	movl	4(%esp), %edx
79	movb	(%edx), %al
801:
81	leal	1(%eax), %ecx
82	lock
83	cmpxchgb %cl, (%edx)
84	jne	1b
85	movzbl	%cl, %eax
86	ret
87	SET_SIZE(atomic_inc_uchar_nv)
88	SET_SIZE(atomic_inc_8_nv)
89
90	ENTRY(atomic_inc_16_nv)
91	ALTENTRY(atomic_inc_ushort_nv)
92	movl	4(%esp), %edx
93	movw	(%edx), %ax
941:
95	leal	1(%eax), %ecx
96	lock
97	cmpxchgw %cx, (%edx)
98	jne	1b
99	movzwl	%cx, %eax
100	ret
101	SET_SIZE(atomic_inc_ushort_nv)
102	SET_SIZE(atomic_inc_16_nv)
103
104	ENTRY(atomic_inc_32_nv)
105	ALTENTRY(atomic_inc_uint_nv)
106	ALTENTRY(atomic_inc_ulong_nv)
107	movl	4(%esp), %edx
108	movl	(%edx), %eax
1091:
110	leal	1(%eax), %ecx
111	lock
112	cmpxchgl %ecx, (%edx)
113	jne	1b
114	movl	%ecx, %eax
115	ret
116	SET_SIZE(atomic_inc_ulong_nv)
117	SET_SIZE(atomic_inc_uint_nv)
118	SET_SIZE(atomic_inc_32_nv)
119
120	ENTRY(atomic_inc_64)
121	ALTENTRY(atomic_inc_64_nv)
122	pushl	%edi
123	pushl	%ebx
124	movl	12(%esp), %edi
125	movl	(%edi), %eax
126	movl	4(%edi), %edx
1271:
128	xorl	%ebx, %ebx
129	xorl	%ecx, %ecx
130	incl	%ebx
131	addl	%eax, %ebx
132	adcl	%edx, %ecx
133	lock
134	cmpxchg8b (%edi)
135	jne	1b
136	movl	%ebx, %eax
137	movl	%ecx, %edx
138	popl	%ebx
139	popl	%edi
140	ret
141	SET_SIZE(atomic_inc_64_nv)
142	SET_SIZE(atomic_inc_64)
143
144	ENTRY(atomic_dec_8)
145	ALTENTRY(atomic_dec_uchar)
146	movl	4(%esp), %eax
147	lock
148	decb	(%eax)
149	ret
150	SET_SIZE(atomic_dec_uchar)
151	SET_SIZE(atomic_dec_8)
152
153	ENTRY(atomic_dec_16)
154	ALTENTRY(atomic_dec_ushort)
155	movl	4(%esp), %eax
156	lock
157	decw	(%eax)
158	ret
159	SET_SIZE(atomic_dec_ushort)
160	SET_SIZE(atomic_dec_16)
161
162	ENTRY(atomic_dec_32)
163	ALTENTRY(atomic_dec_uint)
164	ALTENTRY(atomic_dec_ulong)
165	movl	4(%esp), %eax
166	lock
167	decl	(%eax)
168	ret
169	SET_SIZE(atomic_dec_ulong)
170	SET_SIZE(atomic_dec_uint)
171	SET_SIZE(atomic_dec_32)
172
173	ENTRY(atomic_dec_8_nv)
174	ALTENTRY(atomic_dec_uchar_nv)
175	movl	4(%esp), %edx
176	movb	(%edx), %al
1771:
178	leal	-1(%eax), %ecx
179	lock
180	cmpxchgb %cl, (%edx)
181	jne	1b
182	movzbl	%cl, %eax
183	ret
184	SET_SIZE(atomic_dec_uchar_nv)
185	SET_SIZE(atomic_dec_8_nv)
186
187	ENTRY(atomic_dec_16_nv)
188	ALTENTRY(atomic_dec_ushort_nv)
189	movl	4(%esp), %edx
190	movw	(%edx), %ax
1911:
192	leal	-1(%eax), %ecx
193	lock
194	cmpxchgw %cx, (%edx)
195	jne	1b
196	movzwl	%cx, %eax
197	ret
198	SET_SIZE(atomic_dec_ushort_nv)
199	SET_SIZE(atomic_dec_16_nv)
200
201	ENTRY(atomic_dec_32_nv)
202	ALTENTRY(atomic_dec_uint_nv)
203	ALTENTRY(atomic_dec_ulong_nv)
204	movl	4(%esp), %edx
205	movl	(%edx), %eax
2061:
207	leal	-1(%eax), %ecx
208	lock
209	cmpxchgl %ecx, (%edx)
210	jne	1b
211	movl	%ecx, %eax
212	ret
213	SET_SIZE(atomic_dec_ulong_nv)
214	SET_SIZE(atomic_dec_uint_nv)
215	SET_SIZE(atomic_dec_32_nv)
216
217	ENTRY(atomic_dec_64)
218	ALTENTRY(atomic_dec_64_nv)
219	pushl	%edi
220	pushl	%ebx
221	movl	12(%esp), %edi
222	movl	(%edi), %eax
223	movl	4(%edi), %edx
2241:
225	xorl	%ebx, %ebx
226	xorl	%ecx, %ecx
227	not	%ecx
228	not	%ebx
229	addl	%eax, %ebx
230	adcl	%edx, %ecx
231	lock
232	cmpxchg8b (%edi)
233	jne	1b
234	movl	%ebx, %eax
235	movl	%ecx, %edx
236	popl	%ebx
237	popl	%edi
238	ret
239	SET_SIZE(atomic_dec_64_nv)
240	SET_SIZE(atomic_dec_64)
241
242	ENTRY(atomic_or_8)
243	ALTENTRY(atomic_or_uchar)
244	movl	4(%esp), %eax
245	movb	8(%esp), %cl
246	lock
247	orb	%cl, (%eax)
248	ret
249	SET_SIZE(atomic_or_uchar)
250	SET_SIZE(atomic_or_8)
251
252	ENTRY(atomic_or_16)
253	ALTENTRY(atomic_or_ushort)
254	movl	4(%esp), %eax
255	movw	8(%esp), %cx
256	lock
257	orw	%cx, (%eax)
258	ret
259	SET_SIZE(atomic_or_ushort)
260	SET_SIZE(atomic_or_16)
261
262	ENTRY(atomic_or_32)
263	ALTENTRY(atomic_or_uint)
264	ALTENTRY(atomic_or_ulong)
265	movl	4(%esp), %eax
266	movl	8(%esp), %ecx
267	lock
268	orl	%ecx, (%eax)
269	ret
270	SET_SIZE(atomic_or_ulong)
271	SET_SIZE(atomic_or_uint)
272	SET_SIZE(atomic_or_32)
273
274	ENTRY(atomic_and_8)
275	ALTENTRY(atomic_and_uchar)
276	movl	4(%esp), %eax
277	movb	8(%esp), %cl
278	lock
279	andb	%cl, (%eax)
280	ret
281	SET_SIZE(atomic_and_uchar)
282	SET_SIZE(atomic_and_8)
283
284	ENTRY(atomic_and_16)
285	ALTENTRY(atomic_and_ushort)
286	movl	4(%esp), %eax
287	movw	8(%esp), %cx
288	lock
289	andw	%cx, (%eax)
290	ret
291	SET_SIZE(atomic_and_ushort)
292	SET_SIZE(atomic_and_16)
293
294	ENTRY(atomic_and_32)
295	ALTENTRY(atomic_and_uint)
296	ALTENTRY(atomic_and_ulong)
297	movl	4(%esp), %eax
298	movl	8(%esp), %ecx
299	lock
300	andl	%ecx, (%eax)
301	ret
302	SET_SIZE(atomic_and_ulong)
303	SET_SIZE(atomic_and_uint)
304	SET_SIZE(atomic_and_32)
305
306	ENTRY(atomic_add_8_nv)
307	ALTENTRY(atomic_add_char_nv)
308	movl	4(%esp), %edx
309	movb	(%edx), %al
3101:
311	movl	8(%esp), %ecx
312	addb	%al, %cl
313	lock
314	cmpxchgb %cl, (%edx)
315	jne	1b
316	movzbl	%cl, %eax
317	ret
318	SET_SIZE(atomic_add_char_nv)
319	SET_SIZE(atomic_add_8_nv)
320
321	ENTRY(atomic_add_16_nv)
322	ALTENTRY(atomic_add_short_nv)
323	movl	4(%esp), %edx
324	movw	(%edx), %ax
3251:
326	movl	8(%esp), %ecx
327	addw	%ax, %cx
328	lock
329	cmpxchgw %cx, (%edx)
330	jne	1b
331	movzwl	%cx, %eax
332	ret
333	SET_SIZE(atomic_add_short_nv)
334	SET_SIZE(atomic_add_16_nv)
335
336	ENTRY(atomic_add_32_nv)
337	ALTENTRY(atomic_add_int_nv)
338	ALTENTRY(atomic_add_ptr_nv)
339	ALTENTRY(atomic_add_long_nv)
340	movl	4(%esp), %edx
341	movl	(%edx), %eax
3421:
343	movl	8(%esp), %ecx
344	addl	%eax, %ecx
345	lock
346	cmpxchgl %ecx, (%edx)
347	jne	1b
348	movl	%ecx, %eax
349	ret
350	SET_SIZE(atomic_add_long_nv)
351	SET_SIZE(atomic_add_ptr_nv)
352	SET_SIZE(atomic_add_int_nv)
353	SET_SIZE(atomic_add_32_nv)
354
355	ENTRY(atomic_add_64)
356	ALTENTRY(atomic_add_64_nv)
357	pushl	%edi
358	pushl	%ebx
359	movl	12(%esp), %edi
360	movl	(%edi), %eax
361	movl	4(%edi), %edx
3621:
363	movl	16(%esp), %ebx
364	movl	20(%esp), %ecx
365	addl	%eax, %ebx
366	adcl	%edx, %ecx
367	lock
368	cmpxchg8b (%edi)
369	jne	1b
370	movl	%ebx, %eax
371	movl	%ecx, %edx
372	popl	%ebx
373	popl	%edi
374	ret
375	SET_SIZE(atomic_add_64_nv)
376	SET_SIZE(atomic_add_64)
377
378	ENTRY(atomic_or_8_nv)
379	ALTENTRY(atomic_or_uchar_nv)
380	movl	4(%esp), %edx
381	movb	(%edx), %al
3821:
383	movl	8(%esp), %ecx
384	orb	%al, %cl
385	lock
386	cmpxchgb %cl, (%edx)
387	jne	1b
388	movzbl	%cl, %eax
389	ret
390	SET_SIZE(atomic_or_uchar_nv)
391	SET_SIZE(atomic_or_8_nv)
392
393	ENTRY(atomic_or_16_nv)
394	ALTENTRY(atomic_or_ushort_nv)
395	movl	4(%esp), %edx
396	movw	(%edx), %ax
3971:
398	movl	8(%esp), %ecx
399	orw	%ax, %cx
400	lock
401	cmpxchgw %cx, (%edx)
402	jne	1b
403	movzwl	%cx, %eax
404	ret
405	SET_SIZE(atomic_or_ushort_nv)
406	SET_SIZE(atomic_or_16_nv)
407
408	ENTRY(atomic_or_32_nv)
409	ALTENTRY(atomic_or_uint_nv)
410	ALTENTRY(atomic_or_ulong_nv)
411	movl	4(%esp), %edx
412	movl	(%edx), %eax
4131:
414	movl	8(%esp), %ecx
415	orl	%eax, %ecx
416	lock
417	cmpxchgl %ecx, (%edx)
418	jne	1b
419	movl	%ecx, %eax
420	ret
421	SET_SIZE(atomic_or_ulong_nv)
422	SET_SIZE(atomic_or_uint_nv)
423	SET_SIZE(atomic_or_32_nv)
424
425	ENTRY(atomic_or_64)
426	ALTENTRY(atomic_or_64_nv)
427	pushl	%edi
428	pushl	%ebx
429	movl	12(%esp), %edi
430	movl	(%edi), %eax
431	movl	4(%edi), %edx
4321:
433	movl	16(%esp), %ebx
434	movl	20(%esp), %ecx
435	orl	%eax, %ebx
436	orl	%edx, %ecx
437	lock
438	cmpxchg8b (%edi)
439	jne	1b
440	movl	%ebx, %eax
441	movl	%ecx, %edx
442	popl	%ebx
443	popl	%edi
444	ret
445	SET_SIZE(atomic_or_64_nv)
446	SET_SIZE(atomic_or_64)
447
448	ENTRY(atomic_and_8_nv)
449	ALTENTRY(atomic_and_uchar_nv)
450	movl	4(%esp), %edx
451	movb	(%edx), %al
4521:
453	movl	8(%esp), %ecx
454	andb	%al, %cl
455	lock
456	cmpxchgb %cl, (%edx)
457	jne	1b
458	movzbl	%cl, %eax
459	ret
460	SET_SIZE(atomic_and_uchar_nv)
461	SET_SIZE(atomic_and_8_nv)
462
463	ENTRY(atomic_and_16_nv)
464	ALTENTRY(atomic_and_ushort_nv)
465	movl	4(%esp), %edx
466	movw	(%edx), %ax
4671:
468	movl	8(%esp), %ecx
469	andw	%ax, %cx
470	lock
471	cmpxchgw %cx, (%edx)
472	jne	1b
473	movzwl	%cx, %eax
474	ret
475	SET_SIZE(atomic_and_ushort_nv)
476	SET_SIZE(atomic_and_16_nv)
477
478	ENTRY(atomic_and_32_nv)
479	ALTENTRY(atomic_and_uint_nv)
480	ALTENTRY(atomic_and_ulong_nv)
481	movl	4(%esp), %edx
482	movl	(%edx), %eax
4831:
484	movl	8(%esp), %ecx
485	andl	%eax, %ecx
486	lock
487	cmpxchgl %ecx, (%edx)
488	jne	1b
489	movl	%ecx, %eax
490	ret
491	SET_SIZE(atomic_and_ulong_nv)
492	SET_SIZE(atomic_and_uint_nv)
493	SET_SIZE(atomic_and_32_nv)
494
495	ENTRY(atomic_and_64)
496	ALTENTRY(atomic_and_64_nv)
497	pushl	%edi
498	pushl	%ebx
499	movl	12(%esp), %edi
500	movl	(%edi), %eax
501	movl	4(%edi), %edx
5021:
503	movl	16(%esp), %ebx
504	movl	20(%esp), %ecx
505	andl	%eax, %ebx
506	andl	%edx, %ecx
507	lock
508	cmpxchg8b (%edi)
509	jne	1b
510	movl	%ebx, %eax
511	movl	%ecx, %edx
512	popl	%ebx
513	popl	%edi
514	ret
515	SET_SIZE(atomic_and_64_nv)
516	SET_SIZE(atomic_and_64)
517
518	ENTRY(atomic_cas_8)
519	ALTENTRY(atomic_cas_uchar)
520	movl	4(%esp), %edx
521	movzbl	8(%esp), %eax
522	movb	12(%esp), %cl
523	lock
524	cmpxchgb %cl, (%edx)
525	ret
526	SET_SIZE(atomic_cas_uchar)
527	SET_SIZE(atomic_cas_8)
528
529	ENTRY(atomic_cas_16)
530	ALTENTRY(atomic_cas_ushort)
531	movl	4(%esp), %edx
532	movzwl	8(%esp), %eax
533	movw	12(%esp), %cx
534	lock
535	cmpxchgw %cx, (%edx)
536	ret
537	SET_SIZE(atomic_cas_ushort)
538	SET_SIZE(atomic_cas_16)
539
540	ENTRY(atomic_cas_32)
541	ALTENTRY(atomic_cas_uint)
542	ALTENTRY(atomic_cas_ulong)
543	ALTENTRY(atomic_cas_ptr)
544	movl	4(%esp), %edx
545	movl	8(%esp), %eax
546	movl	12(%esp), %ecx
547	lock
548	cmpxchgl %ecx, (%edx)
549	ret
550	SET_SIZE(atomic_cas_ptr)
551	SET_SIZE(atomic_cas_ulong)
552	SET_SIZE(atomic_cas_uint)
553	SET_SIZE(atomic_cas_32)
554
555	ENTRY(atomic_cas_64)
556	pushl	%ebx
557	pushl	%esi
558	movl	12(%esp), %esi
559	movl	16(%esp), %eax
560	movl	20(%esp), %edx
561	movl	24(%esp), %ebx
562	movl	28(%esp), %ecx
563	lock
564	cmpxchg8b (%esi)
565	popl	%esi
566	popl	%ebx
567	ret
568	SET_SIZE(atomic_cas_64)
569
570	ENTRY(atomic_swap_8)
571	ALTENTRY(atomic_swap_uchar)
572	movl	4(%esp), %edx
573	movzbl	8(%esp), %eax
574	lock
575	xchgb	%al, (%edx)
576	ret
577	SET_SIZE(atomic_swap_uchar)
578	SET_SIZE(atomic_swap_8)
579
580	ENTRY(atomic_swap_16)
581	ALTENTRY(atomic_swap_ushort)
582	movl	4(%esp), %edx
583	movzwl	8(%esp), %eax
584	lock
585	xchgw	%ax, (%edx)
586	ret
587	SET_SIZE(atomic_swap_ushort)
588	SET_SIZE(atomic_swap_16)
589
590	ENTRY(atomic_swap_32)
591	ALTENTRY(atomic_swap_uint)
592	ALTENTRY(atomic_swap_ptr)
593	ALTENTRY(atomic_swap_ulong)
594	movl	4(%esp), %edx
595	movl	8(%esp), %eax
596	lock
597	xchgl	%eax, (%edx)
598	ret
599	SET_SIZE(atomic_swap_ulong)
600	SET_SIZE(atomic_swap_ptr)
601	SET_SIZE(atomic_swap_uint)
602	SET_SIZE(atomic_swap_32)
603
604	ENTRY(atomic_swap_64)
605	pushl	%esi
606	pushl	%ebx
607	movl	12(%esp), %esi
608	movl	16(%esp), %ebx
609	movl	20(%esp), %ecx
610	movl	(%esi), %eax
611	movl	4(%esi), %edx
6121:
613	lock
614	cmpxchg8b (%esi)
615	jne	1b
616	popl	%ebx
617	popl	%esi
618	ret
619	SET_SIZE(atomic_swap_64)
620
621	ENTRY(atomic_set_long_excl)
622	movl	4(%esp), %edx
623	movl	8(%esp), %ecx
624	xorl	%eax, %eax
625	lock
626	btsl	%ecx, (%edx)
627	jnc	1f
628	decl	%eax
6291:
630	ret
631	SET_SIZE(atomic_set_long_excl)
632
633	ENTRY(atomic_clear_long_excl)
634	movl	4(%esp), %edx
635	movl	8(%esp), %ecx
636	xorl	%eax, %eax
637	lock
638	btrl	%ecx, (%edx)
639	jc	1f
640	decl	%eax
6411:
642	ret
643	SET_SIZE(atomic_clear_long_excl)
644
645	ENTRY(membar_enter)
646	ALTENTRY(membar_exit)
647	ALTENTRY(membar_producer)
648	ALTENTRY(membar_consumer)
649	lock
650	xorl	$0, (%esp)
651	ret
652	SET_SIZE(membar_consumer)
653	SET_SIZE(membar_producer)
654	SET_SIZE(membar_exit)
655	SET_SIZE(membar_enter)
656