atomic.h revision 65514
138517Sdfr/*- 238517Sdfr * Copyright (c) 1998 Doug Rabson 338517Sdfr * All rights reserved. 438517Sdfr * 538517Sdfr * Redistribution and use in source and binary forms, with or without 638517Sdfr * modification, are permitted provided that the following conditions 738517Sdfr * are met: 838517Sdfr * 1. Redistributions of source code must retain the above copyright 938517Sdfr * notice, this list of conditions and the following disclaimer. 1038517Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138517Sdfr * notice, this list of conditions and the following disclaimer in the 1238517Sdfr * documentation and/or other materials provided with the distribution. 1338517Sdfr * 1438517Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538517Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638517Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738517Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838517Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938517Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038517Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138517Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238517Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338517Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438517Sdfr * SUCH DAMAGE. 2538517Sdfr * 2650477Speter * $FreeBSD: head/sys/i386/include/atomic.h 65514 2000-09-06 11:21:14Z phk $ 2738517Sdfr */ 2838517Sdfr#ifndef _MACHINE_ATOMIC_H_ 2938517Sdfr#define _MACHINE_ATOMIC_H_ 3038517Sdfr 3138517Sdfr/* 3238517Sdfr * Various simple arithmetic on memory which is atomic in the presence 3348797Salc * of interrupts and multiple processors. 3438517Sdfr * 3548797Salc * atomic_set_char(P, V) (*(u_char*)(P) |= (V)) 3648797Salc * atomic_clear_char(P, V) (*(u_char*)(P) &= ~(V)) 3748797Salc * atomic_add_char(P, V) (*(u_char*)(P) += (V)) 3848797Salc * atomic_subtract_char(P, V) (*(u_char*)(P) -= (V)) 3948797Salc * 4048797Salc * atomic_set_short(P, V) (*(u_short*)(P) |= (V)) 4148797Salc * atomic_clear_short(P, V) (*(u_short*)(P) &= ~(V)) 4248797Salc * atomic_add_short(P, V) (*(u_short*)(P) += (V)) 4348797Salc * atomic_subtract_short(P, V) (*(u_short*)(P) -= (V)) 4448797Salc * 4548797Salc * atomic_set_int(P, V) (*(u_int*)(P) |= (V)) 4648797Salc * atomic_clear_int(P, V) (*(u_int*)(P) &= ~(V)) 4748797Salc * atomic_add_int(P, V) (*(u_int*)(P) += (V)) 4848797Salc * atomic_subtract_int(P, V) (*(u_int*)(P) -= (V)) 4948797Salc * 5048797Salc * atomic_set_long(P, V) (*(u_long*)(P) |= (V)) 5148797Salc * atomic_clear_long(P, V) (*(u_long*)(P) &= ~(V)) 5248797Salc * atomic_add_long(P, V) (*(u_long*)(P) += (V)) 5348797Salc * atomic_subtract_long(P, V) (*(u_long*)(P) -= (V)) 5438517Sdfr */ 5538517Sdfr 5648797Salc/* 5749999Salc * The above functions are expanded inline in the statically-linked 5849999Salc * kernel. Lock prefixes are generated if an SMP kernel is being 5949999Salc * built. 6049999Salc * 6149999Salc * Kernel modules call real functions which are built into the kernel. 6249999Salc * This allows kernel modules to be portable between UP and SMP systems. 6348797Salc */ 6449999Salc#if defined(KLD_MODULE) 6549999Salc#define ATOMIC_ASM(NAME, TYPE, OP, V) \ 6649999Salc extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); 6749999Salc 6865514Sphkextern int atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); 6965514Sphk 7049999Salc#else /* !KLD_MODULE */ 7149999Salc#if defined(SMP) 7265514Sphk#if defined(LOCORE) 7365514Sphk#define MPLOCKED lock ; 7465514Sphk#else 7548797Salc#define MPLOCKED "lock ; " 7665514Sphk#endif 7748796Salc#else 7848797Salc#define MPLOCKED 7948796Salc#endif 8038517Sdfr 8148797Salc/* 8248797Salc * The assembly is volatilized to demark potential before-and-after side 8348797Salc * effects if an interrupt or SMP collision were to occur. 8448797Salc */ 8551938Speter#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) 8651938Speter/* egcs 1.1.2+ version */ 8748797Salc#define ATOMIC_ASM(NAME, TYPE, OP, V) \ 8848797Salcstatic __inline void \ 8949043Salcatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 9048797Salc{ \ 9148797Salc __asm __volatile(MPLOCKED OP \ 9249043Salc : "=m" (*p) \ 9349043Salc : "0" (*p), "ir" (V)); \ 9448797Salc} 9551938Speter 9665514Sphk/* 9765514Sphk * Atomic compare and set, used by the mutex functions 9865514Sphk * 9965514Sphk * if (*dst == exp) *dst = src (all 32 bit words) 10065514Sphk * 10165514Sphk * Returns 0 on failure, non-zero on success 10265514Sphk */ 10365514Sphk 10465514Sphk#if defined(I386_CPU) 10565514Sphkstatic __inline int 10665514Sphkatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 10765514Sphk{ 10865514Sphk int res = exp; 10965514Sphk 11065514Sphk __asm __volatile( 11165514Sphk " pushfl ; " 11265514Sphk " cli ; " 11365514Sphk " cmpl %1,%3 ; " 11465514Sphk " jne 1f ; " 11565514Sphk " movl %2,%3 ; " 11665514Sphk "1: " 11765514Sphk " sete %%al; " 11865514Sphk " movzbl %%al,%0 ; " 11965514Sphk " popfl ; " 12065514Sphk "# atomic_cmpset_int" 12165514Sphk : "=a" (res) /* 0 (result) */ 12265514Sphk : "0" (exp), /* 1 */ 12365514Sphk "r" (src), /* 2 */ 12465514Sphk "m" (*(dst)) /* 3 */ 12565514Sphk : "memory"); 12665514Sphk 12765514Sphk return (res); 12865514Sphk} 12965514Sphk#else /* defined(I386_CPU) */ 13065514Sphkstatic __inline int 13165514Sphkatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 13265514Sphk{ 13365514Sphk int res = exp; 13465514Sphk 13565514Sphk __asm __volatile ( 13665514Sphk " " MPLOCKED " " 13765514Sphk " cmpxchgl %2,%3 ; " 13865514Sphk " setz %%al ; " 13965514Sphk " movzbl %%al,%0 ; " 14065514Sphk "1: " 14165514Sphk "# atomic_cmpset_int" 14265514Sphk : "=a" (res) /* 0 (result) */ 14365514Sphk : "0" (exp), /* 1 */ 14465514Sphk "r" (src), /* 2 */ 14565514Sphk "m" (*(dst)) /* 3 */ 14665514Sphk : "memory"); 14765514Sphk 14865514Sphk return (res); 14965514Sphk} 15065514Sphk#endif /* defined(I386_CPU) */ 15165514Sphk 15251938Speter#else 15351938Speter/* gcc <= 2.8 version */ 15451938Speter#define ATOMIC_ASM(NAME, TYPE, OP, V) \ 15551938Speterstatic __inline void \ 15651938Speteratomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 15751938Speter{ \ 15851938Speter __asm __volatile(MPLOCKED OP \ 15951938Speter : "=m" (*p) \ 16051938Speter : "ir" (V)); \ 16151938Speter} 16251938Speter#endif 16349999Salc#endif /* KLD_MODULE */ 16438517Sdfr 16551937Speter#if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 9) 16651938Speter 16751938Speter/* egcs 1.1.2+ version */ 16860300SobrienATOMIC_ASM(set, char, "orb %b2,%0", v) 16960300SobrienATOMIC_ASM(clear, char, "andb %b2,%0", ~v) 17060300SobrienATOMIC_ASM(add, char, "addb %b2,%0", v) 17160300SobrienATOMIC_ASM(subtract, char, "subb %b2,%0", v) 17238517Sdfr 17360300SobrienATOMIC_ASM(set, short, "orw %w2,%0", v) 17460300SobrienATOMIC_ASM(clear, short, "andw %w2,%0", ~v) 17560300SobrienATOMIC_ASM(add, short, "addw %w2,%0", v) 17660300SobrienATOMIC_ASM(subtract, short, "subw %w2,%0", v) 17738517Sdfr 17848797SalcATOMIC_ASM(set, int, "orl %2,%0", v) 17948797SalcATOMIC_ASM(clear, int, "andl %2,%0", ~v) 18048797SalcATOMIC_ASM(add, int, "addl %2,%0", v) 18148797SalcATOMIC_ASM(subtract, int, "subl %2,%0", v) 18248796Salc 18348797SalcATOMIC_ASM(set, long, "orl %2,%0", v) 18448797SalcATOMIC_ASM(clear, long, "andl %2,%0", ~v) 18548797SalcATOMIC_ASM(add, long, "addl %2,%0", v) 18648797SalcATOMIC_ASM(subtract, long, "subl %2,%0", v) 18748796Salc 18851917Seivind#else 18951917Seivind 19051938Speter/* gcc <= 2.8 version */ 19151938SpeterATOMIC_ASM(set, char, "orb %1,%0", v) 19251938SpeterATOMIC_ASM(clear, char, "andb %1,%0", ~v) 19351938SpeterATOMIC_ASM(add, char, "addb %1,%0", v) 19451938SpeterATOMIC_ASM(subtract, char, "subb %1,%0", v) 19551917Seivind 19651938SpeterATOMIC_ASM(set, short, "orw %1,%0", v) 19751938SpeterATOMIC_ASM(clear, short, "andw %1,%0", ~v) 19851938SpeterATOMIC_ASM(add, short, "addw %1,%0", v) 19951938SpeterATOMIC_ASM(subtract, short, "subw %1,%0", v) 20051917Seivind 20151938SpeterATOMIC_ASM(set, int, "orl %1,%0", v) 20251938SpeterATOMIC_ASM(clear, int, "andl %1,%0", ~v) 20351938SpeterATOMIC_ASM(add, int, "addl %1,%0", v) 20451938SpeterATOMIC_ASM(subtract, int, "subl %1,%0", v) 20551938Speter 20651938SpeterATOMIC_ASM(set, long, "orl %1,%0", v) 20751938SpeterATOMIC_ASM(clear, long, "andl %1,%0", ~v) 20851938SpeterATOMIC_ASM(add, long, "addl %1,%0", v) 20951938SpeterATOMIC_ASM(subtract, long, "subl %1,%0", v) 21051938Speter 21151917Seivind#endif 21251917Seivind 21365514Sphk#ifndef WANT_FUNCTIONS 21465514Sphkstatic __inline int 21565514Sphkatomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 21665514Sphk{ 21765514Sphk 21865514Sphk return ( 21965514Sphk atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, (u_int)src)); 22065514Sphk} 22165514Sphk#endif 22265514Sphk 22338517Sdfr#endif /* ! _MACHINE_ATOMIC_H_ */ 224