atomic.h revision 90515
138517Sdfr/*- 238517Sdfr * Copyright (c) 1998 Doug Rabson 338517Sdfr * All rights reserved. 438517Sdfr * 538517Sdfr * Redistribution and use in source and binary forms, with or without 638517Sdfr * modification, are permitted provided that the following conditions 738517Sdfr * are met: 838517Sdfr * 1. Redistributions of source code must retain the above copyright 938517Sdfr * notice, this list of conditions and the following disclaimer. 1038517Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1138517Sdfr * notice, this list of conditions and the following disclaimer in the 1238517Sdfr * documentation and/or other materials provided with the distribution. 1338517Sdfr * 1438517Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1538517Sdfr * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1638517Sdfr * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1738517Sdfr * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1838517Sdfr * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1938517Sdfr * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2038517Sdfr * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2138517Sdfr * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2238517Sdfr * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2338517Sdfr * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2438517Sdfr * SUCH DAMAGE. 2538517Sdfr * 2650477Speter * $FreeBSD: head/sys/i386/include/atomic.h 90515 2002-02-11 03:41:59Z bde $ 2738517Sdfr */ 2838517Sdfr#ifndef _MACHINE_ATOMIC_H_ 2938517Sdfr#define _MACHINE_ATOMIC_H_ 3038517Sdfr 3138517Sdfr/* 3238517Sdfr * Various simple arithmetic on memory which is atomic in the presence 3348797Salc * of interrupts and multiple processors. 3438517Sdfr * 3548797Salc * atomic_set_char(P, V) (*(u_char*)(P) |= (V)) 3648797Salc * atomic_clear_char(P, V) (*(u_char*)(P) &= ~(V)) 3748797Salc * atomic_add_char(P, V) (*(u_char*)(P) += (V)) 3848797Salc * atomic_subtract_char(P, V) (*(u_char*)(P) -= (V)) 3948797Salc * 4048797Salc * atomic_set_short(P, V) (*(u_short*)(P) |= (V)) 4148797Salc * atomic_clear_short(P, V) (*(u_short*)(P) &= ~(V)) 4248797Salc * atomic_add_short(P, V) (*(u_short*)(P) += (V)) 4348797Salc * atomic_subtract_short(P, V) (*(u_short*)(P) -= (V)) 4448797Salc * 4548797Salc * atomic_set_int(P, V) (*(u_int*)(P) |= (V)) 4648797Salc * atomic_clear_int(P, V) (*(u_int*)(P) &= ~(V)) 4748797Salc * atomic_add_int(P, V) (*(u_int*)(P) += (V)) 4848797Salc * atomic_subtract_int(P, V) (*(u_int*)(P) -= (V)) 4966695Sjhb * atomic_readandclear_int(P) (return *(u_int*)P; *(u_int*)P = 0;) 5048797Salc * 5148797Salc * atomic_set_long(P, V) (*(u_long*)(P) |= (V)) 5248797Salc * atomic_clear_long(P, V) (*(u_long*)(P) &= ~(V)) 5348797Salc * atomic_add_long(P, V) (*(u_long*)(P) += (V)) 5448797Salc * atomic_subtract_long(P, V) (*(u_long*)(P) -= (V)) 5566695Sjhb * atomic_readandclear_long(P) (return *(u_long*)P; *(u_long*)P = 0;) 5638517Sdfr */ 5738517Sdfr 5848797Salc/* 5949999Salc * The above functions are expanded inline in the statically-linked 6049999Salc * kernel. Lock prefixes are generated if an SMP kernel is being 6149999Salc * built. 6249999Salc * 6349999Salc * Kernel modules call real functions which are built into the kernel. 6449999Salc * This allows kernel modules to be portable between UP and SMP systems. 6548797Salc */ 6649999Salc#if defined(KLD_MODULE) 6788117Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 6867351Sjhbvoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); 6949999Salc 7066695Sjhbint atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); 7165514Sphk 7271085Sjhb#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 7371085Sjhbu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ 7471085Sjhbvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v); 7571085Sjhb 7649999Salc#else /* !KLD_MODULE */ 7772358Smarkm 7884679Sjhb/* 7984679Sjhb * For userland, assume the SMP case and use lock prefixes so that 8084679Sjhb * the binaries will run on both types of systems. 8184679Sjhb */ 8284679Sjhb#if defined(SMP) || !defined(_KERNEL) 8348797Salc#define MPLOCKED "lock ; " 8490515Sbde#else 8548797Salc#define MPLOCKED 8690515Sbde#endif 8738517Sdfr 8848797Salc/* 8948797Salc * The assembly is volatilized to demark potential before-and-after side 9048797Salc * effects if an interrupt or SMP collision were to occur. 9148797Salc */ 9288117Sjhb#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 9348797Salcstatic __inline void \ 9449043Salcatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 9548797Salc{ \ 9648797Salc __asm __volatile(MPLOCKED OP \ 9786301Sjhb : "+m" (*p) \ 9888117Sjhb : CONS (V)); \ 9948797Salc} 10051938Speter 10165514Sphk/* 10265514Sphk * Atomic compare and set, used by the mutex functions 10365514Sphk * 10465514Sphk * if (*dst == exp) *dst = src (all 32 bit words) 10565514Sphk * 10665514Sphk * Returns 0 on failure, non-zero on success 10765514Sphk */ 10865514Sphk 10965514Sphk#if defined(I386_CPU) 11065514Sphkstatic __inline int 11165514Sphkatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 11265514Sphk{ 11365514Sphk int res = exp; 11465514Sphk 11565514Sphk __asm __volatile( 11665514Sphk " pushfl ; " 11765514Sphk " cli ; " 11886303Sjhb " cmpl %0,%2 ; " 11965514Sphk " jne 1f ; " 12086303Sjhb " movl %1,%2 ; " 12165514Sphk "1: " 12265514Sphk " sete %%al; " 12365514Sphk " movzbl %%al,%0 ; " 12465514Sphk " popfl ; " 12565514Sphk "# atomic_cmpset_int" 12686303Sjhb : "+a" (res) /* 0 (result) */ 12786303Sjhb : "r" (src), /* 1 */ 12886303Sjhb "m" (*(dst)) /* 2 */ 12965514Sphk : "memory"); 13065514Sphk 13165514Sphk return (res); 13265514Sphk} 13365514Sphk#else /* defined(I386_CPU) */ 13465514Sphkstatic __inline int 13565514Sphkatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 13665514Sphk{ 13765514Sphk int res = exp; 13865514Sphk 13965514Sphk __asm __volatile ( 14065514Sphk " " MPLOCKED " " 14186303Sjhb " cmpxchgl %1,%2 ; " 14265514Sphk " setz %%al ; " 14365514Sphk " movzbl %%al,%0 ; " 14465514Sphk "1: " 14565514Sphk "# atomic_cmpset_int" 14686303Sjhb : "+a" (res) /* 0 (result) */ 14786303Sjhb : "r" (src), /* 1 */ 14886303Sjhb "m" (*(dst)) /* 2 */ 14965514Sphk : "memory"); 15065514Sphk 15165514Sphk return (res); 15265514Sphk} 15365514Sphk#endif /* defined(I386_CPU) */ 15465514Sphk 15571023Sjhb#if defined(I386_CPU) 15667351Sjhb/* 15767351Sjhb * We assume that a = b will do atomic loads and stores. 15871023Sjhb * 15971023Sjhb * XXX: This is _NOT_ safe on a P6 or higher because it does not guarantee 16071023Sjhb * memory ordering. These should only be used on a 386. 16167351Sjhb */ 16271023Sjhb#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 16367351Sjhbstatic __inline u_##TYPE \ 16467351Sjhbatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 16567351Sjhb{ \ 16667351Sjhb return (*p); \ 16767351Sjhb} \ 16867351Sjhb \ 16967351Sjhbstatic __inline void \ 17067351Sjhbatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 17167351Sjhb{ \ 17267351Sjhb *p = v; \ 17367351Sjhb __asm __volatile("" : : : "memory"); \ 17467351Sjhb} 17571023Sjhb#else 17667351Sjhb 17771023Sjhb#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 17871023Sjhbstatic __inline u_##TYPE \ 17971023Sjhbatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 18071023Sjhb{ \ 18171023Sjhb u_##TYPE res; \ 18271023Sjhb \ 18371023Sjhb __asm __volatile(MPLOCKED LOP \ 18471141Sjhb : "=a" (res), /* 0 (result) */\ 18571023Sjhb "+m" (*p) /* 1 */ \ 18688117Sjhb : : "memory"); \ 18771023Sjhb \ 18871023Sjhb return (res); \ 18971023Sjhb} \ 19071023Sjhb \ 19171023Sjhb/* \ 19271023Sjhb * The XCHG instruction asserts LOCK automagically. \ 19371023Sjhb */ \ 19471023Sjhbstatic __inline void \ 19571023Sjhbatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 19671023Sjhb{ \ 19771023Sjhb __asm __volatile(SOP \ 19871023Sjhb : "+m" (*p), /* 0 */ \ 19971023Sjhb "+r" (v) /* 1 */ \ 20071023Sjhb : : "memory"); \ 20171023Sjhb} 20271023Sjhb#endif /* defined(I386_CPU) */ 20371085Sjhb#endif /* KLD_MODULE */ 20467351Sjhb 20588117SjhbATOMIC_ASM(set, char, "orb %b1,%0", "iq", v) 20688117SjhbATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v) 20788117SjhbATOMIC_ASM(add, char, "addb %b1,%0", "iq", v) 20888117SjhbATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v) 20971085Sjhb 21088117SjhbATOMIC_ASM(set, short, "orw %w1,%0", "ir", v) 21188117SjhbATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v) 21288117SjhbATOMIC_ASM(add, short, "addw %w1,%0", "ir", v) 21388117SjhbATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v) 21471085Sjhb 21588117SjhbATOMIC_ASM(set, int, "orl %1,%0", "ir", v) 21688117SjhbATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v) 21788117SjhbATOMIC_ASM(add, int, "addl %1,%0", "ir", v) 21888117SjhbATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v) 21971085Sjhb 22088117SjhbATOMIC_ASM(set, long, "orl %1,%0", "ir", v) 22188117SjhbATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v) 22288117SjhbATOMIC_ASM(add, long, "addl %1,%0", "ir", v) 22388117SjhbATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v) 22471085Sjhb 22571023SjhbATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0") 22671023SjhbATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0") 22771023SjhbATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0") 22871023SjhbATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1", "xchgl %1,%0") 22971023Sjhb 23071085Sjhb#undef ATOMIC_ASM 23167351Sjhb#undef ATOMIC_STORE_LOAD 23267351Sjhb 23371085Sjhb#define atomic_set_acq_char atomic_set_char 23471085Sjhb#define atomic_set_rel_char atomic_set_char 23571085Sjhb#define atomic_clear_acq_char atomic_clear_char 23671085Sjhb#define atomic_clear_rel_char atomic_clear_char 23771085Sjhb#define atomic_add_acq_char atomic_add_char 23871085Sjhb#define atomic_add_rel_char atomic_add_char 23971085Sjhb#define atomic_subtract_acq_char atomic_subtract_char 24071085Sjhb#define atomic_subtract_rel_char atomic_subtract_char 24171085Sjhb 24271085Sjhb#define atomic_set_acq_short atomic_set_short 24371085Sjhb#define atomic_set_rel_short atomic_set_short 24471085Sjhb#define atomic_clear_acq_short atomic_clear_short 24571085Sjhb#define atomic_clear_rel_short atomic_clear_short 24671085Sjhb#define atomic_add_acq_short atomic_add_short 24771085Sjhb#define atomic_add_rel_short atomic_add_short 24871085Sjhb#define atomic_subtract_acq_short atomic_subtract_short 24971085Sjhb#define atomic_subtract_rel_short atomic_subtract_short 25071085Sjhb 25171085Sjhb#define atomic_set_acq_int atomic_set_int 25271085Sjhb#define atomic_set_rel_int atomic_set_int 25371085Sjhb#define atomic_clear_acq_int atomic_clear_int 25471085Sjhb#define atomic_clear_rel_int atomic_clear_int 25571085Sjhb#define atomic_add_acq_int atomic_add_int 25671085Sjhb#define atomic_add_rel_int atomic_add_int 25771085Sjhb#define atomic_subtract_acq_int atomic_subtract_int 25871085Sjhb#define atomic_subtract_rel_int atomic_subtract_int 25971085Sjhb#define atomic_cmpset_acq_int atomic_cmpset_int 26071085Sjhb#define atomic_cmpset_rel_int atomic_cmpset_int 26171085Sjhb 26271085Sjhb#define atomic_set_acq_long atomic_set_long 26371085Sjhb#define atomic_set_rel_long atomic_set_long 26471085Sjhb#define atomic_clear_acq_long atomic_clear_long 26571085Sjhb#define atomic_clear_rel_long atomic_clear_long 26671085Sjhb#define atomic_add_acq_long atomic_add_long 26771085Sjhb#define atomic_add_rel_long atomic_add_long 26871085Sjhb#define atomic_subtract_acq_long atomic_subtract_long 26971085Sjhb#define atomic_subtract_rel_long atomic_subtract_long 27071085Sjhb#define atomic_cmpset_long atomic_cmpset_int 27171085Sjhb#define atomic_cmpset_acq_long atomic_cmpset_acq_int 27271085Sjhb#define atomic_cmpset_rel_long atomic_cmpset_rel_int 27371085Sjhb 27471085Sjhb#define atomic_cmpset_acq_ptr atomic_cmpset_ptr 27571085Sjhb#define atomic_cmpset_rel_ptr atomic_cmpset_ptr 27671085Sjhb 27771085Sjhb#define atomic_set_8 atomic_set_char 27871085Sjhb#define atomic_set_acq_8 atomic_set_acq_char 27971085Sjhb#define atomic_set_rel_8 atomic_set_rel_char 28071085Sjhb#define atomic_clear_8 atomic_clear_char 28171085Sjhb#define atomic_clear_acq_8 atomic_clear_acq_char 28271085Sjhb#define atomic_clear_rel_8 atomic_clear_rel_char 28371085Sjhb#define atomic_add_8 atomic_add_char 28471085Sjhb#define atomic_add_acq_8 atomic_add_acq_char 28571085Sjhb#define atomic_add_rel_8 atomic_add_rel_char 28671085Sjhb#define atomic_subtract_8 atomic_subtract_char 28771085Sjhb#define atomic_subtract_acq_8 atomic_subtract_acq_char 28871085Sjhb#define atomic_subtract_rel_8 atomic_subtract_rel_char 28971085Sjhb#define atomic_load_acq_8 atomic_load_acq_char 29071085Sjhb#define atomic_store_rel_8 atomic_store_rel_char 29171085Sjhb 29271085Sjhb#define atomic_set_16 atomic_set_short 29371085Sjhb#define atomic_set_acq_16 atomic_set_acq_short 29471085Sjhb#define atomic_set_rel_16 atomic_set_rel_short 29571085Sjhb#define atomic_clear_16 atomic_clear_short 29671085Sjhb#define atomic_clear_acq_16 atomic_clear_acq_short 29771085Sjhb#define atomic_clear_rel_16 atomic_clear_rel_short 29871085Sjhb#define atomic_add_16 atomic_add_short 29971085Sjhb#define atomic_add_acq_16 atomic_add_acq_short 30071085Sjhb#define atomic_add_rel_16 atomic_add_rel_short 30171085Sjhb#define atomic_subtract_16 atomic_subtract_short 30271085Sjhb#define atomic_subtract_acq_16 atomic_subtract_acq_short 30371085Sjhb#define atomic_subtract_rel_16 atomic_subtract_rel_short 30471085Sjhb#define atomic_load_acq_16 atomic_load_acq_short 30571085Sjhb#define atomic_store_rel_16 atomic_store_rel_short 30671085Sjhb 30771085Sjhb#define atomic_set_32 atomic_set_int 30871085Sjhb#define atomic_set_acq_32 atomic_set_acq_int 30971085Sjhb#define atomic_set_rel_32 atomic_set_rel_int 31071085Sjhb#define atomic_clear_32 atomic_clear_int 31171085Sjhb#define atomic_clear_acq_32 atomic_clear_acq_int 31271085Sjhb#define atomic_clear_rel_32 atomic_clear_rel_int 31371085Sjhb#define atomic_add_32 atomic_add_int 31471085Sjhb#define atomic_add_acq_32 atomic_add_acq_int 31571085Sjhb#define atomic_add_rel_32 atomic_add_rel_int 31671085Sjhb#define atomic_subtract_32 atomic_subtract_int 31771085Sjhb#define atomic_subtract_acq_32 atomic_subtract_acq_int 31871085Sjhb#define atomic_subtract_rel_32 atomic_subtract_rel_int 31971085Sjhb#define atomic_load_acq_32 atomic_load_acq_int 32071085Sjhb#define atomic_store_rel_32 atomic_store_rel_int 32171085Sjhb#define atomic_cmpset_32 atomic_cmpset_int 32271085Sjhb#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 32371085Sjhb#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 32471085Sjhb#define atomic_readandclear_32 atomic_readandclear_int 32571085Sjhb 32671085Sjhb#if !defined(WANT_FUNCTIONS) 32765514Sphkstatic __inline int 32865514Sphkatomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 32965514Sphk{ 33065514Sphk 33171085Sjhb return (atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, 33271085Sjhb (u_int)src)); 33365514Sphk} 33466695Sjhb 33567351Sjhbstatic __inline void * 33667351Sjhbatomic_load_acq_ptr(volatile void *p) 33767351Sjhb{ 33867351Sjhb return (void *)atomic_load_acq_int((volatile u_int *)p); 33967351Sjhb} 34067351Sjhb 34167351Sjhbstatic __inline void 34267351Sjhbatomic_store_rel_ptr(volatile void *p, void *v) 34367351Sjhb{ 34467351Sjhb atomic_store_rel_int((volatile u_int *)p, (u_int)v); 34567351Sjhb} 34667351Sjhb 34767351Sjhb#define ATOMIC_PTR(NAME) \ 34867351Sjhbstatic __inline void \ 34967351Sjhbatomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 35067351Sjhb{ \ 35167351Sjhb atomic_##NAME##_int((volatile u_int *)p, v); \ 35267351Sjhb} \ 35367351Sjhb \ 35467351Sjhbstatic __inline void \ 35567351Sjhbatomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 35667351Sjhb{ \ 35767351Sjhb atomic_##NAME##_acq_int((volatile u_int *)p, v);\ 35867351Sjhb} \ 35967351Sjhb \ 36067351Sjhbstatic __inline void \ 36167351Sjhbatomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 36267351Sjhb{ \ 36367351Sjhb atomic_##NAME##_rel_int((volatile u_int *)p, v);\ 36467351Sjhb} 36567351Sjhb 36667351SjhbATOMIC_PTR(set) 36767351SjhbATOMIC_PTR(clear) 36867351SjhbATOMIC_PTR(add) 36967351SjhbATOMIC_PTR(subtract) 37067351Sjhb 37167351Sjhb#undef ATOMIC_PTR 37267351Sjhb 37366695Sjhbstatic __inline u_int 37466695Sjhbatomic_readandclear_int(volatile u_int *addr) 37566695Sjhb{ 37666695Sjhb u_int result; 37766695Sjhb 37866695Sjhb __asm __volatile ( 37966695Sjhb " xorl %0,%0 ; " 38066695Sjhb " xchgl %1,%0 ; " 38166695Sjhb "# atomic_readandclear_int" 38266695Sjhb : "=&r" (result) /* 0 (result) */ 38366695Sjhb : "m" (*addr)); /* 1 (addr) */ 38466695Sjhb 38566695Sjhb return (result); 38666695Sjhb} 38766695Sjhb 38866695Sjhbstatic __inline u_long 38966695Sjhbatomic_readandclear_long(volatile u_long *addr) 39066695Sjhb{ 39166695Sjhb u_long result; 39266695Sjhb 39366695Sjhb __asm __volatile ( 39466695Sjhb " xorl %0,%0 ; " 39566695Sjhb " xchgl %1,%0 ; " 39666695Sjhb "# atomic_readandclear_int" 39766695Sjhb : "=&r" (result) /* 0 (result) */ 39866695Sjhb : "m" (*addr)); /* 1 (addr) */ 39966695Sjhb 40066695Sjhb return (result); 40166695Sjhb} 40271085Sjhb#endif /* !defined(WANT_FUNCTIONS) */ 40338517Sdfr#endif /* ! _MACHINE_ATOMIC_H_ */ 404