atomic.h revision 91469
167754Smsmith/*- 267754Smsmith * Copyright (c) 1998 Doug Rabson 367754Smsmith * All rights reserved. 467754Smsmith * 567754Smsmith * Redistribution and use in source and binary forms, with or without 667754Smsmith * modification, are permitted provided that the following conditions 7217365Sjkim * are met: 8281075Sdim * 1. Redistributions of source code must retain the above copyright 970243Smsmith * notice, this list of conditions and the following disclaimer. 1067754Smsmith * 2. Redistributions in binary form must reproduce the above copyright 11217365Sjkim * notice, this list of conditions and the following disclaimer in the 12217365Sjkim * documentation and/or other materials provided with the distribution. 13217365Sjkim * 14217365Sjkim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15217365Sjkim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16217365Sjkim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17217365Sjkim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18217365Sjkim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19217365Sjkim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20217365Sjkim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21217365Sjkim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22217365Sjkim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23217365Sjkim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24217365Sjkim * SUCH DAMAGE. 2567754Smsmith * 26217365Sjkim * $FreeBSD: head/sys/amd64/include/atomic.h 91469 2002-02-28 06:17:05Z bmilekic $ 27217365Sjkim */ 28217365Sjkim#ifndef _MACHINE_ATOMIC_H_ 2967754Smsmith#define _MACHINE_ATOMIC_H_ 30217365Sjkim 31217365Sjkim/* 32217365Sjkim * Various simple arithmetic on memory which is atomic in the presence 33217365Sjkim * of interrupts and multiple processors. 34217365Sjkim * 35217365Sjkim * atomic_set_char(P, V) (*(u_char*)(P) |= (V)) 36217365Sjkim * atomic_clear_char(P, V) (*(u_char*)(P) &= ~(V)) 37217365Sjkim * atomic_add_char(P, V) (*(u_char*)(P) += (V)) 38217365Sjkim * atomic_subtract_char(P, V) (*(u_char*)(P) -= (V)) 39217365Sjkim * 40217365Sjkim * atomic_set_short(P, V) (*(u_short*)(P) |= (V)) 41217365Sjkim * atomic_clear_short(P, V) (*(u_short*)(P) &= ~(V)) 42217365Sjkim * atomic_add_short(P, V) (*(u_short*)(P) += (V)) 4367754Smsmith * atomic_subtract_short(P, V) (*(u_short*)(P) -= (V)) 4467754Smsmith * 4567754Smsmith * atomic_set_int(P, V) (*(u_int*)(P) |= (V)) 4667754Smsmith * atomic_clear_int(P, V) (*(u_int*)(P) &= ~(V)) 4767754Smsmith * atomic_add_int(P, V) (*(u_int*)(P) += (V)) 48193267Sjkim * atomic_subtract_int(P, V) (*(u_int*)(P) -= (V)) 49193267Sjkim * atomic_readandclear_int(P) (return *(u_int*)P; *(u_int*)P = 0;) 50193267Sjkim * 5167754Smsmith * atomic_set_long(P, V) (*(u_long*)(P) |= (V)) 5267754Smsmith * atomic_clear_long(P, V) (*(u_long*)(P) &= ~(V)) 5382367Smsmith * atomic_add_long(P, V) (*(u_long*)(P) += (V)) 5467754Smsmith * atomic_subtract_long(P, V) (*(u_long*)(P) -= (V)) 5577424Smsmith * atomic_readandclear_long(P) (return *(u_long*)P; *(u_long*)P = 0;) 5677424Smsmith */ 5777424Smsmith 5877424Smsmith/* 5977424Smsmith * The above functions are expanded inline in the statically-linked 6077424Smsmith * kernel. Lock prefixes are generated if an SMP kernel is being 6177424Smsmith * built. 6277424Smsmith * 6377424Smsmith * Kernel modules call real functions which are built into the kernel. 64102550Siwasaki * This allows kernel modules to be portable between UP and SMP systems. 6578986Smsmith */ 66102550Siwasaki#if defined(KLD_MODULE) 6767754Smsmith#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 6891116Smsmithvoid atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); 6967754Smsmith 7091116Smsmithint atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src); 7191116Smsmith 72193267Sjkim#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 73193267Sjkimu_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p); \ 74209746Sjkimvoid atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v); 75233250Sjkim 7667754Smsmith#else /* !KLD_MODULE */ 77209746Sjkim 78102550Siwasaki/* 79102550Siwasaki * For userland, assume the SMP case and use lock prefixes so that 8091116Smsmith * the binaries will run on both types of systems. 8177424Smsmith */ 8291116Smsmith#if defined(SMP) || !defined(_KERNEL) 8391116Smsmith#define MPLOCKED lock ; 84114237Snjl#else 8582367Smsmith#define MPLOCKED 86193267Sjkim#endif 8782367Smsmith 88193267Sjkim/* 89193267Sjkim * The assembly is volatilized to demark potential before-and-after side 90193267Sjkim * effects if an interrupt or SMP collision were to occur. 91200553Sjkim */ 92200553Sjkim#define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ 9382367Smsmithstatic __inline void \ 9483174Smsmithatomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 9582367Smsmith{ \ 96114237Snjl __asm __volatile(__XSTRING(MPLOCKED) OP \ 9783174Smsmith : "+m" (*p) \ 9883174Smsmith : CONS (V)); \ 9983174Smsmith} 10083174Smsmith 10183174Smsmith/* 10283174Smsmith * Atomic compare and set, used by the mutex functions 10383174Smsmith * 10483174Smsmith * if (*dst == exp) *dst = src (all 32 bit words) 10583174Smsmith * 10683174Smsmith * Returns 0 on failure, non-zero on success 10783174Smsmith */ 10883174Smsmith 10983174Smsmith#if defined(I386_CPU) 110114237Snjlstatic __inline int 11183174Smsmithatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 11283174Smsmith{ 11383174Smsmith int res = exp; 11482367Smsmith 11583174Smsmith __asm __volatile( 116102550Siwasaki " pushfl ; " 117102550Siwasaki " cli ; " 11883174Smsmith " cmpl %0,%2 ; " 11982367Smsmith " jne 1f ; " 12083174Smsmith " movl %1,%2 ; " 12182367Smsmith "1: " 12283174Smsmith " sete %%al; " 12383174Smsmith " movzbl %%al,%0 ; " 12483174Smsmith " popfl ; " 12583174Smsmith "# atomic_cmpset_int" 12683174Smsmith : "+a" (res) /* 0 (result) */ 12783174Smsmith : "r" (src), /* 1 */ 12899679Siwasaki "m" (*(dst)) /* 2 */ 12999679Siwasaki : "memory"); 13099679Siwasaki 13199679Siwasaki return (res); 13299679Siwasaki} 13399679Siwasaki#else /* defined(I386_CPU) */ 13499679Siwasakistatic __inline int 13599679Siwasakiatomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) 13699679Siwasaki{ 13782367Smsmith int res = exp; 13882367Smsmith 13982367Smsmith __asm __volatile ( 140151937Sjkim " " __XSTRING(MPLOCKED) " " 14182367Smsmith " cmpxchgl %1,%2 ; " 142193267Sjkim " setz %%al ; " 143193267Sjkim " movzbl %%al,%0 ; " 144193267Sjkim "1: " 145193267Sjkim "# atomic_cmpset_int" 146193267Sjkim : "+a" (res) /* 0 (result) */ 147193267Sjkim : "r" (src), /* 1 */ 148114237Snjl "m" (*(dst)) /* 2 */ 14982367Smsmith : "memory"); 150114237Snjl 151200553Sjkim return (res); 15282367Smsmith} 15367754Smsmith#endif /* defined(I386_CPU) */ 15467754Smsmith 15567754Smsmith#if defined(I386_CPU) 156114237Snjl/* 15782367Smsmith * We assume that a = b will do atomic loads and stores. 15882367Smsmith * 15982367Smsmith * XXX: This is _NOT_ safe on a P6 or higher because it does not guarantee 16082367Smsmith * memory ordering. These should only be used on a 386. 16182367Smsmith */ 16282367Smsmith#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 16382367Smsmithstatic __inline u_##TYPE \ 16482367Smsmithatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 16582367Smsmith{ \ 16682367Smsmith return (*p); \ 167102550Siwasaki} \ 16882367Smsmith \ 16982367Smsmithstatic __inline void \ 17082367Smsmithatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 17182367Smsmith{ \ 17282367Smsmith *p = v; \ 17382367Smsmith __asm __volatile("" : : : "memory"); \ 17482367Smsmith} 17582367Smsmith#else 17682367Smsmith 177193267Sjkim#define ATOMIC_STORE_LOAD(TYPE, LOP, SOP) \ 17867754Smsmithstatic __inline u_##TYPE \ 17999679Siwasakiatomic_load_acq_##TYPE(volatile u_##TYPE *p) \ 18067754Smsmith{ \ 18167754Smsmith u_##TYPE res; \ 18267754Smsmith \ 183200553Sjkim __asm __volatile(__XSTRING(MPLOCKED) LOP \ 184200553Sjkim : "=a" (res), /* 0 (result) */\ 185114237Snjl "+m" (*p) /* 1 */ \ 18667754Smsmith : : "memory"); \ 18767754Smsmith \ 188193267Sjkim return (res); \ 189193267Sjkim} \ 190218590Sjkim \ 191218590Sjkim/* \ 192218590Sjkim * The XCHG instruction asserts LOCK automagically. \ 193218590Sjkim */ \ 194193267Sjkimstatic __inline void \ 195193267Sjkimatomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ 196193267Sjkim{ \ 197218590Sjkim __asm __volatile(SOP \ 198218590Sjkim : "+m" (*p), /* 0 */ \ 199218590Sjkim "+r" (v) /* 1 */ \ 200218590Sjkim : : "memory"); \ 201193267Sjkim} 202218590Sjkim#endif /* defined(I386_CPU) */ 203193267Sjkim#endif /* KLD_MODULE */ 204193267Sjkim 205193267SjkimATOMIC_ASM(set, char, "orb %b1,%0", "iq", v) 206193267SjkimATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v) 207193267SjkimATOMIC_ASM(add, char, "addb %b1,%0", "iq", v) 208193267SjkimATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v) 209193267Sjkim 210193267SjkimATOMIC_ASM(set, short, "orw %w1,%0", "ir", v) 211193267SjkimATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v) 212193267SjkimATOMIC_ASM(add, short, "addw %w1,%0", "ir", v) 213193267SjkimATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v) 214193267Sjkim 215193267SjkimATOMIC_ASM(set, int, "orl %1,%0", "ir", v) 216193267SjkimATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v) 217193267SjkimATOMIC_ASM(add, int, "addl %1,%0", "ir", v) 218193267SjkimATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v) 219193267Sjkim 220238381SjkimATOMIC_ASM(set, long, "orl %1,%0", "ir", v) 221238381SjkimATOMIC_ASM(clear, long, "andl %1,%0", "ir", ~v) 222204773SjkimATOMIC_ASM(add, long, "addl %1,%0", "ir", v) 223193267SjkimATOMIC_ASM(subtract, long, "subl %1,%0", "ir", v) 224193267Sjkim 225193267SjkimATOMIC_STORE_LOAD(char, "cmpxchgb %b0,%1", "xchgb %b1,%0") 226193267SjkimATOMIC_STORE_LOAD(short,"cmpxchgw %w0,%1", "xchgw %w1,%0") 227193267SjkimATOMIC_STORE_LOAD(int, "cmpxchgl %0,%1", "xchgl %1,%0") 228193267SjkimATOMIC_STORE_LOAD(long, "cmpxchgl %0,%1", "xchgl %1,%0") 229193267Sjkim 230193267Sjkim#undef ATOMIC_ASM 231193267Sjkim#undef ATOMIC_STORE_LOAD 232238381Sjkim 233238381Sjkim#define atomic_set_acq_char atomic_set_char 234204773Sjkim#define atomic_set_rel_char atomic_set_char 235193267Sjkim#define atomic_clear_acq_char atomic_clear_char 236193267Sjkim#define atomic_clear_rel_char atomic_clear_char 237193267Sjkim#define atomic_add_acq_char atomic_add_char 238193267Sjkim#define atomic_add_rel_char atomic_add_char 239193267Sjkim#define atomic_subtract_acq_char atomic_subtract_char 240193267Sjkim#define atomic_subtract_rel_char atomic_subtract_char 241193267Sjkim 242193267Sjkim#define atomic_set_acq_short atomic_set_short 243193267Sjkim#define atomic_set_rel_short atomic_set_short 244193267Sjkim#define atomic_clear_acq_short atomic_clear_short 245193267Sjkim#define atomic_clear_rel_short atomic_clear_short 246193267Sjkim#define atomic_add_acq_short atomic_add_short 247193267Sjkim#define atomic_add_rel_short atomic_add_short 248193267Sjkim#define atomic_subtract_acq_short atomic_subtract_short 249193267Sjkim#define atomic_subtract_rel_short atomic_subtract_short 250193267Sjkim 251193267Sjkim#define atomic_set_acq_int atomic_set_int 252193267Sjkim#define atomic_set_rel_int atomic_set_int 253193267Sjkim#define atomic_clear_acq_int atomic_clear_int 254193267Sjkim#define atomic_clear_rel_int atomic_clear_int 255193267Sjkim#define atomic_add_acq_int atomic_add_int 256193267Sjkim#define atomic_add_rel_int atomic_add_int 257193267Sjkim#define atomic_subtract_acq_int atomic_subtract_int 258193267Sjkim#define atomic_subtract_rel_int atomic_subtract_int 259193267Sjkim#define atomic_cmpset_acq_int atomic_cmpset_int 260193267Sjkim#define atomic_cmpset_rel_int atomic_cmpset_int 261193267Sjkim 262193267Sjkim#define atomic_set_acq_long atomic_set_long 263193267Sjkim#define atomic_set_rel_long atomic_set_long 264193267Sjkim#define atomic_clear_acq_long atomic_clear_long 265193267Sjkim#define atomic_clear_rel_long atomic_clear_long 266193267Sjkim#define atomic_add_acq_long atomic_add_long 267193267Sjkim#define atomic_add_rel_long atomic_add_long 268193267Sjkim#define atomic_subtract_acq_long atomic_subtract_long 269193267Sjkim#define atomic_subtract_rel_long atomic_subtract_long 270245582Sjkim#define atomic_cmpset_long atomic_cmpset_int 271245582Sjkim#define atomic_cmpset_acq_long atomic_cmpset_acq_int 272193267Sjkim#define atomic_cmpset_rel_long atomic_cmpset_rel_int 273245582Sjkim 274245582Sjkim#define atomic_cmpset_acq_ptr atomic_cmpset_ptr 275245582Sjkim#define atomic_cmpset_rel_ptr atomic_cmpset_ptr 276245582Sjkim 277245582Sjkim#define atomic_set_8 atomic_set_char 278193267Sjkim#define atomic_set_acq_8 atomic_set_acq_char 279193267Sjkim#define atomic_set_rel_8 atomic_set_rel_char 280193267Sjkim#define atomic_clear_8 atomic_clear_char 281193267Sjkim#define atomic_clear_acq_8 atomic_clear_acq_char 282193267Sjkim#define atomic_clear_rel_8 atomic_clear_rel_char 283245582Sjkim#define atomic_add_8 atomic_add_char 284245582Sjkim#define atomic_add_acq_8 atomic_add_acq_char 285245582Sjkim#define atomic_add_rel_8 atomic_add_rel_char 286245582Sjkim#define atomic_subtract_8 atomic_subtract_char 287245582Sjkim#define atomic_subtract_acq_8 atomic_subtract_acq_char 288245582Sjkim#define atomic_subtract_rel_8 atomic_subtract_rel_char 289245582Sjkim#define atomic_load_acq_8 atomic_load_acq_char 290245582Sjkim#define atomic_store_rel_8 atomic_store_rel_char 291245582Sjkim 292193267Sjkim#define atomic_set_16 atomic_set_short 293245582Sjkim#define atomic_set_acq_16 atomic_set_acq_short 294245582Sjkim#define atomic_set_rel_16 atomic_set_rel_short 295245582Sjkim#define atomic_clear_16 atomic_clear_short 296245582Sjkim#define atomic_clear_acq_16 atomic_clear_acq_short 297245582Sjkim#define atomic_clear_rel_16 atomic_clear_rel_short 298193267Sjkim#define atomic_add_16 atomic_add_short 299245582Sjkim#define atomic_add_acq_16 atomic_add_acq_short 300245582Sjkim#define atomic_add_rel_16 atomic_add_rel_short 301245582Sjkim#define atomic_subtract_16 atomic_subtract_short 302245582Sjkim#define atomic_subtract_acq_16 atomic_subtract_acq_short 303245582Sjkim#define atomic_subtract_rel_16 atomic_subtract_rel_short 304245582Sjkim#define atomic_load_acq_16 atomic_load_acq_short 305245582Sjkim#define atomic_store_rel_16 atomic_store_rel_short 306245582Sjkim 307245582Sjkim#define atomic_set_32 atomic_set_int 308245582Sjkim#define atomic_set_acq_32 atomic_set_acq_int 309245582Sjkim#define atomic_set_rel_32 atomic_set_rel_int 310245582Sjkim#define atomic_clear_32 atomic_clear_int 311245582Sjkim#define atomic_clear_acq_32 atomic_clear_acq_int 312245582Sjkim#define atomic_clear_rel_32 atomic_clear_rel_int 313245582Sjkim#define atomic_add_32 atomic_add_int 314245582Sjkim#define atomic_add_acq_32 atomic_add_acq_int 315245582Sjkim#define atomic_add_rel_32 atomic_add_rel_int 316245582Sjkim#define atomic_subtract_32 atomic_subtract_int 317245582Sjkim#define atomic_subtract_acq_32 atomic_subtract_acq_int 318245582Sjkim#define atomic_subtract_rel_32 atomic_subtract_rel_int 319245582Sjkim#define atomic_load_acq_32 atomic_load_acq_int 320245582Sjkim#define atomic_store_rel_32 atomic_store_rel_int 321245582Sjkim#define atomic_cmpset_32 atomic_cmpset_int 322245582Sjkim#define atomic_cmpset_acq_32 atomic_cmpset_acq_int 323193267Sjkim#define atomic_cmpset_rel_32 atomic_cmpset_rel_int 324245582Sjkim#define atomic_readandclear_32 atomic_readandclear_int 325245582Sjkim 326245582Sjkim#if !defined(WANT_FUNCTIONS) 327245582Sjkimstatic __inline int 328245582Sjkimatomic_cmpset_ptr(volatile void *dst, void *exp, void *src) 329245582Sjkim{ 330245582Sjkim 331245582Sjkim return (atomic_cmpset_int((volatile u_int *)dst, (u_int)exp, 332246040Sjkim (u_int)src)); 333245582Sjkim} 334246040Sjkim 335245582Sjkimstatic __inline void * 336245582Sjkimatomic_load_acq_ptr(volatile void *p) 337245582Sjkim{ 338245582Sjkim return (void *)atomic_load_acq_int((volatile u_int *)p); 339245582Sjkim} 340245582Sjkim 341245582Sjkimstatic __inline void 342245582Sjkimatomic_store_rel_ptr(volatile void *p, void *v) 343246040Sjkim{ 344245582Sjkim atomic_store_rel_int((volatile u_int *)p, (u_int)v); 345245582Sjkim} 346246040Sjkim 347245582Sjkim#define ATOMIC_PTR(NAME) \ 348245582Sjkimstatic __inline void \ 349246040Sjkimatomic_##NAME##_ptr(volatile void *p, uintptr_t v) \ 350245582Sjkim{ \ 351245582Sjkim atomic_##NAME##_int((volatile u_int *)p, v); \ 352245582Sjkim} \ 353245582Sjkim \ 354245582Sjkimstatic __inline void \ 355245582Sjkimatomic_##NAME##_acq_ptr(volatile void *p, uintptr_t v) \ 356245582Sjkim{ \ 357245582Sjkim atomic_##NAME##_acq_int((volatile u_int *)p, v);\ 358245582Sjkim} \ 359245582Sjkim \ 360245582Sjkimstatic __inline void \ 361245582Sjkimatomic_##NAME##_rel_ptr(volatile void *p, uintptr_t v) \ 362245582Sjkim{ \ 363245582Sjkim atomic_##NAME##_rel_int((volatile u_int *)p, v);\ 364246040Sjkim} 365246040Sjkim 366246040SjkimATOMIC_PTR(set) 367246040SjkimATOMIC_PTR(clear) 368246040SjkimATOMIC_PTR(add) 369246040SjkimATOMIC_PTR(subtract) 370246040Sjkim 371246040Sjkim#undef ATOMIC_PTR 372245582Sjkim 373245582Sjkimstatic __inline u_int 374245582Sjkimatomic_readandclear_int(volatile u_int *addr) 375245582Sjkim{ 376246040Sjkim u_int result; 377246040Sjkim 378246040Sjkim __asm __volatile ( 379245582Sjkim " xorl %0,%0 ; " 380246040Sjkim " xchgl %1,%0 ; " 381246040Sjkim "# atomic_readandclear_int" 382246040Sjkim : "=&r" (result) /* 0 (result) */ 383245582Sjkim : "m" (*addr)); /* 1 (addr) */ 384245582Sjkim 385246040Sjkim return (result); 386246040Sjkim} 387246040Sjkim 388246040Sjkimstatic __inline u_long 389246040Sjkimatomic_readandclear_long(volatile u_long *addr) 390246040Sjkim{ 391246040Sjkim u_long result; 392246040Sjkim 393246040Sjkim __asm __volatile ( 394246040Sjkim " xorl %0,%0 ; " 395245582Sjkim " xchgl %1,%0 ; " 396245582Sjkim "# atomic_readandclear_int" 397245582Sjkim : "=&r" (result) /* 0 (result) */ 398245582Sjkim : "m" (*addr)); /* 1 (addr) */ 399245582Sjkim 400245582Sjkim return (result); 401245582Sjkim} 402245582Sjkim#endif /* !defined(WANT_FUNCTIONS) */ 403245582Sjkim#endif /* ! _MACHINE_ATOMIC_H_ */ 404246040Sjkim