atomic.h revision 241080
1252602Spjd/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ 2252602Spjd 3252603Spjd/*- 4252602Spjd * Copyright (C) 2003-2004 Olivier Houchard 51553Srgrimes * Copyright (C) 1994-1997 Mark Brinicombe 61553Srgrimes * Copyright (C) 1994 Brini 71553Srgrimes * All rights reserved. 81553Srgrimes * 91553Srgrimes * This code is derived from software written for Brini by Mark Brinicombe 101553Srgrimes * 111553Srgrimes * Redistribution and use in source and binary forms, with or without 121553Srgrimes * modification, are permitted provided that the following conditions 131553Srgrimes * are met: 141553Srgrimes * 1. Redistributions of source code must retain the above copyright 151553Srgrimes * notice, this list of conditions and the following disclaimer. 161553Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 171553Srgrimes * notice, this list of conditions and the following disclaimer in the 181553Srgrimes * documentation and/or other materials provided with the distribution. 191553Srgrimes * 3. All advertising materials mentioning features or use of this software 201553Srgrimes * must display the following acknowledgement: 211553Srgrimes * This product includes software developed by Brini. 221553Srgrimes * 4. The name of Brini may not be used to endorse or promote products 231553Srgrimes * derived from this software without specific prior written permission. 241553Srgrimes * 251553Srgrimes * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR 261553Srgrimes * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 271553Srgrimes * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 281553Srgrimes * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 291553Srgrimes * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 301553Srgrimes * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 311553Srgrimes * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 3230380Scharnier * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 331553Srgrimes * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 341553Srgrimes * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 351553Srgrimes * 361553Srgrimes * $FreeBSD: head/sys/arm/include/atomic.h 241080 2012-10-01 05:12:17Z andrew $ 37117278Scharnier */ 381553Srgrimes 391553Srgrimes#ifndef _MACHINE_ATOMIC_H_ 40117278Scharnier#define _MACHINE_ATOMIC_H_ 4130380Scharnier 421553Srgrimes#include <sys/types.h> 43117278Scharnier 44117278Scharnier#ifndef _KERNEL 45117278Scharnier#include <machine/sysarch.h> 46263234Srwatson#else 471553Srgrimes#include <machine/cpuconf.h> 481553Srgrimes#endif 491553Srgrimes 501553Srgrimes#define mb() 511553Srgrimes#define wmb() 521553Srgrimes#define rmb() 53252603Spjd 54252603Spjd#ifndef I32_bit 551553Srgrimes#define I32_bit (1 << 7) /* IRQ disable */ 561553Srgrimes#endif 571553Srgrimes#ifndef F32_bit 581553Srgrimes#define F32_bit (1 << 6) /* FIQ disable */ 591553Srgrimes#endif 6070284Siedowse 611553Srgrimes/* 621553Srgrimes * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h 631553Srgrimes * here, but that header can't be included here because this is C 6430380Scharnier * code. I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition 651553Srgrimes * out of asm.h so it can be used in both asm and C code. - kientzle@ 661553Srgrimes */ 67252602Spjd#if defined (__ARM_ARCH_7__) || \ 681553Srgrimes defined (__ARM_ARCH_7A__) || \ 691553Srgrimes defined (__ARM_ARCH_6__) || \ 70252602Spjd defined (__ARM_ARCH_6J__) || \ 711553Srgrimes defined (__ARM_ARCH_6K__) || \ 721553Srgrimes defined (__ARM_ARCH_6Z__) || \ 731553Srgrimes defined (__ARM_ARCH_6ZK__) 741553Srgrimesstatic __inline void 7599825Salfred__do_dmb(void) 76252602Spjd{ 771553Srgrimes 781553Srgrimes#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 7917832Spst __asm __volatile("dmb" : : : "memory"); 8017829Spst#else 8117829Spst __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory"); 8210087Sjkh#endif 8310087Sjkh} 8410087Sjkh 8510087Sjkh#define ATOMIC_ACQ_REL_LONG(NAME) \ 8610087Sjkhstatic __inline void \ 8710087Sjkhatomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ 8810087Sjkh{ \ 89252602Spjd atomic_##NAME##_long(p, v); \ 9010087Sjkh __do_dmb(); \ 91252602Spjd} \ 92252602Spjd \ 93252602Spjdstatic __inline void \ 94252602Spjdatomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ 95252602Spjd{ \ 96252602Spjd __do_dmb(); \ 97252602Spjd atomic_##NAME##_long(p, v); \ 9810087Sjkh} 9910087Sjkh 100252603Spjd#define ATOMIC_ACQ_REL(NAME, WIDTH) \ 1011553Srgrimesstatic __inline void \ 1021553Srgrimesatomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 103252603Spjd{ \ 1041553Srgrimes atomic_##NAME##_##WIDTH(p, v); \ 1051553Srgrimes __do_dmb(); \ 1061553Srgrimes} \ 1071553Srgrimes \ 1081553Srgrimesstatic __inline void \ 1091553Srgrimesatomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 1101553Srgrimes{ \ 1111553Srgrimes __do_dmb(); \ 1121553Srgrimes atomic_##NAME##_##WIDTH(p, v); \ 1131553Srgrimes} 114252602Spjd 1151553Srgrimesstatic __inline void 116252602Spjdatomic_set_32(volatile uint32_t *address, uint32_t setmask) 117252602Spjd{ 1181553Srgrimes uint32_t tmp = 0, tmp2 = 0; 1191553Srgrimes 1201553Srgrimes __asm __volatile("1: ldrex %0, [%2]\n" 1211553Srgrimes "orr %0, %0, %3\n" 122252602Spjd "strex %1, %0, [%2]\n" 123201060Sed "cmp %1, #0\n" 124252603Spjd "bne 1b\n" 125252603Spjd : "=&r" (tmp), "+r" (tmp2) 1261553Srgrimes , "+r" (address), "+r" (setmask) : : "cc", "memory"); 12799825Salfred 1281553Srgrimes} 129252602Spjd 130252602Spjdstatic __inline void 131252603Spjdatomic_set_long(volatile u_long *address, u_long setmask) 132252602Spjd{ 133252602Spjd u_long tmp = 0, tmp2 = 0; 134252602Spjd 135252603Spjd __asm __volatile("1: ldrex %0, [%2]\n" 136252602Spjd "orr %0, %0, %3\n" 13799825Salfred "strex %1, %0, [%2]\n" 138252602Spjd "cmp %1, #0\n" 1391553Srgrimes "bne 1b\n" 140252602Spjd : "=&r" (tmp), "+r" (tmp2) 141252602Spjd , "+r" (address), "+r" (setmask) : : "cc", "memory"); 142252602Spjd 1431553Srgrimes} 1441553Srgrimes 1451553Srgrimesstatic __inline void 146252602Spjdatomic_clear_32(volatile uint32_t *address, uint32_t setmask) 147252602Spjd{ 148252602Spjd uint32_t tmp = 0, tmp2 = 0; 149252602Spjd 150252602Spjd __asm __volatile("1: ldrex %0, [%2]\n" 151252602Spjd "bic %0, %0, %3\n" 152252602Spjd "strex %1, %0, [%2]\n" 153252602Spjd "cmp %1, #0\n" 154252602Spjd "bne 1b\n" 155252602Spjd : "=&r" (tmp), "+r" (tmp2) 156252602Spjd ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 157252602Spjd} 158252602Spjd 159252602Spjdstatic __inline void 160252602Spjdatomic_clear_long(volatile u_long *address, u_long setmask) 161252602Spjd{ 162252602Spjd u_long tmp = 0, tmp2 = 0; 163252602Spjd 164252602Spjd __asm __volatile("1: ldrex %0, [%2]\n" 165252602Spjd "bic %0, %0, %3\n" 166252602Spjd "strex %1, %0, [%2]\n" 167252602Spjd "cmp %1, #0\n" 168252602Spjd "bne 1b\n" 169252602Spjd : "=&r" (tmp), "+r" (tmp2) 170252602Spjd ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 171252602Spjd} 172252602Spjd 173252602Spjdstatic __inline u_int32_t 174252602Spjdatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 1751553Srgrimes{ 17699825Salfred uint32_t ret; 1771553Srgrimes 178252602Spjd __asm __volatile("1: ldrex %0, [%1]\n" 1791553Srgrimes "cmp %0, %2\n" 18099825Salfred "movne %0, #0\n" 18117829Spst "bne 2f\n" 18217829Spst "strex %0, %3, [%1]\n" 1831553Srgrimes "cmp %0, #0\n" 184252602Spjd "bne 1b\n" 18530380Scharnier "moveq %0, #1\n" 18630380Scharnier "2:" 18717829Spst : "=&r" (ret) 18817829Spst ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 18917829Spst "memory"); 190252602Spjd return (ret); 191252602Spjd} 19210087Sjkh 19310087Sjkhstatic __inline u_long 19410087Sjkhatomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 195252602Spjd{ 196252602Spjd u_long ret; 19710087Sjkh 19810087Sjkh __asm __volatile("1: ldrex %0, [%1]\n" 199252602Spjd "cmp %0, %2\n" 20030380Scharnier "movne %0, #0\n" 201252602Spjd "bne 2f\n" 202252602Spjd "strex %0, %3, [%1]\n" 203252602Spjd "cmp %0, #0\n" 204252602Spjd "bne 1b\n" 20510087Sjkh "moveq %0, #1\n" 206252602Spjd "2:" 20742508Ssteve : "=&r" (ret) 208252602Spjd ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 20942508Ssteve "memory"); 210252602Spjd return (ret); 21147963Sbrian} 212252602Spjd 21330380Scharnierstatic __inline u_int32_t 214252602Spjdatomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 215252602Spjd{ 216252602Spjd u_int32_t ret = atomic_cmpset_32(p, cmpval, newval); 21710087Sjkh 21830380Scharnier __do_dmb(); 21930380Scharnier return (ret); 2201553Srgrimes} 2211553Srgrimes 2221553Srgrimesstatic __inline u_long 22310087Sjkhatomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 224252605Spjd{ 22510087Sjkh u_long ret = atomic_cmpset_long(p, cmpval, newval); 22610087Sjkh 227117278Scharnier __do_dmb(); 22810087Sjkh return (ret); 22910087Sjkh} 2301553Srgrimes 23153770Scharnierstatic __inline u_int32_t 2321553Srgrimesatomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 2331553Srgrimes{ 2341553Srgrimes 2351553Srgrimes __do_dmb(); 2361553Srgrimes return (atomic_cmpset_32(p, cmpval, newval)); 2371553Srgrimes} 2381553Srgrimes 2391553Srgrimesstatic __inline u_long 2401553Srgrimesatomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 241229403Sed{ 2421553Srgrimes 243252602Spjd __do_dmb(); 2441553Srgrimes return (atomic_cmpset_long(p, cmpval, newval)); 2451553Srgrimes} 2461553Srgrimes 2471553Srgrimes 2481553Srgrimesstatic __inline void 2491553Srgrimesatomic_add_32(volatile u_int32_t *p, u_int32_t val) 2501553Srgrimes{ 2511553Srgrimes uint32_t tmp = 0, tmp2 = 0; 2521553Srgrimes 25399825Salfred __asm __volatile("1: ldrex %0, [%2]\n" 25499825Salfred "add %0, %0, %3\n" 25599825Salfred "strex %1, %0, [%2]\n" 25699825Salfred "cmp %1, #0\n" 25799825Salfred "bne 1b\n" 2581553Srgrimes : "=&r" (tmp), "+r" (tmp2) 2591553Srgrimes ,"+r" (p), "+r" (val) : : "cc", "memory"); 2601553Srgrimes} 261220969Ssimon 262220969Ssimonstatic __inline void 263220969Ssimonatomic_add_long(volatile u_long *p, u_long val) 264220969Ssimon{ 265252602Spjd u_long tmp = 0, tmp2 = 0; 266220969Ssimon 267220969Ssimon __asm __volatile("1: ldrex %0, [%2]\n" 268220969Ssimon "add %0, %0, %3\n" 269220969Ssimon "strex %1, %0, [%2]\n" 270220969Ssimon "cmp %1, #0\n" 271220969Ssimon "bne 1b\n" 272220969Ssimon : "=&r" (tmp), "+r" (tmp2) 2731553Srgrimes ,"+r" (p), "+r" (val) : : "cc", "memory"); 2741553Srgrimes} 27542508Ssteve 276252603Spjdstatic __inline void 277252603Spjdatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 278252603Spjd{ 279252603Spjd uint32_t tmp = 0, tmp2 = 0; 280252603Spjd 281252603Spjd __asm __volatile("1: ldrex %0, [%2]\n" 282255227Spjd "sub %0, %0, %3\n" 283255227Spjd "strex %1, %0, [%2]\n" 284258768Spjd "cmp %1, #0\n" 285255227Spjd "bne 1b\n" 286255227Spjd : "=&r" (tmp), "+r" (tmp2) 287255227Spjd ,"+r" (p), "+r" (val) : : "cc", "memory"); 288252603Spjd} 289252603Spjd 290252603Spjdstatic __inline void 291252603Spjdatomic_subtract_long(volatile u_long *p, u_long val) 29242508Ssteve{ 293252603Spjd u_long tmp = 0, tmp2 = 0; 294252603Spjd 295252603Spjd __asm __volatile("1: ldrex %0, [%2]\n" 296252603Spjd "sub %0, %0, %3\n" 297252603Spjd "strex %1, %0, [%2]\n" 298252603Spjd "cmp %1, #0\n" 299252603Spjd "bne 1b\n" 300252603Spjd : "=&r" (tmp), "+r" (tmp2) 301252603Spjd ,"+r" (p), "+r" (val) : : "cc", "memory"); 302252603Spjd} 303252603Spjd 304252603SpjdATOMIC_ACQ_REL(clear, 32) 305252603SpjdATOMIC_ACQ_REL(add, 32) 306252603SpjdATOMIC_ACQ_REL(subtract, 32) 307252603SpjdATOMIC_ACQ_REL(set, 32) 308252603SpjdATOMIC_ACQ_REL_LONG(clear) 309252603SpjdATOMIC_ACQ_REL_LONG(add) 310252603SpjdATOMIC_ACQ_REL_LONG(subtract) 311252603SpjdATOMIC_ACQ_REL_LONG(set) 312252603Spjd 313252603Spjd#undef ATOMIC_ACQ_REL 314252603Spjd#undef ATOMIC_ACQ_REL_LONG 315252603Spjd 316252603Spjdstatic __inline uint32_t 317252603Spjdatomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 318252603Spjd{ 319252603Spjd uint32_t tmp = 0, tmp2 = 0, ret = 0; 320252603Spjd 321252603Spjd __asm __volatile("1: ldrex %0, [%3]\n" 322252603Spjd "add %1, %0, %4\n" 323252603Spjd "strex %2, %1, [%3]\n" 324252603Spjd "cmp %2, #0\n" 325252603Spjd "bne 1b\n" 326252603Spjd : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 327252603Spjd ,"+r" (p), "+r" (val) : : "cc", "memory"); 328252603Spjd return (ret); 329252603Spjd} 330252603Spjd 331252603Spjdstatic __inline uint32_t 332252603Spjdatomic_readandclear_32(volatile u_int32_t *p) 333252603Spjd{ 334252603Spjd uint32_t ret, tmp = 0, tmp2 = 0; 335252603Spjd 336254486Spjd __asm __volatile("1: ldrex %0, [%3]\n" 337254486Spjd "mov %1, #0\n" 338254486Spjd "strex %2, %1, [%3]\n" 339252603Spjd "cmp %2, #0\n" 340254486Spjd "bne 1b\n" 341252603Spjd : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 342252603Spjd ,"+r" (p) : : "cc", "memory"); 343252603Spjd return (ret); 344252603Spjd} 345252603Spjd 346252603Spjdstatic __inline uint32_t 347252603Spjdatomic_load_acq_32(volatile uint32_t *p) 348252603Spjd{ 349252603Spjd uint32_t v; 350252603Spjd 351252603Spjd v = *p; 352252603Spjd __do_dmb(); 353255219Spjd return (v); 354252603Spjd} 355252605Spjd 356252603Spjdstatic __inline void 357252603Spjdatomic_store_rel_32(volatile uint32_t *p, uint32_t v) 358252603Spjd{ 359252603Spjd 360252603Spjd __do_dmb(); 361252603Spjd *p = v; 362252605Spjd} 363252605Spjd 364252605Spjdstatic __inline u_long 365252605Spjdatomic_fetchadd_long(volatile u_long *p, u_long val) 366252605Spjd{ 367255219Spjd u_long tmp = 0, tmp2 = 0, ret = 0; 368255219Spjd 369255219Spjd __asm __volatile("1: ldrex %0, [%3]\n" 370252605Spjd "add %1, %0, %4\n" 371252605Spjd "strex %2, %1, [%3]\n" 372252605Spjd "cmp %2, #0\n" 373252605Spjd "bne 1b\n" 374252605Spjd : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 375252605Spjd ,"+r" (p), "+r" (val) : : "cc", "memory"); 376252605Spjd return (ret); 3771553Srgrimes} 378252602Spjd 379252602Spjdstatic __inline u_long 3801553Srgrimesatomic_readandclear_long(volatile u_long *p) 3811553Srgrimes{ 3821553Srgrimes u_long ret, tmp = 0, tmp2 = 0; 3831553Srgrimes 3841553Srgrimes __asm __volatile("1: ldrex %0, [%3]\n" 38541895Sdes "mov %1, #0\n" 38670284Siedowse "strex %2, %1, [%3]\n" 38770284Siedowse "cmp %2, #0\n" 3881553Srgrimes "bne 1b\n" 3891553Srgrimes : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 39070284Siedowse ,"+r" (p) : : "cc", "memory"); 39170284Siedowse return (ret); 39270284Siedowse} 39370284Siedowse 39470284Siedowsestatic __inline u_long 3951553Srgrimesatomic_load_acq_long(volatile u_long *p) 3961553Srgrimes{ 3971553Srgrimes u_long v; 3981553Srgrimes 399252602Spjd v = *p; 40070284Siedowse __do_dmb(); 40170284Siedowse return (v); 4021553Srgrimes} 4031553Srgrimes 404252602Spjdstatic __inline void 4051553Srgrimesatomic_store_rel_long(volatile u_long *p, u_long v) 4061553Srgrimes{ 4071553Srgrimes 4081553Srgrimes __do_dmb(); 409252605Spjd *p = v; 4101553Srgrimes} 4111553Srgrimes#else /* < armv6 */ 4121553Srgrimes 4131553Srgrimes#define __with_interrupts_disabled(expr) \ 414255219Spjd do { \ 415255219Spjd u_int cpsr_save, tmp; \ 416252605Spjd \ 417252605Spjd __asm __volatile( \ 418252605Spjd "mrs %0, cpsr;" \ 4191553Srgrimes "orr %1, %0, %2;" \ 4201553Srgrimes "msr cpsr_all, %1;" \ 4211553Srgrimes : "=r" (cpsr_save), "=r" (tmp) \ 422252602Spjd : "I" (I32_bit | F32_bit) \ 4231553Srgrimes : "cc" ); \ 424252602Spjd (expr); \ 4251553Srgrimes __asm __volatile( \ 4261553Srgrimes "msr cpsr_all, %0" \ 4271553Srgrimes : /* no output */ \ 4281553Srgrimes : "r" (cpsr_save) \ 4291553Srgrimes : "cc" ); \ 4301553Srgrimes } while(0) 4311553Srgrimes 4321553Srgrimesstatic __inline uint32_t 4331553Srgrimes__swp(uint32_t val, volatile uint32_t *ptr) 4341553Srgrimes{ 4351553Srgrimes __asm __volatile("swp %0, %2, [%3]" 4361553Srgrimes : "=&r" (val), "=m" (*ptr) 4371553Srgrimes : "r" (val), "r" (ptr), "m" (*ptr) 4381553Srgrimes : "memory"); 43985640Sdillon return (val); 44089572Sdillon} 4411553Srgrimes 4421553Srgrimes 4431553Srgrimes#ifdef _KERNEL 4441553Srgrimesstatic __inline void 4451553Srgrimesatomic_set_32(volatile uint32_t *address, uint32_t setmask) 446252605Spjd{ 4471553Srgrimes __with_interrupts_disabled(*address |= setmask); 4481553Srgrimes} 44917829Spst 450252603Spjdstatic __inline void 45117829Spstatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 452252603Spjd{ 4531553Srgrimes __with_interrupts_disabled(*address &= ~clearmask); 4541553Srgrimes} 455252603Spjd 456252602Spjdstatic __inline u_int32_t 457252602Spjdatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 458252602Spjd{ 459252602Spjd int ret; 4601553Srgrimes 461252603Spjd __with_interrupts_disabled( 462252603Spjd { 463252603Spjd if (*p == cmpval) { 464252603Spjd *p = newval; 465252603Spjd ret = 1; 466252603Spjd } else { 467252603Spjd ret = 0; 468252603Spjd } 469252603Spjd }); 470252603Spjd return (ret); 471252603Spjd} 472252603Spjd 473252603Spjdstatic __inline void 474252603Spjdatomic_add_32(volatile u_int32_t *p, u_int32_t val) 475252603Spjd{ 476252603Spjd __with_interrupts_disabled(*p += val); 477252603Spjd} 478252603Spjd 479252603Spjdstatic __inline void 480252603Spjdatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 481252603Spjd{ 4821553Srgrimes __with_interrupts_disabled(*p -= val); 483252603Spjd} 484252603Spjd 485252603Spjdstatic __inline uint32_t 486252603Spjdatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 487252603Spjd{ 488252603Spjd uint32_t value; 489252603Spjd 490252603Spjd __with_interrupts_disabled( 491252603Spjd { 492252603Spjd value = *p; 493252603Spjd *p += v; 494252603Spjd }); 495252603Spjd return (value); 496252603Spjd} 497252603Spjd 498252603Spjd#else /* !_KERNEL */ 499252603Spjd 500252603Spjdstatic __inline u_int32_t 501252603Spjdatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 502252603Spjd{ 503252603Spjd register int done, ras_start = ARM_RAS_START; 504252603Spjd 505252603Spjd __asm __volatile("1:\n" 506252603Spjd "adr %1, 1b\n" 507252603Spjd "str %1, [%0]\n" 508252603Spjd "adr %1, 2f\n" 509252603Spjd "str %1, [%0, #4]\n" 510252603Spjd "ldr %1, [%2]\n" 511252603Spjd "cmp %1, %3\n" 512252603Spjd "streq %4, [%2]\n" 513252603Spjd "2:\n" 514252603Spjd "mov %1, #0\n" 515252603Spjd "str %1, [%0]\n" 516252603Spjd "mov %1, #0xffffffff\n" 517252603Spjd "str %1, [%0, #4]\n" 518252603Spjd "moveq %1, #1\n" 519252603Spjd "movne %1, #0\n" 520252603Spjd : "+r" (ras_start), "=r" (done) 521252603Spjd ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory"); 522252603Spjd return (done); 523252603Spjd} 524252603Spjd 525252603Spjdstatic __inline void 526252603Spjdatomic_add_32(volatile u_int32_t *p, u_int32_t val) 527252602Spjd{ 52810087Sjkh int start, ras_start = ARM_RAS_START; 529252602Spjd 530252603Spjd __asm __volatile("1:\n" 531252603Spjd "adr %1, 1b\n" 532252603Spjd "str %1, [%0]\n" 533252603Spjd "adr %1, 2f\n" 534252603Spjd "str %1, [%0, #4]\n" 535252603Spjd "ldr %1, [%2]\n" 536252603Spjd "add %1, %1, %3\n" 537252603Spjd "str %1, [%2]\n" 538252603Spjd "2:\n" 53910087Sjkh "mov %1, #0\n" 5401553Srgrimes "str %1, [%0]\n" 5411553Srgrimes "mov %1, #0xffffffff\n" 5421553Srgrimes "str %1, [%0, #4]\n" 543252602Spjd : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 5441553Srgrimes : : "memory"); 5451553Srgrimes} 5461553Srgrimes 5471553Srgrimesstatic __inline void 5481553Srgrimesatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 5491553Srgrimes{ 5501553Srgrimes int start, ras_start = ARM_RAS_START; 5511553Srgrimes 5521553Srgrimes __asm __volatile("1:\n" 5531553Srgrimes "adr %1, 1b\n" 5541553Srgrimes "str %1, [%0]\n" 5551553Srgrimes "adr %1, 2f\n" 55689572Sdillon "str %1, [%0, #4]\n" 5571553Srgrimes "ldr %1, [%2]\n" 5581553Srgrimes "sub %1, %1, %3\n" 5591553Srgrimes "str %1, [%2]\n" 560252602Spjd "2:\n" 5611553Srgrimes "mov %1, #0\n" 562252602Spjd "str %1, [%0]\n" 56362989Skris "mov %1, #0xffffffff\n" 5641553Srgrimes "str %1, [%0, #4]\n" 5651553Srgrimes 5661553Srgrimes : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 5671553Srgrimes : : "memory"); 568252602Spjd} 5691553Srgrimes 570252602Spjdstatic __inline void 571252602Spjdatomic_set_32(volatile uint32_t *address, uint32_t setmask) 5721553Srgrimes{ 5731553Srgrimes int start, ras_start = ARM_RAS_START; 574252602Spjd 5751553Srgrimes __asm __volatile("1:\n" 5761553Srgrimes "adr %1, 1b\n" 577252602Spjd "str %1, [%0]\n" 578252602Spjd "adr %1, 2f\n" 579128186Sluigi "str %1, [%0, #4]\n" 5801553Srgrimes "ldr %1, [%2]\n" 5811553Srgrimes "orr %1, %1, %3\n" 5821553Srgrimes "str %1, [%2]\n" 5831553Srgrimes "2:\n" 5841553Srgrimes "mov %1, #0\n" 5851553Srgrimes "str %1, [%0]\n" 5861553Srgrimes "mov %1, #0xffffffff\n" 5871553Srgrimes "str %1, [%0, #4]\n" 588252602Spjd 5891553Srgrimes : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask) 590252602Spjd : : "memory"); 591252602Spjd} 592252602Spjd 5931553Srgrimesstatic __inline void 5941553Srgrimesatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 595252602Spjd{ 5961553Srgrimes int start, ras_start = ARM_RAS_START; 5971553Srgrimes 5981553Srgrimes __asm __volatile("1:\n" 599252602Spjd "adr %1, 1b\n" 60010087Sjkh "str %1, [%0]\n" 60110087Sjkh "adr %1, 2f\n" 60210087Sjkh "str %1, [%0, #4]\n" 60310087Sjkh "ldr %1, [%2]\n" 60410087Sjkh "bic %1, %1, %3\n" 60510087Sjkh "str %1, [%2]\n" 60610087Sjkh "2:\n" 60710087Sjkh "mov %1, #0\n" 60810087Sjkh "str %1, [%0]\n" 60910087Sjkh "mov %1, #0xffffffff\n" 61010087Sjkh "str %1, [%0, #4]\n" 61199825Salfred : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask) 612252602Spjd : : "memory"); 61310087Sjkh 614252602Spjd} 615252602Spjd 61610087Sjkhstatic __inline uint32_t 61710087Sjkhatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 618252602Spjd{ 619252602Spjd uint32_t start, tmp, ras_start = ARM_RAS_START; 62010087Sjkh 621252602Spjd __asm __volatile("1:\n" 622252602Spjd "adr %1, 1b\n" 62310087Sjkh "str %1, [%0]\n" 624252602Spjd "adr %1, 2f\n" 62510087Sjkh "str %1, [%0, #4]\n" 62610087Sjkh "ldr %1, [%3]\n" 6271553Srgrimes "mov %2, %1\n" 6281553Srgrimes "add %2, %2, %4\n" 6291553Srgrimes "str %2, [%3]\n" 6301553Srgrimes "2:\n" 6311553Srgrimes "mov %2, #0\n" 6321553Srgrimes "str %2, [%0]\n" 6331553Srgrimes "mov %2, #0xffffffff\n" 6341553Srgrimes "str %2, [%0, #4]\n" 6351553Srgrimes : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v) 6361553Srgrimes : : "memory"); 6371553Srgrimes return (start); 6381553Srgrimes} 6391553Srgrimes 6401553Srgrimes#endif /* _KERNEL */ 6411553Srgrimes 6421553Srgrimes 6431553Srgrimesstatic __inline uint32_t 6441553Srgrimesatomic_readandclear_32(volatile u_int32_t *p) 6451553Srgrimes{ 6461553Srgrimes 6471553Srgrimes return (__swp(0, p)); 6481553Srgrimes} 649252602Spjd 6501553Srgrimes#define atomic_cmpset_rel_32 atomic_cmpset_32 651252602Spjd#define atomic_cmpset_acq_32 atomic_cmpset_32 652252602Spjd#define atomic_set_rel_32 atomic_set_32 653252602Spjd#define atomic_set_acq_32 atomic_set_32 654252602Spjd#define atomic_clear_rel_32 atomic_clear_32 655252602Spjd#define atomic_clear_acq_32 atomic_clear_32 6561553Srgrimes#define atomic_add_rel_32 atomic_add_32 6571553Srgrimes#define atomic_add_acq_32 atomic_add_32 6581553Srgrimes#define atomic_subtract_rel_32 atomic_subtract_32 6591553Srgrimes#define atomic_subtract_acq_32 atomic_subtract_32 6601553Srgrimes#define atomic_store_rel_32 atomic_store_32 661252602Spjd#define atomic_store_rel_long atomic_store_long 6621553Srgrimes#define atomic_load_acq_32 atomic_load_32 663252602Spjd#define atomic_load_acq_long atomic_load_long 664252602Spjd#undef __with_interrupts_disabled 665252602Spjd 666252602Spjdstatic __inline void 6671553Srgrimesatomic_add_long(volatile u_long *p, u_long v) 6681553Srgrimes{ 6691553Srgrimes 670252602Spjd atomic_add_32((volatile uint32_t *)p, v); 6711553Srgrimes} 672252602Spjd 673252602Spjdstatic __inline void 6741553Srgrimesatomic_clear_long(volatile u_long *p, u_long v) 675252602Spjd{ 676252602Spjd 6771553Srgrimes atomic_clear_32((volatile uint32_t *)p, v); 6781553Srgrimes} 6791553Srgrimes 680252602Spjdstatic __inline int 6811553Srgrimesatomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe) 6821553Srgrimes{ 6831553Srgrimes 6841553Srgrimes return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe)); 6851553Srgrimes} 6861553Srgrimes 6871553Srgrimesstatic __inline u_long 68810087Sjkhatomic_fetchadd_long(volatile u_long *p, u_long v) 68910087Sjkh{ 69010087Sjkh 691252602Spjd return (atomic_fetchadd_32((volatile uint32_t *)p, v)); 692252602Spjd} 69310087Sjkh 69410087Sjkhstatic __inline void 69510087Sjkhatomic_readandclear_long(volatile u_long *p) 696252602Spjd{ 69710087Sjkh 69810087Sjkh atomic_readandclear_32((volatile uint32_t *)p); 699252602Spjd} 70010087Sjkh 701252602Spjdstatic __inline void 70210087Sjkhatomic_set_long(volatile u_long *p, u_long v) 70310087Sjkh{ 70410087Sjkh 70510087Sjkh atomic_set_32((volatile uint32_t *)p, v); 70610087Sjkh} 70710087Sjkh 708252602Spjdstatic __inline void 70910087Sjkhatomic_subtract_long(volatile u_long *p, u_long v) 71010087Sjkh{ 71110087Sjkh 71210087Sjkh atomic_subtract_32((volatile uint32_t *)p, v); 7131553Srgrimes} 7141553Srgrimes 7151553Srgrimes 7161553Srgrimes 7171553Srgrimes#endif /* Arch >= v6 */ 7181553Srgrimes 7191553Srgrimesstatic __inline int 7201553Srgrimesatomic_load_32(volatile uint32_t *v) 7211553Srgrimes{ 722252602Spjd 723252602Spjd return (*v); 7241553Srgrimes} 725252602Spjd 726252602Spjdstatic __inline void 727252602Spjdatomic_store_32(volatile uint32_t *dst, uint32_t src) 7281553Srgrimes{ 729252602Spjd *dst = src; 730252602Spjd} 73117829Spst 732252602Spjdstatic __inline int 7331553Srgrimesatomic_load_long(volatile u_long *v) 734252602Spjd{ 7351553Srgrimes 7361553Srgrimes return (*v); 7371553Srgrimes} 7381553Srgrimes 7391553Srgrimesstatic __inline void 74089572Sdillonatomic_store_long(volatile u_long *dst, u_long src) 741252602Spjd{ 742252602Spjd *dst = src; 743252602Spjd} 7441553Srgrimes 745252602Spjd#define atomic_add_acq_long atomic_add_long 746252602Spjd#define atomic_add_rel_long atomic_add_long 747252602Spjd#define atomic_subtract_acq_long atomic_subtract_long 7481553Srgrimes#define atomic_subtract_rel_long atomic_subtract_long 7491553Srgrimes#define atomic_clear_acq_long atomic_clear_long 7501553Srgrimes#define atomic_clear_rel_long atomic_clear_long 7511553Srgrimes#define atomic_set_acq_long atomic_set_long 7521553Srgrimes#define atomic_set_rel_long atomic_set_long 7531553Srgrimes#define atomic_cmpset_acq_long atomic_cmpset_long 7541553Srgrimes#define atomic_cmpset_rel_long atomic_cmpset_long 7551553Srgrimes#define atomic_load_acq_long atomic_load_long 7561553Srgrimes 7571553Srgrimes#define atomic_clear_ptr atomic_clear_32 7581553Srgrimes#define atomic_set_ptr atomic_set_32 759252602Spjd#define atomic_cmpset_ptr atomic_cmpset_32 7601553Srgrimes#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 7611553Srgrimes#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 7621553Srgrimes#define atomic_store_ptr atomic_store_32 7631553Srgrimes#define atomic_store_rel_ptr atomic_store_ptr 764252602Spjd 7651553Srgrimes#define atomic_add_int atomic_add_32 7661553Srgrimes#define atomic_add_acq_int atomic_add_acq_32 7671553Srgrimes#define atomic_add_rel_int atomic_add_rel_32 7681553Srgrimes#define atomic_subtract_int atomic_subtract_32 769252602Spjd#define atomic_subtract_acq_int atomic_subtract_acq_32 770252602Spjd#define atomic_subtract_rel_int atomic_subtract_rel_32 771252602Spjd#define atomic_clear_int atomic_clear_32 772252602Spjd#define atomic_clear_acq_int atomic_clear_acq_32 773252602Spjd#define atomic_clear_rel_int atomic_clear_rel_32 7741553Srgrimes#define atomic_set_int atomic_set_32 7751553Srgrimes#define atomic_set_acq_int atomic_set_acq_32 776252602Spjd#define atomic_set_rel_int atomic_set_rel_32 7771553Srgrimes#define atomic_cmpset_int atomic_cmpset_32 7781553Srgrimes#define atomic_cmpset_acq_int atomic_cmpset_acq_32 779252602Spjd#define atomic_cmpset_rel_int atomic_cmpset_rel_32 7801553Srgrimes#define atomic_fetchadd_int atomic_fetchadd_32 7811553Srgrimes#define atomic_readandclear_int atomic_readandclear_32 7821553Srgrimes#define atomic_load_acq_int atomic_load_acq_32 783#define atomic_store_rel_int atomic_store_rel_32 784 785#endif /* _MACHINE_ATOMIC_H_ */ 786