atomic.h revision 241080
1129198Scognet/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ 2129198Scognet 3139735Simp/*- 4129198Scognet * Copyright (C) 2003-2004 Olivier Houchard 5129198Scognet * Copyright (C) 1994-1997 Mark Brinicombe 6129198Scognet * Copyright (C) 1994 Brini 7129198Scognet * All rights reserved. 8129198Scognet * 9129198Scognet * This code is derived from software written for Brini by Mark Brinicombe 10129198Scognet * 11129198Scognet * Redistribution and use in source and binary forms, with or without 12129198Scognet * modification, are permitted provided that the following conditions 13129198Scognet * are met: 14129198Scognet * 1. Redistributions of source code must retain the above copyright 15129198Scognet * notice, this list of conditions and the following disclaimer. 16129198Scognet * 2. Redistributions in binary form must reproduce the above copyright 17129198Scognet * notice, this list of conditions and the following disclaimer in the 18129198Scognet * documentation and/or other materials provided with the distribution. 19129198Scognet * 3. All advertising materials mentioning features or use of this software 20129198Scognet * must display the following acknowledgement: 21129198Scognet * This product includes software developed by Brini. 22129198Scognet * 4. The name of Brini may not be used to endorse or promote products 23129198Scognet * derived from this software without specific prior written permission. 24129198Scognet * 25129198Scognet * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR 26129198Scognet * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27129198Scognet * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28129198Scognet * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29129198Scognet * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 30129198Scognet * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 31129198Scognet * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32129198Scognet * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 33129198Scognet * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 34129198Scognet * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35129198Scognet * 36129198Scognet * $FreeBSD: head/sys/arm/include/atomic.h 241080 2012-10-01 05:12:17Z andrew $ 37129198Scognet */ 38129198Scognet 39129198Scognet#ifndef _MACHINE_ATOMIC_H_ 40129198Scognet#define _MACHINE_ATOMIC_H_ 41129198Scognet 42129198Scognet#include <sys/types.h> 43129198Scognet 44175982Sraj#ifndef _KERNEL 45175982Sraj#include <machine/sysarch.h> 46239268Sgonzo#else 47239268Sgonzo#include <machine/cpuconf.h> 48175982Sraj#endif 49175982Sraj 50239268Sgonzo#define mb() 51239268Sgonzo#define wmb() 52239268Sgonzo#define rmb() 53185162Skmacy 54129198Scognet#ifndef I32_bit 55129198Scognet#define I32_bit (1 << 7) /* IRQ disable */ 56129198Scognet#endif 57129198Scognet#ifndef F32_bit 58129198Scognet#define F32_bit (1 << 6) /* FIQ disable */ 59129198Scognet#endif 60129198Scognet 61239268Sgonzo/* 62239268Sgonzo * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h 63239268Sgonzo * here, but that header can't be included here because this is C 64239268Sgonzo * code. I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition 65239268Sgonzo * out of asm.h so it can be used in both asm and C code. - kientzle@ 66239268Sgonzo */ 67239268Sgonzo#if defined (__ARM_ARCH_7__) || \ 68239268Sgonzo defined (__ARM_ARCH_7A__) || \ 69239268Sgonzo defined (__ARM_ARCH_6__) || \ 70239268Sgonzo defined (__ARM_ARCH_6J__) || \ 71239268Sgonzo defined (__ARM_ARCH_6K__) || \ 72239268Sgonzo defined (__ARM_ARCH_6Z__) || \ 73239268Sgonzo defined (__ARM_ARCH_6ZK__) 74239268Sgonzostatic __inline void 75239268Sgonzo__do_dmb(void) 76239268Sgonzo{ 77239268Sgonzo 78239268Sgonzo#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 79239268Sgonzo __asm __volatile("dmb" : : : "memory"); 80239268Sgonzo#else 81239268Sgonzo __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory"); 82239268Sgonzo#endif 83239268Sgonzo} 84239268Sgonzo 85239268Sgonzo#define ATOMIC_ACQ_REL_LONG(NAME) \ 86239268Sgonzostatic __inline void \ 87239268Sgonzoatomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ 88239268Sgonzo{ \ 89239268Sgonzo atomic_##NAME##_long(p, v); \ 90239268Sgonzo __do_dmb(); \ 91239268Sgonzo} \ 92239268Sgonzo \ 93239268Sgonzostatic __inline void \ 94239268Sgonzoatomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ 95239268Sgonzo{ \ 96239268Sgonzo __do_dmb(); \ 97239268Sgonzo atomic_##NAME##_long(p, v); \ 98239268Sgonzo} 99239268Sgonzo 100239268Sgonzo#define ATOMIC_ACQ_REL(NAME, WIDTH) \ 101239268Sgonzostatic __inline void \ 102239268Sgonzoatomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 103239268Sgonzo{ \ 104239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 105239268Sgonzo __do_dmb(); \ 106239268Sgonzo} \ 107239268Sgonzo \ 108239268Sgonzostatic __inline void \ 109239268Sgonzoatomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 110239268Sgonzo{ \ 111239268Sgonzo __do_dmb(); \ 112239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 113239268Sgonzo} 114239268Sgonzo 115239268Sgonzostatic __inline void 116239268Sgonzoatomic_set_32(volatile uint32_t *address, uint32_t setmask) 117239268Sgonzo{ 118239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 119239268Sgonzo 120239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 121239268Sgonzo "orr %0, %0, %3\n" 122239268Sgonzo "strex %1, %0, [%2]\n" 123239268Sgonzo "cmp %1, #0\n" 124239268Sgonzo "bne 1b\n" 125239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 126241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 127239268Sgonzo 128239268Sgonzo} 129239268Sgonzo 130239268Sgonzostatic __inline void 131239268Sgonzoatomic_set_long(volatile u_long *address, u_long setmask) 132239268Sgonzo{ 133239268Sgonzo u_long tmp = 0, tmp2 = 0; 134239268Sgonzo 135239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 136239268Sgonzo "orr %0, %0, %3\n" 137239268Sgonzo "strex %1, %0, [%2]\n" 138239268Sgonzo "cmp %1, #0\n" 139239268Sgonzo "bne 1b\n" 140239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 141241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 142239268Sgonzo 143239268Sgonzo} 144239268Sgonzo 145239268Sgonzostatic __inline void 146239268Sgonzoatomic_clear_32(volatile uint32_t *address, uint32_t setmask) 147239268Sgonzo{ 148239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 149239268Sgonzo 150239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 151239268Sgonzo "bic %0, %0, %3\n" 152239268Sgonzo "strex %1, %0, [%2]\n" 153239268Sgonzo "cmp %1, #0\n" 154239268Sgonzo "bne 1b\n" 155239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 156241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 157239268Sgonzo} 158239268Sgonzo 159239268Sgonzostatic __inline void 160239268Sgonzoatomic_clear_long(volatile u_long *address, u_long setmask) 161239268Sgonzo{ 162239268Sgonzo u_long tmp = 0, tmp2 = 0; 163239268Sgonzo 164239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 165239268Sgonzo "bic %0, %0, %3\n" 166239268Sgonzo "strex %1, %0, [%2]\n" 167239268Sgonzo "cmp %1, #0\n" 168239268Sgonzo "bne 1b\n" 169239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 170241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 171239268Sgonzo} 172239268Sgonzo 173239268Sgonzostatic __inline u_int32_t 174239268Sgonzoatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 175239268Sgonzo{ 176239268Sgonzo uint32_t ret; 177239268Sgonzo 178239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 179239268Sgonzo "cmp %0, %2\n" 180239268Sgonzo "movne %0, #0\n" 181239268Sgonzo "bne 2f\n" 182239268Sgonzo "strex %0, %3, [%1]\n" 183239268Sgonzo "cmp %0, #0\n" 184239268Sgonzo "bne 1b\n" 185239268Sgonzo "moveq %0, #1\n" 186239268Sgonzo "2:" 187239268Sgonzo : "=&r" (ret) 188241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 189241080Sandrew "memory"); 190239268Sgonzo return (ret); 191239268Sgonzo} 192239268Sgonzo 193239268Sgonzostatic __inline u_long 194239268Sgonzoatomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 195239268Sgonzo{ 196239268Sgonzo u_long ret; 197239268Sgonzo 198239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 199239268Sgonzo "cmp %0, %2\n" 200239268Sgonzo "movne %0, #0\n" 201239268Sgonzo "bne 2f\n" 202239268Sgonzo "strex %0, %3, [%1]\n" 203239268Sgonzo "cmp %0, #0\n" 204239268Sgonzo "bne 1b\n" 205239268Sgonzo "moveq %0, #1\n" 206239268Sgonzo "2:" 207239268Sgonzo : "=&r" (ret) 208241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 209241080Sandrew "memory"); 210239268Sgonzo return (ret); 211239268Sgonzo} 212239268Sgonzo 213239268Sgonzostatic __inline u_int32_t 214239268Sgonzoatomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 215239268Sgonzo{ 216239268Sgonzo u_int32_t ret = atomic_cmpset_32(p, cmpval, newval); 217239268Sgonzo 218239268Sgonzo __do_dmb(); 219239268Sgonzo return (ret); 220239268Sgonzo} 221239268Sgonzo 222239268Sgonzostatic __inline u_long 223239268Sgonzoatomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 224239268Sgonzo{ 225239268Sgonzo u_long ret = atomic_cmpset_long(p, cmpval, newval); 226239268Sgonzo 227239268Sgonzo __do_dmb(); 228239268Sgonzo return (ret); 229239268Sgonzo} 230239268Sgonzo 231239268Sgonzostatic __inline u_int32_t 232239268Sgonzoatomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 233239268Sgonzo{ 234239268Sgonzo 235239268Sgonzo __do_dmb(); 236239268Sgonzo return (atomic_cmpset_32(p, cmpval, newval)); 237239268Sgonzo} 238239268Sgonzo 239239268Sgonzostatic __inline u_long 240239268Sgonzoatomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 241239268Sgonzo{ 242239268Sgonzo 243239268Sgonzo __do_dmb(); 244239268Sgonzo return (atomic_cmpset_long(p, cmpval, newval)); 245239268Sgonzo} 246239268Sgonzo 247239268Sgonzo 248239268Sgonzostatic __inline void 249239268Sgonzoatomic_add_32(volatile u_int32_t *p, u_int32_t val) 250239268Sgonzo{ 251239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 252239268Sgonzo 253239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 254239268Sgonzo "add %0, %0, %3\n" 255239268Sgonzo "strex %1, %0, [%2]\n" 256239268Sgonzo "cmp %1, #0\n" 257239268Sgonzo "bne 1b\n" 258239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 259241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 260239268Sgonzo} 261239268Sgonzo 262239268Sgonzostatic __inline void 263239268Sgonzoatomic_add_long(volatile u_long *p, u_long val) 264239268Sgonzo{ 265239268Sgonzo u_long tmp = 0, tmp2 = 0; 266239268Sgonzo 267239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 268239268Sgonzo "add %0, %0, %3\n" 269239268Sgonzo "strex %1, %0, [%2]\n" 270239268Sgonzo "cmp %1, #0\n" 271239268Sgonzo "bne 1b\n" 272239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 273241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 274239268Sgonzo} 275239268Sgonzo 276239268Sgonzostatic __inline void 277239268Sgonzoatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 278239268Sgonzo{ 279239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 280239268Sgonzo 281239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 282239268Sgonzo "sub %0, %0, %3\n" 283239268Sgonzo "strex %1, %0, [%2]\n" 284239268Sgonzo "cmp %1, #0\n" 285239268Sgonzo "bne 1b\n" 286239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 287241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 288239268Sgonzo} 289239268Sgonzo 290239268Sgonzostatic __inline void 291239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long val) 292239268Sgonzo{ 293239268Sgonzo u_long tmp = 0, tmp2 = 0; 294239268Sgonzo 295239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 296239268Sgonzo "sub %0, %0, %3\n" 297239268Sgonzo "strex %1, %0, [%2]\n" 298239268Sgonzo "cmp %1, #0\n" 299239268Sgonzo "bne 1b\n" 300239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 301241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 302239268Sgonzo} 303239268Sgonzo 304239268SgonzoATOMIC_ACQ_REL(clear, 32) 305239268SgonzoATOMIC_ACQ_REL(add, 32) 306239268SgonzoATOMIC_ACQ_REL(subtract, 32) 307239268SgonzoATOMIC_ACQ_REL(set, 32) 308239268SgonzoATOMIC_ACQ_REL_LONG(clear) 309239268SgonzoATOMIC_ACQ_REL_LONG(add) 310239268SgonzoATOMIC_ACQ_REL_LONG(subtract) 311239268SgonzoATOMIC_ACQ_REL_LONG(set) 312239268Sgonzo 313239268Sgonzo#undef ATOMIC_ACQ_REL 314239268Sgonzo#undef ATOMIC_ACQ_REL_LONG 315239268Sgonzo 316239268Sgonzostatic __inline uint32_t 317239268Sgonzoatomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 318239268Sgonzo{ 319239268Sgonzo uint32_t tmp = 0, tmp2 = 0, ret = 0; 320239268Sgonzo 321239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 322239268Sgonzo "add %1, %0, %4\n" 323239268Sgonzo "strex %2, %1, [%3]\n" 324239268Sgonzo "cmp %2, #0\n" 325239268Sgonzo "bne 1b\n" 326239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 327241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 328239268Sgonzo return (ret); 329239268Sgonzo} 330239268Sgonzo 331239268Sgonzostatic __inline uint32_t 332239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 333239268Sgonzo{ 334239268Sgonzo uint32_t ret, tmp = 0, tmp2 = 0; 335239268Sgonzo 336239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 337239268Sgonzo "mov %1, #0\n" 338239268Sgonzo "strex %2, %1, [%3]\n" 339239268Sgonzo "cmp %2, #0\n" 340239268Sgonzo "bne 1b\n" 341239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 342241080Sandrew ,"+r" (p) : : "cc", "memory"); 343239268Sgonzo return (ret); 344239268Sgonzo} 345239268Sgonzo 346239268Sgonzostatic __inline uint32_t 347239268Sgonzoatomic_load_acq_32(volatile uint32_t *p) 348239268Sgonzo{ 349239268Sgonzo uint32_t v; 350239268Sgonzo 351239268Sgonzo v = *p; 352239268Sgonzo __do_dmb(); 353239268Sgonzo return (v); 354239268Sgonzo} 355239268Sgonzo 356239268Sgonzostatic __inline void 357239268Sgonzoatomic_store_rel_32(volatile uint32_t *p, uint32_t v) 358239268Sgonzo{ 359239268Sgonzo 360239268Sgonzo __do_dmb(); 361239268Sgonzo *p = v; 362239268Sgonzo} 363239268Sgonzo 364239268Sgonzostatic __inline u_long 365239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long val) 366239268Sgonzo{ 367239268Sgonzo u_long tmp = 0, tmp2 = 0, ret = 0; 368239268Sgonzo 369239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 370239268Sgonzo "add %1, %0, %4\n" 371239268Sgonzo "strex %2, %1, [%3]\n" 372239268Sgonzo "cmp %2, #0\n" 373239268Sgonzo "bne 1b\n" 374239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 375241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 376239268Sgonzo return (ret); 377239268Sgonzo} 378239268Sgonzo 379239268Sgonzostatic __inline u_long 380239268Sgonzoatomic_readandclear_long(volatile u_long *p) 381239268Sgonzo{ 382239268Sgonzo u_long ret, tmp = 0, tmp2 = 0; 383239268Sgonzo 384239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 385239268Sgonzo "mov %1, #0\n" 386239268Sgonzo "strex %2, %1, [%3]\n" 387239268Sgonzo "cmp %2, #0\n" 388239268Sgonzo "bne 1b\n" 389239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 390241080Sandrew ,"+r" (p) : : "cc", "memory"); 391239268Sgonzo return (ret); 392239268Sgonzo} 393239268Sgonzo 394239268Sgonzostatic __inline u_long 395239268Sgonzoatomic_load_acq_long(volatile u_long *p) 396239268Sgonzo{ 397239268Sgonzo u_long v; 398239268Sgonzo 399239268Sgonzo v = *p; 400239268Sgonzo __do_dmb(); 401239268Sgonzo return (v); 402239268Sgonzo} 403239268Sgonzo 404239268Sgonzostatic __inline void 405239268Sgonzoatomic_store_rel_long(volatile u_long *p, u_long v) 406239268Sgonzo{ 407239268Sgonzo 408239268Sgonzo __do_dmb(); 409239268Sgonzo *p = v; 410239268Sgonzo} 411239268Sgonzo#else /* < armv6 */ 412239268Sgonzo 413129198Scognet#define __with_interrupts_disabled(expr) \ 414129198Scognet do { \ 415129198Scognet u_int cpsr_save, tmp; \ 416129198Scognet \ 417129198Scognet __asm __volatile( \ 418129198Scognet "mrs %0, cpsr;" \ 419129198Scognet "orr %1, %0, %2;" \ 420129198Scognet "msr cpsr_all, %1;" \ 421129198Scognet : "=r" (cpsr_save), "=r" (tmp) \ 422157725Scognet : "I" (I32_bit | F32_bit) \ 423129198Scognet : "cc" ); \ 424129198Scognet (expr); \ 425129198Scognet __asm __volatile( \ 426129198Scognet "msr cpsr_all, %0" \ 427129198Scognet : /* no output */ \ 428129198Scognet : "r" (cpsr_save) \ 429129198Scognet : "cc" ); \ 430129198Scognet } while(0) 431129198Scognet 432137222Scognetstatic __inline uint32_t 433137222Scognet__swp(uint32_t val, volatile uint32_t *ptr) 434129198Scognet{ 435148453Sjhb __asm __volatile("swp %0, %2, [%3]" 436148453Sjhb : "=&r" (val), "=m" (*ptr) 437151340Sjhb : "r" (val), "r" (ptr), "m" (*ptr) 438148453Sjhb : "memory"); 439137222Scognet return (val); 440129198Scognet} 441129198Scognet 442137222Scognet 443144761Scognet#ifdef _KERNEL 444129198Scognetstatic __inline void 445137222Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 446129198Scognet{ 447144761Scognet __with_interrupts_disabled(*address |= setmask); 448129198Scognet} 449129198Scognet 450129198Scognetstatic __inline void 451129198Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 452129198Scognet{ 453144761Scognet __with_interrupts_disabled(*address &= ~clearmask); 454129198Scognet} 455129198Scognet 456144761Scognetstatic __inline u_int32_t 457144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 458129198Scognet{ 459144761Scognet int ret; 460144761Scognet 461144761Scognet __with_interrupts_disabled( 462144761Scognet { 463144761Scognet if (*p == cmpval) { 464144761Scognet *p = newval; 465144761Scognet ret = 1; 466144761Scognet } else { 467144761Scognet ret = 0; 468144761Scognet } 469144761Scognet }); 470144761Scognet return (ret); 471129198Scognet} 472129198Scognet 473129198Scognetstatic __inline void 474144761Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 475129198Scognet{ 476144761Scognet __with_interrupts_disabled(*p += val); 477129198Scognet} 478129198Scognet 479144761Scognetstatic __inline void 480144761Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 481129198Scognet{ 482144761Scognet __with_interrupts_disabled(*p -= val); 483129198Scognet} 484129198Scognet 485150627Sjhbstatic __inline uint32_t 486150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 487150627Sjhb{ 488150627Sjhb uint32_t value; 489150627Sjhb 490150627Sjhb __with_interrupts_disabled( 491150627Sjhb { 492150627Sjhb value = *p; 493150627Sjhb *p += v; 494150627Sjhb }); 495150627Sjhb return (value); 496150627Sjhb} 497150627Sjhb 498144761Scognet#else /* !_KERNEL */ 499144761Scognet 500129198Scognetstatic __inline u_int32_t 501144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 502129198Scognet{ 503175982Sraj register int done, ras_start = ARM_RAS_START; 504144761Scognet 505144761Scognet __asm __volatile("1:\n" 506174170Scognet "adr %1, 1b\n" 507174170Scognet "str %1, [%0]\n" 508144761Scognet "adr %1, 2f\n" 509175982Sraj "str %1, [%0, #4]\n" 510155355Scognet "ldr %1, [%2]\n" 511144761Scognet "cmp %1, %3\n" 512155355Scognet "streq %4, [%2]\n" 513144761Scognet "2:\n" 514146591Scognet "mov %1, #0\n" 515146591Scognet "str %1, [%0]\n" 516174170Scognet "mov %1, #0xffffffff\n" 517175982Sraj "str %1, [%0, #4]\n" 518144761Scognet "moveq %1, #1\n" 519144761Scognet "movne %1, #0\n" 520175982Sraj : "+r" (ras_start), "=r" (done) 521241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory"); 522137282Scognet return (done); 523129198Scognet} 524129198Scognet 525129198Scognetstatic __inline void 526129198Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 527129198Scognet{ 528175982Sraj int start, ras_start = ARM_RAS_START; 529144761Scognet 530144761Scognet __asm __volatile("1:\n" 531174170Scognet "adr %1, 1b\n" 532174170Scognet "str %1, [%0]\n" 533144761Scognet "adr %1, 2f\n" 534175982Sraj "str %1, [%0, #4]\n" 535155355Scognet "ldr %1, [%2]\n" 536144761Scognet "add %1, %1, %3\n" 537155355Scognet "str %1, [%2]\n" 538144761Scognet "2:\n" 539146591Scognet "mov %1, #0\n" 540146591Scognet "str %1, [%0]\n" 541174170Scognet "mov %1, #0xffffffff\n" 542175982Sraj "str %1, [%0, #4]\n" 543175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 544155391Scognet : : "memory"); 545129198Scognet} 546129198Scognet 547129198Scognetstatic __inline void 548129198Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 549129198Scognet{ 550175982Sraj int start, ras_start = ARM_RAS_START; 551144761Scognet 552144761Scognet __asm __volatile("1:\n" 553174170Scognet "adr %1, 1b\n" 554174170Scognet "str %1, [%0]\n" 555144761Scognet "adr %1, 2f\n" 556175982Sraj "str %1, [%0, #4]\n" 557155355Scognet "ldr %1, [%2]\n" 558144761Scognet "sub %1, %1, %3\n" 559155355Scognet "str %1, [%2]\n" 560144761Scognet "2:\n" 561146591Scognet "mov %1, #0\n" 562146591Scognet "str %1, [%0]\n" 563174170Scognet "mov %1, #0xffffffff\n" 564175982Sraj "str %1, [%0, #4]\n" 565146591Scognet 566175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 567155391Scognet : : "memory"); 568129198Scognet} 569129198Scognet 570144761Scognetstatic __inline void 571144761Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 572144761Scognet{ 573175982Sraj int start, ras_start = ARM_RAS_START; 574144761Scognet 575144761Scognet __asm __volatile("1:\n" 576174170Scognet "adr %1, 1b\n" 577174170Scognet "str %1, [%0]\n" 578144761Scognet "adr %1, 2f\n" 579175982Sraj "str %1, [%0, #4]\n" 580155355Scognet "ldr %1, [%2]\n" 581144761Scognet "orr %1, %1, %3\n" 582155355Scognet "str %1, [%2]\n" 583144761Scognet "2:\n" 584146591Scognet "mov %1, #0\n" 585146591Scognet "str %1, [%0]\n" 586174170Scognet "mov %1, #0xffffffff\n" 587175982Sraj "str %1, [%0, #4]\n" 588146591Scognet 589175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask) 590155391Scognet : : "memory"); 591144761Scognet} 592144761Scognet 593144761Scognetstatic __inline void 594144761Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 595144761Scognet{ 596175982Sraj int start, ras_start = ARM_RAS_START; 597144761Scognet 598144761Scognet __asm __volatile("1:\n" 599174170Scognet "adr %1, 1b\n" 600174170Scognet "str %1, [%0]\n" 601144761Scognet "adr %1, 2f\n" 602175982Sraj "str %1, [%0, #4]\n" 603155355Scognet "ldr %1, [%2]\n" 604144761Scognet "bic %1, %1, %3\n" 605155355Scognet "str %1, [%2]\n" 606144761Scognet "2:\n" 607146591Scognet "mov %1, #0\n" 608146591Scognet "str %1, [%0]\n" 609174170Scognet "mov %1, #0xffffffff\n" 610175982Sraj "str %1, [%0, #4]\n" 611175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask) 612155391Scognet : : "memory"); 613144761Scognet 614144761Scognet} 615150627Sjhb 616150627Sjhbstatic __inline uint32_t 617150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 618150627Sjhb{ 619190603Scognet uint32_t start, tmp, ras_start = ARM_RAS_START; 620150627Sjhb 621150627Sjhb __asm __volatile("1:\n" 622174170Scognet "adr %1, 1b\n" 623174170Scognet "str %1, [%0]\n" 624150627Sjhb "adr %1, 2f\n" 625175982Sraj "str %1, [%0, #4]\n" 626190603Scognet "ldr %1, [%3]\n" 627190603Scognet "mov %2, %1\n" 628190603Scognet "add %2, %2, %4\n" 629190603Scognet "str %2, [%3]\n" 630150627Sjhb "2:\n" 631190603Scognet "mov %2, #0\n" 632190603Scognet "str %2, [%0]\n" 633190603Scognet "mov %2, #0xffffffff\n" 634190603Scognet "str %2, [%0, #4]\n" 635190603Scognet : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v) 636155391Scognet : : "memory"); 637150627Sjhb return (start); 638150627Sjhb} 639150627Sjhb 640144761Scognet#endif /* _KERNEL */ 641144761Scognet 642239268Sgonzo 643239268Sgonzostatic __inline uint32_t 644239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 645239268Sgonzo{ 646239268Sgonzo 647239268Sgonzo return (__swp(0, p)); 648239268Sgonzo} 649239268Sgonzo 650239268Sgonzo#define atomic_cmpset_rel_32 atomic_cmpset_32 651239268Sgonzo#define atomic_cmpset_acq_32 atomic_cmpset_32 652239268Sgonzo#define atomic_set_rel_32 atomic_set_32 653239268Sgonzo#define atomic_set_acq_32 atomic_set_32 654239268Sgonzo#define atomic_clear_rel_32 atomic_clear_32 655239268Sgonzo#define atomic_clear_acq_32 atomic_clear_32 656239268Sgonzo#define atomic_add_rel_32 atomic_add_32 657239268Sgonzo#define atomic_add_acq_32 atomic_add_32 658239268Sgonzo#define atomic_subtract_rel_32 atomic_subtract_32 659239268Sgonzo#define atomic_subtract_acq_32 atomic_subtract_32 660239268Sgonzo#define atomic_store_rel_32 atomic_store_32 661239268Sgonzo#define atomic_store_rel_long atomic_store_long 662239268Sgonzo#define atomic_load_acq_32 atomic_load_32 663239268Sgonzo#define atomic_load_acq_long atomic_load_long 664239268Sgonzo#undef __with_interrupts_disabled 665239268Sgonzo 666239268Sgonzostatic __inline void 667239268Sgonzoatomic_add_long(volatile u_long *p, u_long v) 668239268Sgonzo{ 669239268Sgonzo 670239268Sgonzo atomic_add_32((volatile uint32_t *)p, v); 671239268Sgonzo} 672239268Sgonzo 673239268Sgonzostatic __inline void 674239268Sgonzoatomic_clear_long(volatile u_long *p, u_long v) 675239268Sgonzo{ 676239268Sgonzo 677239268Sgonzo atomic_clear_32((volatile uint32_t *)p, v); 678239268Sgonzo} 679239268Sgonzo 680144761Scognetstatic __inline int 681239268Sgonzoatomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe) 682239268Sgonzo{ 683239268Sgonzo 684239268Sgonzo return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe)); 685239268Sgonzo} 686239268Sgonzo 687239268Sgonzostatic __inline u_long 688239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long v) 689239268Sgonzo{ 690239268Sgonzo 691239268Sgonzo return (atomic_fetchadd_32((volatile uint32_t *)p, v)); 692239268Sgonzo} 693239268Sgonzo 694239268Sgonzostatic __inline void 695239268Sgonzoatomic_readandclear_long(volatile u_long *p) 696239268Sgonzo{ 697239268Sgonzo 698239268Sgonzo atomic_readandclear_32((volatile uint32_t *)p); 699239268Sgonzo} 700239268Sgonzo 701239268Sgonzostatic __inline void 702239268Sgonzoatomic_set_long(volatile u_long *p, u_long v) 703239268Sgonzo{ 704239268Sgonzo 705239268Sgonzo atomic_set_32((volatile uint32_t *)p, v); 706239268Sgonzo} 707239268Sgonzo 708239268Sgonzostatic __inline void 709239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long v) 710239268Sgonzo{ 711239268Sgonzo 712239268Sgonzo atomic_subtract_32((volatile uint32_t *)p, v); 713239268Sgonzo} 714239268Sgonzo 715239268Sgonzo 716239268Sgonzo 717239268Sgonzo#endif /* Arch >= v6 */ 718239268Sgonzo 719239268Sgonzostatic __inline int 720144761Scognetatomic_load_32(volatile uint32_t *v) 721144761Scognet{ 722144761Scognet 723144761Scognet return (*v); 724144761Scognet} 725144761Scognet 726144761Scognetstatic __inline void 727144761Scognetatomic_store_32(volatile uint32_t *dst, uint32_t src) 728144761Scognet{ 729144761Scognet *dst = src; 730144761Scognet} 731144761Scognet 732239268Sgonzostatic __inline int 733239268Sgonzoatomic_load_long(volatile u_long *v) 734144761Scognet{ 735144761Scognet 736239268Sgonzo return (*v); 737144761Scognet} 738144761Scognet 739239268Sgonzostatic __inline void 740239268Sgonzoatomic_store_long(volatile u_long *dst, u_long src) 741239268Sgonzo{ 742239268Sgonzo *dst = src; 743239268Sgonzo} 744129198Scognet 745165786Sticso#define atomic_add_acq_long atomic_add_long 746165786Sticso#define atomic_add_rel_long atomic_add_long 747165786Sticso#define atomic_subtract_acq_long atomic_subtract_long 748165786Sticso#define atomic_subtract_rel_long atomic_subtract_long 749165786Sticso#define atomic_clear_acq_long atomic_clear_long 750165786Sticso#define atomic_clear_rel_long atomic_clear_long 751165786Sticso#define atomic_set_acq_long atomic_set_long 752165786Sticso#define atomic_set_rel_long atomic_set_long 753165786Sticso#define atomic_cmpset_acq_long atomic_cmpset_long 754165786Sticso#define atomic_cmpset_rel_long atomic_cmpset_long 755165786Sticso#define atomic_load_acq_long atomic_load_long 756129198Scognet 757165786Sticso#define atomic_clear_ptr atomic_clear_32 758165786Sticso#define atomic_set_ptr atomic_set_32 759239268Sgonzo#define atomic_cmpset_ptr atomic_cmpset_32 760239268Sgonzo#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 761239268Sgonzo#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 762165786Sticso#define atomic_store_ptr atomic_store_32 763165786Sticso#define atomic_store_rel_ptr atomic_store_ptr 764165786Sticso 765165786Sticso#define atomic_add_int atomic_add_32 766239268Sgonzo#define atomic_add_acq_int atomic_add_acq_32 767239268Sgonzo#define atomic_add_rel_int atomic_add_rel_32 768165786Sticso#define atomic_subtract_int atomic_subtract_32 769239268Sgonzo#define atomic_subtract_acq_int atomic_subtract_acq_32 770239268Sgonzo#define atomic_subtract_rel_int atomic_subtract_rel_32 771165786Sticso#define atomic_clear_int atomic_clear_32 772239268Sgonzo#define atomic_clear_acq_int atomic_clear_acq_32 773239268Sgonzo#define atomic_clear_rel_int atomic_clear_rel_32 774137222Scognet#define atomic_set_int atomic_set_32 775239268Sgonzo#define atomic_set_acq_int atomic_set_acq_32 776239268Sgonzo#define atomic_set_rel_int atomic_set_rel_32 777165786Sticso#define atomic_cmpset_int atomic_cmpset_32 778239268Sgonzo#define atomic_cmpset_acq_int atomic_cmpset_acq_32 779239268Sgonzo#define atomic_cmpset_rel_int atomic_cmpset_rel_32 780165786Sticso#define atomic_fetchadd_int atomic_fetchadd_32 781137222Scognet#define atomic_readandclear_int atomic_readandclear_32 782239268Sgonzo#define atomic_load_acq_int atomic_load_acq_32 783239268Sgonzo#define atomic_store_rel_int atomic_store_rel_32 784165786Sticso 785129198Scognet#endif /* _MACHINE_ATOMIC_H_ */ 786