1129198Scognet/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ 2129198Scognet 3139735Simp/*- 4129198Scognet * Copyright (C) 2003-2004 Olivier Houchard 5129198Scognet * Copyright (C) 1994-1997 Mark Brinicombe 6129198Scognet * Copyright (C) 1994 Brini 7129198Scognet * All rights reserved. 8129198Scognet * 9129198Scognet * This code is derived from software written for Brini by Mark Brinicombe 10129198Scognet * 11129198Scognet * Redistribution and use in source and binary forms, with or without 12129198Scognet * modification, are permitted provided that the following conditions 13129198Scognet * are met: 14129198Scognet * 1. Redistributions of source code must retain the above copyright 15129198Scognet * notice, this list of conditions and the following disclaimer. 16129198Scognet * 2. Redistributions in binary form must reproduce the above copyright 17129198Scognet * notice, this list of conditions and the following disclaimer in the 18129198Scognet * documentation and/or other materials provided with the distribution. 19129198Scognet * 3. All advertising materials mentioning features or use of this software 20129198Scognet * must display the following acknowledgement: 21129198Scognet * This product includes software developed by Brini. 22129198Scognet * 4. The name of Brini may not be used to endorse or promote products 23129198Scognet * derived from this software without specific prior written permission. 24129198Scognet * 25129198Scognet * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR 26129198Scognet * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27129198Scognet * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28129198Scognet * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29129198Scognet * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 30129198Scognet * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 31129198Scognet * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32129198Scognet * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 33129198Scognet * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 34129198Scognet * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35129198Scognet * 36129198Scognet * $FreeBSD$ 37129198Scognet */ 38129198Scognet 39129198Scognet#ifndef _MACHINE_ATOMIC_H_ 40129198Scognet#define _MACHINE_ATOMIC_H_ 41129198Scognet 42129198Scognet#include <sys/types.h> 43129198Scognet 44175982Sraj#ifndef _KERNEL 45175982Sraj#include <machine/sysarch.h> 46239268Sgonzo#else 47239268Sgonzo#include <machine/cpuconf.h> 48175982Sraj#endif 49175982Sraj 50245135Sgonzo#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 51245135Sgonzo#define isb() __asm __volatile("isb" : : : "memory") 52245135Sgonzo#define dsb() __asm __volatile("dsb" : : : "memory") 53245135Sgonzo#define dmb() __asm __volatile("dmb" : : : "memory") 54245135Sgonzo#elif defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \ 55253489Sandrew defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6T2__) || \ 56253489Sandrew defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__) 57245135Sgonzo#define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") 58245135Sgonzo#define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") 59245135Sgonzo#define dmb() __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") 60245135Sgonzo#else 61245135Sgonzo#define isb() 62245135Sgonzo#define dsb() 63245135Sgonzo#define dmb() 64245135Sgonzo#endif 65185162Skmacy 66245135Sgonzo#define mb() dmb() 67245135Sgonzo#define wmb() dmb() 68245135Sgonzo#define rmb() dmb() 69245135Sgonzo 70129198Scognet#ifndef I32_bit 71129198Scognet#define I32_bit (1 << 7) /* IRQ disable */ 72129198Scognet#endif 73129198Scognet#ifndef F32_bit 74129198Scognet#define F32_bit (1 << 6) /* FIQ disable */ 75129198Scognet#endif 76129198Scognet 77239268Sgonzo/* 78239268Sgonzo * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h 79239268Sgonzo * here, but that header can't be included here because this is C 80239268Sgonzo * code. I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition 81239268Sgonzo * out of asm.h so it can be used in both asm and C code. - kientzle@ 82239268Sgonzo */ 83239268Sgonzo#if defined (__ARM_ARCH_7__) || \ 84253489Sandrew defined (__ARM_ARCH_7A__) || \ 85253489Sandrew defined (__ARM_ARCH_6__) || \ 86253489Sandrew defined (__ARM_ARCH_6J__) || \ 87253489Sandrew defined (__ARM_ARCH_6K__) || \ 88253489Sandrew defined (__ARM_ARCH_6T2__) || \ 89253489Sandrew defined (__ARM_ARCH_6Z__) || \ 90239268Sgonzo defined (__ARM_ARCH_6ZK__) 91239268Sgonzostatic __inline void 92239268Sgonzo__do_dmb(void) 93239268Sgonzo{ 94239268Sgonzo 95239268Sgonzo#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 96239268Sgonzo __asm __volatile("dmb" : : : "memory"); 97239268Sgonzo#else 98239268Sgonzo __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory"); 99239268Sgonzo#endif 100239268Sgonzo} 101239268Sgonzo 102239268Sgonzo#define ATOMIC_ACQ_REL_LONG(NAME) \ 103239268Sgonzostatic __inline void \ 104239268Sgonzoatomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ 105239268Sgonzo{ \ 106239268Sgonzo atomic_##NAME##_long(p, v); \ 107239268Sgonzo __do_dmb(); \ 108239268Sgonzo} \ 109239268Sgonzo \ 110239268Sgonzostatic __inline void \ 111239268Sgonzoatomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ 112239268Sgonzo{ \ 113239268Sgonzo __do_dmb(); \ 114239268Sgonzo atomic_##NAME##_long(p, v); \ 115239268Sgonzo} 116239268Sgonzo 117239268Sgonzo#define ATOMIC_ACQ_REL(NAME, WIDTH) \ 118239268Sgonzostatic __inline void \ 119239268Sgonzoatomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 120239268Sgonzo{ \ 121239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 122239268Sgonzo __do_dmb(); \ 123239268Sgonzo} \ 124239268Sgonzo \ 125239268Sgonzostatic __inline void \ 126239268Sgonzoatomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 127239268Sgonzo{ \ 128239268Sgonzo __do_dmb(); \ 129239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 130239268Sgonzo} 131239268Sgonzo 132239268Sgonzostatic __inline void 133239268Sgonzoatomic_set_32(volatile uint32_t *address, uint32_t setmask) 134239268Sgonzo{ 135239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 136239268Sgonzo 137239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 138239268Sgonzo "orr %0, %0, %3\n" 139239268Sgonzo "strex %1, %0, [%2]\n" 140239268Sgonzo "cmp %1, #0\n" 141253489Sandrew "it ne\n" 142239268Sgonzo "bne 1b\n" 143239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 144241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 145239268Sgonzo 146239268Sgonzo} 147239268Sgonzo 148239268Sgonzostatic __inline void 149239268Sgonzoatomic_set_long(volatile u_long *address, u_long setmask) 150239268Sgonzo{ 151239268Sgonzo u_long tmp = 0, tmp2 = 0; 152239268Sgonzo 153239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 154239268Sgonzo "orr %0, %0, %3\n" 155239268Sgonzo "strex %1, %0, [%2]\n" 156239268Sgonzo "cmp %1, #0\n" 157253489Sandrew "it ne\n" 158239268Sgonzo "bne 1b\n" 159239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 160241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 161239268Sgonzo 162239268Sgonzo} 163239268Sgonzo 164239268Sgonzostatic __inline void 165239268Sgonzoatomic_clear_32(volatile uint32_t *address, uint32_t setmask) 166239268Sgonzo{ 167239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 168239268Sgonzo 169239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 170239268Sgonzo "bic %0, %0, %3\n" 171239268Sgonzo "strex %1, %0, [%2]\n" 172239268Sgonzo "cmp %1, #0\n" 173253489Sandrew "it ne\n" 174239268Sgonzo "bne 1b\n" 175239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 176241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 177239268Sgonzo} 178239268Sgonzo 179239268Sgonzostatic __inline void 180239268Sgonzoatomic_clear_long(volatile u_long *address, u_long setmask) 181239268Sgonzo{ 182239268Sgonzo u_long tmp = 0, tmp2 = 0; 183239268Sgonzo 184239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 185239268Sgonzo "bic %0, %0, %3\n" 186239268Sgonzo "strex %1, %0, [%2]\n" 187239268Sgonzo "cmp %1, #0\n" 188253489Sandrew "it ne\n" 189239268Sgonzo "bne 1b\n" 190239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 191241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 192239268Sgonzo} 193239268Sgonzo 194239268Sgonzostatic __inline u_int32_t 195239268Sgonzoatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 196239268Sgonzo{ 197239268Sgonzo uint32_t ret; 198239268Sgonzo 199239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 200239268Sgonzo "cmp %0, %2\n" 201253489Sandrew "it ne\n" 202239268Sgonzo "movne %0, #0\n" 203239268Sgonzo "bne 2f\n" 204239268Sgonzo "strex %0, %3, [%1]\n" 205239268Sgonzo "cmp %0, #0\n" 206253489Sandrew "ite eq\n" 207253489Sandrew "moveq %0, #1\n" 208239268Sgonzo "bne 1b\n" 209239268Sgonzo "2:" 210239268Sgonzo : "=&r" (ret) 211241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 212241080Sandrew "memory"); 213239268Sgonzo return (ret); 214239268Sgonzo} 215239268Sgonzo 216239268Sgonzostatic __inline u_long 217239268Sgonzoatomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 218239268Sgonzo{ 219239268Sgonzo u_long ret; 220239268Sgonzo 221239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 222239268Sgonzo "cmp %0, %2\n" 223253489Sandrew "itt ne\n" 224239268Sgonzo "movne %0, #0\n" 225239268Sgonzo "bne 2f\n" 226239268Sgonzo "strex %0, %3, [%1]\n" 227239268Sgonzo "cmp %0, #0\n" 228253489Sandrew "ite eq\n" 229253489Sandrew "moveq %0, #1\n" 230239268Sgonzo "bne 1b\n" 231239268Sgonzo "2:" 232239268Sgonzo : "=&r" (ret) 233241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 234241080Sandrew "memory"); 235239268Sgonzo return (ret); 236239268Sgonzo} 237239268Sgonzo 238239268Sgonzostatic __inline u_int32_t 239239268Sgonzoatomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 240239268Sgonzo{ 241239268Sgonzo u_int32_t ret = atomic_cmpset_32(p, cmpval, newval); 242239268Sgonzo 243239268Sgonzo __do_dmb(); 244239268Sgonzo return (ret); 245239268Sgonzo} 246239268Sgonzo 247239268Sgonzostatic __inline u_long 248239268Sgonzoatomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 249239268Sgonzo{ 250239268Sgonzo u_long ret = atomic_cmpset_long(p, cmpval, newval); 251239268Sgonzo 252239268Sgonzo __do_dmb(); 253239268Sgonzo return (ret); 254239268Sgonzo} 255239268Sgonzo 256239268Sgonzostatic __inline u_int32_t 257239268Sgonzoatomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 258239268Sgonzo{ 259239268Sgonzo 260239268Sgonzo __do_dmb(); 261239268Sgonzo return (atomic_cmpset_32(p, cmpval, newval)); 262239268Sgonzo} 263239268Sgonzo 264239268Sgonzostatic __inline u_long 265239268Sgonzoatomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 266239268Sgonzo{ 267239268Sgonzo 268239268Sgonzo __do_dmb(); 269239268Sgonzo return (atomic_cmpset_long(p, cmpval, newval)); 270239268Sgonzo} 271239268Sgonzo 272239268Sgonzo 273239268Sgonzostatic __inline void 274239268Sgonzoatomic_add_32(volatile u_int32_t *p, u_int32_t val) 275239268Sgonzo{ 276239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 277239268Sgonzo 278239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 279239268Sgonzo "add %0, %0, %3\n" 280239268Sgonzo "strex %1, %0, [%2]\n" 281239268Sgonzo "cmp %1, #0\n" 282253489Sandrew "it ne\n" 283239268Sgonzo "bne 1b\n" 284239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 285241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 286239268Sgonzo} 287239268Sgonzo 288239268Sgonzostatic __inline void 289239268Sgonzoatomic_add_long(volatile u_long *p, u_long val) 290239268Sgonzo{ 291239268Sgonzo u_long tmp = 0, tmp2 = 0; 292239268Sgonzo 293239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 294239268Sgonzo "add %0, %0, %3\n" 295239268Sgonzo "strex %1, %0, [%2]\n" 296239268Sgonzo "cmp %1, #0\n" 297253489Sandrew "it ne\n" 298239268Sgonzo "bne 1b\n" 299239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 300241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 301239268Sgonzo} 302239268Sgonzo 303239268Sgonzostatic __inline void 304239268Sgonzoatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 305239268Sgonzo{ 306239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 307239268Sgonzo 308239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 309239268Sgonzo "sub %0, %0, %3\n" 310239268Sgonzo "strex %1, %0, [%2]\n" 311239268Sgonzo "cmp %1, #0\n" 312253489Sandrew "it ne\n" 313239268Sgonzo "bne 1b\n" 314239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 315241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 316239268Sgonzo} 317239268Sgonzo 318239268Sgonzostatic __inline void 319239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long val) 320239268Sgonzo{ 321239268Sgonzo u_long tmp = 0, tmp2 = 0; 322239268Sgonzo 323239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 324239268Sgonzo "sub %0, %0, %3\n" 325239268Sgonzo "strex %1, %0, [%2]\n" 326239268Sgonzo "cmp %1, #0\n" 327253489Sandrew "it ne\n" 328239268Sgonzo "bne 1b\n" 329239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 330241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 331239268Sgonzo} 332239268Sgonzo 333239268SgonzoATOMIC_ACQ_REL(clear, 32) 334239268SgonzoATOMIC_ACQ_REL(add, 32) 335239268SgonzoATOMIC_ACQ_REL(subtract, 32) 336239268SgonzoATOMIC_ACQ_REL(set, 32) 337239268SgonzoATOMIC_ACQ_REL_LONG(clear) 338239268SgonzoATOMIC_ACQ_REL_LONG(add) 339239268SgonzoATOMIC_ACQ_REL_LONG(subtract) 340239268SgonzoATOMIC_ACQ_REL_LONG(set) 341239268Sgonzo 342239268Sgonzo#undef ATOMIC_ACQ_REL 343239268Sgonzo#undef ATOMIC_ACQ_REL_LONG 344239268Sgonzo 345239268Sgonzostatic __inline uint32_t 346239268Sgonzoatomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 347239268Sgonzo{ 348239268Sgonzo uint32_t tmp = 0, tmp2 = 0, ret = 0; 349239268Sgonzo 350239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 351239268Sgonzo "add %1, %0, %4\n" 352239268Sgonzo "strex %2, %1, [%3]\n" 353239268Sgonzo "cmp %2, #0\n" 354253489Sandrew "it ne\n" 355239268Sgonzo "bne 1b\n" 356239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 357241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 358239268Sgonzo return (ret); 359239268Sgonzo} 360239268Sgonzo 361239268Sgonzostatic __inline uint32_t 362239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 363239268Sgonzo{ 364239268Sgonzo uint32_t ret, tmp = 0, tmp2 = 0; 365239268Sgonzo 366239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 367239268Sgonzo "mov %1, #0\n" 368239268Sgonzo "strex %2, %1, [%3]\n" 369239268Sgonzo "cmp %2, #0\n" 370253489Sandrew "it ne\n" 371239268Sgonzo "bne 1b\n" 372239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 373241080Sandrew ,"+r" (p) : : "cc", "memory"); 374239268Sgonzo return (ret); 375239268Sgonzo} 376239268Sgonzo 377239268Sgonzostatic __inline uint32_t 378239268Sgonzoatomic_load_acq_32(volatile uint32_t *p) 379239268Sgonzo{ 380239268Sgonzo uint32_t v; 381239268Sgonzo 382239268Sgonzo v = *p; 383239268Sgonzo __do_dmb(); 384239268Sgonzo return (v); 385239268Sgonzo} 386239268Sgonzo 387239268Sgonzostatic __inline void 388239268Sgonzoatomic_store_rel_32(volatile uint32_t *p, uint32_t v) 389239268Sgonzo{ 390239268Sgonzo 391239268Sgonzo __do_dmb(); 392239268Sgonzo *p = v; 393239268Sgonzo} 394239268Sgonzo 395239268Sgonzostatic __inline u_long 396239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long val) 397239268Sgonzo{ 398239268Sgonzo u_long tmp = 0, tmp2 = 0, ret = 0; 399239268Sgonzo 400239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 401239268Sgonzo "add %1, %0, %4\n" 402239268Sgonzo "strex %2, %1, [%3]\n" 403239268Sgonzo "cmp %2, #0\n" 404253489Sandrew "it ne\n" 405239268Sgonzo "bne 1b\n" 406239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 407241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 408239268Sgonzo return (ret); 409239268Sgonzo} 410239268Sgonzo 411239268Sgonzostatic __inline u_long 412239268Sgonzoatomic_readandclear_long(volatile u_long *p) 413239268Sgonzo{ 414239268Sgonzo u_long ret, tmp = 0, tmp2 = 0; 415239268Sgonzo 416239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 417239268Sgonzo "mov %1, #0\n" 418239268Sgonzo "strex %2, %1, [%3]\n" 419239268Sgonzo "cmp %2, #0\n" 420253489Sandrew "it ne\n" 421239268Sgonzo "bne 1b\n" 422239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 423241080Sandrew ,"+r" (p) : : "cc", "memory"); 424239268Sgonzo return (ret); 425239268Sgonzo} 426239268Sgonzo 427239268Sgonzostatic __inline u_long 428239268Sgonzoatomic_load_acq_long(volatile u_long *p) 429239268Sgonzo{ 430239268Sgonzo u_long v; 431239268Sgonzo 432239268Sgonzo v = *p; 433239268Sgonzo __do_dmb(); 434239268Sgonzo return (v); 435239268Sgonzo} 436239268Sgonzo 437239268Sgonzostatic __inline void 438239268Sgonzoatomic_store_rel_long(volatile u_long *p, u_long v) 439239268Sgonzo{ 440239268Sgonzo 441239268Sgonzo __do_dmb(); 442239268Sgonzo *p = v; 443239268Sgonzo} 444239268Sgonzo#else /* < armv6 */ 445239268Sgonzo 446129198Scognet#define __with_interrupts_disabled(expr) \ 447129198Scognet do { \ 448129198Scognet u_int cpsr_save, tmp; \ 449129198Scognet \ 450129198Scognet __asm __volatile( \ 451129198Scognet "mrs %0, cpsr;" \ 452129198Scognet "orr %1, %0, %2;" \ 453129198Scognet "msr cpsr_all, %1;" \ 454129198Scognet : "=r" (cpsr_save), "=r" (tmp) \ 455157725Scognet : "I" (I32_bit | F32_bit) \ 456129198Scognet : "cc" ); \ 457129198Scognet (expr); \ 458129198Scognet __asm __volatile( \ 459129198Scognet "msr cpsr_all, %0" \ 460129198Scognet : /* no output */ \ 461129198Scognet : "r" (cpsr_save) \ 462129198Scognet : "cc" ); \ 463129198Scognet } while(0) 464129198Scognet 465137222Scognetstatic __inline uint32_t 466137222Scognet__swp(uint32_t val, volatile uint32_t *ptr) 467129198Scognet{ 468148453Sjhb __asm __volatile("swp %0, %2, [%3]" 469148453Sjhb : "=&r" (val), "=m" (*ptr) 470151340Sjhb : "r" (val), "r" (ptr), "m" (*ptr) 471148453Sjhb : "memory"); 472137222Scognet return (val); 473129198Scognet} 474129198Scognet 475137222Scognet 476144761Scognet#ifdef _KERNEL 477129198Scognetstatic __inline void 478137222Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 479129198Scognet{ 480144761Scognet __with_interrupts_disabled(*address |= setmask); 481129198Scognet} 482129198Scognet 483129198Scognetstatic __inline void 484129198Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 485129198Scognet{ 486144761Scognet __with_interrupts_disabled(*address &= ~clearmask); 487129198Scognet} 488129198Scognet 489144761Scognetstatic __inline u_int32_t 490144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 491129198Scognet{ 492144761Scognet int ret; 493144761Scognet 494144761Scognet __with_interrupts_disabled( 495144761Scognet { 496144761Scognet if (*p == cmpval) { 497144761Scognet *p = newval; 498144761Scognet ret = 1; 499144761Scognet } else { 500144761Scognet ret = 0; 501144761Scognet } 502144761Scognet }); 503144761Scognet return (ret); 504129198Scognet} 505129198Scognet 506129198Scognetstatic __inline void 507144761Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 508129198Scognet{ 509144761Scognet __with_interrupts_disabled(*p += val); 510129198Scognet} 511129198Scognet 512144761Scognetstatic __inline void 513144761Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 514129198Scognet{ 515144761Scognet __with_interrupts_disabled(*p -= val); 516129198Scognet} 517129198Scognet 518150627Sjhbstatic __inline uint32_t 519150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 520150627Sjhb{ 521150627Sjhb uint32_t value; 522150627Sjhb 523150627Sjhb __with_interrupts_disabled( 524150627Sjhb { 525150627Sjhb value = *p; 526150627Sjhb *p += v; 527150627Sjhb }); 528150627Sjhb return (value); 529150627Sjhb} 530150627Sjhb 531144761Scognet#else /* !_KERNEL */ 532144761Scognet 533129198Scognetstatic __inline u_int32_t 534144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 535129198Scognet{ 536175982Sraj register int done, ras_start = ARM_RAS_START; 537144761Scognet 538144761Scognet __asm __volatile("1:\n" 539174170Scognet "adr %1, 1b\n" 540174170Scognet "str %1, [%0]\n" 541144761Scognet "adr %1, 2f\n" 542175982Sraj "str %1, [%0, #4]\n" 543155355Scognet "ldr %1, [%2]\n" 544144761Scognet "cmp %1, %3\n" 545155355Scognet "streq %4, [%2]\n" 546144761Scognet "2:\n" 547146591Scognet "mov %1, #0\n" 548146591Scognet "str %1, [%0]\n" 549174170Scognet "mov %1, #0xffffffff\n" 550175982Sraj "str %1, [%0, #4]\n" 551144761Scognet "moveq %1, #1\n" 552144761Scognet "movne %1, #0\n" 553175982Sraj : "+r" (ras_start), "=r" (done) 554241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory"); 555137282Scognet return (done); 556129198Scognet} 557129198Scognet 558129198Scognetstatic __inline void 559129198Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 560129198Scognet{ 561175982Sraj int start, ras_start = ARM_RAS_START; 562144761Scognet 563144761Scognet __asm __volatile("1:\n" 564174170Scognet "adr %1, 1b\n" 565174170Scognet "str %1, [%0]\n" 566144761Scognet "adr %1, 2f\n" 567175982Sraj "str %1, [%0, #4]\n" 568155355Scognet "ldr %1, [%2]\n" 569144761Scognet "add %1, %1, %3\n" 570155355Scognet "str %1, [%2]\n" 571144761Scognet "2:\n" 572146591Scognet "mov %1, #0\n" 573146591Scognet "str %1, [%0]\n" 574174170Scognet "mov %1, #0xffffffff\n" 575175982Sraj "str %1, [%0, #4]\n" 576175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 577155391Scognet : : "memory"); 578129198Scognet} 579129198Scognet 580129198Scognetstatic __inline void 581129198Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 582129198Scognet{ 583175982Sraj int start, ras_start = ARM_RAS_START; 584144761Scognet 585144761Scognet __asm __volatile("1:\n" 586174170Scognet "adr %1, 1b\n" 587174170Scognet "str %1, [%0]\n" 588144761Scognet "adr %1, 2f\n" 589175982Sraj "str %1, [%0, #4]\n" 590155355Scognet "ldr %1, [%2]\n" 591144761Scognet "sub %1, %1, %3\n" 592155355Scognet "str %1, [%2]\n" 593144761Scognet "2:\n" 594146591Scognet "mov %1, #0\n" 595146591Scognet "str %1, [%0]\n" 596174170Scognet "mov %1, #0xffffffff\n" 597175982Sraj "str %1, [%0, #4]\n" 598146591Scognet 599175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 600155391Scognet : : "memory"); 601129198Scognet} 602129198Scognet 603144761Scognetstatic __inline void 604144761Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 605144761Scognet{ 606175982Sraj int start, ras_start = ARM_RAS_START; 607144761Scognet 608144761Scognet __asm __volatile("1:\n" 609174170Scognet "adr %1, 1b\n" 610174170Scognet "str %1, [%0]\n" 611144761Scognet "adr %1, 2f\n" 612175982Sraj "str %1, [%0, #4]\n" 613155355Scognet "ldr %1, [%2]\n" 614144761Scognet "orr %1, %1, %3\n" 615155355Scognet "str %1, [%2]\n" 616144761Scognet "2:\n" 617146591Scognet "mov %1, #0\n" 618146591Scognet "str %1, [%0]\n" 619174170Scognet "mov %1, #0xffffffff\n" 620175982Sraj "str %1, [%0, #4]\n" 621146591Scognet 622175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask) 623155391Scognet : : "memory"); 624144761Scognet} 625144761Scognet 626144761Scognetstatic __inline void 627144761Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 628144761Scognet{ 629175982Sraj int start, ras_start = ARM_RAS_START; 630144761Scognet 631144761Scognet __asm __volatile("1:\n" 632174170Scognet "adr %1, 1b\n" 633174170Scognet "str %1, [%0]\n" 634144761Scognet "adr %1, 2f\n" 635175982Sraj "str %1, [%0, #4]\n" 636155355Scognet "ldr %1, [%2]\n" 637144761Scognet "bic %1, %1, %3\n" 638155355Scognet "str %1, [%2]\n" 639144761Scognet "2:\n" 640146591Scognet "mov %1, #0\n" 641146591Scognet "str %1, [%0]\n" 642174170Scognet "mov %1, #0xffffffff\n" 643175982Sraj "str %1, [%0, #4]\n" 644175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask) 645155391Scognet : : "memory"); 646144761Scognet 647144761Scognet} 648150627Sjhb 649150627Sjhbstatic __inline uint32_t 650150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 651150627Sjhb{ 652190603Scognet uint32_t start, tmp, ras_start = ARM_RAS_START; 653150627Sjhb 654150627Sjhb __asm __volatile("1:\n" 655174170Scognet "adr %1, 1b\n" 656174170Scognet "str %1, [%0]\n" 657150627Sjhb "adr %1, 2f\n" 658175982Sraj "str %1, [%0, #4]\n" 659190603Scognet "ldr %1, [%3]\n" 660190603Scognet "mov %2, %1\n" 661190603Scognet "add %2, %2, %4\n" 662190603Scognet "str %2, [%3]\n" 663150627Sjhb "2:\n" 664190603Scognet "mov %2, #0\n" 665190603Scognet "str %2, [%0]\n" 666190603Scognet "mov %2, #0xffffffff\n" 667190603Scognet "str %2, [%0, #4]\n" 668190603Scognet : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v) 669155391Scognet : : "memory"); 670150627Sjhb return (start); 671150627Sjhb} 672150627Sjhb 673144761Scognet#endif /* _KERNEL */ 674144761Scognet 675239268Sgonzo 676239268Sgonzostatic __inline uint32_t 677239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 678239268Sgonzo{ 679239268Sgonzo 680239268Sgonzo return (__swp(0, p)); 681239268Sgonzo} 682239268Sgonzo 683239268Sgonzo#define atomic_cmpset_rel_32 atomic_cmpset_32 684239268Sgonzo#define atomic_cmpset_acq_32 atomic_cmpset_32 685239268Sgonzo#define atomic_set_rel_32 atomic_set_32 686239268Sgonzo#define atomic_set_acq_32 atomic_set_32 687239268Sgonzo#define atomic_clear_rel_32 atomic_clear_32 688239268Sgonzo#define atomic_clear_acq_32 atomic_clear_32 689239268Sgonzo#define atomic_add_rel_32 atomic_add_32 690239268Sgonzo#define atomic_add_acq_32 atomic_add_32 691239268Sgonzo#define atomic_subtract_rel_32 atomic_subtract_32 692239268Sgonzo#define atomic_subtract_acq_32 atomic_subtract_32 693239268Sgonzo#define atomic_store_rel_32 atomic_store_32 694239268Sgonzo#define atomic_store_rel_long atomic_store_long 695239268Sgonzo#define atomic_load_acq_32 atomic_load_32 696239268Sgonzo#define atomic_load_acq_long atomic_load_long 697245475Scognet#define atomic_add_acq_long atomic_add_long 698245475Scognet#define atomic_add_rel_long atomic_add_long 699245475Scognet#define atomic_subtract_acq_long atomic_subtract_long 700245475Scognet#define atomic_subtract_rel_long atomic_subtract_long 701245475Scognet#define atomic_clear_acq_long atomic_clear_long 702245475Scognet#define atomic_clear_rel_long atomic_clear_long 703245475Scognet#define atomic_set_acq_long atomic_set_long 704245475Scognet#define atomic_set_rel_long atomic_set_long 705245475Scognet#define atomic_cmpset_acq_long atomic_cmpset_long 706245475Scognet#define atomic_cmpset_rel_long atomic_cmpset_long 707245475Scognet#define atomic_load_acq_long atomic_load_long 708239268Sgonzo#undef __with_interrupts_disabled 709239268Sgonzo 710239268Sgonzostatic __inline void 711239268Sgonzoatomic_add_long(volatile u_long *p, u_long v) 712239268Sgonzo{ 713239268Sgonzo 714239268Sgonzo atomic_add_32((volatile uint32_t *)p, v); 715239268Sgonzo} 716239268Sgonzo 717239268Sgonzostatic __inline void 718239268Sgonzoatomic_clear_long(volatile u_long *p, u_long v) 719239268Sgonzo{ 720239268Sgonzo 721239268Sgonzo atomic_clear_32((volatile uint32_t *)p, v); 722239268Sgonzo} 723239268Sgonzo 724144761Scognetstatic __inline int 725239268Sgonzoatomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe) 726239268Sgonzo{ 727239268Sgonzo 728239268Sgonzo return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe)); 729239268Sgonzo} 730239268Sgonzo 731239268Sgonzostatic __inline u_long 732239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long v) 733239268Sgonzo{ 734239268Sgonzo 735239268Sgonzo return (atomic_fetchadd_32((volatile uint32_t *)p, v)); 736239268Sgonzo} 737239268Sgonzo 738239268Sgonzostatic __inline void 739239268Sgonzoatomic_readandclear_long(volatile u_long *p) 740239268Sgonzo{ 741239268Sgonzo 742239268Sgonzo atomic_readandclear_32((volatile uint32_t *)p); 743239268Sgonzo} 744239268Sgonzo 745239268Sgonzostatic __inline void 746239268Sgonzoatomic_set_long(volatile u_long *p, u_long v) 747239268Sgonzo{ 748239268Sgonzo 749239268Sgonzo atomic_set_32((volatile uint32_t *)p, v); 750239268Sgonzo} 751239268Sgonzo 752239268Sgonzostatic __inline void 753239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long v) 754239268Sgonzo{ 755239268Sgonzo 756239268Sgonzo atomic_subtract_32((volatile uint32_t *)p, v); 757239268Sgonzo} 758239268Sgonzo 759239268Sgonzo 760239268Sgonzo 761239268Sgonzo#endif /* Arch >= v6 */ 762239268Sgonzo 763239268Sgonzostatic __inline int 764144761Scognetatomic_load_32(volatile uint32_t *v) 765144761Scognet{ 766144761Scognet 767144761Scognet return (*v); 768144761Scognet} 769144761Scognet 770144761Scognetstatic __inline void 771144761Scognetatomic_store_32(volatile uint32_t *dst, uint32_t src) 772144761Scognet{ 773144761Scognet *dst = src; 774144761Scognet} 775144761Scognet 776239268Sgonzostatic __inline int 777239268Sgonzoatomic_load_long(volatile u_long *v) 778144761Scognet{ 779144761Scognet 780239268Sgonzo return (*v); 781144761Scognet} 782144761Scognet 783239268Sgonzostatic __inline void 784239268Sgonzoatomic_store_long(volatile u_long *dst, u_long src) 785239268Sgonzo{ 786239268Sgonzo *dst = src; 787239268Sgonzo} 788129198Scognet 789165786Sticso#define atomic_clear_ptr atomic_clear_32 790165786Sticso#define atomic_set_ptr atomic_set_32 791239268Sgonzo#define atomic_cmpset_ptr atomic_cmpset_32 792239268Sgonzo#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 793239268Sgonzo#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 794165786Sticso#define atomic_store_ptr atomic_store_32 795245475Scognet#define atomic_store_rel_ptr atomic_store_rel_32 796165786Sticso 797165786Sticso#define atomic_add_int atomic_add_32 798239268Sgonzo#define atomic_add_acq_int atomic_add_acq_32 799239268Sgonzo#define atomic_add_rel_int atomic_add_rel_32 800165786Sticso#define atomic_subtract_int atomic_subtract_32 801239268Sgonzo#define atomic_subtract_acq_int atomic_subtract_acq_32 802239268Sgonzo#define atomic_subtract_rel_int atomic_subtract_rel_32 803165786Sticso#define atomic_clear_int atomic_clear_32 804239268Sgonzo#define atomic_clear_acq_int atomic_clear_acq_32 805239268Sgonzo#define atomic_clear_rel_int atomic_clear_rel_32 806137222Scognet#define atomic_set_int atomic_set_32 807239268Sgonzo#define atomic_set_acq_int atomic_set_acq_32 808239268Sgonzo#define atomic_set_rel_int atomic_set_rel_32 809165786Sticso#define atomic_cmpset_int atomic_cmpset_32 810239268Sgonzo#define atomic_cmpset_acq_int atomic_cmpset_acq_32 811239268Sgonzo#define atomic_cmpset_rel_int atomic_cmpset_rel_32 812165786Sticso#define atomic_fetchadd_int atomic_fetchadd_32 813137222Scognet#define atomic_readandclear_int atomic_readandclear_32 814239268Sgonzo#define atomic_load_acq_int atomic_load_acq_32 815239268Sgonzo#define atomic_store_rel_int atomic_store_rel_32 816165786Sticso 817129198Scognet#endif /* _MACHINE_ATOMIC_H_ */ 818