1129198Scognet/* $NetBSD: atomic.h,v 1.1 2002/10/19 12:22:34 bsh Exp $ */ 2129198Scognet 3139735Simp/*- 4129198Scognet * Copyright (C) 2003-2004 Olivier Houchard 5129198Scognet * Copyright (C) 1994-1997 Mark Brinicombe 6129198Scognet * Copyright (C) 1994 Brini 7129198Scognet * All rights reserved. 8129198Scognet * 9129198Scognet * This code is derived from software written for Brini by Mark Brinicombe 10129198Scognet * 11129198Scognet * Redistribution and use in source and binary forms, with or without 12129198Scognet * modification, are permitted provided that the following conditions 13129198Scognet * are met: 14129198Scognet * 1. Redistributions of source code must retain the above copyright 15129198Scognet * notice, this list of conditions and the following disclaimer. 16129198Scognet * 2. Redistributions in binary form must reproduce the above copyright 17129198Scognet * notice, this list of conditions and the following disclaimer in the 18129198Scognet * documentation and/or other materials provided with the distribution. 19129198Scognet * 3. All advertising materials mentioning features or use of this software 20129198Scognet * must display the following acknowledgement: 21129198Scognet * This product includes software developed by Brini. 22129198Scognet * 4. The name of Brini may not be used to endorse or promote products 23129198Scognet * derived from this software without specific prior written permission. 24129198Scognet * 25129198Scognet * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR 26129198Scognet * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27129198Scognet * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 28129198Scognet * IN NO EVENT SHALL BRINI BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29129198Scognet * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 30129198Scognet * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 31129198Scognet * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 32129198Scognet * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 33129198Scognet * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 34129198Scognet * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35129198Scognet * 36129198Scognet * $FreeBSD$ 37129198Scognet */ 38129198Scognet 39129198Scognet#ifndef _MACHINE_ATOMIC_H_ 40129198Scognet#define _MACHINE_ATOMIC_H_ 41129198Scognet 42129198Scognet#include <sys/types.h> 43278613Sian#include <machine/armreg.h> 44129198Scognet 45175982Sraj#ifndef _KERNEL 46175982Sraj#include <machine/sysarch.h> 47239268Sgonzo#else 48239268Sgonzo#include <machine/cpuconf.h> 49175982Sraj#endif 50175982Sraj 51245135Sgonzo#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 52245135Sgonzo#define isb() __asm __volatile("isb" : : : "memory") 53245135Sgonzo#define dsb() __asm __volatile("dsb" : : : "memory") 54245135Sgonzo#define dmb() __asm __volatile("dmb" : : : "memory") 55245135Sgonzo#elif defined (__ARM_ARCH_6__) || defined (__ARM_ARCH_6J__) || \ 56253489Sandrew defined (__ARM_ARCH_6K__) || defined (__ARM_ARCH_6T2__) || \ 57253489Sandrew defined (__ARM_ARCH_6Z__) || defined (__ARM_ARCH_6ZK__) 58245135Sgonzo#define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") 59245135Sgonzo#define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") 60245135Sgonzo#define dmb() __asm __volatile("mcr p15, 0, %0, c7, c10, 5" : : "r" (0) : "memory") 61245135Sgonzo#else 62266387Sian#define isb() __asm __volatile("mcr p15, 0, %0, c7, c5, 4" : : "r" (0) : "memory") 63266387Sian#define dsb() __asm __volatile("mcr p15, 0, %0, c7, c10, 4" : : "r" (0) : "memory") 64266387Sian#define dmb() dsb() 65245135Sgonzo#endif 66185162Skmacy 67245135Sgonzo#define mb() dmb() 68245135Sgonzo#define wmb() dmb() 69245135Sgonzo#define rmb() dmb() 70245135Sgonzo 71129198Scognet 72278613Sian 73239268Sgonzo/* 74239268Sgonzo * It would be nice to use _HAVE_ARMv6_INSTRUCTIONS from machine/asm.h 75239268Sgonzo * here, but that header can't be included here because this is C 76239268Sgonzo * code. I would like to move the _HAVE_ARMv6_INSTRUCTIONS definition 77239268Sgonzo * out of asm.h so it can be used in both asm and C code. - kientzle@ 78239268Sgonzo */ 79239268Sgonzo#if defined (__ARM_ARCH_7__) || \ 80253489Sandrew defined (__ARM_ARCH_7A__) || \ 81253489Sandrew defined (__ARM_ARCH_6__) || \ 82253489Sandrew defined (__ARM_ARCH_6J__) || \ 83253489Sandrew defined (__ARM_ARCH_6K__) || \ 84253489Sandrew defined (__ARM_ARCH_6T2__) || \ 85253489Sandrew defined (__ARM_ARCH_6Z__) || \ 86239268Sgonzo defined (__ARM_ARCH_6ZK__) 87269798Sian#define ARM_HAVE_ATOMIC64 88269798Sian 89239268Sgonzostatic __inline void 90239268Sgonzo__do_dmb(void) 91239268Sgonzo{ 92239268Sgonzo 93239268Sgonzo#if defined (__ARM_ARCH_7__) || defined (__ARM_ARCH_7A__) 94239268Sgonzo __asm __volatile("dmb" : : : "memory"); 95239268Sgonzo#else 96239268Sgonzo __asm __volatile("mcr p15, 0, r0, c7, c10, 5" : : : "memory"); 97239268Sgonzo#endif 98239268Sgonzo} 99239268Sgonzo 100239268Sgonzo#define ATOMIC_ACQ_REL_LONG(NAME) \ 101239268Sgonzostatic __inline void \ 102239268Sgonzoatomic_##NAME##_acq_long(__volatile u_long *p, u_long v) \ 103239268Sgonzo{ \ 104239268Sgonzo atomic_##NAME##_long(p, v); \ 105239268Sgonzo __do_dmb(); \ 106239268Sgonzo} \ 107239268Sgonzo \ 108239268Sgonzostatic __inline void \ 109239268Sgonzoatomic_##NAME##_rel_long(__volatile u_long *p, u_long v) \ 110239268Sgonzo{ \ 111239268Sgonzo __do_dmb(); \ 112239268Sgonzo atomic_##NAME##_long(p, v); \ 113239268Sgonzo} 114239268Sgonzo 115239268Sgonzo#define ATOMIC_ACQ_REL(NAME, WIDTH) \ 116239268Sgonzostatic __inline void \ 117239268Sgonzoatomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 118239268Sgonzo{ \ 119239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 120239268Sgonzo __do_dmb(); \ 121239268Sgonzo} \ 122239268Sgonzo \ 123239268Sgonzostatic __inline void \ 124239268Sgonzoatomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ 125239268Sgonzo{ \ 126239268Sgonzo __do_dmb(); \ 127239268Sgonzo atomic_##NAME##_##WIDTH(p, v); \ 128239268Sgonzo} 129239268Sgonzo 130239268Sgonzostatic __inline void 131239268Sgonzoatomic_set_32(volatile uint32_t *address, uint32_t setmask) 132239268Sgonzo{ 133239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 134239268Sgonzo 135239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 136239268Sgonzo "orr %0, %0, %3\n" 137239268Sgonzo "strex %1, %0, [%2]\n" 138239268Sgonzo "cmp %1, #0\n" 139253489Sandrew "it ne\n" 140239268Sgonzo "bne 1b\n" 141239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 142241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 143239268Sgonzo 144239268Sgonzo} 145239268Sgonzo 146239268Sgonzostatic __inline void 147269798Sianatomic_set_64(volatile uint64_t *p, uint64_t val) 148269798Sian{ 149269798Sian uint64_t tmp; 150269798Sian uint32_t exflag; 151269798Sian 152269798Sian __asm __volatile( 153269798Sian "1: \n" 154269798Sian " ldrexd %[tmp], [%[ptr]]\n" 155269798Sian " orr %Q[tmp], %Q[val]\n" 156269798Sian " orr %R[tmp], %R[val]\n" 157269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 158269798Sian " teq %[exf], #0\n" 159269798Sian " it ne \n" 160269798Sian " bne 1b\n" 161269798Sian : [exf] "=&r" (exflag), 162269798Sian [tmp] "=&r" (tmp) 163269798Sian : [ptr] "r" (p), 164269798Sian [val] "r" (val) 165269798Sian : "cc", "memory"); 166269798Sian} 167269798Sian 168269798Sianstatic __inline void 169239268Sgonzoatomic_set_long(volatile u_long *address, u_long setmask) 170239268Sgonzo{ 171239268Sgonzo u_long tmp = 0, tmp2 = 0; 172239268Sgonzo 173239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 174239268Sgonzo "orr %0, %0, %3\n" 175239268Sgonzo "strex %1, %0, [%2]\n" 176239268Sgonzo "cmp %1, #0\n" 177253489Sandrew "it ne\n" 178239268Sgonzo "bne 1b\n" 179239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 180241080Sandrew , "+r" (address), "+r" (setmask) : : "cc", "memory"); 181239268Sgonzo 182239268Sgonzo} 183239268Sgonzo 184239268Sgonzostatic __inline void 185239268Sgonzoatomic_clear_32(volatile uint32_t *address, uint32_t setmask) 186239268Sgonzo{ 187239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 188239268Sgonzo 189239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 190239268Sgonzo "bic %0, %0, %3\n" 191239268Sgonzo "strex %1, %0, [%2]\n" 192239268Sgonzo "cmp %1, #0\n" 193253489Sandrew "it ne\n" 194239268Sgonzo "bne 1b\n" 195239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 196241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 197239268Sgonzo} 198239268Sgonzo 199239268Sgonzostatic __inline void 200269798Sianatomic_clear_64(volatile uint64_t *p, uint64_t val) 201269798Sian{ 202269798Sian uint64_t tmp; 203269798Sian uint32_t exflag; 204269798Sian 205269798Sian __asm __volatile( 206269798Sian "1: \n" 207269798Sian " ldrexd %[tmp], [%[ptr]]\n" 208269798Sian " bic %Q[tmp], %Q[val]\n" 209269798Sian " bic %R[tmp], %R[val]\n" 210269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 211269798Sian " teq %[exf], #0\n" 212269798Sian " it ne \n" 213269798Sian " bne 1b\n" 214269798Sian : [exf] "=&r" (exflag), 215269798Sian [tmp] "=&r" (tmp) 216269798Sian : [ptr] "r" (p), 217269798Sian [val] "r" (val) 218269798Sian : "cc", "memory"); 219269798Sian} 220269798Sian 221269798Sianstatic __inline void 222239268Sgonzoatomic_clear_long(volatile u_long *address, u_long setmask) 223239268Sgonzo{ 224239268Sgonzo u_long tmp = 0, tmp2 = 0; 225239268Sgonzo 226239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 227239268Sgonzo "bic %0, %0, %3\n" 228239268Sgonzo "strex %1, %0, [%2]\n" 229239268Sgonzo "cmp %1, #0\n" 230253489Sandrew "it ne\n" 231239268Sgonzo "bne 1b\n" 232239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 233241080Sandrew ,"+r" (address), "+r" (setmask) : : "cc", "memory"); 234239268Sgonzo} 235239268Sgonzo 236239268Sgonzostatic __inline u_int32_t 237239268Sgonzoatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 238239268Sgonzo{ 239239268Sgonzo uint32_t ret; 240239268Sgonzo 241239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 242239268Sgonzo "cmp %0, %2\n" 243266408Sian "itt ne\n" 244239268Sgonzo "movne %0, #0\n" 245239268Sgonzo "bne 2f\n" 246239268Sgonzo "strex %0, %3, [%1]\n" 247239268Sgonzo "cmp %0, #0\n" 248253489Sandrew "ite eq\n" 249253489Sandrew "moveq %0, #1\n" 250239268Sgonzo "bne 1b\n" 251239268Sgonzo "2:" 252239268Sgonzo : "=&r" (ret) 253241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 254241080Sandrew "memory"); 255239268Sgonzo return (ret); 256239268Sgonzo} 257239268Sgonzo 258269798Sianstatic __inline int 259269798Sianatomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) 260269798Sian{ 261269798Sian uint64_t tmp; 262269798Sian uint32_t ret; 263269798Sian 264269798Sian __asm __volatile( 265269798Sian "1: \n" 266269798Sian " ldrexd %[tmp], [%[ptr]]\n" 267273667Sian " teq %Q[tmp], %Q[cmpval]\n" 268269798Sian " itee eq \n" 269273667Sian " teqeq %R[tmp], %R[cmpval]\n" 270269798Sian " movne %[ret], #0\n" 271269798Sian " bne 2f\n" 272273667Sian " strexd %[ret], %[newval], [%[ptr]]\n" 273269798Sian " teq %[ret], #0\n" 274269798Sian " it ne \n" 275269798Sian " bne 1b\n" 276269798Sian " mov %[ret], #1\n" 277269798Sian "2: \n" 278269798Sian : [ret] "=&r" (ret), 279269798Sian [tmp] "=&r" (tmp) 280269798Sian : [ptr] "r" (p), 281273667Sian [cmpval] "r" (cmpval), 282273667Sian [newval] "r" (newval) 283269798Sian : "cc", "memory"); 284269798Sian return (ret); 285269798Sian} 286269798Sian 287239268Sgonzostatic __inline u_long 288239268Sgonzoatomic_cmpset_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 289239268Sgonzo{ 290239268Sgonzo u_long ret; 291239268Sgonzo 292239268Sgonzo __asm __volatile("1: ldrex %0, [%1]\n" 293239268Sgonzo "cmp %0, %2\n" 294253489Sandrew "itt ne\n" 295239268Sgonzo "movne %0, #0\n" 296239268Sgonzo "bne 2f\n" 297239268Sgonzo "strex %0, %3, [%1]\n" 298239268Sgonzo "cmp %0, #0\n" 299253489Sandrew "ite eq\n" 300253489Sandrew "moveq %0, #1\n" 301239268Sgonzo "bne 1b\n" 302239268Sgonzo "2:" 303239268Sgonzo : "=&r" (ret) 304241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", 305241080Sandrew "memory"); 306239268Sgonzo return (ret); 307239268Sgonzo} 308239268Sgonzo 309239268Sgonzostatic __inline u_int32_t 310239268Sgonzoatomic_cmpset_acq_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 311239268Sgonzo{ 312239268Sgonzo u_int32_t ret = atomic_cmpset_32(p, cmpval, newval); 313239268Sgonzo 314239268Sgonzo __do_dmb(); 315239268Sgonzo return (ret); 316239268Sgonzo} 317239268Sgonzo 318269798Sianstatic __inline uint64_t 319269798Sianatomic_cmpset_acq_64(volatile uint64_t *p, volatile uint64_t cmpval, volatile uint64_t newval) 320269798Sian{ 321269798Sian uint64_t ret = atomic_cmpset_64(p, cmpval, newval); 322269798Sian 323269798Sian __do_dmb(); 324269798Sian return (ret); 325269798Sian} 326269798Sian 327239268Sgonzostatic __inline u_long 328239268Sgonzoatomic_cmpset_acq_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 329239268Sgonzo{ 330239268Sgonzo u_long ret = atomic_cmpset_long(p, cmpval, newval); 331239268Sgonzo 332239268Sgonzo __do_dmb(); 333239268Sgonzo return (ret); 334239268Sgonzo} 335239268Sgonzo 336239268Sgonzostatic __inline u_int32_t 337239268Sgonzoatomic_cmpset_rel_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 338239268Sgonzo{ 339239268Sgonzo 340239268Sgonzo __do_dmb(); 341239268Sgonzo return (atomic_cmpset_32(p, cmpval, newval)); 342239268Sgonzo} 343239268Sgonzo 344269798Sianstatic __inline uint64_t 345269798Sianatomic_cmpset_rel_64(volatile uint64_t *p, volatile uint64_t cmpval, volatile uint64_t newval) 346269798Sian{ 347269798Sian 348269798Sian __do_dmb(); 349269798Sian return (atomic_cmpset_64(p, cmpval, newval)); 350269798Sian} 351269798Sian 352239268Sgonzostatic __inline u_long 353239268Sgonzoatomic_cmpset_rel_long(volatile u_long *p, volatile u_long cmpval, volatile u_long newval) 354239268Sgonzo{ 355239268Sgonzo 356239268Sgonzo __do_dmb(); 357239268Sgonzo return (atomic_cmpset_long(p, cmpval, newval)); 358239268Sgonzo} 359239268Sgonzo 360239268Sgonzo 361239268Sgonzostatic __inline void 362239268Sgonzoatomic_add_32(volatile u_int32_t *p, u_int32_t val) 363239268Sgonzo{ 364239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 365239268Sgonzo 366239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 367239268Sgonzo "add %0, %0, %3\n" 368239268Sgonzo "strex %1, %0, [%2]\n" 369239268Sgonzo "cmp %1, #0\n" 370253489Sandrew "it ne\n" 371239268Sgonzo "bne 1b\n" 372239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 373241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 374239268Sgonzo} 375239268Sgonzo 376239268Sgonzostatic __inline void 377269798Sianatomic_add_64(volatile uint64_t *p, uint64_t val) 378269798Sian{ 379269798Sian uint64_t tmp; 380269798Sian uint32_t exflag; 381269798Sian 382269798Sian __asm __volatile( 383269798Sian "1: \n" 384269798Sian " ldrexd %[tmp], [%[ptr]]\n" 385269798Sian " adds %Q[tmp], %Q[val]\n" 386269798Sian " adc %R[tmp], %R[val]\n" 387269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 388269798Sian " teq %[exf], #0\n" 389269798Sian " it ne \n" 390269798Sian " bne 1b\n" 391269798Sian : [exf] "=&r" (exflag), 392269798Sian [tmp] "=&r" (tmp) 393269798Sian : [ptr] "r" (p), 394269798Sian [val] "r" (val) 395269798Sian : "cc", "memory"); 396269798Sian} 397269798Sian 398269798Sianstatic __inline void 399239268Sgonzoatomic_add_long(volatile u_long *p, u_long val) 400239268Sgonzo{ 401239268Sgonzo u_long tmp = 0, tmp2 = 0; 402239268Sgonzo 403239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 404239268Sgonzo "add %0, %0, %3\n" 405239268Sgonzo "strex %1, %0, [%2]\n" 406239268Sgonzo "cmp %1, #0\n" 407253489Sandrew "it ne\n" 408239268Sgonzo "bne 1b\n" 409239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 410241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 411239268Sgonzo} 412239268Sgonzo 413239268Sgonzostatic __inline void 414239268Sgonzoatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 415239268Sgonzo{ 416239268Sgonzo uint32_t tmp = 0, tmp2 = 0; 417239268Sgonzo 418239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 419239268Sgonzo "sub %0, %0, %3\n" 420239268Sgonzo "strex %1, %0, [%2]\n" 421239268Sgonzo "cmp %1, #0\n" 422253489Sandrew "it ne\n" 423239268Sgonzo "bne 1b\n" 424239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 425241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 426239268Sgonzo} 427239268Sgonzo 428239268Sgonzostatic __inline void 429269798Sianatomic_subtract_64(volatile uint64_t *p, uint64_t val) 430269798Sian{ 431269798Sian uint64_t tmp; 432269798Sian uint32_t exflag; 433269798Sian 434269798Sian __asm __volatile( 435269798Sian "1: \n" 436269798Sian " ldrexd %[tmp], [%[ptr]]\n" 437269798Sian " subs %Q[tmp], %Q[val]\n" 438269798Sian " sbc %R[tmp], %R[val]\n" 439269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 440269798Sian " teq %[exf], #0\n" 441269798Sian " it ne \n" 442269798Sian " bne 1b\n" 443269798Sian : [exf] "=&r" (exflag), 444269798Sian [tmp] "=&r" (tmp) 445269798Sian : [ptr] "r" (p), 446269798Sian [val] "r" (val) 447269798Sian : "cc", "memory"); 448269798Sian} 449269798Sian 450269798Sianstatic __inline void 451239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long val) 452239268Sgonzo{ 453239268Sgonzo u_long tmp = 0, tmp2 = 0; 454239268Sgonzo 455239268Sgonzo __asm __volatile("1: ldrex %0, [%2]\n" 456239268Sgonzo "sub %0, %0, %3\n" 457239268Sgonzo "strex %1, %0, [%2]\n" 458239268Sgonzo "cmp %1, #0\n" 459253489Sandrew "it ne\n" 460239268Sgonzo "bne 1b\n" 461239268Sgonzo : "=&r" (tmp), "+r" (tmp2) 462241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 463239268Sgonzo} 464239268Sgonzo 465239268SgonzoATOMIC_ACQ_REL(clear, 32) 466239268SgonzoATOMIC_ACQ_REL(add, 32) 467239268SgonzoATOMIC_ACQ_REL(subtract, 32) 468239268SgonzoATOMIC_ACQ_REL(set, 32) 469269798SianATOMIC_ACQ_REL(clear, 64) 470269798SianATOMIC_ACQ_REL(add, 64) 471269798SianATOMIC_ACQ_REL(subtract, 64) 472269798SianATOMIC_ACQ_REL(set, 64) 473239268SgonzoATOMIC_ACQ_REL_LONG(clear) 474239268SgonzoATOMIC_ACQ_REL_LONG(add) 475239268SgonzoATOMIC_ACQ_REL_LONG(subtract) 476239268SgonzoATOMIC_ACQ_REL_LONG(set) 477239268Sgonzo 478239268Sgonzo#undef ATOMIC_ACQ_REL 479239268Sgonzo#undef ATOMIC_ACQ_REL_LONG 480239268Sgonzo 481239268Sgonzostatic __inline uint32_t 482239268Sgonzoatomic_fetchadd_32(volatile uint32_t *p, uint32_t val) 483239268Sgonzo{ 484239268Sgonzo uint32_t tmp = 0, tmp2 = 0, ret = 0; 485239268Sgonzo 486239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 487239268Sgonzo "add %1, %0, %4\n" 488239268Sgonzo "strex %2, %1, [%3]\n" 489239268Sgonzo "cmp %2, #0\n" 490253489Sandrew "it ne\n" 491239268Sgonzo "bne 1b\n" 492239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 493241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 494239268Sgonzo return (ret); 495239268Sgonzo} 496239268Sgonzo 497239268Sgonzostatic __inline uint32_t 498239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 499239268Sgonzo{ 500239268Sgonzo uint32_t ret, tmp = 0, tmp2 = 0; 501239268Sgonzo 502239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 503239268Sgonzo "mov %1, #0\n" 504239268Sgonzo "strex %2, %1, [%3]\n" 505239268Sgonzo "cmp %2, #0\n" 506253489Sandrew "it ne\n" 507239268Sgonzo "bne 1b\n" 508239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 509241080Sandrew ,"+r" (p) : : "cc", "memory"); 510239268Sgonzo return (ret); 511239268Sgonzo} 512239268Sgonzo 513239268Sgonzostatic __inline uint32_t 514239268Sgonzoatomic_load_acq_32(volatile uint32_t *p) 515239268Sgonzo{ 516239268Sgonzo uint32_t v; 517239268Sgonzo 518239268Sgonzo v = *p; 519239268Sgonzo __do_dmb(); 520239268Sgonzo return (v); 521239268Sgonzo} 522239268Sgonzo 523239268Sgonzostatic __inline void 524239268Sgonzoatomic_store_rel_32(volatile uint32_t *p, uint32_t v) 525239268Sgonzo{ 526239268Sgonzo 527239268Sgonzo __do_dmb(); 528239268Sgonzo *p = v; 529239268Sgonzo} 530239268Sgonzo 531269798Sianstatic __inline uint64_t 532269798Sianatomic_fetchadd_64(volatile uint64_t *p, uint64_t val) 533269798Sian{ 534269798Sian uint64_t ret, tmp; 535269798Sian uint32_t exflag; 536269798Sian 537269798Sian __asm __volatile( 538269798Sian "1: \n" 539269798Sian " ldrexd %[ret], [%[ptr]]\n" 540269798Sian " adds %Q[tmp], %Q[ret], %Q[val]\n" 541269798Sian " adc %R[tmp], %R[ret], %R[val]\n" 542269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 543269798Sian " teq %[exf], #0\n" 544269798Sian " it ne \n" 545269798Sian " bne 1b\n" 546269798Sian : [ret] "=&r" (ret), 547269798Sian [exf] "=&r" (exflag), 548269798Sian [tmp] "=&r" (tmp) 549269798Sian : [ptr] "r" (p), 550269798Sian [val] "r" (val) 551269798Sian : "cc", "memory"); 552269798Sian return (ret); 553269798Sian} 554269798Sian 555269798Sianstatic __inline uint64_t 556269798Sianatomic_readandclear_64(volatile uint64_t *p) 557269798Sian{ 558269798Sian uint64_t ret, tmp; 559269798Sian uint32_t exflag; 560269798Sian 561269798Sian __asm __volatile( 562269798Sian "1: \n" 563269798Sian " ldrexd %[ret], [%[ptr]]\n" 564269798Sian " mov %Q[tmp], #0\n" 565269798Sian " mov %R[tmp], #0\n" 566269798Sian " strexd %[exf], %[tmp], [%[ptr]]\n" 567269798Sian " teq %[exf], #0\n" 568269798Sian " it ne \n" 569269798Sian " bne 1b\n" 570269798Sian : [ret] "=&r" (ret), 571269798Sian [exf] "=&r" (exflag), 572269798Sian [tmp] "=&r" (tmp) 573269798Sian : [ptr] "r" (p) 574269798Sian : "cc", "memory"); 575269798Sian return (ret); 576269798Sian} 577269798Sian 578269798Sianstatic __inline uint64_t 579269798Sianatomic_load_64(volatile uint64_t *p) 580269798Sian{ 581269798Sian uint64_t ret; 582269798Sian 583269798Sian /* 584269798Sian * The only way to atomically load 64 bits is with LDREXD which puts the 585283317Sian * exclusive monitor into the exclusive state, so reset it to open state 586283317Sian * with CLREX because we don't actually need to store anything. 587269798Sian */ 588269798Sian __asm __volatile( 589269798Sian "1: \n" 590269798Sian " ldrexd %[ret], [%[ptr]]\n" 591269798Sian " clrex \n" 592269798Sian : [ret] "=&r" (ret) 593269798Sian : [ptr] "r" (p) 594269798Sian : "cc", "memory"); 595269798Sian return (ret); 596269798Sian} 597269798Sian 598269798Sianstatic __inline uint64_t 599269798Sianatomic_load_acq_64(volatile uint64_t *p) 600269798Sian{ 601269798Sian uint64_t ret; 602269798Sian 603269798Sian ret = atomic_load_64(p); 604269798Sian __do_dmb(); 605269798Sian return (ret); 606269798Sian} 607269798Sian 608269798Sianstatic __inline void 609269798Sianatomic_store_64(volatile uint64_t *p, uint64_t val) 610269798Sian{ 611269798Sian uint64_t tmp; 612269798Sian uint32_t exflag; 613269798Sian 614269798Sian /* 615269798Sian * The only way to atomically store 64 bits is with STREXD, which will 616269798Sian * succeed only if paired up with a preceeding LDREXD using the same 617269798Sian * address, so we read and discard the existing value before storing. 618269798Sian */ 619269798Sian __asm __volatile( 620269798Sian "1: \n" 621269798Sian " ldrexd %[tmp], [%[ptr]]\n" 622269798Sian " strexd %[exf], %[val], [%[ptr]]\n" 623269798Sian " teq %[exf], #0\n" 624269798Sian " it ne \n" 625269798Sian " bne 1b\n" 626269798Sian : [tmp] "=&r" (tmp), 627269798Sian [exf] "=&r" (exflag) 628269798Sian : [ptr] "r" (p), 629269798Sian [val] "r" (val) 630269798Sian : "cc", "memory"); 631269798Sian} 632269798Sian 633269798Sianstatic __inline void 634269798Sianatomic_store_rel_64(volatile uint64_t *p, uint64_t val) 635269798Sian{ 636269798Sian 637269798Sian __do_dmb(); 638269798Sian atomic_store_64(p, val); 639269798Sian} 640269798Sian 641239268Sgonzostatic __inline u_long 642239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long val) 643239268Sgonzo{ 644239268Sgonzo u_long tmp = 0, tmp2 = 0, ret = 0; 645239268Sgonzo 646239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 647239268Sgonzo "add %1, %0, %4\n" 648239268Sgonzo "strex %2, %1, [%3]\n" 649239268Sgonzo "cmp %2, #0\n" 650253489Sandrew "it ne\n" 651239268Sgonzo "bne 1b\n" 652239268Sgonzo : "+r" (ret), "=&r" (tmp), "+r" (tmp2) 653241080Sandrew ,"+r" (p), "+r" (val) : : "cc", "memory"); 654239268Sgonzo return (ret); 655239268Sgonzo} 656239268Sgonzo 657239268Sgonzostatic __inline u_long 658239268Sgonzoatomic_readandclear_long(volatile u_long *p) 659239268Sgonzo{ 660239268Sgonzo u_long ret, tmp = 0, tmp2 = 0; 661239268Sgonzo 662239268Sgonzo __asm __volatile("1: ldrex %0, [%3]\n" 663239268Sgonzo "mov %1, #0\n" 664239268Sgonzo "strex %2, %1, [%3]\n" 665239268Sgonzo "cmp %2, #0\n" 666253489Sandrew "it ne\n" 667239268Sgonzo "bne 1b\n" 668239268Sgonzo : "=r" (ret), "=&r" (tmp), "+r" (tmp2) 669241080Sandrew ,"+r" (p) : : "cc", "memory"); 670239268Sgonzo return (ret); 671239268Sgonzo} 672239268Sgonzo 673239268Sgonzostatic __inline u_long 674239268Sgonzoatomic_load_acq_long(volatile u_long *p) 675239268Sgonzo{ 676239268Sgonzo u_long v; 677239268Sgonzo 678239268Sgonzo v = *p; 679239268Sgonzo __do_dmb(); 680239268Sgonzo return (v); 681239268Sgonzo} 682239268Sgonzo 683239268Sgonzostatic __inline void 684239268Sgonzoatomic_store_rel_long(volatile u_long *p, u_long v) 685239268Sgonzo{ 686239268Sgonzo 687239268Sgonzo __do_dmb(); 688239268Sgonzo *p = v; 689239268Sgonzo} 690239268Sgonzo#else /* < armv6 */ 691239268Sgonzo 692129198Scognet#define __with_interrupts_disabled(expr) \ 693129198Scognet do { \ 694129198Scognet u_int cpsr_save, tmp; \ 695129198Scognet \ 696129198Scognet __asm __volatile( \ 697129198Scognet "mrs %0, cpsr;" \ 698129198Scognet "orr %1, %0, %2;" \ 699266144Sian "msr cpsr_fsxc, %1;" \ 700129198Scognet : "=r" (cpsr_save), "=r" (tmp) \ 701278613Sian : "I" (PSR_I | PSR_F) \ 702129198Scognet : "cc" ); \ 703129198Scognet (expr); \ 704129198Scognet __asm __volatile( \ 705266144Sian "msr cpsr_fsxc, %0" \ 706129198Scognet : /* no output */ \ 707129198Scognet : "r" (cpsr_save) \ 708129198Scognet : "cc" ); \ 709129198Scognet } while(0) 710129198Scognet 711137222Scognetstatic __inline uint32_t 712137222Scognet__swp(uint32_t val, volatile uint32_t *ptr) 713129198Scognet{ 714148453Sjhb __asm __volatile("swp %0, %2, [%3]" 715148453Sjhb : "=&r" (val), "=m" (*ptr) 716151340Sjhb : "r" (val), "r" (ptr), "m" (*ptr) 717148453Sjhb : "memory"); 718137222Scognet return (val); 719129198Scognet} 720129198Scognet 721137222Scognet 722144761Scognet#ifdef _KERNEL 723269798Sian#define ARM_HAVE_ATOMIC64 724269798Sian 725129198Scognetstatic __inline void 726137222Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 727129198Scognet{ 728144761Scognet __with_interrupts_disabled(*address |= setmask); 729129198Scognet} 730129198Scognet 731129198Scognetstatic __inline void 732269798Sianatomic_set_64(volatile uint64_t *address, uint64_t setmask) 733269798Sian{ 734269798Sian __with_interrupts_disabled(*address |= setmask); 735269798Sian} 736269798Sian 737269798Sianstatic __inline void 738129198Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 739129198Scognet{ 740144761Scognet __with_interrupts_disabled(*address &= ~clearmask); 741129198Scognet} 742129198Scognet 743269798Sianstatic __inline void 744269798Sianatomic_clear_64(volatile uint64_t *address, uint64_t clearmask) 745269798Sian{ 746269798Sian __with_interrupts_disabled(*address &= ~clearmask); 747269798Sian} 748269798Sian 749144761Scognetstatic __inline u_int32_t 750144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 751129198Scognet{ 752144761Scognet int ret; 753144761Scognet 754144761Scognet __with_interrupts_disabled( 755144761Scognet { 756144761Scognet if (*p == cmpval) { 757144761Scognet *p = newval; 758144761Scognet ret = 1; 759144761Scognet } else { 760144761Scognet ret = 0; 761144761Scognet } 762144761Scognet }); 763144761Scognet return (ret); 764129198Scognet} 765129198Scognet 766269798Sianstatic __inline u_int64_t 767269798Sianatomic_cmpset_64(volatile u_int64_t *p, volatile u_int64_t cmpval, volatile u_int64_t newval) 768269798Sian{ 769269798Sian int ret; 770269798Sian 771269798Sian __with_interrupts_disabled( 772269798Sian { 773269798Sian if (*p == cmpval) { 774269798Sian *p = newval; 775269798Sian ret = 1; 776269798Sian } else { 777269798Sian ret = 0; 778269798Sian } 779269798Sian }); 780269798Sian return (ret); 781269798Sian} 782269798Sian 783129198Scognetstatic __inline void 784144761Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 785129198Scognet{ 786144761Scognet __with_interrupts_disabled(*p += val); 787129198Scognet} 788129198Scognet 789144761Scognetstatic __inline void 790269798Sianatomic_add_64(volatile u_int64_t *p, u_int64_t val) 791269798Sian{ 792269798Sian __with_interrupts_disabled(*p += val); 793269798Sian} 794269798Sian 795269798Sianstatic __inline void 796144761Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 797129198Scognet{ 798144761Scognet __with_interrupts_disabled(*p -= val); 799129198Scognet} 800129198Scognet 801269798Sianstatic __inline void 802269798Sianatomic_subtract_64(volatile u_int64_t *p, u_int64_t val) 803269798Sian{ 804269798Sian __with_interrupts_disabled(*p -= val); 805269798Sian} 806269798Sian 807150627Sjhbstatic __inline uint32_t 808150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 809150627Sjhb{ 810150627Sjhb uint32_t value; 811150627Sjhb 812150627Sjhb __with_interrupts_disabled( 813150627Sjhb { 814150627Sjhb value = *p; 815150627Sjhb *p += v; 816150627Sjhb }); 817150627Sjhb return (value); 818150627Sjhb} 819150627Sjhb 820269798Sianstatic __inline uint64_t 821269798Sianatomic_fetchadd_64(volatile uint64_t *p, uint64_t v) 822269798Sian{ 823269798Sian uint64_t value; 824269798Sian 825269798Sian __with_interrupts_disabled( 826269798Sian { 827269798Sian value = *p; 828269798Sian *p += v; 829269798Sian }); 830269798Sian return (value); 831269798Sian} 832269798Sian 833269798Sianstatic __inline uint64_t 834269798Sianatomic_load_64(volatile uint64_t *p) 835269798Sian{ 836269798Sian uint64_t value; 837269798Sian 838269798Sian __with_interrupts_disabled(value = *p); 839269798Sian return (value); 840269798Sian} 841269798Sian 842269798Sianstatic __inline void 843269798Sianatomic_store_64(volatile uint64_t *p, uint64_t value) 844269798Sian{ 845269798Sian __with_interrupts_disabled(*p = value); 846269798Sian} 847269798Sian 848144761Scognet#else /* !_KERNEL */ 849144761Scognet 850129198Scognetstatic __inline u_int32_t 851144761Scognetatomic_cmpset_32(volatile u_int32_t *p, volatile u_int32_t cmpval, volatile u_int32_t newval) 852129198Scognet{ 853175982Sraj register int done, ras_start = ARM_RAS_START; 854144761Scognet 855144761Scognet __asm __volatile("1:\n" 856174170Scognet "adr %1, 1b\n" 857174170Scognet "str %1, [%0]\n" 858144761Scognet "adr %1, 2f\n" 859175982Sraj "str %1, [%0, #4]\n" 860155355Scognet "ldr %1, [%2]\n" 861144761Scognet "cmp %1, %3\n" 862155355Scognet "streq %4, [%2]\n" 863144761Scognet "2:\n" 864146591Scognet "mov %1, #0\n" 865146591Scognet "str %1, [%0]\n" 866174170Scognet "mov %1, #0xffffffff\n" 867175982Sraj "str %1, [%0, #4]\n" 868144761Scognet "moveq %1, #1\n" 869144761Scognet "movne %1, #0\n" 870175982Sraj : "+r" (ras_start), "=r" (done) 871241080Sandrew ,"+r" (p), "+r" (cmpval), "+r" (newval) : : "cc", "memory"); 872137282Scognet return (done); 873129198Scognet} 874129198Scognet 875129198Scognetstatic __inline void 876129198Scognetatomic_add_32(volatile u_int32_t *p, u_int32_t val) 877129198Scognet{ 878175982Sraj int start, ras_start = ARM_RAS_START; 879144761Scognet 880144761Scognet __asm __volatile("1:\n" 881174170Scognet "adr %1, 1b\n" 882174170Scognet "str %1, [%0]\n" 883144761Scognet "adr %1, 2f\n" 884175982Sraj "str %1, [%0, #4]\n" 885155355Scognet "ldr %1, [%2]\n" 886144761Scognet "add %1, %1, %3\n" 887155355Scognet "str %1, [%2]\n" 888144761Scognet "2:\n" 889146591Scognet "mov %1, #0\n" 890146591Scognet "str %1, [%0]\n" 891174170Scognet "mov %1, #0xffffffff\n" 892175982Sraj "str %1, [%0, #4]\n" 893175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 894155391Scognet : : "memory"); 895129198Scognet} 896129198Scognet 897129198Scognetstatic __inline void 898129198Scognetatomic_subtract_32(volatile u_int32_t *p, u_int32_t val) 899129198Scognet{ 900175982Sraj int start, ras_start = ARM_RAS_START; 901144761Scognet 902144761Scognet __asm __volatile("1:\n" 903174170Scognet "adr %1, 1b\n" 904174170Scognet "str %1, [%0]\n" 905144761Scognet "adr %1, 2f\n" 906175982Sraj "str %1, [%0, #4]\n" 907155355Scognet "ldr %1, [%2]\n" 908144761Scognet "sub %1, %1, %3\n" 909155355Scognet "str %1, [%2]\n" 910144761Scognet "2:\n" 911146591Scognet "mov %1, #0\n" 912146591Scognet "str %1, [%0]\n" 913174170Scognet "mov %1, #0xffffffff\n" 914175982Sraj "str %1, [%0, #4]\n" 915146591Scognet 916175982Sraj : "+r" (ras_start), "=r" (start), "+r" (p), "+r" (val) 917155391Scognet : : "memory"); 918129198Scognet} 919129198Scognet 920144761Scognetstatic __inline void 921144761Scognetatomic_set_32(volatile uint32_t *address, uint32_t setmask) 922144761Scognet{ 923175982Sraj int start, ras_start = ARM_RAS_START; 924144761Scognet 925144761Scognet __asm __volatile("1:\n" 926174170Scognet "adr %1, 1b\n" 927174170Scognet "str %1, [%0]\n" 928144761Scognet "adr %1, 2f\n" 929175982Sraj "str %1, [%0, #4]\n" 930155355Scognet "ldr %1, [%2]\n" 931144761Scognet "orr %1, %1, %3\n" 932155355Scognet "str %1, [%2]\n" 933144761Scognet "2:\n" 934146591Scognet "mov %1, #0\n" 935146591Scognet "str %1, [%0]\n" 936174170Scognet "mov %1, #0xffffffff\n" 937175982Sraj "str %1, [%0, #4]\n" 938146591Scognet 939175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (setmask) 940155391Scognet : : "memory"); 941144761Scognet} 942144761Scognet 943144761Scognetstatic __inline void 944144761Scognetatomic_clear_32(volatile uint32_t *address, uint32_t clearmask) 945144761Scognet{ 946175982Sraj int start, ras_start = ARM_RAS_START; 947144761Scognet 948144761Scognet __asm __volatile("1:\n" 949174170Scognet "adr %1, 1b\n" 950174170Scognet "str %1, [%0]\n" 951144761Scognet "adr %1, 2f\n" 952175982Sraj "str %1, [%0, #4]\n" 953155355Scognet "ldr %1, [%2]\n" 954144761Scognet "bic %1, %1, %3\n" 955155355Scognet "str %1, [%2]\n" 956144761Scognet "2:\n" 957146591Scognet "mov %1, #0\n" 958146591Scognet "str %1, [%0]\n" 959174170Scognet "mov %1, #0xffffffff\n" 960175982Sraj "str %1, [%0, #4]\n" 961175982Sraj : "+r" (ras_start), "=r" (start), "+r" (address), "+r" (clearmask) 962155391Scognet : : "memory"); 963144761Scognet 964144761Scognet} 965150627Sjhb 966150627Sjhbstatic __inline uint32_t 967150627Sjhbatomic_fetchadd_32(volatile uint32_t *p, uint32_t v) 968150627Sjhb{ 969190603Scognet uint32_t start, tmp, ras_start = ARM_RAS_START; 970150627Sjhb 971150627Sjhb __asm __volatile("1:\n" 972174170Scognet "adr %1, 1b\n" 973174170Scognet "str %1, [%0]\n" 974150627Sjhb "adr %1, 2f\n" 975175982Sraj "str %1, [%0, #4]\n" 976190603Scognet "ldr %1, [%3]\n" 977190603Scognet "mov %2, %1\n" 978190603Scognet "add %2, %2, %4\n" 979190603Scognet "str %2, [%3]\n" 980150627Sjhb "2:\n" 981190603Scognet "mov %2, #0\n" 982190603Scognet "str %2, [%0]\n" 983190603Scognet "mov %2, #0xffffffff\n" 984190603Scognet "str %2, [%0, #4]\n" 985190603Scognet : "+r" (ras_start), "=r" (start), "=r" (tmp), "+r" (p), "+r" (v) 986155391Scognet : : "memory"); 987150627Sjhb return (start); 988150627Sjhb} 989150627Sjhb 990144761Scognet#endif /* _KERNEL */ 991144761Scognet 992239268Sgonzo 993239268Sgonzostatic __inline uint32_t 994239268Sgonzoatomic_readandclear_32(volatile u_int32_t *p) 995239268Sgonzo{ 996239268Sgonzo 997239268Sgonzo return (__swp(0, p)); 998239268Sgonzo} 999239268Sgonzo 1000239268Sgonzo#define atomic_cmpset_rel_32 atomic_cmpset_32 1001239268Sgonzo#define atomic_cmpset_acq_32 atomic_cmpset_32 1002239268Sgonzo#define atomic_set_rel_32 atomic_set_32 1003239268Sgonzo#define atomic_set_acq_32 atomic_set_32 1004239268Sgonzo#define atomic_clear_rel_32 atomic_clear_32 1005239268Sgonzo#define atomic_clear_acq_32 atomic_clear_32 1006239268Sgonzo#define atomic_add_rel_32 atomic_add_32 1007239268Sgonzo#define atomic_add_acq_32 atomic_add_32 1008239268Sgonzo#define atomic_subtract_rel_32 atomic_subtract_32 1009239268Sgonzo#define atomic_subtract_acq_32 atomic_subtract_32 1010239268Sgonzo#define atomic_store_rel_32 atomic_store_32 1011239268Sgonzo#define atomic_store_rel_long atomic_store_long 1012239268Sgonzo#define atomic_load_acq_32 atomic_load_32 1013239268Sgonzo#define atomic_load_acq_long atomic_load_long 1014245475Scognet#define atomic_add_acq_long atomic_add_long 1015245475Scognet#define atomic_add_rel_long atomic_add_long 1016245475Scognet#define atomic_subtract_acq_long atomic_subtract_long 1017245475Scognet#define atomic_subtract_rel_long atomic_subtract_long 1018245475Scognet#define atomic_clear_acq_long atomic_clear_long 1019245475Scognet#define atomic_clear_rel_long atomic_clear_long 1020245475Scognet#define atomic_set_acq_long atomic_set_long 1021245475Scognet#define atomic_set_rel_long atomic_set_long 1022245475Scognet#define atomic_cmpset_acq_long atomic_cmpset_long 1023245475Scognet#define atomic_cmpset_rel_long atomic_cmpset_long 1024245475Scognet#define atomic_load_acq_long atomic_load_long 1025239268Sgonzo#undef __with_interrupts_disabled 1026239268Sgonzo 1027239268Sgonzostatic __inline void 1028239268Sgonzoatomic_add_long(volatile u_long *p, u_long v) 1029239268Sgonzo{ 1030239268Sgonzo 1031239268Sgonzo atomic_add_32((volatile uint32_t *)p, v); 1032239268Sgonzo} 1033239268Sgonzo 1034239268Sgonzostatic __inline void 1035239268Sgonzoatomic_clear_long(volatile u_long *p, u_long v) 1036239268Sgonzo{ 1037239268Sgonzo 1038239268Sgonzo atomic_clear_32((volatile uint32_t *)p, v); 1039239268Sgonzo} 1040239268Sgonzo 1041144761Scognetstatic __inline int 1042239268Sgonzoatomic_cmpset_long(volatile u_long *dst, u_long old, u_long newe) 1043239268Sgonzo{ 1044239268Sgonzo 1045239268Sgonzo return (atomic_cmpset_32((volatile uint32_t *)dst, old, newe)); 1046239268Sgonzo} 1047239268Sgonzo 1048239268Sgonzostatic __inline u_long 1049239268Sgonzoatomic_fetchadd_long(volatile u_long *p, u_long v) 1050239268Sgonzo{ 1051239268Sgonzo 1052239268Sgonzo return (atomic_fetchadd_32((volatile uint32_t *)p, v)); 1053239268Sgonzo} 1054239268Sgonzo 1055239268Sgonzostatic __inline void 1056239268Sgonzoatomic_readandclear_long(volatile u_long *p) 1057239268Sgonzo{ 1058239268Sgonzo 1059239268Sgonzo atomic_readandclear_32((volatile uint32_t *)p); 1060239268Sgonzo} 1061239268Sgonzo 1062239268Sgonzostatic __inline void 1063239268Sgonzoatomic_set_long(volatile u_long *p, u_long v) 1064239268Sgonzo{ 1065239268Sgonzo 1066239268Sgonzo atomic_set_32((volatile uint32_t *)p, v); 1067239268Sgonzo} 1068239268Sgonzo 1069239268Sgonzostatic __inline void 1070239268Sgonzoatomic_subtract_long(volatile u_long *p, u_long v) 1071239268Sgonzo{ 1072239268Sgonzo 1073239268Sgonzo atomic_subtract_32((volatile uint32_t *)p, v); 1074239268Sgonzo} 1075239268Sgonzo 1076239268Sgonzo 1077239268Sgonzo 1078239268Sgonzo#endif /* Arch >= v6 */ 1079239268Sgonzo 1080239268Sgonzostatic __inline int 1081144761Scognetatomic_load_32(volatile uint32_t *v) 1082144761Scognet{ 1083144761Scognet 1084144761Scognet return (*v); 1085144761Scognet} 1086144761Scognet 1087144761Scognetstatic __inline void 1088144761Scognetatomic_store_32(volatile uint32_t *dst, uint32_t src) 1089144761Scognet{ 1090144761Scognet *dst = src; 1091144761Scognet} 1092144761Scognet 1093239268Sgonzostatic __inline int 1094239268Sgonzoatomic_load_long(volatile u_long *v) 1095144761Scognet{ 1096144761Scognet 1097239268Sgonzo return (*v); 1098144761Scognet} 1099144761Scognet 1100239268Sgonzostatic __inline void 1101239268Sgonzoatomic_store_long(volatile u_long *dst, u_long src) 1102239268Sgonzo{ 1103239268Sgonzo *dst = src; 1104239268Sgonzo} 1105129198Scognet 1106165786Sticso#define atomic_clear_ptr atomic_clear_32 1107165786Sticso#define atomic_set_ptr atomic_set_32 1108239268Sgonzo#define atomic_cmpset_ptr atomic_cmpset_32 1109239268Sgonzo#define atomic_cmpset_rel_ptr atomic_cmpset_rel_32 1110239268Sgonzo#define atomic_cmpset_acq_ptr atomic_cmpset_acq_32 1111165786Sticso#define atomic_store_ptr atomic_store_32 1112245475Scognet#define atomic_store_rel_ptr atomic_store_rel_32 1113165786Sticso 1114165786Sticso#define atomic_add_int atomic_add_32 1115239268Sgonzo#define atomic_add_acq_int atomic_add_acq_32 1116239268Sgonzo#define atomic_add_rel_int atomic_add_rel_32 1117165786Sticso#define atomic_subtract_int atomic_subtract_32 1118239268Sgonzo#define atomic_subtract_acq_int atomic_subtract_acq_32 1119239268Sgonzo#define atomic_subtract_rel_int atomic_subtract_rel_32 1120165786Sticso#define atomic_clear_int atomic_clear_32 1121239268Sgonzo#define atomic_clear_acq_int atomic_clear_acq_32 1122239268Sgonzo#define atomic_clear_rel_int atomic_clear_rel_32 1123137222Scognet#define atomic_set_int atomic_set_32 1124239268Sgonzo#define atomic_set_acq_int atomic_set_acq_32 1125239268Sgonzo#define atomic_set_rel_int atomic_set_rel_32 1126165786Sticso#define atomic_cmpset_int atomic_cmpset_32 1127239268Sgonzo#define atomic_cmpset_acq_int atomic_cmpset_acq_32 1128239268Sgonzo#define atomic_cmpset_rel_int atomic_cmpset_rel_32 1129165786Sticso#define atomic_fetchadd_int atomic_fetchadd_32 1130137222Scognet#define atomic_readandclear_int atomic_readandclear_32 1131239268Sgonzo#define atomic_load_acq_int atomic_load_acq_32 1132239268Sgonzo#define atomic_store_rel_int atomic_store_rel_32 1133165786Sticso 1134129198Scognet#endif /* _MACHINE_ATOMIC_H_ */ 1135