1/* 2 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <sys/appleapiopts.h> 30#include <ppc/asm.h> // EXT, LEXT 31#include <machine/cpu_capabilities.h> 32#include <machine/commpage.h> 33 34 35/* OSAtomic.h library native implementations. */ 36 37 .text 38 .align 2 39 40atomic_add32: // int32_t OSAtomicAdd32( int32_t amt, int32_t *value ); 411: 42 lwarx r5,0,r4 43 add r6,r3,r5 44 stwcx. r6,0,r4 45 bne-- 1b 46 mr r3,r6 47 blr 48 49 COMMPAGE_DESCRIPTOR(atomic_add32,_COMM_PAGE_ATOMIC_ADD32,0,0,kCommPageBoth) 50 51 52atomic_add64: // int64_t OSAtomicAdd64( int64_t amt, int64_t *value ); 531: 54 ldarx r5,0,r4 55 add r6,r3,r5 56 stdcx. r6,0,r4 57 bne-- 1b 58 mr r3,r6 59 blr 60 61 COMMPAGE_DESCRIPTOR(atomic_add64,_COMM_PAGE_ATOMIC_ADD64,k64Bit,0,kCommPage64) 62 63/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 64/* This is the no-barrier version */ 65compare_and_swap32_on32: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value); 661: 67 lwarx r7,0,r5 68 cmplw r7,r3 69 bne- 2f 70 stwcx. r4,0,r5 71 bne- 1b 72 li r3,1 73 blr 742: 75 li r3,0 // return failure 76 blr 77 78 COMMPAGE_DESCRIPTOR(compare_and_swap32_on32,_COMM_PAGE_COMPARE_AND_SWAP32,0,k64Bit,kCommPageBoth) 79 80 81/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 82/* This is the no-barrier version */ 83compare_and_swap32_on64: // bool OSAtomicCompareAndSwap32( int32_t old, int32_t new, int32_t *value); 841: 85 lwarx r7,0,r5 86 cmplw r7,r3 87 bne-- 2f 88 stwcx. r4,0,r5 89 bne-- 1b 90 li r3,1 91 blr 922: 93 li r8,-8 // on 970, must release reservation 94 li r3,0 // return failure 95 stwcx. r4,r8,r1 // store into red zone to release 96 blr 97 98 COMMPAGE_DESCRIPTOR(compare_and_swap32_on64,_COMM_PAGE_COMPARE_AND_SWAP32,k64Bit,0,kCommPageBoth) 99 100 101/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 102/* This is the no-barrier version */ 103compare_and_swap64: // bool OSAtomicCompareAndSwap64( int64_t old, int64_t new, int64_t *value); 1041: 105 ldarx r7,0,r5 106 cmpld r7,r3 107 bne-- 2f 108 stdcx. r4,0,r5 109 bne-- 1b 110 li r3,1 111 blr 1122: 113 li r8,-8 // on 970, must release reservation 114 li r3,0 // return failure 115 stdcx. r4,r8,r1 // store into red zone to release 116 blr 117 118 COMMPAGE_DESCRIPTOR(compare_and_swap64,_COMM_PAGE_COMPARE_AND_SWAP64,k64Bit,0,kCommPage64) 119 120/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 121/* This version of compare-and-swap incorporates a memory barrier. */ 122compare_and_swap32_on32b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value); 123 eieio // write barrier, NOP'd on a UP 1241: 125 lwarx r7,0,r5 126 cmplw r7,r3 127 bne- 2f 128 stwcx. r4,0,r5 129 bne- 1b 130 isync // read barrier, NOP'd on a UP 131 li r3,1 132 blr 1332: 134 li r3,0 // return failure 135 blr 136 137 COMMPAGE_DESCRIPTOR(compare_and_swap32_on32b,_COMM_PAGE_COMPARE_AND_SWAP32B,0,k64Bit,kCommPageBoth+kCommPageSYNC+kCommPageISYNC) 138 139 140/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 141/* This version of compare-and-swap incorporates a memory barrier. */ 142compare_and_swap32_on64b: // bool OSAtomicCompareAndSwapBarrier32( int32_t old, int32_t new, int32_t *value); 143 lwsync // write barrier, NOP'd on a UP 1441: 145 lwarx r7,0,r5 146 cmplw r7,r3 147 bne-- 2f 148 stwcx. r4,0,r5 149 bne-- 1b 150 isync // read barrier, NOP'd on a UP 151 li r3,1 152 blr 1532: 154 li r8,-8 // on 970, must release reservation 155 li r3,0 // return failure 156 stwcx. r4,r8,r1 // store into red zone to release 157 blr 158 159 COMMPAGE_DESCRIPTOR(compare_and_swap32_on64b,_COMM_PAGE_COMPARE_AND_SWAP32B,k64Bit,0,kCommPageBoth+kCommPageSYNC+kCommPageISYNC) 160 161 162/* WARNING: Libc clients assume compare-and-swap preserves r4, r5, and r9-r12! */ 163/* This version of compare-and-swap incorporates a memory barrier. */ 164compare_and_swap64b: // bool OSAtomicCompareAndSwapBarrier64( int64_t old, int64_t new, int64_t *value); 165 lwsync // write barrier, NOP'd on a UP 1661: 167 ldarx r7,0,r5 168 cmpld r7,r3 169 bne-- 2f 170 stdcx. r4,0,r5 171 bne-- 1b 172 isync // read barrier, NOP'd on a UP 173 li r3,1 174 blr 1752: 176 li r8,-8 // on 970, must release reservation 177 li r3,0 // return failure 178 stdcx. r4,r8,r1 // store into red zone to release 179 blr 180 181 COMMPAGE_DESCRIPTOR(compare_and_swap64b,_COMM_PAGE_COMPARE_AND_SWAP64B,k64Bit,0,kCommPage64+kCommPageSYNC+kCommPageISYNC) 182 183 184atomic_enqueue32: // void OSAtomicEnqueue( void **list, void *new, size_t offset); 1851: 186 lwarx r6,0,r3 // get link to 1st on list 187 stwx r6,r4,r5 // hang list off new node 188 eieio // make sure the "stwx" comes before "stwcx." (nop'd on UP) 189 stwcx. r4,0,r3 // make new 1st on list 190 beqlr++ 191 b 1b 192 193 COMMPAGE_DESCRIPTOR(atomic_enqueue32,_COMM_PAGE_ENQUEUE,0,0,kCommPageSYNC+kCommPage32) 194 195 196atomic_enqueue64: // void OSAtomicEnqueue( void **list, void *new, size_t offset); 1971: 198 ldarx r6,0,r3 // get link to 1st on list 199 stdx r6,r4,r5 // hang list off new node 200 lwsync // make sure the "stdx" comes before the "stdcx." (nop'd on UP) 201 stdcx. r4,0,r3 // make new 1st on list 202 beqlr++ 203 b 1b 204 205 COMMPAGE_DESCRIPTOR(atomic_enqueue64,_COMM_PAGE_ENQUEUE,k64Bit,0,kCommPageSYNC+kCommPage64) 206 207 208atomic_dequeue32_on32: // void* OSAtomicDequeue( void **list, size_t offset); 209 mr r5,r3 2101: 211 lwarx r3,0,r5 // get 1st in list 212 cmpwi r3,0 // null? 213 beqlr // yes, list empty 214 lwzx r6,r3,r4 // get 2nd 215 stwcx. r6,0,r5 // make 2nd first 216 bne-- 1b 217 isync // cancel read-aheads (nop'd on UP) 218 blr 219 220 COMMPAGE_DESCRIPTOR(atomic_dequeue32_on32,_COMM_PAGE_DEQUEUE,0,k64Bit,kCommPageISYNC+kCommPage32) 221 222 223atomic_dequeue32_on64: // void* OSAtomicDequeue( void **list, size_t offset); 224 mr r5,r3 225 li r7,-8 // use red zone to release reservation if necessary 2261: 227 lwarx r3,0,r5 // get 1st in list 228 cmpwi r3,0 // null? 229 beq 2f 230 lwzx r6,r3,r4 // get 2nd 231 stwcx. r6,0,r5 // make 2nd first 232 isync // cancel read-aheads (nop'd on UP) 233 beqlr++ // return next element in r2 234 b 1b // retry (lost reservation) 2352: 236 stwcx. r0,r7,r1 // on 970, release reservation using red zone 237 blr // return null 238 239 COMMPAGE_DESCRIPTOR(atomic_dequeue32_on64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage32) 240 241 242atomic_dequeue64: // void* OSAtomicDequeue( void **list, size_t offset); 243 mr r5,r3 244 li r7,-8 // use red zone to release reservation if necessary 2451: 246 ldarx r3,0,r5 // get 1st in list 247 cmpdi r3,0 // null? 248 beq 2f 249 ldx r6,r3,r4 // get 2nd 250 stdcx. r6,0,r5 // make 2nd first 251 isync // cancel read-aheads (nop'd on UP) 252 beqlr++ // return next element in r2 253 b 1b // retry (lost reservation) 2542: 255 stdcx. r0,r7,r1 // on 970, release reservation using red zone 256 blr // return null 257 258 COMMPAGE_DESCRIPTOR(atomic_dequeue64,_COMM_PAGE_DEQUEUE,k64Bit,0,kCommPageISYNC+kCommPage64) 259 260 261memory_barrier_up: // void OSMemoryBarrier( void ) 262 blr // nothing to do on UP 263 264 COMMPAGE_DESCRIPTOR(memory_barrier_up,_COMM_PAGE_MEMORY_BARRIER,kUP,0,kCommPageBoth) 265 266 267memory_barrier_mp32: // void OSMemoryBarrier( void ) 268 isync // we use eieio in preference to sync... 269 eieio // ...because it is faster 270 blr 271 272 COMMPAGE_DESCRIPTOR(memory_barrier_mp32,_COMM_PAGE_MEMORY_BARRIER,0,kUP+k64Bit,kCommPage32) 273 274 275memory_barrier_mp64: // void OSMemoryBarrier( void ) 276 isync 277 lwsync // on 970, lwsync is faster than eieio 278 blr 279 280 COMMPAGE_DESCRIPTOR(memory_barrier_mp64,_COMM_PAGE_MEMORY_BARRIER,k64Bit,kUP,kCommPageBoth) 281