1/* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31 32#include <ppc/asm.h> 33#include <ppc/proc_reg.h> 34#include <ppc/exception.h> 35#include <assym.s> 36 37/* These routines run in 32 or 64-bit addressing, and handle 38 * 32 and 128 byte caches. They do not use compare instructions 39 * on addresses, since compares are 32/64-bit-mode-specific. 40 */ 41 42#define kDcbf 0x1 43#define kDcbfb 31 44#define kDcbi 0x2 45#define kDcbib 30 46#define kIcbi 0x4 47#define kIcbib 29 48 49 50/* 51 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys); 52 * extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys); 53 * 54 * flush_dcache takes a virtual or physical address and count to flush 55 * and (can be called for multiple virtual pages). 56 * 57 * it flushes the data cache 58 * cache for the address range in question 59 * 60 * if 'phys' is non-zero then physical addresses will be used 61 */ 62 63 64 65 .text 66 .align 5 67 .globl _flush_dcache 68_flush_dcache: 69 li r0,kDcbf // use DCBF instruction 70 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 71 b cache_op_join // join common code 72 73 .align 5 74 .globl _flush_dcache64 75_flush_dcache64: 76 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 77 li r0,kDcbf // use DCBF instruction 78 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 79 mr r4,r5 ; Move count 80 mr r5,r6 ; Move physical flag 81 b cache_op_join // join common code 82 83 84/* 85 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys); 86 * extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys); 87 * 88 * invalidate_dcache takes a virtual or physical address and count to 89 * invalidate and (can be called for multiple virtual pages). 90 * 91 * it invalidates the data cache for the address range in question 92 */ 93 94 .globl _invalidate_dcache 95_invalidate_dcache: 96 li r0,kDcbi // use DCBI instruction 97 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 98 b cache_op_join // join common code 99 100 101 .align 5 102 .globl _invalidate_dcache64 103_invalidate_dcache64: 104 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 105 li r0,kDcbi // use DCBI instruction 106 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 107 mr r4,r5 ; Move count 108 mr r5,r6 ; Move physical flag 109 b cache_op_join // join common code 110 111/* 112 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys); 113 * extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys); 114 * 115 * invalidate_icache takes a virtual or physical address and 116 * count to invalidate, (can be called for multiple virtual pages). 117 * 118 * it invalidates the instruction cache for the address range in question. 119 */ 120 121 .globl _invalidate_icache 122_invalidate_icache: 123 li r0,kIcbi // use ICBI instruction 124 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 125 b cache_op_join // join common code 126 127 128 .align 5 129 .globl _invalidate_icache64 130_invalidate_icache64: 131 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 132 li r0,kIcbi // use ICBI instruction 133 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 134 mr r4,r5 ; Move count 135 mr r5,r6 ; Move physical flag 136 b cache_op_join // join common code 137 138/* 139 * extern void sync_ppage(ppnum_t pa); 140 * 141 * sync_ppage takes a physical page number 142 * 143 * it writes out the data cache and invalidates the instruction 144 * cache for the address range in question 145 */ 146 147 .globl _sync_ppage 148 .align 5 149_sync_ppage: // Should be the most commonly called routine, by far 150 mfsprg r2,2 151 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 152 mtcrf 0x02,r2 ; Move pf64Bit to cr6 153 li r5,1 // set flag for physical addresses 154 li r4,4096 ; Set page size 155 bt++ pf64Bitb,spp64 ; Skip if 64-bit (only they take the hint) 156 rlwinm r3,r3,12,0,19 ; Convert to physical address - 32-bit 157 b cache_op_join ; Join up.... 158 159spp64: sldi r3,r3,12 ; Convert to physical address - 64-bit 160 b cache_op_join ; Join up.... 161 162 163 164/* 165 * extern void sync_cache_virtual(vm_offset_t addr, unsigned count); 166 * 167 * Like "sync_cache", except it takes a virtual address and byte count. 168 * It flushes the data cache, invalidates the I cache, and sync's. 169 */ 170 171 .globl _sync_cache_virtual 172 .align 5 173_sync_cache_virtual: 174 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 175 li r5,0 // set flag for virtual addresses 176 b cache_op_join // join common code 177 178 179/* 180 * extern void sync_cache(vm_offset_t pa, unsigned count); 181 * extern void sync_cache64(addr64_t pa, unsigned count); 182 * 183 * sync_cache takes a physical address and count to sync, thus 184 * must not be called for multiple virtual pages. 185 * 186 * it writes out the data cache and invalidates the instruction 187 * cache for the address range in question 188 */ 189 190 .globl _sync_cache 191 .align 5 192_sync_cache: 193 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 194 li r5,1 // set flag for physical addresses 195 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine 196 b cache_op_join // join common code 197 198 .globl _sync_cache64 199 .align 5 200_sync_cache64: 201 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 202 li r0,kDcbf+kIcbi // we need to dcbf and then icbi 203 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 204 mr r4,r5 ; Copy over the length 205 li r5,1 // set flag for physical addresses 206 207 208 // Common code to handle the cache operations. 209 210cache_op_join: // here with r3=addr, r4=count, r5=phys flag, r0=bits 211 mfsprg r10,2 // r10 <- processor feature flags 212 cmpwi cr5,r5,0 // using physical addresses? 213 mtcrf 0x01,r0 // move kDcbf, kDcbi, and kIcbi bits to CR7 214 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 215 mtcrf 0x02,r10 // move pf64Bit bit to CR6 216 subi r8,r9,1 // r8 <- (linesize-1) 217 beq-- cr5,cache_op_2 // skip if using virtual addresses 218 219 bf-- pf64Bitb,cache_op_not64 // This is not a 64-bit machine 220 221 srdi r12,r3,31 // Slide bit 32 to bit 63 222 cmpldi r12,1 // Are we in the I/O mapped area? 223 beqlr-- // No cache ops allowed here... 224 225cache_op_not64: 226 mflr r12 // save return address 227 bl EXT(ml_set_physical) // turn on physical addressing 228 mtlr r12 // restore return address 229 230 // get r3=first cache line, r4=first line not in set, r6=byte count 231 232cache_op_2: 233 add r7,r3,r4 // point to 1st byte not to operate on 234 andc r3,r3,r8 // r3 <- 1st line to operate on 235 add r4,r7,r8 // round up 236 andc r4,r4,r8 // r4 <- 1st line not to operate on 237 sub. r6,r4,r3 // r6 <- byte count to operate on 238 beq-- cache_op_exit // nothing to do 239 bf-- kDcbfb,cache_op_6 // no need to dcbf 240 241 242 // DCBF loop 243 244cache_op_5: 245 sub. r6,r6,r9 // more to go? 246 dcbf r6,r3 // flush next line to RAM 247 bne cache_op_5 // loop if more to go 248 sync // make sure the data reaches RAM 249 sub r6,r4,r3 // reset count 250 251 252 // ICBI loop 253 254cache_op_6: 255 bf-- kIcbib,cache_op_8 // no need to icbi 256cache_op_7: 257 sub. r6,r6,r9 // more to go? 258 icbi r6,r3 // invalidate next line 259 bne cache_op_7 260 sub r6,r4,r3 // reset count 261 isync 262 sync 263 264 265 // DCBI loop 266 267cache_op_8: 268 bf++ kDcbib,cache_op_exit // no need to dcbi 269cache_op_9: 270 sub. r6,r6,r9 // more to go? 271 dcbi r6,r3 // invalidate next line 272 bne cache_op_9 273 sync 274 275 276 // restore MSR iff necessary and done 277 278cache_op_exit: 279 beqlr-- cr5 // if using virtual addresses, no need to restore MSR 280 b EXT(ml_restore) // restore MSR and return 281 282 283//////////////////////////////////////////////////// 284 285 .align 5 286 .globl _dcache_incoherent_io_store64 287_dcache_incoherent_io_store64: 288 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 289 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 290 mr r4,r5 ; Move count 291 292 // here with r3=addr, r4=count 293 mfsprg r10,2 // r10 <- processor feature flags 294 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 295 mtcrf 0x02,r10 // move pf64Bit bit to CR6 296 subi r8,r9,1 // r8 <- (linesize-1) 297 298 bf-- pf64Bitb,cache_ios_not64 // This is not a 64-bit machine 299 300 srdi r12,r3,31 // Slide bit 32 to bit 63 301 cmpldi r12,1 // Are we in the I/O mapped area? 302 beqlr-- // No cache ops allowed here... 303 304cache_ios_not64: 305 mflr r12 // save return address 306 bl EXT(ml_set_physical) // turn on physical addressing 307 mtlr r12 // restore return address 308 309 // get r3=first cache line, r4=first line not in set, r6=byte count 310 add r7,r3,r4 // point to 1st byte not to operate on 311 andc r3,r3,r8 // r3 <- 1st line to operate on 312 add r4,r7,r8 // round up 313 andc r4,r4,r8 // r4 <- 1st line not to operate on 314 sub. r6,r4,r3 // r6 <- byte count to operate on 315 beq-- cache_ios_exit // nothing to do 316 317 sub. r6,r6,r9 // >1 line? 318 beq cache_ios_last_line // use dcbst on all lines but last 319 320 // DCBST loop 321cache_ios_5: 322 sub. r6,r6,r9 // more to go? 323 dcbst r6,r3 // store next line 324 bne cache_ios_5 // loop if more to go 325 326cache_ios_last_line: 327 sync // flush last line 328 isync 329 dcbf r6,r3 330 sync 331 isync 332 add r6,r6,r3 333 lwz r0,0(r6) // make sure the data reaches RAM (not just the memory controller) 334 isync 335 336 // restore MSR 337cache_ios_exit: 338 b EXT(ml_restore) // restore MSR and return 339 340 341//////////////////////////////////////////////////// 342 343 .align 5 344 .globl _dcache_incoherent_io_flush64 345_dcache_incoherent_io_flush64: 346 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg 347 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits 348 mr r4,r5 ; Move count 349 350 // here with r3=addr, r4=count 351 mfsprg r10,2 // r10 <- processor feature flags 352 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size 353 mtcrf 0x02,r10 // move pf64Bit bit to CR6 354 subi r8,r9,1 // r8 <- (linesize-1) 355 356 bf-- pf64Bitb,cache_iof_not64 // This is not a 64-bit machine 357 358 srdi r12,r3,31 // Slide bit 32 to bit 63 359 cmpldi r12,1 // Are we in the I/O mapped area? 360 beqlr-- // No cache ops allowed here... 361 362cache_iof_not64: 363 mflr r12 // save return address 364 bl EXT(ml_set_physical) // turn on physical addressing 365 mtlr r12 // restore return address 366 367 // get r3=first cache line, r4=first line not in set, r6=byte count 368 add r7,r3,r4 // point to 1st byte not to operate on 369 andc r3,r3,r8 // r3 <- 1st line to operate on 370 add r4,r7,r8 // round up 371 andc r4,r4,r8 // r4 <- 1st line not to operate on 372 sub. r6,r4,r3 // r6 <- byte count to operate on 373 beq-- cache_iof_exit // nothing to do 374 375 // DCBF loop 376cache_iof_5: 377 sub. r6,r6,r9 // more to go? 378 dcbf r6,r3 // store next line 379 bne cache_iof_5 // loop if more to go 380 381cache_iof_last_line: 382 sync // flush last line 383 isync 384 385 // restore MSR 386cache_iof_exit: 387 b EXT(ml_restore) // restore MSR and return 388 389 390