1/*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31
32#include <ppc/asm.h>
33#include <ppc/proc_reg.h>
34#include <ppc/exception.h>
35#include <assym.s>
36
37/* These routines run in 32 or 64-bit addressing, and handle
38 * 32 and 128 byte caches.  They do not use compare instructions
39 * on addresses, since compares are 32/64-bit-mode-specific.
40 */
41
42#define	kDcbf			0x1
43#define	kDcbfb			31
44#define	kDcbi			0x2
45#define	kDcbib			30
46#define	kIcbi			0x4
47#define	kIcbib			29
48
49
50/*
51 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
52 * extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys);
53 *
54 * flush_dcache takes a virtual or physical address and count to flush
55 * and (can be called for multiple virtual pages).
56 *
57 * it flushes the data cache
58 * cache for the address range in question
59 *
60 * if 'phys' is non-zero then physical addresses will be used
61 */
62
63
64
65        .text
66        .align	5
67        .globl	_flush_dcache
68_flush_dcache:
69        li		r0,kDcbf					// use DCBF instruction
70        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
71        b		cache_op_join				// join common code
72
73        .align	5
74        .globl	_flush_dcache64
75_flush_dcache64:
76		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
77		li		r0,kDcbf					// use DCBF instruction
78		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
79		mr		r4,r5						; Move count
80		mr		r5,r6						; Move physical flag
81        b		cache_op_join				// join common code
82
83
84/*
85 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
86 * extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys);
87 *
88 * invalidate_dcache takes a virtual or physical address and count to
89 * invalidate and (can be called for multiple virtual pages).
90 *
91 * it invalidates the data cache for the address range in question
92 */
93
94        .globl	_invalidate_dcache
95_invalidate_dcache:
96        li		r0,kDcbi					// use DCBI instruction
97        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
98        b		cache_op_join				// join common code
99
100
101        .align	5
102        .globl	_invalidate_dcache64
103_invalidate_dcache64:
104		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
105        li		r0,kDcbi					// use DCBI instruction
106		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
107		mr		r4,r5						; Move count
108		mr		r5,r6						; Move physical flag
109        b		cache_op_join				// join common code
110
111/*
112 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
113 * extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys);
114 *
115 * invalidate_icache takes a virtual or physical address and
116 * count to invalidate, (can be called for multiple virtual pages).
117 *
118 * it invalidates the instruction cache for the address range in question.
119 */
120
121        .globl	_invalidate_icache
122_invalidate_icache:
123        li		r0,kIcbi					// use ICBI instruction
124        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
125        b		cache_op_join				// join common code
126
127
128        .align	5
129        .globl	_invalidate_icache64
130_invalidate_icache64:
131		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
132        li		r0,kIcbi					// use ICBI instruction
133		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
134		mr		r4,r5						; Move count
135		mr		r5,r6						; Move physical flag
136        b		cache_op_join				// join common code
137
138/*
139 * extern void sync_ppage(ppnum_t pa);
140 *
141 * sync_ppage takes a physical page number
142 *
143 * it writes out the data cache and invalidates the instruction
144 * cache for the address range in question
145 */
146
147        .globl	_sync_ppage
148        .align	5
149_sync_ppage:								// Should be the most commonly called routine, by far
150		mfsprg	r2,2
151        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
152		mtcrf	0x02,r2						; Move pf64Bit to cr6
153        li		r5,1						// set flag for physical addresses
154		li		r4,4096						; Set page size
155		bt++	pf64Bitb,spp64				; Skip if 64-bit (only they take the hint)
156        rlwinm	r3,r3,12,0,19				; Convert to physical address - 32-bit
157        b		cache_op_join				; Join up....
158
159spp64:	sldi	r3,r3,12					; Convert to physical address - 64-bit
160        b		cache_op_join				; Join up....
161
162
163
164/*
165 * extern void sync_cache_virtual(vm_offset_t addr, unsigned count);
166 *
167 * Like "sync_cache", except it takes a virtual address and byte count.
168 * It flushes the data cache, invalidates the I cache, and sync's.
169 */
170
171        .globl	_sync_cache_virtual
172        .align	5
173_sync_cache_virtual:
174        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
175        li		r5,0						// set flag for virtual addresses
176        b		cache_op_join				// join common code
177
178
179/*
180 * extern void sync_cache(vm_offset_t pa, unsigned count);
181 * extern void sync_cache64(addr64_t pa, unsigned count);
182 *
183 * sync_cache takes a physical address and count to sync, thus
184 * must not be called for multiple virtual pages.
185 *
186 * it writes out the data cache and invalidates the instruction
187 * cache for the address range in question
188 */
189
190        .globl	_sync_cache
191        .align	5
192_sync_cache:
193        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
194        li		r5,1						// set flag for physical addresses
195        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
196        b		cache_op_join				// join common code
197
198        .globl	_sync_cache64
199        .align	5
200_sync_cache64:
201		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
202        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
203		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
204       	mr		r4,r5						; Copy over the length
205        li		r5,1						// set flag for physical addresses
206
207
208        // Common code to handle the cache operations.
209
210cache_op_join:								// here with r3=addr, r4=count, r5=phys flag, r0=bits
211        mfsprg	r10,2						// r10 <- processor feature flags
212        cmpwi	cr5,r5,0					// using physical addresses?
213        mtcrf	0x01,r0						// move kDcbf, kDcbi, and kIcbi bits to CR7
214        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
215        mtcrf	0x02,r10					// move pf64Bit bit to CR6
216        subi	r8,r9,1						// r8 <- (linesize-1)
217        beq--	cr5,cache_op_2				// skip if using virtual addresses
218
219        bf--	pf64Bitb,cache_op_not64		// This is not a 64-bit machine
220
221        srdi	r12,r3,31					// Slide bit 32 to bit 63
222        cmpldi	r12,1						// Are we in the I/O mapped area?
223        beqlr--								// No cache ops allowed here...
224
225cache_op_not64:
226        mflr	r12							// save return address
227        bl		EXT(ml_set_physical)		// turn on physical addressing
228        mtlr	r12							// restore return address
229
230        // get r3=first cache line, r4=first line not in set, r6=byte count
231
232cache_op_2:
233        add		r7,r3,r4					// point to 1st byte not to operate on
234        andc	r3,r3,r8					// r3 <- 1st line to operate on
235        add		r4,r7,r8					// round up
236        andc	r4,r4,r8					// r4 <- 1st line not to operate on
237        sub.	r6,r4,r3					// r6 <- byte count to operate on
238        beq--	cache_op_exit				// nothing to do
239        bf--	kDcbfb,cache_op_6			// no need to dcbf
240
241
242        // DCBF loop
243
244cache_op_5:
245        sub.	r6,r6,r9					// more to go?
246        dcbf	r6,r3						// flush next line to RAM
247        bne		cache_op_5					// loop if more to go
248        sync								// make sure the data reaches RAM
249        sub		r6,r4,r3					// reset count
250
251
252        // ICBI loop
253
254cache_op_6:
255        bf--	kIcbib,cache_op_8			// no need to icbi
256cache_op_7:
257        sub.	r6,r6,r9					// more to go?
258        icbi	r6,r3						// invalidate next line
259        bne		cache_op_7
260        sub		r6,r4,r3					// reset count
261        isync
262        sync
263
264
265        // DCBI loop
266
267cache_op_8:
268        bf++	kDcbib,cache_op_exit		// no need to dcbi
269cache_op_9:
270        sub.	r6,r6,r9					// more to go?
271        dcbi	r6,r3						// invalidate next line
272        bne		cache_op_9
273        sync
274
275
276        // restore MSR iff necessary and done
277
278cache_op_exit:
279        beqlr--	cr5							// if using virtual addresses, no need to restore MSR
280        b		EXT(ml_restore)				// restore MSR and return
281
282
283////////////////////////////////////////////////////
284
285        .align	5
286        .globl	_dcache_incoherent_io_store64
287_dcache_incoherent_io_store64:
288		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
289		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
290		mr		r4,r5						; Move count
291
292											// here with r3=addr, r4=count
293        mfsprg	r10,2						// r10 <- processor feature flags
294        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
295        mtcrf	0x02,r10					// move pf64Bit bit to CR6
296        subi	r8,r9,1						// r8 <- (linesize-1)
297
298        bf--	pf64Bitb,cache_ios_not64	// This is not a 64-bit machine
299
300        srdi	r12,r3,31					// Slide bit 32 to bit 63
301        cmpldi	r12,1						// Are we in the I/O mapped area?
302        beqlr--								// No cache ops allowed here...
303
304cache_ios_not64:
305        mflr	r12							// save return address
306        bl		EXT(ml_set_physical)		// turn on physical addressing
307        mtlr	r12							// restore return address
308
309        // get r3=first cache line, r4=first line not in set, r6=byte count
310        add		r7,r3,r4					// point to 1st byte not to operate on
311        andc	r3,r3,r8					// r3 <- 1st line to operate on
312        add		r4,r7,r8					// round up
313        andc	r4,r4,r8					// r4 <- 1st line not to operate on
314        sub.	r6,r4,r3					// r6 <- byte count to operate on
315        beq--	cache_ios_exit				// nothing to do
316
317        sub.	r6,r6,r9					// >1 line?
318        beq		cache_ios_last_line			// use dcbst on all lines but last
319
320        // DCBST loop
321cache_ios_5:
322        sub.	r6,r6,r9					// more to go?
323        dcbst	r6,r3						// store next line
324        bne		cache_ios_5					// loop if more to go
325
326cache_ios_last_line:
327        sync								// flush last line
328        isync
329        dcbf	r6,r3
330        sync
331        isync
332        add		r6,r6,r3
333        lwz		r0,0(r6)					// make sure the data reaches RAM (not just the memory controller)
334        isync
335
336        // restore MSR
337cache_ios_exit:
338        b		EXT(ml_restore)				// restore MSR and return
339
340
341////////////////////////////////////////////////////
342
343        .align	5
344        .globl	_dcache_incoherent_io_flush64
345_dcache_incoherent_io_flush64:
346		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
347		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
348		mr		r4,r5						; Move count
349
350											// here with r3=addr, r4=count
351        mfsprg	r10,2						// r10 <- processor feature flags
352        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
353        mtcrf	0x02,r10					// move pf64Bit bit to CR6
354        subi	r8,r9,1						// r8 <- (linesize-1)
355
356        bf--	pf64Bitb,cache_iof_not64	// This is not a 64-bit machine
357
358        srdi	r12,r3,31					// Slide bit 32 to bit 63
359        cmpldi	r12,1						// Are we in the I/O mapped area?
360        beqlr--								// No cache ops allowed here...
361
362cache_iof_not64:
363        mflr	r12							// save return address
364        bl		EXT(ml_set_physical)		// turn on physical addressing
365        mtlr	r12							// restore return address
366
367        // get r3=first cache line, r4=first line not in set, r6=byte count
368        add		r7,r3,r4					// point to 1st byte not to operate on
369        andc	r3,r3,r8					// r3 <- 1st line to operate on
370        add		r4,r7,r8					// round up
371        andc	r4,r4,r8					// r4 <- 1st line not to operate on
372        sub.	r6,r4,r3					// r6 <- byte count to operate on
373        beq--	cache_iof_exit				// nothing to do
374
375        // DCBF loop
376cache_iof_5:
377        sub.	r6,r6,r9					// more to go?
378        dcbf	r6,r3						// store next line
379        bne		cache_iof_5					// loop if more to go
380
381cache_iof_last_line:
382        sync								// flush last line
383        isync
384
385        // restore MSR
386cache_iof_exit:
387        b		EXT(ml_restore)				// restore MSR and return
388
389
390