locore32.S revision 91484
1/* $FreeBSD: head/sys/powerpc/aim/locore.S 91484 2002-02-28 11:57:47Z benno $ */
2/* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */
3
4/*
5 * Copyright (C) 2001 Benno Rice
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
26 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28/*
29 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
30 * Copyright (C) 1995, 1996 TooLs GmbH.
31 * All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 *    notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 *    notice, this list of conditions and the following disclaimer in the
40 *    documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 *    must display the following acknowledgement:
43 *	This product includes software developed by TooLs GmbH.
44 * 4. The name of TooLs GmbH may not be used to endorse or promote products
45 *    derived from this software without specific prior written permission.
46 *
47 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
48 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
49 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
50 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
51 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
52 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
53 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
54 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
55 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
56 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
57 */
58
59#include "opt_ddb.h"
60#include "opt_ipkdb.h"
61#include "assym.s"
62
63#include <sys/syscall.h>
64
65#include <machine/trap.h>
66#include <machine/param.h>
67#include <machine/sr.h>
68#include <machine/psl.h>
69#include <machine/asm.h>
70
71/*
72 * Some instructions gas doesn't understand (yet?)
73 */
74#define	bdneq	bdnzf 2,
75
76/*
77 * Globals
78 */
79	.data
80GLOBAL(tmpstk)
81	.space	8208
82GLOBAL(esym)
83	.long	0			/* end of symbol table */
84GLOBAL(proc0paddr)
85	.long	0			/* proc0 p_addr */
86GLOBAL(PTmap)
87	.long	0			/* PTmap */
88GLOBAL(decrnest)
89	.long	0
90
91GLOBAL(intrnames)
92	.asciz	"irq0", "irq1", "irq2", "irq3"
93	.asciz	"irq4", "irq5", "irq6", "irq7"
94	.asciz	"irq8", "irq9", "irq10", "irq11"
95	.asciz	"irq12", "irq13", "irq14", "irq15"
96	.asciz	"irq16", "irq17", "irq18", "irq19"
97	.asciz	"irq20", "irq21", "irq22", "irq23"
98	.asciz	"irq24", "irq25", "irq26", "irq27"
99	.asciz	"irq28", "irq29", "irq30", "irq31"
100	.asciz	"irq32", "irq33", "irq34", "irq35"
101	.asciz	"irq36", "irq37", "irq38", "irq39"
102	.asciz	"irq40", "irq41", "irq42", "irq43"
103	.asciz	"irq44", "irq45", "irq46", "irq47"
104	.asciz	"irq48", "irq49", "irq50", "irq51"
105	.asciz	"irq52", "irq53", "irq54", "irq55"
106	.asciz	"irq56", "irq57", "irq58", "irq59"
107	.asciz	"irq60", "irq61", "irq62", "irq63"
108	.asciz	"clock", "softclock", "softnet", "softserial"
109GLOBAL(eintrnames)
110	.align	4
111GLOBAL(intrcnt)
112	.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
113	.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
114	.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
115	.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
116	.long	0,0,0,0
117GLOBAL(eintrcnt)
118
119GLOBAL(ofmsr)
120	.long	0			/* msr used in Open Firmware */
121
122GLOBAL(powersave)
123	.long	0
124
125/*
126 * File-scope for locore.S
127 */
128idle_u:
129	.long	0			/* fake uarea during idle after exit */
130openfirmware_entry:
131	.long	0			/* openfirmware entry point */
132srsave:
133	.long	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
134
135/*
136 * This symbol is here for the benefit of kvm_mkdb, and is supposed to
137 * mark the start of kernel text.
138 */
139	.text
140	.globl	kernel_text
141kernel_text:
142
143/*
144 * Startup entry.  Note, this must be the first thing in the text
145 * segment!
146 */
147	.text
148	.globl	__start
149__start:
150#ifdef	FIRMWORKSBUGS
151	mfmsr	0
152	andi.	0,0,PSL_IR|PSL_DR
153	beq	1f
154
155	bl	ofwr_init
1561:
157#endif
158	li	8,0
159	li	9,0x100
160	mtctr	9
1611:
162	dcbf	0,8
163	icbi	0,8
164	addi	8,8,0x20
165	bdnz	1b
166	sync
167	isync
168
169	mtibatu	0,0
170	mtibatu	1,0
171	mtibatu	2,0
172	mtibatu	3,0
173	mtdbatu	0,0
174	mtdbatu	1,0
175	mtdbatu	2,0
176	mtdbatu	3,0
177
178	li	9,0x12
179	mtibatl	0,9
180	mtdbatl	0,9
181	li	9,0x1ffe
182	mtibatu	0,9
183	mtdbatu	0,9
184	isync
185
186	lis	8,openfirmware_entry@ha
187	stw	5,openfirmware_entry@l(8) /* save client interface handler */
188	mr	3,5
189
190	lis	1,tmpstk@ha
191	addi	1,1,tmpstk@l
192	addi	1,1,8192
193
194	mfmsr	0
195	lis	9,ofmsr@ha
196	stw	0,ofmsr@l(9)
197
198	bl	OF_init
199
200	lis	4,end@ha
201	addi	4,4,end@l
202	mr	5,4
203	li	9,PAGE_MASK
204	add	4,4,9
205	andc	4,4,9
206	lis	9,OF_buf@ha
207	stw	4,OF_buf@l(9)
208	addi	4,4,PAGE_SIZE
209	lis	9,proc0paddr@ha
210	stw	4,proc0paddr@l(9)
211	addi	4,4,USPACE-FRAMELEN
212	mr	1,4
213	xor	0,0,0
214	stwu	0,-16(1)
215
216	lis	3,kernel_text@ha
217	addi	3,3,kernel_text@l
218#if 0
219	mr	5,6
220#endif
221
222	bl	powerpc_init
223	bl	mi_startup
224	b	OF_exit
225
226#if 0 /* XXX: We may switch back to this in the future. */
227/*
228 * OpenFirmware entry point
229 */
230ENTRY(openfirmware)
231	mflr	0			/* save return address */
232	stw	0,4(1)
233	stwu	1,-16(1)		/* setup stack frame */
234
235	mfmsr	4			/* save msr */
236	stw	4,8(1)
237
238	lis	4,openfirmware_entry@ha	/* get firmware entry point */
239	lwz	4,openfirmware_entry@l(4)
240	mtlr	4
241
242	li	0,0			/* clear battable translations */
243	mtdbatu	2,0
244	mtdbatu	3,0
245	mtibatu	2,0
246	mtibatu	3,0
247
248	lis	4,ofmsr@ha		/* Open Firmware msr */
249	lwz	4,ofmsr@l(4)
250	mtmsr	4
251	isync
252
253	lis	4,srsave@ha		/* save old SR */
254	addi	4,4,srsave@l
255	li	5,0
2561:	mfsrin	0,5
257	stw	0,0(4)
258	addi	4,4,4
259	addis	5,5,0x10000000@h
260	cmpwi	5,0
261	bne	1b
262
263	lis	4,ofw_pmap@ha		/* load OFW SR */
264	addi	4,4,ofw_pmap@l
265	lwz	0,PM_KERNELSR(4)
266	cmpwi	0,0			/* pm_sr[KERNEL_SR] == 0? */
267	beq	2f			/* then skip (not initialized yet) */
268	li	5,0
2691:	lwz	0,0(4)
270	mtsrin	0,5
271	addi	4,4,4
272	addis	5,5,0x10000000@h
273	cmpwi	5,0
274	bne	1b
2752:
276	blrl				/* call Open Firmware */
277
278	mfmsr	4
279	li	5,PSL_IR|PSL_DR
280	andc 	4,4,5
281	mtmsr	4
282	isync
283
284	lis	4,srsave@ha		/* restore saved SR */
285	addi	4,4,srsave@l
286	li	5,0
2871:	lwz	0,0(4)
288	mtsrin	0,5
289	addi	4,4,4
290	addis	5,5,0x10000000@h
291	cmpwi	5,0
292	bne	1b
293
294	lwz	4,8(1)			/* restore msr */
295	mtmsr	4
296	isync
297
298	lwz	1,0(1)			/* and return */
299	lwz	0,4(1)
300	mtlr	0
301	blr
302#endif
303
304/*
305 * Switch to/from OpenFirmware real mode stack
306 *
307 * Note: has to be called as the very first thing in OpenFirmware interface
308 * routines.
309 * E.g.:
310 * int
311 * OF_xxx(arg1, arg2)
312 * type arg1, arg2;
313 * {
314 *	static struct {
315 *		char *name;
316 *		int nargs;
317 *		int nreturns;
318 *		char *method;
319 *		int arg1;
320 *		int arg2;
321 *		int ret;
322 *	} args = {
323 *		"xxx",
324 *		2,
325 *		1,
326 *	};
327 *
328 *	ofw_stack();
329 *	args.arg1 = arg1;
330 *	args.arg2 = arg2;
331 *	if (openfirmware(&args) < 0)
332 *		return -1;
333 *	return args.ret;
334 * }
335 */
336
337	.local	firmstk
338	.comm	firmstk,PAGE_SIZE,8
339
340ENTRY(ofw_stack)
341	mfmsr	8			/* turn off interrupts */
342	andi.	0,8,~(PSL_EE|PSL_RI)@l
343	mtmsr	0
344	stw	8,4(1)			/* abuse return address slot */
345
346	lwz	5,0(1)			/* get length of stack frame */
347	subf	5,1,5
348
349	lis	7,firmstk+PAGE_SIZE-8@ha
350	addi	7,7,firmstk+PAGE_SIZE-8@l
351	lis	6,ofw_back@ha
352	addi	6,6,ofw_back@l
353	subf	4,5,7			/* make room for stack frame on
354					   new stack */
355	stw	6,-4(7)			/* setup return pointer */
356	stwu	1,-8(7)
357
358	stw	7,-8(4)
359
360	addi	3,1,8
361	addi	1,4,-8
362	subi	5,5,8
363
364	cmpw	3,4
365	beqlr
366
367	mr	0,5
368	addi	5,5,-1
369	cmpwi	0,0
370	beqlr
371
3721:	lwz	0,0(3)
373	stw	0,0(4)
374	addi	3,3,1
375	addi	4,4,1
376	mr	0,5
377	addi	5,5,-1
378	cmpwi	0,0
379	bne	1b
380	blr
381
382ofw_back:
383	lwz	1,0(1)			/* get callers original stack pointer */
384
385	lwz	0,4(1)			/* get saved msr from abused slot */
386	mtmsr	0
387
388	lwz	1,0(1)			/* return */
389	lwz	0,4(1)
390	mtlr	0
391	blr
392
393/*
394 * Data used during primary/secondary traps/interrupts
395 */
396#define	tempsave	0x2e0	/* primary save area for trap handling */
397#define	disisave	0x3e0	/* primary save area for dsi/isi traps */
398
399#define	INTSTK	(8*1024)	/* 8K interrupt stack */
400	.data
401	.align	4
402intstk:
403	.space	INTSTK		/* interrupt stack */
404
405GLOBAL(intr_depth)
406	.long	-1		/* in-use marker */
407
408#define	SPILLSTK 1024		/* 1K spill stack */
409
410	.comm	spillstk,SPILLSTK,8
411
412/*
413 * This code gets copied to all the trap vectors
414 * (except ISI/DSI, ALI, the interrupts, and possibly the debugging
415 * traps when using IPKDB).
416 */
417	.text
418	.globl	trapcode,trapsize
419trapcode:
420	mtsprg	1,1			/* save SP */
421	stmw	28,tempsave(0)		/* free r28-r31 */
422	mflr	28			/* save LR */
423	mfcr	29			/* save CR */
424/* Test whether we already had PR set */
425	mfsrr1	31
426	mtcr	31
427	bc	4,17,1f			/* branch if PSL_PR is clear */
428	mfsprg	1,0
429	lwz	1,PC_CURPCB(1)
430	addi	1,1,USPACE		/* stack is top of user struct */
4311:
432	bla	s_trap
433trapsize = .-trapcode
434
435/*
436 * For ALI: has to save DSISR and DAR
437 */
438	.globl	alitrap,alisize
439alitrap:
440	mtsprg	1,1			/* save SP */
441	stmw	28,tempsave(0)		/* free r28-r31 */
442	mfdar	30
443	mfdsisr	31
444	stmw	30,tempsave+16(0)
445	mflr	28			/* save LR */
446	mfcr	29			/* save CR */
447/* Test whether we already had PR set */
448	mfsrr1	31
449	mtcr	31
450	bc	4,17,1f			/* branch if PSL_PR is clear */
451	mfsprg	1,0
452	lwz	1,PC_CURPCB(1)
453	addi	1,1,USPACE		/* stack is top of user struct */
4541:
455	bla	s_trap
456alisize = .-alitrap
457
458/*
459 * Similar to the above for DSI
460 * Has to handle BAT spills
461 * and standard pagetable spills
462 */
463	.globl	dsitrap,dsisize
464dsitrap:
465	stmw	28,disisave(0)		/* free r28-r31 */
466	mfcr	29			/* save CR */
467	mfxer	30			/* save XER */
468	mtsprg	2,30			/* in SPRG2 */
469	mfsrr1	31			/* test kernel mode */
470	mtcr	31
471	bc	12,17,1f		/* branch if PSL_PR is set */
472	mfdar	31			/* get fault address */
473	rlwinm	31,31,7,25,28		/* get segment * 8 */
474
475	/* get batu */
476	addis	31,31,battable@ha
477	lwz	30,battable@l(31)
478	mtcr	30
479	bc	4,30,1f			/* branch if supervisor valid is
480					   false */
481	/* get batl */
482	lwz	31,battable+4@l(31)
483/* We randomly use the highest two bat registers here */
484	mftb	28
485	andi.	28,28,1
486	bne	2f
487	mtdbatu	2,30
488	mtdbatl	2,31
489	b	3f
4902:
491	mtdbatu	3,30
492	mtdbatl	3,31
4933:
494	mfsprg	30,2			/* restore XER */
495	mtxer	30
496	mtcr	29			/* restore CR */
497	lmw	28,disisave(0)		/* restore r28-r31 */
498	rfi				/* return to trapped code */
4991:
500	mflr	28			/* save LR */
501	bla	s_dsitrap
502dsisize = .-dsitrap
503
504/*
505 * Similar to the above for ISI
506 */
507	.globl	isitrap,isisize
508isitrap:
509	stmw	28,disisave(0)		/* free r28-r31 */
510	mflr	28			/* save LR */
511	mfcr	29			/* save CR */
512	mfsrr1	31			/* test kernel mode */
513	mtcr	31
514	bc	12,17,1f		/* branch if PSL_PR is set */
515	mfsrr0	31			/* get fault address */
516	rlwinm	31,31,7,25,28		/* get segment * 8 */
517
518	/* get batu */
519	addis	31,31,battable@ha
520	lwz	30,battable@l(31)
521	mtcr	30
522	bc	4,30,1f			/* branch if supervisor valid is
523					   false */
524	mtibatu	3,30
525
526	/* get batl */
527	lwz	30,battable+4@l(31)
528	mtibatl	3,30
529
530	mtcr	29			/* restore CR */
531	lmw	28,disisave(0)		/* restore r28-r31 */
532	rfi				/* return to trapped code */
5331:
534	bla	s_isitrap
535isisize = .-isitrap
536
537/*
538 * This one for the external interrupt handler.
539 */
540	.globl	extint,extsize
541extint:
542	mtsprg	1,1			/* save SP */
543	stmw	28,tempsave(0)		/* free r28-r31 */
544	mflr	28			/* save LR */
545	mfcr	29			/* save CR */
546	mfxer	30			/* save XER */
547	lis	1,intstk+INTSTK@ha	/* get interrupt stack */
548	addi	1,1,intstk+INTSTK@l
549	lwz	31,0(1)			/* were we already running on intstk? */
550	addic.	31,31,1
551	stw	31,0(1)
552	beq	1f
553	mfsprg	1,1			/* yes, get old SP */
5541:
555	ba	extintr
556extsize = .-extint
557
558/*
559 * And this one for the decrementer interrupt handler.
560 */
561	.globl	decrint,decrsize
562decrint:
563	mtsprg	1,1			/* save SP */
564	stmw	28,tempsave(0)		/* free r28-r31 */
565	lis	28,decrnest@ha
566	lwz	29,decrnest@l(28)
567	cmplwi	0,29,0
568	bne	2f
569	li	29,1
570	stw	29,decrnest@l(28)
571	mflr	28			/* save LR */
572	mfcr	29			/* save CR */
573	mfxer	30			/* save XER */
574	lis	1,intstk+INTSTK@ha	/* get interrupt stack */
575	addi	1,1,intstk+INTSTK@l
576	lwz	31,0(1)			/* were we already running on intstk? */
577	addic.	31,31,1
578	stw	31,0(1)
579	beq	1f
580	mfsprg	1,1			/* yes, get old SP */
5811:
582	ba	decrintr
5832:
584	lmw	28,tempsave(0)
585	rfi
586decrsize = .-decrint
587
588/*
589 * Now the tlb software load for 603 processors:
590 * (Code essentially from the 603e User Manual, Chapter 5, but
591 * corrected a lot.)
592 */
593#define	DMISS	976
594#define	DCMP	977
595#define	HASH1	978
596#define	HASH2	979
597#define	IMISS	980
598#define	ICMP	981
599#define	RPA	982
600
601	.globl	tlbimiss,tlbimsize
602tlbimiss:
603	mfspr	2,HASH1			/* get first pointer */
604	li	1,8
605	mfctr	0			/* save counter */
606	mfspr	3,ICMP			/* get first compare value */
607	addi	2,2,-8			/* predec pointer */
6081:
609	mtctr	1			/* load counter */
6102:
611	lwzu	1,8(2)			/* get next pte */
612	cmpl	0,1,3			/* see if found pte */
613	bdneq	2b			/* loop if not eq */
614	bne	3f			/* not found */
615	lwz	1,4(2)			/* load tlb entry lower word */
616	andi.	3,1,8			/* check G-bit */
617	bne	4f			/* if guarded, take ISI */
618	mtctr	0			/* restore counter */
619	mfspr	0,IMISS			/* get the miss address for the tlbli */
620	mfsrr1	3			/* get the saved cr0 bits */
621	mtcrf	0x80,3			/* and restore */
622	ori	1,1,0x100		/* set the reference bit */
623	mtspr	RPA,1			/* set the pte */
624	srwi	1,1,8			/* get byte 7 of pte */
625	tlbli	0			/* load the itlb */
626	stb	1,6(2)			/* update page table */
627	rfi
628
6293:	/* not found in pteg */
630	andi.	1,3,0x40		/* have we already done second hash? */
631	bne	5f
632	mfspr	2,HASH2			/* get the second pointer */
633	ori	3,3,0x40		/* change the compare value */
634	li	1,8
635	addi	2,2,-8			/* predec pointer */
636	b	1b
6374:	/* guarded */
638	mfsrr1	3
639	andi.	2,3,0xffff		/* clean upper srr1 */
640	oris	2,2,0x8000000@h		/* set srr<4> to flag prot violation */
641	b	6f
6425:	/* not found anywhere */
643	mfsrr1	3
644	andi.	2,3,0xffff		/* clean upper srr1 */
645	oris	2,2,0x40000000@h	/* set srr1<1> to flag pte not found */
6466:
647	mtctr	0			/* restore counter */
648	mtsrr1	2
649	mfmsr	0
650	xoris	0,0,0x20000@h		/* flip the msr<tgpr> bit */
651	mtcrf	0x80,3			/* restore cr0 */
652	mtmsr	0			/* now with native gprs */
653	isync
654	ba	EXC_ISI
655tlbimsize = .-tlbimiss
656
657	.globl	tlbdlmiss,tlbdlmsize
658tlbdlmiss:
659	mfspr	2,HASH1			/* get first pointer */
660	li	1,8
661	mfctr	0			/* save counter */
662	mfspr	3,DCMP			/* get first compare value */
663	addi	2,2,-8			/* predec pointer */
6641:
665	mtctr	1			/* load counter */
6662:
667	lwzu	1,8(2)			/* get next pte */
668	cmpl	0,1,3			/* see if found pte */
669	bdneq	2b			/* loop if not eq */
670	bne	3f			/* not found */
671	lwz	1,4(2)			/* load tlb entry lower word */
672	mtctr	0			/* restore counter */
673	mfspr	0,DMISS			/* get the miss address for the tlbld */
674	mfsrr1	3			/* get the saved cr0 bits */
675	mtcrf	0x80,3			/* and restore */
676	ori	1,1,0x100		/* set the reference bit */
677	mtspr	RPA,1			/* set the pte */
678	srwi	1,1,8			/* get byte 7 of pte */
679	tlbld	0			/* load the dtlb */
680	stb	1,6(2)			/* update page table */
681	rfi
682
6833:	/* not found in pteg */
684	andi.	1,3,0x40		/* have we already done second hash? */
685	bne	5f
686	mfspr	2,HASH2			/* get the second pointer */
687	ori	3,3,0x40		/* change the compare value */
688	li	1,8
689	addi	2,2,-8			/* predec pointer */
690	b	1b
6915:	/* not found anywhere */
692	mfsrr1	3
693	lis	1,0x40000000@h		/* set dsisr<1> to flag pte not found */
694	mtctr	0			/* restore counter */
695	andi.	2,3,0xffff		/* clean upper srr1 */
696	mtsrr1	2
697	mtdsisr	1			/* load the dsisr */
698	mfspr	1,DMISS			/* get the miss address */
699	mtdar	1			/* put in dar */
700	mfmsr	0
701	xoris	0,0,0x20000@h		/* flip the msr<tgpr> bit */
702	mtcrf	0x80,3			/* restore cr0 */
703	mtmsr	0			/* now with native gprs */
704	isync
705	ba	EXC_DSI
706tlbdlmsize = .-tlbdlmiss
707
708	.globl	tlbdsmiss,tlbdsmsize
709tlbdsmiss:
710	mfspr	2,HASH1			/* get first pointer */
711	li	1,8
712	mfctr	0			/* save counter */
713	mfspr	3,DCMP			/* get first compare value */
714	addi	2,2,-8			/* predec pointer */
7151:
716	mtctr	1			/* load counter */
7172:
718	lwzu	1,8(2)			/* get next pte */
719	cmpl	0,1,3			/* see if found pte */
720	bdneq	2b			/* loop if not eq */
721	bne	3f			/* not found */
722	lwz	1,4(2)			/* load tlb entry lower word */
723	andi.	3,1,0x80		/* check the C-bit */
724	beq	4f
7255:
726	mtctr	0			/* restore counter */
727	mfspr	0,DMISS			/* get the miss address for the tlbld */
728	mfsrr1	3			/* get the saved cr0 bits */
729	mtcrf	0x80,3			/* and restore */
730	mtspr	RPA,1			/* set the pte */
731	tlbld	0			/* load the dtlb */
732	rfi
733
7343:	/* not found in pteg */
735	andi.	1,3,0x40		/* have we already done second hash? */
736	bne	5f
737	mfspr	2,HASH2			/* get the second pointer */
738	ori	3,3,0x40		/* change the compare value */
739	li	1,8
740	addi	2,2,-8			/* predec pointer */
741	b	1b
7424:	/* found, but C-bit = 0 */
743	rlwinm.	3,1,30,0,1		/* test PP */
744	bge-	7f
745	andi.	3,1,1
746	beq+	8f
7479:	/* found, but protection violation (PP==00)*/
748	mfsrr1	3
749	lis	1,0xa000000@h		/* indicate protection violation
750					   on store */
751	b	1f
7527:	/* found, PP=1x */
753	mfspr	3,DMISS			/* get the miss address */
754	mfsrin	1,3			/* get the segment register */
755	mfsrr1	3
756	rlwinm	3,3,18,31,31		/* get PR-bit */
757	rlwnm.	2,2,3,1,1		/* get the key */
758	bne-	9b			/* protection violation */
7598:	/* found, set reference/change bits */
760	lwz	1,4(2)			/* reload tlb entry */
761	ori	1,1,0x180
762	sth	1,6(2)
763	b	5b
7645:	/* not found anywhere */
765	mfsrr1	3
766	lis	1,0x42000000@h		/* set dsisr<1> to flag pte not found */
767					/* dsisr<6> to flag store */
7681:
769	mtctr	0			/* restore counter */
770	andi.	2,3,0xffff		/* clean upper srr1 */
771	mtsrr1	2
772	mtdsisr	1			/* load the dsisr */
773	mfspr	1,DMISS			/* get the miss address */
774	mtdar	1			/* put in dar */
775	mfmsr	0
776	xoris	0,0,0x20000@h		/* flip the msr<tgpr> bit */
777	mtcrf	0x80,3			/* restore cr0 */
778	mtmsr	0			/* now with native gprs */
779	isync
780	ba	EXC_DSI
781tlbdsmsize = .-tlbdsmiss
782
783#ifdef DDB
784#define	ddbsave	0xde0		/* primary save area for DDB */
785/*
786 * In case of DDB we want a separate trap catcher for it
787 */
788	.local	ddbstk
789	.comm	ddbstk,INTSTK,8		/* ddb stack */
790
791	.globl	ddblow,ddbsize
792ddblow:
793	mtsprg	1,1			/* save SP */
794	stmw	28,ddbsave(0)		/* free r28-r31 */
795	mflr	28			/* save LR */
796	mfcr	29			/* save CR */
797	lis	1,ddbstk+INTSTK@ha	/* get new SP */
798	addi	1,1,ddbstk+INTSTK@l
799	bla	ddbtrap
800ddbsize = .-ddblow
801#endif	/* DDB */
802
803#ifdef IPKDB
804#define	ipkdbsave	0xde0		/* primary save area for IPKDB */
805/*
806 * In case of IPKDB we want a separate trap catcher for it
807 */
808
809	.local	ipkdbstk
810	.comm	ipkdbstk,INTSTK,8		/* ipkdb stack */
811
812	.globl	ipkdblow,ipkdbsize
813ipkdblow:
814	mtsprg	1,1			/* save SP */
815	stmw	28,ipkdbsave(0)		/* free r28-r31 */
816	mflr	28			/* save LR */
817	mfcr	29			/* save CR */
818	lis	1,ipkdbstk+INTSTK@ha	/* get new SP */
819	addi	1,1,ipkdbstk+INTSTK@l
820	bla	ipkdbtrap
821ipkdbsize = .-ipkdblow
822#endif	/* IPKDB */
823
824/*
825 * FRAME_SETUP assumes:
826 *	SPRG1		SP (1)
827 *	savearea	r28-r31,DAR,DSISR	(DAR & DSISR only for DSI traps)
828 *	28		LR
829 *	29		CR
830 *	1		kernel stack
831 *	LR		trap type
832 *	SRR0/1		as at start of trap
833 */
834#define	FRAME_SETUP(savearea)						\
835/* Have to enable translation to allow access of kernel stack: */	\
836	mfsrr0	30;							\
837	mfsrr1	31;							\
838	stmw	30,savearea+24(0);					\
839	mfmsr	30;							\
840	ori	30,30,(PSL_DR|PSL_IR);					\
841	mtmsr	30;							\
842	isync;								\
843	mfsprg	31,1;							\
844	stwu	31,-FRAMELEN(1);					\
845	stw	0,FRAME_0+8(1);						\
846	stw	31,FRAME_1+8(1);					\
847	stw	28,FRAME_LR+8(1);					\
848	stw	29,FRAME_CR+8(1);					\
849	lmw	28,savearea(0);						\
850	stmw	2,FRAME_2+8(1);						\
851	lmw	28,savearea+16(0);					\
852	mfxer	3;							\
853	mfctr	4;							\
854	mflr	5;							\
855	andi.	5,5,0xff00;						\
856	stw	3,FRAME_XER+8(1);					\
857	stw	4,FRAME_CTR+8(1);					\
858	stw	5,FRAME_EXC+8(1);					\
859	stw	28,FRAME_DAR+8(1);					\
860	stw	29,FRAME_DSISR+8(1);					\
861	stw	30,FRAME_SRR0+8(1);					\
862	stw	31,FRAME_SRR1+8(1)
863
864#define	FRAME_LEAVE(savearea)						\
865/* Now restore regs: */							\
866	lwz	2,FRAME_SRR0+8(1);					\
867	lwz	3,FRAME_SRR1+8(1);					\
868	lwz	4,FRAME_CTR+8(1);					\
869	lwz	5,FRAME_XER+8(1);					\
870	lwz	6,FRAME_LR+8(1);					\
871	lwz	7,FRAME_CR+8(1);					\
872	stw	2,savearea(0);						\
873	stw	3,savearea+4(0);					\
874	mtctr	4;							\
875	mtxer	5;							\
876	mtlr	6;							\
877	mtsprg	1,7;			/* save cr */			\
878	lmw	2,FRAME_2+8(1);						\
879	lwz	0,FRAME_0+8(1);						\
880	lwz	1,FRAME_1+8(1);						\
881	mtsprg	2,2;			/* save r2 & r3 */		\
882	mtsprg	3,3;							\
883/* Disable translation, machine check and recoverability: */		\
884	mfmsr	2;							\
885	andi.	2,2,~(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;			\
886	mtmsr	2;							\
887	isync;								\
888/* Decide whether we return to user mode: */				\
889	lwz	3,savearea+4(0);					\
890	mtcr	3;							\
891	bc	4,17,1f;		/* branch if PSL_PR is false */	\
892/* Restore user & kernel access SR: */					\
893/*	lis	2,curpm@ha;		get real address of pmap */	\
894/*	lwz	2,curpm@l(2);					*/	\
895/*	lwz	3,PM_USRSR(2);					*/	\
896/*	mtsr	USER_SR,3;					*/	\
897/*	lwz	3,PM_KERNELSR(2);				*/	\
898/*	mtsr	KERNEL_SR,3;					*/	\
8991:	mfsprg	2,1;			/* restore cr */		\
900	mtcr	2;							\
901	lwz	2,savearea(0);						\
902	lwz	3,savearea+4(0);					\
903	mtsrr0	2;							\
904	mtsrr1	3;							\
905	mfsprg	2,2;			/* restore r2 & r3 */		\
906	mfsprg	3,3
907
908/*
909 * Preamble code for DSI/ISI traps
910 */
911disitrap:
912	lmw	30,disisave(0)
913	stmw	30,tempsave(0)
914	lmw	30,disisave+8(0)
915	stmw	30,tempsave+8(0)
916	mfdar	30
917	mfdsisr	31
918	stmw	30,tempsave+16(0)
919realtrap:
920/* Test whether we already had PR set */
921	mfsrr1	1
922	mtcr	1
923	mfsprg	1,1			/* restore SP (might have been
924					   overwritten) */
925	bc	4,17,s_trap		/* branch if PSL_PR is false */
926	mfsprg	1,0
927	lwz	1,PC_CURPCB(1)
928	addi	1,1,USPACE		/* stack is top of user struct */
929
930/*
931 * Now the common trap catching code.
932 */
933s_trap:
934/* First have to enable KERNEL mapping */
935	lis	31,KERNEL_SEGMENT@h
936	ori	31,31,KERNEL_SEGMENT@l
937	mtsr	KERNEL_SR,31
938	FRAME_SETUP(tempsave)
939/* Now we can recover interrupts again: */
940	mfmsr	7
941	ori	7,7,(PSL_EE|PSL_FP|PSL_ME|PSL_RI)@l
942	mtmsr	7
943	isync
944/* Call C trap code: */
945	addi	3,1,8
946	mr	30,3
947	bl	trap
948	mr	3,30
949	bl	ast
950	FRAME_LEAVE(tempsave)
951	rfi
952
953/*
954 * DSI second stage fault handler
955 */
956s_dsitrap:
957	mfdsisr	31			/* test whether this may be a
958					   spill fault */
959	mtcr	31
960	mtsprg	1,1			/* save SP */
961	bc	4,1,disitrap		/* branch if table miss is false */
962	lis	1,spillstk+SPILLSTK@ha
963	addi	1,1,spillstk+SPILLSTK@l	/* get spill stack */
964	stwu	1,-52(1)
965	stw	0,48(1)			/* save non-volatile registers */
966	stw	3,44(1)
967	stw	4,40(1)
968	stw	5,36(1)
969	stw	6,32(1)
970	stw	7,28(1)
971	stw	8,24(1)
972	stw	9,20(1)
973	stw	10,16(1)
974	stw	11,12(1)
975	stw	12,8(1)
976	mflr	30			/* save trap type */
977	mfctr	31			/* & CTR */
978	mfdar	3
979s_pte_spill:
980	bl	pmap_pte_spill		/* try a spill */
981	or.	3,3,3
982	mtctr	31			/* restore CTR */
983	mtlr	30			/* and trap type */
984	mfsprg	31,2			/* get saved XER */
985	mtxer	31			/* restore XER */
986	lwz	12,8(1)			/* restore non-volatile registers */
987	lwz	11,12(1)
988	lwz	10,16(1)
989	lwz	9,20(1)
990	lwz	8,24(1)
991	lwz	7,28(1)
992	lwz	6,32(1)
993	lwz	5,36(1)
994	lwz	4,40(1)
995	lwz	3,44(1)
996	lwz	0,48(1)
997	beq	disitrap
998	mfsprg	1,1			/* restore SP */
999	mtcr	29			/* restore CR */
1000	mtlr	28			/* restore LR */
1001	lmw	28,disisave(0)		/* restore r28-r31 */
1002	rfi				/* return to trapped code */
1003
1004/*
1005 * ISI second stage fault handler
1006 */
1007s_isitrap:
1008	mfsrr1	31			/* test whether this may be a
1009					   spill fault */
1010	mtcr	31
1011	mtsprg	1,1			/* save SP */
1012	bc	4,1,disitrap		/* branch if table miss is false */
1013	lis	1,spillstk+SPILLSTK@ha
1014	addi	1,1,spillstk+SPILLSTK@l	/* get spill stack */
1015	stwu	1,-52(1)
1016	stw	0,48(1)			/* save non-volatile registers */
1017	stw	3,44(1)
1018	stw	4,40(1)
1019	stw	5,36(1)
1020	stw	6,32(1)
1021	stw	7,28(1)
1022	stw	8,24(1)
1023	stw	9,20(1)
1024	stw	10,16(1)
1025	stw	11,12(1)
1026	stw	12,8(1)
1027	mfxer	30			/* save XER */
1028	mtsprg	2,30
1029	mflr	30			/* save trap type */
1030	mfctr	31			/* & ctr */
1031	mfsrr0	3
1032	b	s_pte_spill		/* above */
1033
1034/*
1035 * External interrupt second level handler
1036 */
1037#define	INTRENTER							\
1038/* Save non-volatile registers: */					\
1039	stwu	1,-88(1);		/* temporarily */		\
1040	stw	0,84(1);						\
1041	mfsprg	0,1;			/* get original SP */		\
1042	stw	0,0(1);			/* and store it */		\
1043	stw	3,80(1);						\
1044	stw	4,76(1);						\
1045	stw	5,72(1);						\
1046	stw	6,68(1);						\
1047	stw	7,64(1);						\
1048	stw	8,60(1);						\
1049	stw	9,56(1);						\
1050	stw	10,52(1);						\
1051	stw	11,48(1);						\
1052	stw	12,44(1);						\
1053	stw	28,40(1);		/* saved LR */			\
1054	stw	29,36(1);		/* saved CR */			\
1055	stw	30,32(1);		/* saved XER */			\
1056	lmw	28,tempsave(0);		/* restore r28-r31 */		\
1057	mfctr	6;							\
1058	lis	5,intr_depth@ha;					\
1059	lwz	5,intr_depth@l(5);					\
1060	mfsrr0	4;							\
1061	mfsrr1	3;							\
1062	stw	6,28(1);						\
1063	stw	5,20(1);						\
1064	stw	4,12(1);						\
1065	stw	3,8(1);							\
1066/* interrupts are recoverable here, and enable translation */		\
1067	lis	3,(KERNEL_SEGMENT|SR_KS|SR_KP)@h;			\
1068	ori	3,3,(KERNEL_SEGMENT|SR_KS|SR_KP)@l;			\
1069	mtsr	KERNEL_SR,3;						\
1070	mfmsr	5;							\
1071	ori	5,5,(PSL_IR|PSL_DR|PSL_RI);				\
1072	mtmsr	5;							\
1073	isync
1074
1075	.globl	extint_call
1076extintr:
1077	INTRENTER
1078extint_call:
1079	bl	extint_call		/* to be filled in later */
1080
1081intr_exit:
1082/* Disable interrupts (should already be disabled) and MMU here: */
1083	mfmsr	3
1084	andi.	3,3,~(PSL_EE|PSL_ME|PSL_RI|PSL_DR|PSL_IR)@l
1085	mtmsr	3
1086	isync
1087/* restore possibly overwritten registers: */
1088	lwz	12,44(1)
1089	lwz	11,48(1)
1090	lwz	10,52(1)
1091	lwz	9,56(1)
1092	lwz	8,60(1)
1093	lwz	7,64(1)
1094	lwz	6,8(1)
1095	lwz	5,12(1)
1096	lwz	4,28(1)
1097	lwz	3,32(1)
1098	mtsrr1	6
1099	mtsrr0	5
1100	mtctr	4
1101	mtxer	3
1102/* Returning to user mode? */
1103	mtcr	6			/* saved SRR1 */
1104	bc	4,17,1f			/* branch if PSL_PR is false */
1105	mfsprg	3,0			/* get pcpu */
1106	lwz	3,PC_CURPCB(3)		/* get curpcb from pcpu */
1107	lwz	3,PCB_PMR(3)		/* get pmap real address from curpcb */
1108	mtsr	KERNEL_SR,3
1109/* Setup for entry to realtrap: */
1110	lwz	3,0(1)			/* get saved SP */
1111	mtsprg	1,3
1112#if 0 /* XXX */
1113	li	6,EXC_AST
1114#endif
1115	stmw	28,tempsave(0)		/* establish tempsave again */
1116	mtlr	6
1117	lwz	28,40(1)		/* saved LR */
1118	lwz	29,36(1)		/* saved CR */
1119	lwz	6,68(1)
1120	lwz	5,72(1)
1121	lwz	4,76(1)
1122	lwz	3,80(1)
1123	lwz	0,84(1)
1124	lis	30,intr_depth@ha	 /* adjust reentrancy count */
1125	lwz	31,intr_depth@l(30)
1126	addi	31,31,-1
1127	stw	31,intr_depth@l(30)
1128	b	realtrap		/* XXX:	should call ast(frame ptr) */
11291:
1130/* Here is the normal exit of extintr: */
1131	lwz	5,36(1)
1132	lwz	6,40(1)
1133	mtcr	5
1134	mtlr	6
1135	lwz	6,68(1)
1136	lwz	5,72(1)
1137	lis	3,intr_depth@ha		/* adjust reentrancy count */
1138	lwz	4,intr_depth@l(3)
1139	addi	4,4,-1
1140	stw	4,intr_depth@l(3)
1141	lwz	4,76(1)
1142	lwz	3,80(1)
1143	lwz	0,84(1)
1144	lwz	1,0(1)
1145	rfi
1146
1147/*
1148 * Decrementer interrupt second level handler
1149 */
1150decrintr:
1151	INTRENTER
1152	addi	3,1,8			/* intr frame */
1153	bl	decr_intr
1154	lis	28,decrnest@ha
1155	xor	29,29,29
1156	stw	29,decrnest@l(28)
1157	b	intr_exit
1158
1159#ifdef DDB
1160/*
1161 * Deliberate entry to ddbtrap
1162 */
1163	.globl	ddb_trap
1164ddb_trap:
1165	mtsprg	1,1
1166	mfmsr	3
1167	mtsrr1	3
1168	andi.	3,3,~(PSL_EE|PSL_ME|PSL_RI)@l
1169	mtmsr	3			/* disable interrupts */
1170	isync
1171	stmw	28,ddbsave(0)
1172	mflr	28
1173	li	29,EXC_BPT
1174	mtlr	29
1175	mfcr	29
1176	mtsrr0	28
1177
1178/*
1179 * Now the ddb trap catching code.
1180 */
1181ddbtrap:
1182	FRAME_SETUP(ddbsave)
1183/* Call C trap code: */
1184	addi	3,1,8
1185	bl	ddb_trap_glue
1186	or.	3,3,3
1187	bne	ddbleave
1188/* This wasn't for DDB, so switch to real trap: */
1189	lwz	3,FRAME_EXC+8(1)	/* save exception */
1190	stw	3,ddbsave+8(0)
1191	FRAME_LEAVE(ddbsave)
1192	mtsprg	1,1			/* prepare for entrance to realtrap */
1193	stmw	28,tempsave(0)
1194	mflr	28
1195	mfcr	29
1196	lwz	31,ddbsave+8(0)
1197	mtlr	31
1198	b	realtrap
1199ddbleave:
1200	FRAME_LEAVE(ddbsave)
1201	rfi
1202#endif /* DDB */
1203
1204#ifdef IPKDB
1205/*
1206 * Deliberate entry to ipkdbtrap
1207 */
1208	.globl	ipkdb_trap
1209ipkdb_trap:
1210	mtsprg	1,1
1211	mfmsr	3
1212	mtsrr1	3
1213	andi.	3,3,~(PSL_EE|PSL_ME|PSL_RI)@l
1214	mtmsr	3			/* disable interrupts */
1215	isync
1216	stmw	28,ipkdbsave(0)
1217	mflr	28
1218	li	29,EXC_BPT
1219	mtlr	29
1220	mfcr	29
1221	mtsrr0	28
1222
1223/*
1224 * Now the ipkdb trap catching code.
1225 */
1226ipkdbtrap:
1227	FRAME_SETUP(ipkdbsave)
1228/* Call C trap code: */
1229	addi	3,1,8
1230	bl	ipkdb_trap_glue
1231	or.	3,3,3
1232	bne	ipkdbleave
1233/* This wasn't for IPKDB, so switch to real trap: */
1234	lwz	3,FRAME_EXC+8(1)	/* save exception */
1235	stw	3,ipkdbsave+8(0)
1236	FRAME_LEAVE(ipkdbsave)
1237	mtsprg	1,1			/* prepare for entrance to realtrap */
1238	stmw	28,tempsave(0)
1239	mflr	28
1240	mfcr	29
1241	lwz	31,ipkdbsave+8(0)
1242	mtlr	31
1243	b	realtrap
1244ipkdbleave:
1245	FRAME_LEAVE(ipkdbsave)
1246	rfi
1247
1248ipkdbfault:
1249	ba	_ipkdbfault
1250_ipkdbfault:
1251	mfsrr0	3
1252	addi	3,3,4
1253	mtsrr0	3
1254	li	3,-1
1255	rfi
1256
1257/*
1258 * int ipkdbfbyte(unsigned char *p)
1259 */
1260	.globl	ipkdbfbyte
1261ipkdbfbyte:
1262	li	9,EXC_DSI		/* establish new fault routine */
1263	lwz	5,0(9)
1264	lis	6,ipkdbfault@ha
1265	lwz	6,ipkdbfault@l(6)
1266	stw	6,0(9)
1267#ifdef	IPKDBUSERHACK
1268	lis	8,ipkdbsr@ha
1269	lwz	8,ipkdbsr@l(8)
1270	mtsr	USER_SR,8
1271	isync
1272#endif
1273	dcbst	0,9			/* flush data... */
1274	sync
1275	icbi	0,9			/* and instruction caches */
1276	lbz	3,0(3)			/* fetch data */
1277	stw	5,0(9)			/* restore previous fault handler */
1278	dcbst	0,9			/* and flush data... */
1279	sync
1280	icbi	0,9			/* and instruction caches */
1281	blr
1282
1283/*
1284 * int ipkdbsbyte(unsigned char *p, int c)
1285 */
1286	.globl	ipkdbsbyte
1287ipkdbsbyte:
1288	li	9,EXC_DSI		/* establish new fault routine */
1289	lwz	5,0(9)
1290	lis	6,ipkdbfault@ha
1291	lwz	6,ipkdbfault@l(6)
1292	stw	6,0(9)
1293#ifdef	IPKDBUSERHACK
1294	lis	8,ipkdbsr@ha
1295	lwz	8,ipkdbsr@l(8)
1296	mtsr	USER_SR,8
1297	isync
1298#endif
1299	dcbst	0,9			/* flush data... */
1300	sync
1301	icbi	0,9			/* and instruction caches */
1302	mr	6,3
1303	xor	3,3,3
1304	stb	4,0(6)
1305	dcbst	0,6			/* Now do appropriate flushes
1306					   to data... */
1307	sync
1308	icbi	0,6			/* and instruction caches */
1309	stw	5,0(9)			/* restore previous fault handler */
1310	dcbst	0,9			/* and flush data... */
1311	sync
1312	icbi	0,9			/* and instruction caches */
1313	blr
1314#endif	/* IPKDB */
1315
1316/*
1317 * int setfault()
1318 *
1319 * Similar to setjmp to setup for handling faults on accesses to user memory.
1320 * Any routine using this may only call bcopy, either the form below,
1321 * or the (currently used) C code optimized, so it doesn't use any non-volatile
1322 * registers.
1323 */
1324	.globl	setfault
1325setfault:
1326	mflr	0
1327	mfcr	12
1328	mfsprg	4,0
1329	lwz	4,PC_CURPCB(4)
1330	stw	3,PCB_ONFAULT(4)
1331	stw	0,0(3)
1332	stw	1,4(3)
1333	stw	2,8(3)
1334	stmw	12,12(3)
1335	xor	3,3,3
1336	blr
1337
1338/*
1339 * Signal "trampoline" code.
1340 */
1341	.globl	sigcode
1342sigcode:
1343	b	sys_exit
1344esigcode:
1345	.data
1346GLOBAL(szsigcode)
1347	.long	esigcode-sigcode
1348	.text
1349
1350