mp_exception.S revision 114188 
1/*-
2 * Copyright (c) 2002 Jake Burkholder.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <machine/asm.h>
28__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/mp_exception.S 114188 2003-04-29 00:37:41Z jake $");
29
30#include <machine/asi.h>
31#include <machine/ktr.h>
32#include <machine/asmacros.h>
33#include <machine/pstate.h>
34
35#include "assym.s"
36
37	.register	%g2, #ignore
38	.register	%g3, #ignore
39
40#define	IPI_WAIT(r1, r2, r3, r4) \
41	lduw	[PCPU(CPUMASK)], r4 ;  \
42	ATOMIC_CLEAR_INT(r1, r2, r3, r4) ; \
439:	lduw	[r1], r2 ; \
44	brnz,a,pn r2, 9b ; \
45	 nop
46
47/*
48 * Invalidate a physical page in the data cache.  For UltraSPARC I and II.
49 */
50ENTRY(tl_ipi_spitfire_dcache_page_inval)
51#if KTR_COMPILE & KTR_SMP
52	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
53	    , %g1, %g2, %g3, 7, 8, 9)
54	ldx	[%g5 + ICA_PA], %g2
55	stx	%g2, [%g1 + KTR_PARM1]
569:
57#endif
58
59	ldx	[%g5 + ICA_PA], %g6
60	srlx	%g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6
61
62	SET(cache, %g3, %g2)
63	lduw	[%g2 + DC_SIZE], %g3
64	lduw	[%g2 + DC_LINESIZE], %g4
65	sub	%g3, %g4, %g2
66
671:	ldxa	[%g2] ASI_DCACHE_TAG, %g1
68	srlx	%g1, DC_VALID_SHIFT, %g3
69	andcc	%g3, DC_VALID_MASK, %g0
70	bz,pt	%xcc, 2f
71	 set	DC_TAG_MASK, %g3
72	sllx	%g3, DC_TAG_SHIFT, %g3
73	and	%g1, %g3, %g1
74	cmp	%g1, %g6
75	bne,a,pt %xcc, 2f
76	 nop
77	stxa	%g1, [%g2] ASI_DCACHE_TAG
78	membar	#Sync
79
802:	brgz,pt	%g2, 1b
81	 sub	%g2, %g4, %g2
82
83	IPI_WAIT(%g5, %g1, %g2, %g3)
84	retry
85END(tl_ipi_spitfire_dcache_page_inval)
86
87/*
88 * Invalidate a physical page in the instruction cache.  For UltraSPARC I and
89 * II.
90 */
91ENTRY(tl_ipi_spitfire_icache_page_inval)
92#if KTR_COMPILE & KTR_SMP
93	CATR(KTR_SMP, "ipi_icache_page_inval: pa=%#lx"
94	    , %g1, %g2, %g3, 7, 8, 9)
95	ldx	[%g5 + ICA_PA], %g2
96	stx	%g2, [%g1 + KTR_PARM1]
979:
98#endif
99
100	ldx	[%g5 + ICA_PA], %g6
101	srlx	%g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6
102
103	SET(cache, %g3, %g2)
104	lduw	[%g2 + IC_SIZE], %g3
105	lduw	[%g2 + IC_LINESIZE], %g4
106	sub	%g3, %g4, %g2
107
1081:	ldda	[%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */
109	srlx	%g1, IC_VALID_SHIFT, %g3
110	andcc	%g3, IC_VALID_MASK, %g0
111	bz,pt	%xcc, 2f
112	 set	IC_TAG_MASK, %g3
113	sllx	%g3, IC_TAG_SHIFT, %g3
114	and	%g1, %g3, %g1
115	cmp	%g1, %g6
116	bne,a,pt %xcc, 2f
117	 nop
118	stxa	%g1, [%g2] ASI_ICACHE_TAG
119	membar	#Sync
120
1212:	brgz,pt	%g2, 1b
122	 sub	%g2, %g4, %g2
123
124	IPI_WAIT(%g5, %g1, %g2, %g3)
125	retry
126END(tl_ipi_spitfire_icache_page_inval)
127
128/*
129 * Invalidate a physical page in the data cache.  For UltraSPARC III.
130 */
131ENTRY(tl_ipi_cheetah_dcache_page_inval)
132#if KTR_COMPILE & KTR_SMP
133	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
134	    , %g1, %g2, %g3, 7, 8, 9)
135	ldx	[%g5 + ICA_PA], %g2
136	stx	%g2, [%g1 + KTR_PARM1]
1379:
138#endif
139
140	ldx	[%g5 + ICA_PA], %g1
141
142	set	PAGE_SIZE, %g2
143	add	%g1, %g2, %g3
144
145	SET(cache, %g4, %g2)
146	lduw	[%g2 + DC_LINESIZE], %g2
147
1481:	stxa	%g0, [%g1] ASI_DCACHE_INVALIDATE
149	membar	#Sync
150
151	add	%g1, %g2, %g1
152	cmp	%g1, %g3
153	blt,a,pt %xcc, 1b
154	 nop
155
156	IPI_WAIT(%g5, %g1, %g2, %g3)
157	retry
158END(tl_ipi_cheetah_dcache_page_inval)
159
160/*
161 * Trigger a softint at the desired level.
162 */
163ENTRY(tl_ipi_level)
164#if KTR_COMPILE & KTR_SMP
165	CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx"
166	    , %g1, %g2, %g3, 7, 8, 9)
167	lduw	[PCPU(CPUID)], %g2
168	stx	%g2, [%g1 + KTR_PARM1]
169	lduw	[PCPU(MID)], %g2
170	stx	%g2, [%g1 + KTR_PARM2]
171	stx	%g4, [%g1 + KTR_PARM3]
172	stx	%g5, [%g1 + KTR_PARM4]
1739:
174#endif
175
176	mov	1, %g1
177	sllx	%g1, %g5, %g1
178	wr	%g1, 0, %set_softint
179	retry
180END(tl_ipi_level)
181
182/*
183 * Demap a page from the dtlb and/or itlb.
184 */
185ENTRY(tl_ipi_tlb_page_demap)
186#if KTR_COMPILE & KTR_SMP
187	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
188	    , %g1, %g2, %g3, 7, 8, 9)
189	ldx	[%g5 + ITA_PMAP], %g2
190	stx	%g2, [%g1 + KTR_PARM1]
191	ldx	[%g5 + ITA_VA], %g2
192	stx	%g2, [%g1 + KTR_PARM2]
1939:
194#endif
195
196	ldx	[%g5 + ITA_PMAP], %g1
197
198	SET(kernel_pmap_store, %g3, %g2)
199	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
200
201	cmp	%g1, %g2
202	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
203
204	ldx	[%g5 + ITA_VA], %g2
205	or	%g2, %g3, %g2
206
207	stxa	%g0, [%g2] ASI_DMMU_DEMAP
208	stxa	%g0, [%g2] ASI_IMMU_DEMAP
209	membar	#Sync
210
211	IPI_WAIT(%g5, %g1, %g2, %g3)
212	retry
213END(tl_ipi_tlb_page_demap)
214
215/*
216 * Demap a range of pages from the dtlb and itlb.
217 */
218ENTRY(tl_ipi_tlb_range_demap)
219#if KTR_COMPILE & KTR_SMP
220	CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx"
221	    , %g1, %g2, %g3, 7, 8, 9)
222	ldx	[%g5 + ITA_PMAP], %g2
223	stx	%g2, [%g1 + KTR_PARM1]
224	ldx	[%g5 + ITA_START], %g2
225	stx	%g2, [%g1 + KTR_PARM2]
226	ldx	[%g5 + ITA_END], %g2
227	stx	%g2, [%g1 + KTR_PARM3]
2289:
229#endif
230
231	ldx	[%g5 + ITA_PMAP], %g1
232
233	SET(kernel_pmap_store, %g3, %g2)
234	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
235
236	cmp	%g1, %g2
237	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
238
239	ldx	[%g5 + ITA_START], %g1
240	ldx	[%g5 + ITA_END], %g2
241
242	set	PAGE_SIZE, %g6
243
2441:	or	%g1, %g3, %g4
245	stxa	%g0, [%g4] ASI_DMMU_DEMAP
246	stxa	%g0, [%g4] ASI_IMMU_DEMAP
247	membar	#Sync
248
249	add	%g1, %g6, %g1
250	cmp	%g1, %g2
251	blt,a,pt %xcc, 1b
252	 nop
253
254	IPI_WAIT(%g5, %g1, %g2, %g3)
255	retry
256END(tl_ipi_tlb_range_demap)
257
258/*
259 * Demap the primary context from the dtlb and itlb.
260 */
261ENTRY(tl_ipi_tlb_context_demap)
262#if KTR_COMPILE & KTR_SMP
263	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
264	    , %g1, %g2, %g3, 7, 8, 9)
265	ldx	[%g5 + ITA_PMAP], %g2
266	stx	%g2, [%g1 + KTR_PARM1]
267	ldx	[%g5 + ITA_VA], %g2
268	stx	%g2, [%g1 + KTR_PARM2]
2699:
270#endif
271
272	mov	TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
273	stxa	%g0, [%g1] ASI_DMMU_DEMAP
274	stxa	%g0, [%g1] ASI_IMMU_DEMAP
275	membar	#Sync
276
277	IPI_WAIT(%g5, %g1, %g2, %g3)
278	retry
279END(tl_ipi_tlb_context_demap)
280