mp_exception.S revision 166105 
1/*-
2 * Copyright (c) 2002 Jake Burkholder.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <machine/asm.h>
28__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/mp_exception.S 166105 2007-01-19 11:15:34Z marius $");
29
30#include <machine/asi.h>
31#include <machine/asmacros.h>
32#include <machine/cache.h>
33#include <machine/ktr.h>
34#include <machine/pstate.h>
35
36#include "assym.s"
37
38	.register	%g2, #ignore
39	.register	%g3, #ignore
40
41#define	IPI_DONE(r1, r2, r3, r4) \
42	lduw	[PCPU(CPUMASK)], r4 ;  \
43	ATOMIC_CLEAR_INT(r1, r2, r3, r4)
44
45/*
46 * Invalidate a physical page in the data cache.  For UltraSPARC I and II.
47 */
48ENTRY(tl_ipi_spitfire_dcache_page_inval)
49#if KTR_COMPILE & KTR_SMP
50	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
51	    , %g1, %g2, %g3, 7, 8, 9)
52	ldx	[%g5 + ICA_PA], %g2
53	stx	%g2, [%g1 + KTR_PARM1]
549:
55#endif
56
57	ldx	[%g5 + ICA_PA], %g6
58	srlx	%g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6
59
60	SET(cache, %g3, %g2)
61	lduw	[%g2 + DC_SIZE], %g3
62	lduw	[%g2 + DC_LINESIZE], %g4
63	sub	%g3, %g4, %g2
64
651:	ldxa	[%g2] ASI_DCACHE_TAG, %g1
66	srlx	%g1, DC_VALID_SHIFT, %g3
67	andcc	%g3, DC_VALID_MASK, %g0
68	bz,pt	%xcc, 2f
69	 set	DC_TAG_MASK, %g3
70	sllx	%g3, DC_TAG_SHIFT, %g3
71	and	%g1, %g3, %g1
72	cmp	%g1, %g6
73	bne,a,pt %xcc, 2f
74	 nop
75	stxa	%g1, [%g2] ASI_DCACHE_TAG
76	membar	#Sync
77
782:	brgz,pt	%g2, 1b
79	 sub	%g2, %g4, %g2
80
81	IPI_DONE(%g5, %g1, %g2, %g3)
82	retry
83END(tl_ipi_spitfire_dcache_page_inval)
84
85/*
86 * Invalidate a physical page in the instruction cache.  For UltraSPARC I and
87 * II.
88 */
89ENTRY(tl_ipi_spitfire_icache_page_inval)
90#if KTR_COMPILE & KTR_SMP
91	CATR(KTR_SMP, "ipi_icache_page_inval: pa=%#lx"
92	    , %g1, %g2, %g3, 7, 8, 9)
93	ldx	[%g5 + ICA_PA], %g2
94	stx	%g2, [%g1 + KTR_PARM1]
959:
96#endif
97
98	ldx	[%g5 + ICA_PA], %g6
99	srlx	%g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6
100
101	SET(cache, %g3, %g2)
102	lduw	[%g2 + IC_SIZE], %g3
103	lduw	[%g2 + IC_LINESIZE], %g4
104	sub	%g3, %g4, %g2
105
1061:	ldda	[%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */
107	srlx	%g1, IC_VALID_SHIFT, %g3
108	andcc	%g3, IC_VALID_MASK, %g0
109	bz,pt	%xcc, 2f
110	 set	IC_TAG_MASK, %g3
111	sllx	%g3, IC_TAG_SHIFT, %g3
112	and	%g1, %g3, %g1
113	cmp	%g1, %g6
114	bne,a,pt %xcc, 2f
115	 nop
116	stxa	%g1, [%g2] ASI_ICACHE_TAG
117	membar	#Sync
118
1192:	brgz,pt	%g2, 1b
120	 sub	%g2, %g4, %g2
121
122	IPI_DONE(%g5, %g1, %g2, %g3)
123	retry
124END(tl_ipi_spitfire_icache_page_inval)
125
126/*
127 * Invalidate a physical page in the data cache.  For UltraSPARC III.
128 */
129ENTRY(tl_ipi_cheetah_dcache_page_inval)
130#if KTR_COMPILE & KTR_SMP
131	CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx"
132	    , %g1, %g2, %g3, 7, 8, 9)
133	ldx	[%g5 + ICA_PA], %g2
134	stx	%g2, [%g1 + KTR_PARM1]
1359:
136#endif
137
138	ldx	[%g5 + ICA_PA], %g1
139
140	set	PAGE_SIZE, %g2
141	add	%g1, %g2, %g3
142
143	SET(cache, %g4, %g2)
144	lduw	[%g2 + DC_LINESIZE], %g2
145
1461:	stxa	%g0, [%g1] ASI_DCACHE_INVALIDATE
147	membar	#Sync
148
149	add	%g1, %g2, %g1
150	cmp	%g1, %g3
151	blt,a,pt %xcc, 1b
152	 nop
153
154	IPI_DONE(%g5, %g1, %g2, %g3)
155	retry
156END(tl_ipi_cheetah_dcache_page_inval)
157
158/*
159 * Trigger a softint at the desired level.
160 */
161ENTRY(tl_ipi_level)
162#if KTR_COMPILE & KTR_SMP
163	CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx"
164	    , %g1, %g2, %g3, 7, 8, 9)
165	lduw	[PCPU(CPUID)], %g2
166	stx	%g2, [%g1 + KTR_PARM1]
167	lduw	[PCPU(MID)], %g2
168	stx	%g2, [%g1 + KTR_PARM2]
169	stx	%g4, [%g1 + KTR_PARM3]
170	stx	%g5, [%g1 + KTR_PARM4]
1719:
172#endif
173
174	mov	1, %g1
175	sllx	%g1, %g5, %g1
176	wr	%g1, 0, %set_softint
177	retry
178END(tl_ipi_level)
179
180/*
181 * Demap a page from the dtlb and/or itlb.
182 */
183ENTRY(tl_ipi_tlb_page_demap)
184#if KTR_COMPILE & KTR_SMP
185	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
186	    , %g1, %g2, %g3, 7, 8, 9)
187	ldx	[%g5 + ITA_PMAP], %g2
188	stx	%g2, [%g1 + KTR_PARM1]
189	ldx	[%g5 + ITA_VA], %g2
190	stx	%g2, [%g1 + KTR_PARM2]
1919:
192#endif
193
194	ldx	[%g5 + ITA_PMAP], %g1
195
196	SET(kernel_pmap_store, %g3, %g2)
197	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
198
199	cmp	%g1, %g2
200	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
201
202	ldx	[%g5 + ITA_VA], %g2
203	or	%g2, %g3, %g2
204
205	stxa	%g0, [%g2] ASI_DMMU_DEMAP
206	stxa	%g0, [%g2] ASI_IMMU_DEMAP
207	membar	#Sync
208
209	IPI_DONE(%g5, %g1, %g2, %g3)
210	retry
211END(tl_ipi_tlb_page_demap)
212
213/*
214 * Demap a range of pages from the dtlb and itlb.
215 */
216ENTRY(tl_ipi_tlb_range_demap)
217#if KTR_COMPILE & KTR_SMP
218	CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx"
219	    , %g1, %g2, %g3, 7, 8, 9)
220	ldx	[%g5 + ITA_PMAP], %g2
221	stx	%g2, [%g1 + KTR_PARM1]
222	ldx	[%g5 + ITA_START], %g2
223	stx	%g2, [%g1 + KTR_PARM2]
224	ldx	[%g5 + ITA_END], %g2
225	stx	%g2, [%g1 + KTR_PARM3]
2269:
227#endif
228
229	ldx	[%g5 + ITA_PMAP], %g1
230
231	SET(kernel_pmap_store, %g3, %g2)
232	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
233
234	cmp	%g1, %g2
235	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
236
237	ldx	[%g5 + ITA_START], %g1
238	ldx	[%g5 + ITA_END], %g2
239
240	set	PAGE_SIZE, %g6
241
2421:	or	%g1, %g3, %g4
243	stxa	%g0, [%g4] ASI_DMMU_DEMAP
244	stxa	%g0, [%g4] ASI_IMMU_DEMAP
245	membar	#Sync
246
247	add	%g1, %g6, %g1
248	cmp	%g1, %g2
249	blt,a,pt %xcc, 1b
250	 nop
251
252	IPI_DONE(%g5, %g1, %g2, %g3)
253	retry
254END(tl_ipi_tlb_range_demap)
255
256/*
257 * Demap the primary context from the dtlb and itlb.
258 */
259ENTRY(tl_ipi_tlb_context_demap)
260#if KTR_COMPILE & KTR_SMP
261	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
262	    , %g1, %g2, %g3, 7, 8, 9)
263	ldx	[%g5 + ITA_PMAP], %g2
264	stx	%g2, [%g1 + KTR_PARM1]
265	ldx	[%g5 + ITA_VA], %g2
266	stx	%g2, [%g1 + KTR_PARM2]
2679:
268#endif
269
270	mov	TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
271	stxa	%g0, [%g1] ASI_DMMU_DEMAP
272	stxa	%g0, [%g1] ASI_IMMU_DEMAP
273	membar	#Sync
274
275	IPI_DONE(%g5, %g1, %g2, %g3)
276	retry
277END(tl_ipi_tlb_context_demap)
278