1/*-
2 * Copyright (c) 2002 Jake Burkholder.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <machine/asm.h>
28__FBSDID("$FreeBSD$");
29
30#include <machine/asi.h>
31#include <machine/asmacros.h>
32#include <machine/cache.h>
33#include <machine/ktr.h>
34#include <machine/pstate.h>
35
36#include "assym.s"
37
38	.register	%g2, #ignore
39	.register	%g3, #ignore
40
41#define	IPI_DONE(r1, r2, r3, r4, r5, r6)				\
42	rd	%y, r6 ;						\
43	lduw	[PCPU(CPUID)], r2 ;					\
44	mov	_NCPUBITS, r3 ;						\
45	mov	%g0, %y ;						\
46	udiv	r2, r3, r4 ;						\
47	srl	r4, 0, r5 ;						\
48	sllx	r5, PTR_SHIFT, r5 ;					\
49	add	r1, r5, r1 ;						\
50	smul	r4, r3, r3 ;						\
51	sub	r2, r3, r3 ;						\
52	mov	1, r4 ;							\
53	sllx	r4, r3, r4 ;						\
54	wr	r6, %y ;						\
55	ATOMIC_CLEAR_LONG(r1, r2, r3, r4)
56
57/*
58 * Invalidate a physical page in the data cache.  For UltraSPARC I and II.
59 */
60ENTRY(tl_ipi_spitfire_dcache_page_inval)
61#if KTR_COMPILE & KTR_SMP
62	CATR(KTR_SMP, "tl_ipi_spitfire_dcache_page_inval: pa=%#lx"
63	    , %g1, %g2, %g3, 7, 8, 9)
64	ldx	[%g5 + ICA_PA], %g2
65	stx	%g2, [%g1 + KTR_PARM1]
669:
67#endif
68
69	ldx	[%g5 + ICA_PA], %g6
70	srlx	%g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6
71
72	lduw	[PCPU(CACHE) + DC_SIZE], %g3
73	lduw	[PCPU(CACHE) + DC_LINESIZE], %g4
74	sub	%g3, %g4, %g2
75
761:	ldxa	[%g2] ASI_DCACHE_TAG, %g1
77	srlx	%g1, DC_VALID_SHIFT, %g3
78	andcc	%g3, DC_VALID_MASK, %g0
79	bz,pt	%xcc, 2f
80	 set	DC_TAG_MASK, %g3
81	sllx	%g3, DC_TAG_SHIFT, %g3
82	and	%g1, %g3, %g1
83	cmp	%g1, %g6
84	bne,a,pt %xcc, 2f
85	 nop
86	stxa	%g1, [%g2] ASI_DCACHE_TAG
87	membar	#Sync
88
892:	brgz,pt	%g2, 1b
90	 sub	%g2, %g4, %g2
91
92	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
93	retry
94END(tl_ipi_spitfire_dcache_page_inval)
95
96/*
97 * Invalidate a physical page in the instruction cache.  For UltraSPARC I and
98 * II.
99 */
100ENTRY(tl_ipi_spitfire_icache_page_inval)
101#if KTR_COMPILE & KTR_SMP
102	CATR(KTR_SMP, "tl_ipi_spitfire_icache_page_inval: pa=%#lx"
103	    , %g1, %g2, %g3, 7, 8, 9)
104	ldx	[%g5 + ICA_PA], %g2
105	stx	%g2, [%g1 + KTR_PARM1]
1069:
107#endif
108
109	ldx	[%g5 + ICA_PA], %g6
110	srlx	%g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6
111
112	lduw	[PCPU(CACHE) + IC_SIZE], %g3
113	lduw	[PCPU(CACHE) + IC_LINESIZE], %g4
114	sub	%g3, %g4, %g2
115
1161:	ldda	[%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */
117	srlx	%g1, IC_VALID_SHIFT, %g3
118	andcc	%g3, IC_VALID_MASK, %g0
119	bz,pt	%xcc, 2f
120	 set	IC_TAG_MASK, %g3
121	sllx	%g3, IC_TAG_SHIFT, %g3
122	and	%g1, %g3, %g1
123	cmp	%g1, %g6
124	bne,a,pt %xcc, 2f
125	 nop
126	stxa	%g1, [%g2] ASI_ICACHE_TAG
127	membar	#Sync
128
1292:	brgz,pt	%g2, 1b
130	 sub	%g2, %g4, %g2
131
132	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
133	retry
134END(tl_ipi_spitfire_icache_page_inval)
135
136/*
137 * Invalidate a physical page in the data cache.  For UltraSPARC III.
138 */
139ENTRY(tl_ipi_cheetah_dcache_page_inval)
140#if KTR_COMPILE & KTR_SMP
141	CATR(KTR_SMP, "tl_ipi_cheetah_dcache_page_inval: pa=%#lx"
142	    , %g1, %g2, %g3, 7, 8, 9)
143	ldx	[%g5 + ICA_PA], %g2
144	stx	%g2, [%g1 + KTR_PARM1]
1459:
146#endif
147
148	ldx	[%g5 + ICA_PA], %g1
149
150	set	PAGE_SIZE, %g2
151	add	%g1, %g2, %g3
152
153	lduw	[PCPU(CACHE) + DC_LINESIZE], %g2
154
1551:	stxa	%g0, [%g1] ASI_DCACHE_INVALIDATE
156	membar	#Sync
157
158	add	%g1, %g2, %g1
159	cmp	%g1, %g3
160	blt,a,pt %xcc, 1b
161	 nop
162
163	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
164	retry
165END(tl_ipi_cheetah_dcache_page_inval)
166
167/*
168 * Trigger a softint at the desired level.
169 */
170ENTRY(tl_ipi_level)
171#if KTR_COMPILE & KTR_SMP
172	CATR(KTR_SMP, "tl_ipi_level: cpuid=%d mid=%d d1=%#lx d2=%#lx"
173	    , %g1, %g2, %g3, 7, 8, 9)
174	lduw	[PCPU(CPUID)], %g2
175	stx	%g2, [%g1 + KTR_PARM1]
176	lduw	[PCPU(MID)], %g2
177	stx	%g2, [%g1 + KTR_PARM2]
178	stx	%g4, [%g1 + KTR_PARM3]
179	stx	%g5, [%g1 + KTR_PARM4]
1809:
181#endif
182
183	mov	1, %g1
184	sllx	%g1, %g5, %g1
185	wr	%g1, 0, %set_softint
186	retry
187END(tl_ipi_level)
188
189/*
190 * Demap a page from the dtlb and/or itlb.
191 */
192ENTRY(tl_ipi_tlb_page_demap)
193#if KTR_COMPILE & KTR_SMP
194	CATR(KTR_SMP, "ipi_tlb_page_demap: pm=%p va=%#lx"
195	    , %g1, %g2, %g3, 7, 8, 9)
196	ldx	[%g5 + ITA_PMAP], %g2
197	stx	%g2, [%g1 + KTR_PARM1]
198	ldx	[%g5 + ITA_VA], %g2
199	stx	%g2, [%g1 + KTR_PARM2]
2009:
201#endif
202
203	ldx	[%g5 + ITA_PMAP], %g1
204
205	SET(kernel_pmap_store, %g3, %g2)
206	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
207
208	cmp	%g1, %g2
209	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
210
211	ldx	[%g5 + ITA_VA], %g2
212	or	%g2, %g3, %g2
213
214	sethi	%hi(KERNBASE), %g3
215	stxa	%g0, [%g2] ASI_DMMU_DEMAP
216	stxa	%g0, [%g2] ASI_IMMU_DEMAP
217	flush	%g3
218
219	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
220	retry
221END(tl_ipi_tlb_page_demap)
222
223/*
224 * Demap a range of pages from the dtlb and itlb.
225 */
226ENTRY(tl_ipi_tlb_range_demap)
227#if KTR_COMPILE & KTR_SMP
228	CATR(KTR_SMP, "ipi_tlb_range_demap: pm=%p start=%#lx end=%#lx"
229	    , %g1, %g2, %g3, 7, 8, 9)
230	ldx	[%g5 + ITA_PMAP], %g2
231	stx	%g2, [%g1 + KTR_PARM1]
232	ldx	[%g5 + ITA_START], %g2
233	stx	%g2, [%g1 + KTR_PARM2]
234	ldx	[%g5 + ITA_END], %g2
235	stx	%g2, [%g1 + KTR_PARM3]
2369:
237#endif
238
239	ldx	[%g5 + ITA_PMAP], %g1
240
241	SET(kernel_pmap_store, %g3, %g2)
242	mov	TLB_DEMAP_NUCLEUS | TLB_DEMAP_PAGE, %g3
243
244	cmp	%g1, %g2
245	movne	%xcc, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %g3
246
247	ldx	[%g5 + ITA_START], %g1
248	ldx	[%g5 + ITA_END], %g2
249
250	sethi	%hi(KERNBASE), %g6
2511:	or	%g1, %g3, %g4
252	stxa	%g0, [%g4] ASI_DMMU_DEMAP
253	stxa	%g0, [%g4] ASI_IMMU_DEMAP
254	flush	%g6
255
256	set	PAGE_SIZE, %g6
257	add	%g1, %g6, %g1
258	cmp	%g1, %g2
259	blt,a,pt %xcc, 1b
260	 sethi	%hi(KERNBASE), %g6
261
262	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
263	retry
264END(tl_ipi_tlb_range_demap)
265
266/*
267 * Demap the primary context from the dtlb and itlb.
268 */
269ENTRY(tl_ipi_tlb_context_demap)
270#if KTR_COMPILE & KTR_SMP
271	CATR(KTR_SMP, "tl_ipi_tlb_context_demap: pm=%p va=%#lx"
272	    , %g1, %g2, %g3, 7, 8, 9)
273	ldx	[%g5 + ITA_PMAP], %g2
274	stx	%g2, [%g1 + KTR_PARM1]
275	ldx	[%g5 + ITA_VA], %g2
276	stx	%g2, [%g1 + KTR_PARM2]
2779:
278#endif
279
280	mov	TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1
281	sethi	%hi(KERNBASE), %g3
282	stxa	%g0, [%g1] ASI_DMMU_DEMAP
283	stxa	%g0, [%g1] ASI_IMMU_DEMAP
284	flush	%g3
285
286	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
287	retry
288END(tl_ipi_tlb_context_demap)
289
290/*
291 * Read %stick.
292 */
293ENTRY(tl_ipi_stick_rd)
294	ldx	[%g5 + IRA_VAL], %g1
295	rd	%asr24, %g2
296	stx	%g2, [%g1]
297
298	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
299	retry
300END(tl_ipi_stick_rd)
301
302/*
303 * Read %tick.
304 */
305ENTRY(tl_ipi_tick_rd)
306	ldx	[%g5 + IRA_VAL], %g1
307	rd	%tick, %g2
308	stx	%g2, [%g1]
309
310	IPI_DONE(%g5, %g1, %g2, %g3, %g4, %g6)
311	retry
312END(tl_ipi_tick_rd)
313