mp_locore.S revision 203833 
1/*-
2 * Copyright (c) 2002 Jake Burkholder.
3 * Copyright (c) 2008 Marius Strobl <marius@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <machine/asm.h>
29__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/mp_locore.S 203833 2010-02-13 15:36:33Z marius $");
30
31#include <machine/asi.h>
32#include <machine/asmacros.h>
33#include <machine/ktr.h>
34#include <machine/pstate.h>
35#include <machine/smp.h>
36#include <machine/upa.h>
37#include <machine/ver.h>
38
39#include "assym.s"
40
41	.register	%g2, #ignore
42	.register	%g3, #ignore
43
44	.text
45	_ALIGN_TEXT
46	/*
47	 * Initialize misc. state to known values: interrupts disabled, normal
48	 * globals, windows flushed (cr = 0, cs = nwindows - 1), PIL 0 and
49	 * floating point disabled.
50	 * Note that some firmware versions don't implement a clean window
51	 * trap handler so we unfortunately can't clear the windows by setting
52	 * %cleanwin to zero here.
53	 */
541:	wrpr	%g0, PSTATE_NORMAL, %pstate
55	wrpr	%g0, 0, %pil
56	wr	%g0, 0, %fprs
57
58	rdpr	%ver, %l7
59	srlx	%l7, VER_IMPL_SHIFT, %l7
60	sll	%l7, VER_IMPL_SIZE, %l7
61	srl	%l7, VER_IMPL_SIZE, %l7
62	cmp	%l7, CPU_IMPL_ULTRASPARCIIIp
63	bne	%icc, 3f
64	 nop
65
66	/*
67	 * Relocate the locked entry in it16 slot 0 (if existent)
68	 * as part of working around Cheetah+ erratum 34.
69	 */
70
71	setx	TD_V | TD_L, %l1, %l0
72	/*
73	 * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work
74	 * around errata of USIII and beyond.
75	 */
76	ldxa	[%g0] ASI_ITLB_DATA_ACCESS_REG, %g0
77	ldxa	[%g0] ASI_ITLB_DATA_ACCESS_REG, %l6
78	and	%l6, %l0, %l1
79	cmp	%l0, %l1
80	bne	%xcc, 3f
81	 nop
82
83	/* Flush the mapping of slot 0. */
84	ldxa	[%g0] ASI_ITLB_TAG_READ_REG, %l5
85	srlx	%l5, TAR_VPN_SHIFT, %l0
86	sllx	%l0, TAR_VPN_SHIFT, %l0
87	or	%l0, TLB_DEMAP_PRIMARY | TLB_DEMAP_PAGE, %l0
88	stxa	%g0, [%l0] ASI_IMMU_DEMAP
89	/* The USIII-family ignores the address. */
90	flush	%g0
91
92	/*
93	 * Search a replacement slot != 0 and enter the data and tag
94	 * that formerly were in slot 0.
95	 */
96	mov	(1 << TLB_DAR_SLOT_SHIFT), %l4
97	setx	TD_V, %l1, %l0
98	/*
99	 * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work
100	 * around errata of USIII and beyond.
101	 */
1022:	ldxa	[%l4] ASI_ITLB_DATA_ACCESS_REG, %g0
103	ldxa	[%l4] ASI_ITLB_DATA_ACCESS_REG, %l1
104	and	%l1, %l0, %l1
105	cmp	%l0, %l1
106	be,a	%xcc, 2b
107	 add	%l4, (1 << TLB_DAR_SLOT_SHIFT), %l4
108	wr	%g0, ASI_IMMU, %asi
109	stxa	%l5, [%g0 + AA_IMMU_TAR] %asi
110	stxa	%l6, [%l4] ASI_ITLB_DATA_ACCESS_REG
111	/* The USIII-family ignores the address. */
112	flush	%g0
113
1143:	rd	%pc, %l6
115	ldx	[%l6 + (9f-3b)], %l1
116	add	%l6, (11f-3b), %l2
117	clr	%l3
1184:	cmp	%l3, %l1
119	be	%xcc, 8f
120	 nop
121	ldx	[%l2 + TTE_VPN], %l4
122	ldx	[%l2 + TTE_DATA], %l5
123	srlx	%l4, TV_SIZE_BITS, %l4
124	sllx	%l4, PAGE_SHIFT_4M, %l4
125	wr	%g0, ASI_DMMU, %asi
126	stxa	%l4, [%g0 + AA_DMMU_TAR] %asi
127	stxa	%l5, [%g0] ASI_DTLB_DATA_IN_REG
128	membar	#Sync
129
130	cmp	%l7, CPU_IMPL_ULTRASPARCIIIp
131	bne	%icc, 6f
132	 wr	%g0, ASI_IMMU, %asi
133
134	/*
135	 * Search an unused slot != 0 and explicitly enter the data
136	 * and tag there in order to avoid Cheetah+ erratum 34.
137	 */
138	mov	(1 << TLB_DAR_SLOT_SHIFT), %l0
139	setx	TD_V, %o1, %o0
140	/*
141	 * We read ASI_ITLB_DATA_ACCESS_REG twice in order to work
142	 * around errata of USIII and beyond.
143	 */
1445:	ldxa	[%l0] ASI_ITLB_DATA_ACCESS_REG, %g0
145	ldxa	[%l0] ASI_ITLB_DATA_ACCESS_REG, %o1
146	and	%o1, %o0, %o1
147	cmp	%o0, %o1
148	be,a	%xcc, 5b
149	 add	%l0, (1 << TLB_DAR_SLOT_SHIFT), %l0
150	sethi	%hi(KERNBASE), %o0
151	stxa	%l4, [%g0 + AA_IMMU_TAR] %asi
152	stxa	%l5, [%l0] ASI_ITLB_DATA_ACCESS_REG
153	flush	%o0
154	ba	%xcc, 7f
155	 nop
156
1576:	sethi	%hi(KERNBASE), %l0
158	stxa	%l4, [%g0 + AA_IMMU_TAR] %asi
159	stxa	%l5, [%g0] ASI_ITLB_DATA_IN_REG
160	flush	%l0
1617:	add	%l2, 1 << TTE_SHIFT, %l2
162	add	%l3, 1, %l3
163	ba	%xcc, 4b
164	 nop
1658:	ldx	[%l6 + (10f-3b)], %l1
166	jmpl	%l1, %g0
167	 nop
168	_ALIGN_DATA
1699:	.xword	0x0
17010:	.xword	0x0
17111:
172
173DATA(mp_tramp_code)
174	.xword	1b
175DATA(mp_tramp_code_len)
176	.xword	11b-1b
177DATA(mp_tramp_tlb_slots)
178	.xword	9b-1b
179DATA(mp_tramp_func)
180	.xword	10b-1b
181
182/*
183 * void mp_startup(void)
184 */
185ENTRY(mp_startup)
186	SET(cpu_start_args, %l1, %l0)
187
188	mov	CPU_TICKSYNC, %l1
189	membar	#StoreLoad
190	stw	%l1, [%l0 + CSA_STATE]
191
1921:	ldx	[%l0 + CSA_TICK], %l1
193	brz	%l1, 1b
194	 nop
195	wrpr	%l1, 0, %tick
196
197	rdpr	%ver, %l1
198	stx	%l1, [%l0 + CSA_VER]
199
200	srlx	%l1, VER_IMPL_SHIFT, %l1
201	sll	%l1, VER_IMPL_SIZE, %l1
202	srl	%l1, VER_IMPL_SIZE, %l1
203	cmp	%l1, CPU_IMPL_ULTRASPARCIII
204	bl	%icc, 3f
205	 nop
206	mov	CPU_STICKSYNC, %l1
207	membar	#StoreLoad
208	stw	%l1, [%l0 + CSA_STATE]
209
2102:	ldx	[%l0 + CSA_STICK], %l1
211	brz	%l1, 2b
212	 nop
213	wr	%l1, 0, %asr24
214
2153:	UPA_GET_MID(%o0)
216
217#if KTR_COMPILE & KTR_SMP
218	CATR(KTR_SMP, "mp_start: CPU %d entered kernel"
219	    , %g1, %g2, %g3, 7, 8, 9)
220	stx	%o0, [%g1 + KTR_PARM1]
2219:
222#endif
223
224	/*
225	 * Inform the boot processor we have inited.
226	 */
227	mov	CPU_INIT, %l1
228	membar	#LoadStore
229	stw	%l1, [%l0 + CSA_STATE]
230
231	/*
232	 * Wait till its our turn to bootstrap.
233	 */
2344:	lduw	[%l0 + CSA_MID], %l1
235	cmp	%l1, %o0
236	bne	%xcc, 4b
237	 nop
238
239#if KTR_COMPILE & KTR_SMP
240	CATR(KTR_SMP, "_mp_start: CPU %d got start signal"
241	    , %g1, %g2, %g3, 7, 8, 9)
242	stx	%o0, [%g1 + KTR_PARM1]
2439:
244#endif
245
246	add	%l0, CSA_TTES, %l1
247	clr	%l2
248
249	/*
250	 * Map the per-CPU pages.
251	 */
2525:	sllx	%l2, TTE_SHIFT, %l3
253	add	%l1, %l3, %l3
254
255	ldx	[%l3 + TTE_VPN], %l4
256	ldx	[%l3 + TTE_DATA], %l5
257
258	wr	%g0, ASI_DMMU, %asi
259	srlx	%l4, TV_SIZE_BITS, %l4
260	sllx	%l4, PAGE_SHIFT_8K, %l4
261	stxa	%l4, [%g0 + AA_DMMU_TAR] %asi
262	stxa	%l5, [%g0] ASI_DTLB_DATA_IN_REG
263	membar	#Sync
264
265	add	%l2, 1, %l2
266	cmp	%l2, PCPU_PAGES
267	bne	%xcc, 5b
268	 nop
269
270	/*
271	 * Get onto our per-CPU panic stack, which precedes the struct pcpu
272	 * in the per-CPU page.
273	 */
274	ldx	[%l0 + CSA_PCPU], %l1
275	set	PCPU_PAGES * PAGE_SIZE - PC_SIZEOF, %l2
276	add	%l1, %l2, %l1
277	sub	%l1, SPOFF + CCFSZ, %sp
278
279	/*
280	 * Enable interrupts.
281	 */
282	wrpr	%g0, PSTATE_KERNEL, %pstate
283
284#if KTR_COMPILE & KTR_SMP
285	CATR(KTR_SMP,
286	    "_mp_start: bootstrap cpuid=%d mid=%d pcpu=%#lx data=%#lx sp=%#lx"
287	    , %g1, %g2, %g3, 7, 8, 9)
288	lduw	[%l1 + PC_CPUID], %g2
289	stx	%g2, [%g1 + KTR_PARM1]
290	lduw	[%l1 + PC_MID], %g2
291	stx	%g2, [%g1 + KTR_PARM2]
292	stx	%l1, [%g1 + KTR_PARM3]
293	stx	%sp, [%g1 + KTR_PARM5]
2949:
295#endif
296
297	/*
298	 * And away we go.  This doesn't return.
299	 */
300	call	cpu_mp_bootstrap
301	 mov	%l1, %o0
302	sir
303	! NOTREACHED
304END(mp_startup)
305