1/*-
2 * Copyright (c) 2001 Jake Burkholder.
3 * Copyright (c) 2011 Marius Strobl <marius@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <machine/asm.h>
29__FBSDID("$FreeBSD$");
30
31#include <machine/asmacros.h>
32#include <machine/asi.h>
33#include <machine/fsr.h>
34#include <machine/ktr.h>
35#include <machine/pcb.h>
36#include <machine/tstate.h>
37
38#include "assym.s"
39#include "opt_sched.h"
40
41	.register	%g2, #ignore
42	.register	%g3, #ignore
43
44/*
45 * void cpu_throw(struct thread *old, struct thread *new)
46 */
47ENTRY(cpu_throw)
48	save	%sp, -CCFSZ, %sp
49	flushw
50	ba	%xcc, .Lsw1
51	 mov	%g0, %i2
52END(cpu_throw)
53
54/*
55 * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
56 */
57ENTRY(cpu_switch)
58	save	%sp, -CCFSZ, %sp
59
60	/*
61	 * If the current thread was using floating point in the kernel, save
62	 * its context.  The userland floating point context has already been
63	 * saved in that case.
64	 */
65	rd	%fprs, %l2
66	andcc	%l2, FPRS_FEF, %g0
67	bz,a,pt	%xcc, 1f
68	 nop
69	call	savefpctx
70	 add	PCB_REG, PCB_KFP, %o0
71	ba,a,pt	%xcc, 2f
72	 nop
73
74	/*
75	 * If the current thread was using floating point in userland, save
76	 * its context.
77	 */
781:	sub	PCB_REG, TF_SIZEOF, %l2
79	ldx	[%l2 + TF_FPRS], %l3
80	andcc	%l3, FPRS_FEF, %g0
81	bz,a,pt	%xcc, 2f
82	 nop
83	call	savefpctx
84	 add	PCB_REG, PCB_UFP, %o0
85	andn	%l3, FPRS_FEF, %l3
86	stx	%l3, [%l2 + TF_FPRS]
87
88	ldx	[PCB_REG + PCB_FLAGS], %l3
89	or	%l3, PCB_FEF, %l3
90	stx	%l3, [PCB_REG + PCB_FLAGS]
91
92	/*
93	 * Flush the windows out to the stack and save the current frame
94	 * pointer and program counter.
95	 */
962:	flushw
97	wrpr	%g0, 0, %cleanwin
98	stx	%fp, [PCB_REG + PCB_SP]
99	stx	%i7, [PCB_REG + PCB_PC]
100
101	/*
102	 * Load the new thread's frame pointer and program counter, and set
103	 * the current thread and pcb.
104	 */
105.Lsw1:
106#if KTR_COMPILE & KTR_PROC
107	CATR(KTR_PROC, "cpu_switch: new td=%p pc=%#lx fp=%#lx"
108	    , %g1, %g2, %g3, 8, 9, 10)
109	stx	%i1, [%g1 + KTR_PARM1]
110	ldx	[%i1 + TD_PCB], %g2
111	ldx	[%g2 + PCB_PC], %g3
112	stx	%g3, [%g1 + KTR_PARM2]
113	ldx	[%g2 + PCB_SP], %g3
114	stx	%g3, [%g1 + KTR_PARM3]
11510:
116#endif
117	ldx	[%i1 + TD_PCB], %l0
118
119	stx	%i1, [PCPU(CURTHREAD)]
120	stx	%l0, [PCPU(CURPCB)]
121
122	wrpr	%g0, PSTATE_NORMAL, %pstate
123	mov	%l0, PCB_REG
124	wrpr	%g0, PSTATE_ALT, %pstate
125	mov	%l0, PCB_REG
126	wrpr	%g0, PSTATE_KERNEL, %pstate
127
128	ldx	[PCB_REG + PCB_SP], %fp
129	ldx	[PCB_REG + PCB_PC], %i7
130	sub	%fp, CCFSZ, %sp
131
132	/*
133	 * Point to the pmaps of the new process, and of the last non-kernel
134	 * process to run.
135	 */
136	ldx	[%i1 + TD_PROC], %l1
137	ldx	[PCPU(PMAP)], %l2
138	ldx	[%l1 + P_VMSPACE], %i5
139	add	%i5, VM_PMAP, %l1
140
141#if KTR_COMPILE & KTR_PROC
142	CATR(KTR_PROC, "cpu_switch: new pmap=%p old pmap=%p"
143	    , %g1, %g2, %g3, 8, 9, 10)
144	stx	%l1, [%g1 + KTR_PARM1]
145	stx	%l2, [%g1 + KTR_PARM2]
14610:
147#endif
148
149	/*
150	 * If they are the same we are done.
151	 */
152	cmp	%l2, %l1
153	be,a,pn	%xcc, 8f
154	 nop
155
156	/*
157	 * If the new process is a kernel thread we can just leave the old
158	 * context active and avoid recycling its context number.
159	 */
160	SET(vmspace0, %i4, %i3)
161	cmp	%i5, %i3
162	be,a,pn	%xcc, 8f
163	 nop
164
165	/*
166	 * If there was no non-kernel pmap, don't try to deactivate it.
167	 */
168	brz,pn	%l2, 3f
169	 lduw	[PCPU(CPUID)], %l3
170
171	/*
172	 * Mark the pmap of the last non-kernel vmspace to run as no longer
173	 * active on this CPU.
174	 */
175	mov	_NCPUBITS, %l5
176	udivx	%l3, %l5, %l6
177	srl	%l6, 0, %l4
178	sllx	%l4, PTR_SHIFT, %l4
179	add	%l4, PM_ACTIVE, %l4
180	smul	%l6, %l5, %l5
181	sub	%l3, %l5, %l5
182	mov	1, %l6
183	sllx	%l6, %l5, %l5
184#ifdef SMP
185	add	%l2, %l4, %l4
186	membar	#LoadStore | #StoreStore
187	ATOMIC_CLEAR_LONG(%l4, %l6, %l7, %l5)
188#else
189	ldx	[%l2 + %l4], %l6
190	andn	%l6, %l5, %l6
191	stx	%l6, [%l2 + %l4]
192#endif
193
194	/*
195	 * Take away its context number.
196	 */
197	sllx	%l3, INT_SHIFT, %l3
198	add	%l2, PM_CONTEXT, %l4
199	mov	-1, %l5
200	stw	%l5, [%l3 + %l4]
201
2023:	cmp	%i2, %g0
203	be,pn	%xcc, 4f
204	 add	%i0, TD_LOCK, %l4
205#if defined(SCHED_ULE) && defined(SMP)
206	membar	#LoadStore | #StoreStore
207	ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2)
208#else
209	stx	%i2, [%l4]
210#endif
211
212	/*
213	 * Find a new TLB context.  If we've run out we have to flush all
214	 * user mappings from the TLB and reset the context numbers.
215	 */
2164:	lduw	[PCPU(TLB_CTX)], %i3
217	lduw	[PCPU(TLB_CTX_MAX)], %i4
218	cmp	%i3, %i4
219	bne,a,pt %xcc, 5f
220	 nop
221	SET(tlb_flush_user, %i5, %i4)
222	ldx	[%i4], %i5
223	call	%i5
224	 lduw	[PCPU(TLB_CTX_MIN)], %i3
225
226	/*
227	 * Advance next free context.
228	 */
2295:	add	%i3, 1, %i4
230	stw	%i4, [PCPU(TLB_CTX)]
231
232	/*
233	 * Set the new context number in the pmap.
234	 */
235	lduw	[PCPU(CPUID)], %l3
236	sllx	%l3, INT_SHIFT, %i4
237	add	%l1, PM_CONTEXT, %i5
238	stw	%i3, [%i4 + %i5]
239
240	/*
241	 * Mark the pmap as active on this CPU.
242	 */
243	mov	_NCPUBITS, %l5
244	udivx	%l3, %l5, %l6
245	srl	%l6, 0, %l4
246	sllx	%l4, PTR_SHIFT, %l4
247	add	%l4, PM_ACTIVE, %l4
248	smul	%l6, %l5, %l5
249	sub	%l3, %l5, %l5
250	mov	1, %l6
251	sllx	%l6, %l5, %l5
252#ifdef SMP
253	add	%l1, %l4, %l4
254	ATOMIC_SET_LONG(%l4, %l6, %l7, %l5)
255#else
256	ldx	[%l1 + %l4], %l6
257	or	%l6, %l5, %l6
258	stx	%l6, [%l1 + %l4]
259#endif
260
261	/*
262	 * Make note of the change in pmap.
263	 */
264#ifdef SMP
265	PCPU_ADDR(PMAP, %l4)
266	ATOMIC_STORE_LONG(%l4, %l5, %l6, %l1)
267#else
268	stx	%l1, [PCPU(PMAP)]
269#endif
270
271	/*
272	 * Fiddle the hardware bits.  Set the TSB registers and install the
273	 * new context number in the CPU.
274	 */
275	ldx	[%l1 + PM_TSB], %i4
276	mov	AA_DMMU_TSB, %i5
277	stxa	%i4, [%i5] ASI_DMMU
278	mov	AA_IMMU_TSB, %i5
279	stxa	%i4, [%i5] ASI_IMMU
280	setx	TLB_CXR_PGSZ_MASK, %i5, %i4
281	mov	AA_DMMU_PCXR, %i5
282	ldxa	[%i5] ASI_DMMU, %l1
283	and	%l1, %i4, %l1
284	or	%i3, %l1, %i3
285	sethi	%hi(KERNBASE), %i4
286	stxa	%i3, [%i5] ASI_DMMU
287	flush	%i4
288
2896:
290#if defined(SCHED_ULE) && defined(SMP)
291	SET(blocked_lock, %l2, %l1)
292	add	%i1, TD_LOCK, %l2
2937:
294	ATOMIC_LOAD_LONG(%l2, %l3)
295	cmp	%l1, %l3
296	be,a,pn	%xcc, 7b
297	 nop
298#endif
299
300	/*
301	 * Done, return and load the new process's window from the stack.
302	 */
303	ret
304	 restore
305
3068:	cmp	%i2, %g0
307	be,pn	%xcc, 6b
308	 add	%i0, TD_LOCK, %l4
309#if defined(SCHED_ULE) && defined(SMP)
310	membar	#LoadStore | #StoreStore
311	ATOMIC_STORE_LONG(%l4, %l6, %l7, %i2)
312	ba,pt	%xcc, 6b
313	 nop
314#else
315	ba,pt	%xcc, 6b
316	 stx	%i2, [%l4]
317#endif
318END(cpu_switch)
319
320ENTRY(savectx)
321	save	%sp, -CCFSZ, %sp
322	flushw
323	call	savefpctx
324	 add	%i0, PCB_UFP, %o0
325	stx	%fp, [%i0 + PCB_SP]
326	stx	%i7, [%i0 + PCB_PC]
327	ret
328	 restore %g0, 0, %o0
329END(savectx)
330
331/*
332 * void savefpctx(uint32_t *);
333 */
334ENTRY(savefpctx)
335	wr	%g0, FPRS_FEF, %fprs
336	wr	%g0, ASI_BLK_S, %asi
337	stda	%f0, [%o0 + (0 * 64)] %asi
338	stda	%f16, [%o0 + (1 * 64)] %asi
339	stda	%f32, [%o0 + (2 * 64)] %asi
340	stda	%f48, [%o0 + (3 * 64)] %asi
341	membar	#Sync
342	retl
343	 wr	%g0, 0, %fprs
344END(savefpctx)
345