1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1990 The Regents of the University of California.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * William Jolitz.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * $FreeBSD: stable/10/sys/amd64/amd64/cpu_switch.S 335455 2018-06-20 18:51:38Z kib $
34 */
35
36#include <machine/asmacros.h>
37#include <machine/specialreg.h>
38
39#include "assym.s"
40#include "opt_sched.h"
41
42/*****************************************************************************/
43/* Scheduling                                                                */
44/*****************************************************************************/
45
46	.text
47
48#ifdef SMP
49#define LK	lock ;
50#else
51#define LK
52#endif
53
54#if defined(SCHED_ULE) && defined(SMP)
55#define	SETLK	xchgq
56#else
57#define	SETLK	movq
58#endif
59
60/*
61 * cpu_throw()
62 *
63 * This is the second half of cpu_switch(). It is used when the current
64 * thread is either a dummy or slated to die, and we no longer care
65 * about its state.  This is only a slight optimization and is probably
66 * not worth it anymore.  Note that we need to clear the pm_active bits so
67 * we do need the old proc if it still exists.
68 * %rdi = oldtd
69 * %rsi = newtd
70 */
71ENTRY(cpu_throw)
72	movl	PCPU(CPUID),%eax
73	testq	%rdi,%rdi
74	jz	1f
75	/* release bit from old pm_active */
76	movq	PCPU(CURPMAP),%rdx
77	LK btrl	%eax,PM_ACTIVE(%rdx)		/* clear old */
781:
79	movq	TD_PCB(%rsi),%r8		/* newtd->td_pcb */
80	movq	PCB_CR3(%r8),%rcx		/* new address space */
81	jmp	swact
82END(cpu_throw)
83
84/*
85 * cpu_switch(old, new, mtx)
86 *
87 * Save the current thread state, then select the next thread to run
88 * and load its state.
89 * %rdi = oldtd
90 * %rsi = newtd
91 * %rdx = mtx
92 */
93ENTRY(cpu_switch)
94	/* Switch to new thread.  First, save context. */
95	movq	TD_PCB(%rdi),%r8
96	orl	$PCB_FULL_IRET,PCB_FLAGS(%r8)
97
98	movq	(%rsp),%rax			/* Hardware registers */
99	movq	%r15,PCB_R15(%r8)
100	movq	%r14,PCB_R14(%r8)
101	movq	%r13,PCB_R13(%r8)
102	movq	%r12,PCB_R12(%r8)
103	movq	%rbp,PCB_RBP(%r8)
104	movq	%rsp,PCB_RSP(%r8)
105	movq	%rbx,PCB_RBX(%r8)
106	movq	%rax,PCB_RIP(%r8)
107
108	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
109	jnz	store_dr			/* static predict not taken */
110done_store_dr:
111
112	/* have we used fp, and need a save? */
113	cmpq	%rdi,PCPU(FPCURTHREAD)
114	jne	2f
115	movq	PCB_SAVEFPU(%r8),%r8
116	clts
117	cmpl	$0,use_xsave(%rip)
118	jne	1f
119	fxsave	(%r8)
120	jmp	2f
1211:	movq	%rdx,%rcx
122	movl	xsave_mask,%eax
123	movl	xsave_mask+4,%edx
124	.globl	ctx_switch_xsave
125ctx_switch_xsave:
126	/* This is patched to xsaveopt if supported, see fpuinit_bsp1() */
127	xsave	(%r8)
128	movq	%rcx,%rdx
1292:
130
131	/* Save is done.  Now fire up new thread. Leave old vmspace. */
132	movq	TD_PCB(%rsi),%r8
133
134	/* switch address space */
135	movq	PCB_CR3(%r8),%rcx
136	movq	%cr3,%rax
137	cmpq	%rcx,%rax			/* Same address space? */
138	jne	swinact
139	SETLK	%rdx, TD_LOCK(%rdi)		/* Release the old thread */
140	jmp	sw1
141swinact:
142	movl	PCPU(CPUID),%eax
143	/* Release bit from old pmap->pm_active */
144	movq	PCPU(CURPMAP),%r12
145	LK btrl	%eax,PM_ACTIVE(%r12)		/* clear old */
146	SETLK	%rdx,TD_LOCK(%rdi)		/* Release the old thread */
147swact:
148	/* Set bit in new pmap->pm_active */
149	movq	TD_PROC(%rsi),%rdx		/* newproc */
150	movq	P_VMSPACE(%rdx), %rdx
151	addq	$VM_PMAP,%rdx
152	cmpl	$-1,PM_PCID(%rdx)
153	je	1f
154	LK btsl	%eax,PM_SAVE(%rdx)
155	jnc	1f
156	btsq	$63,%rcx			/* CR3_PCID_SAVE */
157	incq	PCPU(PM_SAVE_CNT)
1581:
159	movq	%rcx,%cr3			/* new address space */
160	LK btsl	%eax,PM_ACTIVE(%rdx)		/* set new */
161	movq	%rdx,PCPU(CURPMAP)
162
163	/*
164	 * We might lose the race and other CPU might have changed
165	 * the pmap after we set our bit in pmap->pm_save.  Recheck.
166	 * Reload %cr3 with CR3_PCID_SAVE bit cleared if pmap was
167	 * modified, causing TLB flush for this pcid.
168	 */
169	btrq	$63,%rcx
170	jnc	1f
171	LK btsl	%eax,PM_SAVE(%rdx)
172	jc	1f
173	decq	PCPU(PM_SAVE_CNT)
174	movq	%rcx,%cr3
1751:
176
177sw1:
178#if defined(SCHED_ULE) && defined(SMP)
179	/* Wait for the new thread to become unblocked */
180	movq	$blocked_lock, %rdx
1811:
182	movq	TD_LOCK(%rsi),%rcx
183	cmpq	%rcx, %rdx
184	pause
185	je	1b
186#endif
187	/*
188	 * At this point, we've switched address spaces and are ready
189	 * to load up the rest of the next context.
190	 */
191
192	/* Skip loading user fsbase/gsbase for kthreads */
193	testl	$TDP_KTHREAD,TD_PFLAGS(%rsi)
194	jnz	do_kthread
195
196	/*
197	 * Load ldt register
198	 */
199	movq	TD_PROC(%rsi),%rcx
200	cmpq	$0, P_MD+MD_LDT(%rcx)
201	jne	do_ldt
202	xorl	%eax,%eax
203ld_ldt:	lldt	%ax
204
205	/* Restore fs base in GDT */
206	movl	PCB_FSBASE(%r8),%eax
207	movq	PCPU(FS32P),%rdx
208	movw	%ax,2(%rdx)
209	shrl	$16,%eax
210	movb	%al,4(%rdx)
211	shrl	$8,%eax
212	movb	%al,7(%rdx)
213
214	/* Restore gs base in GDT */
215	movl	PCB_GSBASE(%r8),%eax
216	movq	PCPU(GS32P),%rdx
217	movw	%ax,2(%rdx)
218	shrl	$16,%eax
219	movb	%al,4(%rdx)
220	shrl	$8,%eax
221	movb	%al,7(%rdx)
222
223do_kthread:
224	/* Do we need to reload tss ? */
225	movq	PCPU(TSSP),%rax
226	movq	PCB_TSSP(%r8),%rdx
227	testq	%rdx,%rdx
228	cmovzq	PCPU(COMMONTSSP),%rdx
229	cmpq	%rax,%rdx
230	jne	do_tss
231done_tss:
232	movq	%r8,PCPU(RSP0)
233	movq	%r8,PCPU(CURPCB)
234	/* Update the TSS_RSP0 pointer for the next interrupt */
235	movq	%r8,COMMON_TSS_RSP0(%rdx)
236	movq	%rsi,PCPU(CURTHREAD)		/* into next thread */
237
238	/* Test if debug registers should be restored. */
239	testl	$PCB_DBREGS,PCB_FLAGS(%r8)
240	jnz	load_dr				/* static predict not taken */
241done_load_dr:
242
243	/* Restore context. */
244	movq	PCB_R15(%r8),%r15
245	movq	PCB_R14(%r8),%r14
246	movq	PCB_R13(%r8),%r13
247	movq	PCB_R12(%r8),%r12
248	movq	PCB_RBP(%r8),%rbp
249	movq	PCB_RSP(%r8),%rsp
250	movq	PCB_RBX(%r8),%rbx
251	movq	PCB_RIP(%r8),%rax
252	movq	%rax,(%rsp)
253	movq	PCPU(CURTHREAD),%rdi
254	call	fpu_activate_sw
255	ret
256
257	/*
258	 * We order these strangely for several reasons.
259	 * 1: I wanted to use static branch prediction hints
260	 * 2: Most athlon64/opteron cpus don't have them.  They define
261	 *    a forward branch as 'predict not taken'.  Intel cores have
262	 *    the 'rep' prefix to invert this.
263	 * So, to make it work on both forms of cpu we do the detour.
264	 * We use jumps rather than call in order to avoid the stack.
265	 */
266
267store_dr:
268	movq	%dr7,%rax			/* yes, do the save */
269	movq	%dr0,%r15
270	movq	%dr1,%r14
271	movq	%dr2,%r13
272	movq	%dr3,%r12
273	movq	%dr6,%r11
274	movq	%r15,PCB_DR0(%r8)
275	movq	%r14,PCB_DR1(%r8)
276	movq	%r13,PCB_DR2(%r8)
277	movq	%r12,PCB_DR3(%r8)
278	movq	%r11,PCB_DR6(%r8)
279	movq	%rax,PCB_DR7(%r8)
280	andq	$0x0000fc00, %rax		/* disable all watchpoints */
281	movq	%rax,%dr7
282	jmp	done_store_dr
283
284load_dr:
285	movq	%dr7,%rax
286	movq	PCB_DR0(%r8),%r15
287	movq	PCB_DR1(%r8),%r14
288	movq	PCB_DR2(%r8),%r13
289	movq	PCB_DR3(%r8),%r12
290	movq	PCB_DR6(%r8),%r11
291	movq	PCB_DR7(%r8),%rcx
292	movq	%r15,%dr0
293	movq	%r14,%dr1
294	/* Preserve reserved bits in %dr7 */
295	andq	$0x0000fc00,%rax
296	andq	$~0x0000fc00,%rcx
297	movq	%r13,%dr2
298	movq	%r12,%dr3
299	orq	%rcx,%rax
300	movq	%r11,%dr6
301	movq	%rax,%dr7
302	jmp	done_load_dr
303
304do_tss:	movq	%rdx,PCPU(TSSP)
305	movq	%rdx,%rcx
306	movq	PCPU(TSS),%rax
307	movw	%cx,2(%rax)
308	shrq	$16,%rcx
309	movb	%cl,4(%rax)
310	shrq	$8,%rcx
311	movb	%cl,7(%rax)
312	shrq	$8,%rcx
313	movl	%ecx,8(%rax)
314	movb	$0x89,5(%rax)	/* unset busy */
315	movl	$TSSSEL,%eax
316	ltr	%ax
317	jmp	done_tss
318
319do_ldt:	movq	PCPU(LDT),%rax
320	movq	P_MD+MD_LDT_SD(%rcx),%rdx
321	movq	%rdx,(%rax)
322	movq	P_MD+MD_LDT_SD+8(%rcx),%rdx
323	movq	%rdx,8(%rax)
324	movl	$LDTSEL,%eax
325	jmp	ld_ldt
326END(cpu_switch)
327
328/*
329 * savectx(pcb)
330 * Update pcb, saving current processor state.
331 */
332ENTRY(savectx)
333	/* Save caller's return address. */
334	movq	(%rsp),%rax
335	movq	%rax,PCB_RIP(%rdi)
336
337	movq	%rbx,PCB_RBX(%rdi)
338	movq	%rsp,PCB_RSP(%rdi)
339	movq	%rbp,PCB_RBP(%rdi)
340	movq	%r12,PCB_R12(%rdi)
341	movq	%r13,PCB_R13(%rdi)
342	movq	%r14,PCB_R14(%rdi)
343	movq	%r15,PCB_R15(%rdi)
344
345	movq	%cr0,%rax
346	movq	%rax,PCB_CR0(%rdi)
347	movq	%cr2,%rax
348	movq	%rax,PCB_CR2(%rdi)
349	movq	%cr3,%rax
350	movq	%rax,PCB_CR3(%rdi)
351	movq	%cr4,%rax
352	movq	%rax,PCB_CR4(%rdi)
353
354	movq	%dr0,%rax
355	movq	%rax,PCB_DR0(%rdi)
356	movq	%dr1,%rax
357	movq	%rax,PCB_DR1(%rdi)
358	movq	%dr2,%rax
359	movq	%rax,PCB_DR2(%rdi)
360	movq	%dr3,%rax
361	movq	%rax,PCB_DR3(%rdi)
362	movq	%dr6,%rax
363	movq	%rax,PCB_DR6(%rdi)
364	movq	%dr7,%rax
365	movq	%rax,PCB_DR7(%rdi)
366
367	movl	$MSR_FSBASE,%ecx
368	rdmsr
369	movl	%eax,PCB_FSBASE(%rdi)
370	movl	%edx,PCB_FSBASE+4(%rdi)
371	movl	$MSR_GSBASE,%ecx
372	rdmsr
373	movl	%eax,PCB_GSBASE(%rdi)
374	movl	%edx,PCB_GSBASE+4(%rdi)
375	movl	$MSR_KGSBASE,%ecx
376	rdmsr
377	movl	%eax,PCB_KGSBASE(%rdi)
378	movl	%edx,PCB_KGSBASE+4(%rdi)
379	movl	$MSR_EFER,%ecx
380	rdmsr
381	movl	%eax,PCB_EFER(%rdi)
382	movl	%edx,PCB_EFER+4(%rdi)
383	movl	$MSR_STAR,%ecx
384	rdmsr
385	movl	%eax,PCB_STAR(%rdi)
386	movl	%edx,PCB_STAR+4(%rdi)
387	movl	$MSR_LSTAR,%ecx
388	rdmsr
389	movl	%eax,PCB_LSTAR(%rdi)
390	movl	%edx,PCB_LSTAR+4(%rdi)
391	movl	$MSR_CSTAR,%ecx
392	rdmsr
393	movl	%eax,PCB_CSTAR(%rdi)
394	movl	%edx,PCB_CSTAR+4(%rdi)
395	movl	$MSR_SF_MASK,%ecx
396	rdmsr
397	movl	%eax,PCB_SFMASK(%rdi)
398	movl	%edx,PCB_SFMASK+4(%rdi)
399
400	sgdt	PCB_GDT(%rdi)
401	sidt	PCB_IDT(%rdi)
402	sldt	PCB_LDT(%rdi)
403	str	PCB_TR(%rdi)
404
405	movl	$1,%eax
406	ret
407END(savectx)
408
409/*
410 * resumectx(pcb)
411 * Resuming processor state from pcb.
412 */
413ENTRY(resumectx)
414	/* Switch to KPML4phys. */
415	movq	KPML4phys,%rax
416	movq	%rax,%cr3
417
418	/* Force kernel segment registers. */
419	movl	$KDSEL,%eax
420	movw	%ax,%ds
421	movw	%ax,%es
422	movw	%ax,%ss
423	movl	$KUF32SEL,%eax
424	movw	%ax,%fs
425	movl	$KUG32SEL,%eax
426	movw	%ax,%gs
427
428	movl	$MSR_FSBASE,%ecx
429	movl	PCB_FSBASE(%rdi),%eax
430	movl	4 + PCB_FSBASE(%rdi),%edx
431	wrmsr
432	movl	$MSR_GSBASE,%ecx
433	movl	PCB_GSBASE(%rdi),%eax
434	movl	4 + PCB_GSBASE(%rdi),%edx
435	wrmsr
436	movl	$MSR_KGSBASE,%ecx
437	movl	PCB_KGSBASE(%rdi),%eax
438	movl	4 + PCB_KGSBASE(%rdi),%edx
439	wrmsr
440
441	/* Restore EFER. */
442	movl	$MSR_EFER,%ecx
443	movl	PCB_EFER(%rdi),%eax
444	wrmsr
445
446	/* Restore fast syscall stuff. */
447	movl	$MSR_STAR,%ecx
448	movl	PCB_STAR(%rdi),%eax
449	movl	4 + PCB_STAR(%rdi),%edx
450	wrmsr
451	movl	$MSR_LSTAR,%ecx
452	movl	PCB_LSTAR(%rdi),%eax
453	movl	4 + PCB_LSTAR(%rdi),%edx
454	wrmsr
455	movl	$MSR_CSTAR,%ecx
456	movl	PCB_CSTAR(%rdi),%eax
457	movl	4 + PCB_CSTAR(%rdi),%edx
458	wrmsr
459	movl	$MSR_SF_MASK,%ecx
460	movl	PCB_SFMASK(%rdi),%eax
461	wrmsr
462
463	/* Restore CR0, CR2, CR4 and CR3. */
464	movq	PCB_CR0(%rdi),%rax
465	movq	%rax,%cr0
466	movq	PCB_CR2(%rdi),%rax
467	movq	%rax,%cr2
468	movq	PCB_CR4(%rdi),%rax
469	movq	%rax,%cr4
470	movq	PCB_CR3(%rdi),%rax
471	movq	%rax,%cr3
472
473	/* Restore descriptor tables. */
474	lidt	PCB_IDT(%rdi)
475	lldt	PCB_LDT(%rdi)
476
477#define	SDT_SYSTSS	9
478#define	SDT_SYSBSY	11
479
480	/* Clear "task busy" bit and reload TR. */
481	movq	PCPU(TSS),%rax
482	andb	$(~SDT_SYSBSY | SDT_SYSTSS),5(%rax)
483	movw	PCB_TR(%rdi),%ax
484	ltr	%ax
485
486#undef	SDT_SYSTSS
487#undef	SDT_SYSBSY
488
489	/* Restore debug registers. */
490	movq	PCB_DR0(%rdi),%rax
491	movq	%rax,%dr0
492	movq	PCB_DR1(%rdi),%rax
493	movq	%rax,%dr1
494	movq	PCB_DR2(%rdi),%rax
495	movq	%rax,%dr2
496	movq	PCB_DR3(%rdi),%rax
497	movq	%rax,%dr3
498	movq	PCB_DR6(%rdi),%rax
499	movq	%rax,%dr6
500	movq	PCB_DR7(%rdi),%rax
501	movq	%rax,%dr7
502
503	/* Restore other callee saved registers. */
504	movq	PCB_R15(%rdi),%r15
505	movq	PCB_R14(%rdi),%r14
506	movq	PCB_R13(%rdi),%r13
507	movq	PCB_R12(%rdi),%r12
508	movq	PCB_RBP(%rdi),%rbp
509	movq	PCB_RSP(%rdi),%rsp
510	movq	PCB_RBX(%rdi),%rbx
511
512	/* Restore return address. */
513	movq	PCB_RIP(%rdi),%rax
514	movq	%rax,(%rsp)
515
516	xorl	%eax,%eax
517	ret
518END(resumectx)
519