cpu_switch.S revision 58764 
1/*-
2 * Copyright (c) 1990 The Regents of the University of California.
3 * All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * William Jolitz.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 *    must display the following acknowledgement:
18 *	This product includes software developed by the University of
19 *	California, Berkeley and its contributors.
20 * 4. Neither the name of the University nor the names of its contributors
21 *    may be used to endorse or promote products derived from this software
22 *    without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 * $FreeBSD: head/sys/amd64/amd64/cpu_switch.S 58764 2000-03-29 06:15:43Z dillon $
37 */
38
39#include "npx.h"
40#include "opt_user_ldt.h"
41
42#include <sys/rtprio.h>
43
44#include <machine/asmacros.h>
45
46#ifdef SMP
47#include <machine/pmap.h>
48#include <machine/apic.h>
49#include <machine/smptests.h>		/** GRAB_LOPRIO */
50#include <machine/ipl.h>
51#include <machine/lock.h>
52#endif /* SMP */
53
54#include "assym.s"
55
56
57/*****************************************************************************/
58/* Scheduling                                                                */
59/*****************************************************************************/
60
61	.data
62
63	.globl	_hlt_vector
64_hlt_vector:	.long	_default_halt	/* pointer to halt routine */
65
66	.globl	_panic
67
68#if defined(SWTCH_OPTIM_STATS)
69	.globl	_swtch_optim_stats, _tlb_flush_count
70_swtch_optim_stats:	.long	0		/* number of _swtch_optims */
71_tlb_flush_count:	.long	0
72#endif
73
74	.text
75
76/*
77 * When no processes are on the runq, cpu_switch() branches to _idle
78 * to wait for something to come ready.
79 */
80	ALIGN_TEXT
81	.type	_idle,@function
82_idle:
83	xorl	%ebp,%ebp
84	movl	%ebp,_switchtime
85
86#ifdef SMP
87
88	/* when called, we have the mplock, intr disabled */
89	/* use our idleproc's "context" */
90	movl	_IdlePTD, %ecx
91	movl	%cr3, %eax
92	cmpl	%ecx, %eax
93	je		2f
94#if defined(SWTCH_OPTIM_STATS)
95	decl	_swtch_optim_stats
96	incl	_tlb_flush_count
97#endif
98	movl	%ecx, %cr3
992:
100	/* Keep space for nonexisting return addr, or profiling bombs */
101	movl	$gd_idlestack_top-4, %ecx
102	addl	%fs:0, %ecx
103	movl	%ecx, %esp
104
105	/* update common_tss.tss_esp0 pointer */
106	movl	%ecx, _common_tss + TSS_ESP0
107
108	movl	_cpuid, %esi
109	btrl	%esi, _private_tss
110	jae	1f
111
112	movl	$gd_common_tssd, %edi
113	addl	%fs:0, %edi
114
115	/* move correct tss descriptor into GDT slot, then reload tr */
116	movl	_tss_gdt, %ebx			/* entry in GDT */
117	movl	0(%edi), %eax
118	movl	%eax, 0(%ebx)
119	movl	4(%edi), %eax
120	movl	%eax, 4(%ebx)
121	movl	$GPROC0_SEL*8, %esi		/* GSEL(entry, SEL_KPL) */
122	ltr	%si
1231:
124
125	sti
126
127	/*
128	 * XXX callers of cpu_switch() do a bogus splclock().  Locking should
129	 * be left to cpu_switch().
130	 *
131	 * NOTE: spl*() may only be called while we hold the MP lock (which
132	 * we do).
133	 */
134	call	_spl0
135
136	cli
137
138	/*
139	 * _REALLY_ free the lock, no matter how deep the prior nesting.
140	 * We will recover the nesting on the way out when we have a new
141	 * proc to load.
142	 *
143	 * XXX: we had damn well better be sure we had it before doing this!
144	 */
145	movl	$FREE_LOCK, %eax
146	movl	%eax, _mp_lock
147
148	/* do NOT have lock, intrs disabled */
149	.globl	idle_loop
150idle_loop:
151
152	cmpl	$0,_smp_active
153	jne	1f
154	cmpl	$0,_cpuid
155	je	1f
156	jmp	2f
157
1581:
159	call	_procrunnable
160	testl	%eax,%eax
161	jnz	3f
162
163	/*
164	 * Handle page-zeroing in the idle loop.  Called with interrupts
165	 * disabled and the MP lock released.  Inside vm_page_zero_idle
166	 * we enable interrupts and grab the mplock as required.
167	 */
168	cmpl	$0,_do_page_zero_idle
169	je	2f
170
171	call	_vm_page_zero_idle		/* internal locking */
172	testl	%eax, %eax
173	jnz	idle_loop
1742:
175
176	/* enable intrs for a halt */
177	movl	$0, lapic_tpr			/* 1st candidate for an INT */
178	call	*_hlt_vector			/* wait for interrupt */
179	cli
180	jmp	idle_loop
181
182	/*
183	 * Note that interrupts must be enabled while obtaining the MP lock
184	 * in order to be able to take IPI's while blocked.
185	 */
1863:
187	movl	$LOPRIO_LEVEL, lapic_tpr	/* arbitrate for INTs */
188	sti
189	call	_get_mplock
190	cli
191	call	_procrunnable
192	testl	%eax,%eax
193	CROSSJUMP(jnz, sw1a, jz)
194	call	_rel_mplock
195	jmp	idle_loop
196
197#else /* !SMP */
198
199	movl	$HIDENAME(tmpstk),%esp
200#if defined(OVERLY_CONSERVATIVE_PTD_MGMT)
201#if defined(SWTCH_OPTIM_STATS)
202	incl	_swtch_optim_stats
203#endif
204	movl	_IdlePTD, %ecx
205	movl	%cr3, %eax
206	cmpl	%ecx, %eax
207	je		2f
208#if defined(SWTCH_OPTIM_STATS)
209	decl	_swtch_optim_stats
210	incl	_tlb_flush_count
211#endif
212	movl	%ecx, %cr3
2132:
214#endif
215
216	/* update common_tss.tss_esp0 pointer */
217	movl	%esp, _common_tss + TSS_ESP0
218
219	movl	$0, %esi
220	btrl	%esi, _private_tss
221	jae	1f
222
223	movl	$_common_tssd, %edi
224
225	/* move correct tss descriptor into GDT slot, then reload tr */
226	movl	_tss_gdt, %ebx			/* entry in GDT */
227	movl	0(%edi), %eax
228	movl	%eax, 0(%ebx)
229	movl	4(%edi), %eax
230	movl	%eax, 4(%ebx)
231	movl	$GPROC0_SEL*8, %esi		/* GSEL(entry, SEL_KPL) */
232	ltr	%si
2331:
234
235	sti
236
237	/*
238	 * XXX callers of cpu_switch() do a bogus splclock().  Locking should
239	 * be left to cpu_switch().
240	 */
241	call	_spl0
242
243	ALIGN_TEXT
244idle_loop:
245	cli
246	call	_procrunnable
247	testl	%eax,%eax
248	CROSSJUMP(jnz, sw1a, jz)
249	call	_vm_page_zero_idle
250	testl	%eax, %eax
251	jnz	idle_loop
252	call	*_hlt_vector			/* wait for interrupt */
253	jmp	idle_loop
254
255#endif /* SMP */
256
257CROSSJUMPTARGET(_idle)
258
259ENTRY(default_halt)
260	sti
261#ifndef SMP
262	hlt					/* XXX:	 until a wakeup IPI */
263#endif
264	ret
265
266/*
267 * cpu_switch()
268 */
269ENTRY(cpu_switch)
270
271	/* switch to new process. first, save context as needed */
272	movl	_curproc,%ecx
273
274	/* if no process to save, don't bother */
275	testl	%ecx,%ecx
276	je	sw1
277
278#ifdef SMP
279	movb	P_ONCPU(%ecx), %al		/* save "last" cpu */
280	movb	%al, P_LASTCPU(%ecx)
281	movb	$0xff, P_ONCPU(%ecx)		/* "leave" the cpu */
282#endif /* SMP */
283	movl	P_VMSPACE(%ecx), %edx
284#ifdef SMP
285	movl	_cpuid, %eax
286#else
287	xorl	%eax, %eax
288#endif /* SMP */
289	btrl	%eax, VM_PMAP+PM_ACTIVE(%edx)
290
291	movl	P_ADDR(%ecx),%edx
292
293	movl	(%esp),%eax			/* Hardware registers */
294	movl	%eax,PCB_EIP(%edx)
295	movl	%ebx,PCB_EBX(%edx)
296	movl	%esp,PCB_ESP(%edx)
297	movl	%ebp,PCB_EBP(%edx)
298	movl	%esi,PCB_ESI(%edx)
299	movl	%edi,PCB_EDI(%edx)
300	movl	%gs,PCB_GS(%edx)
301
302	/* test if debug regisers should be saved */
303	movb    PCB_FLAGS(%edx),%al
304	andb    $PCB_DBREGS,%al
305	jz      1f                              /* no, skip over */
306	movl    %dr7,%eax                       /* yes, do the save */
307	movl    %eax,PCB_DR7(%edx)
308	andl    $0x0000ff00, %eax               /* disable all watchpoints */
309	movl    %eax,%dr7
310	movl    %dr6,%eax
311	movl    %eax,PCB_DR6(%edx)
312	movl    %dr3,%eax
313	movl    %eax,PCB_DR3(%edx)
314	movl    %dr2,%eax
315	movl    %eax,PCB_DR2(%edx)
316	movl    %dr1,%eax
317	movl    %eax,PCB_DR1(%edx)
318	movl    %dr0,%eax
319	movl    %eax,PCB_DR0(%edx)
3201:
321
322#ifdef SMP
323	movl	_mp_lock, %eax
324	/* XXX FIXME: we should be saving the local APIC TPR */
325#ifdef DIAGNOSTIC
326	cmpl	$FREE_LOCK, %eax		/* is it free? */
327	je	badsw4				/* yes, bad medicine! */
328#endif /* DIAGNOSTIC */
329	andl	$COUNT_FIELD, %eax		/* clear CPU portion */
330	movl	%eax, PCB_MPNEST(%edx)		/* store it */
331#endif /* SMP */
332
333#if NNPX > 0
334	/* have we used fp, and need a save? */
335	cmpl	%ecx,_npxproc
336	jne	1f
337	addl	$PCB_SAVEFPU,%edx		/* h/w bugs make saving complicated */
338	pushl	%edx
339	call	_npxsave			/* do it in a big C function */
340	popl	%eax
3411:
342#endif	/* NNPX > 0 */
343
344	movl	$0,_curproc			/* out of process */
345
346	/* save is done, now choose a new process or idle */
347sw1:
348	cli
349
350#ifdef SMP
351	/* Stop scheduling if smp_active goes zero and we are not BSP */
352	cmpl	$0,_smp_active
353	jne	1f
354	cmpl	$0,_cpuid
355	CROSSJUMP(je, _idle, jne)		/* wind down */
3561:
357#endif
358
359sw1a:
360	call	_chooseproc			/* trash ecx, edx, ret eax*/
361	testl	%eax,%eax
362	CROSSJUMP(je, _idle, jne)		/* if no proc, idle */
363	movl	%eax,%ecx
364
365	xorl	%eax,%eax
366	andl	$~AST_RESCHED,_astpending
367
368#ifdef	DIAGNOSTIC
369	cmpl	%eax,P_WCHAN(%ecx)
370	jne	badsw1
371	cmpb	$SRUN,P_STAT(%ecx)
372	jne	badsw2
373#endif
374
375	movl	P_ADDR(%ecx),%edx
376
377#if defined(SWTCH_OPTIM_STATS)
378	incl	_swtch_optim_stats
379#endif
380	/* switch address space */
381	movl	%cr3,%ebx
382	cmpl	PCB_CR3(%edx),%ebx
383	je	4f
384#if defined(SWTCH_OPTIM_STATS)
385	decl	_swtch_optim_stats
386	incl	_tlb_flush_count
387#endif
388	movl	PCB_CR3(%edx),%ebx
389	movl	%ebx,%cr3
3904:
391
392#ifdef SMP
393	movl	_cpuid, %esi
394#else
395	xorl	%esi, %esi
396#endif
397	cmpl	$0, PCB_EXT(%edx)		/* has pcb extension? */
398	je	1f
399	btsl	%esi, _private_tss		/* mark use of private tss */
400	movl	PCB_EXT(%edx), %edi		/* new tss descriptor */
401	jmp	2f
4021:
403
404	/* update common_tss.tss_esp0 pointer */
405	movl	%edx, %ebx			/* pcb */
406	addl	$(UPAGES * PAGE_SIZE - 16), %ebx
407	movl	%ebx, _common_tss + TSS_ESP0
408
409	btrl	%esi, _private_tss
410	jae	3f
411#ifdef SMP
412	movl	$gd_common_tssd, %edi
413	addl	%fs:0, %edi
414#else
415	movl	$_common_tssd, %edi
416#endif
4172:
418	/* move correct tss descriptor into GDT slot, then reload tr */
419	movl	_tss_gdt, %ebx			/* entry in GDT */
420	movl	0(%edi), %eax
421	movl	%eax, 0(%ebx)
422	movl	4(%edi), %eax
423	movl	%eax, 4(%ebx)
424	movl	$GPROC0_SEL*8, %esi		/* GSEL(entry, SEL_KPL) */
425	ltr	%si
4263:
427	movl	P_VMSPACE(%ecx), %ebx
428#ifdef SMP
429	movl	_cpuid, %eax
430#else
431	xorl	%eax, %eax
432#endif
433	btsl	%eax, VM_PMAP+PM_ACTIVE(%ebx)
434
435	/* restore context */
436	movl	PCB_EBX(%edx),%ebx
437	movl	PCB_ESP(%edx),%esp
438	movl	PCB_EBP(%edx),%ebp
439	movl	PCB_ESI(%edx),%esi
440	movl	PCB_EDI(%edx),%edi
441	movl	PCB_EIP(%edx),%eax
442	movl	%eax,(%esp)
443
444#ifdef SMP
445#ifdef GRAB_LOPRIO				/* hold LOPRIO for INTs */
446#ifdef CHEAP_TPR
447	movl	$0, lapic_tpr
448#else
449	andl	$~APIC_TPR_PRIO, lapic_tpr
450#endif /** CHEAP_TPR */
451#endif /** GRAB_LOPRIO */
452	movl	_cpuid,%eax
453	movb	%al, P_ONCPU(%ecx)
454#endif /* SMP */
455	movl	%edx, _curpcb
456	movl	%ecx, _curproc			/* into next process */
457
458#ifdef SMP
459	movl	_cpu_lockid, %eax
460	orl	PCB_MPNEST(%edx), %eax		/* add next count from PROC */
461	movl	%eax, _mp_lock			/* load the mp_lock */
462	/* XXX FIXME: we should be restoring the local APIC TPR */
463#endif /* SMP */
464
465#ifdef	USER_LDT
466	cmpl	$0, PCB_USERLDT(%edx)
467	jnz	1f
468	movl	__default_ldt,%eax
469	cmpl	_currentldt,%eax
470	je	2f
471	lldt	__default_ldt
472	movl	%eax,_currentldt
473	jmp	2f
4741:	pushl	%edx
475	call	_set_user_ldt
476	popl	%edx
4772:
478#endif
479
480	/* This must be done after loading the user LDT. */
481	.globl	cpu_switch_load_gs
482cpu_switch_load_gs:
483	movl	PCB_GS(%edx),%gs
484
485	/* test if debug regisers should be restored */
486	movb    PCB_FLAGS(%edx),%al
487	andb    $PCB_DBREGS,%al
488	jz      1f                              /* no, skip over */
489	movl    PCB_DR6(%edx),%eax              /* yes, do the restore */
490	movl    %eax,%dr6
491	movl    PCB_DR3(%edx),%eax
492	movl    %eax,%dr3
493	movl    PCB_DR2(%edx),%eax
494	movl    %eax,%dr2
495	movl    PCB_DR1(%edx),%eax
496	movl    %eax,%dr1
497	movl    PCB_DR0(%edx),%eax
498	movl    %eax,%dr0
499	movl    PCB_DR7(%edx),%eax
500	movl    %eax,%dr7
5011:
502
503	sti
504	ret
505
506CROSSJUMPTARGET(sw1a)
507
508#ifdef DIAGNOSTIC
509badsw1:
510	pushl	$sw0_1
511	call	_panic
512
513sw0_1:	.asciz	"cpu_switch: has wchan"
514
515badsw2:
516	pushl	$sw0_2
517	call	_panic
518
519sw0_2:	.asciz	"cpu_switch: not SRUN"
520#endif
521
522#if defined(SMP) && defined(DIAGNOSTIC)
523badsw4:
524	pushl	$sw0_4
525	call	_panic
526
527sw0_4:	.asciz	"cpu_switch: do not have lock"
528#endif /* SMP && DIAGNOSTIC */
529
530/*
531 * savectx(pcb)
532 * Update pcb, saving current processor state.
533 */
534ENTRY(savectx)
535	/* fetch PCB */
536	movl	4(%esp),%ecx
537
538	/* caller's return address - child won't execute this routine */
539	movl	(%esp),%eax
540	movl	%eax,PCB_EIP(%ecx)
541
542	movl	%ebx,PCB_EBX(%ecx)
543	movl	%esp,PCB_ESP(%ecx)
544	movl	%ebp,PCB_EBP(%ecx)
545	movl	%esi,PCB_ESI(%ecx)
546	movl	%edi,PCB_EDI(%ecx)
547	movl	%gs,PCB_GS(%ecx)
548
549#if NNPX > 0
550	/*
551	 * If npxproc == NULL, then the npx h/w state is irrelevant and the
552	 * state had better already be in the pcb.  This is true for forks
553	 * but not for dumps (the old book-keeping with FP flags in the pcb
554	 * always lost for dumps because the dump pcb has 0 flags).
555	 *
556	 * If npxproc != NULL, then we have to save the npx h/w state to
557	 * npxproc's pcb and copy it to the requested pcb, or save to the
558	 * requested pcb and reload.  Copying is easier because we would
559	 * have to handle h/w bugs for reloading.  We used to lose the
560	 * parent's npx state for forks by forgetting to reload.
561	 */
562	movl	_npxproc,%eax
563	testl	%eax,%eax
564	je	1f
565
566	pushl	%ecx
567	movl	P_ADDR(%eax),%eax
568	leal	PCB_SAVEFPU(%eax),%eax
569	pushl	%eax
570	pushl	%eax
571	call	_npxsave
572	addl	$4,%esp
573	popl	%eax
574	popl	%ecx
575
576	pushl	$PCB_SAVEFPU_SIZE
577	leal	PCB_SAVEFPU(%ecx),%ecx
578	pushl	%ecx
579	pushl	%eax
580	call	_bcopy
581	addl	$12,%esp
582#endif	/* NNPX > 0 */
583
5841:
585	ret
586