swtch.s revision 55316
1/*- 2 * Copyright (c) 1990 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * William Jolitz. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * $FreeBSD: head/sys/i386/i386/swtch.s 55312 2000-01-02 15:18:12Z phk $ 37 */ 38 39#include "npx.h" 40#include "opt_user_ldt.h" 41 42#include <sys/rtprio.h> 43 44#include <machine/asmacros.h> 45 46#ifdef SMP 47#include <machine/pmap.h> 48#include <machine/apic.h> 49#include <machine/smptests.h> /** GRAB_LOPRIO */ 50#include <machine/ipl.h> 51#include <machine/lock.h> 52#endif /* SMP */ 53 54#include "assym.s" 55 56 57/*****************************************************************************/ 58/* Scheduling */ 59/*****************************************************************************/ 60 61 .data 62 63 .globl _hlt_vector 64_hlt_vector: .long _default_halt /* pointer to halt routine */ 65 66 .globl _panic 67 68 .globl _want_resched 69_want_resched: .long 0 /* we need to re-run the scheduler */ 70#if defined(SWTCH_OPTIM_STATS) 71 .globl _swtch_optim_stats, _tlb_flush_count 72_swtch_optim_stats: .long 0 /* number of _swtch_optims */ 73_tlb_flush_count: .long 0 74#endif 75 76 .text 77 78/* 79 * When no processes are on the runq, cpu_switch() branches to _idle 80 * to wait for something to come ready. 81 */ 82 ALIGN_TEXT 83 .type _idle,@function 84_idle: 85 xorl %ebp,%ebp 86 movl %ebp,_switchtime 87 88#ifdef SMP 89 90 /* when called, we have the mplock, intr disabled */ 91 /* use our idleproc's "context" */ 92 movl _IdlePTD, %ecx 93 movl %cr3, %eax 94 cmpl %ecx, %eax 95 je 2f 96#if defined(SWTCH_OPTIM_STATS) 97 decl _swtch_optim_stats 98 incl _tlb_flush_count 99#endif 100 movl %ecx, %cr3 1012: 102 /* Keep space for nonexisting return addr, or profiling bombs */ 103 movl $gd_idlestack_top-4, %ecx 104 addl %fs:0, %ecx 105 movl %ecx, %esp 106 107 /* update common_tss.tss_esp0 pointer */ 108 movl %ecx, _common_tss + TSS_ESP0 109 110 movl _cpuid, %esi 111 btrl %esi, _private_tss 112 jae 1f 113 114 movl $gd_common_tssd, %edi 115 addl %fs:0, %edi 116 117 /* move correct tss descriptor into GDT slot, then reload tr */ 118 movl _tss_gdt, %ebx /* entry in GDT */ 119 movl 0(%edi), %eax 120 movl %eax, 0(%ebx) 121 movl 4(%edi), %eax 122 movl %eax, 4(%ebx) 123 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 124 ltr %si 1251: 126 127 sti 128 129 /* 130 * XXX callers of cpu_switch() do a bogus splclock(). Locking should 131 * be left to cpu_switch(). 132 */ 133 call _spl0 134 135 cli 136 137 /* 138 * _REALLY_ free the lock, no matter how deep the prior nesting. 139 * We will recover the nesting on the way out when we have a new 140 * proc to load. 141 * 142 * XXX: we had damn well better be sure we had it before doing this! 143 */ 144 movl $FREE_LOCK, %eax 145 movl %eax, _mp_lock 146 147 /* do NOT have lock, intrs disabled */ 148 .globl idle_loop 149idle_loop: 150 151 cmpl $0,_smp_active 152 jne 1f 153 cmpl $0,_cpuid 154 je 1f 155 jmp 2f 156 1571: 158 call _procrunnable 159 testl %eax,%eax 160 jnz 3f 161 162 cmpl $0,_do_page_zero_idle 163 je 2f 164 165 /* XXX appears to cause panics */ 166 /* 167 * Inside zero_idle we enable interrupts and grab the mplock 168 * as needed. It needs to be careful about entry/exit mutexes. 169 */ 170 call _vm_page_zero_idle /* internal locking */ 171 testl %eax, %eax 172 jnz idle_loop 1732: 174 175 /* enable intrs for a halt */ 176 movl $0, lapic_tpr /* 1st candidate for an INT */ 177 call *_hlt_vector /* wait for interrupt */ 178 cli 179 jmp idle_loop 180 1813: 182 movl $LOPRIO_LEVEL, lapic_tpr /* arbitrate for INTs */ 183 call _get_mplock 184 call _procrunnable 185 testl %eax,%eax 186 CROSSJUMP(jnz, sw1a, jz) 187 call _rel_mplock 188 jmp idle_loop 189 190#else /* !SMP */ 191 192 movl $HIDENAME(tmpstk),%esp 193#if defined(OVERLY_CONSERVATIVE_PTD_MGMT) 194#if defined(SWTCH_OPTIM_STATS) 195 incl _swtch_optim_stats 196#endif 197 movl _IdlePTD, %ecx 198 movl %cr3, %eax 199 cmpl %ecx, %eax 200 je 2f 201#if defined(SWTCH_OPTIM_STATS) 202 decl _swtch_optim_stats 203 incl _tlb_flush_count 204#endif 205 movl %ecx, %cr3 2062: 207#endif 208 209 /* update common_tss.tss_esp0 pointer */ 210 movl %esp, _common_tss + TSS_ESP0 211 212 movl $0, %esi 213 btrl %esi, _private_tss 214 jae 1f 215 216 movl $_common_tssd, %edi 217 218 /* move correct tss descriptor into GDT slot, then reload tr */ 219 movl _tss_gdt, %ebx /* entry in GDT */ 220 movl 0(%edi), %eax 221 movl %eax, 0(%ebx) 222 movl 4(%edi), %eax 223 movl %eax, 4(%ebx) 224 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 225 ltr %si 2261: 227 228 sti 229 230 /* 231 * XXX callers of cpu_switch() do a bogus splclock(). Locking should 232 * be left to cpu_switch(). 233 */ 234 call _spl0 235 236 ALIGN_TEXT 237idle_loop: 238 cli 239 call _procrunnable 240 testl %eax,%eax 241 CROSSJUMP(jnz, sw1a, jz) 242 call _vm_page_zero_idle 243 testl %eax, %eax 244 jnz idle_loop 245 call *_hlt_vector /* wait for interrupt */ 246 jmp idle_loop 247 248#endif /* SMP */ 249 250CROSSJUMPTARGET(_idle) 251 252ENTRY(default_halt) 253 sti 254#ifndef SMP 255 hlt /* XXX: until a wakeup IPI */ 256#endif 257 ret 258 259/* 260 * cpu_switch() 261 */ 262ENTRY(cpu_switch) 263 264 /* switch to new process. first, save context as needed */ 265 movl _curproc,%ecx 266 267 /* if no process to save, don't bother */ 268 testl %ecx,%ecx 269 je sw1 270 271#ifdef SMP 272 movb P_ONCPU(%ecx), %al /* save "last" cpu */ 273 movb %al, P_LASTCPU(%ecx) 274 movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ 275#endif /* SMP */ 276 movl P_VMSPACE(%ecx), %edx 277#ifdef SMP 278 movl _cpuid, %eax 279#else 280 xorl %eax, %eax 281#endif /* SMP */ 282 btrl %eax, VM_PMAP+PM_ACTIVE(%edx) 283 284 movl P_ADDR(%ecx),%edx 285 286 movl (%esp),%eax /* Hardware registers */ 287 movl %eax,PCB_EIP(%edx) 288 movl %ebx,PCB_EBX(%edx) 289 movl %esp,PCB_ESP(%edx) 290 movl %ebp,PCB_EBP(%edx) 291 movl %esi,PCB_ESI(%edx) 292 movl %edi,PCB_EDI(%edx) 293 movl %gs,PCB_GS(%edx) 294 295 /* test if debug regisers should be saved */ 296 movb PCB_FLAGS(%edx),%al 297 andb $PCB_DBREGS,%al 298 jz 1f /* no, skip over */ 299 movl %dr7,%eax /* yes, do the save */ 300 movl %eax,PCB_DR7(%edx) 301 andl $0x0000ff00, %eax /* disable all watchpoints */ 302 movl %eax,%dr7 303 movl %dr6,%eax 304 movl %eax,PCB_DR6(%edx) 305 movl %dr3,%eax 306 movl %eax,PCB_DR3(%edx) 307 movl %dr2,%eax 308 movl %eax,PCB_DR2(%edx) 309 movl %dr1,%eax 310 movl %eax,PCB_DR1(%edx) 311 movl %dr0,%eax 312 movl %eax,PCB_DR0(%edx) 3131: 314 315#ifdef SMP 316 movl _mp_lock, %eax 317 /* XXX FIXME: we should be saving the local APIC TPR */ 318#ifdef DIAGNOSTIC 319 cmpl $FREE_LOCK, %eax /* is it free? */ 320 je badsw4 /* yes, bad medicine! */ 321#endif /* DIAGNOSTIC */ 322 andl $COUNT_FIELD, %eax /* clear CPU portion */ 323 movl %eax, PCB_MPNEST(%edx) /* store it */ 324#endif /* SMP */ 325 326#if NNPX > 0 327 /* have we used fp, and need a save? */ 328 cmpl %ecx,_npxproc 329 jne 1f 330 addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ 331 pushl %edx 332 call _npxsave /* do it in a big C function */ 333 popl %eax 3341: 335#endif /* NNPX > 0 */ 336 337 movl $0,_curproc /* out of process */ 338 339 /* save is done, now choose a new process or idle */ 340sw1: 341 cli 342 343#ifdef SMP 344 /* Stop scheduling if smp_active goes zero and we are not BSP */ 345 cmpl $0,_smp_active 346 jne 1f 347 cmpl $0,_cpuid 348 CROSSJUMP(je, _idle, jne) /* wind down */ 3491: 350#endif 351 352sw1a: 353 call _chooseproc /* trash ecx, edx, ret eax*/ 354 testl %eax,%eax 355 CROSSJUMP(je, _idle, jne) /* if no proc, idle */ 356 movl %eax,%ecx 357 358 movl $0,%eax 359 movl %eax,_want_resched 360 361#ifdef DIAGNOSTIC 362 cmpl %eax,P_WCHAN(%ecx) 363 jne badsw1 364 cmpb $SRUN,P_STAT(%ecx) 365 jne badsw2 366#endif 367 368 movl P_ADDR(%ecx),%edx 369 370#if defined(SWTCH_OPTIM_STATS) 371 incl _swtch_optim_stats 372#endif 373 /* switch address space */ 374 movl %cr3,%ebx 375 cmpl PCB_CR3(%edx),%ebx 376 je 4f 377#if defined(SWTCH_OPTIM_STATS) 378 decl _swtch_optim_stats 379 incl _tlb_flush_count 380#endif 381 movl PCB_CR3(%edx),%ebx 382 movl %ebx,%cr3 3834: 384 385#ifdef SMP 386 movl _cpuid, %esi 387#else 388 xorl %esi, %esi 389#endif 390 cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ 391 je 1f 392 btsl %esi, _private_tss /* mark use of private tss */ 393 movl PCB_EXT(%edx), %edi /* new tss descriptor */ 394 jmp 2f 3951: 396 397 /* update common_tss.tss_esp0 pointer */ 398 movl %edx, %ebx /* pcb */ 399 addl $(UPAGES * PAGE_SIZE - 16), %ebx 400 movl %ebx, _common_tss + TSS_ESP0 401 402 btrl %esi, _private_tss 403 jae 3f 404#ifdef SMP 405 movl $gd_common_tssd, %edi 406 addl %fs:0, %edi 407#else 408 movl $_common_tssd, %edi 409#endif 4102: 411 /* move correct tss descriptor into GDT slot, then reload tr */ 412 movl _tss_gdt, %ebx /* entry in GDT */ 413 movl 0(%edi), %eax 414 movl %eax, 0(%ebx) 415 movl 4(%edi), %eax 416 movl %eax, 4(%ebx) 417 movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ 418 ltr %si 4193: 420 movl P_VMSPACE(%ecx), %ebx 421#ifdef SMP 422 movl _cpuid, %eax 423#else 424 xorl %eax, %eax 425#endif 426 btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) 427 428 /* restore context */ 429 movl PCB_EBX(%edx),%ebx 430 movl PCB_ESP(%edx),%esp 431 movl PCB_EBP(%edx),%ebp 432 movl PCB_ESI(%edx),%esi 433 movl PCB_EDI(%edx),%edi 434 movl PCB_EIP(%edx),%eax 435 movl %eax,(%esp) 436 437#ifdef SMP 438#ifdef GRAB_LOPRIO /* hold LOPRIO for INTs */ 439#ifdef CHEAP_TPR 440 movl $0, lapic_tpr 441#else 442 andl $~APIC_TPR_PRIO, lapic_tpr 443#endif /** CHEAP_TPR */ 444#endif /** GRAB_LOPRIO */ 445 movl _cpuid,%eax 446 movb %al, P_ONCPU(%ecx) 447#endif /* SMP */ 448 movl %edx, _curpcb 449 movl %ecx, _curproc /* into next process */ 450 451#ifdef SMP 452 movl _cpu_lockid, %eax 453 orl PCB_MPNEST(%edx), %eax /* add next count from PROC */ 454 movl %eax, _mp_lock /* load the mp_lock */ 455 /* XXX FIXME: we should be restoring the local APIC TPR */ 456#endif /* SMP */ 457 458#ifdef USER_LDT 459 cmpl $0, PCB_USERLDT(%edx) 460 jnz 1f 461 movl __default_ldt,%eax 462 cmpl _currentldt,%eax 463 je 2f 464 lldt __default_ldt 465 movl %eax,_currentldt 466 jmp 2f 4671: pushl %edx 468 call _set_user_ldt 469 popl %edx 4702: 471#endif 472 473 /* This must be done after loading the user LDT. */ 474 .globl cpu_switch_load_gs 475cpu_switch_load_gs: 476 movl PCB_GS(%edx),%gs 477 478 /* test if debug regisers should be restored */ 479 movb PCB_FLAGS(%edx),%al 480 andb $PCB_DBREGS,%al 481 jz 1f /* no, skip over */ 482 movl PCB_DR6(%edx),%eax /* yes, do the restore */ 483 movl %eax,%dr6 484 movl PCB_DR3(%edx),%eax 485 movl %eax,%dr3 486 movl PCB_DR2(%edx),%eax 487 movl %eax,%dr2 488 movl PCB_DR1(%edx),%eax 489 movl %eax,%dr1 490 movl PCB_DR0(%edx),%eax 491 movl %eax,%dr0 492 movl PCB_DR7(%edx),%eax 493 movl %eax,%dr7 4941: 495 496 sti 497 ret 498 499CROSSJUMPTARGET(sw1a) 500 501#ifdef DIAGNOSTIC 502badsw1: 503 pushl $sw0_1 504 call _panic 505 506sw0_1: .asciz "cpu_switch: has wchan" 507 508badsw2: 509 pushl $sw0_2 510 call _panic 511 512sw0_2: .asciz "cpu_switch: not SRUN" 513#endif 514 515#if defined(SMP) && defined(DIAGNOSTIC) 516badsw4: 517 pushl $sw0_4 518 call _panic 519 520sw0_4: .asciz "cpu_switch: do not have lock" 521#endif /* SMP && DIAGNOSTIC */ 522 523/* 524 * savectx(pcb) 525 * Update pcb, saving current processor state. 526 */ 527ENTRY(savectx) 528 /* fetch PCB */ 529 movl 4(%esp),%ecx 530 531 /* caller's return address - child won't execute this routine */ 532 movl (%esp),%eax 533 movl %eax,PCB_EIP(%ecx) 534 535 movl %ebx,PCB_EBX(%ecx) 536 movl %esp,PCB_ESP(%ecx) 537 movl %ebp,PCB_EBP(%ecx) 538 movl %esi,PCB_ESI(%ecx) 539 movl %edi,PCB_EDI(%ecx) 540 movl %gs,PCB_GS(%ecx) 541 542#if NNPX > 0 543 /* 544 * If npxproc == NULL, then the npx h/w state is irrelevant and the 545 * state had better already be in the pcb. This is true for forks 546 * but not for dumps (the old book-keeping with FP flags in the pcb 547 * always lost for dumps because the dump pcb has 0 flags). 548 * 549 * If npxproc != NULL, then we have to save the npx h/w state to 550 * npxproc's pcb and copy it to the requested pcb, or save to the 551 * requested pcb and reload. Copying is easier because we would 552 * have to handle h/w bugs for reloading. We used to lose the 553 * parent's npx state for forks by forgetting to reload. 554 */ 555 movl _npxproc,%eax 556 testl %eax,%eax 557 je 1f 558 559 pushl %ecx 560 movl P_ADDR(%eax),%eax 561 leal PCB_SAVEFPU(%eax),%eax 562 pushl %eax 563 pushl %eax 564 call _npxsave 565 addl $4,%esp 566 popl %eax 567 popl %ecx 568 569 pushl $PCB_SAVEFPU_SIZE 570 leal PCB_SAVEFPU(%ecx),%ecx 571 pushl %ecx 572 pushl %eax 573 call _bcopy 574 addl $12,%esp 575#endif /* NNPX > 0 */ 576 5771: 578 ret 579