cpu_switch.S revision 27133
1/*- 2 * Copyright (c) 1990 The Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * William Jolitz. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * $Id: swtch.s,v 1.53 1997/06/22 16:03:35 peter Exp $ 37 */ 38 39#include "npx.h" 40#include "opt_user_ldt.h" 41 42#include <sys/rtprio.h> 43 44#include <machine/asmacros.h> 45#include <machine/ipl.h> 46#include <machine/smptests.h> /** TEST_LOPRIO */ 47 48#if defined(SMP) 49#include <machine/pmap.h> 50#include <machine/apic.h> 51#endif 52 53#include "assym.s" 54 55 56/*****************************************************************************/ 57/* Scheduling */ 58/*****************************************************************************/ 59 60/* 61 * The following primitives manipulate the run queues. 62 * _whichqs tells which of the 32 queues _qs 63 * have processes in them. setrunqueue puts processes into queues, Remrq 64 * removes them from queues. The running process is on no queue, 65 * other processes are on a queue related to p->p_priority, divided by 4 66 * actually to shrink the 0-127 range of priorities into the 32 available 67 * queues. 68 */ 69 .data 70#ifndef SMP 71 .globl _curpcb 72_curpcb: .long 0 /* pointer to curproc's PCB area */ 73#endif 74 .globl _whichqs, _whichrtqs, _whichidqs 75 76_whichqs: .long 0 /* which run queues have data */ 77_whichrtqs: .long 0 /* which realtime run queues have data */ 78_whichidqs: .long 0 /* which idletime run queues have data */ 79 .globl _hlt_vector 80_hlt_vector: .long _default_halt /* pointer to halt routine */ 81 82 83 .globl _qs,_cnt,_panic 84 85 .globl _want_resched 86_want_resched: .long 0 /* we need to re-run the scheduler */ 87 88 .text 89/* 90 * setrunqueue(p) 91 * 92 * Call should be made at spl6(), and p->p_stat should be SRUN 93 */ 94ENTRY(setrunqueue) 95 movl 4(%esp),%eax 96#ifdef DIAGNOSTIC 97 cmpb $SRUN,P_STAT(%eax) 98 je set1 99 pushl $set2 100 call _panic 101set1: 102#endif 103 cmpw $RTP_PRIO_NORMAL,P_RTPRIO_TYPE(%eax) /* normal priority process? */ 104 je set_nort 105 106 movzwl P_RTPRIO_PRIO(%eax),%edx 107 108 cmpw $RTP_PRIO_REALTIME,P_RTPRIO_TYPE(%eax) /* realtime priority? */ 109 jne set_id /* must be idle priority */ 110 111set_rt: 112 btsl %edx,_whichrtqs /* set q full bit */ 113 shll $3,%edx 114 addl $_rtqs,%edx /* locate q hdr */ 115 movl %edx,P_FORW(%eax) /* link process on tail of q */ 116 movl P_BACK(%edx),%ecx 117 movl %ecx,P_BACK(%eax) 118 movl %eax,P_BACK(%edx) 119 movl %eax,P_FORW(%ecx) 120 ret 121 122set_id: 123 btsl %edx,_whichidqs /* set q full bit */ 124 shll $3,%edx 125 addl $_idqs,%edx /* locate q hdr */ 126 movl %edx,P_FORW(%eax) /* link process on tail of q */ 127 movl P_BACK(%edx),%ecx 128 movl %ecx,P_BACK(%eax) 129 movl %eax,P_BACK(%edx) 130 movl %eax,P_FORW(%ecx) 131 ret 132 133set_nort: /* Normal (RTOFF) code */ 134 movzbl P_PRI(%eax),%edx 135 shrl $2,%edx 136 btsl %edx,_whichqs /* set q full bit */ 137 shll $3,%edx 138 addl $_qs,%edx /* locate q hdr */ 139 movl %edx,P_FORW(%eax) /* link process on tail of q */ 140 movl P_BACK(%edx),%ecx 141 movl %ecx,P_BACK(%eax) 142 movl %eax,P_BACK(%edx) 143 movl %eax,P_FORW(%ecx) 144 ret 145 146set2: .asciz "setrunqueue" 147 148/* 149 * Remrq(p) 150 * 151 * Call should be made at spl6(). 152 */ 153ENTRY(remrq) 154 movl 4(%esp),%eax 155 cmpw $RTP_PRIO_NORMAL,P_RTPRIO_TYPE(%eax) /* normal priority process? */ 156 je rem_nort 157 158 movzwl P_RTPRIO_PRIO(%eax),%edx 159 160 cmpw $RTP_PRIO_REALTIME,P_RTPRIO_TYPE(%eax) /* normal priority process? */ 161 jne rem_id 162 163 btrl %edx,_whichrtqs /* clear full bit, panic if clear already */ 164 jb rem1rt 165 pushl $rem3rt 166 call _panic 167rem1rt: 168 pushl %edx 169 movl P_FORW(%eax),%ecx /* unlink process */ 170 movl P_BACK(%eax),%edx 171 movl %edx,P_BACK(%ecx) 172 movl P_BACK(%eax),%ecx 173 movl P_FORW(%eax),%edx 174 movl %edx,P_FORW(%ecx) 175 popl %edx 176 movl $_rtqs,%ecx 177 shll $3,%edx 178 addl %edx,%ecx 179 cmpl P_FORW(%ecx),%ecx /* q still has something? */ 180 je rem2rt 181 shrl $3,%edx /* yes, set bit as still full */ 182 btsl %edx,_whichrtqs 183rem2rt: 184 ret 185rem_id: 186 btrl %edx,_whichidqs /* clear full bit, panic if clear already */ 187 jb rem1id 188 pushl $rem3id 189 call _panic 190rem1id: 191 pushl %edx 192 movl P_FORW(%eax),%ecx /* unlink process */ 193 movl P_BACK(%eax),%edx 194 movl %edx,P_BACK(%ecx) 195 movl P_BACK(%eax),%ecx 196 movl P_FORW(%eax),%edx 197 movl %edx,P_FORW(%ecx) 198 popl %edx 199 movl $_idqs,%ecx 200 shll $3,%edx 201 addl %edx,%ecx 202 cmpl P_FORW(%ecx),%ecx /* q still has something? */ 203 je rem2id 204 shrl $3,%edx /* yes, set bit as still full */ 205 btsl %edx,_whichidqs 206rem2id: 207 ret 208 209rem_nort: 210 movzbl P_PRI(%eax),%edx 211 shrl $2,%edx 212 btrl %edx,_whichqs /* clear full bit, panic if clear already */ 213 jb rem1 214 pushl $rem3 215 call _panic 216rem1: 217 pushl %edx 218 movl P_FORW(%eax),%ecx /* unlink process */ 219 movl P_BACK(%eax),%edx 220 movl %edx,P_BACK(%ecx) 221 movl P_BACK(%eax),%ecx 222 movl P_FORW(%eax),%edx 223 movl %edx,P_FORW(%ecx) 224 popl %edx 225 movl $_qs,%ecx 226 shll $3,%edx 227 addl %edx,%ecx 228 cmpl P_FORW(%ecx),%ecx /* q still has something? */ 229 je rem2 230 shrl $3,%edx /* yes, set bit as still full */ 231 btsl %edx,_whichqs 232rem2: 233 ret 234 235rem3: .asciz "remrq" 236rem3rt: .asciz "remrq.rt" 237rem3id: .asciz "remrq.id" 238 239/* 240 * When no processes are on the runq, cpu_switch() branches to _idle 241 * to wait for something to come ready. 242 * 243 * NOTE: on an SMP system this routine is a startup-only code path. 244 * once initialization is over, meaning the idle procs have been 245 * created, we should NEVER branch here. 246 */ 247 ALIGN_TEXT 248_idle: 249#ifdef SMP 250 movl _smp_active, %eax 251 cmpl $0, %eax 252 jnz badsw3 253#endif /* SMP */ 254 xorl %ebp,%ebp 255 movl $HIDENAME(tmpstk),%esp 256 movl _IdlePTD,%ecx 257 movl %ecx,%cr3 258 259 /* update common_tss.tss_esp0 pointer */ 260 movl $_common_tss, %eax 261 movl %esp, TSS_ESP0(%eax) 262 263#ifdef TSS_IS_CACHED /* example only */ 264 /* Reload task register to force reload of selector */ 265 movl _tssptr, %ebx 266 andb $~0x02, 5(%ebx) /* Flip 386BSY -> 386TSS */ 267 movl _gsel_tss, %ebx 268 ltr %bx 269#endif 270 271 sti 272 273 /* 274 * XXX callers of cpu_switch() do a bogus splclock(). Locking should 275 * be left to cpu_switch(). 276 */ 277 call _spl0 278 279 ALIGN_TEXT 280idle_loop: 281 cli 282 cmpl $0,_whichrtqs /* real-time queue */ 283 CROSSJUMP(jne, sw1a, je) 284 cmpl $0,_whichqs /* normal queue */ 285 CROSSJUMP(jne, nortqr, je) 286 cmpl $0,_whichidqs /* 'idle' queue */ 287 CROSSJUMP(jne, idqr, je) 288 call _vm_page_zero_idle 289 testl %eax, %eax 290 jnz idle_loop 291 sti 292 call *_hlt_vector /* wait for interrupt */ 293 jmp idle_loop 294 295CROSSJUMPTARGET(_idle) 296 297ENTRY(default_halt) 298 hlt 299 ret 300 301/* 302 * cpu_switch() 303 */ 304ENTRY(cpu_switch) 305 306 /* switch to new process. first, save context as needed */ 307 movl _curproc,%ecx 308 309 /* if no process to save, don't bother */ 310 testl %ecx,%ecx 311 je sw1 312 313#ifdef SMP 314 movb P_ONCPU(%ecx), %al /* save "last" cpu */ 315 movb %al, P_LASTCPU(%ecx) 316 movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ 317#endif 318 319 movl P_ADDR(%ecx),%ecx 320 321 movl (%esp),%eax /* Hardware registers */ 322 movl %eax,PCB_EIP(%ecx) 323 movl %ebx,PCB_EBX(%ecx) 324 movl %esp,PCB_ESP(%ecx) 325 movl %ebp,PCB_EBP(%ecx) 326 movl %esi,PCB_ESI(%ecx) 327 movl %edi,PCB_EDI(%ecx) 328 movl %fs,PCB_FS(%ecx) 329 movl %gs,PCB_GS(%ecx) 330 331#ifdef SMP 332 movl _mp_lock, %eax 333 cmpl $0xffffffff, %eax /* is it free? */ 334 je badsw4 /* yes, bad medicine! */ 335 andl $0x00ffffff, %eax /* clear CPU portion */ 336 movl %eax,PCB_MPNEST(%ecx) /* store it */ 337#endif /* SMP */ 338 339#if NNPX > 0 340 /* have we used fp, and need a save? */ 341 movl _curproc,%eax 342 cmpl %eax,_npxproc 343 jne 1f 344 addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ 345 pushl %ecx 346 call _npxsave /* do it in a big C function */ 347 popl %eax 3481: 349#endif /* NNPX > 0 */ 350 351 movl $0,_curproc /* out of process */ 352 353 /* save is done, now choose a new process or idle */ 354sw1: 355 cli 356sw1a: 357 movl _whichrtqs,%edi /* pick next p. from rtqs */ 358 testl %edi,%edi 359 jz nortqr /* no realtime procs */ 360 361 /* XXX - bsf is sloow */ 362 bsfl %edi,%ebx /* find a full q */ 363 jz nortqr /* no proc on rt q - try normal ... */ 364 365 /* XX update whichqs? */ 366 btrl %ebx,%edi /* clear q full status */ 367 leal _rtqs(,%ebx,8),%eax /* select q */ 368 movl %eax,%esi 369 370 movl P_FORW(%eax),%ecx /* unlink from front of process q */ 371 movl P_FORW(%ecx),%edx 372 movl %edx,P_FORW(%eax) 373 movl P_BACK(%ecx),%eax 374 movl %eax,P_BACK(%edx) 375 376 cmpl P_FORW(%ecx),%esi /* q empty */ 377 je rt3 378 btsl %ebx,%edi /* nope, set to indicate not empty */ 379rt3: 380 movl %edi,_whichrtqs /* update q status */ 381 jmp swtch_com 382 383 /* old sw1a */ 384/* Normal process priority's */ 385nortqr: 386 movl _whichqs,%edi 3872: 388 /* XXX - bsf is sloow */ 389 bsfl %edi,%ebx /* find a full q */ 390 jz idqr /* if none, idle */ 391 392 /* XX update whichqs? */ 393 btrl %ebx,%edi /* clear q full status */ 394 leal _qs(,%ebx,8),%eax /* select q */ 395 movl %eax,%esi 396 397 movl P_FORW(%eax),%ecx /* unlink from front of process q */ 398 movl P_FORW(%ecx),%edx 399 movl %edx,P_FORW(%eax) 400 movl P_BACK(%ecx),%eax 401 movl %eax,P_BACK(%edx) 402 403 cmpl P_FORW(%ecx),%esi /* q empty */ 404 je 3f 405 btsl %ebx,%edi /* nope, set to indicate not empty */ 4063: 407 movl %edi,_whichqs /* update q status */ 408 jmp swtch_com 409 410idqr: /* was sw1a */ 411 movl _whichidqs,%edi /* pick next p. from idqs */ 412 413 /* XXX - bsf is sloow */ 414 bsfl %edi,%ebx /* find a full q */ 415 CROSSJUMP(je, _idle, jne) /* if no proc, idle */ 416 417 /* XX update whichqs? */ 418 btrl %ebx,%edi /* clear q full status */ 419 leal _idqs(,%ebx,8),%eax /* select q */ 420 movl %eax,%esi 421 422 movl P_FORW(%eax),%ecx /* unlink from front of process q */ 423 movl P_FORW(%ecx),%edx 424 movl %edx,P_FORW(%eax) 425 movl P_BACK(%ecx),%eax 426 movl %eax,P_BACK(%edx) 427 428 cmpl P_FORW(%ecx),%esi /* q empty */ 429 je id3 430 btsl %ebx,%edi /* nope, set to indicate not empty */ 431id3: 432 movl %edi,_whichidqs /* update q status */ 433 434swtch_com: 435 movl $0,%eax 436 movl %eax,_want_resched 437 438#ifdef DIAGNOSTIC 439 cmpl %eax,P_WCHAN(%ecx) 440 jne badsw1 441 cmpb $SRUN,P_STAT(%ecx) 442 jne badsw2 443#endif 444 445 movl %eax,P_BACK(%ecx) /* isolate process to run */ 446 movl P_ADDR(%ecx),%edx 447 movl PCB_CR3(%edx),%ebx 448 449#if defined(SMP) 450 /* Grab the private PT pointer from the outgoing process's PTD */ 451 movl $_PTD,%esi 452 movl 4*MPPTDI(%esi), %eax /* fetch cpu's prv pt */ 453#endif 454 455 /* switch address space */ 456 movl %ebx,%cr3 457 458#if defined(SMP) 459 /* Copy the private PT to the new process's PTD */ 460 /* XXX yuck, the _PTD changes when we switch, so we have to 461 * reload %cr3 after changing the address space. 462 * We need to fix this by storing a pointer to the virtual 463 * location of the per-process PTD in the PCB or something quick. 464 * Dereferencing proc->vm_map->pmap->p_pdir[] is painful in asm. 465 */ 466 movl %eax, 4*MPPTDI(%esi) /* restore cpu's prv page */ 467 468 /* XXX: we have just changed the page tables.. reload.. */ 469 movl %ebx,%cr3 470#endif 471 472#ifdef HOW_TO_SWITCH_TSS /* example only */ 473 /* Fix up tss pointer to floating pcb/stack structure */ 474 /* XXX probably lots faster to store the 64 bits of tss entry 475 * in the pcb somewhere and copy them on activation. 476 */ 477 movl _tssptr, %ebx 478 movl %edx, %eax /* edx = pcb/tss */ 479 movw %ax, 2(%ebx) /* store bits 0->15 */ 480 roll $16, %eax /* swap upper and lower */ 481 movb %al, 4(%ebx) /* store bits 16->23 */ 482 movb %ah, 7(%ebx) /* store bits 24->31 */ 483 andb $~0x02, 5(%ebx) /* Flip 386BSY -> 386TSS */ 484#endif 485 486 /* update common_tss.tss_esp0 pointer */ 487 movl $_common_tss, %eax 488 movl %edx, %ebx /* pcb */ 489 addl $(UPAGES * PAGE_SIZE), %ebx 490 movl %ebx, TSS_ESP0(%eax) 491 492#ifdef TSS_IS_CACHED /* example only */ 493 /* Reload task register to force reload of selector */ 494 movl _tssptr, %ebx 495 andb $~0x02, 5(%ebx) /* Flip 386BSY -> 386TSS */ 496 movl _gsel_tss, %ebx 497 ltr %bx 498#endif 499 500 /* restore context */ 501 movl PCB_EBX(%edx),%ebx 502 movl PCB_ESP(%edx),%esp 503 movl PCB_EBP(%edx),%ebp 504 movl PCB_ESI(%edx),%esi 505 movl PCB_EDI(%edx),%edi 506 movl PCB_EIP(%edx),%eax 507 movl %eax,(%esp) 508 509#ifdef SMP 510 movl _cpuid,%eax 511 movb %al, P_ONCPU(%ecx) 512#endif 513 movl %edx,_curpcb 514 movl %ecx,_curproc /* into next process */ 515 516#ifdef SMP 517#if defined(TEST_LOPRIO) 518 /* Set us to prefer to get irq's from the apic since we have the lock */ 519 movl lapic_tpr, %eax /* get TPR register contents */ 520 andl $0xffffff00, %eax /* clear the prio field */ 521 movl %eax, lapic_tpr /* now hold loprio for INTs */ 522#endif /* TEST_LOPRIO */ 523 movl _cpu_lockid,%eax 524 orl PCB_MPNEST(%edx), %eax /* add next count from PROC */ 525 movl %eax, _mp_lock /* load the mp_lock */ 526#endif /* SMP */ 527 528#ifdef USER_LDT 529 cmpl $0, PCB_USERLDT(%edx) 530 jnz 1f 531 movl __default_ldt,%eax 532 cmpl _currentldt,%eax 533 je 2f 534 lldt __default_ldt 535 movl %eax,_currentldt 536 jmp 2f 5371: pushl %edx 538 call _set_user_ldt 539 popl %edx 5402: 541#endif 542 543 /* This must be done after loading the user LDT. */ 544 .globl cpu_switch_load_fs 545cpu_switch_load_fs: 546 movl PCB_FS(%edx),%fs 547 .globl cpu_switch_load_gs 548cpu_switch_load_gs: 549 movl PCB_GS(%edx),%gs 550 551 sti 552 ret 553 554CROSSJUMPTARGET(idqr) 555CROSSJUMPTARGET(nortqr) 556CROSSJUMPTARGET(sw1a) 557 558#ifdef DIAGNOSTIC 559badsw1: 560 pushl $sw0_1 561 call _panic 562 563sw0_1: .asciz "cpu_switch: has wchan" 564 565badsw2: 566 pushl $sw0_2 567 call _panic 568 569sw0_2: .asciz "cpu_switch: not SRUN" 570#endif 571 572#ifdef SMP 573badsw3: 574 pushl $sw0_3 575 call _panic 576 577sw0_3: .asciz "cpu_switch: went idle with smp_active" 578 579badsw4: 580 pushl $sw0_4 581 call _panic 582 583sw0_4: .asciz "cpu_switch: do not have lock" 584#endif 585 586/* 587 * savectx(pcb) 588 * Update pcb, saving current processor state. 589 */ 590ENTRY(savectx) 591 /* fetch PCB */ 592 movl 4(%esp),%ecx 593 594 /* caller's return address - child won't execute this routine */ 595 movl (%esp),%eax 596 movl %eax,PCB_EIP(%ecx) 597 598 movl %ebx,PCB_EBX(%ecx) 599 movl %esp,PCB_ESP(%ecx) 600 movl %ebp,PCB_EBP(%ecx) 601 movl %esi,PCB_ESI(%ecx) 602 movl %edi,PCB_EDI(%ecx) 603 movl %fs,PCB_FS(%ecx) 604 movl %gs,PCB_GS(%ecx) 605 606#if NNPX > 0 607 /* 608 * If npxproc == NULL, then the npx h/w state is irrelevant and the 609 * state had better already be in the pcb. This is true for forks 610 * but not for dumps (the old book-keeping with FP flags in the pcb 611 * always lost for dumps because the dump pcb has 0 flags). 612 * 613 * If npxproc != NULL, then we have to save the npx h/w state to 614 * npxproc's pcb and copy it to the requested pcb, or save to the 615 * requested pcb and reload. Copying is easier because we would 616 * have to handle h/w bugs for reloading. We used to lose the 617 * parent's npx state for forks by forgetting to reload. 618 */ 619 movl _npxproc,%eax 620 testl %eax,%eax 621 je 1f 622 623 pushl %ecx 624 movl P_ADDR(%eax),%eax 625 leal PCB_SAVEFPU(%eax),%eax 626 pushl %eax 627 pushl %eax 628 call _npxsave 629 addl $4,%esp 630 popl %eax 631 popl %ecx 632 633 pushl $PCB_SAVEFPU_SIZE 634 leal PCB_SAVEFPU(%ecx),%ecx 635 pushl %ecx 636 pushl %eax 637 call _bcopy 638 addl $12,%esp 639#endif /* NNPX > 0 */ 640 6411: 642 ret 643