cpu_switch.S revision 124850
1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * William Jolitz. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by the University of 20 * California, Berkeley and its contributors. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * $FreeBSD: head/sys/amd64/amd64/cpu_switch.S 124850 2004-01-23 01:04:28Z peter $ 38 */ 39 40#include <machine/asmacros.h> 41 42#include "assym.s" 43 44/*****************************************************************************/ 45/* Scheduling */ 46/*****************************************************************************/ 47 48 .text 49 50/* 51 * cpu_throw() 52 * 53 * This is the second half of cpu_swtch(). It is used when the current 54 * thread is either a dummy or slated to die, and we no longer care 55 * about its state. This is only a slight optimization and is probably 56 * not worth it anymore. Note that we need to clear the pm_active bits so 57 * we do need the old proc if it still exists. 58 * %rdi = oldtd 59 * %rsi = newtd 60 */ 61ENTRY(cpu_throw) 62 movl PCPU(CPUID), %eax 63 testq %rdi,%rdi /* no thread? */ 64 jz 1f 65 /* release bit from old pm_active */ 66 movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ 67 movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ 68#ifdef SMP 69 lock 70#endif 71 btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 721: 73 movq TD_PCB(%rsi),%rdx /* newtd->td_proc */ 74 movq PCB_CR3(%rdx),%rdx 75 movq %rdx,%cr3 /* new address space */ 76 /* set bit in new pm_active */ 77 movq TD_PROC(%rsi),%rdx 78 movq P_VMSPACE(%rdx), %rdx 79#ifdef SMP 80 lock 81#endif 82 btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 83 jmp sw1 84 85/* 86 * cpu_switch(old, new) 87 * 88 * Save the current thread state, then select the next thread to run 89 * and load its state. 90 * %rdi = oldtd 91 * %rsi = newtd 92 */ 93ENTRY(cpu_switch) 94 95 /* Switch to new thread. First, save context. */ 96#ifdef INVARIANTS 97 testq %rdi,%rdi /* no thread? */ 98 jz badsw2 /* no, panic */ 99#endif 100 101 movq TD_PCB(%rdi),%r8 102 103 movq (%rsp),%rax /* Hardware registers */ 104 movq %rax,PCB_RIP(%r8) 105 movq %rbx,PCB_RBX(%r8) 106 movq %rsp,PCB_RSP(%r8) 107 movq %rbp,PCB_RBP(%r8) 108 movq %r12,PCB_R12(%r8) 109 movq %r13,PCB_R13(%r8) 110 movq %r14,PCB_R14(%r8) 111 movq %r15,PCB_R15(%r8) 112 pushfq /* PSL */ 113 popq PCB_RFLAGS(%r8) 114 115 /* Save userland %fs */ 116 movl $MSR_FSBASE,%ecx 117 rdmsr 118 movl %eax,PCB_FSBASE(%r8) 119 movl %edx,PCB_FSBASE+4(%r8) 120 121 /* Save userland %gs */ 122 movl $MSR_KGSBASE,%ecx 123 rdmsr 124 movl %eax,PCB_GSBASE(%r8) 125 movl %edx,PCB_GSBASE+4(%r8) 126 127 /* Save segment selector numbers */ 128 movl %ds,PCB_DS(%r8) 129 movl %es,PCB_ES(%r8) 130 movl %fs,PCB_FS(%r8) 131 movl %gs,PCB_GS(%r8) 132 133 /* have we used fp, and need a save? */ 134 cmpq %rdi,PCPU(FPCURTHREAD) 135 jne 1f 136 pushq %rdi 137 pushq %rsi 138 addq $PCB_SAVEFPU,%r8 /* h/w bugs make saving complicated */ 139 movq %r8, %rdi 140 call fpusave /* do it in a big C function */ 141 popq %rsi 142 popq %rdi 1431: 144 145 /* Save is done. Now fire up new thread. Leave old vmspace. */ 146#ifdef INVARIANTS 147 testq %rsi,%rsi /* no thread? */ 148 jz badsw3 /* no, panic */ 149#endif 150 movq TD_PCB(%rsi),%r8 151 movl PCPU(CPUID), %eax 152 153 /* switch address space */ 154 movq PCB_CR3(%r8),%rdx 155 movq %rdx,%cr3 /* new address space */ 156 157 /* Release bit from old pmap->pm_active */ 158 movq TD_PROC(%rdi), %rdx /* oldproc */ 159 movq P_VMSPACE(%rdx), %rdx 160#ifdef SMP 161 lock 162#endif 163 btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 164 165 /* Set bit in new pmap->pm_active */ 166 movq TD_PROC(%rsi),%rdx /* newproc */ 167 movq P_VMSPACE(%rdx), %rdx 168#ifdef SMP 169 lock 170#endif 171 btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 172 173sw1: 174 /* 175 * At this point, we've switched address spaces and are ready 176 * to load up the rest of the next context. 177 */ 178 movq TD_PCB(%rsi),%r8 179 180 /* Restore segment selector numbers */ 181 movl PCB_DS(%r8),%ds 182 movl PCB_ES(%r8),%es 183 movl PCB_FS(%r8),%fs 184 185 /* Restore userland %gs while preserving kernel gsbase */ 186 movl $MSR_GSBASE,%ecx 187 rdmsr 188 movl PCB_GS(%r8),%gs 189 wrmsr 190 191 /* Restore userland %fs */ 192 movl $MSR_FSBASE,%ecx 193 movl PCB_FSBASE(%r8),%eax 194 movl PCB_FSBASE+4(%r8),%edx 195 wrmsr 196 197 /* Restore userland %gs */ 198 movl $MSR_KGSBASE,%ecx 199 movl PCB_GSBASE(%r8),%eax 200 movl PCB_GSBASE+4(%r8),%edx 201 wrmsr 202 203 /* Update the TSS_RSP0 pointer for the next interrupt */ 204 movq PCPU(TSSP), %rax 205 addq $COMMON_TSS_RSP0, %rax 206 leaq -16(%r8), %rbx 207 movq %rbx, (%rax) 208 movq %rbx, PCPU(RSP0) 209 210 /* Restore context. */ 211 movq PCB_RBX(%r8),%rbx 212 movq PCB_RSP(%r8),%rsp 213 movq PCB_RBP(%r8),%rbp 214 movq PCB_R12(%r8),%r12 215 movq PCB_R13(%r8),%r13 216 movq PCB_R14(%r8),%r14 217 movq PCB_R15(%r8),%r15 218 movq PCB_RIP(%r8),%rax 219 movq %rax,(%rsp) 220 pushq PCB_RFLAGS(%r8) 221 popfq 222 223 movq %r8, PCPU(CURPCB) 224 movq %rsi, PCPU(CURTHREAD) /* into next thread */ 225 226 ret 227 228#ifdef INVARIANTS 229badsw1: 230 pushq %rax 231 pushq %rcx 232 pushq %rdx 233 pushq %rbx 234 pushq %rbp 235 pushq %rsi 236 pushq %rdi 237 pushq %r8 238 pushq %r9 239 pushq %r10 240 pushq %r11 241 pushq %r12 242 pushq %r13 243 pushq %r14 244 pushq %r15 245 pushq $sw0_1 246 call __panic 247sw0_1: .asciz "cpu_throw: no newthread supplied" 248 249badsw2: 250 pushq %rax 251 pushq %rcx 252 pushq %rdx 253 pushq %rbx 254 pushq %rbp 255 pushq %rsi 256 pushq %rdi 257 pushq %r8 258 pushq %r9 259 pushq %r10 260 pushq %r11 261 pushq %r12 262 pushq %r13 263 pushq %r14 264 pushq %r15 265 pushq $sw0_2 266 call __panic 267sw0_2: .asciz "cpu_switch: no curthread supplied" 268 269badsw3: 270 pushq %rax 271 pushq %rcx 272 pushq %rdx 273 pushq %rbx 274 pushq %rbp 275 pushq %rsi 276 pushq %rdi 277 pushq %r8 278 pushq %r9 279 pushq %r10 280 pushq %r11 281 pushq %r12 282 pushq %r13 283 pushq %r14 284 pushq %r15 285 pushq $sw0_3 286 call __panic 287sw0_3: .asciz "cpu_switch: no newthread supplied" 288#endif 289 290noswitch: .asciz "cpu_switch: called!" 291nothrow: .asciz "cpu_throw: called!" 292/* 293 * savectx(pcb) 294 * Update pcb, saving current processor state. 295 */ 296ENTRY(savectx) 297 /* Fetch PCB. */ 298 movq %rdi,%rcx 299 300 /* Save caller's return address. */ 301 movq (%rsp),%rax 302 movq %rax,PCB_RIP(%rcx) 303 304 movq %cr3,%rax 305 movq %rax,PCB_CR3(%rcx) 306 307 movq %rbx,PCB_RBX(%rcx) 308 movq %rsp,PCB_RSP(%rcx) 309 movq %rbp,PCB_RBP(%rcx) 310 movq %r12,PCB_R12(%rcx) 311 movq %r13,PCB_R13(%rcx) 312 movq %r14,PCB_R14(%rcx) 313 movq %r15,PCB_R15(%rcx) 314 pushfq 315 popq PCB_RFLAGS(%rcx) 316 317 /* 318 * If fpcurthread == NULL, then the fpu h/w state is irrelevant and the 319 * state had better already be in the pcb. This is true for forks 320 * but not for dumps (the old book-keeping with FP flags in the pcb 321 * always lost for dumps because the dump pcb has 0 flags). 322 * 323 * If fpcurthread != NULL, then we have to save the fpu h/w state to 324 * fpcurthread's pcb and copy it to the requested pcb, or save to the 325 * requested pcb and reload. Copying is easier because we would 326 * have to handle h/w bugs for reloading. We used to lose the 327 * parent's fpu state for forks by forgetting to reload. 328 */ 329 pushfq 330 cli 331 movq PCPU(FPCURTHREAD),%rax 332 testq %rax,%rax 333 je 1f 334 335 pushq %rcx 336 pushq %rax 337 movq TD_PCB(%rax),%rdi 338 leaq PCB_SAVEFPU(%rdi),%rdi 339 call fpusave 340 popq %rax 341 popq %rcx 342 343 movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */ 344 leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */ 345 movq %rax,%rdi /* arg 1 */ 346 call bcopy 3471: 348 popfq 349 350 ret 351