cpu_switch.S revision 170368
1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to Berkeley by 7 * William Jolitz. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * $FreeBSD: head/sys/amd64/amd64/cpu_switch.S 170368 2007-06-06 07:35:08Z davidxu $ 34 */ 35 36#include <machine/asmacros.h> 37#include <machine/specialreg.h> 38 39#include "assym.s" 40 41/*****************************************************************************/ 42/* Scheduling */ 43/*****************************************************************************/ 44 45 .text 46 47#ifdef SMP 48#define LK lock ; 49#else 50#define LK 51#endif 52 53/* 54 * cpu_throw() 55 * 56 * This is the second half of cpu_switch(). It is used when the current 57 * thread is either a dummy or slated to die, and we no longer care 58 * about its state. This is only a slight optimization and is probably 59 * not worth it anymore. Note that we need to clear the pm_active bits so 60 * we do need the old proc if it still exists. 61 * %rdi = oldtd 62 * %rsi = newtd 63 */ 64ENTRY(cpu_throw) 65 movl PCPU(CPUID), %eax 66 testq %rdi,%rdi /* no thread? */ 67 jz 1f 68 /* release bit from old pm_active */ 69 movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */ 70 movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */ 71 LK btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */ 721: 73 movq TD_PCB(%rsi),%rdx /* newtd->td_proc */ 74 movq PCB_CR3(%rdx),%rdx 75 movq %rdx,%cr3 /* new address space */ 76 jmp swact 77 78/* 79 * cpu_switch(old, new, mtx) 80 * 81 * Save the current thread state, then select the next thread to run 82 * and load its state. 83 * %rdi = oldtd 84 * %rsi = newtd 85 * %rdx = mtx 86 */ 87ENTRY(cpu_switch) 88 /* Switch to new thread. First, save context. */ 89 movq TD_PCB(%rdi),%r8 90 91 movq (%rsp),%rax /* Hardware registers */ 92 movq %rax,PCB_RIP(%r8) 93 movq %rbx,PCB_RBX(%r8) 94 movq %rsp,PCB_RSP(%r8) 95 movq %rbp,PCB_RBP(%r8) 96 movq %r12,PCB_R12(%r8) 97 movq %r13,PCB_R13(%r8) 98 movq %r14,PCB_R14(%r8) 99 movq %r15,PCB_R15(%r8) 100 101 testl $PCB_32BIT,PCB_FLAGS(%r8) 102 jz 1f /* no, skip over */ 103 104 /* Save userland %gs */ 105 movl %gs,PCB_GS(%r8) 106 movq PCB_GS32P(%r8),%rax 107 movq (%rax),%rax 108 movq %rax,PCB_GS32SD(%r8) 109 1101: 111 /* Test if debug registers should be saved. */ 112 testl $PCB_DBREGS,PCB_FLAGS(%r8) 113 jz 1f /* no, skip over */ 114 movq %dr7,%rax /* yes, do the save */ 115 movq %rax,PCB_DR7(%r8) 116 andq $0x0000fc00, %rax /* disable all watchpoints */ 117 movq %rax,%dr7 118 movq %dr6,%rax 119 movq %rax,PCB_DR6(%r8) 120 movq %dr3,%rax 121 movq %rax,PCB_DR3(%r8) 122 movq %dr2,%rax 123 movq %rax,PCB_DR2(%r8) 124 movq %dr1,%rax 125 movq %rax,PCB_DR1(%r8) 126 movq %dr0,%rax 127 movq %rax,PCB_DR0(%r8) 1281: 129 130 /* have we used fp, and need a save? */ 131 cmpq %rdi,PCPU(FPCURTHREAD) 132 jne 1f 133 addq $PCB_SAVEFPU,%r8 134 clts 135 fxsave (%r8) 136 smsw %ax 137 orb $CR0_TS,%al 138 lmsw %ax 139 xorl %eax,%eax 140 movq %rax,PCPU(FPCURTHREAD) 1411: 142 143 /* Save is done. Now fire up new thread. Leave old vmspace. */ 144 movq TD_PCB(%rsi),%r8 145 146 /* switch address space */ 147 movq PCB_CR3(%r8),%rcx 148 movq %cr3,%rax 149 cmpq %rcx,%rax /* Same address space? */ 150 jne swinact 151 movq %rdx, TD_LOCK(%rdi) /* Release the old thread */ 152 /* Wait for the new thread to become unblocked */ 153 movq $blocked_lock, %rdx 1541: 155 movq TD_LOCK(%rsi),%rcx 156 cmpq %rcx, %rdx 157 je 1b 158 jmp sw1 159swinact: 160 movq %rcx,%cr3 /* new address space */ 161 movl PCPU(CPUID), %eax 162 /* Release bit from old pmap->pm_active */ 163 movq TD_PROC(%rdi), %rcx /* oldproc */ 164 movq P_VMSPACE(%rcx), %rcx 165 LK btrl %eax, VM_PMAP+PM_ACTIVE(%rcx) /* clear old */ 166 movq %rdx, TD_LOCK(%rdi) /* Release the old thread */ 167swact: 168 /* Wait for the new thread to become unblocked */ 169 movq $blocked_lock, %rdx 1701: 171 movq TD_LOCK(%rsi),%rcx 172 cmpq %rcx, %rdx 173 je 1b 174 175 /* Set bit in new pmap->pm_active */ 176 movq TD_PROC(%rsi),%rdx /* newproc */ 177 movq P_VMSPACE(%rdx), %rdx 178 LK btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */ 179 180sw1: 181 /* 182 * At this point, we've switched address spaces and are ready 183 * to load up the rest of the next context. 184 */ 185 movq TD_PCB(%rsi),%r8 186 187 /* Restore userland %fs */ 188 movl $MSR_FSBASE,%ecx 189 movl PCB_FSBASE(%r8),%eax 190 movl PCB_FSBASE+4(%r8),%edx 191 wrmsr 192 193 /* Restore userland %gs */ 194 movl $MSR_KGSBASE,%ecx 195 movl PCB_GSBASE(%r8),%eax 196 movl PCB_GSBASE+4(%r8),%edx 197 wrmsr 198 199 /* Update the TSS_RSP0 pointer for the next interrupt */ 200 movq PCPU(TSSP), %rax 201 addq $COMMON_TSS_RSP0, %rax 202 leaq -16(%r8), %rbx 203 movq %rbx, (%rax) 204 movq %rbx, PCPU(RSP0) 205 206 movq %r8, PCPU(CURPCB) 207 movq %rsi, PCPU(CURTHREAD) /* into next thread */ 208 209 testl $PCB_32BIT,PCB_FLAGS(%r8) 210 jz 1f /* no, skip over */ 211 212 /* Restore userland %gs while preserving kernel gsbase */ 213 movq PCB_GS32P(%r8),%rax 214 movq PCB_GS32SD(%r8),%rbx 215 movq %rbx,(%rax) 216 movl $MSR_GSBASE,%ecx 217 rdmsr 218 movl PCB_GS(%r8),%gs 219 wrmsr 220 2211: 222 /* Restore context. */ 223 movq PCB_RBX(%r8),%rbx 224 movq PCB_RSP(%r8),%rsp 225 movq PCB_RBP(%r8),%rbp 226 movq PCB_R12(%r8),%r12 227 movq PCB_R13(%r8),%r13 228 movq PCB_R14(%r8),%r14 229 movq PCB_R15(%r8),%r15 230 movq PCB_RIP(%r8),%rax 231 movq %rax,(%rsp) 232 233 /* Test if debug registers should be restored. */ 234 testl $PCB_DBREGS,PCB_FLAGS(%r8) 235 jz 1f 236 movq PCB_DR6(%r8),%rax 237 movq %rax,%dr6 238 movq PCB_DR3(%r8),%rax 239 movq %rax,%dr3 240 movq PCB_DR2(%r8),%rax 241 movq %rax,%dr2 242 movq PCB_DR1(%r8),%rax 243 movq %rax,%dr1 244 movq PCB_DR0(%r8),%rax 245 movq %rax,%dr0 246 /* But preserve reserved bits in %dr7 */ 247 movq %dr7,%rax 248 andq $0x0000fc00,%rax 249 movq PCB_DR7(%r8),%rcx 250 andq $~0x0000fc00,%rcx 251 orq %rcx,%rax 252 movq %rax,%dr7 2531: 254 ret 255 256/* 257 * savectx(pcb) 258 * Update pcb, saving current processor state. 259 */ 260ENTRY(savectx) 261 /* Fetch PCB. */ 262 movq %rdi,%rcx 263 264 /* Save caller's return address. */ 265 movq (%rsp),%rax 266 movq %rax,PCB_RIP(%rcx) 267 268 movq %cr3,%rax 269 movq %rax,PCB_CR3(%rcx) 270 271 movq %rbx,PCB_RBX(%rcx) 272 movq %rsp,PCB_RSP(%rcx) 273 movq %rbp,PCB_RBP(%rcx) 274 movq %r12,PCB_R12(%rcx) 275 movq %r13,PCB_R13(%rcx) 276 movq %r14,PCB_R14(%rcx) 277 movq %r15,PCB_R15(%rcx) 278 279 /* 280 * If fpcurthread == NULL, then the fpu h/w state is irrelevant and the 281 * state had better already be in the pcb. This is true for forks 282 * but not for dumps (the old book-keeping with FP flags in the pcb 283 * always lost for dumps because the dump pcb has 0 flags). 284 * 285 * If fpcurthread != NULL, then we have to save the fpu h/w state to 286 * fpcurthread's pcb and copy it to the requested pcb, or save to the 287 * requested pcb and reload. Copying is easier because we would 288 * have to handle h/w bugs for reloading. We used to lose the 289 * parent's fpu state for forks by forgetting to reload. 290 */ 291 pushfq 292 cli 293 movq PCPU(FPCURTHREAD),%rax 294 testq %rax,%rax 295 je 1f 296 297 movq TD_PCB(%rax),%rdi 298 leaq PCB_SAVEFPU(%rdi),%rdi 299 clts 300 fxsave (%rdi) 301 smsw %ax 302 orb $CR0_TS,%al 303 lmsw %ax 304 305 movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */ 306 leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */ 307 /* arg 1 (%rdi) already loaded */ 308 call bcopy 3091: 310 popfq 311 312 ret 313