exception.S revision 129653
1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 4. Neither the name of the University nor the names of its contributors 15 * may be used to endorse or promote products derived from this software 16 * without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * $FreeBSD: head/sys/amd64/amd64/exception.S 129653 2004-05-24 12:08:56Z bde $ 31 */ 32 33#include "opt_atpic.h" 34#include "opt_ia32.h" 35 36#include <machine/asmacros.h> 37#include <machine/psl.h> 38#include <machine/trap.h> 39 40#include "assym.s" 41 42 .text 43 44/*****************************************************************************/ 45/* Trap handling */ 46/*****************************************************************************/ 47/* 48 * Trap and fault vector routines. 49 * 50 * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 51 * state on the stack but also disables interrupts. This is important for 52 * us for the use of the swapgs instruction. We cannot be interrupted 53 * until the GS.base value is correct. For most traps, we automatically 54 * then enable interrupts if the interrupted context had them enabled. 55 * This is equivalent to the i386 port's use of SDT_SYS386TGT. 56 * 57 * The cpu will push a certain amount of state onto the kernel stack for 58 * the current process. See amd64/include/frame.h. 59 * This includes the current RFLAGS (status register, which includes 60 * the interrupt disable state prior to the trap), the code segment register, 61 * and the return instruction pointer are pushed by the cpu. The cpu 62 * will also push an 'error' code for certain traps. We push a dummy 63 * error code for those traps where the cpu doesn't in order to maintain 64 * a consistent frame. We also push a contrived 'trap number'. 65 * 66 * The cpu does not push the general registers, we must do that, and we 67 * must restore them prior to calling 'iret'. The cpu adjusts the %cs and 68 * %ss segment registers, but does not mess with %ds, %es, or %fs. Thus we 69 * must load them with appropriate values for supervisor mode operation. 70 */ 71 72MCOUNT_LABEL(user) 73MCOUNT_LABEL(btrap) 74 75/* Traps that we leave interrupts disabled for.. */ 76#define TRAP_NOEN(a) \ 77 subq $TF_RIP,%rsp; \ 78 movq $(a),TF_TRAPNO(%rsp) ; \ 79 movq $0,TF_ADDR(%rsp) ; \ 80 movq $0,TF_ERR(%rsp) ; \ 81 jmp alltraps_noen 82IDTVEC(dbg) 83 TRAP_NOEN(T_TRCTRAP) 84IDTVEC(bpt) 85 TRAP_NOEN(T_BPTFLT) 86 87/* Regular traps; The cpu does not supply tf_err for these. */ 88#define TRAP(a) \ 89 subq $TF_RIP,%rsp; \ 90 movq $(a),TF_TRAPNO(%rsp) ; \ 91 movq $0,TF_ADDR(%rsp) ; \ 92 movq $0,TF_ERR(%rsp) ; \ 93 jmp alltraps 94IDTVEC(div) 95 TRAP(T_DIVIDE) 96IDTVEC(nmi) 97 TRAP(T_NMI) 98IDTVEC(ofl) 99 TRAP(T_OFLOW) 100IDTVEC(bnd) 101 TRAP(T_BOUND) 102IDTVEC(ill) 103 TRAP(T_PRIVINFLT) 104IDTVEC(dna) 105 TRAP(T_DNA) 106IDTVEC(fpusegm) 107 TRAP(T_FPOPFLT) 108IDTVEC(mchk) 109 TRAP(T_MCHK) 110IDTVEC(rsvd) 111 TRAP(T_RESERVED) 112IDTVEC(fpu) 113 TRAP(T_ARITHTRAP) 114IDTVEC(xmm) 115 TRAP(T_XMMFLT) 116 117/* This group of traps have tf_err already pushed by the cpu */ 118#define TRAP_ERR(a) \ 119 subq $TF_ERR,%rsp; \ 120 movq $(a),TF_TRAPNO(%rsp) ; \ 121 movq $0,TF_ADDR(%rsp) ; \ 122 jmp alltraps_noen 123IDTVEC(tss) 124 TRAP_ERR(T_TSSFLT) 125IDTVEC(missing) 126 TRAP_ERR(T_SEGNPFLT) 127IDTVEC(stk) 128 TRAP_ERR(T_STKFLT) 129IDTVEC(prot) 130 TRAP_ERR(T_PROTFLT) 131IDTVEC(align) 132 TRAP_ERR(T_ALIGNFLT) 133 134 /* 135 * alltraps entry point. Use swapgs if this is the first time in the 136 * kernel from userland. Reenable interrupts if they were enabled 137 * before the trap. This approximates SDT_SYS386TGT on the i386 port. 138 */ 139 140 SUPERALIGN_TEXT 141 .globl alltraps 142 .type alltraps,@function 143alltraps: 144 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 145 jz alltraps_testi /* already running with kernel GS.base */ 146 swapgs 147alltraps_testi: 148 testl $PSL_I,TF_RFLAGS(%rsp) 149 jz alltraps_pushregs 150 sti 151alltraps_pushregs: 152 movq %rdi,TF_RDI(%rsp) 153alltraps_pushregs_no_rdi: 154 movq %rsi,TF_RSI(%rsp) 155 movq %rdx,TF_RDX(%rsp) 156 movq %rcx,TF_RCX(%rsp) 157 movq %r8,TF_R8(%rsp) 158 movq %r9,TF_R9(%rsp) 159 movq %rax,TF_RAX(%rsp) 160 movq %rbx,TF_RBX(%rsp) 161 movq %rbp,TF_RBP(%rsp) 162 movq %r10,TF_R10(%rsp) 163 movq %r11,TF_R11(%rsp) 164 movq %r12,TF_R12(%rsp) 165 movq %r13,TF_R13(%rsp) 166 movq %r14,TF_R14(%rsp) 167 movq %r15,TF_R15(%rsp) 168alltraps_with_regs_pushed: 169 FAKE_MCOUNT(TF_RIP(%rsp)) 170calltrap: 171 call trap 172 MEXITCOUNT 173 jmp doreti /* Handle any pending ASTs */ 174 175 /* 176 * alltraps_noen entry point. Unlike alltraps above, we want to 177 * leave the interrupts disabled. This corresponds to 178 * SDT_SYS386IGT on the i386 port. 179 */ 180 SUPERALIGN_TEXT 181 .globl alltraps_noen 182 .type alltraps_noen,@function 183alltraps_noen: 184 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 185 jz alltraps_pushregs /* already running with kernel GS.base */ 186 swapgs 187 jmp alltraps_pushregs 188 189IDTVEC(dblfault) 190 subq $TF_ERR,%rsp 191 movq $T_DOUBLEFLT,TF_TRAPNO(%rsp) 192 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 193 jz 1f /* already running with kernel GS.base */ 194 swapgs 1951: call dblfault_handler 1962: hlt 197 jmp 2b 198 199IDTVEC(page) 200 subq $TF_ERR,%rsp 201 movq $T_PAGEFLT,TF_TRAPNO(%rsp) 202 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 203 jz 1f /* already running with kernel GS.base */ 204 swapgs 2051: movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 206 movq %cr2,%rdi /* preserve %cr2 before .. */ 207 movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 208 testl $PSL_I,TF_RFLAGS(%rsp) 209 jz alltraps_pushregs_no_rdi 210 sti 211 jmp alltraps_pushregs_no_rdi 212 213/* 214 * Fast syscall entry point. We enter here with just our new %cs/%ss set, 215 * and the new privilige level. We are still running on the old user stack 216 * pointer. We have to juggle a few things around to find our stack etc. 217 * swapgs gives us access to our PCPU space only. 218 */ 219IDTVEC(fast_syscall) 220 swapgs 221 movq %rsp,PCPU(SCRATCH_RSP) 222 movq PCPU(RSP0),%rsp 223 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 224 subq $TF_SIZE,%rsp 225 /* defer TF_RSP till we have a spare register */ 226 movq %r11,TF_RFLAGS(%rsp) 227 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 228 movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 229 movq %r11,TF_RSP(%rsp) /* user stack pointer */ 230 sti 231 movq $KUDSEL,TF_SS(%rsp) 232 movq $KUCSEL,TF_CS(%rsp) 233 movq $2,TF_ERR(%rsp) 234 movq %rdi,TF_RDI(%rsp) /* arg 1 */ 235 movq %rsi,TF_RSI(%rsp) /* arg 2 */ 236 movq %rdx,TF_RDX(%rsp) /* arg 3 */ 237 movq %r10,TF_RCX(%rsp) /* arg 4 */ 238 movq %r8,TF_R8(%rsp) /* arg 5 */ 239 movq %r9,TF_R9(%rsp) /* arg 6 */ 240 movq %rax,TF_RAX(%rsp) /* syscall number */ 241 movq %rbx,TF_RBX(%rsp) /* C preserved */ 242 movq %rbp,TF_RBP(%rsp) /* C preserved */ 243 movq %r12,TF_R12(%rsp) /* C preserved */ 244 movq %r13,TF_R13(%rsp) /* C preserved */ 245 movq %r14,TF_R14(%rsp) /* C preserved */ 246 movq %r15,TF_R15(%rsp) /* C preserved */ 247 FAKE_MCOUNT(TF_RIP(%rsp)) 248 call syscall 249 movq PCPU(CURPCB),%rax 250 testq $PCB_FULLCTX,PCB_FLAGS(%rax) 251 jne 3f 2521: /* Check for and handle AST's on return to userland */ 253 cli 254 movq PCPU(CURTHREAD),%rax 255 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 256 je 2f 257 sti 258 movq %rsp, %rdi 259 call ast 260 jmp 1b 2612: /* restore preserved registers */ 262 MEXITCOUNT 263 movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ 264 movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ 265 movq TF_RDX(%rsp),%rdx /* return value 2 */ 266 movq TF_RAX(%rsp),%rax /* return value 1 */ 267 movq TF_RBX(%rsp),%rbx /* C preserved */ 268 movq TF_RBP(%rsp),%rbp /* C preserved */ 269 movq TF_R12(%rsp),%r12 /* C preserved */ 270 movq TF_R13(%rsp),%r13 /* C preserved */ 271 movq TF_R14(%rsp),%r14 /* C preserved */ 272 movq TF_R15(%rsp),%r15 /* C preserved */ 273 movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ 274 movq TF_RIP(%rsp),%rcx /* original %rip */ 275 movq TF_RSP(%rsp),%r9 /* user stack pointer */ 276 movq %r9,%rsp /* original %rsp */ 277 swapgs 278 sysretq 2793: /* Requested full context restore, use doreti for that */ 280 andq $~PCB_FULLCTX,PCB_FLAGS(%rax) 281 MEXITCOUNT 282 jmp doreti 283 284/* 285 * Here for CYA insurance, in case a "syscall" instruction gets 286 * issued from 32 bit compatability mode. MSR_CSTAR has to point 287 * to *something* if EFER_SCE is enabled. 288 */ 289IDTVEC(fast_syscall32) 290 sysret 291 292ENTRY(fork_trampoline) 293 movq %r12, %rdi /* function */ 294 movq %rbx, %rsi /* arg1 */ 295 movq %rsp, %rdx /* trapframe pointer */ 296 call fork_exit 297 MEXITCOUNT 298 jmp doreti /* Handle any ASTs */ 299 300/* 301 * To efficiently implement classification of trap and interrupt handlers 302 * for profiling, there must be only trap handlers between the labels btrap 303 * and bintr, and only interrupt handlers between the labels bintr and 304 * eintr. This is implemented (partly) by including files that contain 305 * some of the handlers. Before including the files, set up a normal asm 306 * environment so that the included files doen't need to know that they are 307 * included. 308 */ 309 310#ifdef IA32 311 .data 312 .align 4 313 .text 314 SUPERALIGN_TEXT 315 316#include <amd64/ia32/ia32_exception.S> 317#endif 318 319 .data 320 .align 4 321 .text 322 SUPERALIGN_TEXT 323MCOUNT_LABEL(bintr) 324 325#include <amd64/amd64/apic_vector.S> 326 327#ifdef DEV_ATPIC 328 .data 329 .align 4 330 .text 331 SUPERALIGN_TEXT 332 333#include <amd64/isa/atpic_vector.S> 334#endif 335 336 .text 337MCOUNT_LABEL(eintr) 338 339/* 340 * void doreti(struct trapframe) 341 * 342 * Handle return from interrupts, traps and syscalls. 343 */ 344 .text 345 SUPERALIGN_TEXT 346 .type doreti,@function 347doreti: 348 FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 349 /* 350 * Check if ASTs can be handled now. 351 */ 352 testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 353 jz doreti_exit /* can't handle ASTs now if not */ 354 355doreti_ast: 356 /* 357 * Check for ASTs atomically with returning. Disabling CPU 358 * interrupts provides sufficient locking eve in the SMP case, 359 * since we will be informed of any new ASTs by an IPI. 360 */ 361 cli 362 movq PCPU(CURTHREAD),%rax 363 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 364 je doreti_exit 365 sti 366 movq %rsp, %rdi /* pass a pointer to the trapframe */ 367 call ast 368 jmp doreti_ast 369 370 /* 371 * doreti_exit: pop registers, iret. 372 * 373 * The segment register pop is a special case, since it may 374 * fault if (for example) a sigreturn specifies bad segment 375 * registers. The fault is handled in trap.c. 376 */ 377doreti_exit: 378 MEXITCOUNT 379 movq TF_RDI(%rsp),%rdi 380 movq TF_RSI(%rsp),%rsi 381 movq TF_RDX(%rsp),%rdx 382 movq TF_RCX(%rsp),%rcx 383 movq TF_R8(%rsp),%r8 384 movq TF_R9(%rsp),%r9 385 movq TF_RAX(%rsp),%rax 386 movq TF_RBX(%rsp),%rbx 387 movq TF_RBP(%rsp),%rbp 388 movq TF_R10(%rsp),%r10 389 movq TF_R11(%rsp),%r11 390 movq TF_R12(%rsp),%r12 391 movq TF_R13(%rsp),%r13 392 movq TF_R14(%rsp),%r14 393 movq TF_R15(%rsp),%r15 394 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 395 jz 1f /* keep running with kernel GS.base */ 396 cli 397 swapgs 3981: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 399 .globl doreti_iret 400doreti_iret: 401 iretq 402 403 /* 404 * doreti_iret_fault and friends. Alternative return code for 405 * the case where we get a fault in the doreti_exit code 406 * above. trap() (i386/i386/trap.c) catches this specific 407 * case, sends the process a signal and continues in the 408 * corresponding place in the code below. 409 */ 410 ALIGN_TEXT 411 .globl doreti_iret_fault 412doreti_iret_fault: 413 subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 414 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 415 jz 1f /* already running with kernel GS.base */ 416 swapgs 4171: testl $PSL_I,TF_RFLAGS(%rsp) 418 jz 2f 419 sti 4202: movq %rdi,TF_RDI(%rsp) 421 movq %rsi,TF_RSI(%rsp) 422 movq %rdx,TF_RDX(%rsp) 423 movq %rcx,TF_RCX(%rsp) 424 movq %r8,TF_R8(%rsp) 425 movq %r9,TF_R9(%rsp) 426 movq %rax,TF_RAX(%rsp) 427 movq %rbx,TF_RBX(%rsp) 428 movq %rbp,TF_RBP(%rsp) 429 movq %r10,TF_R10(%rsp) 430 movq %r11,TF_R11(%rsp) 431 movq %r12,TF_R12(%rsp) 432 movq %r13,TF_R13(%rsp) 433 movq %r14,TF_R14(%rsp) 434 movq %r15,TF_R15(%rsp) 435 movq $T_PROTFLT,TF_TRAPNO(%rsp) 436 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 437 jmp alltraps_with_regs_pushed 438