exception.S revision 287147
1/*- 2 * Copyright (c) 1989, 1990 William F. Jolitz. 3 * Copyright (c) 1990 The Regents of the University of California. 4 * Copyright (c) 2007 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * Portions of this software were developed by A. Joseph Koshy under 8 * sponsorship from the FreeBSD Foundation and Google, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 4. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * $FreeBSD: releng/9.3/sys/amd64/amd64/exception.S 287147 2015-08-25 20:49:05Z delphij $ 35 */ 36 37#include "opt_atpic.h" 38#include "opt_compat.h" 39#include "opt_hwpmc_hooks.h" 40#include "opt_kdtrace.h" 41 42#include <machine/asmacros.h> 43#include <machine/psl.h> 44#include <machine/trap.h> 45#include <machine/specialreg.h> 46 47#include "assym.s" 48 49#ifdef KDTRACE_HOOKS 50 .bss 51 .globl dtrace_invop_jump_addr 52 .align 8 53 .type dtrace_invop_jump_addr,@object 54 .size dtrace_invop_jump_addr,8 55dtrace_invop_jump_addr: 56 .zero 8 57 .globl dtrace_invop_calltrap_addr 58 .align 8 59 .type dtrace_invop_calltrap_addr,@object 60 .size dtrace_invop_calltrap_addr,8 61dtrace_invop_calltrap_addr: 62 .zero 8 63#endif 64 .text 65#ifdef HWPMC_HOOKS 66 ENTRY(start_exceptions) 67#endif 68 69/*****************************************************************************/ 70/* Trap handling */ 71/*****************************************************************************/ 72/* 73 * Trap and fault vector routines. 74 * 75 * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 76 * state on the stack but also disables interrupts. This is important for 77 * us for the use of the swapgs instruction. We cannot be interrupted 78 * until the GS.base value is correct. For most traps, we automatically 79 * then enable interrupts if the interrupted context had them enabled. 80 * This is equivalent to the i386 port's use of SDT_SYS386TGT. 81 * 82 * The cpu will push a certain amount of state onto the kernel stack for 83 * the current process. See amd64/include/frame.h. 84 * This includes the current RFLAGS (status register, which includes 85 * the interrupt disable state prior to the trap), the code segment register, 86 * and the return instruction pointer are pushed by the cpu. The cpu 87 * will also push an 'error' code for certain traps. We push a dummy 88 * error code for those traps where the cpu doesn't in order to maintain 89 * a consistent frame. We also push a contrived 'trap number'. 90 * 91 * The CPU does not push the general registers, so we must do that, and we 92 * must restore them prior to calling 'iret'. The CPU adjusts %cs and %ss 93 * but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for 94 * for the kernel mode operation shortly, without changes to the selector 95 * loaded. Since superuser long mode works with any selectors loaded into 96 * segment registers other then %cs, which makes them mostly unused in long 97 * mode, and kernel does not reference %fs, leave them alone. The segment 98 * registers are reloaded on return to the usermode. 99 */ 100 101MCOUNT_LABEL(user) 102MCOUNT_LABEL(btrap) 103 104/* Traps that we leave interrupts disabled for.. */ 105#define TRAP_NOEN(a) \ 106 subq $TF_RIP,%rsp; \ 107 movl $(a),TF_TRAPNO(%rsp) ; \ 108 movq $0,TF_ADDR(%rsp) ; \ 109 movq $0,TF_ERR(%rsp) ; \ 110 jmp alltraps_noen 111IDTVEC(dbg) 112 TRAP_NOEN(T_TRCTRAP) 113IDTVEC(bpt) 114 TRAP_NOEN(T_BPTFLT) 115#ifdef KDTRACE_HOOKS 116IDTVEC(dtrace_ret) 117 TRAP_NOEN(T_DTRACE_RET) 118#endif 119 120/* Regular traps; The cpu does not supply tf_err for these. */ 121#define TRAP(a) \ 122 subq $TF_RIP,%rsp; \ 123 movl $(a),TF_TRAPNO(%rsp) ; \ 124 movq $0,TF_ADDR(%rsp) ; \ 125 movq $0,TF_ERR(%rsp) ; \ 126 jmp alltraps 127IDTVEC(div) 128 TRAP(T_DIVIDE) 129IDTVEC(ofl) 130 TRAP(T_OFLOW) 131IDTVEC(bnd) 132 TRAP(T_BOUND) 133IDTVEC(ill) 134 TRAP(T_PRIVINFLT) 135IDTVEC(dna) 136 TRAP(T_DNA) 137IDTVEC(fpusegm) 138 TRAP(T_FPOPFLT) 139IDTVEC(mchk) 140 TRAP(T_MCHK) 141IDTVEC(rsvd) 142 TRAP(T_RESERVED) 143IDTVEC(fpu) 144 TRAP(T_ARITHTRAP) 145IDTVEC(xmm) 146 TRAP(T_XMMFLT) 147 148/* This group of traps have tf_err already pushed by the cpu */ 149#define TRAP_ERR(a) \ 150 subq $TF_ERR,%rsp; \ 151 movl $(a),TF_TRAPNO(%rsp) ; \ 152 movq $0,TF_ADDR(%rsp) ; \ 153 jmp alltraps 154IDTVEC(tss) 155 TRAP_ERR(T_TSSFLT) 156IDTVEC(missing) 157 subq $TF_ERR,%rsp 158 movl $T_SEGNPFLT,TF_TRAPNO(%rsp) 159 jmp prot_addrf 160IDTVEC(stk) 161 subq $TF_ERR,%rsp 162 movl $T_STKFLT,TF_TRAPNO(%rsp) 163 jmp prot_addrf 164IDTVEC(align) 165 TRAP_ERR(T_ALIGNFLT) 166 167 /* 168 * alltraps entry point. Use swapgs if this is the first time in the 169 * kernel from userland. Reenable interrupts if they were enabled 170 * before the trap. This approximates SDT_SYS386TGT on the i386 port. 171 */ 172 SUPERALIGN_TEXT 173 .globl alltraps 174 .type alltraps,@function 175alltraps: 176 movq %rdi,TF_RDI(%rsp) 177 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 178 jz alltraps_testi /* already running with kernel GS.base */ 179 swapgs 180 movq PCPU(CURPCB),%rdi 181 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 182 movw %fs,TF_FS(%rsp) 183 movw %gs,TF_GS(%rsp) 184 movw %es,TF_ES(%rsp) 185 movw %ds,TF_DS(%rsp) 186alltraps_testi: 187 testl $PSL_I,TF_RFLAGS(%rsp) 188 jz alltraps_pushregs_no_rdi 189 sti 190alltraps_pushregs_no_rdi: 191 movq %rsi,TF_RSI(%rsp) 192 movq %rdx,TF_RDX(%rsp) 193 movq %rcx,TF_RCX(%rsp) 194 movq %r8,TF_R8(%rsp) 195 movq %r9,TF_R9(%rsp) 196 movq %rax,TF_RAX(%rsp) 197 movq %rbx,TF_RBX(%rsp) 198 movq %rbp,TF_RBP(%rsp) 199 movq %r10,TF_R10(%rsp) 200 movq %r11,TF_R11(%rsp) 201 movq %r12,TF_R12(%rsp) 202 movq %r13,TF_R13(%rsp) 203 movq %r14,TF_R14(%rsp) 204 movq %r15,TF_R15(%rsp) 205 movl $TF_HASSEGS,TF_FLAGS(%rsp) 206 cld 207 FAKE_MCOUNT(TF_RIP(%rsp)) 208#ifdef KDTRACE_HOOKS 209 /* 210 * DTrace Function Boundary Trace (fbt) probes are triggered 211 * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint 212 * interrupt. For all other trap types, just handle them in 213 * the usual way. 214 */ 215 cmpl $T_BPTFLT,TF_TRAPNO(%rsp) 216 jne calltrap 217 218 /* Check if there is no DTrace hook registered. */ 219 cmpq $0,dtrace_invop_jump_addr 220 je calltrap 221 222 /* 223 * Set our jump address for the jump back in the event that 224 * the breakpoint wasn't caused by DTrace at all. 225 */ 226 movq $calltrap,dtrace_invop_calltrap_addr(%rip) 227 228 /* Jump to the code hooked in by DTrace. */ 229 movq dtrace_invop_jump_addr,%rax 230 jmpq *dtrace_invop_jump_addr 231#endif 232 .globl calltrap 233 .type calltrap,@function 234calltrap: 235 movq %rsp,%rdi 236 call trap 237 MEXITCOUNT 238 jmp doreti /* Handle any pending ASTs */ 239 240 /* 241 * alltraps_noen entry point. Unlike alltraps above, we want to 242 * leave the interrupts disabled. This corresponds to 243 * SDT_SYS386IGT on the i386 port. 244 */ 245 SUPERALIGN_TEXT 246 .globl alltraps_noen 247 .type alltraps_noen,@function 248alltraps_noen: 249 movq %rdi,TF_RDI(%rsp) 250 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 251 jz 1f /* already running with kernel GS.base */ 252 swapgs 253 movq PCPU(CURPCB),%rdi 254 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 2551: movw %fs,TF_FS(%rsp) 256 movw %gs,TF_GS(%rsp) 257 movw %es,TF_ES(%rsp) 258 movw %ds,TF_DS(%rsp) 259 jmp alltraps_pushregs_no_rdi 260 261IDTVEC(dblfault) 262 subq $TF_ERR,%rsp 263 movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) 264 movq $0,TF_ADDR(%rsp) 265 movq $0,TF_ERR(%rsp) 266 movq %rdi,TF_RDI(%rsp) 267 movq %rsi,TF_RSI(%rsp) 268 movq %rdx,TF_RDX(%rsp) 269 movq %rcx,TF_RCX(%rsp) 270 movq %r8,TF_R8(%rsp) 271 movq %r9,TF_R9(%rsp) 272 movq %rax,TF_RAX(%rsp) 273 movq %rbx,TF_RBX(%rsp) 274 movq %rbp,TF_RBP(%rsp) 275 movq %r10,TF_R10(%rsp) 276 movq %r11,TF_R11(%rsp) 277 movq %r12,TF_R12(%rsp) 278 movq %r13,TF_R13(%rsp) 279 movq %r14,TF_R14(%rsp) 280 movq %r15,TF_R15(%rsp) 281 movw %fs,TF_FS(%rsp) 282 movw %gs,TF_GS(%rsp) 283 movw %es,TF_ES(%rsp) 284 movw %ds,TF_DS(%rsp) 285 movl $TF_HASSEGS,TF_FLAGS(%rsp) 286 cld 287 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 288 jz 1f /* already running with kernel GS.base */ 289 swapgs 2901: 291 movq %rsp,%rdi 292 call dblfault_handler 2932: 294 hlt 295 jmp 2b 296 297IDTVEC(page) 298 subq $TF_ERR,%rsp 299 movl $T_PAGEFLT,TF_TRAPNO(%rsp) 300 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 301 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 302 jz 1f /* already running with kernel GS.base */ 303 swapgs 304 movq PCPU(CURPCB),%rdi 305 andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 3061: movq %cr2,%rdi /* preserve %cr2 before .. */ 307 movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 308 movw %fs,TF_FS(%rsp) 309 movw %gs,TF_GS(%rsp) 310 movw %es,TF_ES(%rsp) 311 movw %ds,TF_DS(%rsp) 312 testl $PSL_I,TF_RFLAGS(%rsp) 313 jz alltraps_pushregs_no_rdi 314 sti 315 jmp alltraps_pushregs_no_rdi 316 317 /* 318 * We have to special-case this one. If we get a trap in doreti() at 319 * the iretq stage, we'll reenter with the wrong gs state. We'll have 320 * to do a special the swapgs in this case even coming from the kernel. 321 * XXX linux has a trap handler for their equivalent of load_gs(). 322 */ 323IDTVEC(prot) 324 subq $TF_ERR,%rsp 325 movl $T_PROTFLT,TF_TRAPNO(%rsp) 326prot_addrf: 327 movq $0,TF_ADDR(%rsp) 328 movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 329 leaq doreti_iret(%rip),%rdi 330 cmpq %rdi,TF_RIP(%rsp) 331 je 1f /* kernel but with user gsbase!! */ 332 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 333 jz 2f /* already running with kernel GS.base */ 3341: swapgs 3352: movq PCPU(CURPCB),%rdi 336 orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ 337 movw %fs,TF_FS(%rsp) 338 movw %gs,TF_GS(%rsp) 339 movw %es,TF_ES(%rsp) 340 movw %ds,TF_DS(%rsp) 341 testl $PSL_I,TF_RFLAGS(%rsp) 342 jz alltraps_pushregs_no_rdi 343 sti 344 jmp alltraps_pushregs_no_rdi 345 346/* 347 * Fast syscall entry point. We enter here with just our new %cs/%ss set, 348 * and the new privilige level. We are still running on the old user stack 349 * pointer. We have to juggle a few things around to find our stack etc. 350 * swapgs gives us access to our PCPU space only. 351 * 352 * We do not support invoking this from a custom %cs or %ss (e.g. using 353 * entries from an LDT). 354 */ 355IDTVEC(fast_syscall) 356 swapgs 357 movq %rsp,PCPU(SCRATCH_RSP) 358 movq PCPU(RSP0),%rsp 359 /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 360 subq $TF_SIZE,%rsp 361 /* defer TF_RSP till we have a spare register */ 362 movq %r11,TF_RFLAGS(%rsp) 363 movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 364 movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 365 movq %r11,TF_RSP(%rsp) /* user stack pointer */ 366 movw %fs,TF_FS(%rsp) 367 movw %gs,TF_GS(%rsp) 368 movw %es,TF_ES(%rsp) 369 movw %ds,TF_DS(%rsp) 370 movq PCPU(CURPCB),%r11 371 andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) 372 sti 373 movq $KUDSEL,TF_SS(%rsp) 374 movq $KUCSEL,TF_CS(%rsp) 375 movq $2,TF_ERR(%rsp) 376 movq %rdi,TF_RDI(%rsp) /* arg 1 */ 377 movq %rsi,TF_RSI(%rsp) /* arg 2 */ 378 movq %rdx,TF_RDX(%rsp) /* arg 3 */ 379 movq %r10,TF_RCX(%rsp) /* arg 4 */ 380 movq %r8,TF_R8(%rsp) /* arg 5 */ 381 movq %r9,TF_R9(%rsp) /* arg 6 */ 382 movq %rax,TF_RAX(%rsp) /* syscall number */ 383 movq %rbx,TF_RBX(%rsp) /* C preserved */ 384 movq %rbp,TF_RBP(%rsp) /* C preserved */ 385 movq %r12,TF_R12(%rsp) /* C preserved */ 386 movq %r13,TF_R13(%rsp) /* C preserved */ 387 movq %r14,TF_R14(%rsp) /* C preserved */ 388 movq %r15,TF_R15(%rsp) /* C preserved */ 389 movl $TF_HASSEGS,TF_FLAGS(%rsp) 390 cld 391 FAKE_MCOUNT(TF_RIP(%rsp)) 392 movq PCPU(CURTHREAD),%rdi 393 movq %rsp,TD_FRAME(%rdi) 394 movl TF_RFLAGS(%rsp),%esi 395 andl $PSL_T,%esi 396 call amd64_syscall 3971: movq PCPU(CURPCB),%rax 398 /* Disable interrupts before testing PCB_FULL_IRET. */ 399 cli 400 testl $PCB_FULL_IRET,PCB_FLAGS(%rax) 401 jnz 3f 402 /* Check for and handle AST's on return to userland. */ 403 movq PCPU(CURTHREAD),%rax 404 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 405 jne 2f 406 /* Restore preserved registers. */ 407 MEXITCOUNT 408 movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ 409 movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ 410 movq TF_RDX(%rsp),%rdx /* return value 2 */ 411 movq TF_RAX(%rsp),%rax /* return value 1 */ 412 movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ 413 movq TF_RIP(%rsp),%rcx /* original %rip */ 414 movq TF_RSP(%rsp),%rsp /* user stack pointer */ 415 swapgs 416 sysretq 417 4182: /* AST scheduled. */ 419 sti 420 movq %rsp,%rdi 421 call ast 422 jmp 1b 423 4243: /* Requested full context restore, use doreti for that. */ 425 MEXITCOUNT 426 jmp doreti 427 428/* 429 * Here for CYA insurance, in case a "syscall" instruction gets 430 * issued from 32 bit compatability mode. MSR_CSTAR has to point 431 * to *something* if EFER_SCE is enabled. 432 */ 433IDTVEC(fast_syscall32) 434 sysret 435 436/* 437 * NMI handling is special. 438 * 439 * First, NMIs do not respect the state of the processor's RFLAGS.IF 440 * bit. The NMI handler may be entered at any time, including when 441 * the processor is in a critical section with RFLAGS.IF == 0. 442 * The processor's GS.base value could be invalid on entry to the 443 * handler. 444 * 445 * Second, the processor treats NMIs specially, blocking further NMIs 446 * until an 'iretq' instruction is executed. We thus need to execute 447 * the NMI handler with interrupts disabled, to prevent a nested interrupt 448 * from executing an 'iretq' instruction and inadvertently taking the 449 * processor out of NMI mode. 450 * 451 * Third, the NMI handler runs on its own stack (tss_ist2). The canonical 452 * GS.base value for the processor is stored just above the bottom of its 453 * NMI stack. For NMIs taken from kernel mode, the current value in 454 * the processor's GS.base is saved at entry to C-preserved register %r12, 455 * the canonical value for GS.base is then loaded into the processor, and 456 * the saved value is restored at exit time. For NMIs taken from user mode, 457 * the cheaper 'SWAPGS' instructions are used for swapping GS.base. 458 */ 459 460IDTVEC(nmi) 461 subq $TF_RIP,%rsp 462 movl $(T_NMI),TF_TRAPNO(%rsp) 463 movq $0,TF_ADDR(%rsp) 464 movq $0,TF_ERR(%rsp) 465 movq %rdi,TF_RDI(%rsp) 466 movq %rsi,TF_RSI(%rsp) 467 movq %rdx,TF_RDX(%rsp) 468 movq %rcx,TF_RCX(%rsp) 469 movq %r8,TF_R8(%rsp) 470 movq %r9,TF_R9(%rsp) 471 movq %rax,TF_RAX(%rsp) 472 movq %rbx,TF_RBX(%rsp) 473 movq %rbp,TF_RBP(%rsp) 474 movq %r10,TF_R10(%rsp) 475 movq %r11,TF_R11(%rsp) 476 movq %r12,TF_R12(%rsp) 477 movq %r13,TF_R13(%rsp) 478 movq %r14,TF_R14(%rsp) 479 movq %r15,TF_R15(%rsp) 480 movw %fs,TF_FS(%rsp) 481 movw %gs,TF_GS(%rsp) 482 movw %es,TF_ES(%rsp) 483 movw %ds,TF_DS(%rsp) 484 movl $TF_HASSEGS,TF_FLAGS(%rsp) 485 cld 486 xorl %ebx,%ebx 487 testb $SEL_RPL_MASK,TF_CS(%rsp) 488 jnz nmi_fromuserspace 489 /* 490 * We've interrupted the kernel. Preserve GS.base in %r12. 491 */ 492 movl $MSR_GSBASE,%ecx 493 rdmsr 494 movq %rax,%r12 495 shlq $32,%rdx 496 orq %rdx,%r12 497 /* Retrieve and load the canonical value for GS.base. */ 498 movq TF_SIZE(%rsp),%rdx 499 movl %edx,%eax 500 shrq $32,%rdx 501 wrmsr 502 jmp nmi_calltrap 503nmi_fromuserspace: 504 incl %ebx 505 swapgs 506/* Note: this label is also used by ddb and gdb: */ 507nmi_calltrap: 508 FAKE_MCOUNT(TF_RIP(%rsp)) 509 movq %rsp,%rdi 510 call trap 511 MEXITCOUNT 512#ifdef HWPMC_HOOKS 513 /* 514 * Capture a userspace callchain if needed. 515 * 516 * - Check if the current trap was from user mode. 517 * - Check if the current thread is valid. 518 * - Check if the thread requires a user call chain to be 519 * captured. 520 * 521 * We are still in NMI mode at this point. 522 */ 523 testl %ebx,%ebx 524 jz nocallchain /* not from userspace */ 525 movq PCPU(CURTHREAD),%rax 526 orq %rax,%rax /* curthread present? */ 527 jz nocallchain 528 testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ 529 jz nocallchain 530 /* 531 * A user callchain is to be captured, so: 532 * - Move execution to the regular kernel stack, to allow for 533 * nested NMI interrupts. 534 * - Take the processor out of "NMI" mode by faking an "iret". 535 * - Enable interrupts, so that copyin() can work. 536 */ 537 movq %rsp,%rsi /* source stack pointer */ 538 movq $TF_SIZE,%rcx 539 movq PCPU(RSP0),%rdx 540 subq %rcx,%rdx 541 movq %rdx,%rdi /* destination stack pointer */ 542 543 shrq $3,%rcx /* trap frame size in long words */ 544 cld 545 rep 546 movsq /* copy trapframe */ 547 548 movl %ss,%eax 549 pushq %rax /* tf_ss */ 550 pushq %rdx /* tf_rsp (on kernel stack) */ 551 pushfq /* tf_rflags */ 552 movl %cs,%eax 553 pushq %rax /* tf_cs */ 554 pushq $outofnmi /* tf_rip */ 555 iretq 556outofnmi: 557 /* 558 * At this point the processor has exited NMI mode and is running 559 * with interrupts turned off on the normal kernel stack. 560 * 561 * If a pending NMI gets recognized at or after this point, it 562 * will cause a kernel callchain to be traced. 563 * 564 * We turn interrupts back on, and call the user callchain capture hook. 565 */ 566 movq pmc_hook,%rax 567 orq %rax,%rax 568 jz nocallchain 569 movq PCPU(CURTHREAD),%rdi /* thread */ 570 movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ 571 movq %rsp,%rdx /* frame */ 572 sti 573 call *%rax 574 cli 575nocallchain: 576#endif 577 testl %ebx,%ebx 578 jnz doreti_exit 579nmi_kernelexit: 580 /* 581 * Put back the preserved MSR_GSBASE value. 582 */ 583 movl $MSR_GSBASE,%ecx 584 movq %r12,%rdx 585 movl %edx,%eax 586 shrq $32,%rdx 587 wrmsr 588nmi_restoreregs: 589 movq TF_RDI(%rsp),%rdi 590 movq TF_RSI(%rsp),%rsi 591 movq TF_RDX(%rsp),%rdx 592 movq TF_RCX(%rsp),%rcx 593 movq TF_R8(%rsp),%r8 594 movq TF_R9(%rsp),%r9 595 movq TF_RAX(%rsp),%rax 596 movq TF_RBX(%rsp),%rbx 597 movq TF_RBP(%rsp),%rbp 598 movq TF_R10(%rsp),%r10 599 movq TF_R11(%rsp),%r11 600 movq TF_R12(%rsp),%r12 601 movq TF_R13(%rsp),%r13 602 movq TF_R14(%rsp),%r14 603 movq TF_R15(%rsp),%r15 604 addq $TF_RIP,%rsp 605 jmp doreti_iret 606 607ENTRY(fork_trampoline) 608 movq %r12,%rdi /* function */ 609 movq %rbx,%rsi /* arg1 */ 610 movq %rsp,%rdx /* trapframe pointer */ 611 call fork_exit 612 MEXITCOUNT 613 jmp doreti /* Handle any ASTs */ 614 615/* 616 * To efficiently implement classification of trap and interrupt handlers 617 * for profiling, there must be only trap handlers between the labels btrap 618 * and bintr, and only interrupt handlers between the labels bintr and 619 * eintr. This is implemented (partly) by including files that contain 620 * some of the handlers. Before including the files, set up a normal asm 621 * environment so that the included files doen't need to know that they are 622 * included. 623 */ 624 625#ifdef COMPAT_FREEBSD32 626 .data 627 .p2align 4 628 .text 629 SUPERALIGN_TEXT 630 631#include <amd64/ia32/ia32_exception.S> 632#endif 633 634 .data 635 .p2align 4 636 .text 637 SUPERALIGN_TEXT 638MCOUNT_LABEL(bintr) 639 640#include <amd64/amd64/apic_vector.S> 641 642#ifdef DEV_ATPIC 643 .data 644 .p2align 4 645 .text 646 SUPERALIGN_TEXT 647 648#include <amd64/amd64/atpic_vector.S> 649#endif 650 651 .text 652MCOUNT_LABEL(eintr) 653 654/* 655 * void doreti(struct trapframe) 656 * 657 * Handle return from interrupts, traps and syscalls. 658 */ 659 .text 660 SUPERALIGN_TEXT 661 .type doreti,@function 662doreti: 663 FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 664 /* 665 * Check if ASTs can be handled now. 666 */ 667 testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 668 jz doreti_exit /* can't handle ASTs now if not */ 669 670doreti_ast: 671 /* 672 * Check for ASTs atomically with returning. Disabling CPU 673 * interrupts provides sufficient locking even in the SMP case, 674 * since we will be informed of any new ASTs by an IPI. 675 */ 676 cli 677 movq PCPU(CURTHREAD),%rax 678 testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 679 je doreti_exit 680 sti 681 movq %rsp,%rdi /* pass a pointer to the trapframe */ 682 call ast 683 jmp doreti_ast 684 685 /* 686 * doreti_exit: pop registers, iret. 687 * 688 * The segment register pop is a special case, since it may 689 * fault if (for example) a sigreturn specifies bad segment 690 * registers. The fault is handled in trap.c. 691 */ 692doreti_exit: 693 MEXITCOUNT 694 movq PCPU(CURPCB),%r8 695 696 /* 697 * Do not reload segment registers for kernel. 698 * Since we do not reload segments registers with sane 699 * values on kernel entry, descriptors referenced by 700 * segments registers might be not valid. This is fatal 701 * for user mode, but is not a problem for the kernel. 702 */ 703 testb $SEL_RPL_MASK,TF_CS(%rsp) 704 jz ld_regs 705 testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 706 jz ld_regs 707 testl $TF_HASSEGS,TF_FLAGS(%rsp) 708 je set_segs 709 710do_segs: 711 /* Restore %fs and fsbase */ 712 movw TF_FS(%rsp),%ax 713 .globl ld_fs 714ld_fs: 715 movw %ax,%fs 716 cmpw $KUF32SEL,%ax 717 jne 1f 718 movl $MSR_FSBASE,%ecx 719 movl PCB_FSBASE(%r8),%eax 720 movl PCB_FSBASE+4(%r8),%edx 721 .globl ld_fsbase 722ld_fsbase: 723 wrmsr 7241: 725 /* Restore %gs and gsbase */ 726 movw TF_GS(%rsp),%si 727 pushfq 728 cli 729 movl $MSR_GSBASE,%ecx 730 /* Save current kernel %gs base into %r12d:%r13d */ 731 rdmsr 732 movl %eax,%r12d 733 movl %edx,%r13d 734 .globl ld_gs 735ld_gs: 736 movw %si,%gs 737 /* Save user %gs base into %r14d:%r15d */ 738 rdmsr 739 movl %eax,%r14d 740 movl %edx,%r15d 741 /* Restore kernel %gs base */ 742 movl %r12d,%eax 743 movl %r13d,%edx 744 wrmsr 745 popfq 746 /* 747 * Restore user %gs base, either from PCB if used for TLS, or 748 * from the previously saved msr read. 749 */ 750 movl $MSR_KGSBASE,%ecx 751 cmpw $KUG32SEL,%si 752 jne 1f 753 movl PCB_GSBASE(%r8),%eax 754 movl PCB_GSBASE+4(%r8),%edx 755 jmp ld_gsbase 7561: 757 movl %r14d,%eax 758 movl %r15d,%edx 759 .globl ld_gsbase 760ld_gsbase: 761 wrmsr /* May trap if non-canonical, but only for TLS. */ 762 .globl ld_es 763ld_es: 764 movw TF_ES(%rsp),%es 765 .globl ld_ds 766ld_ds: 767 movw TF_DS(%rsp),%ds 768ld_regs: 769 movq TF_RDI(%rsp),%rdi 770 movq TF_RSI(%rsp),%rsi 771 movq TF_RDX(%rsp),%rdx 772 movq TF_RCX(%rsp),%rcx 773 movq TF_R8(%rsp),%r8 774 movq TF_R9(%rsp),%r9 775 movq TF_RAX(%rsp),%rax 776 movq TF_RBX(%rsp),%rbx 777 movq TF_RBP(%rsp),%rbp 778 movq TF_R10(%rsp),%r10 779 movq TF_R11(%rsp),%r11 780 movq TF_R12(%rsp),%r12 781 movq TF_R13(%rsp),%r13 782 movq TF_R14(%rsp),%r14 783 movq TF_R15(%rsp),%r15 784 testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 785 jz 1f /* keep running with kernel GS.base */ 786 cli 787 swapgs 7881: 789 addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 790 .globl doreti_iret 791doreti_iret: 792 iretq 793 794set_segs: 795 movw $KUDSEL,%ax 796 movw %ax,TF_DS(%rsp) 797 movw %ax,TF_ES(%rsp) 798 movw $KUF32SEL,TF_FS(%rsp) 799 movw $KUG32SEL,TF_GS(%rsp) 800 jmp do_segs 801 802 /* 803 * doreti_iret_fault. Alternative return code for 804 * the case where we get a fault in the doreti_exit code 805 * above. trap() (amd64/amd64/trap.c) catches this specific 806 * case, sends the process a signal and continues in the 807 * corresponding place in the code below. 808 */ 809 ALIGN_TEXT 810 .globl doreti_iret_fault 811doreti_iret_fault: 812 subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 813 testl $PSL_I,TF_RFLAGS(%rsp) 814 jz 1f 815 sti 8161: 817 movw %fs,TF_FS(%rsp) 818 movw %gs,TF_GS(%rsp) 819 movw %es,TF_ES(%rsp) 820 movw %ds,TF_DS(%rsp) 821 movl $TF_HASSEGS,TF_FLAGS(%rsp) 822 movq %rdi,TF_RDI(%rsp) 823 movq %rsi,TF_RSI(%rsp) 824 movq %rdx,TF_RDX(%rsp) 825 movq %rcx,TF_RCX(%rsp) 826 movq %r8,TF_R8(%rsp) 827 movq %r9,TF_R9(%rsp) 828 movq %rax,TF_RAX(%rsp) 829 movq %rbx,TF_RBX(%rsp) 830 movq %rbp,TF_RBP(%rsp) 831 movq %r10,TF_R10(%rsp) 832 movq %r11,TF_R11(%rsp) 833 movq %r12,TF_R12(%rsp) 834 movq %r13,TF_R13(%rsp) 835 movq %r14,TF_R14(%rsp) 836 movq %r15,TF_R15(%rsp) 837 movl $T_PROTFLT,TF_TRAPNO(%rsp) 838 movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 839 movq $0,TF_ADDR(%rsp) 840 FAKE_MCOUNT(TF_RIP(%rsp)) 841 jmp calltrap 842 843 ALIGN_TEXT 844 .globl ds_load_fault 845ds_load_fault: 846 movl $T_PROTFLT,TF_TRAPNO(%rsp) 847 testl $PSL_I,TF_RFLAGS(%rsp) 848 jz 1f 849 sti 8501: 851 movq %rsp,%rdi 852 call trap 853 movw $KUDSEL,TF_DS(%rsp) 854 jmp doreti 855 856 ALIGN_TEXT 857 .globl es_load_fault 858es_load_fault: 859 movl $T_PROTFLT,TF_TRAPNO(%rsp) 860 testl $PSL_I,TF_RFLAGS(%rsp) 861 jz 1f 862 sti 8631: 864 movq %rsp,%rdi 865 call trap 866 movw $KUDSEL,TF_ES(%rsp) 867 jmp doreti 868 869 ALIGN_TEXT 870 .globl fs_load_fault 871fs_load_fault: 872 testl $PSL_I,TF_RFLAGS(%rsp) 873 jz 1f 874 sti 8751: 876 movl $T_PROTFLT,TF_TRAPNO(%rsp) 877 movq %rsp,%rdi 878 call trap 879 movw $KUF32SEL,TF_FS(%rsp) 880 jmp doreti 881 882 ALIGN_TEXT 883 .globl gs_load_fault 884gs_load_fault: 885 popfq 886 movl $T_PROTFLT,TF_TRAPNO(%rsp) 887 testl $PSL_I,TF_RFLAGS(%rsp) 888 jz 1f 889 sti 8901: 891 movq %rsp,%rdi 892 call trap 893 movw $KUG32SEL,TF_GS(%rsp) 894 jmp doreti 895 896 ALIGN_TEXT 897 .globl fsbase_load_fault 898fsbase_load_fault: 899 movl $T_PROTFLT,TF_TRAPNO(%rsp) 900 testl $PSL_I,TF_RFLAGS(%rsp) 901 jz 1f 902 sti 9031: 904 movq %rsp,%rdi 905 call trap 906 movq PCPU(CURTHREAD),%r8 907 movq TD_PCB(%r8),%r8 908 movq $0,PCB_FSBASE(%r8) 909 jmp doreti 910 911 ALIGN_TEXT 912 .globl gsbase_load_fault 913gsbase_load_fault: 914 movl $T_PROTFLT,TF_TRAPNO(%rsp) 915 testl $PSL_I,TF_RFLAGS(%rsp) 916 jz 1f 917 sti 9181: 919 movq %rsp,%rdi 920 call trap 921 movq PCPU(CURTHREAD),%r8 922 movq TD_PCB(%r8),%r8 923 movq $0,PCB_GSBASE(%r8) 924 jmp doreti 925 926#ifdef HWPMC_HOOKS 927 ENTRY(end_exceptions) 928#endif 929