1757Sdg/*- 299703Sjulian * Copyright (c) 1989, 1990 William F. Jolitz. 3757Sdg * Copyright (c) 1990 The Regents of the University of California. 4174395Sjkoshy * Copyright (c) 2007 The FreeBSD Foundation 5757Sdg * All rights reserved. 6757Sdg * 7174395Sjkoshy * Portions of this software were developed by A. Joseph Koshy under 8174395Sjkoshy * sponsorship from the FreeBSD Foundation and Google, Inc. 9174395Sjkoshy * 10757Sdg * Redistribution and use in source and binary forms, with or without 11757Sdg * modification, are permitted provided that the following conditions 12757Sdg * are met: 13757Sdg * 1. Redistributions of source code must retain the above copyright 14757Sdg * notice, this list of conditions and the following disclaimer. 15757Sdg * 2. Redistributions in binary form must reproduce the above copyright 16757Sdg * notice, this list of conditions and the following disclaimer in the 17757Sdg * documentation and/or other materials provided with the distribution. 18757Sdg * 4. Neither the name of the University nor the names of its contributors 19757Sdg * may be used to endorse or promote products derived from this software 20757Sdg * without specific prior written permission. 21757Sdg * 22757Sdg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23757Sdg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24757Sdg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25757Sdg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26757Sdg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27757Sdg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28757Sdg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29757Sdg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30757Sdg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31757Sdg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32757Sdg * SUCH DAMAGE. 33757Sdg * 3450477Speter * $FreeBSD$ 35757Sdg */ 36757Sdg 37129653Sbde#include "opt_atpic.h" 38133854Sobrien#include "opt_compat.h" 39174395Sjkoshy#include "opt_hwpmc_hooks.h" 40179279Sjb#include "opt_kdtrace.h" 41129653Sbde 4228921Sfsmp#include <machine/asmacros.h> 4330786Sbde#include <machine/psl.h> 4430786Sbde#include <machine/trap.h> 45190620Skib#include <machine/specialreg.h> 46757Sdg 4730786Sbde#include "assym.s" 4830786Sbde 49179279Sjb#ifdef KDTRACE_HOOKS 50179279Sjb .bss 51179279Sjb .globl dtrace_invop_jump_addr 52179279Sjb .align 8 53207570Skib .type dtrace_invop_jump_addr,@object 54207570Skib .size dtrace_invop_jump_addr,8 55179279Sjbdtrace_invop_jump_addr: 56179279Sjb .zero 8 57179279Sjb .globl dtrace_invop_calltrap_addr 58179279Sjb .align 8 59207570Skib .type dtrace_invop_calltrap_addr,@object 60207570Skib .size dtrace_invop_calltrap_addr,8 61179279Sjbdtrace_invop_calltrap_addr: 62179279Sjb .zero 8 63179279Sjb#endif 64757Sdg .text 65174395Sjkoshy#ifdef HWPMC_HOOKS 66174395Sjkoshy ENTRY(start_exceptions) 67174395Sjkoshy#endif 68757Sdg 69757Sdg/*****************************************************************************/ 70757Sdg/* Trap handling */ 71757Sdg/*****************************************************************************/ 72757Sdg/* 7358717Sdillon * Trap and fault vector routines. 7458717Sdillon * 75114928Speter * All traps are 'interrupt gates', SDT_SYSIGT. An interrupt gate pushes 76114928Speter * state on the stack but also disables interrupts. This is important for 77114928Speter * us for the use of the swapgs instruction. We cannot be interrupted 78114928Speter * until the GS.base value is correct. For most traps, we automatically 79114928Speter * then enable interrupts if the interrupted context had them enabled. 80114928Speter * This is equivalent to the i386 port's use of SDT_SYS386TGT. 8158717Sdillon * 8258717Sdillon * The cpu will push a certain amount of state onto the kernel stack for 83251988Skib * the current process. See amd64/include/frame.h. 84251988Skib * This includes the current RFLAGS (status register, which includes 8558717Sdillon * the interrupt disable state prior to the trap), the code segment register, 86251988Skib * and the return instruction pointer are pushed by the cpu. The cpu 87251988Skib * will also push an 'error' code for certain traps. We push a dummy 88251988Skib * error code for those traps where the cpu doesn't in order to maintain 8958717Sdillon * a consistent frame. We also push a contrived 'trap number'. 9058717Sdillon * 91251988Skib * The CPU does not push the general registers, so we must do that, and we 92251988Skib * must restore them prior to calling 'iret'. The CPU adjusts %cs and %ss 93251988Skib * but does not mess with %ds, %es, %gs or %fs. We swap the %gs base for 94251988Skib * for the kernel mode operation shortly, without changes to the selector 95251988Skib * loaded. Since superuser long mode works with any selectors loaded into 96251988Skib * segment registers other then %cs, which makes them mostly unused in long 97251988Skib * mode, and kernel does not reference %fs, leave them alone. The segment 98251988Skib * registers are reloaded on return to the usermode. 991321Sdg */ 1001321Sdg 1011321SdgMCOUNT_LABEL(user) 1021321SdgMCOUNT_LABEL(btrap) 1031321Sdg 104114952Speter/* Traps that we leave interrupts disabled for.. */ 105114952Speter#define TRAP_NOEN(a) \ 106114952Speter subq $TF_RIP,%rsp; \ 107190620Skib movl $(a),TF_TRAPNO(%rsp) ; \ 108114952Speter movq $0,TF_ADDR(%rsp) ; \ 109114952Speter movq $0,TF_ERR(%rsp) ; \ 110114952Speter jmp alltraps_noen 111114952SpeterIDTVEC(dbg) 112114952Speter TRAP_NOEN(T_TRCTRAP) 113114952SpeterIDTVEC(bpt) 114114952Speter TRAP_NOEN(T_BPTFLT) 115211924Srpaulo#ifdef KDTRACE_HOOKS 116211924SrpauloIDTVEC(dtrace_ret) 117211924Srpaulo TRAP_NOEN(T_DTRACE_RET) 118211924Srpaulo#endif 119114952Speter 120114952Speter/* Regular traps; The cpu does not supply tf_err for these. */ 121114952Speter#define TRAP(a) \ 122114952Speter subq $TF_RIP,%rsp; \ 123190620Skib movl $(a),TF_TRAPNO(%rsp) ; \ 124114952Speter movq $0,TF_ADDR(%rsp) ; \ 125114952Speter movq $0,TF_ERR(%rsp) ; \ 126114952Speter jmp alltraps 127757SdgIDTVEC(div) 128114952Speter TRAP(T_DIVIDE) 129757SdgIDTVEC(ofl) 130114952Speter TRAP(T_OFLOW) 131757SdgIDTVEC(bnd) 132114952Speter TRAP(T_BOUND) 133757SdgIDTVEC(ill) 134114952Speter TRAP(T_PRIVINFLT) 135757SdgIDTVEC(dna) 136114952Speter TRAP(T_DNA) 137757SdgIDTVEC(fpusegm) 138114952Speter TRAP(T_FPOPFLT) 139114952SpeterIDTVEC(mchk) 140114952Speter TRAP(T_MCHK) 141114952SpeterIDTVEC(rsvd) 142114952Speter TRAP(T_RESERVED) 143114952SpeterIDTVEC(fpu) 144114952Speter TRAP(T_ARITHTRAP) 145114952SpeterIDTVEC(xmm) 146114952Speter TRAP(T_XMMFLT) 147114952Speter 148114952Speter/* This group of traps have tf_err already pushed by the cpu */ 149114952Speter#define TRAP_ERR(a) \ 150114952Speter subq $TF_ERR,%rsp; \ 151190620Skib movl $(a),TF_TRAPNO(%rsp) ; \ 152114952Speter movq $0,TF_ADDR(%rsp) ; \ 153147568Speter jmp alltraps 154757SdgIDTVEC(tss) 155114952Speter TRAP_ERR(T_TSSFLT) 156757SdgIDTVEC(missing) 157287146Sdelphij subq $TF_ERR,%rsp 158287146Sdelphij movl $T_SEGNPFLT,TF_TRAPNO(%rsp) 159287146Sdelphij jmp prot_addrf 160757SdgIDTVEC(stk) 161287146Sdelphij subq $TF_ERR,%rsp 162287146Sdelphij movl $T_STKFLT,TF_TRAPNO(%rsp) 163287146Sdelphij jmp prot_addrf 1645603SbdeIDTVEC(align) 165114952Speter TRAP_ERR(T_ALIGNFLT) 166140553Speter 16758717Sdillon /* 168114928Speter * alltraps entry point. Use swapgs if this is the first time in the 169114928Speter * kernel from userland. Reenable interrupts if they were enabled 170114928Speter * before the trap. This approximates SDT_SYS386TGT on the i386 port. 17158717Sdillon */ 172757Sdg SUPERALIGN_TEXT 17373011Sjake .globl alltraps 17473011Sjake .type alltraps,@function 17573011Sjakealltraps: 176195486Skib movq %rdi,TF_RDI(%rsp) 177114928Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 178114928Speter jz alltraps_testi /* already running with kernel GS.base */ 179114928Speter swapgs 180195486Skib movq PCPU(CURPCB),%rdi 181216673Sjkim andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 182190620Skib movw %fs,TF_FS(%rsp) 183190620Skib movw %gs,TF_GS(%rsp) 184190620Skib movw %es,TF_ES(%rsp) 185190620Skib movw %ds,TF_DS(%rsp) 186114928Speteralltraps_testi: 187114928Speter testl $PSL_I,TF_RFLAGS(%rsp) 188195486Skib jz alltraps_pushregs_no_rdi 189114928Speter sti 190114952Speteralltraps_pushregs_no_rdi: 191114349Speter movq %rsi,TF_RSI(%rsp) 192114349Speter movq %rdx,TF_RDX(%rsp) 193114349Speter movq %rcx,TF_RCX(%rsp) 194114349Speter movq %r8,TF_R8(%rsp) 195114349Speter movq %r9,TF_R9(%rsp) 196114349Speter movq %rax,TF_RAX(%rsp) 197114349Speter movq %rbx,TF_RBX(%rsp) 198114349Speter movq %rbp,TF_RBP(%rsp) 199114349Speter movq %r10,TF_R10(%rsp) 200114349Speter movq %r11,TF_R11(%rsp) 201114349Speter movq %r12,TF_R12(%rsp) 202114349Speter movq %r13,TF_R13(%rsp) 203114349Speter movq %r14,TF_R14(%rsp) 204114349Speter movq %r15,TF_R15(%rsp) 205190620Skib movl $TF_HASSEGS,TF_FLAGS(%rsp) 206209483Skib cld 207129623Sbde FAKE_MCOUNT(TF_RIP(%rsp)) 208179279Sjb#ifdef KDTRACE_HOOKS 209179279Sjb /* 210179279Sjb * DTrace Function Boundary Trace (fbt) probes are triggered 211179279Sjb * by int3 (0xcc) which causes the #BP (T_BPTFLT) breakpoint 212179279Sjb * interrupt. For all other trap types, just handle them in 213179279Sjb * the usual way. 214179279Sjb */ 215190620Skib cmpl $T_BPTFLT,TF_TRAPNO(%rsp) 216179279Sjb jne calltrap 217179279Sjb 218179279Sjb /* Check if there is no DTrace hook registered. */ 219179279Sjb cmpq $0,dtrace_invop_jump_addr 220179279Sjb je calltrap 221179279Sjb 222179279Sjb /* 223179279Sjb * Set our jump address for the jump back in the event that 224179279Sjb * the breakpoint wasn't caused by DTrace at all. 225179279Sjb */ 226207570Skib movq $calltrap,dtrace_invop_calltrap_addr(%rip) 227179279Sjb 228179279Sjb /* Jump to the code hooked in by DTrace. */ 229207570Skib movq dtrace_invop_jump_addr,%rax 230179279Sjb jmpq *dtrace_invop_jump_addr 231179279Sjb#endif 232146461Speter .globl calltrap 233146461Speter .type calltrap,@function 234757Sdgcalltrap: 235207570Skib movq %rsp,%rdi 23673011Sjake call trap 237129623Sbde MEXITCOUNT 238114349Speter jmp doreti /* Handle any pending ASTs */ 239757Sdg 240114928Speter /* 241114928Speter * alltraps_noen entry point. Unlike alltraps above, we want to 242114928Speter * leave the interrupts disabled. This corresponds to 243114928Speter * SDT_SYS386IGT on the i386 port. 244114928Speter */ 245114928Speter SUPERALIGN_TEXT 246114928Speter .globl alltraps_noen 247114928Speter .type alltraps_noen,@function 248114928Speteralltraps_noen: 249195486Skib movq %rdi,TF_RDI(%rsp) 250114928Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 251190620Skib jz 1f /* already running with kernel GS.base */ 252114928Speter swapgs 253195486Skib movq PCPU(CURPCB),%rdi 254216673Sjkim andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 255190620Skib1: movw %fs,TF_FS(%rsp) 256190620Skib movw %gs,TF_GS(%rsp) 257190620Skib movw %es,TF_ES(%rsp) 258190620Skib movw %ds,TF_DS(%rsp) 259195486Skib jmp alltraps_pushregs_no_rdi 260114928Speter 261114928SpeterIDTVEC(dblfault) 262114952Speter subq $TF_ERR,%rsp 263190620Skib movl $T_DOUBLEFLT,TF_TRAPNO(%rsp) 264173659Sjhb movq $0,TF_ADDR(%rsp) 265173659Sjhb movq $0,TF_ERR(%rsp) 266173659Sjhb movq %rdi,TF_RDI(%rsp) 267173659Sjhb movq %rsi,TF_RSI(%rsp) 268173659Sjhb movq %rdx,TF_RDX(%rsp) 269173659Sjhb movq %rcx,TF_RCX(%rsp) 270173659Sjhb movq %r8,TF_R8(%rsp) 271173659Sjhb movq %r9,TF_R9(%rsp) 272173659Sjhb movq %rax,TF_RAX(%rsp) 273173659Sjhb movq %rbx,TF_RBX(%rsp) 274173659Sjhb movq %rbp,TF_RBP(%rsp) 275173659Sjhb movq %r10,TF_R10(%rsp) 276173659Sjhb movq %r11,TF_R11(%rsp) 277173659Sjhb movq %r12,TF_R12(%rsp) 278173659Sjhb movq %r13,TF_R13(%rsp) 279173659Sjhb movq %r14,TF_R14(%rsp) 280173659Sjhb movq %r15,TF_R15(%rsp) 281190620Skib movw %fs,TF_FS(%rsp) 282190620Skib movw %gs,TF_GS(%rsp) 283190620Skib movw %es,TF_ES(%rsp) 284190620Skib movw %ds,TF_DS(%rsp) 285190620Skib movl $TF_HASSEGS,TF_FLAGS(%rsp) 286209483Skib cld 287114928Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 288114928Speter jz 1f /* already running with kernel GS.base */ 289114928Speter swapgs 290207570Skib1: 291207570Skib movq %rsp,%rdi 292173659Sjhb call dblfault_handler 293207570Skib2: 294207570Skib hlt 295114928Speter jmp 2b 296114928Speter 297114952SpeterIDTVEC(page) 298114952Speter subq $TF_ERR,%rsp 299190620Skib movl $T_PAGEFLT,TF_TRAPNO(%rsp) 300195486Skib movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 301114952Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 302114952Speter jz 1f /* already running with kernel GS.base */ 303114952Speter swapgs 304195486Skib movq PCPU(CURPCB),%rdi 305216673Sjkim andl $~PCB_FULL_IRET,PCB_FLAGS(%rdi) 306195486Skib1: movq %cr2,%rdi /* preserve %cr2 before .. */ 307114952Speter movq %rdi,TF_ADDR(%rsp) /* enabling interrupts. */ 308190620Skib movw %fs,TF_FS(%rsp) 309190620Skib movw %gs,TF_GS(%rsp) 310190620Skib movw %es,TF_ES(%rsp) 311190620Skib movw %ds,TF_DS(%rsp) 312114952Speter testl $PSL_I,TF_RFLAGS(%rsp) 313114952Speter jz alltraps_pushregs_no_rdi 314114952Speter sti 315114952Speter jmp alltraps_pushregs_no_rdi 316114952Speter 317147677Speter /* 318147677Speter * We have to special-case this one. If we get a trap in doreti() at 319147677Speter * the iretq stage, we'll reenter with the wrong gs state. We'll have 320147677Speter * to do a special the swapgs in this case even coming from the kernel. 321147677Speter * XXX linux has a trap handler for their equivalent of load_gs(). 322147677Speter */ 323147677SpeterIDTVEC(prot) 324147677Speter subq $TF_ERR,%rsp 325190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 326287146Sdelphijprot_addrf: 327147677Speter movq $0,TF_ADDR(%rsp) 328147677Speter movq %rdi,TF_RDI(%rsp) /* free up a GP register */ 329147677Speter leaq doreti_iret(%rip),%rdi 330147677Speter cmpq %rdi,TF_RIP(%rsp) 331190620Skib je 1f /* kernel but with user gsbase!! */ 332147677Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 333190620Skib jz 2f /* already running with kernel GS.base */ 334190620Skib1: swapgs 335195486Skib2: movq PCPU(CURPCB),%rdi 336216673Sjkim orl $PCB_FULL_IRET,PCB_FLAGS(%rdi) /* always full iret from GPF */ 337195486Skib movw %fs,TF_FS(%rsp) 338190620Skib movw %gs,TF_GS(%rsp) 339190620Skib movw %es,TF_ES(%rsp) 340190620Skib movw %ds,TF_DS(%rsp) 341147677Speter testl $PSL_I,TF_RFLAGS(%rsp) 342147677Speter jz alltraps_pushregs_no_rdi 343147677Speter sti 344147677Speter jmp alltraps_pushregs_no_rdi 345147677Speter 346757Sdg/* 347114349Speter * Fast syscall entry point. We enter here with just our new %cs/%ss set, 348114349Speter * and the new privilige level. We are still running on the old user stack 349114349Speter * pointer. We have to juggle a few things around to find our stack etc. 350114349Speter * swapgs gives us access to our PCPU space only. 351220430Sjhb * 352220430Sjhb * We do not support invoking this from a custom %cs or %ss (e.g. using 353220430Sjhb * entries from an LDT). 3546380Ssos */ 355114349SpeterIDTVEC(fast_syscall) 356114928Speter swapgs 357114349Speter movq %rsp,PCPU(SCRATCH_RSP) 358122849Speter movq PCPU(RSP0),%rsp 359119924Speter /* Now emulate a trapframe. Make the 8 byte alignment odd for call. */ 360121103Speter subq $TF_SIZE,%rsp 361114349Speter /* defer TF_RSP till we have a spare register */ 362114349Speter movq %r11,TF_RFLAGS(%rsp) 363114928Speter movq %rcx,TF_RIP(%rsp) /* %rcx original value is in %r10 */ 364114928Speter movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ 365114928Speter movq %r11,TF_RSP(%rsp) /* user stack pointer */ 366190620Skib movw %fs,TF_FS(%rsp) 367190620Skib movw %gs,TF_GS(%rsp) 368190620Skib movw %es,TF_ES(%rsp) 369190620Skib movw %ds,TF_DS(%rsp) 370195486Skib movq PCPU(CURPCB),%r11 371216673Sjkim andl $~PCB_FULL_IRET,PCB_FLAGS(%r11) 372114928Speter sti 373114928Speter movq $KUDSEL,TF_SS(%rsp) 374114349Speter movq $KUCSEL,TF_CS(%rsp) 375114349Speter movq $2,TF_ERR(%rsp) 376114349Speter movq %rdi,TF_RDI(%rsp) /* arg 1 */ 377114349Speter movq %rsi,TF_RSI(%rsp) /* arg 2 */ 378114349Speter movq %rdx,TF_RDX(%rsp) /* arg 3 */ 379114349Speter movq %r10,TF_RCX(%rsp) /* arg 4 */ 380114349Speter movq %r8,TF_R8(%rsp) /* arg 5 */ 381114349Speter movq %r9,TF_R9(%rsp) /* arg 6 */ 382114349Speter movq %rax,TF_RAX(%rsp) /* syscall number */ 383114349Speter movq %rbx,TF_RBX(%rsp) /* C preserved */ 384114349Speter movq %rbp,TF_RBP(%rsp) /* C preserved */ 385114349Speter movq %r12,TF_R12(%rsp) /* C preserved */ 386114349Speter movq %r13,TF_R13(%rsp) /* C preserved */ 387114349Speter movq %r14,TF_R14(%rsp) /* C preserved */ 388114349Speter movq %r15,TF_R15(%rsp) /* C preserved */ 389190620Skib movl $TF_HASSEGS,TF_FLAGS(%rsp) 390209483Skib cld 391129623Sbde FAKE_MCOUNT(TF_RIP(%rsp)) 392225475Skib movq PCPU(CURTHREAD),%rdi 393225475Skib movq %rsp,TD_FRAME(%rdi) 394225475Skib movl TF_RFLAGS(%rsp),%esi 395225475Skib andl $PSL_T,%esi 396225475Skib call amd64_syscall 397220452Sjhb1: movq PCPU(CURPCB),%rax 398220460Skib /* Disable interrupts before testing PCB_FULL_IRET. */ 399220460Skib cli 400220431Sjhb testl $PCB_FULL_IRET,PCB_FLAGS(%rax) 401220452Sjhb jnz 3f 402220452Sjhb /* Check for and handle AST's on return to userland. */ 403220430Sjhb movq PCPU(CURTHREAD),%rax 404220430Sjhb testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 405225575Skib jne 2f 406225575Skib /* Restore preserved registers. */ 407129623Sbde MEXITCOUNT 408220430Sjhb movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ 409220430Sjhb movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ 410220430Sjhb movq TF_RDX(%rsp),%rdx /* return value 2 */ 411220430Sjhb movq TF_RAX(%rsp),%rax /* return value 1 */ 412220430Sjhb movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ 413220430Sjhb movq TF_RIP(%rsp),%rcx /* original %rip */ 414225575Skib movq TF_RSP(%rsp),%rsp /* user stack pointer */ 415220430Sjhb swapgs 416220430Sjhb sysretq 417225575Skib 418225575Skib2: /* AST scheduled. */ 419225575Skib sti 420225575Skib movq %rsp,%rdi 421225575Skib call ast 422225575Skib jmp 1b 423225575Skib 424220430Sjhb3: /* Requested full context restore, use doreti for that. */ 425220430Sjhb MEXITCOUNT 42673011Sjake jmp doreti 4276380Ssos 428114349Speter/* 429114349Speter * Here for CYA insurance, in case a "syscall" instruction gets 430114349Speter * issued from 32 bit compatability mode. MSR_CSTAR has to point 431114349Speter * to *something* if EFER_SCE is enabled. 432114349Speter */ 433114349SpeterIDTVEC(fast_syscall32) 434114349Speter sysret 435114349Speter 436149526Sjkoshy/* 437149526Sjkoshy * NMI handling is special. 438149526Sjkoshy * 439149526Sjkoshy * First, NMIs do not respect the state of the processor's RFLAGS.IF 440188065Sjkoshy * bit. The NMI handler may be entered at any time, including when 441188065Sjkoshy * the processor is in a critical section with RFLAGS.IF == 0. 442188065Sjkoshy * The processor's GS.base value could be invalid on entry to the 443188065Sjkoshy * handler. 444149526Sjkoshy * 445149526Sjkoshy * Second, the processor treats NMIs specially, blocking further NMIs 446188065Sjkoshy * until an 'iretq' instruction is executed. We thus need to execute 447188065Sjkoshy * the NMI handler with interrupts disabled, to prevent a nested interrupt 448188065Sjkoshy * from executing an 'iretq' instruction and inadvertently taking the 449188065Sjkoshy * processor out of NMI mode. 450174395Sjkoshy * 451188065Sjkoshy * Third, the NMI handler runs on its own stack (tss_ist2). The canonical 452188065Sjkoshy * GS.base value for the processor is stored just above the bottom of its 453188065Sjkoshy * NMI stack. For NMIs taken from kernel mode, the current value in 454188065Sjkoshy * the processor's GS.base is saved at entry to C-preserved register %r12, 455188065Sjkoshy * the canonical value for GS.base is then loaded into the processor, and 456188065Sjkoshy * the saved value is restored at exit time. For NMIs taken from user mode, 457188065Sjkoshy * the cheaper 'SWAPGS' instructions are used for swapping GS.base. 458149526Sjkoshy */ 459149526Sjkoshy 460149526SjkoshyIDTVEC(nmi) 461149526Sjkoshy subq $TF_RIP,%rsp 462190620Skib movl $(T_NMI),TF_TRAPNO(%rsp) 463149526Sjkoshy movq $0,TF_ADDR(%rsp) 464149526Sjkoshy movq $0,TF_ERR(%rsp) 465149526Sjkoshy movq %rdi,TF_RDI(%rsp) 466149526Sjkoshy movq %rsi,TF_RSI(%rsp) 467149526Sjkoshy movq %rdx,TF_RDX(%rsp) 468149526Sjkoshy movq %rcx,TF_RCX(%rsp) 469149526Sjkoshy movq %r8,TF_R8(%rsp) 470149526Sjkoshy movq %r9,TF_R9(%rsp) 471149526Sjkoshy movq %rax,TF_RAX(%rsp) 472149526Sjkoshy movq %rbx,TF_RBX(%rsp) 473149526Sjkoshy movq %rbp,TF_RBP(%rsp) 474149526Sjkoshy movq %r10,TF_R10(%rsp) 475149526Sjkoshy movq %r11,TF_R11(%rsp) 476149526Sjkoshy movq %r12,TF_R12(%rsp) 477149526Sjkoshy movq %r13,TF_R13(%rsp) 478149526Sjkoshy movq %r14,TF_R14(%rsp) 479149526Sjkoshy movq %r15,TF_R15(%rsp) 480190620Skib movw %fs,TF_FS(%rsp) 481190620Skib movw %gs,TF_GS(%rsp) 482190620Skib movw %es,TF_ES(%rsp) 483190620Skib movw %ds,TF_DS(%rsp) 484190620Skib movl $TF_HASSEGS,TF_FLAGS(%rsp) 485209483Skib cld 486149526Sjkoshy xorl %ebx,%ebx 487149526Sjkoshy testb $SEL_RPL_MASK,TF_CS(%rsp) 488188065Sjkoshy jnz nmi_fromuserspace 489188065Sjkoshy /* 490188065Sjkoshy * We've interrupted the kernel. Preserve GS.base in %r12. 491188065Sjkoshy */ 492149526Sjkoshy movl $MSR_GSBASE,%ecx 493149526Sjkoshy rdmsr 494188065Sjkoshy movq %rax,%r12 495188065Sjkoshy shlq $32,%rdx 496188065Sjkoshy orq %rdx,%r12 497188065Sjkoshy /* Retrieve and load the canonical value for GS.base. */ 498188065Sjkoshy movq TF_SIZE(%rsp),%rdx 499188065Sjkoshy movl %edx,%eax 500188065Sjkoshy shrq $32,%rdx 501188065Sjkoshy wrmsr 502188065Sjkoshy jmp nmi_calltrap 503188065Sjkoshynmi_fromuserspace: 504149526Sjkoshy incl %ebx 505149526Sjkoshy swapgs 506149526Sjkoshy/* Note: this label is also used by ddb and gdb: */ 507149526Sjkoshynmi_calltrap: 508149526Sjkoshy FAKE_MCOUNT(TF_RIP(%rsp)) 509207570Skib movq %rsp,%rdi 510149526Sjkoshy call trap 511149526Sjkoshy MEXITCOUNT 512174395Sjkoshy#ifdef HWPMC_HOOKS 513174395Sjkoshy /* 514188065Sjkoshy * Capture a userspace callchain if needed. 515251988Skib * 516188065Sjkoshy * - Check if the current trap was from user mode. 517188065Sjkoshy * - Check if the current thread is valid. 518188065Sjkoshy * - Check if the thread requires a user call chain to be 519188065Sjkoshy * captured. 520188065Sjkoshy * 521188065Sjkoshy * We are still in NMI mode at this point. 522174395Sjkoshy */ 523188065Sjkoshy testl %ebx,%ebx 524188065Sjkoshy jz nocallchain /* not from userspace */ 525188065Sjkoshy movq PCPU(CURTHREAD),%rax 526188065Sjkoshy orq %rax,%rax /* curthread present? */ 527174395Sjkoshy jz nocallchain 528174395Sjkoshy testl $TDP_CALLCHAIN,TD_PFLAGS(%rax) /* flagged for capture? */ 529174395Sjkoshy jz nocallchain 530174395Sjkoshy /* 531174395Sjkoshy * A user callchain is to be captured, so: 532174395Sjkoshy * - Move execution to the regular kernel stack, to allow for 533174395Sjkoshy * nested NMI interrupts. 534174395Sjkoshy * - Take the processor out of "NMI" mode by faking an "iret". 535174395Sjkoshy * - Enable interrupts, so that copyin() can work. 536174395Sjkoshy */ 537174395Sjkoshy movq %rsp,%rsi /* source stack pointer */ 538174395Sjkoshy movq $TF_SIZE,%rcx 539186076Sjkoshy movq PCPU(RSP0),%rdx 540186076Sjkoshy subq %rcx,%rdx 541186076Sjkoshy movq %rdx,%rdi /* destination stack pointer */ 542174395Sjkoshy 543174395Sjkoshy shrq $3,%rcx /* trap frame size in long words */ 544174395Sjkoshy cld 545174395Sjkoshy rep 546174395Sjkoshy movsq /* copy trapframe */ 547174395Sjkoshy 548174395Sjkoshy movl %ss,%eax 549174395Sjkoshy pushq %rax /* tf_ss */ 550186076Sjkoshy pushq %rdx /* tf_rsp (on kernel stack) */ 551174395Sjkoshy pushfq /* tf_rflags */ 552174395Sjkoshy movl %cs,%eax 553174395Sjkoshy pushq %rax /* tf_cs */ 554174395Sjkoshy pushq $outofnmi /* tf_rip */ 555174395Sjkoshy iretq 556174395Sjkoshyoutofnmi: 557174395Sjkoshy /* 558174395Sjkoshy * At this point the processor has exited NMI mode and is running 559174395Sjkoshy * with interrupts turned off on the normal kernel stack. 560174395Sjkoshy * 561251988Skib * If a pending NMI gets recognized at or after this point, it 562186037Sjkoshy * will cause a kernel callchain to be traced. 563186037Sjkoshy * 564186037Sjkoshy * We turn interrupts back on, and call the user callchain capture hook. 565174395Sjkoshy */ 566186037Sjkoshy movq pmc_hook,%rax 567186037Sjkoshy orq %rax,%rax 568186037Sjkoshy jz nocallchain 569186037Sjkoshy movq PCPU(CURTHREAD),%rdi /* thread */ 570186037Sjkoshy movq $PMC_FN_USER_CALLCHAIN,%rsi /* command */ 571186037Sjkoshy movq %rsp,%rdx /* frame */ 572174395Sjkoshy sti 573186037Sjkoshy call *%rax 574187221Skib cli 575174395Sjkoshynocallchain: 576174395Sjkoshy#endif 577149526Sjkoshy testl %ebx,%ebx 578190620Skib jnz doreti_exit 579251988Skibnmi_kernelexit: 580188065Sjkoshy /* 581188065Sjkoshy * Put back the preserved MSR_GSBASE value. 582188065Sjkoshy */ 583188065Sjkoshy movl $MSR_GSBASE,%ecx 584188065Sjkoshy movq %r12,%rdx 585188065Sjkoshy movl %edx,%eax 586188065Sjkoshy shrq $32,%rdx 587188065Sjkoshy wrmsr 588149526Sjkoshynmi_restoreregs: 589149526Sjkoshy movq TF_RDI(%rsp),%rdi 590149526Sjkoshy movq TF_RSI(%rsp),%rsi 591149526Sjkoshy movq TF_RDX(%rsp),%rdx 592149526Sjkoshy movq TF_RCX(%rsp),%rcx 593149526Sjkoshy movq TF_R8(%rsp),%r8 594149526Sjkoshy movq TF_R9(%rsp),%r9 595149526Sjkoshy movq TF_RAX(%rsp),%rax 596149526Sjkoshy movq TF_RBX(%rsp),%rbx 597149526Sjkoshy movq TF_RBP(%rsp),%rbp 598149526Sjkoshy movq TF_R10(%rsp),%r10 599149526Sjkoshy movq TF_R11(%rsp),%r11 600149526Sjkoshy movq TF_R12(%rsp),%r12 601149526Sjkoshy movq TF_R13(%rsp),%r13 602149526Sjkoshy movq TF_R14(%rsp),%r14 603149526Sjkoshy movq TF_R15(%rsp),%r15 604149526Sjkoshy addq $TF_RIP,%rsp 605207958Skib jmp doreti_iret 606149526Sjkoshy 60724691SpeterENTRY(fork_trampoline) 608207570Skib movq %r12,%rdi /* function */ 609207570Skib movq %rbx,%rsi /* arg1 */ 610207570Skib movq %rsp,%rdx /* trapframe pointer */ 61173011Sjake call fork_exit 612129623Sbde MEXITCOUNT 613114349Speter jmp doreti /* Handle any ASTs */ 61424691Speter 615129653Sbde/* 616129653Sbde * To efficiently implement classification of trap and interrupt handlers 617129653Sbde * for profiling, there must be only trap handlers between the labels btrap 618129653Sbde * and bintr, and only interrupt handlers between the labels bintr and 619129653Sbde * eintr. This is implemented (partly) by including files that contain 620129653Sbde * some of the handlers. Before including the files, set up a normal asm 621129653Sbde * environment so that the included files doen't need to know that they are 622129653Sbde * included. 623129653Sbde */ 624129653Sbde 625205014Snwhitehorn#ifdef COMPAT_FREEBSD32 62699703Sjulian .data 627129656Sbde .p2align 4 628129653Sbde .text 629129653Sbde SUPERALIGN_TEXT 63099703Sjulian 631129653Sbde#include <amd64/ia32/ia32_exception.S> 632129653Sbde#endif 633129653Sbde 634129653Sbde .data 635129656Sbde .p2align 4 636129653Sbde .text 637129653Sbde SUPERALIGN_TEXT 638129653SbdeMCOUNT_LABEL(bintr) 639129653Sbde 640129653Sbde#include <amd64/amd64/apic_vector.S> 641129653Sbde 642129653Sbde#ifdef DEV_ATPIC 643129653Sbde .data 644129656Sbde .p2align 4 645129653Sbde .text 646129653Sbde SUPERALIGN_TEXT 647129653Sbde 648204309Sattilio#include <amd64/amd64/atpic_vector.S> 649129653Sbde#endif 650129653Sbde 651129653Sbde .text 652129653SbdeMCOUNT_LABEL(eintr) 653129653Sbde 6543156Sbde/* 65599703Sjulian * void doreti(struct trapframe) 65699703Sjulian * 65799703Sjulian * Handle return from interrupts, traps and syscalls. 6583156Sbde */ 6593156Sbde .text 6603156Sbde SUPERALIGN_TEXT 66199703Sjulian .type doreti,@function 66299703Sjuliandoreti: 663129623Sbde FAKE_MCOUNT($bintr) /* init "from" bintr -> doreti */ 66499703Sjulian /* 665114349Speter * Check if ASTs can be handled now. 66699703Sjulian */ 667114349Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* are we returning to user mode? */ 66899703Sjulian jz doreti_exit /* can't handle ASTs now if not */ 66999703Sjulian 67099703Sjuliandoreti_ast: 67199703Sjulian /* 67299703Sjulian * Check for ASTs atomically with returning. Disabling CPU 673220452Sjhb * interrupts provides sufficient locking even in the SMP case, 67499703Sjulian * since we will be informed of any new ASTs by an IPI. 67599703Sjulian */ 67699703Sjulian cli 677114349Speter movq PCPU(CURTHREAD),%rax 678114349Speter testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) 67999703Sjulian je doreti_exit 68099703Sjulian sti 681207570Skib movq %rsp,%rdi /* pass a pointer to the trapframe */ 68299703Sjulian call ast 68399703Sjulian jmp doreti_ast 68499703Sjulian 68599703Sjulian /* 68699703Sjulian * doreti_exit: pop registers, iret. 68799703Sjulian * 68899703Sjulian * The segment register pop is a special case, since it may 68999703Sjulian * fault if (for example) a sigreturn specifies bad segment 69099703Sjulian * registers. The fault is handled in trap.c. 69199703Sjulian */ 69299703Sjuliandoreti_exit: 693129623Sbde MEXITCOUNT 694220452Sjhb movq PCPU(CURPCB),%r8 695190620Skib 696190620Skib /* 697190620Skib * Do not reload segment registers for kernel. 698190620Skib * Since we do not reload segments registers with sane 699190620Skib * values on kernel entry, descriptors referenced by 700207570Skib * segments registers might be not valid. This is fatal 701207570Skib * for user mode, but is not a problem for the kernel. 702190620Skib */ 703190620Skib testb $SEL_RPL_MASK,TF_CS(%rsp) 704190620Skib jz ld_regs 705216673Sjkim testl $PCB_FULL_IRET,PCB_FLAGS(%r8) 706216634Sjkim jz ld_regs 707190620Skib testl $TF_HASSEGS,TF_FLAGS(%rsp) 708190620Skib je set_segs 709190620Skib 710190620Skibdo_segs: 711190620Skib /* Restore %fs and fsbase */ 712190620Skib movw TF_FS(%rsp),%ax 713190620Skib .globl ld_fs 714207570Skibld_fs: 715207570Skib movw %ax,%fs 716190620Skib cmpw $KUF32SEL,%ax 717190620Skib jne 1f 718190620Skib movl $MSR_FSBASE,%ecx 719190620Skib movl PCB_FSBASE(%r8),%eax 720190620Skib movl PCB_FSBASE+4(%r8),%edx 721206459Skib .globl ld_fsbase 722207570Skibld_fsbase: 723207570Skib wrmsr 724190620Skib1: 725190620Skib /* Restore %gs and gsbase */ 726190620Skib movw TF_GS(%rsp),%si 727190620Skib pushfq 728190620Skib cli 729190620Skib movl $MSR_GSBASE,%ecx 730267083Skib /* Save current kernel %gs base into %r12d:%r13d */ 731190620Skib rdmsr 732267083Skib movl %eax,%r12d 733267083Skib movl %edx,%r13d 734190620Skib .globl ld_gs 735207570Skibld_gs: 736207570Skib movw %si,%gs 737267083Skib /* Save user %gs base into %r14d:%r15d */ 738267083Skib rdmsr 739267083Skib movl %eax,%r14d 740267083Skib movl %edx,%r15d 741267083Skib /* Restore kernel %gs base */ 742267083Skib movl %r12d,%eax 743267083Skib movl %r13d,%edx 744190620Skib wrmsr 745190620Skib popfq 746267083Skib /* 747267083Skib * Restore user %gs base, either from PCB if used for TLS, or 748267083Skib * from the previously saved msr read. 749267083Skib */ 750267083Skib movl $MSR_KGSBASE,%ecx 751190620Skib cmpw $KUG32SEL,%si 752190620Skib jne 1f 753190620Skib movl PCB_GSBASE(%r8),%eax 754190620Skib movl PCB_GSBASE+4(%r8),%edx 755267083Skib jmp ld_gsbase 756267083Skib1: 757267083Skib movl %r14d,%eax 758267083Skib movl %r15d,%edx 759206459Skib .globl ld_gsbase 760207570Skibld_gsbase: 761267083Skib wrmsr /* May trap if non-canonical, but only for TLS. */ 762207570Skib .globl ld_es 763207570Skibld_es: 764207570Skib movw TF_ES(%rsp),%es 765190620Skib .globl ld_ds 766207570Skibld_ds: 767207570Skib movw TF_DS(%rsp),%ds 768207570Skibld_regs: 769207570Skib movq TF_RDI(%rsp),%rdi 770114349Speter movq TF_RSI(%rsp),%rsi 771114349Speter movq TF_RDX(%rsp),%rdx 772114349Speter movq TF_RCX(%rsp),%rcx 773114349Speter movq TF_R8(%rsp),%r8 774114349Speter movq TF_R9(%rsp),%r9 775114349Speter movq TF_RAX(%rsp),%rax 776114349Speter movq TF_RBX(%rsp),%rbx 777114349Speter movq TF_RBP(%rsp),%rbp 778114349Speter movq TF_R10(%rsp),%r10 779114349Speter movq TF_R11(%rsp),%r11 780114349Speter movq TF_R12(%rsp),%r12 781114349Speter movq TF_R13(%rsp),%r13 782114349Speter movq TF_R14(%rsp),%r14 783114349Speter movq TF_R15(%rsp),%r15 784114928Speter testb $SEL_RPL_MASK,TF_CS(%rsp) /* Did we come from kernel? */ 785114928Speter jz 1f /* keep running with kernel GS.base */ 786114928Speter cli 787114928Speter swapgs 788207570Skib1: 789207570Skib addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ 79099703Sjulian .globl doreti_iret 79199703Sjuliandoreti_iret: 792114349Speter iretq 79399703Sjulian 794190620Skibset_segs: 795190620Skib movw $KUDSEL,%ax 796190620Skib movw %ax,TF_DS(%rsp) 797190620Skib movw %ax,TF_ES(%rsp) 798190620Skib movw $KUF32SEL,TF_FS(%rsp) 799190620Skib movw $KUG32SEL,TF_GS(%rsp) 800190620Skib jmp do_segs 801190620Skib 802114928Speter /* 803146507Speter * doreti_iret_fault. Alternative return code for 80499703Sjulian * the case where we get a fault in the doreti_exit code 805146507Speter * above. trap() (amd64/amd64/trap.c) catches this specific 80699703Sjulian * case, sends the process a signal and continues in the 80799703Sjulian * corresponding place in the code below. 80899703Sjulian */ 80999703Sjulian ALIGN_TEXT 81099703Sjulian .globl doreti_iret_fault 81199703Sjuliandoreti_iret_fault: 812114349Speter subq $TF_RIP,%rsp /* space including tf_err, tf_trapno */ 813181823Skib testl $PSL_I,TF_RFLAGS(%rsp) 814181823Skib jz 1f 815114928Speter sti 816207570Skib1: 817207570Skib movw %fs,TF_FS(%rsp) 818190620Skib movw %gs,TF_GS(%rsp) 819190620Skib movw %es,TF_ES(%rsp) 820190620Skib movw %ds,TF_DS(%rsp) 821190620Skib movl $TF_HASSEGS,TF_FLAGS(%rsp) 822190620Skib movq %rdi,TF_RDI(%rsp) 823114349Speter movq %rsi,TF_RSI(%rsp) 824114349Speter movq %rdx,TF_RDX(%rsp) 825114349Speter movq %rcx,TF_RCX(%rsp) 826114349Speter movq %r8,TF_R8(%rsp) 827114349Speter movq %r9,TF_R9(%rsp) 828114349Speter movq %rax,TF_RAX(%rsp) 829114349Speter movq %rbx,TF_RBX(%rsp) 830114349Speter movq %rbp,TF_RBP(%rsp) 831114349Speter movq %r10,TF_R10(%rsp) 832114349Speter movq %r11,TF_R11(%rsp) 833114349Speter movq %r12,TF_R12(%rsp) 834114349Speter movq %r13,TF_R13(%rsp) 835114349Speter movq %r14,TF_R14(%rsp) 836114349Speter movq %r15,TF_R15(%rsp) 837190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 838114349Speter movq $0,TF_ERR(%rsp) /* XXX should be the error code */ 839146507Speter movq $0,TF_ADDR(%rsp) 840146461Speter FAKE_MCOUNT(TF_RIP(%rsp)) 841146461Speter jmp calltrap 842190620Skib 843190620Skib ALIGN_TEXT 844190620Skib .globl ds_load_fault 845190620Skibds_load_fault: 846190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 847251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 848251033Skib jz 1f 849251033Skib sti 850251033Skib1: 851207570Skib movq %rsp,%rdi 852195535Skib call trap 853190620Skib movw $KUDSEL,TF_DS(%rsp) 854195535Skib jmp doreti 855190620Skib 856190620Skib ALIGN_TEXT 857190620Skib .globl es_load_fault 858190620Skibes_load_fault: 859190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 860251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 861251033Skib jz 1f 862251033Skib sti 863251033Skib1: 864207570Skib movq %rsp,%rdi 865195535Skib call trap 866190620Skib movw $KUDSEL,TF_ES(%rsp) 867195535Skib jmp doreti 868190620Skib 869190620Skib ALIGN_TEXT 870190620Skib .globl fs_load_fault 871190620Skibfs_load_fault: 872251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 873251033Skib jz 1f 874251033Skib sti 875251033Skib1: 876190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 877207570Skib movq %rsp,%rdi 878195535Skib call trap 879190620Skib movw $KUF32SEL,TF_FS(%rsp) 880195535Skib jmp doreti 881190620Skib 882190620Skib ALIGN_TEXT 883190620Skib .globl gs_load_fault 884190620Skibgs_load_fault: 885190620Skib popfq 886190620Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 887251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 888251033Skib jz 1f 889251033Skib sti 890251033Skib1: 891207570Skib movq %rsp,%rdi 892195535Skib call trap 893190620Skib movw $KUG32SEL,TF_GS(%rsp) 894195535Skib jmp doreti 895206459Skib 896206459Skib ALIGN_TEXT 897206459Skib .globl fsbase_load_fault 898206459Skibfsbase_load_fault: 899206459Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 900251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 901251033Skib jz 1f 902251033Skib sti 903251033Skib1: 904207570Skib movq %rsp,%rdi 905206459Skib call trap 906206459Skib movq PCPU(CURTHREAD),%r8 907206459Skib movq TD_PCB(%r8),%r8 908206459Skib movq $0,PCB_FSBASE(%r8) 909206459Skib jmp doreti 910206459Skib 911206459Skib ALIGN_TEXT 912206459Skib .globl gsbase_load_fault 913206459Skibgsbase_load_fault: 914206459Skib movl $T_PROTFLT,TF_TRAPNO(%rsp) 915251033Skib testl $PSL_I,TF_RFLAGS(%rsp) 916251033Skib jz 1f 917251033Skib sti 918251033Skib1: 919207570Skib movq %rsp,%rdi 920206459Skib call trap 921206459Skib movq PCPU(CURTHREAD),%r8 922206459Skib movq TD_PCB(%r8),%r8 923206459Skib movq $0,PCB_GSBASE(%r8) 924206459Skib jmp doreti 925206459Skib 926174395Sjkoshy#ifdef HWPMC_HOOKS 927174395Sjkoshy ENTRY(end_exceptions) 928174395Sjkoshy#endif 929