1/* 2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29* @OSF_COPYRIGHT@ 30*/ 31/* 32* Mach Operating System 33* Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University 34* All Rights Reserved. 35* 36* Permission to use, copy, modify and distribute this software and its 37* documentation is hereby granted, provided that both the copyright 38* notice and this permission notice appear in all copies of the 39* software, derivative works or modified versions, and any portions 40* thereof, and that both notices appear in supporting documentation. 41* 42* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45* 46* Carnegie Mellon requests users of this software to return to 47* 48* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49* School of Computer Science 50* Carnegie Mellon University 51* Pittsburgh PA 15213-3890 52* 53* any improvements or extensions that they make and grant Carnegie Mellon 54* the rights to redistribute these changes. 55*/ 56/* 57*/ 58 59/* 60* Hardware trap/fault handler. 61 */ 62 63#include <mach_kdp.h> 64#include <mach_ldebug.h> 65 66#include <types.h> 67#include <i386/eflags.h> 68#include <i386/trap.h> 69#include <i386/pmap.h> 70#include <i386/fpu.h> 71#include <i386/misc_protos.h> /* panic_io_port_read() */ 72#include <i386/lapic.h> 73 74#include <mach/exception.h> 75#include <mach/kern_return.h> 76#include <mach/vm_param.h> 77#include <mach/i386/thread_status.h> 78 79#include <vm/vm_kern.h> 80#include <vm/vm_fault.h> 81 82#include <kern/kern_types.h> 83#include <kern/processor.h> 84#include <kern/thread.h> 85#include <kern/task.h> 86#include <kern/sched.h> 87#include <kern/sched_prim.h> 88#include <kern/exception.h> 89#include <kern/spl.h> 90#include <kern/misc_protos.h> 91#include <kern/debug.h> 92 93#include <sys/kdebug.h> 94 95#include <string.h> 96 97#include <i386/postcode.h> 98#include <i386/mp_desc.h> 99#include <i386/proc_reg.h> 100#if CONFIG_MCA 101#include <i386/machine_check.h> 102#endif 103#include <mach/i386/syscall_sw.h> 104 105#include <libkern/OSDebug.h> 106#include <i386/cpu_threads.h> 107#include <machine/pal_routines.h> 108 109extern void throttle_lowpri_io(int); 110extern void kprint_state(x86_saved_state64_t *saved_state); 111 112/* 113 * Forward declarations 114 */ 115static void user_page_fault_continue(kern_return_t kret); 116#ifdef __i386__ 117static void panic_trap(x86_saved_state32_t *saved_state); 118static void set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip); 119extern void panic_64(x86_saved_state_t *, int, const char *, boolean_t); 120#else 121static void panic_trap(x86_saved_state64_t *saved_state); 122static void set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip); 123#endif 124 125volatile perfCallback perfTrapHook = NULL; /* Pointer to CHUD trap hook routine */ 126 127#if CONFIG_DTRACE 128/* See <rdar://problem/4613924> */ 129perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ 130 131extern boolean_t dtrace_tally_fault(user_addr_t); 132#endif 133 134extern boolean_t pmap_smep_enabled; 135 136void 137thread_syscall_return( 138 kern_return_t ret) 139{ 140 thread_t thr_act = current_thread(); 141 boolean_t is_mach; 142 int code; 143 144 pal_register_cache_state(thr_act, DIRTY); 145 146 if (thread_is_64bit(thr_act)) { 147 x86_saved_state64_t *regs; 148 149 regs = USER_REGS64(thr_act); 150 151 code = (int) (regs->rax & SYSCALL_NUMBER_MASK); 152 is_mach = (regs->rax & SYSCALL_CLASS_MASK) 153 == (SYSCALL_CLASS_MACH << SYSCALL_CLASS_SHIFT); 154 if (kdebug_enable && is_mach) { 155 /* Mach trap */ 156 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 157 MACHDBG_CODE(DBG_MACH_EXCP_SC,code)|DBG_FUNC_END, 158 ret, 0, 0, 0, 0); 159 } 160 regs->rax = ret; 161#if DEBUG 162 if (is_mach) 163 DEBUG_KPRINT_SYSCALL_MACH( 164 "thread_syscall_return: 64-bit mach ret=%u\n", 165 ret); 166 else 167 DEBUG_KPRINT_SYSCALL_UNIX( 168 "thread_syscall_return: 64-bit unix ret=%u\n", 169 ret); 170#endif 171 } else { 172 x86_saved_state32_t *regs; 173 174 regs = USER_REGS32(thr_act); 175 176 code = ((int) regs->eax); 177 is_mach = (code < 0); 178 if (kdebug_enable && is_mach) { 179 /* Mach trap */ 180 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 181 MACHDBG_CODE(DBG_MACH_EXCP_SC,-code)|DBG_FUNC_END, 182 ret, 0, 0, 0, 0); 183 } 184 regs->eax = ret; 185#if DEBUG 186 if (is_mach) 187 DEBUG_KPRINT_SYSCALL_MACH( 188 "thread_syscall_return: 32-bit mach ret=%u\n", 189 ret); 190 else 191 DEBUG_KPRINT_SYSCALL_UNIX( 192 "thread_syscall_return: 32-bit unix ret=%u\n", 193 ret); 194#endif 195 } 196 throttle_lowpri_io(TRUE); 197 198 thread_exception_return(); 199 /*NOTREACHED*/ 200} 201 202 203static inline void 204user_page_fault_continue( 205 kern_return_t kr) 206{ 207 thread_t thread = current_thread(); 208 user_addr_t vaddr; 209 210 if (thread_is_64bit(thread)) { 211 x86_saved_state64_t *uregs; 212 213 uregs = USER_REGS64(thread); 214 215 vaddr = (user_addr_t)uregs->cr2; 216 } else { 217 x86_saved_state32_t *uregs; 218 219 uregs = USER_REGS32(thread); 220 221 vaddr = uregs->cr2; 222 } 223 224 225 /* PAL debug hook */ 226 pal_dbg_page_fault( thread, vaddr, kr ); 227 228 i386_exception(EXC_BAD_ACCESS, kr, vaddr); 229 /*NOTREACHED*/ 230} 231 232/* 233 * Fault recovery in copyin/copyout routines. 234 */ 235struct recovery { 236 uintptr_t fault_addr; 237 uintptr_t recover_addr; 238}; 239 240extern struct recovery recover_table[]; 241extern struct recovery recover_table_end[]; 242 243const char * trap_type[] = {TRAP_NAMES}; 244unsigned TRAP_TYPES = sizeof(trap_type)/sizeof(trap_type[0]); 245 246extern void PE_incoming_interrupt(int interrupt); 247 248#if defined(__x86_64__) && DEBUG 249void 250kprint_state(x86_saved_state64_t *saved_state) 251{ 252 kprintf("current_cpu_datap() 0x%lx\n", (uintptr_t)current_cpu_datap()); 253 kprintf("Current GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_GS_BASE)); 254 kprintf("Kernel GS base MSR 0x%llx\n", rdmsr64(MSR_IA32_KERNEL_GS_BASE)); 255 kprintf("state at 0x%lx:\n", (uintptr_t) saved_state); 256 257 kprintf(" rdi 0x%llx\n", saved_state->rdi); 258 kprintf(" rsi 0x%llx\n", saved_state->rsi); 259 kprintf(" rdx 0x%llx\n", saved_state->rdx); 260 kprintf(" r10 0x%llx\n", saved_state->r10); 261 kprintf(" r8 0x%llx\n", saved_state->r8); 262 kprintf(" r9 0x%llx\n", saved_state->r9); 263 kprintf(" v_arg6 0x%llx\n", saved_state->v_arg6); 264 kprintf(" v_arg7 0x%llx\n", saved_state->v_arg7); 265 kprintf(" v_arg8 0x%llx\n", saved_state->v_arg8); 266 267 kprintf(" cr2 0x%llx\n", saved_state->cr2); 268 kprintf("real cr2 0x%lx\n", get_cr2()); 269 kprintf(" r15 0x%llx\n", saved_state->r15); 270 kprintf(" r14 0x%llx\n", saved_state->r14); 271 kprintf(" r13 0x%llx\n", saved_state->r13); 272 kprintf(" r12 0x%llx\n", saved_state->r12); 273 kprintf(" r11 0x%llx\n", saved_state->r11); 274 kprintf(" rbp 0x%llx\n", saved_state->rbp); 275 kprintf(" rbx 0x%llx\n", saved_state->rbx); 276 kprintf(" rcx 0x%llx\n", saved_state->rcx); 277 kprintf(" rax 0x%llx\n", saved_state->rax); 278 279 kprintf(" gs 0x%x\n", saved_state->gs); 280 kprintf(" fs 0x%x\n", saved_state->fs); 281 282 kprintf(" isf.trapno 0x%x\n", saved_state->isf.trapno); 283 kprintf(" isf._pad 0x%x\n", saved_state->isf._pad); 284 kprintf(" isf.trapfn 0x%llx\n", saved_state->isf.trapfn); 285 kprintf(" isf.err 0x%llx\n", saved_state->isf.err); 286 kprintf(" isf.rip 0x%llx\n", saved_state->isf.rip); 287 kprintf(" isf.cs 0x%llx\n", saved_state->isf.cs); 288 kprintf(" isf.rflags 0x%llx\n", saved_state->isf.rflags); 289 kprintf(" isf.rsp 0x%llx\n", saved_state->isf.rsp); 290 kprintf(" isf.ss 0x%llx\n", saved_state->isf.ss); 291} 292#endif 293 294 295/* 296 * Non-zero indicates latency assert is enabled and capped at valued 297 * absolute time units. 298 */ 299 300uint64_t interrupt_latency_cap = 0; 301boolean_t ilat_assert = FALSE; 302 303void 304interrupt_latency_tracker_setup(void) { 305 uint32_t ilat_cap_us; 306 if (PE_parse_boot_argn("interrupt_latency_cap_us", &ilat_cap_us, sizeof(ilat_cap_us))) { 307 interrupt_latency_cap = ilat_cap_us * NSEC_PER_USEC; 308 nanoseconds_to_absolutetime(interrupt_latency_cap, &interrupt_latency_cap); 309 } else { 310 interrupt_latency_cap = LockTimeOut; 311 } 312 PE_parse_boot_argn("-interrupt_latency_assert_enable", &ilat_assert, sizeof(ilat_assert)); 313} 314 315void interrupt_reset_latency_stats(void) { 316 uint32_t i; 317 for (i = 0; i < real_ncpus; i++) { 318 cpu_data_ptr[i]->cpu_max_observed_int_latency = 319 cpu_data_ptr[i]->cpu_max_observed_int_latency_vector = 0; 320 } 321} 322 323void interrupt_populate_latency_stats(char *buf, unsigned bufsize) { 324 uint32_t i, tcpu = ~0; 325 uint64_t cur_max = 0; 326 327 for (i = 0; i < real_ncpus; i++) { 328 if (cur_max < cpu_data_ptr[i]->cpu_max_observed_int_latency) { 329 cur_max = cpu_data_ptr[i]->cpu_max_observed_int_latency; 330 tcpu = i; 331 } 332 } 333 334 if (tcpu < real_ncpus) 335 snprintf(buf, bufsize, "0x%x 0x%x 0x%llx", tcpu, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency_vector, cpu_data_ptr[tcpu]->cpu_max_observed_int_latency); 336} 337 338/* 339 * Handle interrupts: 340 * - local APIC interrupts (IPIs, timers, etc) are handled by the kernel, 341 * - device interrupts go to the platform expert. 342 */ 343void 344interrupt(x86_saved_state_t *state) 345{ 346 uint64_t rip; 347 uint64_t rsp; 348 int interrupt_num; 349 boolean_t user_mode = FALSE; 350 int ipl; 351 int cnum = cpu_number(); 352 int itype = 0; 353 354 if (is_saved_state64(state) == TRUE) { 355 x86_saved_state64_t *state64; 356 357 state64 = saved_state64(state); 358 rip = state64->isf.rip; 359 rsp = state64->isf.rsp; 360 interrupt_num = state64->isf.trapno; 361#ifdef __x86_64__ 362 if(state64->isf.cs & 0x03) 363#endif 364 user_mode = TRUE; 365 } else { 366 x86_saved_state32_t *state32; 367 368 state32 = saved_state32(state); 369 if (state32->cs & 0x03) 370 user_mode = TRUE; 371 rip = state32->eip; 372 rsp = state32->uesp; 373 interrupt_num = state32->trapno; 374 } 375 376 if (cpu_data_ptr[cnum]->lcpu.package->num_idle == topoParms.nLThreadsPerPackage) 377 cpu_data_ptr[cnum]->cpu_hwIntpexits[interrupt_num]++; 378 379 if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_INTERPROCESSOR_INTERRUPT)) 380 itype = 1; 381 else if (interrupt_num == (LAPIC_DEFAULT_INTERRUPT_BASE + LAPIC_TIMER_INTERRUPT)) 382 itype = 2; 383 else 384 itype = 3; 385 386 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 387 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, 388 interrupt_num, 389 (user_mode ? rip : VM_KERNEL_UNSLIDE(rip)), 390 user_mode, itype, 0); 391 392 SCHED_STATS_INTERRUPT(current_processor()); 393 394 ipl = get_preemption_level(); 395 396 /* 397 * Handle local APIC interrupts 398 * else call platform expert for devices. 399 */ 400 if (!lapic_interrupt(interrupt_num, state)) 401 PE_incoming_interrupt(interrupt_num); 402 403 if (__improbable(get_preemption_level() != ipl)) { 404 panic("Preemption level altered by interrupt vector 0x%x: initial 0x%x, final: 0x%x\n", interrupt_num, ipl, get_preemption_level()); 405 } 406 407 408 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 409 MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END, 410 interrupt_num, 0, 0, 0, 0); 411 412 if (cpu_data_ptr[cnum]->cpu_nested_istack) { 413 cpu_data_ptr[cnum]->cpu_nested_istack_events++; 414 } 415 else { 416 uint64_t int_latency = mach_absolute_time() - cpu_data_ptr[cnum]->cpu_int_event_time; 417 if (ilat_assert && (int_latency > interrupt_latency_cap) && !machine_timeout_suspended()) { 418 panic("Interrupt vector 0x%x exceeded interrupt latency threshold, 0x%llx absolute time delta, prior signals: 0x%x, current signals: 0x%x", interrupt_num, int_latency, cpu_data_ptr[cnum]->cpu_prior_signals, cpu_data_ptr[cnum]->cpu_signals); 419 } 420 if (int_latency > cpu_data_ptr[cnum]->cpu_max_observed_int_latency) { 421 cpu_data_ptr[cnum]->cpu_max_observed_int_latency = int_latency; 422 cpu_data_ptr[cnum]->cpu_max_observed_int_latency_vector = interrupt_num; 423 } 424 } 425 426 /* 427 * Having serviced the interrupt first, look at the interrupted stack depth. 428 */ 429 if (!user_mode) { 430 uint64_t depth = cpu_data_ptr[cnum]->cpu_kernel_stack 431 + sizeof(struct x86_kernel_state) 432 + sizeof(struct i386_exception_link *) 433 - rsp; 434 if (depth > kernel_stack_depth_max) { 435 kernel_stack_depth_max = (vm_offset_t)depth; 436 KERNEL_DEBUG_CONSTANT( 437 MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_DEPTH), 438 (long) depth, (long) VM_KERNEL_UNSLIDE(rip), 0, 0, 0); 439 } 440 } 441} 442 443static inline void 444reset_dr7(void) 445{ 446 long dr7 = 0x400; /* magic dr7 reset value; 32 bit on i386, 64 bit on x86_64 */ 447 __asm__ volatile("mov %0,%%dr7" : : "r" (dr7)); 448} 449#if MACH_KDP 450unsigned kdp_has_active_watchpoints = 0; 451#define NO_WATCHPOINTS (!kdp_has_active_watchpoints) 452#else 453#define NO_WATCHPOINTS 1 454#endif 455/* 456 * Trap from kernel mode. Only page-fault errors are recoverable, 457 * and then only in special circumstances. All other errors are 458 * fatal. Return value indicates if trap was handled. 459 */ 460 461void 462kernel_trap( 463 x86_saved_state_t *state, 464 uintptr_t *lo_spp) 465{ 466#ifdef __i386__ 467 x86_saved_state32_t *saved_state; 468#else 469 x86_saved_state64_t *saved_state; 470#endif 471 int code; 472 user_addr_t vaddr; 473 int type; 474 vm_map_t map = 0; /* protected by T_PAGE_FAULT */ 475 kern_return_t result = KERN_FAILURE; 476 thread_t thread; 477 ast_t *myast; 478 boolean_t intr; 479 vm_prot_t prot; 480 struct recovery *rp; 481 vm_offset_t kern_ip; 482#if NCOPY_WINDOWS > 0 483 int fault_in_copy_window = -1; 484#endif 485 int is_user = 0; 486 487 thread = current_thread(); 488 489#ifdef __i386__ 490 if (__improbable(is_saved_state64(state))) { 491 panic_64(state, 0, "Kernel trap with 64-bit state", FALSE); 492 } 493 494 saved_state = saved_state32(state); 495 496 /* Record cpu where state was captured (trampolines don't set this) */ 497 saved_state->cpu = cpu_number(); 498 499 vaddr = (user_addr_t)saved_state->cr2; 500 type = saved_state->trapno; 501 code = saved_state->err & 0xffff; 502 intr = (saved_state->efl & EFL_IF) != 0; /* state of ints at trap */ 503 kern_ip = (vm_offset_t)saved_state->eip; 504#else 505 if (__improbable(is_saved_state32(state))) 506 panic("kernel_trap(%p) with 32-bit state", state); 507 saved_state = saved_state64(state); 508 509 /* Record cpu where state was captured */ 510 saved_state->isf.cpu = cpu_number(); 511 512 vaddr = (user_addr_t)saved_state->cr2; 513 type = saved_state->isf.trapno; 514 code = (int)(saved_state->isf.err & 0xffff); 515 intr = (saved_state->isf.rflags & EFL_IF) != 0; /* state of ints at trap */ 516 kern_ip = (vm_offset_t)saved_state->isf.rip; 517#endif 518 519 myast = ast_pending(); 520 521 perfASTCallback astfn = perfASTHook; 522 if (__improbable(astfn != NULL)) { 523 if (*myast & AST_CHUD_ALL) 524 astfn(AST_CHUD_ALL, myast); 525 } else 526 *myast &= ~AST_CHUD_ALL; 527 528 /* 529 * Is there a hook? 530 */ 531 perfCallback fn = perfTrapHook; 532 if (__improbable(fn != NULL)) { 533 if (fn(type, NULL, 0, 0) == KERN_SUCCESS) { 534 /* 535 * If it succeeds, we are done... 536 */ 537 return; 538 } 539 } 540 541#if CONFIG_DTRACE 542 if (__improbable(tempDTraceTrapHook != NULL)) { 543 if (tempDTraceTrapHook(type, state, lo_spp, 0) == KERN_SUCCESS) { 544 /* 545 * If it succeeds, we are done... 546 */ 547 return; 548 } 549 } 550#endif /* CONFIG_DTRACE */ 551 552 /* 553 * we come here with interrupts off as we don't want to recurse 554 * on preemption below. but we do want to re-enable interrupts 555 * as soon we possibly can to hold latency down 556 */ 557 if (__improbable(T_PREEMPT == type)) { 558 ast_taken(AST_PREEMPTION, FALSE); 559 560 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 561 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, 562 0, 0, 0, VM_KERNEL_UNSLIDE(kern_ip), 0); 563 return; 564 } 565 566 if (T_PAGE_FAULT == type) { 567 /* 568 * assume we're faulting in the kernel map 569 */ 570 map = kernel_map; 571 572 if (__probable(thread != THREAD_NULL && thread->map != kernel_map)) { 573#if NCOPY_WINDOWS > 0 574 vm_offset_t copy_window_base; 575 vm_offset_t kvaddr; 576 int window_index; 577 578 kvaddr = (vm_offset_t)vaddr; 579 /* 580 * must determine if fault occurred in 581 * the copy window while pre-emption is 582 * disabled for this processor so that 583 * we only need to look at the window 584 * associated with this processor 585 */ 586 copy_window_base = current_cpu_datap()->cpu_copywindow_base; 587 588 if (kvaddr >= copy_window_base && kvaddr < (copy_window_base + (NBPDE * NCOPY_WINDOWS)) ) { 589 590 window_index = (int)((kvaddr - copy_window_base) / NBPDE); 591 592 if (thread->machine.copy_window[window_index].user_base != (user_addr_t)-1) { 593 594 kvaddr -= (copy_window_base + (NBPDE * window_index)); 595 vaddr = thread->machine.copy_window[window_index].user_base + kvaddr; 596 597 map = thread->map; 598 fault_in_copy_window = window_index; 599 } 600 is_user = -1; 601 } 602#else 603 if (__probable(vaddr < VM_MAX_USER_PAGE_ADDRESS)) { 604 /* fault occurred in userspace */ 605 map = thread->map; 606 is_user = -1; 607 608 /* Intercept a potential Supervisor Mode Execute 609 * Protection fault. These criteria identify 610 * both NX faults and SMEP faults, but both 611 * are fatal. We avoid checking PTEs (racy). 612 * (The VM could just redrive a SMEP fault, hence 613 * the intercept). 614 */ 615 if (__improbable((code == (T_PF_PROT | T_PF_EXECUTE)) && (pmap_smep_enabled) && (saved_state->isf.rip == vaddr))) { 616 goto debugger_entry; 617 } 618 619 /* 620 * If we're not sharing cr3 with the user 621 * and we faulted in copyio, 622 * then switch cr3 here and dismiss the fault. 623 */ 624 if (no_shared_cr3 && 625 (thread->machine.specFlags&CopyIOActive) && 626 map->pmap->pm_cr3 != get_cr3_base()) { 627 pmap_assert(current_cpu_datap()->cpu_pmap_pcid_enabled == FALSE); 628 set_cr3_raw(map->pmap->pm_cr3); 629 return; 630 } 631 } 632#endif 633 } 634 } 635 user_addr_t kd_vaddr = is_user ? vaddr : VM_KERNEL_UNSLIDE(vaddr); 636 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 637 (MACHDBG_CODE(DBG_MACH_EXCP_KTRAP_x86, type)) | DBG_FUNC_NONE, 638 (unsigned)(kd_vaddr >> 32), (unsigned)kd_vaddr, is_user, 639 VM_KERNEL_UNSLIDE(kern_ip), 0); 640 641 642 (void) ml_set_interrupts_enabled(intr); 643 644 switch (type) { 645 646 case T_NO_FPU: 647 fpnoextflt(); 648 return; 649 650 case T_FPU_FAULT: 651 fpextovrflt(); 652 return; 653 654 case T_FLOATING_POINT_ERROR: 655 fpexterrflt(); 656 return; 657 658 case T_SSE_FLOAT_ERROR: 659 fpSSEexterrflt(); 660 return; 661 case T_DEBUG: 662#ifdef __i386__ 663 if ((saved_state->efl & EFL_TF) == 0 && NO_WATCHPOINTS) 664#else 665 if ((saved_state->isf.rflags & EFL_TF) == 0 && NO_WATCHPOINTS) 666#endif 667 { 668 /* We've somehow encountered a debug 669 * register match that does not belong 670 * to the kernel debugger. 671 * This isn't supposed to happen. 672 */ 673 reset_dr7(); 674 return; 675 } 676 goto debugger_entry; 677#ifdef __x86_64__ 678 case T_INT3: 679 goto debugger_entry; 680#endif 681 case T_PAGE_FAULT: 682 683#if CONFIG_DTRACE 684 if (thread != THREAD_NULL && thread->options & TH_OPT_DTRACE) { /* Executing under dtrace_probe? */ 685 if (dtrace_tally_fault(vaddr)) { /* Should a fault under dtrace be ignored? */ 686 /* 687 * DTrace has "anticipated" the possibility of this fault, and has 688 * established the suitable recovery state. Drop down now into the 689 * recovery handling code in "case T_GENERAL_PROTECTION:". 690 */ 691 goto FALL_THROUGH; 692 } 693 } 694#endif /* CONFIG_DTRACE */ 695 696 prot = VM_PROT_READ; 697 698 if (code & T_PF_WRITE) 699 prot |= VM_PROT_WRITE; 700#if PAE 701 if (code & T_PF_EXECUTE) 702 prot |= VM_PROT_EXECUTE; 703#endif 704 705 result = vm_fault(map, 706 vm_map_trunc_page(vaddr), 707 prot, 708 FALSE, 709 THREAD_UNINT, NULL, 0); 710 711 if (result == KERN_SUCCESS) { 712#if NCOPY_WINDOWS > 0 713 if (fault_in_copy_window != -1) { 714 ml_set_interrupts_enabled(FALSE); 715 copy_window_fault(thread, map, 716 fault_in_copy_window); 717 (void) ml_set_interrupts_enabled(intr); 718 } 719#endif /* NCOPY_WINDOWS > 0 */ 720 return; 721 } 722 /* 723 * fall through 724 */ 725#if CONFIG_DTRACE 726FALL_THROUGH: 727#endif /* CONFIG_DTRACE */ 728 729 case T_GENERAL_PROTECTION: 730 /* 731 * If there is a failure recovery address 732 * for this fault, go there. 733 */ 734 for (rp = recover_table; rp < recover_table_end; rp++) { 735 if (kern_ip == rp->fault_addr) { 736 set_recovery_ip(saved_state, rp->recover_addr); 737 return; 738 } 739 } 740 741 /* 742 * Check thread recovery address also. 743 */ 744 if (thread != THREAD_NULL && thread->recover) { 745 set_recovery_ip(saved_state, thread->recover); 746 thread->recover = 0; 747 return; 748 } 749 /* 750 * Unanticipated page-fault errors in kernel 751 * should not happen. 752 * 753 * fall through... 754 */ 755 default: 756 /* 757 * Exception 15 is reserved but some chips may generate it 758 * spuriously. Seen at startup on AMD Athlon-64. 759 */ 760 if (type == 15) { 761 kprintf("kernel_trap() ignoring spurious trap 15\n"); 762 return; 763 } 764debugger_entry: 765 /* Ensure that the i386_kernel_state at the base of the 766 * current thread's stack (if any) is synchronized with the 767 * context at the moment of the trap, to facilitate 768 * access through the debugger. 769 */ 770 sync_iss_to_iks(state); 771#if MACH_KDP 772 if (current_debugger != KDB_CUR_DB) { 773 if (kdp_i386_trap(type, saved_state, result, (vm_offset_t)vaddr)) 774 return; 775 } 776#endif 777 } 778 pal_cli(); 779 panic_trap(saved_state); 780 /* 781 * NO RETURN 782 */ 783} 784 785 786#ifdef __i386__ 787static void 788set_recovery_ip(x86_saved_state32_t *saved_state, vm_offset_t ip) 789{ 790 saved_state->eip = ip; 791} 792#else 793static void 794set_recovery_ip(x86_saved_state64_t *saved_state, vm_offset_t ip) 795{ 796 saved_state->isf.rip = ip; 797} 798#endif 799 800 801#ifdef __i386__ 802static void 803panic_trap(x86_saved_state32_t *regs) 804{ 805 const char *trapname = "Unknown"; 806 pal_cr_t cr0, cr2, cr3, cr4; 807 808 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 ); 809 810 /* 811 * Issue an I/O port read if one has been requested - this is an 812 * event logic analyzers can use as a trigger point. 813 */ 814 panic_io_port_read(); 815 816 kprintf("panic trap number 0x%x, eip 0x%x\n", regs->trapno, regs->eip); 817 kprintf("cr0 0x%08x cr2 0x%08x cr3 0x%08x cr4 0x%08x\n", 818 cr0, cr2, cr3, cr4); 819 820 if (regs->trapno < TRAP_TYPES) 821 trapname = trap_type[regs->trapno]; 822#undef panic 823 panic("Kernel trap at 0x%08x, type %d=%s, registers:\n" 824 "CR0: 0x%08x, CR2: 0x%08x, CR3: 0x%08x, CR4: 0x%08x\n" 825 "EAX: 0x%08x, EBX: 0x%08x, ECX: 0x%08x, EDX: 0x%08x\n" 826 "CR2: 0x%08x, EBP: 0x%08x, ESI: 0x%08x, EDI: 0x%08x\n" 827 "EFL: 0x%08x, EIP: 0x%08x, CS: 0x%08x, DS: 0x%08x\n" 828 "Error code: 0x%08x%s\n", 829 regs->eip, regs->trapno, trapname, cr0, cr2, cr3, cr4, 830 regs->eax,regs->ebx,regs->ecx,regs->edx, 831 regs->cr2,regs->ebp,regs->esi,regs->edi, 832 regs->efl,regs->eip,regs->cs & 0xFFFF, regs->ds & 0xFFFF, regs->err, 833 virtualized ? " VMM" : ""); 834 /* 835 * This next statement is not executed, 836 * but it's needed to stop the compiler using tail call optimization 837 * for the panic call - which confuses the subsequent backtrace. 838 */ 839 cr0 = 0; 840} 841#else 842 843 844static void 845panic_trap(x86_saved_state64_t *regs) 846{ 847 const char *trapname = "Unknown"; 848 pal_cr_t cr0, cr2, cr3, cr4; 849 boolean_t potential_smep_fault = FALSE, potential_kernel_NX_fault = FALSE; 850 851 pal_get_control_registers( &cr0, &cr2, &cr3, &cr4 ); 852 assert(ml_get_interrupts_enabled() == FALSE); 853 current_cpu_datap()->cpu_fatal_trap_state = regs; 854 /* 855 * Issue an I/O port read if one has been requested - this is an 856 * event logic analyzers can use as a trigger point. 857 */ 858 panic_io_port_read(); 859 860 kprintf("panic trap number 0x%x, rip 0x%016llx\n", 861 regs->isf.trapno, regs->isf.rip); 862 kprintf("cr0 0x%016llx cr2 0x%016llx cr3 0x%016llx cr4 0x%016llx\n", 863 cr0, cr2, cr3, cr4); 864 865 if (regs->isf.trapno < TRAP_TYPES) 866 trapname = trap_type[regs->isf.trapno]; 867 868 if ((regs->isf.trapno == T_PAGE_FAULT) && (regs->isf.err == (T_PF_PROT | T_PF_EXECUTE)) && (regs->isf.rip == regs->cr2)) { 869 if (pmap_smep_enabled && (regs->isf.rip < VM_MAX_USER_PAGE_ADDRESS)) { 870 potential_smep_fault = TRUE; 871 } else if (regs->isf.rip >= VM_MIN_KERNEL_AND_KEXT_ADDRESS) { 872 potential_kernel_NX_fault = TRUE; 873 } 874 } 875 876#undef panic 877 panic("Kernel trap at 0x%016llx, type %d=%s, registers:\n" 878 "CR0: 0x%016llx, CR2: 0x%016llx, CR3: 0x%016llx, CR4: 0x%016llx\n" 879 "RAX: 0x%016llx, RBX: 0x%016llx, RCX: 0x%016llx, RDX: 0x%016llx\n" 880 "RSP: 0x%016llx, RBP: 0x%016llx, RSI: 0x%016llx, RDI: 0x%016llx\n" 881 "R8: 0x%016llx, R9: 0x%016llx, R10: 0x%016llx, R11: 0x%016llx\n" 882 "R12: 0x%016llx, R13: 0x%016llx, R14: 0x%016llx, R15: 0x%016llx\n" 883 "RFL: 0x%016llx, RIP: 0x%016llx, CS: 0x%016llx, SS: 0x%016llx\n" 884 "Fault CR2: 0x%016llx, Error code: 0x%016llx, Fault CPU: 0x%x%s%s%s\n", 885 regs->isf.rip, regs->isf.trapno, trapname, 886 cr0, cr2, cr3, cr4, 887 regs->rax, regs->rbx, regs->rcx, regs->rdx, 888 regs->isf.rsp, regs->rbp, regs->rsi, regs->rdi, 889 regs->r8, regs->r9, regs->r10, regs->r11, 890 regs->r12, regs->r13, regs->r14, regs->r15, 891 regs->isf.rflags, regs->isf.rip, regs->isf.cs & 0xFFFF, 892 regs->isf.ss & 0xFFFF,regs->cr2, regs->isf.err, regs->isf.cpu, 893 virtualized ? " VMM" : "", 894 potential_kernel_NX_fault ? " Kernel NX fault" : "", 895 potential_smep_fault ? " SMEP/User NX fault" : ""); 896 /* 897 * This next statement is not executed, 898 * but it's needed to stop the compiler using tail call optimization 899 * for the panic call - which confuses the subsequent backtrace. 900 */ 901 cr0 = 0; 902} 903#endif 904 905#if CONFIG_DTRACE 906extern kern_return_t dtrace_user_probe(x86_saved_state_t *); 907#endif 908 909/* 910 * Trap from user mode. 911 */ 912void 913user_trap( 914 x86_saved_state_t *saved_state) 915{ 916 int exc; 917 int err; 918 mach_exception_code_t code; 919 mach_exception_subcode_t subcode; 920 int type; 921 user_addr_t vaddr; 922 vm_prot_t prot; 923 thread_t thread = current_thread(); 924 ast_t *myast; 925 kern_return_t kret; 926 user_addr_t rip; 927 unsigned long dr6 = 0; /* 32 bit for i386, 64 bit for x86_64 */ 928 929 assert((is_saved_state32(saved_state) && !thread_is_64bit(thread)) || 930 (is_saved_state64(saved_state) && thread_is_64bit(thread))); 931 932 if (is_saved_state64(saved_state)) { 933 x86_saved_state64_t *regs; 934 935 regs = saved_state64(saved_state); 936 937 /* Record cpu where state was captured */ 938 regs->isf.cpu = cpu_number(); 939 940 type = regs->isf.trapno; 941 err = (int)regs->isf.err & 0xffff; 942 vaddr = (user_addr_t)regs->cr2; 943 rip = (user_addr_t)regs->isf.rip; 944 } else { 945 x86_saved_state32_t *regs; 946 947 regs = saved_state32(saved_state); 948 949 /* Record cpu where state was captured */ 950 regs->cpu = cpu_number(); 951 952 type = regs->trapno; 953 err = regs->err & 0xffff; 954 vaddr = (user_addr_t)regs->cr2; 955 rip = (user_addr_t)regs->eip; 956 } 957 958 if ((type == T_DEBUG) && thread->machine.ids) { 959 unsigned long clear = 0; 960 /* Stash and clear this processor's DR6 value, in the event 961 * this was a debug register match 962 */ 963 __asm__ volatile ("mov %%db6, %0" : "=r" (dr6)); 964 __asm__ volatile ("mov %0, %%db6" : : "r" (clear)); 965 } 966 967 pal_sti(); 968 969 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 970 (MACHDBG_CODE(DBG_MACH_EXCP_UTRAP_x86, type)) | DBG_FUNC_NONE, 971 (unsigned)(vaddr>>32), (unsigned)vaddr, 972 (unsigned)(rip>>32), (unsigned)rip, 0); 973 974 code = 0; 975 subcode = 0; 976 exc = 0; 977 978#if DEBUG_TRACE 979 kprintf("user_trap(0x%08x) type=%d vaddr=0x%016llx\n", 980 saved_state, type, vaddr); 981#endif 982 983 perfASTCallback astfn = perfASTHook; 984 if (__improbable(astfn != NULL)) { 985 myast = ast_pending(); 986 if (*myast & AST_CHUD_ALL) { 987 astfn(AST_CHUD_ALL, myast); 988 } 989 } 990 991 /* Is there a hook? */ 992 perfCallback fn = perfTrapHook; 993 if (__improbable(fn != NULL)) { 994 if (fn(type, saved_state, 0, 0) == KERN_SUCCESS) 995 return; /* If it succeeds, we are done... */ 996 } 997 998 /* 999 * DTrace does not consume all user traps, only INT_3's for now. 1000 * Avoid needlessly calling tempDTraceTrapHook here, and let the 1001 * INT_3 case handle them. 1002 */ 1003 DEBUG_KPRINT_SYSCALL_MASK(1, 1004 "user_trap: type=0x%x(%s) err=0x%x cr2=%p rip=%p\n", 1005 type, trap_type[type], err, (void *)(long) vaddr, (void *)(long) rip); 1006 1007 switch (type) { 1008 1009 case T_DIVIDE_ERROR: 1010 exc = EXC_ARITHMETIC; 1011 code = EXC_I386_DIV; 1012 break; 1013 1014 case T_DEBUG: 1015 { 1016 pcb_t pcb; 1017 /* 1018 * Update the PCB with this processor's DR6 value 1019 * in the event this was a debug register match. 1020 */ 1021 pcb = THREAD_TO_PCB(thread); 1022 if (pcb->ids) { 1023 /* 1024 * We can get and set the status register 1025 * in 32-bit mode even on a 64-bit thread 1026 * because the high order bits are not 1027 * used on x86_64 1028 */ 1029 if (thread_is_64bit(thread)) { 1030 x86_debug_state64_t *ids = pcb->ids; 1031 ids->dr6 = dr6; 1032 } else { /* 32 bit thread */ 1033 x86_debug_state32_t *ids = pcb->ids; 1034 ids->dr6 = (uint32_t) dr6; 1035 } 1036 } 1037 exc = EXC_BREAKPOINT; 1038 code = EXC_I386_SGL; 1039 break; 1040 } 1041 case T_INT3: 1042#if CONFIG_DTRACE 1043 if (dtrace_user_probe(saved_state) == KERN_SUCCESS) 1044 return; /* If it succeeds, we are done... */ 1045#endif 1046 exc = EXC_BREAKPOINT; 1047 code = EXC_I386_BPT; 1048 break; 1049 1050 case T_OVERFLOW: 1051 exc = EXC_ARITHMETIC; 1052 code = EXC_I386_INTO; 1053 break; 1054 1055 case T_OUT_OF_BOUNDS: 1056 exc = EXC_SOFTWARE; 1057 code = EXC_I386_BOUND; 1058 break; 1059 1060 case T_INVALID_OPCODE: 1061 exc = EXC_BAD_INSTRUCTION; 1062 code = EXC_I386_INVOP; 1063 break; 1064 1065 case T_NO_FPU: 1066 fpnoextflt(); 1067 return; 1068 1069 case T_FPU_FAULT: 1070 fpextovrflt(); /* Propagates exception directly, doesn't return */ 1071 return; 1072 1073 case T_INVALID_TSS: /* invalid TSS == iret with NT flag set */ 1074 exc = EXC_BAD_INSTRUCTION; 1075 code = EXC_I386_INVTSSFLT; 1076 subcode = err; 1077 break; 1078 1079 case T_SEGMENT_NOT_PRESENT: 1080 exc = EXC_BAD_INSTRUCTION; 1081 code = EXC_I386_SEGNPFLT; 1082 subcode = err; 1083 break; 1084 1085 case T_STACK_FAULT: 1086 exc = EXC_BAD_INSTRUCTION; 1087 code = EXC_I386_STKFLT; 1088 subcode = err; 1089 break; 1090 1091 case T_GENERAL_PROTECTION: 1092 /* 1093 * There's a wide range of circumstances which generate this 1094 * class of exception. From user-space, many involve bad 1095 * addresses (such as a non-canonical 64-bit address). 1096 * So we map this to EXC_BAD_ACCESS (and thereby SIGSEGV). 1097 * The trouble is cr2 doesn't contain the faulting address; 1098 * we'd need to decode the faulting instruction to really 1099 * determine this. We'll leave that to debuggers. 1100 * However, attempted execution of privileged instructions 1101 * (e.g. cli) also generate GP faults and so we map these to 1102 * to EXC_BAD_ACCESS (and thence SIGSEGV) also - rather than 1103 * EXC_BAD_INSTRUCTION which is more accurate. We just can't 1104 * win! 1105 */ 1106 exc = EXC_BAD_ACCESS; 1107 code = EXC_I386_GPFLT; 1108 subcode = err; 1109 break; 1110 1111 case T_PAGE_FAULT: 1112 { 1113 prot = VM_PROT_READ; 1114 1115 if (err & T_PF_WRITE) 1116 prot |= VM_PROT_WRITE; 1117#if PAE 1118 if (__improbable(err & T_PF_EXECUTE)) 1119 prot |= VM_PROT_EXECUTE; 1120#endif 1121 kret = vm_fault(thread->map, vm_map_trunc_page(vaddr), 1122 prot, FALSE, 1123 THREAD_ABORTSAFE, NULL, 0); 1124 1125 if (__probable((kret == KERN_SUCCESS) || (kret == KERN_ABORTED))) { 1126 thread_exception_return(); 1127 /* NOTREACHED */ 1128 } 1129 1130 user_page_fault_continue(kret); 1131 } /* NOTREACHED */ 1132 break; 1133 1134 case T_SSE_FLOAT_ERROR: 1135 fpSSEexterrflt(); /* Propagates exception directly, doesn't return */ 1136 return; 1137 1138 1139 case T_FLOATING_POINT_ERROR: 1140 fpexterrflt(); /* Propagates exception directly, doesn't return */ 1141 return; 1142 1143 case T_DTRACE_RET: 1144#if CONFIG_DTRACE 1145 if (dtrace_user_probe(saved_state) == KERN_SUCCESS) 1146 return; /* If it succeeds, we are done... */ 1147#endif 1148 /* 1149 * If we get an INT 0x7f when we do not expect to, 1150 * treat it as an illegal instruction 1151 */ 1152 exc = EXC_BAD_INSTRUCTION; 1153 code = EXC_I386_INVOP; 1154 break; 1155 1156 default: 1157 panic("Unexpected user trap, type %d", type); 1158 return; 1159 } 1160 /* Note: Codepaths that directly return from user_trap() have pending 1161 * ASTs processed in locore 1162 */ 1163 i386_exception(exc, code, subcode); 1164 /* NOTREACHED */ 1165} 1166 1167 1168/* 1169 * Handle AST traps for i386. 1170 */ 1171 1172extern void log_thread_action (thread_t, char *); 1173 1174void 1175i386_astintr(int preemption) 1176{ 1177 ast_t mask = AST_ALL; 1178 spl_t s; 1179 1180 if (preemption) 1181 mask = AST_PREEMPTION; 1182 1183 s = splsched(); 1184 1185 ast_taken(mask, s); 1186 1187 splx(s); 1188} 1189 1190/* 1191 * Handle exceptions for i386. 1192 * 1193 * If we are an AT bus machine, we must turn off the AST for a 1194 * delayed floating-point exception. 1195 * 1196 * If we are providing floating-point emulation, we may have 1197 * to retrieve the real register values from the floating point 1198 * emulator. 1199 */ 1200void 1201i386_exception( 1202 int exc, 1203 mach_exception_code_t code, 1204 mach_exception_subcode_t subcode) 1205{ 1206 mach_exception_data_type_t codes[EXCEPTION_CODE_MAX]; 1207 1208 DEBUG_KPRINT_SYSCALL_MACH("i386_exception: exc=%d code=0x%llx subcode=0x%llx\n", 1209 exc, code, subcode); 1210 codes[0] = code; /* new exception interface */ 1211 codes[1] = subcode; 1212 exception_triage(exc, codes, 2); 1213 /*NOTREACHED*/ 1214} 1215 1216 1217/* Synchronize a thread's i386_kernel_state (if any) with the given 1218 * i386_saved_state_t obtained from the trap/IPI handler; called in 1219 * kernel_trap() prior to entering the debugger, and when receiving 1220 * an "MP_KDP" IPI. 1221 */ 1222 1223void 1224sync_iss_to_iks(x86_saved_state_t *saved_state) 1225{ 1226 struct x86_kernel_state *iks; 1227 vm_offset_t kstack; 1228 boolean_t record_active_regs = FALSE; 1229 1230 /* The PAL may have a special way to sync registers */ 1231 if( saved_state->flavor == THREAD_STATE_NONE ) 1232 pal_get_kern_regs( saved_state ); 1233 1234 if ((kstack = current_thread()->kernel_stack) != 0) { 1235#ifdef __i386__ 1236 x86_saved_state32_t *regs = saved_state32(saved_state); 1237#else 1238 x86_saved_state64_t *regs = saved_state64(saved_state); 1239#endif 1240 1241 iks = STACK_IKS(kstack); 1242 1243 /* Did we take the trap/interrupt in kernel mode? */ 1244#ifdef __i386__ 1245 if (regs == USER_REGS32(current_thread())) 1246 record_active_regs = TRUE; 1247 else { 1248 iks->k_ebx = regs->ebx; 1249 iks->k_esp = (int)regs; 1250 iks->k_ebp = regs->ebp; 1251 iks->k_edi = regs->edi; 1252 iks->k_esi = regs->esi; 1253 iks->k_eip = regs->eip; 1254 } 1255#else 1256 if (regs == USER_REGS64(current_thread())) 1257 record_active_regs = TRUE; 1258 else { 1259 iks->k_rbx = regs->rbx; 1260 iks->k_rsp = regs->isf.rsp; 1261 iks->k_rbp = regs->rbp; 1262 iks->k_r12 = regs->r12; 1263 iks->k_r13 = regs->r13; 1264 iks->k_r14 = regs->r14; 1265 iks->k_r15 = regs->r15; 1266 iks->k_rip = regs->isf.rip; 1267 } 1268#endif 1269 } 1270 1271 if (record_active_regs == TRUE) { 1272#ifdef __i386__ 1273 /* Show the trap handler path */ 1274 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx)); 1275 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp)); 1276 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp)); 1277 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi)); 1278 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi)); 1279 /* "Current" instruction pointer */ 1280 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip)); 1281#else 1282 /* Show the trap handler path */ 1283 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx)); 1284 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp)); 1285 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp)); 1286 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12)); 1287 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13)); 1288 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14)); 1289 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15)); 1290 /* "Current" instruction pointer */ 1291 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" 1292 : "=m" (iks->k_rip) 1293 : 1294 : "rax"); 1295#endif 1296 } 1297} 1298 1299/* 1300 * This is used by the NMI interrupt handler (from mp.c) to 1301 * uncondtionally sync the trap handler context to the IKS 1302 * irrespective of whether the NMI was fielded in kernel 1303 * or user space. 1304 */ 1305void 1306sync_iss_to_iks_unconditionally(__unused x86_saved_state_t *saved_state) { 1307 struct x86_kernel_state *iks; 1308 vm_offset_t kstack; 1309 1310 if ((kstack = current_thread()->kernel_stack) != 0) { 1311 iks = STACK_IKS(kstack); 1312#ifdef __i386__ 1313 /* Display the trap handler path */ 1314 __asm__ volatile("movl %%ebx, %0" : "=m" (iks->k_ebx)); 1315 __asm__ volatile("movl %%esp, %0" : "=m" (iks->k_esp)); 1316 __asm__ volatile("movl %%ebp, %0" : "=m" (iks->k_ebp)); 1317 __asm__ volatile("movl %%edi, %0" : "=m" (iks->k_edi)); 1318 __asm__ volatile("movl %%esi, %0" : "=m" (iks->k_esi)); 1319 /* "Current" instruction pointer */ 1320 __asm__ volatile("movl $1f, %0\n1:" : "=m" (iks->k_eip)); 1321#else 1322 /* Display the trap handler path */ 1323 __asm__ volatile("movq %%rbx, %0" : "=m" (iks->k_rbx)); 1324 __asm__ volatile("movq %%rsp, %0" : "=m" (iks->k_rsp)); 1325 __asm__ volatile("movq %%rbp, %0" : "=m" (iks->k_rbp)); 1326 __asm__ volatile("movq %%r12, %0" : "=m" (iks->k_r12)); 1327 __asm__ volatile("movq %%r13, %0" : "=m" (iks->k_r13)); 1328 __asm__ volatile("movq %%r14, %0" : "=m" (iks->k_r14)); 1329 __asm__ volatile("movq %%r15, %0" : "=m" (iks->k_r15)); 1330 /* "Current" instruction pointer */ 1331 __asm__ volatile("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:" : "=m" (iks->k_rip)::"rax"); 1332#endif 1333 } 1334} 1335