1/* 2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58#include <i386/asm.h> 59#include <assym.s> 60#include <i386/eflags.h> 61#include <i386/trap.h> 62#include <i386/rtclock_asm.h> 63#define _ARCH_I386_ASM_HELP_H_ /* Prevent inclusion of user header */ 64#include <mach/i386/syscall_sw.h> 65#include <i386/postcode.h> 66#include <i386/proc_reg.h> 67#include <mach/exception_types.h> 68 69/* 70 * Low-memory handlers. 71 */ 72#define LO_ALLINTRS EXT(lo_allintrs32) 73#define LO_ALLTRAPS EXT(lo_alltraps32) 74#define LO_SYSENTER EXT(lo_sysenter32) 75#define LO_UNIX_SCALL EXT(lo_unix_scall32) 76#define LO_MACH_SCALL EXT(lo_mach_scall32) 77#define LO_MDEP_SCALL EXT(lo_mdep_scall32) 78 79#define HI_DATA(lo_addr) ( (EXT(lo_addr) - EXT(hi_remap_data)) + HIGH_IDT_BASE ) 80#define HI_TEXT(lo_text) ( (EXT(lo_text) - EXT(hi_remap_text)) + HIGH_MEM_BASE ) 81 82/* 83 * Interrupt descriptor table and code vectors for it. 84 */ 85#define IDT_BASE_ENTRY(vec,seg,type) \ 86 .data ;\ 87 .long EXT(vec) - EXT(hi_remap_text) + HIGH_MEM_BASE ; \ 88 .word seg ;\ 89 .byte 0 ;\ 90 .byte type ;\ 91 .text 92 93#define IDT_BASE_ENTRY_INT(vec,seg,type) \ 94 .data ;\ 95 .long vec - EXT(hi_remap_text) + HIGH_MEM_BASE ; \ 96 .word seg ;\ 97 .byte 0 ;\ 98 .byte type ;\ 99 .text 100 101#define IDT_BASE_ENTRY_TG(vec,seg,type) \ 102 .data ;\ 103 .long 0 ; \ 104 .word seg ;\ 105 .byte 0 ;\ 106 .byte type ;\ 107 .text 108 109#define IDT_ENTRY(vec,type) IDT_BASE_ENTRY(vec,KERNEL32_CS,type) 110#define IDT_ENTRY_INT(vec,type) IDT_BASE_ENTRY_INT(vec,KERNEL32_CS,type) 111 112/* 113 * No error code. Clear error code and push trap number. 114 */ 115#define EXCEPTION(n,name) \ 116 IDT_ENTRY(name,K_INTR_GATE);\ 117Entry(name) ;\ 118 pushl $0 ;\ 119 pushl $(n) ;\ 120 pusha ;\ 121 movl $(LO_ALLTRAPS),%ebx ;\ 122 jmp enter_lohandler 123 124 125/* 126 * Interrupt from user. Clear error code and push trap number. 127 */ 128#define EXCEP_USR(n,name) \ 129 IDT_ENTRY(name,U_INTR_GATE);\ 130Entry(name) ;\ 131 pushl $0 ;\ 132 pushl $(n) ;\ 133 pusha ;\ 134 movl $(LO_ALLTRAPS),%ebx ;\ 135 jmp enter_lohandler 136 137 138/* 139 * Special interrupt code. 140 */ 141#define EXCEP_SPC(n,name) \ 142 IDT_ENTRY(name,K_INTR_GATE) 143 144/* 145 * Special interrupt code from user. 146 */ 147#define EXCEP_SPC_USR(n,name) \ 148 IDT_ENTRY(name,U_INTR_GATE) 149 150 151/* 152 * Extra-special interrupt code. Note that no offset may be 153 * specified in a task gate descriptor, so name is ignored. 154 */ 155 156/* Double-fault fatal handler */ 157#define DF_FATAL_TASK(n,name) \ 158 IDT_BASE_ENTRY_TG(0,DF_TSS,K_TASK_GATE) 159 160/* machine-check handler */ 161#define MC_FATAL_TASK(n,name) \ 162 IDT_BASE_ENTRY_TG(0,MC_TSS,K_TASK_GATE) 163 164/* 165 * Error code has been pushed. Push trap number. 166 */ 167#define EXCEP_ERR(n,name) \ 168 IDT_ENTRY(name,K_INTR_GATE) ;\ 169Entry(name) ;\ 170 pushl $(n) ;\ 171 pusha ;\ 172 movl $(LO_ALLTRAPS),%ebx ;\ 173 jmp enter_lohandler 174 175 176/* 177 * Interrupt. 178 */ 179#define INTERRUPT(n) \ 180 IDT_ENTRY_INT(L_ ## n,K_INTR_GATE) ;\ 181 .align FALIGN ;\ 182L_ ## n: ;\ 183 pushl $0 ;\ 184 pushl $(n) ;\ 185 pusha ;\ 186 movl $(LO_ALLINTRS),%ebx ;\ 187 jmp enter_lohandler 188 189 190 .data 191 .align 12 192Entry(master_idt) 193Entry(hi_remap_data) 194 .text 195 .align 12 196Entry(hi_remap_text) 197 198EXCEPTION(0x00,t_zero_div) 199EXCEP_SPC(0x01,hi_debug) 200INTERRUPT(0x02) /* NMI */ 201EXCEP_USR(0x03,t_int3) 202EXCEP_USR(0x04,t_into) 203EXCEP_USR(0x05,t_bounds) 204EXCEPTION(0x06,t_invop) 205EXCEPTION(0x07,t_nofpu) 206DF_FATAL_TASK(0x08,df_task_start) 207EXCEPTION(0x09,a_fpu_over) 208EXCEPTION(0x0a,a_inv_tss) 209EXCEP_SPC(0x0b,hi_segnp) 210EXCEP_ERR(0x0c,t_stack_fault) 211EXCEP_SPC(0x0d,hi_gen_prot) 212EXCEP_SPC(0x0e,hi_page_fault) 213EXCEPTION(0x0f,t_trap_0f) 214EXCEPTION(0x10,t_fpu_err) 215EXCEPTION(0x11,t_trap_11) 216MC_FATAL_TASK(0x12,mc_task_start) 217EXCEPTION(0x13,t_sse_err) 218EXCEPTION(0x14,t_trap_14) 219EXCEPTION(0x15,t_trap_15) 220EXCEPTION(0x16,t_trap_16) 221EXCEPTION(0x17,t_trap_17) 222EXCEPTION(0x18,t_trap_18) 223EXCEPTION(0x19,t_trap_19) 224EXCEPTION(0x1a,t_trap_1a) 225EXCEPTION(0x1b,t_trap_1b) 226EXCEPTION(0x1c,t_trap_1c) 227EXCEPTION(0x1d,t_trap_1d) 228EXCEPTION(0x1e,t_trap_1e) 229EXCEPTION(0x1f,t_trap_1f) 230 231INTERRUPT(0x20) 232INTERRUPT(0x21) 233INTERRUPT(0x22) 234INTERRUPT(0x23) 235INTERRUPT(0x24) 236INTERRUPT(0x25) 237INTERRUPT(0x26) 238INTERRUPT(0x27) 239INTERRUPT(0x28) 240INTERRUPT(0x29) 241INTERRUPT(0x2a) 242INTERRUPT(0x2b) 243INTERRUPT(0x2c) 244INTERRUPT(0x2d) 245INTERRUPT(0x2e) 246INTERRUPT(0x2f) 247 248INTERRUPT(0x30) 249INTERRUPT(0x31) 250INTERRUPT(0x32) 251INTERRUPT(0x33) 252INTERRUPT(0x34) 253INTERRUPT(0x35) 254INTERRUPT(0x36) 255INTERRUPT(0x37) 256INTERRUPT(0x38) 257INTERRUPT(0x39) 258INTERRUPT(0x3a) 259INTERRUPT(0x3b) 260INTERRUPT(0x3c) 261INTERRUPT(0x3d) 262INTERRUPT(0x3e) 263INTERRUPT(0x3f) 264 265INTERRUPT(0x40) 266INTERRUPT(0x41) 267INTERRUPT(0x42) 268INTERRUPT(0x43) 269INTERRUPT(0x44) 270INTERRUPT(0x45) 271INTERRUPT(0x46) 272INTERRUPT(0x47) 273INTERRUPT(0x48) 274INTERRUPT(0x49) 275INTERRUPT(0x4a) 276INTERRUPT(0x4b) 277INTERRUPT(0x4c) 278INTERRUPT(0x4d) 279INTERRUPT(0x4e) 280INTERRUPT(0x4f) 281 282INTERRUPT(0x50) 283INTERRUPT(0x51) 284INTERRUPT(0x52) 285INTERRUPT(0x53) 286INTERRUPT(0x54) 287INTERRUPT(0x55) 288INTERRUPT(0x56) 289INTERRUPT(0x57) 290INTERRUPT(0x58) 291INTERRUPT(0x59) 292INTERRUPT(0x5a) 293INTERRUPT(0x5b) 294INTERRUPT(0x5c) 295INTERRUPT(0x5d) 296INTERRUPT(0x5e) 297INTERRUPT(0x5f) 298 299INTERRUPT(0x60) 300INTERRUPT(0x61) 301INTERRUPT(0x62) 302INTERRUPT(0x63) 303INTERRUPT(0x64) 304INTERRUPT(0x65) 305INTERRUPT(0x66) 306INTERRUPT(0x67) 307INTERRUPT(0x68) 308INTERRUPT(0x69) 309INTERRUPT(0x6a) 310INTERRUPT(0x6b) 311INTERRUPT(0x6c) 312INTERRUPT(0x6d) 313INTERRUPT(0x6e) 314INTERRUPT(0x6f) 315 316INTERRUPT(0x70) 317INTERRUPT(0x71) 318INTERRUPT(0x72) 319INTERRUPT(0x73) 320INTERRUPT(0x74) 321INTERRUPT(0x75) 322INTERRUPT(0x76) 323INTERRUPT(0x77) 324INTERRUPT(0x78) 325INTERRUPT(0x79) 326INTERRUPT(0x7a) 327INTERRUPT(0x7b) 328INTERRUPT(0x7c) 329INTERRUPT(0x7d) 330INTERRUPT(0x7e) 331EXCEP_USR(0x7f, t_dtrace_ret) 332 333EXCEP_SPC_USR(0x80,hi_unix_scall) 334EXCEP_SPC_USR(0x81,hi_mach_scall) 335EXCEP_SPC_USR(0x82,hi_mdep_scall) 336INTERRUPT(0x83) 337INTERRUPT(0x84) 338INTERRUPT(0x85) 339INTERRUPT(0x86) 340INTERRUPT(0x87) 341INTERRUPT(0x88) 342INTERRUPT(0x89) 343INTERRUPT(0x8a) 344INTERRUPT(0x8b) 345INTERRUPT(0x8c) 346INTERRUPT(0x8d) 347INTERRUPT(0x8e) 348INTERRUPT(0x8f) 349 350INTERRUPT(0x90) 351INTERRUPT(0x91) 352INTERRUPT(0x92) 353INTERRUPT(0x93) 354INTERRUPT(0x94) 355INTERRUPT(0x95) 356INTERRUPT(0x96) 357INTERRUPT(0x97) 358INTERRUPT(0x98) 359INTERRUPT(0x99) 360INTERRUPT(0x9a) 361INTERRUPT(0x9b) 362INTERRUPT(0x9c) 363INTERRUPT(0x9d) 364INTERRUPT(0x9e) 365INTERRUPT(0x9f) 366 367INTERRUPT(0xa0) 368INTERRUPT(0xa1) 369INTERRUPT(0xa2) 370INTERRUPT(0xa3) 371INTERRUPT(0xa4) 372INTERRUPT(0xa5) 373INTERRUPT(0xa6) 374INTERRUPT(0xa7) 375INTERRUPT(0xa8) 376INTERRUPT(0xa9) 377INTERRUPT(0xaa) 378INTERRUPT(0xab) 379INTERRUPT(0xac) 380INTERRUPT(0xad) 381INTERRUPT(0xae) 382INTERRUPT(0xaf) 383 384INTERRUPT(0xb0) 385INTERRUPT(0xb1) 386INTERRUPT(0xb2) 387INTERRUPT(0xb3) 388INTERRUPT(0xb4) 389INTERRUPT(0xb5) 390INTERRUPT(0xb6) 391INTERRUPT(0xb7) 392INTERRUPT(0xb8) 393INTERRUPT(0xb9) 394INTERRUPT(0xba) 395INTERRUPT(0xbb) 396INTERRUPT(0xbc) 397INTERRUPT(0xbd) 398INTERRUPT(0xbe) 399INTERRUPT(0xbf) 400 401INTERRUPT(0xc0) 402INTERRUPT(0xc1) 403INTERRUPT(0xc2) 404INTERRUPT(0xc3) 405INTERRUPT(0xc4) 406INTERRUPT(0xc5) 407INTERRUPT(0xc6) 408INTERRUPT(0xc7) 409INTERRUPT(0xc8) 410INTERRUPT(0xc9) 411INTERRUPT(0xca) 412INTERRUPT(0xcb) 413INTERRUPT(0xcc) 414INTERRUPT(0xcd) 415INTERRUPT(0xce) 416INTERRUPT(0xcf) 417 418INTERRUPT(0xd0) 419INTERRUPT(0xd1) 420INTERRUPT(0xd2) 421INTERRUPT(0xd3) 422INTERRUPT(0xd4) 423INTERRUPT(0xd5) 424INTERRUPT(0xd6) 425INTERRUPT(0xd7) 426INTERRUPT(0xd8) 427INTERRUPT(0xd9) 428INTERRUPT(0xda) 429INTERRUPT(0xdb) 430INTERRUPT(0xdc) 431INTERRUPT(0xdd) 432INTERRUPT(0xde) 433INTERRUPT(0xdf) 434 435INTERRUPT(0xe0) 436INTERRUPT(0xe1) 437INTERRUPT(0xe2) 438INTERRUPT(0xe3) 439INTERRUPT(0xe4) 440INTERRUPT(0xe5) 441INTERRUPT(0xe6) 442INTERRUPT(0xe7) 443INTERRUPT(0xe8) 444INTERRUPT(0xe9) 445INTERRUPT(0xea) 446INTERRUPT(0xeb) 447INTERRUPT(0xec) 448INTERRUPT(0xed) 449INTERRUPT(0xee) 450INTERRUPT(0xef) 451 452INTERRUPT(0xf0) 453INTERRUPT(0xf1) 454INTERRUPT(0xf2) 455INTERRUPT(0xf3) 456INTERRUPT(0xf4) 457INTERRUPT(0xf5) 458INTERRUPT(0xf6) 459INTERRUPT(0xf7) 460INTERRUPT(0xf8) 461INTERRUPT(0xf9) 462INTERRUPT(0xfa) 463INTERRUPT(0xfb) 464INTERRUPT(0xfc) 465INTERRUPT(0xfd) 466INTERRUPT(0xfe) 467EXCEPTION(0xff,t_preempt) 468 469 470 .data 471Entry(lo_kernel_cr3) 472 .long 0 473 .long 0 474 475 .text 476 477 478/* 479 * Trap/interrupt entry points. 480 * 481 * All traps must create the following save area on the PCB "stack": 482 * 483 * gs 484 * fs 485 * es 486 * ds 487 * edi 488 * esi 489 * ebp 490 * cr2 if page fault - otherwise unused 491 * ebx 492 * edx 493 * ecx 494 * eax 495 * trap number 496 * error code 497 * eip 498 * cs 499 * eflags 500 * user esp - if from user 501 * user ss - if from user 502 */ 503 504ret_to_kernel: 505 jmp *1f 5061: .long HI_TEXT(hi_ret_to_kernel) 507 508ret_to_user: 509 jmp *1f 5101: .long HI_TEXT(hi_ret_to_user) 511 512Entry(hi_ret_to_user) 513 movl %esp,%ebx 514 movl %gs:CPU_ACTIVE_THREAD,%ecx 515 subl TH_PCB_ISS(%ecx),%ebx 516 movl $(WINDOWS_CLEAN),TH_COPYIO_STATE(%ecx) 517 518 movl TH_PCB_IDS(%ecx),%eax /* get debug state struct */ 519 cmpl $0,%eax /* is there a debug state */ 520 je 1f /* branch if not */ 521 movl DS_DR0(%eax), %ecx /* Load the 32 bit debug registers */ 522 movl %ecx, %db0 523 movl DS_DR1(%eax), %ecx 524 movl %ecx, %db1 525 movl DS_DR2(%eax), %ecx 526 movl %ecx, %db2 527 movl DS_DR3(%eax), %ecx 528 movl %ecx, %db3 529 movl DS_DR7(%eax), %eax 5301: 531 addl %gs:CPU_HI_ISS,%ebx /* rebase PCB save area to high addr */ 532 movl %gs:CPU_TASK_CR3,%ecx 533 movl %ecx,%gs:CPU_ACTIVE_CR3 534 movl %ebx,%esp /* switch to hi based PCB stack */ 535 movl %ecx,%cr3 /* switch to user's address space */ 536 537 cmpl $0,%eax /* is dr7 set to something? */ 538 je 2f /* branch if not */ 539 movl %eax,%db7 /* Set dr7 */ 5402: 541 542Entry(hi_ret_to_kernel) 543 544 popl %eax /* ignore flavor of saved state */ 545EXT(ret_popl_gs): 546 popl %gs /* restore segment registers */ 547EXT(ret_popl_fs): 548 popl %fs 549EXT(ret_popl_es): 550 popl %es 551EXT(ret_popl_ds): 552 popl %ds 553 554 popa /* restore general registers */ 555 addl $8,%esp /* discard trap number and error code */ 556 557 cmpl $(SYSENTER_CS),4(%esp) /* test for fast entry/exit */ 558 je fast_exit 559EXT(ret_iret): 560 iret /* return from interrupt */ 561fast_exit: 562 popl %edx /* user return eip */ 563 popl %ecx /* pop and toss cs */ 564 andl $(~EFL_IF),(%esp) /* clear intrs enabled, see sti below */ 565 popf /* flags - carry denotes failure */ 566 popl %ecx /* user return esp */ 567 sti /* interrupts enabled after sysexit */ 568 sysexit 569 570 571Entry(hi_unix_scall) 572 pushl %eax /* save system call number */ 573 pushl $0 /* clear trap number slot */ 574 pusha /* save the general registers */ 575 movl $(LO_UNIX_SCALL),%ebx 576 jmp enter_lohandler 577 578 579Entry(hi_mach_scall) 580 pushl %eax /* save system call number */ 581 pushl $0 /* clear trap number slot */ 582 pusha /* save the general registers */ 583 movl $(LO_MACH_SCALL),%ebx 584 jmp enter_lohandler 585 586 587Entry(hi_mdep_scall) 588 pushl %eax /* save system call number */ 589 pushl $0 /* clear trap number slot */ 590 pusha /* save the general registers */ 591 movl $(LO_MDEP_SCALL),%ebx 592 jmp enter_lohandler 593 594 595/* 596 * sysenter entry point 597 * Requires user code to set up: 598 * edx: user instruction pointer (return address) 599 * ecx: user stack pointer 600 * on which is pushed stub ret addr and saved ebx 601 * Return to user-space is made using sysexit. 602 * Note: sysenter/sysexit cannot be used for calls returning a value in edx, 603 * or requiring ecx to be preserved. 604 */ 605Entry(hi_sysenter) 606 movl (%esp), %esp /* switch from intr stack to pcb */ 607 /* 608 * Push values on to the PCB stack 609 * to cons up the saved state. 610 */ 611 pushl $(USER_DS) /* ss */ 612 pushl %ecx /* uesp */ 613 pushf /* flags */ 614 /* 615 * Clear, among others, the Nested Task (NT) flags bit; 616 * This is cleared by INT, but not by SYSENTER. 617 */ 618 pushl $0 619 popfl 620 pushl $(SYSENTER_CS) /* cs */ 621hi_sysenter_2: 622 pushl %edx /* eip */ 623 pushl %eax /* err/eax - syscall code */ 624 pushl $0 /* clear trap number slot */ 625 pusha /* save the general registers */ 626 orl $(EFL_IF),R32_EFLAGS-R32_EDI(%esp) /* (edi was last reg pushed) */ 627 movl $(LO_SYSENTER),%ebx 628enter_lohandler: 629 pushl %ds 630 pushl %es 631 pushl %fs 632 pushl %gs 633 pushl $(SS_32) /* 32-bit state flavor */ 634enter_lohandler1: 635 mov %ss,%eax 636 mov %eax,%ds 637 mov %eax,%fs 638 mov %eax,%es /* switch to kernel data seg */ 639 mov $(CPU_DATA_GS),%eax 640 mov %eax,%gs 641 cld /* clear direction flag */ 642 /* 643 * Switch to kernel's address space if necessary 644 */ 645 movl HI_DATA(lo_kernel_cr3),%ecx 646 movl %cr3,%eax 647 cmpl %eax,%ecx 648 je 1f 649 movl %ecx,%cr3 650 movl %ecx,%gs:CPU_ACTIVE_CR3 6511: 652 testb $3,R32_CS(%esp) 653 jz 2f 654 movl %esp,%edx /* came from user mode */ 655 xor %ebp, %ebp 656 subl %gs:CPU_HI_ISS,%edx 657 movl %gs:CPU_ACTIVE_THREAD,%ecx 658 addl TH_PCB_ISS(%ecx),%edx /* rebase the high stack to a low address */ 659 movl %edx,%esp 660 cmpl $0, TH_PCB_IDS(%ecx) /* Is there a debug register state? */ 661 je 2f 662 movl $0, %ecx /* If so, reset DR7 (the control) */ 663 movl %ecx, %dr7 6642: 665 movl R32_TRAPNO(%esp),%ecx // Get the interrupt vector 666 addl $1,%gs:hwIntCnt(,%ecx,4) // Bump the count 667 jmp *%ebx 668 669 670/* 671 * Page fault traps save cr2. 672 */ 673Entry(hi_page_fault) 674 pushl $(T_PAGE_FAULT) /* mark a page fault trap */ 675 pusha /* save the general registers */ 676 movl %cr2,%eax /* get the faulting address */ 677 movl %eax,R32_CR2-R32_EDI(%esp)/* save in esp save slot */ 678 679 movl $(LO_ALLTRAPS),%ebx 680 jmp enter_lohandler 681 682 683 684/* 685 * Debug trap. Check for single-stepping across system call into 686 * kernel. If this is the case, taking the debug trap has turned 687 * off single-stepping - save the flags register with the trace 688 * bit set. 689 */ 690Entry(hi_debug) 691 testb $3,4(%esp) 692 jnz hi_debug_trap 693 /* trap came from kernel mode */ 694 cmpl $(HI_TEXT(hi_mach_scall)),(%esp) 695 jne 6f 696 addl $12,%esp /* remove eip/cs/eflags from debug_trap */ 697 jmp EXT(hi_mach_scall) /* continue system call entry */ 6986: 699 cmpl $(HI_TEXT(hi_mdep_scall)),(%esp) 700 jne 5f 701 addl $12,%esp /* remove eip/cs/eflags from debug_trap */ 702 jmp EXT(hi_mdep_scall) /* continue system call entry */ 7035: 704 cmpl $(HI_TEXT(hi_unix_scall)),(%esp) 705 jne 4f 706 addl $12,%esp /* remove eip/cs/eflags from debug_trap */ 707 jmp EXT(hi_unix_scall) /* continue system call entry */ 7084: 709 cmpl $(HI_TEXT(hi_sysenter)),(%esp) 710 jne hi_debug_trap 711 /* 712 * eip/cs/flags have been pushed on intr stack 713 * We have to switch to pcb stack and copy eflags. 714 * Note: setting the cs selector to SYSENTER_TF_CS 715 * will cause the return to user path to take the iret path so 716 * that eflags (containing the trap bit) is set atomically. 717 * In unix_syscall this is tested so that we'll rewind the pc 718 * to account for with sysenter or int entry. 719 */ 720 addl $8,%esp /* remove eip/cs */ 721 pushl %ecx /* save %ecx */ 722 movl 8(%esp),%ecx /* top of intr stack -> pcb stack */ 723 xchgl %ecx,%esp /* switch to pcb stack */ 724 pushl $(USER_DS) /* ss */ 725 pushl %ss:(%ecx) /* %ecx into uesp slot */ 726 pushl %ss:4(%ecx) /* eflags */ 727 movl %ss:(%ecx),%ecx /* restore %ecx */ 728 pushl $(SYSENTER_TF_CS) /* cs - not SYSENTER_CS for iret path */ 729 jmp hi_sysenter_2 /* continue sysenter entry */ 730hi_debug_trap: 731 pushl $0 732 pushl $(T_DEBUG) /* handle as user trap */ 733 pusha /* save the general registers */ 734 movl $(LO_ALLTRAPS),%ebx 735 jmp enter_lohandler 736 737 738 739/* 740 * General protection or segment-not-present fault. 741 * Check for a GP/NP fault in the kernel_return 742 * sequence; if there, report it as a GP/NP fault on the user's instruction. 743 * 744 * esp-> 0: trap code (NP or GP) 745 * 4: segment number in error 746 * 8 eip 747 * 12 cs 748 * 16 eflags 749 * 20 old registers (trap is from kernel) 750 */ 751Entry(hi_gen_prot) 752 pushl $(T_GENERAL_PROTECTION) /* indicate fault type */ 753 jmp trap_check_kernel_exit /* check for kernel exit sequence */ 754 755Entry(hi_segnp) 756 pushl $(T_SEGMENT_NOT_PRESENT) 757 /* indicate fault type */ 758trap_check_kernel_exit: 759 testb $3,12(%esp) 760 jnz hi_take_trap 761 /* trap was from kernel mode, so */ 762 /* check for the kernel exit sequence */ 763 cmpl $(HI_TEXT(ret_iret)),8(%esp) /* on IRET? */ 764 je fault_iret 765 cmpl $(HI_TEXT(ret_popl_ds)),8(%esp) /* popping DS? */ 766 je fault_popl_ds 767 cmpl $(HI_TEXT(ret_popl_es)),8(%esp) /* popping ES? */ 768 je fault_popl_es 769 cmpl $(HI_TEXT(ret_popl_fs)),8(%esp) /* popping FS? */ 770 je fault_popl_fs 771 cmpl $(HI_TEXT(ret_popl_gs)),8(%esp) /* popping GS? */ 772 je fault_popl_gs 773hi_take_trap: 774 pusha /* save the general registers */ 775 movl $(LO_ALLTRAPS),%ebx 776 jmp enter_lohandler 777 778 779/* 780 * GP/NP fault on IRET: CS or SS is in error. 781 * All registers contain the user's values. 782 * 783 * on SP is 784 * 0 trap number 785 * 4 errcode 786 * 8 eip 787 * 12 cs --> trapno 788 * 16 efl --> errcode 789 * 20 user eip 790 * 24 user cs 791 * 28 user eflags 792 * 32 user esp 793 * 36 user ss 794 */ 795fault_iret: 796 movl %eax,8(%esp) /* save eax (we don`t need saved eip) */ 797 popl %eax /* get trap number */ 798 movl %eax,12-4(%esp) /* put in user trap number */ 799 popl %eax /* get error code */ 800 movl %eax,16-8(%esp) /* put in user errcode */ 801 popl %eax /* restore eax */ 802 /* now treat as fault from user */ 803 pusha /* save the general registers */ 804 movl $(LO_ALLTRAPS),%ebx 805 jmp enter_lohandler 806 807/* 808 * Fault restoring a segment register. The user's registers are still 809 * saved on the stack. The offending segment register has not been 810 * popped. 811 */ 812fault_popl_ds: 813 popl %eax /* get trap number */ 814 popl %edx /* get error code */ 815 addl $12,%esp /* pop stack to user regs */ 816 jmp push_es /* (DS on top of stack) */ 817fault_popl_es: 818 popl %eax /* get trap number */ 819 popl %edx /* get error code */ 820 addl $12,%esp /* pop stack to user regs */ 821 jmp push_fs /* (ES on top of stack) */ 822fault_popl_fs: 823 popl %eax /* get trap number */ 824 popl %edx /* get error code */ 825 addl $12,%esp /* pop stack to user regs */ 826 jmp push_gs /* (FS on top of stack) */ 827fault_popl_gs: 828 popl %eax /* get trap number */ 829 popl %edx /* get error code */ 830 addl $12,%esp /* pop stack to user regs */ 831 jmp push_none /* (GS on top of stack) */ 832 833push_es: 834 pushl %es /* restore es, */ 835push_fs: 836 pushl %fs /* restore fs, */ 837push_gs: 838 pushl %gs /* restore gs. */ 839push_none: 840 pushl $(SS_32) /* 32-bit state flavor */ 841 movl %eax,R32_TRAPNO(%esp) /* set trap number */ 842 movl %edx,R32_ERR(%esp) /* set error code */ 843 /* now treat as fault from user */ 844 /* except that segment registers are */ 845 /* already pushed */ 846 movl $(LO_ALLTRAPS),%ebx 847 jmp enter_lohandler1 848 849 850 .text 851 852 853Entry(hi_remap_etext) 854 855 856/* 857 * All 32 bit task 'exceptions' enter lo_alltraps: 858 * esp -> x86_saved_state_t 859 * 860 * The rest of the state is set up as: 861 * cr3 -> kernel directory 862 * esp -> low based stack 863 * gs -> CPU_DATA_GS 864 * cs -> KERNEL32_CS 865 * ss/ds/es -> KERNEL_DS 866 * 867 * interrupts disabled 868 * direction flag cleared 869 */ 870Entry(lo_alltraps32) 871 movl R32_CS(%esp),%eax /* assume 32-bit state */ 872 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */ 873 jne 1f 874 movl R64_CS(%esp),%eax /* 64-bit user mode */ 8751: 876 testb $3,%al 877 jz trap_from_kernel 878 /* user mode trap */ 879 TIME_TRAP_UENTRY 880 881 movl %gs:CPU_ACTIVE_THREAD,%ecx 882 movl TH_TASK(%ecx),%ebx 883 884 /* Check for active vtimers in the current task */ 885 TASK_VTIMER_CHECK(%ebx, %ecx) 886 887 movl %gs:CPU_KERNEL_STACK,%ebx 888 xchgl %ebx,%esp /* switch to kernel stack */ 889 890 CCALL1(user_trap, %ebx) /* call user trap routine */ 891 /* user_trap() unmasks interrupts */ 892 cli /* hold off intrs - critical section */ 893 xorl %ecx,%ecx /* don't check if we're in the PFZ */ 894 895/* 896 * Return from trap or system call, checking for ASTs. 897 * On lowbase PCB stack with intrs disabled 898 */ 899Entry(return_from_trap32) 900 movl %gs:CPU_ACTIVE_THREAD, %esp 901 movl TH_PCB_ISS(%esp), %esp /* switch back to PCB stack */ 902 movl %gs:CPU_PENDING_AST, %eax 903 testl %eax, %eax 904 je EXT(return_to_user) /* branch if no AST */ 905LEXT(return_from_trap_with_ast) 906 movl %gs:CPU_KERNEL_STACK, %ebx 907 xchgl %ebx, %esp /* switch to kernel stack */ 908 909 testl %ecx, %ecx /* see if we need to check for an EIP in the PFZ */ 910 je 2f /* no, go handle the AST */ 911 cmpl $(SS_64), SS_FLAVOR(%ebx) /* are we a 64-bit task? */ 912 je 1f 913 /* no... 32-bit user mode */ 914 movl R32_EIP(%ebx), %eax 915 pushl %ebx /* save PCB stack */ 916 xorl %ebp, %ebp /* clear frame pointer */ 917 CCALL1(commpage_is_in_pfz32, %eax) 918 popl %ebx /* retrieve pointer to PCB stack */ 919 testl %eax, %eax 920 je 2f /* not in the PFZ... go service AST */ 921 movl %eax, R32_EBX(%ebx) /* let the PFZ know we've pended an AST */ 922 xchgl %ebx, %esp /* switch back to PCB stack */ 923 jmp EXT(return_to_user) 9241: /* 64-bit user mode */ 925 movl R64_RIP(%ebx), %ecx 926 movl R64_RIP+4(%ebx), %eax 927 pushl %ebx /* save PCB stack */ 928 xorl %ebp, %ebp /* clear frame pointer */ 929 CCALL2(commpage_is_in_pfz64, %ecx, %eax) 930 popl %ebx /* retrieve pointer to PCB stack */ 931 testl %eax, %eax 932 je 2f /* not in the PFZ... go service AST */ 933 movl %eax, R64_RBX(%ebx) /* let the PFZ know we've pended an AST */ 934 xchgl %ebx, %esp /* switch back to PCB stack */ 935 jmp EXT(return_to_user) 9362: 937 sti /* interrupts always enabled on return to user mode */ 938 xorl %ebp, %ebp /* Clear framepointer */ 939 CCALL1(i386_astintr, $0) /* take the AST */ 940 cli 941 xorl %ecx, %ecx /* don't check if we're in the PFZ */ 942 jmp EXT(return_from_trap32) /* and check again (rare) */ 943 944 945/* 946 * Trap from kernel mode. No need to switch stacks. 947 * Interrupts must be off here - we will set them to state at time of trap 948 * as soon as it's safe for us to do so and not recurse doing preemption 949 */ 950trap_from_kernel: 951 movl %esp, %eax /* saved state addr */ 952 pushl R32_EIP(%esp) /* Simulate a CALL from fault point */ 953 pushl %ebp /* Extend framepointer chain */ 954 movl %esp, %ebp 955 CCALL1WITHSP(kernel_trap, %eax) /* Call kernel trap handler */ 956 popl %ebp 957 addl $4, %esp 958 cli 959 960 movl %gs:CPU_PENDING_AST,%eax /* get pending asts */ 961 testl $ AST_URGENT,%eax /* any urgent preemption? */ 962 je ret_to_kernel /* no, nothing to do */ 963 cmpl $ T_PREEMPT,R32_TRAPNO(%esp) 964 je ret_to_kernel /* T_PREEMPT handled in kernel_trap() */ 965 testl $ EFL_IF,R32_EFLAGS(%esp) /* interrupts disabled? */ 966 je ret_to_kernel 967 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */ 968 jne ret_to_kernel 969 movl %gs:CPU_KERNEL_STACK,%eax 970 movl %esp,%ecx 971 xorl %eax,%ecx 972 and EXT(kernel_stack_mask),%ecx 973 testl %ecx,%ecx /* are we on the kernel stack? */ 974 jne ret_to_kernel /* no, skip it */ 975 976 CCALL1(i386_astintr, $1) /* take the AST */ 977 978 jmp ret_to_kernel 979 980 981/* 982 * All interrupts on all tasks enter here with: 983 * esp-> -> x86_saved_state_t 984 * 985 * cr3 -> kernel directory 986 * esp -> low based stack 987 * gs -> CPU_DATA_GS 988 * cs -> KERNEL32_CS 989 * ss/ds/es -> KERNEL_DS 990 * 991 * interrupts disabled 992 * direction flag cleared 993 */ 994Entry(lo_allintrs32) 995 /* 996 * test whether already on interrupt stack 997 */ 998 movl %gs:CPU_INT_STACK_TOP,%ecx 999 cmpl %esp,%ecx 1000 jb 1f 1001 leal -INTSTACK_SIZE(%ecx),%edx 1002 cmpl %esp,%edx 1003 jb int_from_intstack 10041: 1005 xchgl %ecx,%esp /* switch to interrupt stack */ 1006 1007 movl %cr0,%eax /* get cr0 */ 1008 orl $(CR0_TS),%eax /* or in TS bit */ 1009 movl %eax,%cr0 /* set cr0 */ 1010 1011 subl $8, %esp /* for 16-byte stack alignment */ 1012 pushl %ecx /* save pointer to old stack */ 1013 movl %ecx,%gs:CPU_INT_STATE /* save intr state */ 1014 1015 TIME_INT_ENTRY /* do timing */ 1016 1017 movl %gs:CPU_ACTIVE_THREAD,%ecx 1018 movl TH_TASK(%ecx),%ebx 1019 1020 /* Check for active vtimers in the current task */ 1021 TASK_VTIMER_CHECK(%ebx, %ecx) 1022 1023 incl %gs:CPU_PREEMPTION_LEVEL 1024 incl %gs:CPU_INTERRUPT_LEVEL 1025 1026 movl %gs:CPU_INT_STATE, %eax 1027 CCALL1(interrupt, %eax) /* call generic interrupt routine */ 1028 1029 cli /* just in case we returned with intrs enabled */ 1030 xorl %eax,%eax 1031 movl %eax,%gs:CPU_INT_STATE /* clear intr state pointer */ 1032 1033 decl %gs:CPU_INTERRUPT_LEVEL 1034 decl %gs:CPU_PREEMPTION_LEVEL 1035 1036 TIME_INT_EXIT /* do timing */ 1037 1038 movl %gs:CPU_ACTIVE_THREAD,%eax 1039 movl TH_PCB_FPS(%eax),%eax /* get pcb's ifps */ 1040 testl %eax, %eax /* Is there a context */ 1041 je 1f /* Branch if not */ 1042 cmpl $0, FP_VALID(%eax) /* Check fp_valid */ 1043 jne 1f /* Branch if valid */ 1044 clts /* Clear TS */ 1045 jmp 2f 10461: 1047 movl %cr0,%eax /* get cr0 */ 1048 orl $(CR0_TS),%eax /* or in TS bit */ 1049 movl %eax,%cr0 /* set cr0 */ 10502: 1051 popl %esp /* switch back to old stack */ 1052 1053 /* Load interrupted code segment into %eax */ 1054 movl R32_CS(%esp),%eax /* assume 32-bit state */ 1055 cmpl $(SS_64),SS_FLAVOR(%esp)/* 64-bit? */ 1056 jne 3f 1057 movl R64_CS(%esp),%eax /* 64-bit user mode */ 10583: 1059 testb $3,%al /* user mode, */ 1060 jnz ast_from_interrupt_user /* go handle potential ASTs */ 1061 /* 1062 * we only want to handle preemption requests if 1063 * the interrupt fell in the kernel context 1064 * and preemption isn't disabled 1065 */ 1066 movl %gs:CPU_PENDING_AST,%eax 1067 testl $ AST_URGENT,%eax /* any urgent requests? */ 1068 je ret_to_kernel /* no, nothing to do */ 1069 1070 cmpl $0,%gs:CPU_PREEMPTION_LEVEL /* preemption disabled? */ 1071 jne ret_to_kernel /* yes, skip it */ 1072 1073 movl %gs:CPU_KERNEL_STACK,%eax 1074 movl %esp,%ecx 1075 xorl %eax,%ecx 1076 and EXT(kernel_stack_mask),%ecx 1077 testl %ecx,%ecx /* are we on the kernel stack? */ 1078 jne ret_to_kernel /* no, skip it */ 1079 1080 /* 1081 * Take an AST from kernel space. We don't need (and don't want) 1082 * to do as much as the case where the interrupt came from user 1083 * space. 1084 */ 1085 CCALL1(i386_astintr, $1) 1086 1087 jmp ret_to_kernel 1088 1089 1090/* 1091 * nested int - simple path, can't preempt etc on way out 1092 */ 1093int_from_intstack: 1094 incl %gs:CPU_PREEMPTION_LEVEL 1095 incl %gs:CPU_INTERRUPT_LEVEL 1096 1097 movl %esp, %edx /* x86_saved_state */ 1098 CCALL1(interrupt, %edx) 1099 1100 decl %gs:CPU_INTERRUPT_LEVEL 1101 decl %gs:CPU_PREEMPTION_LEVEL 1102 1103 jmp ret_to_kernel 1104 1105/* 1106 * Take an AST from an interrupted user 1107 */ 1108ast_from_interrupt_user: 1109 movl %gs:CPU_PENDING_AST,%eax 1110 testl %eax,%eax /* pending ASTs? */ 1111 je ret_to_user /* no, nothing to do */ 1112 1113 TIME_TRAP_UENTRY 1114 1115 movl $1, %ecx /* check if we're in the PFZ */ 1116 jmp EXT(return_from_trap_with_ast) /* return */ 1117 1118 1119/* 1120 * 32bit Tasks 1121 * System call entries via INTR_GATE or sysenter: 1122 * 1123 * esp -> x86_saved_state32_t 1124 * cr3 -> kernel directory 1125 * esp -> low based stack 1126 * gs -> CPU_DATA_GS 1127 * cs -> KERNEL32_CS 1128 * ss/ds/es -> KERNEL_DS 1129 * 1130 * interrupts disabled 1131 * direction flag cleared 1132 */ 1133 1134Entry(lo_sysenter32) 1135 /* 1136 * We can be here either for a mach syscall or a unix syscall, 1137 * as indicated by the sign of the code: 1138 */ 1139 movl R32_EAX(%esp),%eax 1140 testl %eax,%eax 1141 js EXT(lo_mach_scall32) /* < 0 => mach */ 1142 /* > 0 => unix */ 1143 1144Entry(lo_unix_scall32) 1145 TIME_TRAP_UENTRY 1146 1147 movl %gs:CPU_KERNEL_STACK,%edi 1148 xchgl %edi,%esp /* switch to kernel stack */ 1149 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ 1150 movl TH_TASK(%ecx),%ebx /* point to current task */ 1151 incl TH_SYSCALLS_UNIX(%ecx) /* increment call count */ 1152 1153 /* Check for active vtimers in the current task */ 1154 TASK_VTIMER_CHECK(%ebx, %ecx) 1155 1156 sti 1157 1158 CCALL1(unix_syscall, %edi) 1159 /* 1160 * always returns through thread_exception_return 1161 */ 1162 1163 1164Entry(lo_mach_scall32) 1165 TIME_TRAP_UENTRY 1166 1167 movl %gs:CPU_KERNEL_STACK,%edi 1168 xchgl %edi,%esp /* switch to kernel stack */ 1169 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ 1170 movl TH_TASK(%ecx),%ebx /* point to current task */ 1171 incl TH_SYSCALLS_MACH(%ecx) /* increment call count */ 1172 1173 /* Check for active vtimers in the current task */ 1174 TASK_VTIMER_CHECK(%ebx, %ecx) 1175 1176 sti 1177 1178 CCALL1(mach_call_munger, %edi) 1179 /* 1180 * always returns through thread_exception_return 1181 */ 1182 1183 1184Entry(lo_mdep_scall32) 1185 TIME_TRAP_UENTRY 1186 1187 movl %gs:CPU_KERNEL_STACK,%edi 1188 xchgl %edi,%esp /* switch to kernel stack */ 1189 movl %gs:CPU_ACTIVE_THREAD,%ecx /* get current thread */ 1190 movl TH_TASK(%ecx),%ebx /* point to current task */ 1191 1192 /* Check for active vtimers in the current task */ 1193 TASK_VTIMER_CHECK(%ebx, %ecx) 1194 1195 sti 1196 1197 CCALL1(machdep_syscall, %edi) 1198 /* 1199 * always returns through thread_exception_return 1200 */ 1201 1202 1203LEXT(return_to_user) 1204 TIME_TRAP_UEXIT 1205 jmp ret_to_user 1206 1207 1208/* 1209 * Double-fault exception handler task. The last gasp... 1210 */ 1211Entry(df_task_start) 1212 CCALL1(panic_double_fault32, $(T_DOUBLE_FAULT)) 1213 hlt 1214 1215 1216/* 1217 * machine-check handler task. The last gasp... 1218 */ 1219Entry(mc_task_start) 1220 CCALL1(panic_machine_check32, $(T_MACHINE_CHECK)) 1221 hlt 1222