1/* 2 * Copyright 2017, Data61 3 * Commonwealth Scientific and Industrial Research Organisation (CSIRO) 4 * ABN 41 687 119 230. 5 * 6 * This software may be distributed and modified according to the terms of 7 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 8 * See "LICENSE_GPLv2.txt" for details. 9 * 10 * @TAG(DATA61_GPL) 11 */ 12 13#include <machine/assembler.h> 14#include <config.h> 15#include <plat_mode/machine/hardware.h> 16 17/* 18 * The exception in 64-bit mode: 19 * All interrupt handlers pointed by the IDT are in 64-bit code. (this does not apply to SMI handler) 20 * The size of interrupt-stack pushes is fixed at 64 bits; and the processor uses 8-byte, zero extended 21 * stores. 22 * The stack pointer (SS:RSP) is pushed unconditionally on interrupts. 23 * The new SS is set to NULL if there is a change in CPL. 24 * Only 64-bit interrupt and trap gates can be referenced in x86-64 mode. 25 * No 32-bit interrupt or trap gate type exists in x86-64 mode. 26 * The RSP is aligned to a 16-byte boundary before pushing the stack frame. 27 * In x86-64 mode, when stacks are switched as part of a 64-bit mode privilege-level 28 * change, a new SS descriptor is not loaded. x86-64 mode loads only an inner-level 29 * RSP from the TSS. The new SS selector is forced to NULL and the SS selctor's RPL 30 * field is set to the new CPL. The old SS and RSP are saved on the new stack. 31 * 32 * Stack Usage with Privilege-Level Change 33 * 34 * SS +40 35 * RSP +32 36 * RFLAGS +24 37 * CS +16 38 * RIP +8 39 * ErrCode 0 - RSP 40 * 41 * About Segment in x86-64 mode 42 * ES, DS and SS segment registers are not used in 64-bit mode, their 43 * fields (base, limit and attribute) in segment descriptor registers 44 * are ignored. Some forms of segment load instructions are also invalid. 45 * Address caculations that reference the DS, ES or SS segments are treated 46 * as if the segment base is zero. Mode change does not change the contents 47 * of the segment registers or associated descriptor register. These registers 48 * are also not changed during 64-bit mode exectuion, unless explicit 49 * segment loads are performed. 50 * 51 * In order to setup compability mode for an application, segment-load 52 * instructions (mov to Sreg, pop Sreg) work normally in 64-bit mode. An 53 * entry is read from the system descriptor table (GDT or LDT) and is loaded 54 * in the hidden portion of the segment descriptor. The descriptor-register 55 * base, limit and attribute fields are all loaded. However, the contents 56 * of the data and stack segment selector and the descriptor registers are ignored 57 */ 58 59#ifdef ENABLE_SMP_SUPPORT 60/* If using multicore our gs base is set to point to a nodeInfo_t structure. 61 * Inside that is the 'currentThreadUserContext' that points to the first 62 * register we want to push in the case of the fastsyscall trap. See the 63 * comment in the nodeInfo struct for more details 64 */ 65#define MAYBE_SWAPGS swapgs 66#define LOAD_USER_CONTEXT movq %gs:16, %rsp 67#define LOAD_USER_CONTEXT_OFFSET(x) LOAD_USER_CONTEXT; addq $((x) * 8), %rsp 68#define LOAD_KERNEL_STACK movq %gs:0, %rsp 69#define LOAD_IRQ_STACK(x) movq %gs:8, %x 70#else 71/* The location in the user context we want is in ksCurThread after the fpu 72 * state (CONFIG_XSAVE_SIZE) and then the end of the user context is after 73 * 23 words (23 == n_contextRegisters). By default (in the case of a fast 74 * syscall trap) we skip 7 registers (SS, CS, RCX, R11, FaultIP, TLS_BASE 75 * and RSP) and are ready to push Error. 76 */ 77#define MAYBE_SWAPGS 78#define LOAD_USER_CONTEXT_OFFSET(x) movq (ksCurThread), %rsp; addq $(CONFIG_XSAVE_SIZE + 23*8 - 7*8 + (x)*8), %rsp 79#define LOAD_USER_CONTEXT LOAD_USER_CONTEXT_OFFSET(0) 80#define LOAD_KERNEL_STACK leaq kernel_stack_alloc + (1 << CONFIG_KERNEL_STACK_BITS), %rsp 81#define LOAD_IRQ_STACK(x) leaq x64KSIRQStack, %x 82#endif 83 84#ifdef CONFIG_KERNEL_SKIM_WINDOW 85/* If using PCIDs then our final value is not an address with a valid 32-bit relocation for 86 * the linker, but rather requires a full 64-bit representation. To work around this we 87 * use movabs to generate a full 64-bit immediate if using PCID, but lea if not. We prefer 88 * lea where possible as it has a more efficient instruction representation 89 */ 90#ifdef CONFIG_SUPPORT_PCID 91#define LOAD_KERNEL_AS(reg) \ 92 movabs $x64KSKernelPML4 - KERNEL_BASE_OFFSET + (1 << 63), %reg; \ 93 movq %reg, %cr3; 94#else /* !CONFIG_SUPPORT_PCID */ 95#define LOAD_KERNEL_AS(reg) \ 96 lea x64KSKernelPML4 - KERNEL_BASE_OFFSET, %reg; \ 97 movq %reg, %cr3; 98#endif /* CONFIG_SUPPORT_PCID */ 99#else /* !CONFIG_KERNEL_SKIM_WINDOW */ 100 101#define LOAD_KERNEL_AS(reg) 102 103#endif /* CONFIG_KERNEL_SKIM_WINDOW */ 104 105/* Registers to be pushed after an interrupt 106 %rcx must be pushed beforehand */ 107#define INT_SAVE_STATE \ 108 push %r11; \ 109 /* skip FaultIP TLS_BASE, RSP, NextIP, Error, RFLAGS */ \ 110 subq $(6 * 8), %rsp; \ 111 push %r15; \ 112 push %r9; \ 113 push %r8; \ 114 push %r10; \ 115 push %rdx; \ 116 push %r14; \ 117 push %r13; \ 118 push %r12; \ 119 push %rbp; \ 120 push %rbx; \ 121 push %rax; \ 122 push %rsi; \ 123 push %rdi 124 125/* Kernel exception handler if interrupt vector < 32. */ 126#define EXPAND_EXCEPT_1(number) \ 127 /* use saved stack */ \ 128 movq 32(%rsp), %rsp; \ 129 push %rcx; \ 130 movq $0x##number, %rcx; \ 131 jmp kernel_exception 132 133/* Kernel exception handler if interrupt vector >= 32. 134 Either a normal interrupt from the idle thread 135 or a nested interrupt. */ 136#define EXPAND_EXCEPT_0(number) \ 137 /* Check the saved esp, if its 0 we came from */\ 138 /* the idle thread and have a normal interrupt*/\ 139 addq $48, %rsp; \ 140 cmpq $0, -16(%rsp); \ 141 je 2b; \ 142 /* nested interrupt, use saved stack */ \ 143 movq -16(%rsp), %rsp; \ 144 /* skip 128 bytes as we need to respect the */ \ 145 /* red zone */ \ 146 subq $128, %rsp; \ 147 push %rcx; \ 148 movq $0x##number, %rcx; \ 149 jmp nested_interrupt 150 151#define INT_HANDLE_COMMON_EXCEPT(number,except) _expand_except_(except)(number) 152#define _expand_except_(except) EXPAND_EXCEPT_##except 153 154#define INT_HANDLER_COMMON(number,error_code,except) \ 155.global int_##number; \ 156.type int_##number, %function; \ 157int_##number: \ 158 error_code; \ 159 /* Check CPL */ \ 160 testq $3, 16(%rsp); \ 161 jz 1f; \ 1622: \ 163 LOAD_KERNEL_AS(rsp) \ 164 /* we need to not skip RSP, TLS_BASE, FaultIP, R11 and RCX for now */ \ 165 MAYBE_SWAPGS; \ 166 LOAD_USER_CONTEXT_OFFSET(5); \ 167 push %rcx; \ 168 movq $0x##number, %rcx; \ 169 jmp handle_interrupt; \ 1701: \ 171 INT_HANDLE_COMMON_EXCEPT(number,except); \ 172.size int_##number, . - int_##number; 173 174#define INT_HANDLER_WITH_ERR_CODE(number,except) INT_HANDLER_COMMON(number,,except) 175#define INT_HANDLER_WITHOUT_ERR_CODE(number,except) INT_HANDLER_COMMON(number,pushq $0x0,except) 176 177.section .text 178.code64 179 180INT_HANDLER_WITHOUT_ERR_CODE(00,1) 181INT_HANDLER_WITHOUT_ERR_CODE(01,1) 182INT_HANDLER_WITHOUT_ERR_CODE(02,1) 183INT_HANDLER_WITHOUT_ERR_CODE(03,1) 184INT_HANDLER_WITHOUT_ERR_CODE(04,1) 185INT_HANDLER_WITHOUT_ERR_CODE(05,1) 186INT_HANDLER_WITHOUT_ERR_CODE(06,1) 187INT_HANDLER_WITHOUT_ERR_CODE(07,1) 188INT_HANDLER_WITH_ERR_CODE(08,1) 189INT_HANDLER_WITHOUT_ERR_CODE(09,1) 190INT_HANDLER_WITH_ERR_CODE(0a,1) 191INT_HANDLER_WITH_ERR_CODE(0b,1) 192INT_HANDLER_WITH_ERR_CODE(0c,1) 193INT_HANDLER_WITH_ERR_CODE(0d,1) 194INT_HANDLER_WITH_ERR_CODE(0e,1) 195INT_HANDLER_WITHOUT_ERR_CODE(0f,1) 196 197INT_HANDLER_WITHOUT_ERR_CODE(10,1) 198INT_HANDLER_WITH_ERR_CODE(11,1) 199INT_HANDLER_WITHOUT_ERR_CODE(12,1) 200INT_HANDLER_WITHOUT_ERR_CODE(13,1) 201INT_HANDLER_WITHOUT_ERR_CODE(14,1) 202INT_HANDLER_WITHOUT_ERR_CODE(15,1) 203INT_HANDLER_WITHOUT_ERR_CODE(16,1) 204INT_HANDLER_WITHOUT_ERR_CODE(17,1) 205INT_HANDLER_WITHOUT_ERR_CODE(18,1) 206INT_HANDLER_WITHOUT_ERR_CODE(19,1) 207INT_HANDLER_WITHOUT_ERR_CODE(1a,1) 208INT_HANDLER_WITHOUT_ERR_CODE(1b,1) 209INT_HANDLER_WITHOUT_ERR_CODE(1c,1) 210INT_HANDLER_WITHOUT_ERR_CODE(1d,1) 211INT_HANDLER_WITHOUT_ERR_CODE(1e,1) 212INT_HANDLER_WITHOUT_ERR_CODE(1f,1) 213 214INT_HANDLER_WITHOUT_ERR_CODE(20,0) 215INT_HANDLER_WITHOUT_ERR_CODE(21,0) 216INT_HANDLER_WITHOUT_ERR_CODE(22,0) 217INT_HANDLER_WITHOUT_ERR_CODE(23,0) 218INT_HANDLER_WITHOUT_ERR_CODE(24,0) 219INT_HANDLER_WITHOUT_ERR_CODE(25,0) 220INT_HANDLER_WITHOUT_ERR_CODE(26,0) 221INT_HANDLER_WITHOUT_ERR_CODE(27,0) 222INT_HANDLER_WITHOUT_ERR_CODE(28,0) 223INT_HANDLER_WITHOUT_ERR_CODE(29,0) 224INT_HANDLER_WITHOUT_ERR_CODE(2a,0) 225INT_HANDLER_WITHOUT_ERR_CODE(2b,0) 226INT_HANDLER_WITHOUT_ERR_CODE(2c,0) 227INT_HANDLER_WITHOUT_ERR_CODE(2d,0) 228INT_HANDLER_WITHOUT_ERR_CODE(2e,0) 229INT_HANDLER_WITHOUT_ERR_CODE(2f,0) 230 231INT_HANDLER_WITHOUT_ERR_CODE(30,0) 232INT_HANDLER_WITHOUT_ERR_CODE(31,0) 233INT_HANDLER_WITHOUT_ERR_CODE(32,0) 234INT_HANDLER_WITHOUT_ERR_CODE(33,0) 235INT_HANDLER_WITHOUT_ERR_CODE(34,0) 236INT_HANDLER_WITHOUT_ERR_CODE(35,0) 237INT_HANDLER_WITHOUT_ERR_CODE(36,0) 238INT_HANDLER_WITHOUT_ERR_CODE(37,0) 239INT_HANDLER_WITHOUT_ERR_CODE(38,0) 240INT_HANDLER_WITHOUT_ERR_CODE(39,0) 241INT_HANDLER_WITHOUT_ERR_CODE(3a,0) 242INT_HANDLER_WITHOUT_ERR_CODE(3b,0) 243INT_HANDLER_WITHOUT_ERR_CODE(3c,0) 244INT_HANDLER_WITHOUT_ERR_CODE(3d,0) 245INT_HANDLER_WITHOUT_ERR_CODE(3e,0) 246INT_HANDLER_WITHOUT_ERR_CODE(3f,0) 247 248INT_HANDLER_WITHOUT_ERR_CODE(40,0) 249INT_HANDLER_WITHOUT_ERR_CODE(41,0) 250INT_HANDLER_WITHOUT_ERR_CODE(42,0) 251INT_HANDLER_WITHOUT_ERR_CODE(43,0) 252INT_HANDLER_WITHOUT_ERR_CODE(44,0) 253INT_HANDLER_WITHOUT_ERR_CODE(45,0) 254INT_HANDLER_WITHOUT_ERR_CODE(46,0) 255INT_HANDLER_WITHOUT_ERR_CODE(47,0) 256INT_HANDLER_WITHOUT_ERR_CODE(48,0) 257INT_HANDLER_WITHOUT_ERR_CODE(49,0) 258INT_HANDLER_WITHOUT_ERR_CODE(4a,0) 259INT_HANDLER_WITHOUT_ERR_CODE(4b,0) 260INT_HANDLER_WITHOUT_ERR_CODE(4c,0) 261INT_HANDLER_WITHOUT_ERR_CODE(4d,0) 262INT_HANDLER_WITHOUT_ERR_CODE(4e,0) 263INT_HANDLER_WITHOUT_ERR_CODE(4f,0) 264 265INT_HANDLER_WITHOUT_ERR_CODE(50,0) 266INT_HANDLER_WITHOUT_ERR_CODE(51,0) 267INT_HANDLER_WITHOUT_ERR_CODE(52,0) 268INT_HANDLER_WITHOUT_ERR_CODE(53,0) 269INT_HANDLER_WITHOUT_ERR_CODE(54,0) 270INT_HANDLER_WITHOUT_ERR_CODE(55,0) 271INT_HANDLER_WITHOUT_ERR_CODE(56,0) 272INT_HANDLER_WITHOUT_ERR_CODE(57,0) 273INT_HANDLER_WITHOUT_ERR_CODE(58,0) 274INT_HANDLER_WITHOUT_ERR_CODE(59,0) 275INT_HANDLER_WITHOUT_ERR_CODE(5a,0) 276INT_HANDLER_WITHOUT_ERR_CODE(5b,0) 277INT_HANDLER_WITHOUT_ERR_CODE(5c,0) 278INT_HANDLER_WITHOUT_ERR_CODE(5d,0) 279INT_HANDLER_WITHOUT_ERR_CODE(5e,0) 280INT_HANDLER_WITHOUT_ERR_CODE(5f,0) 281 282INT_HANDLER_WITHOUT_ERR_CODE(60,0) 283INT_HANDLER_WITHOUT_ERR_CODE(61,0) 284INT_HANDLER_WITHOUT_ERR_CODE(62,0) 285INT_HANDLER_WITHOUT_ERR_CODE(63,0) 286INT_HANDLER_WITHOUT_ERR_CODE(64,0) 287INT_HANDLER_WITHOUT_ERR_CODE(65,0) 288INT_HANDLER_WITHOUT_ERR_CODE(66,0) 289INT_HANDLER_WITHOUT_ERR_CODE(67,0) 290INT_HANDLER_WITHOUT_ERR_CODE(68,0) 291INT_HANDLER_WITHOUT_ERR_CODE(69,0) 292INT_HANDLER_WITHOUT_ERR_CODE(6a,0) 293INT_HANDLER_WITHOUT_ERR_CODE(6b,0) 294INT_HANDLER_WITHOUT_ERR_CODE(6c,0) 295INT_HANDLER_WITHOUT_ERR_CODE(6d,0) 296INT_HANDLER_WITHOUT_ERR_CODE(6e,0) 297INT_HANDLER_WITHOUT_ERR_CODE(6f,0) 298 299INT_HANDLER_WITHOUT_ERR_CODE(70,0) 300INT_HANDLER_WITHOUT_ERR_CODE(71,0) 301INT_HANDLER_WITHOUT_ERR_CODE(72,0) 302INT_HANDLER_WITHOUT_ERR_CODE(73,0) 303INT_HANDLER_WITHOUT_ERR_CODE(74,0) 304INT_HANDLER_WITHOUT_ERR_CODE(75,0) 305INT_HANDLER_WITHOUT_ERR_CODE(76,0) 306INT_HANDLER_WITHOUT_ERR_CODE(77,0) 307INT_HANDLER_WITHOUT_ERR_CODE(78,0) 308INT_HANDLER_WITHOUT_ERR_CODE(79,0) 309INT_HANDLER_WITHOUT_ERR_CODE(7a,0) 310INT_HANDLER_WITHOUT_ERR_CODE(7b,0) 311INT_HANDLER_WITHOUT_ERR_CODE(7c,0) 312INT_HANDLER_WITHOUT_ERR_CODE(7d,0) 313INT_HANDLER_WITHOUT_ERR_CODE(7e,0) 314INT_HANDLER_WITHOUT_ERR_CODE(7f,0) 315 316INT_HANDLER_WITHOUT_ERR_CODE(80,0) 317INT_HANDLER_WITHOUT_ERR_CODE(81,0) 318INT_HANDLER_WITHOUT_ERR_CODE(82,0) 319INT_HANDLER_WITHOUT_ERR_CODE(83,0) 320INT_HANDLER_WITHOUT_ERR_CODE(84,0) 321INT_HANDLER_WITHOUT_ERR_CODE(85,0) 322INT_HANDLER_WITHOUT_ERR_CODE(86,0) 323INT_HANDLER_WITHOUT_ERR_CODE(87,0) 324INT_HANDLER_WITHOUT_ERR_CODE(88,0) 325INT_HANDLER_WITHOUT_ERR_CODE(89,0) 326INT_HANDLER_WITHOUT_ERR_CODE(8a,0) 327INT_HANDLER_WITHOUT_ERR_CODE(8b,0) 328INT_HANDLER_WITHOUT_ERR_CODE(8c,0) 329INT_HANDLER_WITHOUT_ERR_CODE(8d,0) 330INT_HANDLER_WITHOUT_ERR_CODE(8e,0) 331INT_HANDLER_WITHOUT_ERR_CODE(8f,0) 332 333INT_HANDLER_WITHOUT_ERR_CODE(90,0) 334INT_HANDLER_WITHOUT_ERR_CODE(91,0) 335INT_HANDLER_WITHOUT_ERR_CODE(92,0) 336INT_HANDLER_WITHOUT_ERR_CODE(93,0) 337INT_HANDLER_WITHOUT_ERR_CODE(94,0) 338INT_HANDLER_WITHOUT_ERR_CODE(95,0) 339INT_HANDLER_WITHOUT_ERR_CODE(96,0) 340INT_HANDLER_WITHOUT_ERR_CODE(97,0) 341INT_HANDLER_WITHOUT_ERR_CODE(98,0) 342INT_HANDLER_WITHOUT_ERR_CODE(99,0) 343INT_HANDLER_WITHOUT_ERR_CODE(9a,0) 344INT_HANDLER_WITHOUT_ERR_CODE(9b,0) 345INT_HANDLER_WITHOUT_ERR_CODE(9c,0) 346INT_HANDLER_WITHOUT_ERR_CODE(9d,0) 347INT_HANDLER_WITHOUT_ERR_CODE(9e,0) 348INT_HANDLER_WITHOUT_ERR_CODE(9f,0) 349 350INT_HANDLER_WITHOUT_ERR_CODE(a0,0) 351INT_HANDLER_WITHOUT_ERR_CODE(a1,0) 352INT_HANDLER_WITHOUT_ERR_CODE(a2,0) 353INT_HANDLER_WITHOUT_ERR_CODE(a3,0) 354INT_HANDLER_WITHOUT_ERR_CODE(a4,0) 355INT_HANDLER_WITHOUT_ERR_CODE(a5,0) 356INT_HANDLER_WITHOUT_ERR_CODE(a6,0) 357INT_HANDLER_WITHOUT_ERR_CODE(a7,0) 358INT_HANDLER_WITHOUT_ERR_CODE(a8,0) 359INT_HANDLER_WITHOUT_ERR_CODE(a9,0) 360INT_HANDLER_WITHOUT_ERR_CODE(aa,0) 361INT_HANDLER_WITHOUT_ERR_CODE(ab,0) 362INT_HANDLER_WITHOUT_ERR_CODE(ac,0) 363INT_HANDLER_WITHOUT_ERR_CODE(ad,0) 364INT_HANDLER_WITHOUT_ERR_CODE(ae,0) 365INT_HANDLER_WITHOUT_ERR_CODE(af,0) 366 367INT_HANDLER_WITHOUT_ERR_CODE(b0,0) 368INT_HANDLER_WITHOUT_ERR_CODE(b1,0) 369INT_HANDLER_WITHOUT_ERR_CODE(b2,0) 370INT_HANDLER_WITHOUT_ERR_CODE(b3,0) 371INT_HANDLER_WITHOUT_ERR_CODE(b4,0) 372INT_HANDLER_WITHOUT_ERR_CODE(b5,0) 373INT_HANDLER_WITHOUT_ERR_CODE(b6,0) 374INT_HANDLER_WITHOUT_ERR_CODE(b7,0) 375INT_HANDLER_WITHOUT_ERR_CODE(b8,0) 376INT_HANDLER_WITHOUT_ERR_CODE(b9,0) 377INT_HANDLER_WITHOUT_ERR_CODE(ba,0) 378INT_HANDLER_WITHOUT_ERR_CODE(bb,0) 379INT_HANDLER_WITHOUT_ERR_CODE(bc,0) 380INT_HANDLER_WITHOUT_ERR_CODE(bd,0) 381INT_HANDLER_WITHOUT_ERR_CODE(be,0) 382INT_HANDLER_WITHOUT_ERR_CODE(bf,0) 383 384INT_HANDLER_WITHOUT_ERR_CODE(c0,0) 385INT_HANDLER_WITHOUT_ERR_CODE(c1,0) 386INT_HANDLER_WITHOUT_ERR_CODE(c2,0) 387INT_HANDLER_WITHOUT_ERR_CODE(c3,0) 388INT_HANDLER_WITHOUT_ERR_CODE(c4,0) 389INT_HANDLER_WITHOUT_ERR_CODE(c5,0) 390INT_HANDLER_WITHOUT_ERR_CODE(c6,0) 391INT_HANDLER_WITHOUT_ERR_CODE(c7,0) 392INT_HANDLER_WITHOUT_ERR_CODE(c8,0) 393INT_HANDLER_WITHOUT_ERR_CODE(c9,0) 394INT_HANDLER_WITHOUT_ERR_CODE(ca,0) 395INT_HANDLER_WITHOUT_ERR_CODE(cb,0) 396INT_HANDLER_WITHOUT_ERR_CODE(cc,0) 397INT_HANDLER_WITHOUT_ERR_CODE(cd,0) 398INT_HANDLER_WITHOUT_ERR_CODE(ce,0) 399INT_HANDLER_WITHOUT_ERR_CODE(cf,0) 400 401INT_HANDLER_WITHOUT_ERR_CODE(d0,0) 402INT_HANDLER_WITHOUT_ERR_CODE(d1,0) 403INT_HANDLER_WITHOUT_ERR_CODE(d2,0) 404INT_HANDLER_WITHOUT_ERR_CODE(d3,0) 405INT_HANDLER_WITHOUT_ERR_CODE(d4,0) 406INT_HANDLER_WITHOUT_ERR_CODE(d5,0) 407INT_HANDLER_WITHOUT_ERR_CODE(d6,0) 408INT_HANDLER_WITHOUT_ERR_CODE(d7,0) 409INT_HANDLER_WITHOUT_ERR_CODE(d8,0) 410INT_HANDLER_WITHOUT_ERR_CODE(d9,0) 411INT_HANDLER_WITHOUT_ERR_CODE(da,0) 412INT_HANDLER_WITHOUT_ERR_CODE(db,0) 413INT_HANDLER_WITHOUT_ERR_CODE(dc,0) 414INT_HANDLER_WITHOUT_ERR_CODE(dd,0) 415INT_HANDLER_WITHOUT_ERR_CODE(de,0) 416INT_HANDLER_WITHOUT_ERR_CODE(df,0) 417 418INT_HANDLER_WITHOUT_ERR_CODE(e0,0) 419INT_HANDLER_WITHOUT_ERR_CODE(e1,0) 420INT_HANDLER_WITHOUT_ERR_CODE(e2,0) 421INT_HANDLER_WITHOUT_ERR_CODE(e3,0) 422INT_HANDLER_WITHOUT_ERR_CODE(e4,0) 423INT_HANDLER_WITHOUT_ERR_CODE(e5,0) 424INT_HANDLER_WITHOUT_ERR_CODE(e6,0) 425INT_HANDLER_WITHOUT_ERR_CODE(e7,0) 426INT_HANDLER_WITHOUT_ERR_CODE(e8,0) 427INT_HANDLER_WITHOUT_ERR_CODE(e9,0) 428INT_HANDLER_WITHOUT_ERR_CODE(ea,0) 429INT_HANDLER_WITHOUT_ERR_CODE(eb,0) 430INT_HANDLER_WITHOUT_ERR_CODE(ec,0) 431INT_HANDLER_WITHOUT_ERR_CODE(ed,0) 432INT_HANDLER_WITHOUT_ERR_CODE(ee,0) 433INT_HANDLER_WITHOUT_ERR_CODE(ef,0) 434 435INT_HANDLER_WITHOUT_ERR_CODE(f0,0) 436INT_HANDLER_WITHOUT_ERR_CODE(f1,0) 437INT_HANDLER_WITHOUT_ERR_CODE(f2,0) 438INT_HANDLER_WITHOUT_ERR_CODE(f3,0) 439INT_HANDLER_WITHOUT_ERR_CODE(f4,0) 440INT_HANDLER_WITHOUT_ERR_CODE(f5,0) 441INT_HANDLER_WITHOUT_ERR_CODE(f6,0) 442INT_HANDLER_WITHOUT_ERR_CODE(f7,0) 443INT_HANDLER_WITHOUT_ERR_CODE(f8,0) 444INT_HANDLER_WITHOUT_ERR_CODE(f9,0) 445INT_HANDLER_WITHOUT_ERR_CODE(fa,0) 446INT_HANDLER_WITHOUT_ERR_CODE(fb,0) 447INT_HANDLER_WITHOUT_ERR_CODE(fc,0) 448INT_HANDLER_WITHOUT_ERR_CODE(fd,0) 449INT_HANDLER_WITHOUT_ERR_CODE(fe,0) 450INT_HANDLER_WITHOUT_ERR_CODE(ff,0) 451 452BEGIN_FUNC(handle_interrupt) 453 # push the rest of the registers 454 INT_SAVE_STATE 455 456 # switch to kernel stack 457 LOAD_KERNEL_STACK 458 459 # Set the arguments for c_x64_handle_interrupt 460 movq %rcx, %rdi 461 movq %rax, %rsi 462 463 # gtfo to C land, we will not return 464 call c_x64_handle_interrupt 465END_FUNC(handle_interrupt) 466 467BEGIN_FUNC(nested_interrupt) 468 # we got an interrupt from the kernel, call into c to save the irq number, 469 # then return back to where we were 470 INT_SAVE_STATE 471 movq %rcx, %rdi 472 call c_nested_interrupt 473 # disable the interrupt flag so we don't take any more interrupts 474 LOAD_IRQ_STACK(rbx) 475 andq $~0x200, 24(%rbx) 476 # return 477interrupt_return: 478 popq %rdi 479 popq %rsi 480 popq %rax 481 popq %rbx 482 popq %rbp 483 popq %r12 484 popq %r13 485 popq %r14 486 popq %rdx 487 popq %r10 488 popq %r8 489 popq %r9 490 popq %r15 491 /* skip RFLAGS, Error NextIP RSP, TLS_BASE, FaultIP */ 492 addq $48, %rsp 493 popq %r11 494 popq %rcx 495 LOAD_IRQ_STACK(rsp) 496 addq $8, %rsp 497 iretq 498END_FUNC(nested_interrupt) 499 500BEGIN_FUNC(kernel_exception) 501 # push registers 502 INT_SAVE_STATE 503#if defined(CONFIG_HARDWARE_DEBUG_API) 504 /* Before giving up and panicking, we need to test for the extra case that 505 * this might be a kernel exception that is the result of EFLAGS.TF being 506 * set when SYSENTER was called. 507 * 508 * Since EFLAGS.TF is not disabled by SYSENTER, single-stepping continues 509 * into the kernel, and so causes a debug-exception in kernel code, since 510 * the CPU is trying to single-step the kernel code. 511 * 512 * So we test for EFLAGS.TF, and if it's set, we unset it, and let the 513 * exception continue. The debug exception handler will notice that it was 514 * kernel exception, and handle it appropriately -- that really just means 515 * setting EFLAGS.TF before SYSEXIT so that single-stepping resumes in the 516 * userspace thread. 517 */ 518 LOAD_IRQ_STACK(rdx) 519 movq 24(%rdx), %rax 520 movq $(1<<8), %rbx 521 testq %rbx, %rax 522 je .not_eflags_tf 523 524 /* Else it was EFLAGS.TF that caused the kernel exception on SYSENTER. 525 * So, unset the EFLAGS.TF on the stack and this causes the syscall that we 526 * will return to, to be able to execute properly. 527 * 528 * It will then be the debug exception handler's responsibility to re-set 529 * EFLAGS.TF for the userspace thread before it returns. 530 * 531 * So at this point we want to just unset EFLAGS.TF and IRET immediately. 532 */ 533 andq $~(1<<8), %rax 534 movq %rax, 24(%rdx) 535 536 /* Begin popping registers to IRET now. We don't need to consider any 537 * unexpected side effects because we are just immediately returning after 538 * entering. 539 */ 540 popq %rdi 541 popq %rsi 542 popq %rax 543 popq %rbx 544 popq %rbp 545 popq %r12 546 popq %r13 547 popq %r14 548 popq %rdx 549 popq %r10 550 popq %r8 551 popq %r9 552 popq %r15 553 /* skip RFLAGS, Error NextIP RSP, TLS_BASE, FaultIP */ 554 addq $48, %rsp 555 popq %r11 556 popq %rcx 557 LOAD_IRQ_STACK(rsp) 558 addq $8, %rsp 559 iretq 560 561.not_eflags_tf: 562#endif /* CONFIG_HARDWARE_DEBUG_API */ 563 564 movq %rcx, %rdi 565 LOAD_IRQ_STACK(rsi) 566 movq 0(%rsi), %rsi # error code 567 LOAD_IRQ_STACK(rdx) 568 movq 8(%rdx), %rdx # RIP of the exception 569 LOAD_IRQ_STACK(rcx) 570 movq 32(%rcx), %rcx # RSP of the exception 571 LOAD_IRQ_STACK(r8) 572 movq 24(%r8), %r8 # RFLAGS 573 # handleKernelException(vector, errorcode, RIP, RSP, RFLAGS, CR0, CR2, CR3, CR4) 574 movq %cr0, %r9 575 movq %cr4, %r11 576 push %r11 577 movq %cr3, %r11 578 push %r11 579 movq %cr2, %r11 580 push %r11 581 call handleKernelException 582 addq $24, %rsp 583 # Set RIP in the saved register context to the new IP returned from handleKernelException 584 LOAD_IRQ_STACK(r8) 585 movq %rax, 8(%r8) 586 jmp interrupt_return 587END_FUNC(kernel_exception) 588 589# For a fast syscall the RFLAGS have been placed in 590# r11, the instruction *AFTER* the syscall is in 591# rcx. The current CS and SS have been loaded from 592# IA32_LSTAR (along with this code location). Additionally 593# the current RFLAGS (after saving) have been masked 594# with IA32_FMASK. 595BEGIN_FUNC(handle_fastsyscall) 596 LOAD_KERNEL_AS(rsp) 597 MAYBE_SWAPGS 598 LOAD_USER_CONTEXT 599 pushq $-1 # set Error -1 to mean entry via syscall 600 push %rcx # save NextIP 601 push %r11 # save RFLAGS 602 push %r15 # save R15 (message register) 603 push %r9 # save R9 (message register) 604 push %r8 # save R8 (message register) 605 push %r10 # save R10 (message register) 606 push %rdx # save RDX (syscall number) 607 push %r14 608 push %r13 609 push %r12 610 push %rbp 611 push %rbx 612 push %rax 613 push %rsi # save RSI (msgInfo register) 614 push %rdi # save RDI (capRegister) 615 616 # switch to kernel stack 617 LOAD_KERNEL_STACK 618 619 # RSI, RDI and RDX are already correct for calling c_handle_syscall 620 # gtfo to C land, we will not return 621 jmp c_handle_syscall 622END_FUNC(handle_fastsyscall) 623 624# Handle Syscall (coming via sysenter) 625# Assume following register contents when called: 626# RAX : syscall number 627# RCX : user ESP 628# RDX : user EIP (pointing to the sysenter instruction) 629# RSP : NULL 630BEGIN_FUNC(handle_syscall) 631 /* We need to save r11, rdx TLS_BASE and RSP */ 632 LOAD_KERNEL_AS(rsp) 633 MAYBE_SWAPGS 634 LOAD_USER_CONTEXT_OFFSET(4) 635 push %r11 636 push %rdx # save FaultIP 637 subq $8, %rsp # skip TLS_BASE 638 push %rcx # save RSP 639 push $-1 # set Error -1 to mean entry via syscall 640 push %rdx # save FaultIP (which will need to be updated later) 641 pushf # save RFLAGS 642 orq $0x200, (%rsp) # set interrupt bit in save RFLAGS 643 push %r15 # save R15 (message register) 644 push %r9 # save R9 (message register) 645 push %r8 # save R8 (message register) 646 push %r10 # save R10 (message register) 647 subq $8, %rsp # skip RDX 648 push %r14 649 push %r13 650 push %r12 651 push %rbp 652 push %rbx 653 push %rax # save RAX (syscall number) 654 push %rsi # save RSI (msgInfo register) 655 push %rdi # save RDI (capRegister) 656 657 # switch to kernel stack 658 LOAD_KERNEL_STACK 659 660 # RSI, RDI are already correct for calling c_handle_syscall 661 movq %rax, %rdx 662 # gtfo to C land, we will not return 663 call c_handle_syscall 664END_FUNC(handle_syscall) 665 666# Handle vmexit 667# RSP points to the end of the VCPUs general purpose register array 668#ifdef CONFIG_VTX 669BEGIN_FUNC(handle_vmexit) 670 MAYBE_SWAPGS 671 push %rbp 672 push %rdi 673 push %rsi 674 push %rdx 675 push %rcx 676 push %rbx 677 push %rax 678 679 # switch to kernel stack 680 LOAD_KERNEL_STACK 681 # Handle the vmexit, we will not return 682 call c_handle_vmexit 683END_FUNC(handle_vmexit) 684#endif /* CONFIG_VTX */ 685