1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <machine/assembler.h> 8#include <config.h> 9#include <hardware.h> 10 11/* 12 * The exception in 64-bit mode: 13 * All interrupt handlers pointed by the IDT are in 64-bit code. (this does not apply to SMI handler) 14 * The size of interrupt-stack pushes is fixed at 64 bits; and the processor uses 8-byte, zero extended 15 * stores. 16 * The stack pointer (SS:RSP) is pushed unconditionally on interrupts. 17 * The new SS is set to NULL if there is a change in CPL. 18 * Only 64-bit interrupt and trap gates can be referenced in x86-64 mode. 19 * No 32-bit interrupt or trap gate type exists in x86-64 mode. 20 * The RSP is aligned to a 16-byte boundary before pushing the stack frame. 21 * In x86-64 mode, when stacks are switched as part of a 64-bit mode privilege-level 22 * change, a new SS descriptor is not loaded. x86-64 mode loads only an inner-level 23 * RSP from the TSS. The new SS selector is forced to NULL and the SS selctor's RPL 24 * field is set to the new CPL. The old SS and RSP are saved on the new stack. 25 * 26 * Stack Usage with Privilege-Level Change 27 * 28 * SS +40 29 * RSP +32 30 * RFLAGS +24 31 * CS +16 32 * RIP +8 33 * ErrCode 0 - RSP 34 * 35 * About Segment in x86-64 mode 36 * ES, DS and SS segment registers are not used in 64-bit mode, their 37 * fields (base, limit and attribute) in segment descriptor registers 38 * are ignored. Some forms of segment load instructions are also invalid. 39 * Address caculations that reference the DS, ES or SS segments are treated 40 * as if the segment base is zero. Mode change does not change the contents 41 * of the segment registers or associated descriptor register. These registers 42 * are also not changed during 64-bit mode exectuion, unless explicit 43 * segment loads are performed. 44 * 45 * In order to setup compability mode for an application, segment-load 46 * instructions (mov to Sreg, pop Sreg) work normally in 64-bit mode. An 47 * entry is read from the system descriptor table (GDT or LDT) and is loaded 48 * in the hidden portion of the segment descriptor. The descriptor-register 49 * base, limit and attribute fields are all loaded. However, the contents 50 * of the data and stack segment selector and the descriptor registers are ignored 51 */ 52 53#ifdef ENABLE_SMP_SUPPORT 54/* If using multicore our gs base is set to point to a nodeInfo_t structure. 55 * Inside that is the 'currentThreadUserContext' that points to the first 56 * register we want to push in the case of the fastsyscall trap. See the 57 * comment in the nodeInfo struct for more details 58 */ 59#define MAYBE_SWAPGS swapgs 60#define LOAD_USER_CONTEXT movq %gs:16, %rsp 61/* The user context under SMP is always set to Error + 1 for the current 62 * thread */ 63#define LOAD_USER_CONTEXT_OFFSET(x) LOAD_USER_CONTEXT; addq $((x) * 8), %rsp 64#define LOAD_KERNEL_STACK movq %gs:0, %rsp 65#define LOAD_IRQ_STACK(x) movq %gs:8, %x 66#else 67/* The location in the user context we want is in ksCurThread after the 68 * fpu state (CONFIG_XSAVE_SIZE) and then the end of the user context is 69 * after 22 words (22 == n_immContextRegisters). By default (in the case 70 * of a fast syscall trap) we skip 6 registers (SS, CS, RCX, R11, 71 * FaultIP, and RSP) and are ready to push Error. 72 */ 73#define MAYBE_SWAPGS 74#define LOAD_USER_CONTEXT_OFFSET(x) movq (ksCurThread), %rsp; addq $(CONFIG_XSAVE_SIZE + 22*8 - 6*8 + (x)*8), %rsp 75#define LOAD_USER_CONTEXT LOAD_USER_CONTEXT_OFFSET(0) 76#define LOAD_KERNEL_STACK leaq kernel_stack_alloc + (1 << CONFIG_KERNEL_STACK_BITS), %rsp 77#define LOAD_IRQ_STACK(x) leaq x64KSIRQStack, %x 78#endif 79 80#ifdef CONFIG_KERNEL_SKIM_WINDOW 81/* If using PCIDs then our final value is not an address with a valid 32-bit relocation for 82 * the linker, but rather requires a full 64-bit representation. To work around this we 83 * use movabs to generate a full 64-bit immediate if using PCID, but lea if not. We prefer 84 * lea where possible as it has a more efficient instruction representation 85 */ 86#ifdef CONFIG_SUPPORT_PCID 87#define LOAD_KERNEL_AS(reg) \ 88 movabs $x64KSKernelPML4 - KERNEL_ELF_BASE_OFFSET + (1 << 63), %reg; \ 89 movq %reg, %cr3; 90#else /* !CONFIG_SUPPORT_PCID */ 91#define LOAD_KERNEL_AS(reg) \ 92 lea x64KSKernelPML4 - KERNEL_ELF_BASE_OFFSET, %reg; \ 93 movq %reg, %cr3; 94#endif /* CONFIG_SUPPORT_PCID */ 95#else /* !CONFIG_KERNEL_SKIM_WINDOW */ 96 97#define LOAD_KERNEL_AS(reg) 98 99#endif /* CONFIG_KERNEL_SKIM_WINDOW */ 100 101/* Registers to be pushed after an interrupt 102 %rcx must be pushed beforehand */ 103#define INT_SAVE_STATE \ 104 push %r11; \ 105 /* skip FaultIP, RSP, Error, NextIP, RFLAGS */ \ 106 subq $(5 * 8), %rsp; \ 107 push %r15; \ 108 push %r9; \ 109 push %r8; \ 110 push %r10; \ 111 push %rdx; \ 112 push %r14; \ 113 push %r13; \ 114 push %r12; \ 115 push %rbp; \ 116 push %rbx; \ 117 push %rax; \ 118 push %rsi; \ 119 push %rdi 120 121/* Kernel exception handler if interrupt vector < 32. */ 122#define EXPAND_EXCEPT_1(number) \ 123 /* use saved stack */ \ 124 movq 32(%rsp), %rsp; \ 125 push %rcx; \ 126 movq $0x##number, %rcx; \ 127 jmp kernel_exception 128 129/* Kernel exception handler if interrupt vector >= 32. 130 Either a normal interrupt from the idle thread 131 or a nested interrupt. */ 132#define EXPAND_EXCEPT_0(number) \ 133 /* Check the saved esp, if its 0 we came from */\ 134 /* the idle thread and have a normal interrupt*/\ 135 addq $48, %rsp; \ 136 cmpq $0, -16(%rsp); \ 137 je 2b; \ 138 /* nested interrupt, use saved stack */ \ 139 movq -16(%rsp), %rsp; \ 140 /* skip 128 bytes as we need to respect the */ \ 141 /* red zone */ \ 142 subq $128, %rsp; \ 143 push %rcx; \ 144 movq $0x##number, %rcx; \ 145 jmp nested_interrupt 146 147#define INT_HANDLE_COMMON_EXCEPT(number,except) _expand_except_(except)(number) 148#define _expand_except_(except) EXPAND_EXCEPT_##except 149 150#define INT_HANDLER_COMMON(number,error_code,except) \ 151.global int_##number; \ 152.type int_##number, %function; \ 153int_##number: \ 154 error_code; \ 155 /* Check CPL */ \ 156 testq $3, 16(%rsp); \ 157 jz 1f; \ 1582: \ 159 LOAD_KERNEL_AS(rsp) \ 160 /* we need to not skip RSP, FaultIP, R11 and RCX for now */ \ 161 MAYBE_SWAPGS; \ 162 LOAD_USER_CONTEXT_OFFSET(4); \ 163 push %rcx; \ 164 movq $0x##number, %rcx; \ 165 jmp handle_interrupt; \ 1661: \ 167 INT_HANDLE_COMMON_EXCEPT(number,except); \ 168.size int_##number, . - int_##number; 169 170#define INT_HANDLER_WITH_ERR_CODE(number,except) INT_HANDLER_COMMON(number,,except) 171#define INT_HANDLER_WITHOUT_ERR_CODE(number,except) INT_HANDLER_COMMON(number,pushq $0x0,except) 172 173.section .text 174.code64 175 176INT_HANDLER_WITHOUT_ERR_CODE(00,1) 177INT_HANDLER_WITHOUT_ERR_CODE(01,1) 178INT_HANDLER_WITHOUT_ERR_CODE(02,1) 179INT_HANDLER_WITHOUT_ERR_CODE(03,1) 180INT_HANDLER_WITHOUT_ERR_CODE(04,1) 181INT_HANDLER_WITHOUT_ERR_CODE(05,1) 182INT_HANDLER_WITHOUT_ERR_CODE(06,1) 183INT_HANDLER_WITHOUT_ERR_CODE(07,1) 184INT_HANDLER_WITH_ERR_CODE(08,1) 185INT_HANDLER_WITHOUT_ERR_CODE(09,1) 186INT_HANDLER_WITH_ERR_CODE(0a,1) 187INT_HANDLER_WITH_ERR_CODE(0b,1) 188INT_HANDLER_WITH_ERR_CODE(0c,1) 189INT_HANDLER_WITH_ERR_CODE(0d,1) 190INT_HANDLER_WITH_ERR_CODE(0e,1) 191INT_HANDLER_WITHOUT_ERR_CODE(0f,1) 192 193INT_HANDLER_WITHOUT_ERR_CODE(10,1) 194INT_HANDLER_WITH_ERR_CODE(11,1) 195INT_HANDLER_WITHOUT_ERR_CODE(12,1) 196INT_HANDLER_WITHOUT_ERR_CODE(13,1) 197INT_HANDLER_WITHOUT_ERR_CODE(14,1) 198INT_HANDLER_WITHOUT_ERR_CODE(15,1) 199INT_HANDLER_WITHOUT_ERR_CODE(16,1) 200INT_HANDLER_WITHOUT_ERR_CODE(17,1) 201INT_HANDLER_WITHOUT_ERR_CODE(18,1) 202INT_HANDLER_WITHOUT_ERR_CODE(19,1) 203INT_HANDLER_WITHOUT_ERR_CODE(1a,1) 204INT_HANDLER_WITHOUT_ERR_CODE(1b,1) 205INT_HANDLER_WITHOUT_ERR_CODE(1c,1) 206INT_HANDLER_WITHOUT_ERR_CODE(1d,1) 207INT_HANDLER_WITHOUT_ERR_CODE(1e,1) 208INT_HANDLER_WITHOUT_ERR_CODE(1f,1) 209 210INT_HANDLER_WITHOUT_ERR_CODE(20,0) 211INT_HANDLER_WITHOUT_ERR_CODE(21,0) 212INT_HANDLER_WITHOUT_ERR_CODE(22,0) 213INT_HANDLER_WITHOUT_ERR_CODE(23,0) 214INT_HANDLER_WITHOUT_ERR_CODE(24,0) 215INT_HANDLER_WITHOUT_ERR_CODE(25,0) 216INT_HANDLER_WITHOUT_ERR_CODE(26,0) 217INT_HANDLER_WITHOUT_ERR_CODE(27,0) 218INT_HANDLER_WITHOUT_ERR_CODE(28,0) 219INT_HANDLER_WITHOUT_ERR_CODE(29,0) 220INT_HANDLER_WITHOUT_ERR_CODE(2a,0) 221INT_HANDLER_WITHOUT_ERR_CODE(2b,0) 222INT_HANDLER_WITHOUT_ERR_CODE(2c,0) 223INT_HANDLER_WITHOUT_ERR_CODE(2d,0) 224INT_HANDLER_WITHOUT_ERR_CODE(2e,0) 225INT_HANDLER_WITHOUT_ERR_CODE(2f,0) 226 227INT_HANDLER_WITHOUT_ERR_CODE(30,0) 228INT_HANDLER_WITHOUT_ERR_CODE(31,0) 229INT_HANDLER_WITHOUT_ERR_CODE(32,0) 230INT_HANDLER_WITHOUT_ERR_CODE(33,0) 231INT_HANDLER_WITHOUT_ERR_CODE(34,0) 232INT_HANDLER_WITHOUT_ERR_CODE(35,0) 233INT_HANDLER_WITHOUT_ERR_CODE(36,0) 234INT_HANDLER_WITHOUT_ERR_CODE(37,0) 235INT_HANDLER_WITHOUT_ERR_CODE(38,0) 236INT_HANDLER_WITHOUT_ERR_CODE(39,0) 237INT_HANDLER_WITHOUT_ERR_CODE(3a,0) 238INT_HANDLER_WITHOUT_ERR_CODE(3b,0) 239INT_HANDLER_WITHOUT_ERR_CODE(3c,0) 240INT_HANDLER_WITHOUT_ERR_CODE(3d,0) 241INT_HANDLER_WITHOUT_ERR_CODE(3e,0) 242INT_HANDLER_WITHOUT_ERR_CODE(3f,0) 243 244INT_HANDLER_WITHOUT_ERR_CODE(40,0) 245INT_HANDLER_WITHOUT_ERR_CODE(41,0) 246INT_HANDLER_WITHOUT_ERR_CODE(42,0) 247INT_HANDLER_WITHOUT_ERR_CODE(43,0) 248INT_HANDLER_WITHOUT_ERR_CODE(44,0) 249INT_HANDLER_WITHOUT_ERR_CODE(45,0) 250INT_HANDLER_WITHOUT_ERR_CODE(46,0) 251INT_HANDLER_WITHOUT_ERR_CODE(47,0) 252INT_HANDLER_WITHOUT_ERR_CODE(48,0) 253INT_HANDLER_WITHOUT_ERR_CODE(49,0) 254INT_HANDLER_WITHOUT_ERR_CODE(4a,0) 255INT_HANDLER_WITHOUT_ERR_CODE(4b,0) 256INT_HANDLER_WITHOUT_ERR_CODE(4c,0) 257INT_HANDLER_WITHOUT_ERR_CODE(4d,0) 258INT_HANDLER_WITHOUT_ERR_CODE(4e,0) 259INT_HANDLER_WITHOUT_ERR_CODE(4f,0) 260 261INT_HANDLER_WITHOUT_ERR_CODE(50,0) 262INT_HANDLER_WITHOUT_ERR_CODE(51,0) 263INT_HANDLER_WITHOUT_ERR_CODE(52,0) 264INT_HANDLER_WITHOUT_ERR_CODE(53,0) 265INT_HANDLER_WITHOUT_ERR_CODE(54,0) 266INT_HANDLER_WITHOUT_ERR_CODE(55,0) 267INT_HANDLER_WITHOUT_ERR_CODE(56,0) 268INT_HANDLER_WITHOUT_ERR_CODE(57,0) 269INT_HANDLER_WITHOUT_ERR_CODE(58,0) 270INT_HANDLER_WITHOUT_ERR_CODE(59,0) 271INT_HANDLER_WITHOUT_ERR_CODE(5a,0) 272INT_HANDLER_WITHOUT_ERR_CODE(5b,0) 273INT_HANDLER_WITHOUT_ERR_CODE(5c,0) 274INT_HANDLER_WITHOUT_ERR_CODE(5d,0) 275INT_HANDLER_WITHOUT_ERR_CODE(5e,0) 276INT_HANDLER_WITHOUT_ERR_CODE(5f,0) 277 278INT_HANDLER_WITHOUT_ERR_CODE(60,0) 279INT_HANDLER_WITHOUT_ERR_CODE(61,0) 280INT_HANDLER_WITHOUT_ERR_CODE(62,0) 281INT_HANDLER_WITHOUT_ERR_CODE(63,0) 282INT_HANDLER_WITHOUT_ERR_CODE(64,0) 283INT_HANDLER_WITHOUT_ERR_CODE(65,0) 284INT_HANDLER_WITHOUT_ERR_CODE(66,0) 285INT_HANDLER_WITHOUT_ERR_CODE(67,0) 286INT_HANDLER_WITHOUT_ERR_CODE(68,0) 287INT_HANDLER_WITHOUT_ERR_CODE(69,0) 288INT_HANDLER_WITHOUT_ERR_CODE(6a,0) 289INT_HANDLER_WITHOUT_ERR_CODE(6b,0) 290INT_HANDLER_WITHOUT_ERR_CODE(6c,0) 291INT_HANDLER_WITHOUT_ERR_CODE(6d,0) 292INT_HANDLER_WITHOUT_ERR_CODE(6e,0) 293INT_HANDLER_WITHOUT_ERR_CODE(6f,0) 294 295INT_HANDLER_WITHOUT_ERR_CODE(70,0) 296INT_HANDLER_WITHOUT_ERR_CODE(71,0) 297INT_HANDLER_WITHOUT_ERR_CODE(72,0) 298INT_HANDLER_WITHOUT_ERR_CODE(73,0) 299INT_HANDLER_WITHOUT_ERR_CODE(74,0) 300INT_HANDLER_WITHOUT_ERR_CODE(75,0) 301INT_HANDLER_WITHOUT_ERR_CODE(76,0) 302INT_HANDLER_WITHOUT_ERR_CODE(77,0) 303INT_HANDLER_WITHOUT_ERR_CODE(78,0) 304INT_HANDLER_WITHOUT_ERR_CODE(79,0) 305INT_HANDLER_WITHOUT_ERR_CODE(7a,0) 306INT_HANDLER_WITHOUT_ERR_CODE(7b,0) 307INT_HANDLER_WITHOUT_ERR_CODE(7c,0) 308INT_HANDLER_WITHOUT_ERR_CODE(7d,0) 309INT_HANDLER_WITHOUT_ERR_CODE(7e,0) 310INT_HANDLER_WITHOUT_ERR_CODE(7f,0) 311 312INT_HANDLER_WITHOUT_ERR_CODE(80,0) 313INT_HANDLER_WITHOUT_ERR_CODE(81,0) 314INT_HANDLER_WITHOUT_ERR_CODE(82,0) 315INT_HANDLER_WITHOUT_ERR_CODE(83,0) 316INT_HANDLER_WITHOUT_ERR_CODE(84,0) 317INT_HANDLER_WITHOUT_ERR_CODE(85,0) 318INT_HANDLER_WITHOUT_ERR_CODE(86,0) 319INT_HANDLER_WITHOUT_ERR_CODE(87,0) 320INT_HANDLER_WITHOUT_ERR_CODE(88,0) 321INT_HANDLER_WITHOUT_ERR_CODE(89,0) 322INT_HANDLER_WITHOUT_ERR_CODE(8a,0) 323INT_HANDLER_WITHOUT_ERR_CODE(8b,0) 324INT_HANDLER_WITHOUT_ERR_CODE(8c,0) 325INT_HANDLER_WITHOUT_ERR_CODE(8d,0) 326INT_HANDLER_WITHOUT_ERR_CODE(8e,0) 327INT_HANDLER_WITHOUT_ERR_CODE(8f,0) 328 329INT_HANDLER_WITHOUT_ERR_CODE(90,0) 330INT_HANDLER_WITHOUT_ERR_CODE(91,0) 331INT_HANDLER_WITHOUT_ERR_CODE(92,0) 332INT_HANDLER_WITHOUT_ERR_CODE(93,0) 333INT_HANDLER_WITHOUT_ERR_CODE(94,0) 334INT_HANDLER_WITHOUT_ERR_CODE(95,0) 335INT_HANDLER_WITHOUT_ERR_CODE(96,0) 336INT_HANDLER_WITHOUT_ERR_CODE(97,0) 337INT_HANDLER_WITHOUT_ERR_CODE(98,0) 338INT_HANDLER_WITHOUT_ERR_CODE(99,0) 339INT_HANDLER_WITHOUT_ERR_CODE(9a,0) 340INT_HANDLER_WITHOUT_ERR_CODE(9b,0) 341INT_HANDLER_WITHOUT_ERR_CODE(9c,0) 342INT_HANDLER_WITHOUT_ERR_CODE(9d,0) 343INT_HANDLER_WITHOUT_ERR_CODE(9e,0) 344INT_HANDLER_WITHOUT_ERR_CODE(9f,0) 345 346INT_HANDLER_WITHOUT_ERR_CODE(a0,0) 347INT_HANDLER_WITHOUT_ERR_CODE(a1,0) 348INT_HANDLER_WITHOUT_ERR_CODE(a2,0) 349INT_HANDLER_WITHOUT_ERR_CODE(a3,0) 350INT_HANDLER_WITHOUT_ERR_CODE(a4,0) 351INT_HANDLER_WITHOUT_ERR_CODE(a5,0) 352INT_HANDLER_WITHOUT_ERR_CODE(a6,0) 353INT_HANDLER_WITHOUT_ERR_CODE(a7,0) 354INT_HANDLER_WITHOUT_ERR_CODE(a8,0) 355INT_HANDLER_WITHOUT_ERR_CODE(a9,0) 356INT_HANDLER_WITHOUT_ERR_CODE(aa,0) 357INT_HANDLER_WITHOUT_ERR_CODE(ab,0) 358INT_HANDLER_WITHOUT_ERR_CODE(ac,0) 359INT_HANDLER_WITHOUT_ERR_CODE(ad,0) 360INT_HANDLER_WITHOUT_ERR_CODE(ae,0) 361INT_HANDLER_WITHOUT_ERR_CODE(af,0) 362 363INT_HANDLER_WITHOUT_ERR_CODE(b0,0) 364INT_HANDLER_WITHOUT_ERR_CODE(b1,0) 365INT_HANDLER_WITHOUT_ERR_CODE(b2,0) 366INT_HANDLER_WITHOUT_ERR_CODE(b3,0) 367INT_HANDLER_WITHOUT_ERR_CODE(b4,0) 368INT_HANDLER_WITHOUT_ERR_CODE(b5,0) 369INT_HANDLER_WITHOUT_ERR_CODE(b6,0) 370INT_HANDLER_WITHOUT_ERR_CODE(b7,0) 371INT_HANDLER_WITHOUT_ERR_CODE(b8,0) 372INT_HANDLER_WITHOUT_ERR_CODE(b9,0) 373INT_HANDLER_WITHOUT_ERR_CODE(ba,0) 374INT_HANDLER_WITHOUT_ERR_CODE(bb,0) 375INT_HANDLER_WITHOUT_ERR_CODE(bc,0) 376INT_HANDLER_WITHOUT_ERR_CODE(bd,0) 377INT_HANDLER_WITHOUT_ERR_CODE(be,0) 378INT_HANDLER_WITHOUT_ERR_CODE(bf,0) 379 380INT_HANDLER_WITHOUT_ERR_CODE(c0,0) 381INT_HANDLER_WITHOUT_ERR_CODE(c1,0) 382INT_HANDLER_WITHOUT_ERR_CODE(c2,0) 383INT_HANDLER_WITHOUT_ERR_CODE(c3,0) 384INT_HANDLER_WITHOUT_ERR_CODE(c4,0) 385INT_HANDLER_WITHOUT_ERR_CODE(c5,0) 386INT_HANDLER_WITHOUT_ERR_CODE(c6,0) 387INT_HANDLER_WITHOUT_ERR_CODE(c7,0) 388INT_HANDLER_WITHOUT_ERR_CODE(c8,0) 389INT_HANDLER_WITHOUT_ERR_CODE(c9,0) 390INT_HANDLER_WITHOUT_ERR_CODE(ca,0) 391INT_HANDLER_WITHOUT_ERR_CODE(cb,0) 392INT_HANDLER_WITHOUT_ERR_CODE(cc,0) 393INT_HANDLER_WITHOUT_ERR_CODE(cd,0) 394INT_HANDLER_WITHOUT_ERR_CODE(ce,0) 395INT_HANDLER_WITHOUT_ERR_CODE(cf,0) 396 397INT_HANDLER_WITHOUT_ERR_CODE(d0,0) 398INT_HANDLER_WITHOUT_ERR_CODE(d1,0) 399INT_HANDLER_WITHOUT_ERR_CODE(d2,0) 400INT_HANDLER_WITHOUT_ERR_CODE(d3,0) 401INT_HANDLER_WITHOUT_ERR_CODE(d4,0) 402INT_HANDLER_WITHOUT_ERR_CODE(d5,0) 403INT_HANDLER_WITHOUT_ERR_CODE(d6,0) 404INT_HANDLER_WITHOUT_ERR_CODE(d7,0) 405INT_HANDLER_WITHOUT_ERR_CODE(d8,0) 406INT_HANDLER_WITHOUT_ERR_CODE(d9,0) 407INT_HANDLER_WITHOUT_ERR_CODE(da,0) 408INT_HANDLER_WITHOUT_ERR_CODE(db,0) 409INT_HANDLER_WITHOUT_ERR_CODE(dc,0) 410INT_HANDLER_WITHOUT_ERR_CODE(dd,0) 411INT_HANDLER_WITHOUT_ERR_CODE(de,0) 412INT_HANDLER_WITHOUT_ERR_CODE(df,0) 413 414INT_HANDLER_WITHOUT_ERR_CODE(e0,0) 415INT_HANDLER_WITHOUT_ERR_CODE(e1,0) 416INT_HANDLER_WITHOUT_ERR_CODE(e2,0) 417INT_HANDLER_WITHOUT_ERR_CODE(e3,0) 418INT_HANDLER_WITHOUT_ERR_CODE(e4,0) 419INT_HANDLER_WITHOUT_ERR_CODE(e5,0) 420INT_HANDLER_WITHOUT_ERR_CODE(e6,0) 421INT_HANDLER_WITHOUT_ERR_CODE(e7,0) 422INT_HANDLER_WITHOUT_ERR_CODE(e8,0) 423INT_HANDLER_WITHOUT_ERR_CODE(e9,0) 424INT_HANDLER_WITHOUT_ERR_CODE(ea,0) 425INT_HANDLER_WITHOUT_ERR_CODE(eb,0) 426INT_HANDLER_WITHOUT_ERR_CODE(ec,0) 427INT_HANDLER_WITHOUT_ERR_CODE(ed,0) 428INT_HANDLER_WITHOUT_ERR_CODE(ee,0) 429INT_HANDLER_WITHOUT_ERR_CODE(ef,0) 430 431INT_HANDLER_WITHOUT_ERR_CODE(f0,0) 432INT_HANDLER_WITHOUT_ERR_CODE(f1,0) 433INT_HANDLER_WITHOUT_ERR_CODE(f2,0) 434INT_HANDLER_WITHOUT_ERR_CODE(f3,0) 435INT_HANDLER_WITHOUT_ERR_CODE(f4,0) 436INT_HANDLER_WITHOUT_ERR_CODE(f5,0) 437INT_HANDLER_WITHOUT_ERR_CODE(f6,0) 438INT_HANDLER_WITHOUT_ERR_CODE(f7,0) 439INT_HANDLER_WITHOUT_ERR_CODE(f8,0) 440INT_HANDLER_WITHOUT_ERR_CODE(f9,0) 441INT_HANDLER_WITHOUT_ERR_CODE(fa,0) 442INT_HANDLER_WITHOUT_ERR_CODE(fb,0) 443INT_HANDLER_WITHOUT_ERR_CODE(fc,0) 444INT_HANDLER_WITHOUT_ERR_CODE(fd,0) 445INT_HANDLER_WITHOUT_ERR_CODE(fe,0) 446INT_HANDLER_WITHOUT_ERR_CODE(ff,0) 447 448BEGIN_FUNC(handle_interrupt) 449 # push the rest of the registers 450 INT_SAVE_STATE 451 452 # switch to kernel stack 453 LOAD_KERNEL_STACK 454 455 # Set the arguments for c_x64_handle_interrupt 456 movq %rcx, %rdi 457 movq %rax, %rsi 458 459 # gtfo to C land, we will not return 460 call c_x64_handle_interrupt 461END_FUNC(handle_interrupt) 462 463BEGIN_FUNC(nested_interrupt) 464 # we got an interrupt from the kernel, call into c to save the irq number, 465 # then return back to where we were 466 INT_SAVE_STATE 467 movq %rcx, %rdi 468 call c_nested_interrupt 469 # disable the interrupt flag so we don't take any more interrupts 470 LOAD_IRQ_STACK(rbx) 471 andq $~0x200, 24(%rbx) 472 # return 473interrupt_return: 474 popq %rdi 475 popq %rsi 476 popq %rax 477 popq %rbx 478 popq %rbp 479 popq %r12 480 popq %r13 481 popq %r14 482 popq %rdx 483 popq %r10 484 popq %r8 485 popq %r9 486 popq %r15 487 /* skip RFLAGS, Error, NextIP, RSP, FaultIP */ 488 addq $(5 * 8), %rsp 489 popq %r11 490 popq %rcx 491 LOAD_IRQ_STACK(rsp) 492 addq $8, %rsp 493 iretq 494END_FUNC(nested_interrupt) 495 496BEGIN_FUNC(kernel_exception) 497 # push registers 498 INT_SAVE_STATE 499#if defined(CONFIG_HARDWARE_DEBUG_API) 500 /* Before giving up and panicking, we need to test for the extra case that 501 * this might be a kernel exception that is the result of EFLAGS.TF being 502 * set when SYSENTER was called. 503 * 504 * Since EFLAGS.TF is not disabled by SYSENTER, single-stepping continues 505 * into the kernel, and so causes a debug-exception in kernel code, since 506 * the CPU is trying to single-step the kernel code. 507 * 508 * So we test for EFLAGS.TF, and if it's set, we unset it, and let the 509 * exception continue. The debug exception handler will notice that it was 510 * kernel exception, and handle it appropriately -- that really just means 511 * setting EFLAGS.TF before SYSEXIT so that single-stepping resumes in the 512 * userspace thread. 513 */ 514 LOAD_IRQ_STACK(rdx) 515 movq 24(%rdx), %rax 516 movq $(1<<8), %rbx 517 testq %rbx, %rax 518 je .not_eflags_tf 519 520 /* Else it was EFLAGS.TF that caused the kernel exception on SYSENTER. 521 * So, unset the EFLAGS.TF on the stack and this causes the syscall that we 522 * will return to, to be able to execute properly. 523 * 524 * It will then be the debug exception handler's responsibility to re-set 525 * EFLAGS.TF for the userspace thread before it returns. 526 * 527 * So at this point we want to just unset EFLAGS.TF and IRET immediately. 528 */ 529 andq $~(1<<8), %rax 530 movq %rax, 24(%rdx) 531 532 /* Begin popping registers to IRET now. We don't need to consider any 533 * unexpected side effects because we are just immediately returning after 534 * entering. 535 */ 536 popq %rdi 537 popq %rsi 538 popq %rax 539 popq %rbx 540 popq %rbp 541 popq %r12 542 popq %r13 543 popq %r14 544 popq %rdx 545 popq %r10 546 popq %r8 547 popq %r9 548 popq %r15 549 /* skip RFLAGS, NextIP, Error, RSP, FaultIP */ 550 addq $(5 * 8), %rsp 551 popq %r11 552 popq %rcx 553 LOAD_IRQ_STACK(rsp) 554 addq $8, %rsp 555 iretq 556 557.not_eflags_tf: 558#endif /* CONFIG_HARDWARE_DEBUG_API */ 559 560 movq %rcx, %rdi 561 LOAD_IRQ_STACK(rsi) 562 movq 0(%rsi), %rsi # error code 563 LOAD_IRQ_STACK(rdx) 564 movq 8(%rdx), %rdx # RIP of the exception 565 LOAD_IRQ_STACK(rcx) 566 movq 32(%rcx), %rcx # RSP of the exception 567 LOAD_IRQ_STACK(r8) 568 movq 24(%r8), %r8 # RFLAGS 569 # handleKernelException(vector, errorcode, RIP, RSP, RFLAGS, CR0, CR2, CR3, CR4) 570 movq %cr0, %r9 571 movq %cr4, %r11 572 push %r11 573 movq %cr3, %r11 574 push %r11 575 movq %cr2, %r11 576 push %r11 577 call handleKernelException 578 addq $24, %rsp 579 # Set RIP in the saved register context to the new IP returned from handleKernelException 580 LOAD_IRQ_STACK(r8) 581 movq %rax, 8(%r8) 582 jmp interrupt_return 583END_FUNC(kernel_exception) 584 585# For a fast syscall the RFLAGS have been placed in 586# r11, the instruction *AFTER* the syscall is in 587# rcx. The current CS and SS have been loaded from 588# IA32_LSTAR (along with this code location). Additionally 589# the current RFLAGS (after saving) have been masked 590# with IA32_FMASK. 591BEGIN_FUNC(handle_fastsyscall) 592 LOAD_KERNEL_AS(rsp) 593 MAYBE_SWAPGS 594 LOAD_USER_CONTEXT 595 pushq $-1 # set Error -1 to mean entry via syscall 596 push %rcx # save NextIP 597 push %r11 # save RFLAGS 598 push %r15 # save R15 (message register) 599 push %r9 # save R9 (message register) 600 push %r8 # save R8 (message register) 601 push %r10 # save R10 (message register) 602 push %rdx # save RDX (syscall number) 603 push %r14 604 push %r13 605 push %r12 606 push %rbp 607 push %rbx 608 push %rax 609 push %rsi # save RSI (msgInfo register) 610 push %rdi # save RDI (capRegister) 611 612 # switch to kernel stack 613 LOAD_KERNEL_STACK 614 615 # RSI, RDI and RDX are already correct for calling c_handle_syscall 616 # gtfo to C land, we will not return 617#ifdef CONFIG_KERNEL_MCS 618 # mov reply to correct register for calling c_handle_syscall 619 movq %r12, %rcx 620#endif 621 jmp c_handle_syscall 622END_FUNC(handle_fastsyscall) 623 624# Handle Syscall (coming via sysenter) 625# Assume following register contents when called: 626# RAX : syscall number 627# RCX : user ESP 628# RDX : user EIP (pointing to the sysenter instruction) 629# RSP : NULL 630BEGIN_FUNC(handle_syscall) 631 /* We need to save r11, rdx TLS_BASE and RSP */ 632 LOAD_KERNEL_AS(rsp) 633 MAYBE_SWAPGS 634 LOAD_USER_CONTEXT_OFFSET(3) 635 push %r11 636 push %rdx # save FaultIP 637 push %rcx # save RSP 638 push $-1 # set Error -1 to mean entry via syscall 639 push %rdx # save FaultIP (which will need to be updated later) 640 pushf # save RFLAGS 641 orq $0x200, (%rsp) # set interrupt bit in save RFLAGS 642 push %r15 # save R15 (message register) 643 push %r9 # save R9 (message register) 644 push %r8 # save R8 (message register) 645 push %r10 # save R10 (message register) 646 subq $8, %rsp # skip RDX 647 push %r14 648 push %r13 649 push %r12 650 push %rbp 651 push %rbx 652 push %rax # save RAX (syscall number) 653 push %rsi # save RSI (msgInfo register) 654 push %rdi # save RDI (capRegister) 655 656 # switch to kernel stack 657 LOAD_KERNEL_STACK 658 659 # RSI, RDI are already correct for calling c_handle_syscall 660 movq %rax, %rdx 661 # gtfo to C land, we will not return 662#ifdef CONFIG_KERNEL_MCS 663 # mov reply to correct register for calling c_handle_syscall 664 movq %r12, %rcx 665#endif 666 call c_handle_syscall 667END_FUNC(handle_syscall) 668 669# Handle vmexit 670# RSP points to the end of the VCPUs general purpose register array 671#ifdef CONFIG_VTX 672BEGIN_FUNC(handle_vmexit) 673 MAYBE_SWAPGS 674 push %rbp 675 push %rdi 676 push %rsi 677 push %rdx 678 push %rcx 679 push %rbx 680 push %rax 681 682 # switch to kernel stack 683 LOAD_KERNEL_STACK 684 # Handle the vmexit, we will not return 685 call c_handle_vmexit 686END_FUNC(handle_vmexit) 687#endif /* CONFIG_VTX */ 688