1/* 2 * Copyright (c) 2015,2016 ETH Zurich. 3 * All rights reserved. 4 * 5 * This file is distributed under the terms in the attached LICENSE file. 6 * If you do not find this file, copies can be found by writing to: 7 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group. 8 */ 9 10#include <asmoffsets.h> 11#include <offsets.h> 12#include <exceptions.h> 13 14.global do_resume 15.global vectors 16 17.macro invalid_exception vector 18 /* Just stick the trap frame on the kernel stack - we're about to panic 19 * anyway. */ 20 21 add sp, sp, #-(34 * 8) 22 23 /* Spill the GPRs */ 24 stp x0, x1, [sp] 25 stp x2, x3, [sp, #( 2 * 8)] 26 stp x4, x5, [sp, #( 4 * 8)] 27 stp x6, x7, [sp, #( 6 * 8)] 28 stp x8, x9, [sp, #( 8 * 8)] 29 stp x10, x11, [sp, #(10 * 8)] 30 stp x12, x13, [sp, #(12 * 8)] 31 stp x14, x15, [sp, #(14 * 8)] 32 stp x16, x17, [sp, #(16 * 8)] 33 stp x18, x19, [sp, #(18 * 8)] 34 stp x20, x21, [sp, #(20 * 8)] 35 stp x22, x23, [sp, #(22 * 8)] 36 stp x24, x25, [sp, #(24 * 8)] 37 stp x26, x27, [sp, #(26 * 8)] 38 stp x28, x29, [sp, #(28 * 8)] 39 40 /* Stack pointer */ 41 mrs x0, sp_el0 42 stp x30, x0, [sp, #(30 * 8)] 43 44 mrs x0, elr_el1 45 mrs x1, spsr_el1 46 stp x0, x1, [sp, #(32 * 8)] 47 48 /* Exception Syndrome Register */ 49 mrs x2, esr_el1 50 51 /* Exception vector */ 52 mov x3, \vector 53 54 /* Base of the register save area. */ 55 mov x4, sp 56 57 /* Arguments: x0 = EPC, x1 = SPSR, x2 = ESR, x3 = vector, x4 = save area. */ 58 b fatal_kernel_fault 59.endm 60 61/**********************************/ 62/*** Start of exception vectors ***/ 63/**********************************/ 64 65/* The AArch64 exception vectors occupy 2kiB */ 66.align 11 67vectors: 68 69/* Offset 0x000 */ 70/* Exceptions from the current EL, on the EL0 stack. We never do this. */ 71/* Each table entry occupies 128B, which lets us put up to 32 instructions 72 * here before we branch. */ 73.align 7 /* 0x000 */ 74el1_sp_el0_sync: 75 invalid_exception AARCH64_EVECTOR_EL0_SYNC 76.align 7 /* 0x080 */ 77el1_sp_el0_irq: 78 invalid_exception AARCH64_EVECTOR_EL0_IRQ /* TODO: Waait, wenn we do wait for interrupt in kernel, that shouldnt be invalid */ 79.align 7 /* 0x100 */ 80el1_sp_el0_fiq: 81 invalid_exception AARCH64_EVECTOR_EL0_FIQ 82.align 7 /* 0x180 */ 83el1_sp_el0_serror: 84 invalid_exception AARCH64_EVECTOR_EL0_SERROR 85 86/* Offset 0x200 */ 87/* Exceptions from the kernel itself, at EL1. */ 88.align 7 /* 0x200 */ 89el1_sync: 90 invalid_exception AARCH64_EVECTOR_EL1_SYNC 91.align 7 /* 0x280 */ 92el1_irq: 93 invalid_exception AARCH64_EVECTOR_EL1_IRQ 94.align 7 /* 0x300 */ 95el1_fiq: 96 invalid_exception AARCH64_EVECTOR_EL1_FIQ 97.align 7 /* 0x380 */ 98el1_serror: 99 invalid_exception AARCH64_EVECTOR_EL1_SERROR 100 101/* Offset 0x400 */ 102/* Exceptions from user level, EL0, executing AArch64. For any of these four 103 * exceptions, the stack pointer is SP_EL1, which is left at the top of 104 * 'kernel_stack'. */ 105 .align 7 /* 0x400 */ 106/* 107 * Synchronous exceptions from a lower execution level using AArch64: SVC 108 * (syscall), data abort, prefetch abort and undefined instruction. 109 * 110 * Assumption: 111 * when coming from a syscall (SVC) the arguments are in registers x0-x11 112 */ 113el0_aarch64_sync: 114 /* Reenable breakpoints and aborts. Interrupts remain disabled. */ 115 msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */ 116 117 /* Spill a few working registers. 118 * Registers x0-x11 contain the syscall arguments. We use x12-, as these won't 119 * need to be restored if we're doing a syscall; they're caller-saved. We 120 * preserve x0-x6 in registers unless we branch to the abort path, so that they're 121 * immediately available to the syscall handler, sys_syscall. 122 */ 123 stp x14, x15, [sp, #-(2 * 8)]! 124 stp x12, x13, [sp, #-(2 * 8)]! 125 126 /* The EL1 thread ID register holds the address of the currently-running 127 * dispatcher's shared control block. */ 128 mrs x13, tpidr_el1 129 130 /* x13 = dispatcher_shared_aarch64 */ 131 132 /* Exception PC */ 133 mrs x12, elr_el1 134 135 /* x12 = EPC, x13 = dispatcher_shared_aarch64 */ 136 137 /* Check whether the current dispatcher is disabled. See el0_aarch64_irq 138 * for a description of this test. */ 139 ldp x14, x15, [x13, #OFFSETOF_DISP_CRIT_PC_LOW] 140 cmp x14, x12 141 ccmp x15, x12, #0, ls 142 ldr w14, [x13, #OFFSETOF_DISP_DISABLED] 143 ccmp x14, xzr, #0, ls 144 /* NE <-> (low <= PC && PC < high) || disabled != 0 */ 145 146 /* Figure out what sort of exception we've got. All paths need this. */ 147 mrs x14, esr_el1 /* Exception Syndrome Register */ 148 lsr x14, x14, #26 /* Exception Class field is bits [31:26] */ 149 150 /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */ 151 152 /* Faults while disabled should be rare, if the critical section is short, 153 * and will always be within the dispatcher code. Therefore we branch out 154 * to the slowpath here. */ 155 b.ne el0_sync_disabled 156 157 /* 13 instructions to here. */ 158 159 /* All exceptions use the 'enabled' area if the dispatcher is enabled. */ 160 add x13, x13, #OFFSETOF_DISP_ENABLED_AREA 161 162 /* x12 = EPC, x13 = base of save area, x14 = EC */ 163 164 /* now we branch off as we are running out of space */ 165 b save_syscall_context 166 167 /* 15 instructions to here. */ 168 169.align 7 /* 0x480 */ 170/* An interrupt at user level */ 171el0_aarch64_irq: 172 /* Reenable breakpoints and aborts. Interrupts remain disabled. */ 173 msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */ 174 175 /* Free scratch registers. */ 176 stp x2, x3, [sp, #-(2 * 8)]! 177 stp x0, x1, [sp, #-(2 * 8)]! 178 179 /* Find the dispatcher. */ 180 mrs x3, tpidr_el1 181 182 /* Get the exception address (EPC) */ 183 mrs x1, elr_el1 184 185 /* x0 = crit_pc_low, x1 = EPC, 186 x2 = crit_pc_high, x3 = dispatcher_shared_aarch64 */ 187 188 /* Dispatcher disabled? */ 189 ldp x0, x2, [x3, #OFFSETOF_DISP_CRIT_PC_LOW] 190 191 /* Condition LS holds iff low <= PC. */ 192 cmp x0, x1 193 /* Short-circuit computation of P /\ Q: If the success condition for P 194 * (low <= PC) holds, here LS (Less or Same), then test Q (PC < high), 195 * giving either HI (true) or LS (false). Otherwise, set the condition 196 * flags to a failing state for Q (LS), as the conjunction cannot hold if 197 * the P does not. */ 198 ccmp x2, x1, #0, ls 199 /* If HI (C=1, Z=0) holds, then the test executed and passed, which means 200 * that P held previously (low <= PC), and Q holds now (PC < high). 201 * Otherwise LS holds, and thus either HI held before this test, and thus 202 * PC < low, or the test executed and failed, in which case high <= PC. 203 * Thus condition HI holds iff low <= PC /\ PC < high. */ 204 ldr w0, [x3, #OFFSETOF_DISP_DISABLED] 205 /* We do the same to calculate (P /\ Q) \/ R: If P /\ Q doesn't hold, we 206 * need to test R (is the disabled flag 0), giving either EQ or NE. If 207 * P /\ Q holds, we skip the test, as it doesn't affect the result, and 208 * instead set the condition code directly NE, i.e. disabled=1. */ 209 ccmp x0, xzr, #0, ls 210 /* NE <-> (low <= PC && PC < high) || disabled != 0 */ 211 212 /* x1 = EPC, x3 = dispatcher_shared_aarch64 */ 213 214 /* Choose the right save area, using the condition flags. */ 215 add x0, x3, #OFFSETOF_DISP_ENABLED_AREA 216 add x2, x3, #OFFSETOF_DISP_DISABLED_AREA 217 csel x0, x2, x0, ne 218 219 /* x0 = save area, x1 = EPC */ 220 221 /* 13 instructions. */ 222 223 /* Save the register context, starting from x4. x0-x3, the scratch 224 * registers, can be copied to the trap frame from within the C handler, 225 * as can the user stack pointer and SPSR, which are sitting in their own 226 * system registers. */ 227 stp x4, x5, [x0, #( 4 * 8)] 228 stp x6, x7, [x0, #( 6 * 8)] 229 stp x8, x9, [x0, #( 8 * 8)] 230 stp x10, x11, [x0, #(10 * 8)] 231 stp x12, x13, [x0, #(12 * 8)] 232 stp x14, x15, [x0, #(14 * 8)] 233 stp x16, x17, [x0, #(16 * 8)] 234 stp x18, x19, [x0, #(18 * 8)] 235 stp x20, x21, [x0, #(20 * 8)] 236 stp x22, x23, [x0, #(22 * 8)] 237 stp x24, x25, [x0, #(24 * 8)] 238 stp x26, x27, [x0, #(26 * 8)] 239 stp x28, x29, [x0, #(28 * 8)] 240 str x30, [x0, #(30 * 8)] 241 242 /* 27 instructions. */ 243 244 /* Load the saved scratch registers, and pass them as arguments to the 245 * handler. We can't save them ourselves as we've run out of 246 * instructions. We need to do at least this, to clear our stack frame. 247 * */ 248 ldp x2, x3, [sp], #16 /* x0, x1 */ 249 ldp x4, x5, [sp], #16 /* x2, x3 */ 250 251 /* x0 = save area, x1 = EPC, 252 * x2 = user x0, x3 = user x1, 253 * x4 = user x2, x5 = user x3 */ 254 b handle_irq 255 256 /* 30 instructions. */ 257 258.align 7 /* 0x500 */ 259/* We don't implement fast IRQs */ 260el0_aarch64_fiq: 261 invalid_exception AARCH64_EVECTOR_EL0_FIQ 262 263.align 7 /* 0x580 */ 264/* A delayed abort. We don't handle this. */ 265el0_aarch64_serror: 266 invalid_exception AARCH64_EVECTOR_EL0_SERROR 267 268/* Offset 0x600 */ 269/* Exceptions from user level, EL0, executing AArch32. This is currently 270 * unimplemented. */ 271.align 7 /* 0x600 */ 272el0_aarch32_sync: 273 invalid_exception AARCH32_EVECTOR_EL0_SYNC 274.align 7 /* 0x680 */ 275el0_aarch32_irq: 276 invalid_exception AARCH32_EVECTOR_EL0_IRQ 277.align 7 /* 0x700 */ 278el0_aarch32_fiq: 279 invalid_exception AARCH32_EVECTOR_EL0_FIQ 280.align 7 /* 0x780 */ 281el0_aarch32_serror: 282 invalid_exception AARCH32_EVECTOR_EL0_SERROR 283 284.align 11 285/********************************/ 286/*** End of exception vectors ***/ 287/********************************/ 288 289/* The tail of the user syscall handler doesn't fit in the table. */ 290save_syscall_context: 291 /* x12 = EPC, x13 = base of save area, x14 = EC */ 292 293 /* 294 * We need to save callee save registers r19-30 no matter what, so get on with it. 295 */ 296 297 /* Callee-saved registers */ 298 stp x19, x20, [x13, #(19 * 8)] 299 stp x21, x22, [x13, #(21 * 8)] 300 stp x23, x24, [x13, #(23 * 8)] 301 stp x25, x26, [x13, #(25 * 8)] 302 stp x27, x28, [x13, #(27 * 8)] 303 stp x29, x30, [x13, #(29 * 8)] /* FP & LR */ 304 305 stp q0, q1, [x13, #(34 * 8)] 306 stp q2, q3, [x13, #(38 * 8)] 307 stp q4, q5, [x13, #(42 * 8)] 308 stp q6, q7, [x13, #(46 * 8)] 309 stp q8, q9, [x13, #(50 * 8)] 310 stp q10, q11, [x13, #(54 * 8)] 311 stp q12, q13, [x13, #(58 * 8)] 312 stp q14, q15, [x13, #(62 * 8)] 313 stp q16, q17, [x13, #(66 * 8)] 314 stp q18, q19, [x13, #(70 * 8)] 315 stp q20, q21, [x13, #(74 * 8)] 316 stp q22, q23, [x13, #(78 * 8)] 317 stp q24, q25, [x13, #(82 * 8)] 318 stp q26, q27, [x13, #(86 * 8)] 319 stp q28, q29, [x13, #(90 * 8)] 320 stp q30, q31, [x13, #(94 * 8)] 321 322 /* High registers are now available. */ 323 324 /* User SP and PC */ 325 mrs x20, sp_el0 326 stp x20, x12, [x13, #(31 * 8)] 327 328 /* SPSR */ 329 mrs x19, spsr_el1 330 str x19, [x13, #(33 * 8)] 331 332 /* Is this a syscall? */ 333 cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */ 334 b.ne el0_abort_enabled 335 336 /* 337 * we need to save r7-r11 as those are the syscall arguments which are passed 338 * on the stack on a function call, so we need to move them to the trap frame 339 */ 340 stp x7, x8, [x13, #(7 * 8)] 341 stp x9, x10, [x13, #(9 * 8)] 342 str x11, [x13, #(11 * 8)] 343 344 /* If we're here, this is a syscall and we don't need to restore the 345 * scratch registers. Just throw the stack frame away. */ 346 add sp, sp, #(4 * 8) 347 348 /* Pass the address of the trap frame as argument 8. */ 349 mov x7, x13 350 351 /* Jump to the common (C) syscall handler. */ 352 b sys_syscall 353 354 355/* The tail of the user abort handler doesn't fit in the table. */ 356el0_abort_enabled: 357 /* x12 = EPC, x13 = base of save area, x14 = EC */ 358 359 /* 360 * we need to save the caller-saved registers. As it is an 361 * abort, those weren't saved by prior to the call. 362 */ 363 364 365 /* We saved x12-x15 in our stack frame. We load them from 366 * memory so we can store them we use x20-23 as they where 367 * saved in the common path */ 368 ldp x20, x21, [sp], #16 /* x12, x13 */ 369 ldp x22, x23, [sp], #16 /* x14, x15 */ 370 371 /* now we can store the caller-saved registers in the trap frame */ 372 stp x0, x1, [x13] 373 stp x2, x3, [x13, #( 2 * 8)] 374 stp x4, x5, [x13, #( 4 * 8)] 375 stp x6, x7, [x13, #( 6 * 8)] 376 stp x8, x9, [x13, #( 8 * 8)] 377 stp x10, x11, [x13, #( 10 * 8)] 378 stp x20, x21, [x13, #(12 * 8)] 379 stp x22, x23, [x13, #(14 * 8)] 380 stp x16, x17, [x13, #(16 * 8)] 381 str x18, [x13, #(18 * 8)] 382 383 /* x19-x30, SP, SPSR and ELR were saved in the common path. */ 384 385 /* x12 = EPC, x13 = base of save area, x14 = EC */ 386 387 /* All registers are now available. */ 388 389 /* Pass the EPC and the save area address to the handler. */ 390 mov x0, x12 391 mov x1, x14 392 mov x2, x13 393 394 /* Now we can jump to the handler. */ 395 b handle_user_fault 396 397/* A synchronous exception while the dispatcher is disabled. This is the 398 * slowpath. In fact, libbarrelfish currently (2015) just refuses to continue 399 * if this ever happens. */ 400el0_sync_disabled: 401 /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */ 402 403 /* Filter out aborts. */ 404 cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */ 405 b.ne el0_abort_disabled 406 407 /* Use the 'disabled' area. */ 408 add x13, x13, #OFFSETOF_DISP_DISABLED_AREA 409 410 /* Jump back into the syscall path. */ 411 b save_syscall_context 412 413/* This is the *really* unexpected path: a page fault in the dispatcher 414 * critical section. It's (relatively) slow, and libbarrelfish doesn't 415 * actually handle it at present (2015). */ 416el0_abort_disabled: 417 418 /* x12 = EPC, x13 = base of save area, x14 = EC */ 419 420 /* x11 = EC, 421 x13 = dispatcher_shared_aarch64 */ 422 423 /* Use the 'trap' area. */ 424 add x13, x13, #OFFSETOF_DISP_TRAP_AREA 425 426 /* Save the reduced context. */ 427 /* Callee-saved registers */ 428 stp x19, x20, [x13, #(19 * 8)] 429 stp x21, x22, [x13, #(21 * 8)] 430 stp x23, x24, [x13, #(23 * 8)] 431 stp x25, x26, [x13, #(25 * 8)] 432 stp x27, x28, [x13, #(27 * 8)] 433 stp x29, x30, [x13, #(29 * 8)] /* FP & LR */ 434 435 /* SPSR */ 436 mrs x19, spsr_el1 437 str x19, [x13, #(31 * 8)] 438 439 /* User PC and SP */ 440 mrs x20, sp_el0 441 stp x20, x9, [x13, #(32 * 8)] 442 443 /* x12 = EPC, x13 = base of save area, x14 = EC */ 444 445 /* Now reuse the 'enabled' abort handler. */ 446 b el0_abort_enabled 447 448/* Restore the dispatcher's execution context. x0 holds the base of the 449 * appropriate save area. */ 450do_resume: 451 /* Skip to the end... */ 452 add x0, x0, #((34 + 30 * 2) * 8) 453 454 /* Reset the kernel stack pointer. */ 455 ldr x1, =kernel_stack // load the address of the kernel sack 456 ldr x2, [x1] // read the entry of the kernel stack 457 mov sp, x2 458 459 /* Flush the TLB - XXX kill this with fire. */ 460 dsb sy 461 tlbi vmalle1 462 dsb sy 463 isb 464 465 /* Restore the NEON registers. */ 466 ldp q30, q31, [x0], #-(2 * 16) 467 ldp q28, q29, [x0], #-(2 * 16) 468 ldp q26, q27, [x0], #-(2 * 16) 469 ldp q24, q25, [x0], #-(2 * 16) 470 ldp q22, q23, [x0], #-(2 * 16) 471 ldp q20, q21, [x0], #-(2 * 16) 472 ldp q18, q19, [x0], #-(2 * 16) 473 ldp q16, q17, [x0], #-(2 * 16) 474 ldp q14, q15, [x0], #-(2 * 16) 475 ldp q12, q13, [x0], #-(2 * 16) 476 ldp q10, q11, [x0], #-(2 * 16) 477 ldp q8, q9, [x0], #-(2 * 16) 478 ldp q6, q7, [x0], #-(2 * 16) 479 ldp q4, q5, [x0], #-(2 * 16) 480 ldp q2, q3, [x0], #-(2 * 16) 481 ldp q0, q1, [x0], #-(2 * 8) 482 483 /* Restore SPSR, PC (ELR) and SP, which need temporary registers, before 484 * we restore those. */ 485 ldp x2, x3, [x0], #-8 /* pc, spsr */ 486 ldr x1, [x0], #-16 /* stack */ 487 msr spsr_el1, x3 488 msr elr_el1, x2 489 msr sp_el0, x1 490 491 /* Restore the general-purpose registers. */ 492 ldp x29, x30, [x0], #-(2 * 8) 493 ldp x27, x28, [x0], #-(2 * 8) 494 ldp x25, x26, [x0], #-(2 * 8) 495 ldp x23, x24, [x0], #-(2 * 8) 496 ldp x21, x22, [x0], #-(2 * 8) 497 ldp x19, x20, [x0], #-(2 * 8) 498 ldp x17, x18, [x0], #-(2 * 8) 499 ldp x15, x16, [x0], #-(2 * 8) 500 ldp x13, x14, [x0], #-(2 * 8) 501 ldp x11, x12, [x0], #-(2 * 8) 502 ldp x9, x10, [x0], #-(2 * 8) 503 ldp x7, x8, [x0], #-(2 * 8) 504 ldp x5, x6, [x0], #-(2 * 8) 505 ldp x3, x4, [x0], #-(2 * 8) 506 ldp x1, x2, [x0], #-(1 * 8) 507 ldr x0, [x0] 508 509 /* Return from exception. This clears the load exclusive monitor. */ 510 eret 511