1/*
2 * Copyright (c) 2015,2016 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetstr. 6, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#include <asmoffsets.h>
11#include <offsets.h>
12#include <exceptions.h>
13
14.global do_resume
15.global vectors
16
17.macro invalid_exception vector
18    /* Just stick the trap frame on the kernel stack - we're about to panic
19     * anyway.  */
20
21    add sp, sp, #-(34 * 8)
22
23    /* Spill the GPRs */
24    stp  x0,  x1, [sp]
25    stp  x2,  x3, [sp, #( 2 * 8)]
26    stp  x4,  x5, [sp, #( 4 * 8)]
27    stp  x6,  x7, [sp, #( 6 * 8)]
28    stp  x8,  x9, [sp, #( 8 * 8)]
29    stp x10, x11, [sp, #(10 * 8)]
30    stp x12, x13, [sp, #(12 * 8)]
31    stp x14, x15, [sp, #(14 * 8)]
32    stp x16, x17, [sp, #(16 * 8)]
33    stp x18, x19, [sp, #(18 * 8)]
34    stp x20, x21, [sp, #(20 * 8)]
35    stp x22, x23, [sp, #(22 * 8)]
36    stp x24, x25, [sp, #(24 * 8)]
37    stp x26, x27, [sp, #(26 * 8)]
38    stp x28, x29, [sp, #(28 * 8)]
39
40    /* Stack pointer */
41    mrs x0, sp_el0
42    stp x30, x0, [sp, #(30 * 8)]
43
44    mrs x0, elr_el1
45    mrs x1, spsr_el1
46    stp x0, x1, [sp, #(32 * 8)]
47
48    /* Exception Syndrome Register */
49    mrs x2, esr_el1
50
51    /* Exception vector */
52    mov x3, \vector
53
54    /* Base of the register save area. */
55    mov x4, sp
56
57    /* Arguments: x0 = EPC, x1 = SPSR, x2 = ESR, x3 = vector, x4 = save area. */
58    b fatal_kernel_fault
59.endm
60
61/**********************************/
62/*** Start of exception vectors ***/
63/**********************************/
64
65/* The AArch64 exception vectors occupy 2kiB */
66.align 11
67vectors:
68
69/* Offset 0x000 */
70/* Exceptions from the current EL, on the EL0 stack.  We never do this. */
71/* Each table entry occupies 128B, which lets us put up to 32 instructions
72 * here before we branch. */
73.align 7 /* 0x000 */
74el1_sp_el0_sync:
75    invalid_exception AARCH64_EVECTOR_EL0_SYNC
76.align 7 /* 0x080 */
77el1_sp_el0_irq:
78    invalid_exception AARCH64_EVECTOR_EL0_IRQ /* TODO: Waait, wenn we do wait for interrupt in kernel, that shouldnt be invalid */
79.align 7 /* 0x100 */
80el1_sp_el0_fiq:
81    invalid_exception AARCH64_EVECTOR_EL0_FIQ
82.align 7 /* 0x180 */
83el1_sp_el0_serror:
84    invalid_exception AARCH64_EVECTOR_EL0_SERROR
85
86/* Offset 0x200 */
87/* Exceptions from the kernel itself, at EL1. */
88.align 7 /* 0x200 */
89el1_sync:
90    invalid_exception AARCH64_EVECTOR_EL1_SYNC
91.align 7 /* 0x280 */
92el1_irq:
93    invalid_exception AARCH64_EVECTOR_EL1_IRQ
94.align 7 /* 0x300 */
95el1_fiq:
96    invalid_exception AARCH64_EVECTOR_EL1_FIQ
97.align 7 /* 0x380 */
98el1_serror:
99    invalid_exception AARCH64_EVECTOR_EL1_SERROR
100
101/* Offset 0x400 */
102/* Exceptions from user level, EL0, executing AArch64.  For any of these four
103 * exceptions, the stack pointer is SP_EL1, which is left at the top of
104 * 'kernel_stack'. */
105 .align 7 /* 0x400 */
106/*
107 * Synchronous exceptions from a lower execution level using AArch64: SVC
108 * (syscall), data abort, prefetch abort and undefined instruction.
109 *
110 * Assumption:
111 * when coming from a syscall (SVC) the arguments are in registers x0-x11
112 */
113el0_aarch64_sync:
114    /* Reenable breakpoints and aborts.  Interrupts remain disabled. */
115    msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */
116
117    /* Spill a few working registers.
118     * Registers x0-x11 contain the syscall arguments. We use x12-, as these won't
119     * need to be restored if we're doing a syscall; they're caller-saved.  We
120     * preserve x0-x6 in registers unless we branch to the abort path, so that they're
121     * immediately available to the syscall handler, sys_syscall.
122     */
123    stp x14, x15, [sp, #-(2 * 8)]!
124    stp x12, x13, [sp, #-(2 * 8)]!
125
126    /* The EL1 thread ID register holds the address of the currently-running
127     * dispatcher's shared control block. */
128    mrs x13, tpidr_el1
129
130    /* x13 = dispatcher_shared_aarch64 */
131
132    /* Exception PC */
133    mrs x12, elr_el1
134
135    /* x12 = EPC, x13 = dispatcher_shared_aarch64 */
136
137    /* Check whether the current dispatcher is disabled.  See el0_aarch64_irq
138     * for a description of this test. */
139    ldp x14, x15, [x13, #OFFSETOF_DISP_CRIT_PC_LOW]
140    cmp x14, x12
141    ccmp x15, x12, #0, ls
142    ldr w14, [x13, #OFFSETOF_DISP_DISABLED]
143    ccmp x14, xzr, #0, ls
144    /* NE <-> (low <= PC && PC < high) || disabled != 0 */
145
146    /* Figure out what sort of exception we've got.  All paths need this. */
147    mrs x14, esr_el1  /* Exception Syndrome Register */
148    lsr x14, x14, #26 /* Exception Class field is bits [31:26] */
149
150    /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */
151
152    /* Faults while disabled should be rare, if the critical section is short,
153     * and will always be within the dispatcher code.  Therefore we branch out
154     * to the slowpath here. */
155    b.ne el0_sync_disabled
156
157    /* 13 instructions to here. */
158
159    /* All exceptions use the 'enabled' area if the dispatcher is enabled. */
160    add x13, x13, #OFFSETOF_DISP_ENABLED_AREA
161
162    /* x12 = EPC, x13 = base of save area, x14 = EC */
163
164    /* now we branch off as we are running out of space */
165    b save_syscall_context
166
167    /* 15 instructions to here. */
168
169.align 7 /* 0x480 */
170/* An interrupt at user level */
171el0_aarch64_irq:
172    /* Reenable breakpoints and aborts.  Interrupts remain disabled. */
173    msr daifset, #3 /* IRQ and FIQ masked, Debug and Abort enabled. */
174
175    /* Free scratch registers. */
176    stp x2, x3, [sp, #-(2 * 8)]!
177    stp x0, x1, [sp, #-(2 * 8)]!
178
179    /* Find the dispatcher. */
180    mrs x3, tpidr_el1
181
182    /* Get the exception address (EPC) */
183    mrs x1, elr_el1
184
185    /* x0 = crit_pc_low, x1 = EPC,
186       x2 = crit_pc_high, x3 = dispatcher_shared_aarch64 */
187
188    /* Dispatcher disabled? */
189    ldp x0, x2, [x3, #OFFSETOF_DISP_CRIT_PC_LOW]
190
191    /* Condition LS holds iff low <= PC. */
192    cmp x0, x1
193    /* Short-circuit computation of P /\ Q: If the success condition for P
194     * (low <= PC) holds, here LS (Less or Same), then test Q (PC < high),
195     * giving either HI (true) or LS (false).  Otherwise, set the condition
196     * flags to a failing state for Q (LS), as the conjunction cannot hold if
197     * the P does not. */
198    ccmp x2, x1, #0, ls
199    /* If HI (C=1, Z=0) holds, then the test executed and passed, which means
200     * that P held previously (low <= PC), and Q holds now (PC < high).
201     * Otherwise LS holds, and thus either HI held before this test, and thus
202     * PC < low, or the test executed and failed, in which case high <= PC.
203     * Thus condition HI holds iff low <= PC /\ PC < high. */
204    ldr w0, [x3, #OFFSETOF_DISP_DISABLED]
205    /* We do the same to calculate (P /\ Q) \/ R: If P /\ Q doesn't hold, we
206     * need to test R (is the disabled flag 0), giving either EQ or NE.  If
207     * P /\ Q holds, we skip the test, as it doesn't affect the result, and
208     * instead set the condition code directly NE, i.e. disabled=1. */
209    ccmp x0, xzr, #0, ls
210    /* NE <-> (low <= PC && PC < high) || disabled != 0 */
211
212    /* x1 = EPC, x3 = dispatcher_shared_aarch64 */
213
214    /* Choose the right save area, using the condition flags. */
215    add x0, x3, #OFFSETOF_DISP_ENABLED_AREA
216    add x2, x3, #OFFSETOF_DISP_DISABLED_AREA
217    csel x0, x2, x0, ne
218
219    /* x0 = save area, x1 = EPC */
220
221    /* 13 instructions. */
222
223    /* Save the register context, starting from x4.  x0-x3, the scratch
224     * registers, can be copied to the trap frame from within the C handler,
225     * as can the user stack pointer and SPSR, which are sitting in their own
226     * system registers. */
227    stp  x4,  x5, [x0, #( 4 * 8)]
228    stp  x6,  x7, [x0, #( 6 * 8)]
229    stp  x8,  x9, [x0, #( 8 * 8)]
230    stp x10, x11, [x0, #(10 * 8)]
231    stp x12, x13, [x0, #(12 * 8)]
232    stp x14, x15, [x0, #(14 * 8)]
233    stp x16, x17, [x0, #(16 * 8)]
234    stp x18, x19, [x0, #(18 * 8)]
235    stp x20, x21, [x0, #(20 * 8)]
236    stp x22, x23, [x0, #(22 * 8)]
237    stp x24, x25, [x0, #(24 * 8)]
238    stp x26, x27, [x0, #(26 * 8)]
239    stp x28, x29, [x0, #(28 * 8)]
240    str x30,      [x0, #(30 * 8)]
241
242    /* 27 instructions. */
243
244    /* Load the saved scratch registers, and pass them as arguments to the
245     * handler.  We can't save them ourselves as we've run out of
246     * instructions.  We need to do at least this, to clear our stack frame.
247     * */
248    ldp x2, x3, [sp], #16 /* x0, x1 */
249    ldp x4, x5, [sp], #16 /* x2, x3 */
250
251    /* x0 = save area, x1 = EPC,
252     * x2 = user x0, x3 = user x1,
253     * x4 = user x2, x5 = user x3 */
254    b handle_irq
255
256    /* 30 instructions. */
257
258.align 7 /* 0x500 */
259/* We don't implement fast IRQs */
260el0_aarch64_fiq:
261    invalid_exception AARCH64_EVECTOR_EL0_FIQ
262
263.align 7 /* 0x580 */
264/* A delayed abort.  We don't handle this. */
265el0_aarch64_serror:
266    invalid_exception AARCH64_EVECTOR_EL0_SERROR
267
268/* Offset 0x600 */
269/* Exceptions from user level, EL0, executing AArch32.  This is currently
270 * unimplemented. */
271.align 7 /* 0x600 */
272el0_aarch32_sync:
273    invalid_exception AARCH32_EVECTOR_EL0_SYNC
274.align 7 /* 0x680 */
275el0_aarch32_irq:
276    invalid_exception AARCH32_EVECTOR_EL0_IRQ
277.align 7 /* 0x700 */
278el0_aarch32_fiq:
279    invalid_exception AARCH32_EVECTOR_EL0_FIQ
280.align 7 /* 0x780 */
281el0_aarch32_serror:
282    invalid_exception AARCH32_EVECTOR_EL0_SERROR
283
284.align 11
285/********************************/
286/*** End of exception vectors ***/
287/********************************/
288
289/* The tail of the user syscall handler doesn't fit in the table. */
290save_syscall_context:
291    /* x12 = EPC, x13 = base of save area, x14 = EC */
292
293    /*
294     * We need to save callee save registers r19-30 no matter what, so get on with it.
295     */
296
297    /* Callee-saved registers */
298    stp x19, x20, [x13, #(19 * 8)]
299    stp x21, x22, [x13, #(21 * 8)]
300    stp x23, x24, [x13, #(23 * 8)]
301    stp x25, x26, [x13, #(25 * 8)]
302    stp x27, x28, [x13, #(27 * 8)]
303    stp x29, x30, [x13, #(29 * 8)] /* FP & LR */
304
305    stp q0, q1, [x13, #(34 * 8)]
306    stp q2, q3, [x13, #(38 * 8)]
307    stp q4, q5, [x13, #(42 * 8)]
308    stp q6, q7, [x13, #(46 * 8)]
309    stp q8, q9, [x13, #(50 * 8)]
310    stp q10, q11, [x13, #(54 * 8)]
311    stp q12, q13, [x13, #(58 * 8)]
312    stp q14, q15, [x13, #(62 * 8)]
313    stp q16, q17, [x13, #(66 * 8)]
314    stp q18, q19, [x13, #(70 * 8)]
315    stp q20, q21, [x13, #(74 * 8)]
316    stp q22, q23, [x13, #(78 * 8)]
317    stp q24, q25, [x13, #(82 * 8)]
318    stp q26, q27, [x13, #(86 * 8)]
319    stp q28, q29, [x13, #(90 * 8)]
320    stp q30, q31, [x13, #(94 * 8)]
321
322    /* High registers are now available. */
323
324    /* User SP and PC */
325    mrs x20, sp_el0
326    stp x20, x12, [x13, #(31 * 8)]
327
328    /* SPSR */
329    mrs x19, spsr_el1
330    str x19, [x13, #(33 * 8)]
331
332    /* Is this a syscall? */
333    cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */
334    b.ne el0_abort_enabled
335
336    /*
337     * we need to save r7-r11 as those are the syscall arguments which are passed
338     * on the stack on a function call, so we need to move them to the trap frame
339     */
340    stp x7, x8,     [x13, #(7 * 8)]
341    stp x9, x10,    [x13, #(9 * 8)]
342    str x11,        [x13, #(11 * 8)]
343
344    /* If we're here, this is a syscall and we don't need to restore the
345     * scratch registers. Just throw the stack frame away. */
346    add sp, sp, #(4 * 8)
347
348    /* Pass the address of the trap frame as argument 8. */
349    mov x7, x13
350
351    /* Jump to the common (C) syscall handler. */
352    b sys_syscall
353
354
355/* The tail of the user abort handler doesn't fit in the table. */
356el0_abort_enabled:
357    /* x12 = EPC, x13 = base of save area, x14 = EC */
358
359    /*
360     * we need to save the caller-saved registers. As it is an
361     * abort, those weren't saved by prior to the call.
362     */
363
364
365    /* We saved x12-x15 in our stack frame. We load them from
366     * memory so we can store them we use x20-23 as they where
367     * saved in the common path */
368    ldp x20,  x21,  [sp], #16 /* x12, x13 */
369    ldp x22,  x23,  [sp], #16 /* x14, x15 */
370
371    /* now we can store the caller-saved registers in the trap frame */
372    stp x0,   x1,  [x13]
373    stp x2,   x3,  [x13, #(  2 * 8)]
374    stp x4,   x5,  [x13, #(  4 * 8)]
375    stp x6,   x7,  [x13, #(  6 * 8)]
376    stp x8,   x9,  [x13, #(  8 * 8)]
377    stp x10, x11,  [x13, #( 10 * 8)]
378    stp x20, x21,  [x13, #(12 * 8)]
379    stp x22, x23,  [x13, #(14 * 8)]
380    stp x16, x17,  [x13, #(16 * 8)]
381    str x18,       [x13, #(18 * 8)]
382
383    /* x19-x30, SP, SPSR and ELR were saved in the common path. */
384
385    /* x12 = EPC, x13 = base of save area, x14 = EC */
386
387    /* All registers are now available. */
388
389    /* Pass the EPC and the save area address to the handler. */
390    mov x0, x12
391    mov x1, x14
392    mov x2, x13
393
394    /* Now we can jump to the handler. */
395    b handle_user_fault
396
397/* A synchronous exception while the dispatcher is disabled.  This is the
398 * slowpath.  In fact, libbarrelfish currently (2015) just refuses to continue
399 * if this ever happens. */
400el0_sync_disabled:
401    /* x12 = EPC, x14 = EC, x13 = dispatcher_shared_aarch64 */
402
403    /* Filter out aborts. */
404    cmp x14, #0x15 /* SVC or HVC from AArch64 EL0 */
405    b.ne el0_abort_disabled
406
407    /* Use the 'disabled' area. */
408    add x13, x13, #OFFSETOF_DISP_DISABLED_AREA
409
410    /* Jump back into the syscall path. */
411    b save_syscall_context
412
413/* This is the *really* unexpected path: a page fault in the dispatcher
414 * critical section.  It's (relatively) slow, and libbarrelfish doesn't
415 * actually handle it at present (2015). */
416el0_abort_disabled:
417
418    /* x12 = EPC, x13 = base of save area, x14 = EC */
419
420    /* x11 = EC,
421       x13 = dispatcher_shared_aarch64 */
422
423    /* Use the 'trap' area. */
424    add x13, x13, #OFFSETOF_DISP_TRAP_AREA
425
426    /* Save the reduced context. */
427    /* Callee-saved registers */
428    stp x19, x20, [x13, #(19 * 8)]
429    stp x21, x22, [x13, #(21 * 8)]
430    stp x23, x24, [x13, #(23 * 8)]
431    stp x25, x26, [x13, #(25 * 8)]
432    stp x27, x28, [x13, #(27 * 8)]
433    stp x29, x30, [x13, #(29 * 8)] /* FP & LR */
434
435    /* SPSR */
436    mrs x19, spsr_el1
437    str x19, [x13, #(31 * 8)]
438
439    /* User PC and SP */
440    mrs x20, sp_el0
441    stp x20, x9, [x13, #(32 * 8)]
442
443    /* x12 = EPC, x13 = base of save area, x14 = EC */
444
445    /* Now reuse the 'enabled' abort handler. */
446    b el0_abort_enabled
447
448/* Restore the dispatcher's execution context.  x0 holds the base of the
449 * appropriate save area. */
450do_resume:
451    /* Skip to the end... */
452    add x0, x0, #((34 + 30 * 2) * 8)
453
454    /* Reset the kernel stack pointer. */
455    ldr x1, =kernel_stack   // load the address of the kernel sack
456    ldr x2,  [x1]           // read the entry of the kernel stack
457    mov sp, x2
458
459    /* Flush the TLB - XXX kill this with fire. */
460    dsb sy
461    tlbi vmalle1
462    dsb sy
463    isb
464
465    /* Restore the NEON registers. */
466    ldp q30, q31, [x0], #-(2 * 16)
467    ldp q28, q29, [x0], #-(2 * 16)
468    ldp q26, q27, [x0], #-(2 * 16)
469    ldp q24, q25, [x0], #-(2 * 16)
470    ldp q22, q23, [x0], #-(2 * 16)
471    ldp q20, q21, [x0], #-(2 * 16)
472    ldp q18, q19, [x0], #-(2 * 16)
473    ldp q16, q17, [x0], #-(2 * 16)
474    ldp q14, q15, [x0], #-(2 * 16)
475    ldp q12, q13, [x0], #-(2 * 16)
476    ldp q10, q11, [x0], #-(2 * 16)
477    ldp q8, q9, [x0], #-(2 * 16)
478    ldp q6, q7, [x0], #-(2 * 16)
479    ldp q4, q5, [x0], #-(2 * 16)
480    ldp q2, q3, [x0], #-(2 * 16)
481    ldp q0, q1, [x0], #-(2 * 8)
482
483    /* Restore SPSR, PC (ELR) and SP, which need temporary registers, before
484     * we restore those. */
485    ldp x2, x3, [x0], #-8 /* pc, spsr */
486    ldr x1, [x0], #-16 /* stack */
487    msr spsr_el1, x3
488    msr elr_el1,  x2
489    msr sp_el0,   x1
490
491    /* Restore the general-purpose registers. */
492    ldp x29, x30, [x0], #-(2 * 8)
493    ldp x27, x28, [x0], #-(2 * 8)
494    ldp x25, x26, [x0], #-(2 * 8)
495    ldp x23, x24, [x0], #-(2 * 8)
496    ldp x21, x22, [x0], #-(2 * 8)
497    ldp x19, x20, [x0], #-(2 * 8)
498    ldp x17, x18, [x0], #-(2 * 8)
499    ldp x15, x16, [x0], #-(2 * 8)
500    ldp x13, x14, [x0], #-(2 * 8)
501    ldp x11, x12, [x0], #-(2 * 8)
502    ldp x9,  x10, [x0], #-(2 * 8)
503    ldp x7,  x8,  [x0], #-(2 * 8)
504    ldp x5,  x6,  [x0], #-(2 * 8)
505    ldp x3,  x4,  [x0], #-(2 * 8)
506    ldp x1,  x2,  [x0], #-(1 * 8)
507    ldr x0,       [x0]
508
509    /* Return from exception.  This clears the load exclusive monitor. */
510    eret
511