1/*
2 * Copyright 2019-2022 Haiku, Inc. All Rights Reserved.
3 * Distributed under the terms of the MIT License.
4 */
5#include <int.h>
6
7#include <arch/smp.h>
8#include <boot/kernel_args.h>
9#include <device_manager.h>
10#include <kscheduler.h>
11#include <ksyscalls.h>
12#include <interrupt_controller.h>
13#include <smp.h>
14#include <thread.h>
15#include <timer.h>
16#include <util/AutoLock.h>
17#include <util/DoublyLinkedList.h>
18#include <util/kernel_cpp.h>
19#include <vm/vm.h>
20#include <vm/vm_priv.h>
21#include <vm/VMAddressSpace.h>
22#include "syscall_numbers.h"
23#include "VMSAv8TranslationMap.h"
24#include <string.h>
25
26#include "soc.h"
27#include "arch_int_gicv2.h"
28
29#define TRACE_ARCH_INT
30#ifdef TRACE_ARCH_INT
31#	define TRACE(x) dprintf x
32#else
33#	define TRACE(x) ;
34#endif
35
36//#define TRACE_ARCH_INT_IFRAMES
37
38// An iframe stack used in the early boot process when we don't have
39// threads yet.
40struct iframe_stack gBootFrameStack;
41
42// In order to avoid store/restore of large FPU state, it is assumed that
43// this code and page fault handling doesn't use FPU.
44// Instead this is called manually when handling IRQ or syscall.
45extern "C" void _fp_save(aarch64_fpu_state *fpu);
46extern "C" void _fp_restore(aarch64_fpu_state *fpu);
47
48void
49arch_int_enable_io_interrupt(int32 irq)
50{
51	InterruptController *ic = InterruptController::Get();
52	if (ic != NULL)
53		ic->EnableInterrupt(irq);
54}
55
56
57void
58arch_int_disable_io_interrupt(int32 irq)
59{
60	InterruptController *ic = InterruptController::Get();
61	if (ic != NULL)
62		ic->DisableInterrupt(irq);
63}
64
65
66int32
67arch_int_assign_to_cpu(int32 irq, int32 cpu)
68{
69	// Not yet supported.
70	return 0;
71}
72
73
74static void
75print_iframe(const char *event, struct iframe *frame)
76{
77	if (event)
78		dprintf("Exception: %s\n", event);
79
80	dprintf("ELR=%016lx SPSR=%016lx\n",
81		frame->elr, frame->spsr);
82	dprintf("LR=%016lx  SP  =%016lx\n",
83		frame->lr, frame->sp);
84}
85
86
87status_t
88arch_int_init(kernel_args *args)
89{
90	return B_OK;
91}
92
93
94status_t
95arch_int_init_post_vm(kernel_args *args)
96{
97	InterruptController *ic = NULL;
98	if (strcmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2) == 0) {
99		ic = new(std::nothrow) GICv2InterruptController(
100			args->arch_args.interrupt_controller.regs1.start,
101			args->arch_args.interrupt_controller.regs2.start);
102	}
103
104	if (ic == NULL)
105		return B_ERROR;
106
107	return B_OK;
108}
109
110
111status_t
112arch_int_init_io(kernel_args* args)
113{
114	return B_OK;
115}
116
117
118status_t
119arch_int_init_post_device_manager(struct kernel_args *args)
120{
121	return B_ENTRY_NOT_FOUND;
122}
123
124
125// TODO: reuse things from VMSAv8TranslationMap
126
127
128static int page_bits = 12;
129
130static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12);
131static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3);
132static constexpr uint64_t kAttrSWDBM = (1UL << 55);
133static constexpr uint64_t kAttrAF = (1UL << 10);
134static constexpr uint64_t kAttrAP2 = (1UL << 7);
135
136
137static uint64_t*
138TableFromPa(phys_addr_t pa)
139{
140	return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa);
141}
142
143
144static bool
145fixup_entry(phys_addr_t ptPa, int level, addr_t va, bool wr)
146{
147	int tableBits = page_bits - 3;
148	uint64_t tableMask = (1UL << tableBits) - 1;
149
150	int shift = tableBits * (3 - level) + page_bits;
151	uint64_t entrySize = 1UL << shift;
152	uint64_t entryMask = entrySize - 1;
153
154	int index = (va >> shift) & tableMask;
155
156	uint64_t *pte = &TableFromPa(ptPa)[index];
157
158	int type = *pte & 0x3;
159	uint64_t addr = *pte & kPteAddrMask;
160
161	if ((level == 3 && type == 0x3) || (level < 3 && type == 0x1)) {
162		if (!wr && (*pte & kAttrAF) == 0) {
163			atomic_or64((int64*)pte, kAttrAF);
164			return true;
165		}
166		if (wr && (*pte & kAttrSWDBM) != 0 && (*pte & kAttrAP2) != 0) {
167			atomic_and64((int64*)pte, ~kAttrAP2);
168			asm("tlbi vaae1is, %0 \n dsb ish"::"r"(va >> page_bits));
169			return true;
170		}
171	} else if (level < 3 && type == 0x3) {
172		return fixup_entry(addr, level + 1, va, wr);
173	}
174
175	return false;
176}
177
178
179void
180after_exception()
181{
182	Thread* thread = thread_get_current_thread();
183	cpu_status state = disable_interrupts();
184	if (thread->cpu->invoke_scheduler) {
185		SpinLocker schedulerLocker(thread->scheduler_lock);
186		scheduler_reschedule(B_THREAD_READY);
187		schedulerLocker.Unlock();
188		restore_interrupts(state);
189	} else if (thread->post_interrupt_callback != NULL) {
190		void (*callback)(void*) = thread->post_interrupt_callback;
191		void* data = thread->post_interrupt_data;
192
193		thread->post_interrupt_callback = NULL;
194		thread->post_interrupt_data = NULL;
195
196		restore_interrupts(state);
197
198		callback(data);
199	}
200}
201
202
203// Little helper class for handling the
204// iframe stack as used by KDL.
205class IFrameScope {
206public:
207	IFrameScope(struct iframe *iframe) {
208		fThread = thread_get_current_thread();
209		if (fThread)
210			arm64_push_iframe(&fThread->arch_info.iframes, iframe);
211		else
212			arm64_push_iframe(&gBootFrameStack, iframe);
213	}
214
215	virtual ~IFrameScope() {
216		// pop iframe
217		if (fThread)
218			arm64_pop_iframe(&fThread->arch_info.iframes);
219		else
220			arm64_pop_iframe(&gBootFrameStack);
221	}
222private:
223	Thread* fThread;
224};
225
226
227extern "C" void
228do_sync_handler(iframe * frame)
229{
230#ifdef TRACE_ARCH_INT_IFRAMES
231	print_iframe("Sync abort", frame);
232#endif
233
234	IFrameScope scope(frame);
235
236	bool isExec = false;
237	switch (ESR_ELx_EXCEPTION(frame->esr)) {
238		case EXCP_INSN_ABORT_L:
239		case EXCP_INSN_ABORT:
240			isExec = true;
241		case EXCP_DATA_ABORT_L:
242		case EXCP_DATA_ABORT:
243		{
244			bool write = (frame->esr & ISS_DATA_WnR) != 0;
245			bool known = false;
246
247			int initialLevel = VMSAv8TranslationMap::CalcStartLevel(48, 12);
248			phys_addr_t ptPa;
249			bool addrType = (frame->far & (1UL << 63)) != 0;
250			if (addrType)
251				ptPa = READ_SPECIALREG(TTBR1_EL1);
252			else
253				ptPa = READ_SPECIALREG(TTBR0_EL1);
254
255			switch (frame->esr & ISS_DATA_DFSC_MASK) {
256				case ISS_DATA_DFSC_TF_L0:
257				case ISS_DATA_DFSC_TF_L1:
258				case ISS_DATA_DFSC_TF_L2:
259				case ISS_DATA_DFSC_TF_L3:
260					known = true;
261				break;
262
263				case ISS_DATA_DFSC_AFF_L1:
264				case ISS_DATA_DFSC_AFF_L2:
265				case ISS_DATA_DFSC_AFF_L3:
266					known = true;
267					if (fixup_entry(ptPa, initialLevel, frame->far, false))
268						return;
269				break;
270
271				case ISS_DATA_DFSC_PF_L1:
272				case ISS_DATA_DFSC_PF_L2:
273				case ISS_DATA_DFSC_PF_L3:
274					known = true;
275					if (write && fixup_entry(ptPa, initialLevel, frame->far, true))
276						return;
277				break;
278			}
279
280			if (!known)
281				break;
282
283			if (debug_debugger_running()) {
284				Thread* thread = thread_get_current_thread();
285				if (thread != NULL) {
286					cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
287					if (cpu->fault_handler != 0) {
288						debug_set_page_fault_info(frame->far, frame->elr,
289							write ? DEBUG_PAGE_FAULT_WRITE : 0);
290						frame->elr = cpu->fault_handler;
291						frame->sp = cpu->fault_handler_stack_pointer;
292						return;
293					}
294				}
295			}
296
297			Thread *thread = thread_get_current_thread();
298			ASSERT(thread);
299
300			bool isUser = (frame->spsr & PSR_M_MASK) == PSR_M_EL0t;
301
302			if ((frame->spsr & PSR_I) != 0) {
303				// interrupts disabled
304				uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler);
305				if (thread->fault_handler != 0) {
306					frame->elr = handler;
307					return;
308				}
309			} else if (thread->page_faults_allowed != 0) {
310				dprintf("PF: %lx\n", frame->far);
311				enable_interrupts();
312				addr_t ret = 0;
313				vm_page_fault(frame->far, frame->elr, write, isExec, isUser, &ret);
314				if (ret != 0)
315					frame->elr = ret;
316				return;
317			}
318
319			panic("unhandled pagefault! FAR=%lx ELR=%lx ESR=%lx",
320				frame->far, frame->elr, frame->esr);
321			break;
322		}
323
324		case EXCP_SVC64:
325		{
326			uint32 imm = (frame->esr & 0xffff);
327
328			uint32 count = imm & 0x1f;
329			uint32 syscall = imm >> 5;
330
331			uint64_t args[20];
332			if (count > 20) {
333				frame->x[0] = B_ERROR;
334				return;
335			}
336
337			memset(args, 0, sizeof(args));
338			memcpy(args, frame->x, (count < 8 ? count : 8) * 8);
339
340			if (count > 8) {
341				if (!IS_USER_ADDRESS(frame->sp)
342					|| user_memcpy(&args[8], (void*)frame->sp, (count - 8) * 8) != B_OK) {
343					frame->x[0] = B_BAD_ADDRESS;
344					return;
345				}
346			}
347
348			_fp_save(&frame->fpu);
349
350			thread_at_kernel_entry(system_time());
351
352			enable_interrupts();
353			syscall_dispatcher(syscall, (void*)args, &frame->x[0]);
354
355			{
356				disable_interrupts();
357				atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
358				if ((thread_get_current_thread()->flags
359					& (THREAD_FLAGS_SIGNALS_PENDING
360					| THREAD_FLAGS_DEBUG_THREAD
361					| THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) {
362					enable_interrupts();
363					thread_at_kernel_exit();
364				} else {
365					thread_at_kernel_exit_no_signals();
366				}
367				if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) {
368					panic("syscall restart");
369				}
370			}
371
372			_fp_restore(&frame->fpu);
373
374			return;
375		}
376	}
377
378	panic("unhandled exception! FAR=%lx ELR=%lx ESR=%lx (EC=%lx)",
379		frame->far, frame->elr, frame->esr, (frame->esr >> 26) & 0x3f);
380}
381
382
383extern "C" void
384do_error_handler(iframe * frame)
385{
386#ifdef TRACE_ARCH_INT_IFRAMES
387	print_iframe("Error", frame);
388#endif
389
390	IFrameScope scope(frame);
391
392	panic("unhandled error! FAR=%lx ELR=%lx ESR=%lx", frame->far, frame->elr, frame->esr);
393}
394
395
396extern "C" void
397do_irq_handler(iframe * frame)
398{
399#ifdef TRACE_ARCH_INT_IFRAMES
400	print_iframe("IRQ", frame);
401#endif
402
403	IFrameScope scope(frame);
404
405	_fp_save(&frame->fpu);
406
407	InterruptController *ic = InterruptController::Get();
408	if (ic != NULL)
409		ic->HandleInterrupt();
410
411	after_exception();
412
413	_fp_restore(&frame->fpu);
414}
415
416
417extern "C" void
418do_fiq_handler(iframe * frame)
419{
420#ifdef TRACE_ARCH_INT_IFRAMES
421	print_iframe("FIQ", frame);
422#endif
423
424	IFrameScope scope(frame);
425
426	panic("do_fiq_handler");
427}
428