1/*
2 * Copyright 2003-2023, Haiku Inc. All rights reserved.
3 * Distributed under the terms of the MIT License.
4 *
5 * Authors:
6 * 		Axel D��rfler <axeld@pinc-software.de>
7 * 		Ingo Weinhold <bonefish@cs.tu-berlin.de>
8 * 		Fran��ois Revol <revol@free.fr>
9 *              Ithamar R. Adema <ithamar@upgrade-android.com>
10 *
11 * Copyright 2001, Travis Geiselbrecht. All rights reserved.
12 * Distributed under the terms of the NewOS License.
13 */
14
15
16#include <int.h>
17
18#include <arch_cpu_defs.h>
19#include <arch/smp.h>
20#include <boot/kernel_args.h>
21#include <device_manager.h>
22#include <kscheduler.h>
23#include <interrupt_controller.h>
24#include <ksyscalls.h>
25#include <smp.h>
26#include <syscall_numbers.h>
27#include <thread.h>
28#include <timer.h>
29#include <AutoDeleterDrivers.h>
30#include <util/AutoLock.h>
31#include <util/DoublyLinkedList.h>
32#include <util/kernel_cpp.h>
33#include <vm/vm.h>
34#include <vm/vm_priv.h>
35#include <vm/VMAddressSpace.h>
36#include <algorithm>
37#include <string.h>
38
39#include <drivers/bus/FDT.h>
40#include "arch_int_gicv2.h"
41#include "soc.h"
42
43#include "soc_pxa.h"
44#include "soc_omap3.h"
45#include "soc_sun4i.h"
46
47#include "ARMVMTranslationMap.h"
48
49//#define TRACE_ARCH_INT
50#ifdef TRACE_ARCH_INT
51#	define TRACE(x...) dprintf(x)
52#else
53#	define TRACE(x...) ;
54#endif
55
56#define VECTORPAGE_SIZE		64
57#define USER_VECTOR_ADDR_LOW	0x00000000
58#define USER_VECTOR_ADDR_HIGH	0xffff0000
59
60extern int _vectors_start;
61extern int _vectors_end;
62
63static area_id sUserVectorPageArea;
64static void *sUserVectorPageAddress;
65//static fdt_module_info *sFdtModule;
66
67// An iframe stack used in the early boot process when we don't have
68// threads yet.
69struct iframe_stack gBootFrameStack;
70
71
72void
73arch_int_enable_io_interrupt(int32 irq)
74{
75	TRACE("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq);
76	InterruptController *ic = InterruptController::Get();
77	if (ic != NULL)
78		ic->EnableInterrupt(irq);
79}
80
81
82void
83arch_int_disable_io_interrupt(int32 irq)
84{
85	TRACE("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq);
86	InterruptController *ic = InterruptController::Get();
87	if (ic != NULL)
88		ic->DisableInterrupt(irq);
89}
90
91
92/* arch_int_*_interrupts() and friends are in arch_asm.S */
93
94int32
95arch_int_assign_to_cpu(int32 irq, int32 cpu)
96{
97	// Not yet supported.
98	return 0;
99}
100
101
102static void
103print_iframe(const char *event, struct iframe *frame)
104{
105	if (event)
106		dprintf("Exception: %s\n", event);
107
108	dprintf("R00=%08x R01=%08x R02=%08x R03=%08x\n"
109		"R04=%08x R05=%08x R06=%08x R07=%08x\n",
110		frame->r0, frame->r1, frame->r2, frame->r3,
111		frame->r4, frame->r5, frame->r6, frame->r7);
112	dprintf("R08=%08x R09=%08x R10=%08x R11=%08x\n"
113		"R12=%08x SPs=%08x LRs=%08x PC =%08x\n",
114		frame->r8, frame->r9, frame->r10, frame->r11,
115		frame->r12, frame->svc_sp, frame->svc_lr, frame->pc);
116	dprintf("             SPu=%08x LRu=%08x CPSR=%08x\n",
117		frame->usr_sp, frame->usr_lr, frame->spsr);
118}
119
120
121extern "C" void arm_vector_init(void);
122
123
124status_t
125arch_int_init(kernel_args *args)
126{
127	TRACE("arch_int_init\n");
128
129	// copy vector code to vector page
130	memcpy((void*)USER_VECTOR_ADDR_HIGH, &_vectors_start, VECTORPAGE_SIZE);
131
132	// initialize stack for vectors
133	arm_vector_init();
134
135	// enable high vectors
136	arm_set_sctlr(arm_get_sctlr() | SCTLR_HIGH_VECTORS);
137
138	return B_OK;
139}
140
141
142status_t
143arch_int_init_post_vm(kernel_args *args)
144{
145	TRACE("arch_int_init_post_vm\n");
146
147	sUserVectorPageAddress = (addr_t*)USER_VECTOR_ADDR_HIGH;
148	sUserVectorPageArea = create_area("user_vectorpage",
149		(void **)&sUserVectorPageAddress, B_EXACT_ADDRESS,
150		B_PAGE_SIZE, B_ALREADY_WIRED, B_READ_AREA | B_EXECUTE_AREA);
151
152	if (sUserVectorPageArea < 0)
153		panic("user vector page @ %p could not be created (%x)!",
154			sUserVectorPageAddress, sUserVectorPageArea);
155
156	if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2,
157		sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
158		InterruptController *ic = new(std::nothrow) GICv2InterruptController(
159			args->arch_args.interrupt_controller.regs1.start,
160			args->arch_args.interrupt_controller.regs2.start);
161		if (ic == NULL)
162			return B_NO_MEMORY;
163	} else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_OMAP3,
164		sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
165		InterruptController *ic = new(std::nothrow) OMAP3InterruptController(
166			args->arch_args.interrupt_controller.regs1.start);
167		if (ic == NULL)
168			return B_NO_MEMORY;
169	} else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_PXA,
170		sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
171		InterruptController *ic = new(std::nothrow) PXAInterruptController(
172			args->arch_args.interrupt_controller.regs1.start);
173		if (ic == NULL)
174			return B_NO_MEMORY;
175	} else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_SUN4I,
176		sizeof(args->arch_args.interrupt_controller.kind)) == 0) {
177		InterruptController *ic = new(std::nothrow) Sun4iInterruptController(
178			args->arch_args.interrupt_controller.regs1.start);
179		if (ic == NULL)
180			return B_NO_MEMORY;
181	} else {
182		panic("No interrupt controllers found!\n");
183	}
184
185	return B_OK;
186}
187
188
189status_t
190arch_int_init_io(kernel_args* args)
191{
192	return B_OK;
193}
194
195
196status_t
197arch_int_init_post_device_manager(struct kernel_args *args)
198{
199	return B_ENTRY_NOT_FOUND;
200}
201
202
203// Little helper class for handling the
204// iframe stack as used by KDL.
205class IFrameScope {
206public:
207	IFrameScope(struct iframe *iframe) {
208		fThread = thread_get_current_thread();
209		if (fThread)
210			arm_push_iframe(&fThread->arch_info.iframes, iframe);
211		else
212			arm_push_iframe(&gBootFrameStack, iframe);
213	}
214
215	virtual ~IFrameScope() {
216		// pop iframe
217		if (fThread)
218			arm_pop_iframe(&fThread->arch_info.iframes);
219		else
220			arm_pop_iframe(&gBootFrameStack);
221	}
222private:
223	Thread* fThread;
224};
225
226
227extern "C" void
228arch_arm_undefined(struct iframe *iframe)
229{
230	print_iframe("Undefined Instruction", iframe);
231	IFrameScope scope(iframe); // push/pop iframe
232
233	panic("not handled!");
234}
235
236
237extern "C" void
238arch_arm_syscall(struct iframe *iframe)
239{
240#ifdef TRACE_ARCH_INT
241	print_iframe("Software interrupt", iframe);
242#endif
243
244	IFrameScope scope(iframe);
245
246	uint32_t syscall = *(uint32_t *)(iframe->pc-4) & 0x00ffffff;
247	TRACE("syscall number: %d\n", syscall);
248
249	uint32_t args[20];
250	if (syscall < kSyscallCount) {
251		TRACE("syscall(%s,%d)\n",
252			kExtendedSyscallInfos[syscall].name,
253			kExtendedSyscallInfos[syscall].parameter_count);
254
255		int argSize = kSyscallInfos[syscall].parameter_size;
256		memcpy(args, &iframe->r0, std::min<int>(argSize, 4 * sizeof(uint32)));
257		if (argSize > 4 * sizeof(uint32)) {
258			status_t res = user_memcpy(&args[4], (void *)iframe->usr_sp,
259				(argSize - 4 * sizeof(uint32)));
260			if (res < B_OK) {
261				dprintf("can't read syscall arguments on user stack\n");
262				iframe->r0 = res;
263				return;
264			}
265		}
266	}
267
268	thread_get_current_thread()->arch_info.userFrame = iframe;
269	thread_get_current_thread()->arch_info.oldR0 = iframe->r0;
270	thread_at_kernel_entry(system_time());
271
272	enable_interrupts();
273
274	uint64 returnValue = 0;
275	syscall_dispatcher(syscall, (void*)args, &returnValue);
276
277	TRACE("returning %" B_PRId64 "\n", returnValue);
278	iframe->r0 = returnValue;
279
280	disable_interrupts();
281	atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
282	if ((thread_get_current_thread()->flags & (THREAD_FLAGS_SIGNALS_PENDING
283			| THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) {
284		enable_interrupts();
285		thread_at_kernel_exit();
286	} else {
287		thread_at_kernel_exit_no_signals();
288	}
289	if ((thread_get_current_thread()->flags & THREAD_FLAGS_RESTART_SYSCALL) != 0) {
290		atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL);
291		atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED);
292		iframe->r0 = thread_get_current_thread()->arch_info.oldR0;
293		iframe->pc -= 4;
294	}
295
296	thread_get_current_thread()->arch_info.userFrame = NULL;
297}
298
299
300static bool
301arch_arm_handle_access_flag_fault(addr_t far, uint32 fsr, bool isWrite, bool isExec)
302{
303	VMAddressSpacePutter addressSpace;
304	if (IS_KERNEL_ADDRESS(far))
305		addressSpace.SetTo(VMAddressSpace::GetKernel());
306	else if (IS_USER_ADDRESS(far))
307		addressSpace.SetTo(VMAddressSpace::GetCurrent());
308
309	if (!addressSpace.IsSet())
310		return false;
311
312	ARMVMTranslationMap *map = (ARMVMTranslationMap *)addressSpace->TranslationMap();
313
314	if ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_ACCESS_FLAG_FAULT) {
315		phys_addr_t physAddr;
316		uint32 pageFlags;
317
318		map->QueryInterrupt(far, &physAddr, &pageFlags);
319
320		if ((PAGE_PRESENT & pageFlags) == 0)
321			return false;
322
323		if ((pageFlags & PAGE_ACCESSED) == 0) {
324			map->SetFlags(far, PAGE_ACCESSED);
325			return true;
326		}
327	}
328
329	if (isWrite && ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_PERMISSION_FAULT_L2)) {
330		phys_addr_t physAddr;
331		uint32 pageFlags;
332
333		map->QueryInterrupt(far, &physAddr, &pageFlags);
334
335		if ((PAGE_PRESENT & pageFlags) == 0)
336			return false;
337
338		if (((pageFlags & B_KERNEL_WRITE_AREA) && ((pageFlags & PAGE_MODIFIED) == 0))) {
339			map->SetFlags(far, PAGE_MODIFIED);
340			return true;
341		}
342	}
343
344	return false;
345}
346
347
348static void
349arch_arm_page_fault(struct iframe *frame, addr_t far, uint32 fsr, bool isWrite, bool isExec)
350{
351	if (arch_arm_handle_access_flag_fault(far, fsr, isWrite, isExec))
352		return;
353
354	Thread *thread = thread_get_current_thread();
355	bool isUser = (frame->spsr & CPSR_MODE_MASK) == CPSR_MODE_USR;
356	addr_t newip = 0;
357
358#ifdef TRACE_ARCH_INT
359	print_iframe("Page Fault", frame);
360	dprintf("FAR: %08lx, FSR: %08x, isUser: %d, isWrite: %d, isExec: %d, thread: %s\n", far, fsr, isUser, isWrite, isExec, thread->name);
361#endif
362
363	IFrameScope scope(frame);
364
365	if (debug_debugger_running()) {
366		// If this CPU or this thread has a fault handler, we're allowed to be
367		// here.
368		if (thread != NULL) {
369			cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
370
371			if (cpu->fault_handler != 0) {
372				debug_set_page_fault_info(far, frame->pc,
373					isWrite ? DEBUG_PAGE_FAULT_WRITE : 0);
374				frame->svc_sp = cpu->fault_handler_stack_pointer;
375				frame->pc = cpu->fault_handler;
376				return;
377			}
378
379			if (thread->fault_handler != 0) {
380				kprintf("ERROR: thread::fault_handler used in kernel "
381					"debugger!\n");
382				debug_set_page_fault_info(far, frame->pc,
383						isWrite ? DEBUG_PAGE_FAULT_WRITE : 0);
384				frame->pc = reinterpret_cast<uintptr_t>(thread->fault_handler);
385				return;
386			}
387		}
388
389		// otherwise, not really
390		panic("page fault in debugger without fault handler! Touching "
391			"address %p from pc %p\n", (void *)far, (void *)frame->pc);
392		return;
393	} else if (isExec && !isUser && (far < KERNEL_BASE) &&
394		(((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L1) || ((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L2))) {
395		panic("PXN violation trying to execute user-mapped address 0x%08" B_PRIxADDR " from kernel mode\n",
396			far);
397	} else if (!isExec && ((fsr & 0x060f) == FSR_FS_ALIGNMENT_FAULT)) {
398		panic("unhandled alignment exception\n");
399	} else if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) {
400		panic("unhandled access flag fault\n");
401	} else if ((frame->spsr & CPSR_I) != 0) {
402		// interrupts disabled
403
404		// If a page fault handler is installed, we're allowed to be here.
405		// TODO: Now we are generally allowing user_memcpy() with interrupts
406		// disabled, which in most cases is a bug. We should add some thread
407		// flag allowing to explicitly indicate that this handling is desired.
408		uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler);
409		if (thread && thread->fault_handler != 0) {
410			if (frame->pc != handler) {
411				frame->pc = handler;
412				return;
413			}
414
415			// The fault happened at the fault handler address. This is a
416			// certain infinite loop.
417			panic("page fault, interrupts disabled, fault handler loop. "
418				"Touching address %p from pc %p\n", (void*)far,
419				(void*)frame->pc);
420		}
421
422		// If we are not running the kernel startup the page fault was not
423		// allowed to happen and we must panic.
424		panic("page fault, but interrupts were disabled. Touching address "
425			"%p from pc %p\n", (void *)far, (void *)frame->pc);
426		return;
427	} else if (thread != NULL && thread->page_faults_allowed < 1) {
428		panic("page fault not allowed at this place. Touching address "
429			"%p from pc %p\n", (void *)far, (void *)frame->pc);
430		return;
431	}
432
433	enable_interrupts();
434
435	vm_page_fault(far, frame->pc, isWrite, isExec, isUser, &newip);
436
437	if (newip != 0) {
438		// the page fault handler wants us to modify the iframe to set the
439		// IP the cpu will return to to be this ip
440		frame->pc = newip;
441	}
442}
443
444
445extern "C" void
446arch_arm_data_abort(struct iframe *frame)
447{
448	addr_t dfar = arm_get_dfar();
449	uint32 dfsr = arm_get_dfsr();
450	bool isWrite = (dfsr & FSR_WNR) == FSR_WNR;
451
452	arch_arm_page_fault(frame, dfar, dfsr, isWrite, false);
453}
454
455
456extern "C" void
457arch_arm_prefetch_abort(struct iframe *frame)
458{
459	addr_t ifar = arm_get_ifar();
460	uint32 ifsr = arm_get_ifsr();
461
462	arch_arm_page_fault(frame, ifar, ifsr, false, true);
463}
464
465
466extern "C" void
467arch_arm_irq(struct iframe *iframe)
468{
469	IFrameScope scope(iframe);
470
471	InterruptController *ic = InterruptController::Get();
472	if (ic != NULL)
473		ic->HandleInterrupt();
474
475	Thread* thread = thread_get_current_thread();
476	cpu_status state = disable_interrupts();
477	if (thread->cpu->invoke_scheduler) {
478		SpinLocker schedulerLocker(thread->scheduler_lock);
479		scheduler_reschedule(B_THREAD_READY);
480		schedulerLocker.Unlock();
481		restore_interrupts(state);
482	} else if (thread->post_interrupt_callback != NULL) {
483		void (*callback)(void*) = thread->post_interrupt_callback;
484		void* data = thread->post_interrupt_data;
485
486		thread->post_interrupt_callback = NULL;
487		thread->post_interrupt_data = NULL;
488
489		restore_interrupts(state);
490
491		callback(data);
492	}
493}
494
495
496extern "C" void
497arch_arm_fiq(struct iframe *iframe)
498{
499	IFrameScope scope(iframe);
500
501	panic("FIQ not implemented yet!");
502}
503