1/*
2 * Copyright 2003-2011, Haiku, Inc. All rights reserved.
3 * Distributed under the terms of the MIT License.
4 *
5 * Authors:
6 *      Adrien Destugues, pulkomandy@pulkomandy.tk
7 */
8
9
10#include <int.h>
11#include <cpu.h>
12#include <thread.h>
13#include <vm/vm_priv.h>
14#include <ksyscalls.h>
15#include <syscall_numbers.h>
16#include <arch_cpu_defs.h>
17#include <arch_thread_types.h>
18#include <arch/debug.h>
19#include <util/AutoLock.h>
20#include <Htif.h>
21#include <Plic.h>
22#include <Clint.h>
23#include <AutoDeleterDrivers.h>
24#include <ScopeExit.h>
25#include "RISCV64VMTranslationMap.h"
26
27#include <algorithm>
28
29
30static uint32 sPlicContexts[SMP_MAX_CPUS];
31
32
33//#pragma mark debug output
34
35static void
36WriteMode(int mode)
37{
38	switch (mode) {
39		case modeU: dprintf("u"); break;
40		case modeS: dprintf("s"); break;
41		case modeM: dprintf("m"); break;
42		default: dprintf("%d", mode);
43	}
44}
45
46
47static void
48WriteModeSet(uint32_t val)
49{
50	bool first = true;
51	dprintf("{");
52	for (int i = 0; i < 32; i++) {
53		if (((1LL << i) & val) != 0) {
54			if (first) first = false; else dprintf(", ");
55			WriteMode(i);
56		}
57	}
58	dprintf("}");
59}
60
61
62static void
63WriteExt(uint64_t val)
64{
65	switch (val) {
66		case 0: dprintf("off"); break;
67		case 1: dprintf("initial"); break;
68		case 2: dprintf("clean"); break;
69		case 3: dprintf("dirty"); break;
70		default: dprintf("%" B_PRId64, val);
71	}
72}
73
74
75static void
76WriteSstatus(uint64_t val)
77{
78	SstatusReg status{.val = val};
79	dprintf("(");
80	dprintf("ie: "); WriteModeSet(status.ie);
81	dprintf(", pie: "); WriteModeSet(status.pie);
82	dprintf(", spp: "); WriteMode(status.spp);
83	dprintf(", fs: "); WriteExt(status.fs);
84	dprintf(", xs: "); WriteExt(status.xs);
85	dprintf(", sum: %d", (int)status.sum);
86	dprintf(", mxr: %d", (int)status.mxr);
87	dprintf(", uxl: %d", (int)status.uxl);
88	dprintf(", sd: %d", (int)status.sd);
89	dprintf(")");
90}
91
92
93static void
94WriteInterrupt(uint64_t val)
95{
96	switch (val) {
97		case 0 + modeU: dprintf("uSoft"); break;
98		case 0 + modeS: dprintf("sSoft"); break;
99		case 0 + modeM: dprintf("mSoft"); break;
100		case 4 + modeU: dprintf("uTimer"); break;
101		case 4 + modeS: dprintf("sTimer"); break;
102		case 4 + modeM: dprintf("mTimer"); break;
103		case 8 + modeU: dprintf("uExtern"); break;
104		case 8 + modeS: dprintf("sExtern"); break;
105		case 8 + modeM: dprintf("mExtern"); break;
106		default: dprintf("%" B_PRId64, val);
107	}
108}
109
110
111static void
112WriteInterruptSet(uint64_t val)
113{
114	bool first = true;
115	dprintf("{");
116	for (int i = 0; i < 64; i++) {
117		if (((1LL << i) & val) != 0) {
118			if (first) first = false; else dprintf(", ");
119			WriteInterrupt(i);
120		}
121	}
122	dprintf("}");
123}
124
125
126static void
127WriteCause(uint64_t cause)
128{
129	if ((cause & causeInterrupt) == 0) {
130		dprintf("exception ");
131		switch (cause) {
132			case causeExecMisalign: dprintf("execMisalign"); break;
133			case causeExecAccessFault: dprintf("execAccessFault"); break;
134			case causeIllegalInst: dprintf("illegalInst"); break;
135			case causeBreakpoint: dprintf("breakpoint"); break;
136			case causeLoadMisalign: dprintf("loadMisalign"); break;
137			case causeLoadAccessFault: dprintf("loadAccessFault"); break;
138			case causeStoreMisalign: dprintf("storeMisalign"); break;
139			case causeStoreAccessFault: dprintf("storeAccessFault"); break;
140			case causeUEcall: dprintf("uEcall"); break;
141			case causeSEcall: dprintf("sEcall"); break;
142			case causeMEcall: dprintf("mEcall"); break;
143			case causeExecPageFault: dprintf("execPageFault"); break;
144			case causeLoadPageFault: dprintf("loadPageFault"); break;
145			case causeStorePageFault: dprintf("storePageFault"); break;
146			default: dprintf("%" B_PRId64, cause);
147			}
148	} else {
149		dprintf("interrupt "); WriteInterrupt(cause & ~causeInterrupt);
150	}
151}
152
153
154const static char* registerNames[] = {
155	" ra", " t6", " sp", " gp",
156	" tp", " t0", " t1", " t2",
157	" t5", " s1", " a0", " a1",
158	" a2", " a3", " a4", " a5",
159	" a6", " a7", " s2", " s3",
160	" s4", " s5", " s6", " s7",
161	" s8", " s9", "s10", "s11",
162	" t3", " t4", " fp", "epc"
163};
164
165
166static void WriteRegisters(iframe* frame)
167{
168	uint64* regs = &frame->ra;
169	for (int i = 0; i < 32; i += 4) {
170		dprintf(
171			"  %s: 0x%016" B_PRIx64
172			"  %s: 0x%016" B_PRIx64
173			"  %s: 0x%016" B_PRIx64
174			"  %s: 0x%016" B_PRIx64 "\n",
175			registerNames[i + 0], regs[i + 0],
176			registerNames[i + 1], regs[i + 1],
177			registerNames[i + 2], regs[i + 2],
178			registerNames[i + 3], regs[i + 3]
179		);
180	}
181}
182
183
184static void
185DumpMemory(uint64* adr, size_t len)
186{
187	while (len > 0) {
188		if ((addr_t)adr % 0x10 == 0)
189			dprintf("%08" B_PRIxADDR " ", (addr_t)adr);
190		uint64 val;
191		if (user_memcpy(&val, adr++, sizeof(val)) < B_OK) {
192			dprintf(" ????????????????");
193		} else {
194			dprintf(" %016" B_PRIx64, val);
195		}
196		if ((addr_t)adr % 0x10 == 0)
197			dprintf("\n");
198		len -= 8;
199	}
200	if ((addr_t)adr % 0x10 != 0)
201		dprintf("\n");
202
203	dprintf("%08" B_PRIxADDR "\n\n", (addr_t)adr);
204}
205
206
207void
208WriteTrapInfo(iframe* frame)
209{
210	InterruptsLocker locker;
211	dprintf("STrap("); WriteCause(frame->cause); dprintf(")\n");
212	dprintf("  sstatus: "); WriteSstatus(frame->status); dprintf("\n");
213//	dprintf("  sie: "); WriteInterruptSet(Sie()); dprintf("\n");
214//	dprintf("  sip: "); WriteInterruptSet(Sip()); dprintf("\n");
215	//dprintf("  stval: "); WritePC(Stval()); dprintf("\n");
216	dprintf("  stval: 0x%" B_PRIx64 "\n", frame->tval);
217//	dprintf("  tp: 0x%" B_PRIxADDR "(%s)\n", Tp(),
218//		thread_get_current_thread()->name);
219
220	WriteRegisters(frame);
221#if 0
222	dprintf("  kernel stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n",
223		thread_get_current_thread()->kernel_stack_base,
224		thread_get_current_thread()->kernel_stack_top - 1
225	);
226	dprintf("  user stack: %#" B_PRIxADDR " - %#" B_PRIxADDR "\n",
227		thread_get_current_thread()->user_stack_base,
228		thread_get_current_thread()->user_stack_base +
229		thread_get_current_thread()->user_stack_size - 1
230	);
231	if (thread_get_current_thread()->arch_info.userFrame != NULL) {
232		WriteRegisters(thread_get_current_thread()->arch_info.userFrame);
233
234		dprintf("Stack memory dump:\n");
235		DumpMemory(
236			(uint64*)thread_get_current_thread()->arch_info.userFrame->sp,
237			thread_get_current_thread()->user_stack_base +
238			thread_get_current_thread()->user_stack_size -
239			thread_get_current_thread()->arch_info.userFrame->sp
240		);
241//		if (true) {
242//		} else {
243//			DumpMemory((uint64*)frame->sp, thread_get_current_thread()->kernel_stack_top - frame->sp);
244//		}
245	}
246#endif
247}
248
249
250//#pragma mark -
251
252static void
253SendSignal(debug_exception_type type, uint32 signalNumber, int32 signalCode,
254	addr_t signalAddress = 0, int32 signalError = B_ERROR)
255{
256	if (SstatusReg{.val = Sstatus()}.spp == modeU) {
257		struct sigaction action;
258		Thread* thread = thread_get_current_thread();
259
260		//DoStackTrace(Fp(), 0);
261
262		enable_interrupts();
263
264		// If the thread has a signal handler for the signal, we simply send it
265		// the signal. Otherwise we notify the user debugger first.
266		if ((sigaction(signalNumber, NULL, &action) == 0
267				&& action.sa_handler != SIG_DFL
268				&& action.sa_handler != SIG_IGN)
269			|| user_debug_exception_occurred(type, signalNumber)) {
270			Signal signal(signalNumber, signalCode, signalError,
271				thread->team->id);
272			signal.SetAddress((void*)signalAddress);
273			send_signal_to_thread(thread, signal, 0);
274		}
275	} else {
276		panic("Unexpected exception occurred in kernel mode!");
277	}
278}
279
280
281static void
282AfterInterrupt()
283{
284	if (debug_debugger_running())
285		return;
286
287	Thread* thread = thread_get_current_thread();
288	cpu_status state = disable_interrupts();
289	if (thread->cpu->invoke_scheduler) {
290		SpinLocker schedulerLocker(thread->scheduler_lock);
291		scheduler_reschedule(B_THREAD_READY);
292		schedulerLocker.Unlock();
293		restore_interrupts(state);
294	} else if (thread->post_interrupt_callback != NULL) {
295		void (*callback)(void*) = thread->post_interrupt_callback;
296		void* data = thread->post_interrupt_data;
297
298		thread->post_interrupt_callback = NULL;
299		thread->post_interrupt_data = NULL;
300
301		restore_interrupts(state);
302
303		callback(data);
304	}
305}
306
307
308static bool
309SetAccessedFlags(addr_t addr, bool isWrite)
310{
311	VMAddressSpacePutter addressSpace;
312	if (IS_KERNEL_ADDRESS(addr))
313		addressSpace.SetTo(VMAddressSpace::GetKernel());
314	else if (IS_USER_ADDRESS(addr))
315		addressSpace.SetTo(VMAddressSpace::GetCurrent());
316
317	if(!addressSpace.IsSet())
318		return false;
319
320	RISCV64VMTranslationMap* map
321		= (RISCV64VMTranslationMap*)addressSpace->TranslationMap();
322
323	phys_addr_t physAdr;
324	uint32 pageFlags;
325	map->QueryInterrupt(addr, &physAdr, &pageFlags);
326
327	if ((PAGE_PRESENT & pageFlags) == 0)
328		return false;
329
330	if (isWrite) {
331		if (
332			((B_WRITE_AREA | B_KERNEL_WRITE_AREA) & pageFlags) != 0
333			&& ((PAGE_ACCESSED | PAGE_MODIFIED) & pageFlags)
334				!= (PAGE_ACCESSED | PAGE_MODIFIED)
335		) {
336			map->SetFlags(addr, PAGE_ACCESSED | PAGE_MODIFIED);
337			return true;
338		}
339	} else {
340		if (
341			((B_READ_AREA | B_KERNEL_READ_AREA) & pageFlags) != 0
342			&& (PAGE_ACCESSED & pageFlags) == 0
343		) {
344			map->SetFlags(addr, PAGE_ACCESSED);
345			return true;
346		}
347	}
348	return false;
349}
350
351
352extern "C" void
353STrap(iframe* frame)
354{
355	// dprintf("STrap("); WriteCause(Scause()); dprintf(")\n");
356
357/*
358	iframe oldFrame = *frame;
359	const auto& frameChangeChecker = MakeScopeExit([&]() {
360			InterruptsLocker locker;
361			bool first = true;
362			for (int i = 0; i < 32; i++) {
363				uint64 oldVal = ((int64*)&oldFrame)[i];
364				uint64 newVal = ((int64*)frame)[i];
365				if (oldVal != newVal) {
366					if (first) {
367						dprintf("FrameChangeChecker, thread: %" B_PRId32 "(%s)\n", thread_get_current_thread()->id, thread_get_current_thread()->name);
368						first = false;
369					}
370					dprintf("  %s: %#" B_PRIxADDR " -> %#" B_PRIxADDR "\n", registerNames[i], oldVal, newVal);
371				}
372			}
373
374			if (frame->epc == 0)
375				panic("FrameChangeChecker: EPC = 0");
376	});
377*/
378	switch (frame->cause) {
379		case causeExecPageFault:
380		case causeLoadPageFault:
381		case causeStorePageFault: {
382			if (SetAccessedFlags(Stval(), frame->cause == causeStorePageFault))
383				return;
384		}
385	}
386
387	if (SstatusReg{.val = frame->status}.spp == modeU) {
388		thread_get_current_thread()->arch_info.userFrame = frame;
389		thread_get_current_thread()->arch_info.oldA0 = frame->a0;
390		thread_at_kernel_entry(system_time());
391	}
392	const auto& kernelExit = ScopeExit([&]() {
393		if (SstatusReg{.val = frame->status}.spp == modeU) {
394			disable_interrupts();
395			atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED);
396			if ((thread_get_current_thread()->flags
397				& (THREAD_FLAGS_SIGNALS_PENDING
398				| THREAD_FLAGS_DEBUG_THREAD
399				| THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) {
400				enable_interrupts();
401				thread_at_kernel_exit();
402			} else {
403				thread_at_kernel_exit_no_signals();
404			}
405			if ((THREAD_FLAGS_RESTART_SYSCALL & thread_get_current_thread()->flags) != 0) {
406				atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL);
407				atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED);
408
409				frame->a0 = thread_get_current_thread()->arch_info.oldA0;
410				frame->epc -= 4;
411			}
412			thread_get_current_thread()->arch_info.userFrame = NULL;
413		}
414	});
415
416	switch (frame->cause) {
417		case causeIllegalInst: {
418			return SendSignal(B_INVALID_OPCODE_EXCEPTION, SIGILL, ILL_ILLOPC,
419				frame->epc);
420		}
421		case causeExecMisalign:
422		case causeLoadMisalign:
423		case causeStoreMisalign: {
424			return SendSignal(B_ALIGNMENT_EXCEPTION, SIGBUS, BUS_ADRALN,
425				Stval());
426		}
427		case causeBreakpoint: {
428			if (SstatusReg{.val = frame->status}.spp == modeU) {
429				user_debug_breakpoint_hit(false);
430			} else {
431				panic("hit kernel breakpoint");
432			}
433			return;
434		}
435		case causeExecAccessFault:
436		case causeLoadAccessFault:
437		case causeStoreAccessFault: {
438			return SendSignal(B_SEGMENT_VIOLATION, SIGBUS, BUS_ADRERR,
439				Stval());
440		}
441		case causeExecPageFault:
442		case causeLoadPageFault:
443		case causeStorePageFault: {
444			uint64 stval = Stval();
445
446			if (debug_debugger_running()) {
447				Thread* thread = thread_get_current_thread();
448				if (thread != NULL) {
449					cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
450					if (cpu->fault_handler != 0) {
451						debug_set_page_fault_info(stval, frame->epc,
452							(frame->cause == causeStorePageFault)
453								? DEBUG_PAGE_FAULT_WRITE : 0);
454						frame->epc = cpu->fault_handler;
455						frame->sp = cpu->fault_handler_stack_pointer;
456						return;
457					}
458
459					if (thread->fault_handler != 0) {
460						kprintf("ERROR: thread::fault_handler used in kernel "
461							"debugger!\n");
462						debug_set_page_fault_info(stval, frame->epc,
463							frame->cause == causeStorePageFault
464								? DEBUG_PAGE_FAULT_WRITE : 0);
465						frame->epc = (addr_t)thread->fault_handler;
466						return;
467					}
468				}
469
470				panic("page fault in debugger without fault handler! Touching "
471					"address %p from ip %p\n", (void*)stval, (void*)frame->epc);
472				return;
473			}
474
475			if (SstatusReg{.val = frame->status}.pie == 0) {
476				// user_memcpy() failure
477				Thread* thread = thread_get_current_thread();
478				if (thread != NULL && thread->fault_handler != 0) {
479					addr_t handler = (addr_t)(thread->fault_handler);
480					if (frame->epc != handler) {
481						frame->epc = handler;
482						return;
483					}
484				}
485				panic("page fault with interrupts disabled@!dump_virt_page %#" B_PRIx64, stval);
486			}
487
488			addr_t newIP = 0;
489			enable_interrupts();
490
491			vm_page_fault(stval, frame->epc, frame->cause == causeStorePageFault,
492				frame->cause == causeExecPageFault,
493				SstatusReg{.val = frame->status}.spp == modeU, &newIP);
494
495			if (newIP != 0)
496				frame->epc = newIP;
497
498			return;
499		}
500		case causeInterrupt + sSoftInt: {
501			ClearBitsSip(1 << sSoftInt);
502			// dprintf("sSoftInt(%" B_PRId32 ")\n", smp_get_current_cpu());
503			smp_intercpu_int_handler(smp_get_current_cpu());
504			AfterInterrupt();
505			return;
506		}
507		case causeInterrupt + sTimerInt: {
508			ClearBitsSie(1 << sTimerInt);
509			// dprintf("sTimerInt(%" B_PRId32 ")\n", smp_get_current_cpu());
510			timer_interrupt();
511			AfterInterrupt();
512			return;
513		}
514		case causeInterrupt + sExternInt: {
515			uint64 irq = gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete;
516			int_io_interrupt_handler(irq, true);
517			gPlicRegs->contexts[sPlicContexts[smp_get_current_cpu()]].claimAndComplete = irq;
518			AfterInterrupt();
519			return;
520		}
521		case causeUEcall: {
522			frame->epc += 4; // skip ecall
523			uint64 syscall = frame->t0;
524			uint64 args[20];
525			if (syscall < (uint64)kSyscallCount) {
526				uint32 argCnt = kExtendedSyscallInfos[syscall].parameter_count;
527				memcpy(&args[0], &frame->a0,
528					sizeof(uint64)*std::min<uint32>(argCnt, 8));
529				if (argCnt > 8) {
530					if (status_t res = user_memcpy(&args[8], (void*)frame->sp,
531						sizeof(uint64)*(argCnt - 8)) < B_OK) {
532						dprintf("can't read syscall arguments on user "
533							"stack\n");
534						frame->a0 = res;
535						return;
536					}
537				}
538			}
539/*
540			switch (syscall) {
541				case SYSCALL_READ_PORT_ETC:
542				case SYSCALL_WRITE_PORT_ETC:
543					DoStackTrace(Fp(), 0);
544					break;
545			}
546*/
547			// dprintf("syscall: %s\n", kExtendedSyscallInfos[syscall].name);
548
549			enable_interrupts();
550			uint64 returnValue = 0;
551			syscall_dispatcher(syscall, (void*)args, &returnValue);
552			frame->a0 = returnValue;
553			return;
554		}
555	}
556	panic("unhandled STrap");
557}
558
559
560//#pragma mark -
561
562status_t
563arch_int_init(kernel_args* args)
564{
565	dprintf("arch_int_init()\n");
566
567	for (uint32 i = 0; i < args->num_cpus; i++) {
568		dprintf("  CPU %" B_PRIu32 ":\n", i);
569		dprintf("    hartId: %" B_PRIu32 "\n", args->arch_args.hartIds[i]);
570		dprintf("    plicContext: %" B_PRIu32 "\n", args->arch_args.plicContexts[i]);
571	}
572
573	for (uint32 i = 0; i < args->num_cpus; i++)
574		sPlicContexts[i] = args->arch_args.plicContexts[i];
575
576	// TODO: read from FDT
577	reserve_io_interrupt_vectors(128, 0, INTERRUPT_TYPE_IRQ);
578
579	for (uint32 i = 0; i < args->num_cpus; i++)
580		gPlicRegs->contexts[sPlicContexts[i]].priorityThreshold = 0;
581
582	return B_OK;
583}
584
585
586status_t
587arch_int_init_post_vm(kernel_args* args)
588{
589	return B_OK;
590}
591
592
593status_t
594arch_int_init_post_device_manager(struct kernel_args* args)
595{
596	return B_OK;
597}
598
599
600status_t
601arch_int_init_io(kernel_args* args)
602{
603	return B_OK;
604}
605
606
607void
608arch_int_enable_io_interrupt(int32 irq)
609{
610	dprintf("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq);
611	gPlicRegs->priority[irq] = 1;
612	gPlicRegs->enable[sPlicContexts[0]][irq / 32] |= 1 << (irq % 32);
613}
614
615
616void
617arch_int_disable_io_interrupt(int32 irq)
618{
619	dprintf("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq);
620	gPlicRegs->priority[irq] = 0;
621	gPlicRegs->enable[sPlicContexts[0]][irq / 32] &= ~(1 << (irq % 32));
622}
623
624
625int32
626arch_int_assign_to_cpu(int32 irq, int32 cpu)
627{
628	// Not yet supported.
629	return 0;
630}
631
632
633#undef arch_int_enable_interrupts
634#undef arch_int_disable_interrupts
635#undef arch_int_restore_interrupts
636#undef arch_int_are_interrupts_enabled
637
638
639extern "C" void
640arch_int_enable_interrupts()
641{
642	arch_int_enable_interrupts_inline();
643}
644
645
646extern "C" int
647arch_int_disable_interrupts()
648{
649	return arch_int_disable_interrupts_inline();
650}
651
652
653extern "C" void
654arch_int_restore_interrupts(int oldState)
655{
656	arch_int_restore_interrupts_inline(oldState);
657}
658
659
660extern "C" bool
661arch_int_are_interrupts_enabled()
662{
663	return arch_int_are_interrupts_enabled_inline();
664}
665