1/*
2 * Copyright 2018, J��r��me Duval, jerome.duval@gmail.com.
3 * Copyright 2008-2011, Michael Lotz, mmlr@mlotz.ch.
4 * Copyright 2010, Clemens Zeidler, haiku@clemens-zeidler.de.
5 * Copyright 2009-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
6 * Copyright 2002-2010, Axel D��rfler, axeld@pinc-software.de.
7 * Distributed under the terms of the MIT License.
8 *
9 * Copyright 2001, Travis Geiselbrecht. All rights reserved.
10 * Distributed under the terms of the NewOS License.
11 */
12
13
14#include <cpu.h>
15#include <int.h>
16#include <kscheduler.h>
17#include <team.h>
18#include <thread.h>
19#include <util/AutoLock.h>
20#include <vm/vm.h>
21#include <vm/vm_priv.h>
22
23#include <arch/cpu.h>
24#include <arch/int.h>
25
26#include <arch/x86/apic.h>
27#include <arch/x86/descriptors.h>
28#include <arch/x86/msi.h>
29#include <arch/x86/msi_priv.h>
30
31#include <stdio.h>
32
33// interrupt controllers
34#include <arch/x86/ioapic.h>
35#include <arch/x86/pic.h>
36
37
38//#define TRACE_ARCH_INT
39#ifdef TRACE_ARCH_INT
40#	define TRACE(x) dprintf x
41#else
42#	define TRACE(x) ;
43#endif
44
45
46static irq_source sVectorSources[NUM_IO_VECTORS];
47
48static const char *kInterruptNames[] = {
49	/*  0 */ "Divide Error Exception",
50	/*  1 */ "Debug Exception",
51	/*  2 */ "NMI Interrupt",
52	/*  3 */ "Breakpoint Exception",
53	/*  4 */ "Overflow Exception",
54	/*  5 */ "BOUND Range Exceeded Exception",
55	/*  6 */ "Invalid Opcode Exception",
56	/*  7 */ "Device Not Available Exception",
57	/*  8 */ "Double Fault Exception",
58	/*  9 */ "Coprocessor Segment Overrun",
59	/* 10 */ "Invalid TSS Exception",
60	/* 11 */ "Segment Not Present",
61	/* 12 */ "Stack Fault Exception",
62	/* 13 */ "General Protection Exception",
63	/* 14 */ "Page-Fault Exception",
64	/* 15 */ "-",
65	/* 16 */ "x87 FPU Floating-Point Error",
66	/* 17 */ "Alignment Check Exception",
67	/* 18 */ "Machine-Check Exception",
68	/* 19 */ "SIMD Floating-Point Exception",
69};
70static const int kInterruptNameCount = 20;
71
72static const interrupt_controller* sCurrentPIC = NULL;
73
74
75static const char*
76exception_name(int number, char* buffer, int32 bufferSize)
77{
78	if (number >= 0 && number < kInterruptNameCount)
79		return kInterruptNames[number];
80
81	snprintf(buffer, bufferSize, "exception %d", number);
82	return buffer;
83}
84
85
86void
87x86_invalid_exception(iframe* frame)
88{
89	Thread* thread = thread_get_current_thread();
90	char name[32];
91	panic("unhandled trap 0x%lx (%s) at ip 0x%lx, thread %" B_PRId32 "!\n",
92		frame->vector, exception_name(frame->vector, name, sizeof(name)),
93		frame->ip, thread ? thread->id : -1);
94}
95
96
97void
98x86_fatal_exception(iframe* frame)
99{
100	char name[32];
101	panic("Fatal exception \"%s\" occurred! Error code: 0x%lx\n",
102		exception_name(frame->vector, name, sizeof(name)), frame->error_code);
103}
104
105
106void
107x86_unexpected_exception(iframe* frame)
108{
109	debug_exception_type type;
110	uint32 signalNumber;
111	int32 signalCode;
112	addr_t signalAddress = 0;
113	int32 signalError = B_ERROR;
114
115	switch (frame->vector) {
116		case 0:		// Divide Error Exception (#DE)
117			type = B_DIVIDE_ERROR;
118			signalNumber = SIGFPE;
119			signalCode = FPE_INTDIV;
120			signalAddress = frame->ip;
121			break;
122
123		case 4:		// Overflow Exception (#OF)
124			type = B_OVERFLOW_EXCEPTION;
125			signalNumber = SIGFPE;
126			signalCode = FPE_INTOVF;
127			signalAddress = frame->ip;
128			break;
129
130		case 5:		// BOUND Range Exceeded Exception (#BR)
131			type = B_BOUNDS_CHECK_EXCEPTION;
132			signalNumber = SIGTRAP;
133			signalCode = SI_USER;
134			break;
135
136		case 6:		// Invalid Opcode Exception (#UD)
137			type = B_INVALID_OPCODE_EXCEPTION;
138			signalNumber = SIGILL;
139			signalCode = ILL_ILLOPC;
140			signalAddress = frame->ip;
141			break;
142
143		case 12: 	// Stack Fault (#SS)
144			type = B_STACK_FAULT;
145			signalNumber = SIGBUS;
146			signalCode = BUS_ADRERR;
147			signalAddress = frame->ip;
148			break;
149
150		case 13: 	// General Protection Exception (#GP)
151			type = B_GENERAL_PROTECTION_FAULT;
152			signalNumber = SIGILL;
153			signalCode = ILL_PRVOPC;	// or ILL_PRVREG
154			signalAddress = frame->ip;
155			break;
156
157		case 16: 	// x87 FPU Floating-Point Error (#MF)
158			type = B_FLOATING_POINT_EXCEPTION;
159			signalNumber = SIGFPE;
160			signalCode = FPE_FLTDIV;
161				// TODO: Determine the correct cause via the FPU status
162				// register!
163			signalAddress = frame->ip;
164			break;
165
166		case 17: 	// Alignment Check Exception (#AC)
167			type = B_ALIGNMENT_EXCEPTION;
168			signalNumber = SIGBUS;
169			signalCode = BUS_ADRALN;
170			// TODO: Also get the address (from where?). Since we don't enable
171			// alignment checking this exception should never happen, though.
172			signalError = EFAULT;
173			break;
174
175		case 19: 	// SIMD Floating-Point Exception (#XF)
176			type = B_FLOATING_POINT_EXCEPTION;
177			signalNumber = SIGFPE;
178			signalCode = FPE_FLTDIV;
179				// TODO: Determine the correct cause via the MXCSR register!
180			signalAddress = frame->ip;
181			break;
182
183		default:
184			x86_invalid_exception(frame);
185			return;
186	}
187
188	if (IFRAME_IS_USER(frame)) {
189		struct sigaction action;
190		Thread* thread = thread_get_current_thread();
191
192		enable_interrupts();
193
194		// If the thread has a signal handler for the signal, we simply send it
195		// the signal. Otherwise we notify the user debugger first.
196		if ((sigaction(signalNumber, NULL, &action) == 0
197				&& action.sa_handler != SIG_DFL
198				&& action.sa_handler != SIG_IGN)
199			|| user_debug_exception_occurred(type, signalNumber)) {
200			Signal signal(signalNumber, signalCode, signalError,
201				thread->team->id);
202			signal.SetAddress((void*)signalAddress);
203			send_signal_to_thread(thread, signal, 0);
204		}
205	} else {
206		char name[32];
207		panic("Unexpected exception \"%s\" occurred in kernel mode! "
208			"Error code: 0x%lx\n",
209			exception_name(frame->vector, name, sizeof(name)),
210			frame->error_code);
211	}
212}
213
214
215void
216x86_hardware_interrupt(struct iframe* frame)
217{
218	int32 vector = frame->vector - ARCH_INTERRUPT_BASE;
219	bool levelTriggered = false;
220	Thread* thread = thread_get_current_thread();
221
222	if (sCurrentPIC->is_spurious_interrupt(vector)) {
223		TRACE(("got spurious interrupt at vector %ld\n", vector));
224		return;
225	}
226
227	levelTriggered = sCurrentPIC->is_level_triggered_interrupt(vector);
228
229	if (!levelTriggered) {
230		// if it's not handled by the current pic then it's an apic generated
231		// interrupt like local interrupts, msi or ipi.
232		if (!sCurrentPIC->end_of_interrupt(vector))
233			apic_end_of_interrupt();
234	}
235
236	int_io_interrupt_handler(vector, levelTriggered);
237
238	if (levelTriggered) {
239		if (!sCurrentPIC->end_of_interrupt(vector))
240			apic_end_of_interrupt();
241	}
242
243	cpu_status state = disable_interrupts();
244	if (thread->cpu->invoke_scheduler) {
245		SpinLocker schedulerLocker(thread->scheduler_lock);
246		scheduler_reschedule(B_THREAD_READY);
247		schedulerLocker.Unlock();
248		restore_interrupts(state);
249	} else if (thread->post_interrupt_callback != NULL) {
250		void (*callback)(void*) = thread->post_interrupt_callback;
251		void* data = thread->post_interrupt_data;
252
253		thread->post_interrupt_callback = NULL;
254		thread->post_interrupt_data = NULL;
255
256		restore_interrupts(state);
257
258		callback(data);
259	}
260}
261
262
263void
264x86_page_fault_exception(struct iframe* frame)
265{
266	Thread* thread = thread_get_current_thread();
267	addr_t cr2 = x86_read_cr2();
268	addr_t newip;
269
270	if (debug_debugger_running()) {
271		// If this CPU or this thread has a fault handler, we're allowed to be
272		// here.
273		if (thread != NULL) {
274			cpu_ent* cpu = &gCPU[smp_get_current_cpu()];
275			if (cpu->fault_handler != 0) {
276				debug_set_page_fault_info(cr2, frame->ip,
277					(frame->error_code & PGFAULT_W) != 0
278						? DEBUG_PAGE_FAULT_WRITE : 0);
279				frame->ip = cpu->fault_handler;
280				frame->bp = cpu->fault_handler_stack_pointer;
281				return;
282			}
283
284			if (thread->fault_handler != 0) {
285				kprintf("ERROR: thread::fault_handler used in kernel "
286					"debugger!\n");
287				debug_set_page_fault_info(cr2, frame->ip,
288					(frame->error_code & PGFAULT_W) != 0
289						? DEBUG_PAGE_FAULT_WRITE : 0);
290				frame->ip = reinterpret_cast<uintptr_t>(thread->fault_handler);
291				return;
292			}
293		}
294
295		// otherwise, not really
296		panic("page fault in debugger without fault handler! Touching "
297			"address %p from ip %p\n", (void*)cr2, (void*)frame->ip);
298		return;
299	} else if (!IFRAME_IS_USER(frame)
300		&& (frame->error_code & PGFAULT_I) != 0
301		&& (x86_read_cr4() & IA32_CR4_SMEP) != 0) {
302		// check that: 1. come not from userland,
303		// 2. is an instruction fetch, 3. smep is enabled
304		panic("SMEP violation user-mapped address %p touched from kernel %p\n",
305			(void*)cr2, (void*)frame->ip);
306	} else if ((frame->flags & X86_EFLAGS_ALIGNMENT_CHECK) == 0
307		&& !IFRAME_IS_USER(frame)
308		&& (frame->error_code & PGFAULT_P) != 0
309		&& (x86_read_cr4() & IA32_CR4_SMAP) != 0) {
310		// check that: 1. AC flag is not set, 2. come not from userland,
311		// 3. is a page-protection violation, 4. smap is enabled
312		panic("SMAP violation user-mapped address %p touched from kernel %p\n",
313			(void*)cr2, (void*)frame->ip);
314	} else if ((frame->flags & X86_EFLAGS_INTERRUPT) == 0) {
315		// interrupts disabled
316
317		// If a page fault handler is installed, we're allowed to be here.
318		// TODO: Now we are generally allowing user_memcpy() with interrupts
319		// disabled, which in most cases is a bug. We should add some thread
320		// flag allowing to explicitly indicate that this handling is desired.
321		if (thread != NULL && thread->fault_handler != 0) {
322			uintptr_t handler
323				= reinterpret_cast<uintptr_t>(thread->fault_handler);
324			if (frame->ip != handler) {
325				frame->ip = handler;
326				return;
327			}
328
329			// The fault happened at the fault handler address. This is a
330			// certain infinite loop.
331			panic("page fault, interrupts disabled, fault handler loop. "
332				"Touching address %p from ip %p\n", (void*)cr2,
333				(void*)frame->ip);
334		}
335
336		// If we are not running the kernel startup the page fault was not
337		// allowed to happen and we must panic.
338		panic("page fault, but interrupts were disabled. Touching address "
339			"%p from ip %p\n", (void*)cr2, (void*)frame->ip);
340		return;
341	} else if (thread != NULL && thread->page_faults_allowed < 1) {
342		panic("page fault not allowed at this place. Touching address "
343			"%p from ip %p\n", (void*)cr2, (void*)frame->ip);
344		return;
345	}
346
347	enable_interrupts();
348
349	vm_page_fault(cr2, frame->ip,
350		(frame->error_code & PGFAULT_W) != 0,		// write access
351		(frame->error_code & PGFAULT_I) != 0,		// instruction fetch
352		IFRAME_IS_USER(frame),						// userland
353		&newip);
354	if (newip != 0) {
355		// the page fault handler wants us to modify the iframe to set the
356		// IP the cpu will return to this ip
357		frame->ip = newip;
358	}
359}
360
361
362void
363x86_set_irq_source(int32 irq, irq_source source)
364{
365	sVectorSources[irq] = source;
366}
367
368
369// #pragma mark -
370
371
372void
373arch_int_enable_io_interrupt(int32 irq)
374{
375	sCurrentPIC->enable_io_interrupt(irq);
376}
377
378
379void
380arch_int_disable_io_interrupt(int32 irq)
381{
382	sCurrentPIC->disable_io_interrupt(irq);
383}
384
385
386void
387arch_int_configure_io_interrupt(int32 irq, uint32 config)
388{
389	sCurrentPIC->configure_io_interrupt(irq, config);
390}
391
392
393#undef arch_int_enable_interrupts
394#undef arch_int_disable_interrupts
395#undef arch_int_restore_interrupts
396#undef arch_int_are_interrupts_enabled
397
398
399void
400arch_int_enable_interrupts(void)
401{
402	arch_int_enable_interrupts_inline();
403}
404
405
406int
407arch_int_disable_interrupts(void)
408{
409	return arch_int_disable_interrupts_inline();
410}
411
412
413void
414arch_int_restore_interrupts(int oldState)
415{
416	arch_int_restore_interrupts_inline(oldState);
417}
418
419
420bool
421arch_int_are_interrupts_enabled(void)
422{
423	return arch_int_are_interrupts_enabled_inline();
424}
425
426
427int32
428arch_int_assign_to_cpu(int32 irq, int32 cpu)
429{
430	switch (sVectorSources[irq]) {
431		case IRQ_SOURCE_IOAPIC:
432			if (sCurrentPIC->assign_interrupt_to_cpu != NULL)
433				sCurrentPIC->assign_interrupt_to_cpu(irq, cpu);
434			break;
435
436		case IRQ_SOURCE_MSI:
437			msi_assign_interrupt_to_cpu(irq, cpu);
438			break;
439
440		default:
441			break;
442	}
443	return cpu;
444}
445
446
447status_t
448arch_int_init(kernel_args* args)
449{
450	// setup the standard programmable interrupt controller
451	pic_init();
452	return B_OK;
453}
454
455
456status_t
457arch_int_init_post_vm(kernel_args* args)
458{
459	// Always init the local apic as it can be used for timers even if we
460	// don't end up using the io apic
461	apic_init(args);
462	return B_OK;
463}
464
465
466status_t
467arch_int_init_io(kernel_args* args)
468{
469	msi_init(args);
470	ioapic_preinit(args);
471	return B_OK;
472}
473
474
475status_t
476arch_int_init_post_device_manager(kernel_args* args)
477{
478	return B_OK;
479}
480
481
482void
483arch_int_set_interrupt_controller(const interrupt_controller& controller)
484{
485	sCurrentPIC = &controller;
486}
487