1/* 2 * Copyright 2003-2023, Haiku Inc. All rights reserved. 3 * Distributed under the terms of the MIT License. 4 * 5 * Authors: 6 * Axel D��rfler <axeld@pinc-software.de> 7 * Ingo Weinhold <bonefish@cs.tu-berlin.de> 8 * Fran��ois Revol <revol@free.fr> 9 * Ithamar R. Adema <ithamar@upgrade-android.com> 10 * 11 * Copyright 2001, Travis Geiselbrecht. All rights reserved. 12 * Distributed under the terms of the NewOS License. 13 */ 14 15 16#include <int.h> 17 18#include <arch_cpu_defs.h> 19#include <arch/smp.h> 20#include <boot/kernel_args.h> 21#include <device_manager.h> 22#include <kscheduler.h> 23#include <interrupt_controller.h> 24#include <ksyscalls.h> 25#include <smp.h> 26#include <syscall_numbers.h> 27#include <thread.h> 28#include <timer.h> 29#include <AutoDeleterDrivers.h> 30#include <util/AutoLock.h> 31#include <util/DoublyLinkedList.h> 32#include <util/kernel_cpp.h> 33#include <vm/vm.h> 34#include <vm/vm_priv.h> 35#include <vm/VMAddressSpace.h> 36#include <algorithm> 37#include <string.h> 38 39#include <drivers/bus/FDT.h> 40#include "arch_int_gicv2.h" 41#include "soc.h" 42 43#include "soc_pxa.h" 44#include "soc_omap3.h" 45#include "soc_sun4i.h" 46 47#include "ARMVMTranslationMap.h" 48 49//#define TRACE_ARCH_INT 50#ifdef TRACE_ARCH_INT 51# define TRACE(x...) dprintf(x) 52#else 53# define TRACE(x...) ; 54#endif 55 56#define VECTORPAGE_SIZE 64 57#define USER_VECTOR_ADDR_LOW 0x00000000 58#define USER_VECTOR_ADDR_HIGH 0xffff0000 59 60extern int _vectors_start; 61extern int _vectors_end; 62 63static area_id sUserVectorPageArea; 64static void *sUserVectorPageAddress; 65//static fdt_module_info *sFdtModule; 66 67// An iframe stack used in the early boot process when we don't have 68// threads yet. 69struct iframe_stack gBootFrameStack; 70 71 72void 73arch_int_enable_io_interrupt(int32 irq) 74{ 75 TRACE("arch_int_enable_io_interrupt(%" B_PRId32 ")\n", irq); 76 InterruptController *ic = InterruptController::Get(); 77 if (ic != NULL) 78 ic->EnableInterrupt(irq); 79} 80 81 82void 83arch_int_disable_io_interrupt(int32 irq) 84{ 85 TRACE("arch_int_disable_io_interrupt(%" B_PRId32 ")\n", irq); 86 InterruptController *ic = InterruptController::Get(); 87 if (ic != NULL) 88 ic->DisableInterrupt(irq); 89} 90 91 92/* arch_int_*_interrupts() and friends are in arch_asm.S */ 93 94int32 95arch_int_assign_to_cpu(int32 irq, int32 cpu) 96{ 97 // Not yet supported. 98 return 0; 99} 100 101 102static void 103print_iframe(const char *event, struct iframe *frame) 104{ 105 if (event) 106 dprintf("Exception: %s\n", event); 107 108 dprintf("R00=%08x R01=%08x R02=%08x R03=%08x\n" 109 "R04=%08x R05=%08x R06=%08x R07=%08x\n", 110 frame->r0, frame->r1, frame->r2, frame->r3, 111 frame->r4, frame->r5, frame->r6, frame->r7); 112 dprintf("R08=%08x R09=%08x R10=%08x R11=%08x\n" 113 "R12=%08x SPs=%08x LRs=%08x PC =%08x\n", 114 frame->r8, frame->r9, frame->r10, frame->r11, 115 frame->r12, frame->svc_sp, frame->svc_lr, frame->pc); 116 dprintf(" SPu=%08x LRu=%08x CPSR=%08x\n", 117 frame->usr_sp, frame->usr_lr, frame->spsr); 118} 119 120 121extern "C" void arm_vector_init(void); 122 123 124status_t 125arch_int_init(kernel_args *args) 126{ 127 TRACE("arch_int_init\n"); 128 129 // copy vector code to vector page 130 memcpy((void*)USER_VECTOR_ADDR_HIGH, &_vectors_start, VECTORPAGE_SIZE); 131 132 // initialize stack for vectors 133 arm_vector_init(); 134 135 // enable high vectors 136 arm_set_sctlr(arm_get_sctlr() | SCTLR_HIGH_VECTORS); 137 138 return B_OK; 139} 140 141 142status_t 143arch_int_init_post_vm(kernel_args *args) 144{ 145 TRACE("arch_int_init_post_vm\n"); 146 147 sUserVectorPageAddress = (addr_t*)USER_VECTOR_ADDR_HIGH; 148 sUserVectorPageArea = create_area("user_vectorpage", 149 (void **)&sUserVectorPageAddress, B_EXACT_ADDRESS, 150 B_PAGE_SIZE, B_ALREADY_WIRED, B_READ_AREA | B_EXECUTE_AREA); 151 152 if (sUserVectorPageArea < 0) 153 panic("user vector page @ %p could not be created (%x)!", 154 sUserVectorPageAddress, sUserVectorPageArea); 155 156 if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_GICV2, 157 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 158 InterruptController *ic = new(std::nothrow) GICv2InterruptController( 159 args->arch_args.interrupt_controller.regs1.start, 160 args->arch_args.interrupt_controller.regs2.start); 161 if (ic == NULL) 162 return B_NO_MEMORY; 163 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_OMAP3, 164 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 165 InterruptController *ic = new(std::nothrow) OMAP3InterruptController( 166 args->arch_args.interrupt_controller.regs1.start); 167 if (ic == NULL) 168 return B_NO_MEMORY; 169 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_PXA, 170 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 171 InterruptController *ic = new(std::nothrow) PXAInterruptController( 172 args->arch_args.interrupt_controller.regs1.start); 173 if (ic == NULL) 174 return B_NO_MEMORY; 175 } else if (strncmp(args->arch_args.interrupt_controller.kind, INTC_KIND_SUN4I, 176 sizeof(args->arch_args.interrupt_controller.kind)) == 0) { 177 InterruptController *ic = new(std::nothrow) Sun4iInterruptController( 178 args->arch_args.interrupt_controller.regs1.start); 179 if (ic == NULL) 180 return B_NO_MEMORY; 181 } else { 182 panic("No interrupt controllers found!\n"); 183 } 184 185 return B_OK; 186} 187 188 189status_t 190arch_int_init_io(kernel_args* args) 191{ 192 return B_OK; 193} 194 195 196status_t 197arch_int_init_post_device_manager(struct kernel_args *args) 198{ 199 return B_ENTRY_NOT_FOUND; 200} 201 202 203// Little helper class for handling the 204// iframe stack as used by KDL. 205class IFrameScope { 206public: 207 IFrameScope(struct iframe *iframe) { 208 fThread = thread_get_current_thread(); 209 if (fThread) 210 arm_push_iframe(&fThread->arch_info.iframes, iframe); 211 else 212 arm_push_iframe(&gBootFrameStack, iframe); 213 } 214 215 virtual ~IFrameScope() { 216 // pop iframe 217 if (fThread) 218 arm_pop_iframe(&fThread->arch_info.iframes); 219 else 220 arm_pop_iframe(&gBootFrameStack); 221 } 222private: 223 Thread* fThread; 224}; 225 226 227extern "C" void 228arch_arm_undefined(struct iframe *iframe) 229{ 230 print_iframe("Undefined Instruction", iframe); 231 IFrameScope scope(iframe); // push/pop iframe 232 233 panic("not handled!"); 234} 235 236 237extern "C" void 238arch_arm_syscall(struct iframe *iframe) 239{ 240#ifdef TRACE_ARCH_INT 241 print_iframe("Software interrupt", iframe); 242#endif 243 244 IFrameScope scope(iframe); 245 246 uint32_t syscall = *(uint32_t *)(iframe->pc-4) & 0x00ffffff; 247 TRACE("syscall number: %d\n", syscall); 248 249 uint32_t args[20]; 250 if (syscall < kSyscallCount) { 251 TRACE("syscall(%s,%d)\n", 252 kExtendedSyscallInfos[syscall].name, 253 kExtendedSyscallInfos[syscall].parameter_count); 254 255 int argSize = kSyscallInfos[syscall].parameter_size; 256 memcpy(args, &iframe->r0, std::min<int>(argSize, 4 * sizeof(uint32))); 257 if (argSize > 4 * sizeof(uint32)) { 258 status_t res = user_memcpy(&args[4], (void *)iframe->usr_sp, 259 (argSize - 4 * sizeof(uint32))); 260 if (res < B_OK) { 261 dprintf("can't read syscall arguments on user stack\n"); 262 iframe->r0 = res; 263 return; 264 } 265 } 266 } 267 268 thread_get_current_thread()->arch_info.userFrame = iframe; 269 thread_get_current_thread()->arch_info.oldR0 = iframe->r0; 270 thread_at_kernel_entry(system_time()); 271 272 enable_interrupts(); 273 274 uint64 returnValue = 0; 275 syscall_dispatcher(syscall, (void*)args, &returnValue); 276 277 TRACE("returning %" B_PRId64 "\n", returnValue); 278 iframe->r0 = returnValue; 279 280 disable_interrupts(); 281 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_SYSCALL_RESTARTED); 282 if ((thread_get_current_thread()->flags & (THREAD_FLAGS_SIGNALS_PENDING 283 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_TRAP_FOR_CORE_DUMP)) != 0) { 284 enable_interrupts(); 285 thread_at_kernel_exit(); 286 } else { 287 thread_at_kernel_exit_no_signals(); 288 } 289 if ((thread_get_current_thread()->flags & THREAD_FLAGS_RESTART_SYSCALL) != 0) { 290 atomic_and(&thread_get_current_thread()->flags, ~THREAD_FLAGS_RESTART_SYSCALL); 291 atomic_or(&thread_get_current_thread()->flags, THREAD_FLAGS_SYSCALL_RESTARTED); 292 iframe->r0 = thread_get_current_thread()->arch_info.oldR0; 293 iframe->pc -= 4; 294 } 295 296 thread_get_current_thread()->arch_info.userFrame = NULL; 297} 298 299 300static bool 301arch_arm_handle_access_flag_fault(addr_t far, uint32 fsr, bool isWrite, bool isExec) 302{ 303 VMAddressSpacePutter addressSpace; 304 if (IS_KERNEL_ADDRESS(far)) 305 addressSpace.SetTo(VMAddressSpace::GetKernel()); 306 else if (IS_USER_ADDRESS(far)) 307 addressSpace.SetTo(VMAddressSpace::GetCurrent()); 308 309 if (!addressSpace.IsSet()) 310 return false; 311 312 ARMVMTranslationMap *map = (ARMVMTranslationMap *)addressSpace->TranslationMap(); 313 314 if ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_ACCESS_FLAG_FAULT) { 315 phys_addr_t physAddr; 316 uint32 pageFlags; 317 318 map->QueryInterrupt(far, &physAddr, &pageFlags); 319 320 if ((PAGE_PRESENT & pageFlags) == 0) 321 return false; 322 323 if ((pageFlags & PAGE_ACCESSED) == 0) { 324 map->SetFlags(far, PAGE_ACCESSED); 325 return true; 326 } 327 } 328 329 if (isWrite && ((fsr & (FSR_FS_MASK | FSR_LPAE_MASK)) == FSR_FS_PERMISSION_FAULT_L2)) { 330 phys_addr_t physAddr; 331 uint32 pageFlags; 332 333 map->QueryInterrupt(far, &physAddr, &pageFlags); 334 335 if ((PAGE_PRESENT & pageFlags) == 0) 336 return false; 337 338 if (((pageFlags & B_KERNEL_WRITE_AREA) && ((pageFlags & PAGE_MODIFIED) == 0))) { 339 map->SetFlags(far, PAGE_MODIFIED); 340 return true; 341 } 342 } 343 344 return false; 345} 346 347 348static void 349arch_arm_page_fault(struct iframe *frame, addr_t far, uint32 fsr, bool isWrite, bool isExec) 350{ 351 if (arch_arm_handle_access_flag_fault(far, fsr, isWrite, isExec)) 352 return; 353 354 Thread *thread = thread_get_current_thread(); 355 bool isUser = (frame->spsr & CPSR_MODE_MASK) == CPSR_MODE_USR; 356 addr_t newip = 0; 357 358#ifdef TRACE_ARCH_INT 359 print_iframe("Page Fault", frame); 360 dprintf("FAR: %08lx, FSR: %08x, isUser: %d, isWrite: %d, isExec: %d, thread: %s\n", far, fsr, isUser, isWrite, isExec, thread->name); 361#endif 362 363 IFrameScope scope(frame); 364 365 if (debug_debugger_running()) { 366 // If this CPU or this thread has a fault handler, we're allowed to be 367 // here. 368 if (thread != NULL) { 369 cpu_ent* cpu = &gCPU[smp_get_current_cpu()]; 370 371 if (cpu->fault_handler != 0) { 372 debug_set_page_fault_info(far, frame->pc, 373 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 374 frame->svc_sp = cpu->fault_handler_stack_pointer; 375 frame->pc = cpu->fault_handler; 376 return; 377 } 378 379 if (thread->fault_handler != 0) { 380 kprintf("ERROR: thread::fault_handler used in kernel " 381 "debugger!\n"); 382 debug_set_page_fault_info(far, frame->pc, 383 isWrite ? DEBUG_PAGE_FAULT_WRITE : 0); 384 frame->pc = reinterpret_cast<uintptr_t>(thread->fault_handler); 385 return; 386 } 387 } 388 389 // otherwise, not really 390 panic("page fault in debugger without fault handler! Touching " 391 "address %p from pc %p\n", (void *)far, (void *)frame->pc); 392 return; 393 } else if (isExec && !isUser && (far < KERNEL_BASE) && 394 (((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L1) || ((fsr & 0x060f) == FSR_FS_PERMISSION_FAULT_L2))) { 395 panic("PXN violation trying to execute user-mapped address 0x%08" B_PRIxADDR " from kernel mode\n", 396 far); 397 } else if (!isExec && ((fsr & 0x060f) == FSR_FS_ALIGNMENT_FAULT)) { 398 panic("unhandled alignment exception\n"); 399 } else if ((fsr & 0x060f) == FSR_FS_ACCESS_FLAG_FAULT) { 400 panic("unhandled access flag fault\n"); 401 } else if ((frame->spsr & CPSR_I) != 0) { 402 // interrupts disabled 403 404 // If a page fault handler is installed, we're allowed to be here. 405 // TODO: Now we are generally allowing user_memcpy() with interrupts 406 // disabled, which in most cases is a bug. We should add some thread 407 // flag allowing to explicitly indicate that this handling is desired. 408 uintptr_t handler = reinterpret_cast<uintptr_t>(thread->fault_handler); 409 if (thread && thread->fault_handler != 0) { 410 if (frame->pc != handler) { 411 frame->pc = handler; 412 return; 413 } 414 415 // The fault happened at the fault handler address. This is a 416 // certain infinite loop. 417 panic("page fault, interrupts disabled, fault handler loop. " 418 "Touching address %p from pc %p\n", (void*)far, 419 (void*)frame->pc); 420 } 421 422 // If we are not running the kernel startup the page fault was not 423 // allowed to happen and we must panic. 424 panic("page fault, but interrupts were disabled. Touching address " 425 "%p from pc %p\n", (void *)far, (void *)frame->pc); 426 return; 427 } else if (thread != NULL && thread->page_faults_allowed < 1) { 428 panic("page fault not allowed at this place. Touching address " 429 "%p from pc %p\n", (void *)far, (void *)frame->pc); 430 return; 431 } 432 433 enable_interrupts(); 434 435 vm_page_fault(far, frame->pc, isWrite, isExec, isUser, &newip); 436 437 if (newip != 0) { 438 // the page fault handler wants us to modify the iframe to set the 439 // IP the cpu will return to to be this ip 440 frame->pc = newip; 441 } 442} 443 444 445extern "C" void 446arch_arm_data_abort(struct iframe *frame) 447{ 448 addr_t dfar = arm_get_dfar(); 449 uint32 dfsr = arm_get_dfsr(); 450 bool isWrite = (dfsr & FSR_WNR) == FSR_WNR; 451 452 arch_arm_page_fault(frame, dfar, dfsr, isWrite, false); 453} 454 455 456extern "C" void 457arch_arm_prefetch_abort(struct iframe *frame) 458{ 459 addr_t ifar = arm_get_ifar(); 460 uint32 ifsr = arm_get_ifsr(); 461 462 arch_arm_page_fault(frame, ifar, ifsr, false, true); 463} 464 465 466extern "C" void 467arch_arm_irq(struct iframe *iframe) 468{ 469 IFrameScope scope(iframe); 470 471 InterruptController *ic = InterruptController::Get(); 472 if (ic != NULL) 473 ic->HandleInterrupt(); 474 475 Thread* thread = thread_get_current_thread(); 476 cpu_status state = disable_interrupts(); 477 if (thread->cpu->invoke_scheduler) { 478 SpinLocker schedulerLocker(thread->scheduler_lock); 479 scheduler_reschedule(B_THREAD_READY); 480 schedulerLocker.Unlock(); 481 restore_interrupts(state); 482 } else if (thread->post_interrupt_callback != NULL) { 483 void (*callback)(void*) = thread->post_interrupt_callback; 484 void* data = thread->post_interrupt_data; 485 486 thread->post_interrupt_callback = NULL; 487 thread->post_interrupt_data = NULL; 488 489 restore_interrupts(state); 490 491 callback(data); 492 } 493} 494 495 496extern "C" void 497arch_arm_fiq(struct iframe *iframe) 498{ 499 IFrameScope scope(iframe); 500 501 panic("FIQ not implemented yet!"); 502} 503