138451Smsmith/* 238451Smsmith * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. 338451Smsmith * Distributed under the terms of the MIT License. 438451Smsmith */ 538451Smsmith 638451Smsmith 738451Smsmith#include "long.h" 838451Smsmith 938451Smsmith#include <algorithm> 1038451Smsmith 1138451Smsmith#include <KernelExport.h> 1238451Smsmith 1338451Smsmith// Include the x86_64 version of descriptors.h 1438451Smsmith#define __x86_64__ 1538451Smsmith#include <arch/x86/descriptors.h> 1638451Smsmith#undef __x86_64__ 1738451Smsmith 1838451Smsmith#include <arch_system_info.h> 1938451Smsmith#include <boot/platform.h> 2038451Smsmith#include <boot/heap.h> 2138451Smsmith#include <boot/stage2.h> 2238451Smsmith#include <boot/stdio.h> 2338451Smsmith#include <kernel.h> 2438451Smsmith#include <safemode.h> 2538451Smsmith 2638451Smsmith#include "debug.h" 2738451Smsmith#include "mmu.h" 2838451Smsmith#include "smp.h" 2938451Smsmith 3038451Smsmith 3138451Smsmithstatic const uint64 kTableMappingFlags = 0x7; 3238451Smsmithstatic const uint64 kLargePageMappingFlags = 0x183; 3338451Smsmithstatic const uint64 kPageMappingFlags = 0x103; 3438451Smsmith // Global, R/W, Present 3538451Smsmith 3638451Smsmithextern "C" void long_enter_kernel(int currentCPU, uint64 stackTop); 3738451Smsmith 3838451Smsmithextern uint64 gLongGDT; 3938451Smsmithextern uint32 gLongPhysicalPMLTop; 4038451Smsmithextern bool gLongLA57; 4184221Sdillonextern uint64 gLongKernelEntry; 4284221Sdillon 4384221Sdillon 4438451Smsmith/*! Convert a 32-bit address to a 64-bit address. */ 4538451Smsmithstatic inline uint64 4638451Smsmithfix_address(uint64 address) 4738451Smsmith{ 4838451Smsmith if (address >= KERNEL_LOAD_BASE) 49113159Speter return address + KERNEL_FIXUP_FOR_LONG_MODE; 50113159Speter else 51103949Smike return address; 5255137Speter} 5338451Smsmith 5438451Smsmith 5538451Smsmithtemplate<typename Type> 5638451Smsmithinline void 5738451Smsmithfix_address(FixedWidthPointer<Type>& p) 5838451Smsmith{ 5938451Smsmith if (p != NULL) 6038451Smsmith p.SetTo(fix_address(p.Get())); 61113159Speter} 62113159Speter 63156518Sjkim 6438451Smsmithstatic void 6538451Smsmithlong_gdt_init() 6638451Smsmith{ 6738451Smsmith STATIC_ASSERT(BOOT_GDT_SEGMENT_COUNT > KERNEL_CODE_SEGMENT 6838451Smsmith && BOOT_GDT_SEGMENT_COUNT > KERNEL_DATA_SEGMENT 6938451Smsmith && BOOT_GDT_SEGMENT_COUNT > USER_CODE_SEGMENT 7038451Smsmith && BOOT_GDT_SEGMENT_COUNT > USER_DATA_SEGMENT); 7138451Smsmith 7238451Smsmith clear_segment_descriptor(&gBootGDT[0]); 7338451Smsmith 7438451Smsmith // Set up code/data segments (TSS segments set up later in the kernel). 7538451Smsmith set_segment_descriptor(&gBootGDT[KERNEL_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, 7638451Smsmith DPL_KERNEL); 7738451Smsmith set_segment_descriptor(&gBootGDT[KERNEL_DATA_SEGMENT], DT_DATA_WRITEABLE, 7838451Smsmith DPL_KERNEL); 7938451Smsmith set_segment_descriptor(&gBootGDT[USER_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, 8038451Smsmith DPL_USER); 8138451Smsmith set_segment_descriptor(&gBootGDT[USER_DATA_SEGMENT], DT_DATA_WRITEABLE, 8238451Smsmith DPL_USER); 8338451Smsmith 8438451Smsmith // Used by long_enter_kernel(). 8538451Smsmith gLongGDT = fix_address((addr_t)gBootGDT); 8638451Smsmith dprintf("GDT at 0x%llx\n", gLongGDT); 8738451Smsmith} 8838451Smsmith 8938451Smsmith 9038451Smsmithstatic void 9138451Smsmithlong_mmu_init() 9238451Smsmith{ 9338451Smsmith uint64* pmlTop; 9438451Smsmith // Allocate the top level PMLTop. 9538451Smsmith pmlTop = (uint64*)mmu_allocate_page(&gKernelArgs.arch_args.phys_pgdir); 9638451Smsmith memset(pmlTop, 0, B_PAGE_SIZE); 9738451Smsmith gKernelArgs.arch_args.vir_pgdir = fix_address((uint64)(addr_t)pmlTop); 9840805Smsmith 9940805Smsmith // Store the virtual memory usage information. 10040805Smsmith gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT; 10140805Smsmith gKernelArgs.virtual_allocated_range[0].size = mmu_get_virtual_usage(); 10240805Smsmith gKernelArgs.num_virtual_allocated_ranges = 1; 10340805Smsmith gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT 10440805Smsmith + gKernelArgs.virtual_allocated_range[0].size, 0x200000); 10540805Smsmith 10640805Smsmith // Find the highest physical memory address. We map all physical memory 10738451Smsmith // into the kernel address space, so we want to make sure we map everything 108113159Speter // we have available. 109113159Speter uint64 maxAddress = 0; 110113159Speter for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 111113159Speter maxAddress = std::max(maxAddress, 11238451Smsmith gKernelArgs.physical_memory_range[i].start 11338451Smsmith + gKernelArgs.physical_memory_range[i].size); 114156518Sjkim } 115113159Speter 116156518Sjkim // Want to map at least 4GB, there may be stuff other than usable RAM that 11738451Smsmith // could be in the first 4GB of physical address space. 118113159Speter maxAddress = std::max(maxAddress, (uint64)0x100000000ll); 119113159Speter maxAddress = ROUNDUP(maxAddress, 0x40000000); 12038451Smsmith 121156518Sjkim // Currently only use 1 PDPT (512GB). This will need to change if someone 122156518Sjkim // wants to use Haiku on a box with more than 512GB of RAM but that's 123113159Speter // probably not going to happen any time soon. 12438451Smsmith if (maxAddress / 0x40000000 > 512) 125113159Speter panic("Can't currently support more than 512GB of RAM!"); 12638451Smsmith 12738451Smsmith uint64* pml4 = pmlTop; 12838451Smsmith addr_t physicalAddress; 12938451Smsmith cpuid_info info; 13038451Smsmith if (get_current_cpuid(&info, 7, 0) == B_OK 13138451Smsmith && (info.regs.ecx & IA32_FEATURE_LA57) != 0) { 13238451Smsmith 13338451Smsmith if (get_safemode_boolean(B_SAFEMODE_256_TB_MEMORY_LIMIT, false)) { 13438451Smsmith // LA57 has been disabled! 13538451Smsmith dprintf("la57 disabled per safemode setting\n"); 13638451Smsmith } else { 13738451Smsmith dprintf("la57 enabled\n"); 13838451Smsmith gLongLA57 = true; 13938451Smsmith pml4 = (uint64*)mmu_allocate_page(&physicalAddress); 14038451Smsmith memset(pml4, 0, B_PAGE_SIZE); 14138451Smsmith pmlTop[511] = physicalAddress | kTableMappingFlags; 14238451Smsmith pmlTop[0] = physicalAddress | kTableMappingFlags; 14338451Smsmith } 14438451Smsmith } 14538451Smsmith 14638451Smsmith uint64* pdpt; 14738451Smsmith uint64* pageDir; 14838451Smsmith uint64* pageTable; 14938451Smsmith 15038451Smsmith // Create page tables for the physical map area. Also map this PDPT 15138451Smsmith // temporarily at the bottom of the address space so that we are identity 15238451Smsmith // mapped. 15338451Smsmith 15438451Smsmith pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 15538451Smsmith memset(pdpt, 0, B_PAGE_SIZE); 15638451Smsmith pml4[510] = physicalAddress | kTableMappingFlags; 15738451Smsmith pml4[0] = physicalAddress | kTableMappingFlags; 15838451Smsmith 159113159Speter for (uint64 i = 0; i < maxAddress; i += 0x40000000) { 160113159Speter pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 161113159Speter memset(pageDir, 0, B_PAGE_SIZE); 16238451Smsmith pdpt[i / 0x40000000] = physicalAddress | kTableMappingFlags; 16338451Smsmith 164113159Speter for (uint64 j = 0; j < 0x40000000; j += 0x200000) { 165113159Speter pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags; 166113159Speter } 167156518Sjkim 16838451Smsmith mmu_free(pageDir, B_PAGE_SIZE); 16938451Smsmith } 17038451Smsmith 171113159Speter mmu_free(pdpt, B_PAGE_SIZE); 17238451Smsmith 17338451Smsmith // Allocate tables for the kernel mappings. 17438451Smsmith pdpt = (uint64*)mmu_allocate_page(&physicalAddress); 17538451Smsmith memset(pdpt, 0, B_PAGE_SIZE); 17638451Smsmith pml4[511] = physicalAddress | kTableMappingFlags; 17738451Smsmith 17838451Smsmith pageDir = (uint64*)mmu_allocate_page(&physicalAddress); 17938451Smsmith memset(pageDir, 0, B_PAGE_SIZE); 18038451Smsmith pdpt[510] = physicalAddress | kTableMappingFlags; 18138451Smsmith 18238451Smsmith // We can now allocate page tables and duplicate the mappings across from 18338451Smsmith // the 32-bit address space to them. 18438451Smsmith pageTable = NULL; 18538451Smsmith for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size 18638451Smsmith / B_PAGE_SIZE; i++) { 187113159Speter if ((i % 512) == 0) { 188113159Speter if (pageTable) 18938451Smsmith mmu_free(pageTable, B_PAGE_SIZE); 19038451Smsmith 191113159Speter pageTable = (uint64*)mmu_allocate_page(&physicalAddress); 192113159Speter memset(pageTable, 0, B_PAGE_SIZE); 193156518Sjkim pageDir[i / 512] = physicalAddress | kTableMappingFlags; 194113159Speter } 19538451Smsmith 19638451Smsmith // Get the physical address to map. 19738451Smsmith if (!mmu_get_virtual_mapping(KERNEL_LOAD_BASE + (i * B_PAGE_SIZE), 19838451Smsmith &physicalAddress)) 19938451Smsmith continue; 20038451Smsmith 20138451Smsmith pageTable[i % 512] = physicalAddress | kPageMappingFlags; 20238451Smsmith } 20338451Smsmith 20438451Smsmith if (pageTable) 20538451Smsmith mmu_free(pageTable, B_PAGE_SIZE); 20638451Smsmith mmu_free(pageDir, B_PAGE_SIZE); 20738451Smsmith mmu_free(pdpt, B_PAGE_SIZE); 20838451Smsmith if (pml4 != pmlTop) 20938451Smsmith mmu_free(pml4, B_PAGE_SIZE); 21038451Smsmith 21138451Smsmith // Sort the address ranges. 21238451Smsmith sort_address_ranges(gKernelArgs.physical_memory_range, 21338451Smsmith gKernelArgs.num_physical_memory_ranges); 21438451Smsmith sort_address_ranges(gKernelArgs.physical_allocated_range, 21538451Smsmith gKernelArgs.num_physical_allocated_ranges); 21638451Smsmith sort_address_ranges(gKernelArgs.virtual_allocated_range, 21738451Smsmith gKernelArgs.num_virtual_allocated_ranges); 21838451Smsmith 21938451Smsmith dprintf("phys memory ranges:\n"); 22038451Smsmith for (uint32 i = 0; i < gKernelArgs.num_physical_memory_ranges; i++) { 22138451Smsmith dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 22238451Smsmith gKernelArgs.physical_memory_range[i].start, 22338451Smsmith gKernelArgs.physical_memory_range[i].size); 22438451Smsmith } 22538451Smsmith 22638451Smsmith dprintf("allocated phys memory ranges:\n"); 22738451Smsmith for (uint32 i = 0; i < gKernelArgs.num_physical_allocated_ranges; i++) { 22838451Smsmith dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 22938451Smsmith gKernelArgs.physical_allocated_range[i].start, 23038451Smsmith gKernelArgs.physical_allocated_range[i].size); 23138451Smsmith } 23238451Smsmith 23338451Smsmith dprintf("allocated virt memory ranges:\n"); 23438451Smsmith for (uint32 i = 0; i < gKernelArgs.num_virtual_allocated_ranges; i++) { 23538451Smsmith dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 23638451Smsmith gKernelArgs.virtual_allocated_range[i].start, 23738451Smsmith gKernelArgs.virtual_allocated_range[i].size); 23838451Smsmith } 23938451Smsmith 24038451Smsmith gLongPhysicalPMLTop = gKernelArgs.arch_args.phys_pgdir; 241113159Speter} 24238451Smsmith 243156518Sjkim 24438451Smsmithstatic void 24538451Smsmithconvert_preloaded_image(preloaded_elf64_image* image) 246113159Speter{ 24738451Smsmith fix_address(image->next); 24838451Smsmith fix_address(image->name); 24938451Smsmith fix_address(image->debug_string_table); 25038451Smsmith fix_address(image->syms); 251113159Speter fix_address(image->rel); 25238451Smsmith fix_address(image->rela); 25338451Smsmith fix_address(image->pltrel); 25438451Smsmith fix_address(image->debug_symbols); 25538451Smsmith} 25638451Smsmith 25738451Smsmith 25838451Smsmith/*! Convert all addresses in kernel_args to 64-bit addresses. */ 25938451Smsmithstatic void 26038451Smsmithconvert_kernel_args() 26138451Smsmith{ 26238451Smsmith fix_address(gKernelArgs.boot_volume); 26338451Smsmith fix_address(gKernelArgs.vesa_modes); 26438451Smsmith fix_address(gKernelArgs.edid_info); 26538451Smsmith fix_address(gKernelArgs.debug_output); 26638451Smsmith fix_address(gKernelArgs.previous_debug_output); 26738451Smsmith fix_address(gKernelArgs.boot_splash); 26838451Smsmith fix_address(gKernelArgs.ucode_data); 26938451Smsmith fix_address(gKernelArgs.arch_args.apic); 27038451Smsmith fix_address(gKernelArgs.arch_args.hpet); 27138451Smsmith 27238451Smsmith convert_preloaded_image(static_cast<preloaded_elf64_image*>( 27338451Smsmith gKernelArgs.kernel_image.Pointer())); 27438451Smsmith fix_address(gKernelArgs.kernel_image); 27538451Smsmith 27638451Smsmith // Iterate over the preloaded images. Must save the next address before 27738451Smsmith // converting, as the next pointer will be converted. 27838451Smsmith preloaded_image* image = gKernelArgs.preloaded_images; 27938451Smsmith fix_address(gKernelArgs.preloaded_images); 28038451Smsmith while (image != NULL) { 281113159Speter preloaded_image* next = image->next; 282113159Speter convert_preloaded_image(static_cast<preloaded_elf64_image*>(image)); 28338451Smsmith image = next; 284113159Speter } 285113159Speter 286113159Speter // Set correct kernel args range addresses. 287113159Speter dprintf("kernel args ranges:\n"); 28838451Smsmith for (uint32 i = 0; i < gKernelArgs.num_kernel_args_ranges; i++) { 289113159Speter gKernelArgs.kernel_args_range[i].start = fix_address( 290113159Speter gKernelArgs.kernel_args_range[i].start); 291113159Speter dprintf(" base %#018" B_PRIx64 ", length %#018" B_PRIx64 "\n", 292113159Speter gKernelArgs.kernel_args_range[i].start, 293113159Speter gKernelArgs.kernel_args_range[i].size); 29438451Smsmith } 29538451Smsmith 296113159Speter // Fix driver settings files. 297113159Speter driver_settings_file* file = gKernelArgs.driver_settings; 298113159Speter fix_address(gKernelArgs.driver_settings); 299113159Speter while (file != NULL) { 300113159Speter driver_settings_file* next = file->next; 301113159Speter fix_address(file->next); 302113159Speter fix_address(file->buffer); 303113159Speter file = next; 304113159Speter } 305113159Speter} 306113159Speter 30738451Smsmith 30838451Smsmithstatic void 309113159Speterenable_sse() 31038451Smsmith{ 31138451Smsmith x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION); 312113159Speter x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU)); 313113159Speter} 314113159Speter 31538451Smsmith 316113159Speterstatic void 317113159Speterlong_smp_start_kernel(void) 318113159Speter{ 319113159Speter uint32 cpu = smp_get_current_cpu(); 320113159Speter 321113159Speter // Important. Make sure supervisor threads can fault on read only pages... 322113159Speter asm("movl %%eax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1)); 323113159Speter asm("cld"); 32438451Smsmith asm("fninit"); 32538451Smsmith enable_sse(); 32638451Smsmith 32738451Smsmith // Fix our kernel stack address. 32838451Smsmith gKernelArgs.cpu_kstack[cpu].start 32938451Smsmith = fix_address(gKernelArgs.cpu_kstack[cpu].start); 33038451Smsmith 33138451Smsmith long_enter_kernel(cpu, gKernelArgs.cpu_kstack[cpu].start 33238451Smsmith + gKernelArgs.cpu_kstack[cpu].size); 33338451Smsmith 33438451Smsmith panic("Shouldn't get here"); 33538451Smsmith} 33638451Smsmith 33738451Smsmith 33838451Smsmithvoid 33938451Smsmithlong_start_kernel() 34038451Smsmith{ 34138451Smsmith // Check whether long mode is supported. 34238451Smsmith cpuid_info info; 34338451Smsmith get_current_cpuid(&info, 0x80000001, 0); 34438451Smsmith if ((info.regs.edx & (1 << 29)) == 0) 345113159Speter panic("64-bit kernel requires a 64-bit CPU"); 346113159Speter 347113159Speter enable_sse(); 34838451Smsmith 34938451Smsmith preloaded_elf64_image *image = static_cast<preloaded_elf64_image *>( 350113159Speter gKernelArgs.kernel_image.Pointer()); 351156518Sjkim 352156518Sjkim smp_init_other_cpus(); 35338451Smsmith 35438451Smsmith long_gdt_init(); 355113159Speter debug_cleanup(); 356113159Speter long_mmu_init(); 357113159Speter convert_kernel_args(); 358113159Speter 359113159Speter // Save the kernel entry point address. 360113159Speter gLongKernelEntry = image->elf_header.e_entry; 361113159Speter dprintf("kernel entry at %#llx\n", gLongKernelEntry); 362113159Speter 363113159Speter // Fix our kernel stack address. 364113159Speter gKernelArgs.cpu_kstack[0].start 365113159Speter = fix_address(gKernelArgs.cpu_kstack[0].start); 366113159Speter 367113159Speter // We're about to enter the kernel -- disable console output. 368113159Speter stdout = NULL; 369113159Speter 370113159Speter smp_boot_other_cpus(long_smp_start_kernel); 371113159Speter 372113159Speter // Enter the kernel! 373113159Speter long_enter_kernel(0, gKernelArgs.cpu_kstack[0].start 374113159Speter + gKernelArgs.cpu_kstack[0].size); 375113159Speter 376113159Speter panic("Shouldn't get here"); 377113159Speter} 378113159Speter