1/*
2 * Copyright 2010, Ithamar R. Adema, ithamar.adema@team-embedded.nl
3 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
4 * Copyright 2002-2007, Axel D��rfler, axeld@pinc-software.de. All rights reserved.
5 * Distributed under the terms of the MIT License.
6 *
7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8 * Distributed under the terms of the NewOS License.
9 */
10
11
12#include "paging/32bit/ARMPagingMethod32Bit.h"
13
14#include <stdlib.h>
15#include <string.h>
16
17#include <AutoDeleter.h>
18
19#include <arch/smp.h>
20#include <arch_system_info.h>
21#include <boot/kernel_args.h>
22#include <int.h>
23#include <thread.h>
24#include <vm/vm.h>
25#include <vm/VMAddressSpace.h>
26
27#include "paging/32bit/ARMPagingStructures32Bit.h"
28#include "paging/32bit/ARMVMTranslationMap32Bit.h"
29#include "paging/arm_physical_page_mapper.h"
30#include "paging/arm_physical_page_mapper_large_memory.h"
31
32
33//#define TRACE_ARM_PAGING_METHOD_32_BIT
34#ifdef TRACE_ARM_PAGING_METHOD_32_BIT
35#	define TRACE(x...) dprintf(x)
36#else
37#	define TRACE(x...) ;
38#endif
39
40
41#define MAX_INITIAL_POOLS	\
42	(ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, 1024) / 1024)
43
44
45using ARMLargePhysicalPageMapper::PhysicalPageSlot;
46
47
48// #pragma mark - ARMPagingMethod32Bit::PhysicalPageSlotPool
49
50struct ARMPagingMethod32Bit::PhysicalPageSlotPool
51	: ARMLargePhysicalPageMapper::PhysicalPageSlotPool {
52public:
53	virtual						~PhysicalPageSlotPool();
54
55			status_t			InitInitial(kernel_args* args);
56			status_t			InitInitialPostArea(kernel_args* args);
57
58			void				Init(area_id dataArea, void* data,
59									area_id virtualArea, addr_t virtualBase);
60
61	virtual	status_t			AllocatePool(
62									ARMLargePhysicalPageMapper
63										::PhysicalPageSlotPool*& _pool);
64	virtual	void				Map(phys_addr_t physicalAddress,
65									addr_t virtualAddress);
66
67public:
68	static	PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS];
69
70private:
71	area_id					fDataArea;
72	area_id					fVirtualArea;
73	addr_t					fVirtualBase;
74	page_table_entry*		fPageTable;
75};
76
77
78ARMPagingMethod32Bit::PhysicalPageSlotPool
79	ARMPagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool[
80		MAX_INITIAL_POOLS];
81
82
83ARMPagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool()
84{
85}
86
87
88status_t
89ARMPagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args)
90{
91	// allocate a virtual address range for the pages to be mapped into
92	addr_t virtualBase = vm_allocate_early(args, 1024 * B_PAGE_SIZE, 0, 0,
93		kPageTableAlignment);
94	if (virtualBase == 0) {
95		panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
96			"physical page pool space in virtual address space!");
97		return B_ERROR;
98	}
99
100	// allocate memory for the page table and data
101	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
102	page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args,
103		areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
104	if (pageTable == 0) {
105		panic("ARMPagingMethod32Bit::PhysicalPageSlotPool::InitInitial(): "
106			"Failed to allocate memory for page table!");
107		return B_ERROR;
108	}
109
110	// prepare the page table
111	_EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE);
112
113	// init the pool structure and add the initial pool
114	Init(-1, pageTable, -1, (addr_t)virtualBase);
115
116	return B_OK;
117}
118
119
120status_t
121ARMPagingMethod32Bit::PhysicalPageSlotPool::InitInitialPostArea(
122	kernel_args* args)
123{
124	// create an area for the (already allocated) data
125	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
126	void* temp = fPageTable;
127	area_id area = create_area("physical page pool", &temp,
128		B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
129		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
130	if (area < 0) {
131		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
132			"create area for physical page pool.");
133		return area;
134	}
135	fDataArea = area;
136
137	// create an area for the virtual address space
138	temp = (void*)fVirtualBase;
139	area = vm_create_null_area(VMAddressSpace::KernelID(),
140		"physical page pool space", &temp, B_EXACT_ADDRESS,
141		1024 * B_PAGE_SIZE, 0);
142	if (area < B_OK) {
143		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
144			"create area for physical page pool space.");
145		return area;
146	}
147	fVirtualArea = area;
148
149	return B_OK;
150}
151
152
153void
154ARMPagingMethod32Bit::PhysicalPageSlotPool::Init(area_id dataArea, void* data,
155	area_id virtualArea, addr_t virtualBase)
156{
157	fDataArea = dataArea;
158	fVirtualArea = virtualArea;
159	fVirtualBase = virtualBase;
160	fPageTable = (page_table_entry*)data;
161
162	// init slot list
163	fSlots = (PhysicalPageSlot*)(fPageTable + 1024);
164	addr_t slotAddress = virtualBase;
165	for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) {
166		PhysicalPageSlot* slot = &fSlots[i];
167		slot->next = slot + 1;
168		slot->pool = this;
169		slot->address = slotAddress;
170	}
171
172	fSlots[1023].next = NULL;
173		// terminate list
174}
175
176
177void
178ARMPagingMethod32Bit::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
179	addr_t virtualAddress)
180{
181	page_table_entry& pte = fPageTable[
182		(virtualAddress - fVirtualBase) / B_PAGE_SIZE];
183	pte = (physicalAddress & ARM_PTE_ADDRESS_MASK)
184		| ARM_MMU_L2_TYPE_SMALLNEW
185		| ARM_MMU_L2_FLAG_B | ARM_MMU_L2_FLAG_C
186		| ARM_MMU_L2_FLAG_HAIKU_KERNEL_RW | ARM_MMU_L2_FLAG_XN;
187
188	arch_cpu_invalidate_TLB_page(virtualAddress);
189}
190
191
192status_t
193ARMPagingMethod32Bit::PhysicalPageSlotPool::AllocatePool(
194	ARMLargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
195{
196	// create the pool structure
197	PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
198	if (pool == NULL)
199		return B_NO_MEMORY;
200	ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
201
202	// create an area that can contain the page table and the slot
203	// structures
204	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
205	void* data;
206	virtual_address_restrictions virtualRestrictions = {};
207	virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
208	physical_address_restrictions physicalRestrictions = {};
209	area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
210		PAGE_ALIGN(areaSize), B_FULL_LOCK,
211		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
212		&virtualRestrictions, &physicalRestrictions, &data);
213	if (dataArea < 0)
214		return dataArea;
215
216	// create the null area for the virtual address space
217	void* virtualBase;
218	area_id virtualArea = vm_create_null_area(
219		VMAddressSpace::KernelID(), "physical page pool space",
220		&virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE,
221		CREATE_AREA_PRIORITY_VIP);
222	if (virtualArea < 0) {
223		delete_area(dataArea);
224		return virtualArea;
225	}
226
227	// prepare the page table
228	memset(data, 0, B_PAGE_SIZE);
229
230	// get the page table's physical address
231	phys_addr_t physicalTable;
232	ARMVMTranslationMap32Bit* map = static_cast<ARMVMTranslationMap32Bit*>(
233		VMAddressSpace::Kernel()->TranslationMap());
234	uint32 dummyFlags;
235	cpu_status state = disable_interrupts();
236	map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
237	restore_interrupts(state);
238
239	// put the page table into the page directory
240	int32 index = VADDR_TO_PDENT((addr_t)virtualBase);
241	page_directory_entry* entry
242		= &map->PagingStructures32Bit()->pgdir_virt[index];
243	PutPageTableInPageDir(entry, physicalTable, ARM_MMU_L1_FLAG_PXN);
244	ARMPagingStructures32Bit::UpdateAllPageDirs(index, *entry);
245
246	// init the pool structure
247	pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase);
248	poolDeleter.Detach();
249	_pool = pool;
250	return B_OK;
251}
252
253
254// #pragma mark - ARMPagingMethod32Bit
255
256
257ARMPagingMethod32Bit::ARMPagingMethod32Bit()
258	:
259	fKernelPhysicalPageDirectory(0),
260	fKernelVirtualPageDirectory(NULL),
261	fPhysicalPageMapper(NULL),
262	fKernelPhysicalPageMapper(NULL)
263{
264}
265
266
267ARMPagingMethod32Bit::~ARMPagingMethod32Bit()
268{
269}
270
271
272status_t
273ARMPagingMethod32Bit::Init(kernel_args* args,
274	VMPhysicalPageMapper** _physicalPageMapper)
275{
276	TRACE("ARMPagingMethod32Bit::Init(): entry\n");
277
278	fKernelPhysicalPageDirectory = args->arch_args.phys_pgdir;
279	fKernelVirtualPageDirectory = (page_directory_entry*)
280		args->arch_args.vir_pgdir;
281
282#ifdef TRACE_ARM_PAGING_METHOD_32_BIT
283	TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n",
284		fKernelVirtualPageDirectory, fKernelPhysicalPageDirectory);
285#endif
286
287	ARMPagingStructures32Bit::StaticInit();
288
289	// create the initial pools for the physical page mapper
290	int32 poolCount = _GetInitialPoolCount();
291	PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool;
292
293	for (int32 i = 0; i < poolCount; i++) {
294		new(&pool[i]) PhysicalPageSlotPool;
295		status_t error = pool[i].InitInitial(args);
296		if (error != B_OK) {
297			panic("ARMPagingMethod32Bit::Init(): Failed to create initial pool "
298				"for physical page mapper!");
299			return error;
300		}
301	}
302
303	// create physical page mapper
304	large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool),
305		fPhysicalPageMapper, fKernelPhysicalPageMapper);
306		// TODO: Select the best page mapper!
307
308	TRACE("ARMPagingMethod32Bit::Init(): done\n");
309
310	*_physicalPageMapper = fPhysicalPageMapper;
311	return B_OK;
312}
313
314
315status_t
316ARMPagingMethod32Bit::InitPostArea(kernel_args* args)
317{
318	void *temp;
319	area_id area;
320
321	temp = (void*)fKernelVirtualPageDirectory;
322	area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, args->arch_args.next_pagetable,
323		B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
324	ASSERT_PRINT(area >= 0, "Failed mapping the kernel page directory: 0x%08lx!", area);
325
326	int32 poolCount = _GetInitialPoolCount();
327	for (int32 i = 0; i < poolCount; i++) {
328		status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i]
329			.InitInitialPostArea(args);
330		if (error != B_OK)
331			return error;
332	}
333
334	return B_OK;
335}
336
337
338status_t
339ARMPagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
340{
341	ARMVMTranslationMap32Bit* map = new(std::nothrow) ARMVMTranslationMap32Bit;
342	if (map == NULL)
343		return B_NO_MEMORY;
344
345	status_t error = map->Init(kernel);
346	if (error != B_OK) {
347		delete map;
348		return error;
349	}
350
351	*_map = map;
352	return B_OK;
353}
354
355
356static void
357get_free_pgtable(kernel_args* args, phys_addr_t* phys_addr, addr_t* virt_addr)
358{
359	if (args->arch_args.next_pagetable >= args->arch_args.last_pagetable)
360		panic("ran out of early page tables");
361
362	phys_addr_t phys = args->arch_args.phys_pgdir + args->arch_args.next_pagetable;
363	addr_t virt = args->arch_args.vir_pgdir + args->arch_args.next_pagetable;
364	args->arch_args.next_pagetable += ARM_MMU_L2_COARSE_TABLE_SIZE;
365
366	*phys_addr = phys;
367	*virt_addr = virt;
368}
369
370status_t
371ARMPagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress,
372	phys_addr_t physicalAddress, uint8 attributes,
373	page_num_t (*get_free_page)(kernel_args*))
374{
375	// check to see if a page table exists for this range
376	int index = VADDR_TO_PDENT(virtualAddress);
377	if ((fKernelVirtualPageDirectory[index] & ARM_PDE_TYPE_MASK) == 0) {
378		phys_addr_t pgtable_phys;
379		addr_t pgtable_virt;
380		page_directory_entry *e;
381
382		// we need to allocate a pgtable
383		get_free_pgtable(args, &pgtable_phys, &pgtable_virt);
384
385		TRACE("ARMPagingMethod32Bit::MapEarly(): asked for free page for "
386			"pgtable. phys=%#" B_PRIxPHYSADDR ", virt=%#" B_PRIxADDR "\n",
387			pgtable_phys, pgtable_virt);
388
389		// zero it out in it's new mapping
390		memset((void*)pgtable_virt, 0, B_PAGE_SIZE);
391
392		// put it in the pgdir
393		e = &fKernelVirtualPageDirectory[index];
394		PutPageTableInPageDir(e, pgtable_phys,
395			(virtualAddress < KERNEL_BASE) ? ARM_MMU_L1_FLAG_PXN : 0);
396	}
397
398	phys_addr_t ptEntryPhys = fKernelVirtualPageDirectory[index] & ARM_PDE_ADDRESS_MASK;
399	addr_t ptEntryVirt = ptEntryPhys - args->arch_args.phys_pgdir + args->arch_args.vir_pgdir;
400	page_table_entry* ptEntry = (page_table_entry*)ptEntryVirt;
401	ptEntry += VADDR_TO_PTENT(virtualAddress);
402
403	ASSERT_PRINT(
404		(*ptEntry & ARM_PTE_TYPE_MASK) == 0,
405		"virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32
406		", existing pte: %#" B_PRIx32, virtualAddress, fKernelVirtualPageDirectory[index],
407		*ptEntry);
408
409	// now, fill in the pentry
410	PutPageTableEntryInTable(ptEntry,
411		physicalAddress, attributes | PAGE_ACCESSED | PAGE_MODIFIED, 0,
412		IS_KERNEL_ADDRESS(virtualAddress));
413
414	return B_OK;
415}
416
417
418bool
419ARMPagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress,
420	uint32 protection)
421{
422#if 0
423	// We only trust the kernel team's page directory. So switch to it first.
424	// Always set it to make sure the TLBs don't contain obsolete data.
425	uint32 physicalPageDirectory = x86_read_cr3();
426	x86_write_cr3(fKernelPhysicalPageDirectory);
427
428	// get the page directory entry for the address
429	page_directory_entry pageDirectoryEntry;
430	uint32 index = VADDR_TO_PDENT(virtualAddress);
431
432	if (physicalPageDirectory == fKernelPhysicalPageDirectory) {
433		pageDirectoryEntry = fKernelVirtualPageDirectory[index];
434	} else if (fPhysicalPageMapper != NULL) {
435		// map the original page directory and get the entry
436		void* handle;
437		addr_t virtualPageDirectory;
438		status_t error = fPhysicalPageMapper->GetPageDebug(
439			physicalPageDirectory, &virtualPageDirectory, &handle);
440		if (error == B_OK) {
441			pageDirectoryEntry
442				= ((page_directory_entry*)virtualPageDirectory)[index];
443			fPhysicalPageMapper->PutPageDebug(virtualPageDirectory, handle);
444		} else
445			pageDirectoryEntry = 0;
446	} else
447		pageDirectoryEntry = 0;
448
449	// map the page table and get the entry
450	page_table_entry pageTableEntry;
451	index = VADDR_TO_PTENT(virtualAddress);
452
453	if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0
454			&& fPhysicalPageMapper != NULL) {
455		void* handle;
456		addr_t virtualPageTable;
457		status_t error = fPhysicalPageMapper->GetPageDebug(
458			pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable,
459			&handle);
460		if (error == B_OK) {
461			pageTableEntry = ((page_table_entry*)virtualPageTable)[index];
462			fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
463		} else
464			pageTableEntry = 0;
465	} else
466		pageTableEntry = 0;
467
468	// switch back to the original page directory
469	if (physicalPageDirectory != fKernelPhysicalPageDirectory)
470		x86_write_cr3(physicalPageDirectory);
471
472	if ((pageTableEntry & X86_PTE_PRESENT) == 0)
473		return false;
474
475	// present means kernel-readable, so check for writable
476	return (protection & B_KERNEL_WRITE_AREA) == 0
477		|| (pageTableEntry & X86_PTE_WRITABLE) != 0;
478#endif
479	//IRA: fix the above!
480	return true;
481}
482
483
484/*static*/ void
485ARMPagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry,
486	phys_addr_t pgtablePhysical, uint32 attributes)
487{
488	dsb();
489
490	*entry = (pgtablePhysical & ARM_PDE_ADDRESS_MASK) | ARM_MMU_L1_TYPE_COARSE | attributes;
491
492	dsb();
493	isb();
494}
495
496
497/*static*/ void
498ARMPagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry,
499	phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
500	bool globalPage)
501{
502	page_table_entry page = (physicalAddress & ARM_PTE_ADDRESS_MASK)
503		| ARM_MMU_L2_TYPE_SMALLNEW
504		| MemoryTypeToPageTableEntryFlags(memoryType)
505		| AttributesToPageTableEntryFlags(attributes)
506		| (globalPage ? 0 : ARM_MMU_L2_FLAG_NG);
507
508	// put it in the page table
509	*(volatile page_table_entry*)entry = page;
510
511	dsb();
512	isb();
513}
514
515
516inline int32
517ARMPagingMethod32Bit::_GetInitialPoolCount()
518{
519	int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU
520			+ EXTRA_SLOTS;
521	return (requiredSlots + 1023) / 1024;
522}
523
524
525/*static*/ void
526ARMPagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables,
527	addr_t address, size_t size)
528{
529	ARMPagingMethod32Bit* method = ARMPagingMethod32Bit::Method();
530	memset(pageTables, 0, 256 * (size / (B_PAGE_SIZE * 256)));
531
532	// put the array of pgtables directly into the kernel pagedir
533	// these will be wired and kept mapped into virtual space to be easy to get
534	// to
535	{
536		addr_t virtualTable = (addr_t)pageTables;
537
538		for (size_t i = 0; i < (size / (B_PAGE_SIZE * 256));
539				i++, virtualTable += 256*sizeof(page_directory_entry)) {
540			phys_addr_t physicalTable = 0;
541			_EarlyQuery(virtualTable, &physicalTable);
542			page_directory_entry* entry = method->KernelVirtualPageDirectory()
543				+ VADDR_TO_PDENT(address) + i;
544			PutPageTableInPageDir(entry, physicalTable,
545				(address < KERNEL_BASE) ? ARM_MMU_L1_FLAG_PXN : 0);
546		}
547	}
548}
549
550
551//! TODO: currently assumes this translation map is active
552/*static*/ status_t
553ARMPagingMethod32Bit::_EarlyQuery(addr_t virtualAddress,
554	phys_addr_t *_physicalAddress)
555{
556	ARMPagingMethod32Bit* method = ARMPagingMethod32Bit::Method();
557	int index = VADDR_TO_PDENT(virtualAddress);
558	if ((method->KernelVirtualPageDirectory()[index] & ARM_PDE_TYPE_MASK) == 0) {
559		// no pagetable here
560		return B_ERROR;
561	}
562
563	phys_addr_t ptEntryPhys = method->KernelVirtualPageDirectory()[index] & ARM_PDE_ADDRESS_MASK;
564	addr_t ptEntryVirt = ptEntryPhys -
565		(uint32_t)method->KernelPhysicalPageDirectory() +
566		(uint32_t)method->KernelVirtualPageDirectory();
567
568	page_table_entry* entry = (page_table_entry*)ptEntryVirt;
569	entry += VADDR_TO_PTENT(virtualAddress);
570
571	if ((*entry & ARM_PTE_TYPE_MASK) == 0) {
572		// page mapping not valid
573		return B_ERROR;
574	}
575
576	*_physicalAddress = (*entry & ARM_PTE_ADDRESS_MASK)
577		| VADDR_TO_PGOFF(virtualAddress);
578
579	return B_OK;
580}
581