1/*
2 * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
3 * Copyright 2002-2007, Axel Dörfler, axeld@pinc-software.de. All rights reserved.
4 * Distributed under the terms of the MIT License.
5 *
6 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
7 * Distributed under the terms of the NewOS License.
8 */
9
10
11#include "paging/32bit/X86PagingMethod32Bit.h"
12
13#include <stdlib.h>
14#include <string.h>
15
16#include <AutoDeleter.h>
17
18#include <arch_system_info.h>
19#include <boot/kernel_args.h>
20#include <int.h>
21#include <thread.h>
22#include <vm/vm.h>
23#include <vm/VMAddressSpace.h>
24
25#include "paging/32bit/X86PagingStructures32Bit.h"
26#include "paging/32bit/X86VMTranslationMap32Bit.h"
27#include "paging/x86_physical_page_mapper.h"
28#include "paging/x86_physical_page_mapper_large_memory.h"
29
30
31//#define TRACE_X86_PAGING_METHOD_32_BIT
32#ifdef TRACE_X86_PAGING_METHOD_32_BIT
33#	define TRACE(x...) dprintf(x)
34#else
35#	define TRACE(x...) ;
36#endif
37
38
39using X86LargePhysicalPageMapper::PhysicalPageSlot;
40
41
42// #pragma mark - X86PagingMethod32Bit::PhysicalPageSlotPool
43
44
45struct X86PagingMethod32Bit::PhysicalPageSlotPool
46	: X86LargePhysicalPageMapper::PhysicalPageSlotPool {
47public:
48	virtual						~PhysicalPageSlotPool();
49
50			status_t			InitInitial(kernel_args* args);
51			status_t			InitInitialPostArea(kernel_args* args);
52
53			void				Init(area_id dataArea, void* data,
54									area_id virtualArea, addr_t virtualBase);
55
56	virtual	status_t			AllocatePool(
57									X86LargePhysicalPageMapper
58										::PhysicalPageSlotPool*& _pool);
59	virtual	void				Map(phys_addr_t physicalAddress,
60									addr_t virtualAddress);
61
62public:
63	static	PhysicalPageSlotPool sInitialPhysicalPagePool;
64
65private:
66	area_id					fDataArea;
67	area_id					fVirtualArea;
68	addr_t					fVirtualBase;
69	page_table_entry*		fPageTable;
70};
71
72
73X86PagingMethod32Bit::PhysicalPageSlotPool
74	X86PagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool;
75
76
77X86PagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool()
78{
79}
80
81
82status_t
83X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitial(kernel_args* args)
84{
85	// allocate a virtual address range for the pages to be mapped into
86	addr_t virtualBase = vm_allocate_early(args, 1024 * B_PAGE_SIZE, 0, 0,
87		kPageTableAlignment);
88	if (virtualBase == 0) {
89		panic("LargeMemoryPhysicalPageMapper::Init(): Failed to reserve "
90			"physical page pool space in virtual address space!");
91		return B_ERROR;
92	}
93
94	// allocate memory for the page table and data
95	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
96	page_table_entry* pageTable = (page_table_entry*)vm_allocate_early(args,
97		areaSize, ~0L, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, 0);
98
99	// prepare the page table
100	_EarlyPreparePageTables(pageTable, virtualBase, 1024 * B_PAGE_SIZE);
101
102	// init the pool structure and add the initial pool
103	Init(-1, pageTable, -1, (addr_t)virtualBase);
104
105	return B_OK;
106}
107
108
109status_t
110X86PagingMethod32Bit::PhysicalPageSlotPool::InitInitialPostArea(
111	kernel_args* args)
112{
113	// create an area for the (already allocated) data
114	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
115	void* temp = fPageTable;
116	area_id area = create_area("physical page pool", &temp,
117		B_EXACT_ADDRESS, areaSize, B_ALREADY_WIRED,
118		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
119	if (area < B_OK) {
120		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
121			"create area for physical page pool.");
122		return area;
123	}
124	fDataArea = area;
125
126	// create an area for the virtual address space
127	temp = (void*)fVirtualBase;
128	area = vm_create_null_area(VMAddressSpace::KernelID(),
129		"physical page pool space", &temp, B_EXACT_ADDRESS,
130		1024 * B_PAGE_SIZE, 0);
131	if (area < B_OK) {
132		panic("LargeMemoryPhysicalPageMapper::InitPostArea(): Failed to "
133			"create area for physical page pool space.");
134		return area;
135	}
136	fVirtualArea = area;
137
138	return B_OK;
139}
140
141
142void
143X86PagingMethod32Bit::PhysicalPageSlotPool::Init(area_id dataArea, void* data,
144	area_id virtualArea, addr_t virtualBase)
145{
146	fDataArea = dataArea;
147	fVirtualArea = virtualArea;
148	fVirtualBase = virtualBase;
149	fPageTable = (page_table_entry*)data;
150
151	// init slot list
152	fSlots = (PhysicalPageSlot*)(fPageTable + 1024);
153	addr_t slotAddress = virtualBase;
154	for (int32 i = 0; i < 1024; i++, slotAddress += B_PAGE_SIZE) {
155		PhysicalPageSlot* slot = &fSlots[i];
156		slot->next = slot + 1;
157		slot->pool = this;
158		slot->address = slotAddress;
159	}
160
161	fSlots[1023].next = NULL;
162		// terminate list
163}
164
165
166void
167X86PagingMethod32Bit::PhysicalPageSlotPool::Map(phys_addr_t physicalAddress,
168	addr_t virtualAddress)
169{
170	page_table_entry& pte = fPageTable[
171		(virtualAddress - fVirtualBase) / B_PAGE_SIZE];
172	pte = (physicalAddress & X86_PTE_ADDRESS_MASK)
173		| X86_PTE_WRITABLE | X86_PTE_GLOBAL | X86_PTE_PRESENT;
174
175	invalidate_TLB(virtualAddress);
176}
177
178
179status_t
180X86PagingMethod32Bit::PhysicalPageSlotPool::AllocatePool(
181	X86LargePhysicalPageMapper::PhysicalPageSlotPool*& _pool)
182{
183	// create the pool structure
184	PhysicalPageSlotPool* pool = new(std::nothrow) PhysicalPageSlotPool;
185	if (pool == NULL)
186		return B_NO_MEMORY;
187	ObjectDeleter<PhysicalPageSlotPool> poolDeleter(pool);
188
189	// create an area that can contain the page table and the slot
190	// structures
191	size_t areaSize = B_PAGE_SIZE + sizeof(PhysicalPageSlot[1024]);
192	void* data;
193	virtual_address_restrictions virtualRestrictions = {};
194	virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
195	physical_address_restrictions physicalRestrictions = {};
196	area_id dataArea = create_area_etc(B_SYSTEM_TEAM, "physical page pool",
197		PAGE_ALIGN(areaSize), B_FULL_LOCK,
198		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
199		&virtualRestrictions, &physicalRestrictions, &data);
200	if (dataArea < 0)
201		return dataArea;
202
203	// create the null area for the virtual address space
204	void* virtualBase;
205	area_id virtualArea = vm_create_null_area(
206		VMAddressSpace::KernelID(), "physical page pool space",
207		&virtualBase, B_ANY_KERNEL_BLOCK_ADDRESS, 1024 * B_PAGE_SIZE,
208		CREATE_AREA_PRIORITY_VIP);
209	if (virtualArea < 0) {
210		delete_area(dataArea);
211		return virtualArea;
212	}
213
214	// prepare the page table
215	memset(data, 0, B_PAGE_SIZE);
216
217	// get the page table's physical address
218	phys_addr_t physicalTable;
219	X86VMTranslationMap32Bit* map = static_cast<X86VMTranslationMap32Bit*>(
220		VMAddressSpace::Kernel()->TranslationMap());
221	uint32 dummyFlags;
222	cpu_status state = disable_interrupts();
223	map->QueryInterrupt((addr_t)data, &physicalTable, &dummyFlags);
224	restore_interrupts(state);
225
226	// put the page table into the page directory
227	int32 index = (addr_t)virtualBase / (B_PAGE_SIZE * 1024);
228	page_directory_entry* entry
229		= &map->PagingStructures32Bit()->pgdir_virt[index];
230	PutPageTableInPageDir(entry, physicalTable,
231		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
232	X86PagingStructures32Bit::UpdateAllPageDirs(index, *entry);
233
234	// init the pool structure
235	pool->Init(dataArea, data, virtualArea, (addr_t)virtualBase);
236	poolDeleter.Detach();
237	_pool = pool;
238	return B_OK;
239}
240
241
242// #pragma mark - X86PagingMethod32Bit
243
244
245X86PagingMethod32Bit::X86PagingMethod32Bit()
246	:
247	fPageHole(NULL),
248	fPageHolePageDir(NULL),
249	fKernelPhysicalPageDirectory(0),
250	fKernelVirtualPageDirectory(NULL),
251	fPhysicalPageMapper(NULL),
252	fKernelPhysicalPageMapper(NULL)
253{
254}
255
256
257X86PagingMethod32Bit::~X86PagingMethod32Bit()
258{
259}
260
261
262status_t
263X86PagingMethod32Bit::Init(kernel_args* args,
264	VMPhysicalPageMapper** _physicalPageMapper)
265{
266	TRACE("X86PagingMethod32Bit::Init(): entry\n");
267
268	// page hole set up in stage2
269	fPageHole = (page_table_entry*)(addr_t)args->arch_args.page_hole;
270	// calculate where the pgdir would be
271	fPageHolePageDir = (page_directory_entry*)
272		(((addr_t)args->arch_args.page_hole)
273			+ (B_PAGE_SIZE * 1024 - B_PAGE_SIZE));
274	// clear out the bottom 2 GB, unmap everything
275	memset(fPageHolePageDir + FIRST_USER_PGDIR_ENT, 0,
276		sizeof(page_directory_entry) * NUM_USER_PGDIR_ENTS);
277
278	fKernelPhysicalPageDirectory = args->arch_args.phys_pgdir;
279	fKernelVirtualPageDirectory = (page_directory_entry*)(addr_t)
280		args->arch_args.vir_pgdir;
281
282#ifdef TRACE_X86_PAGING_METHOD_32_BIT
283	TRACE("page hole: %p, page dir: %p\n", fPageHole, fPageHolePageDir);
284	TRACE("page dir: %p (physical: %#" B_PRIx32 ")\n",
285		fKernelVirtualPageDirectory, fKernelPhysicalPageDirectory);
286#endif
287
288	X86PagingStructures32Bit::StaticInit();
289
290	// create the initial pool for the physical page mapper
291	PhysicalPageSlotPool* pool
292		= new(&PhysicalPageSlotPool::sInitialPhysicalPagePool)
293			PhysicalPageSlotPool;
294	status_t error = pool->InitInitial(args);
295	if (error != B_OK) {
296		panic("X86PagingMethod32Bit::Init(): Failed to create initial pool "
297			"for physical page mapper!");
298		return error;
299	}
300
301	// create physical page mapper
302	large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper,
303		fKernelPhysicalPageMapper);
304		// TODO: Select the best page mapper!
305
306	// enable global page feature if available
307	if (x86_check_feature(IA32_FEATURE_PGE, FEATURE_COMMON)) {
308		// this prevents kernel pages from being flushed from TLB on
309		// context-switch
310		x86_write_cr4(x86_read_cr4() | IA32_CR4_GLOBAL_PAGES);
311	}
312
313	TRACE("X86PagingMethod32Bit::Init(): done\n");
314
315	*_physicalPageMapper = fPhysicalPageMapper;
316	return B_OK;
317}
318
319
320status_t
321X86PagingMethod32Bit::InitPostArea(kernel_args* args)
322{
323	// now that the vm is initialized, create an area that represents
324	// the page hole
325	void *temp;
326	status_t error;
327	area_id area;
328
329	// unmap the page hole hack we were using before
330	fKernelVirtualPageDirectory[1023] = 0;
331	fPageHolePageDir = NULL;
332	fPageHole = NULL;
333
334	temp = (void*)fKernelVirtualPageDirectory;
335	area = create_area("kernel_pgdir", &temp, B_EXACT_ADDRESS, B_PAGE_SIZE,
336		B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
337	if (area < B_OK)
338		return area;
339
340	error = PhysicalPageSlotPool::sInitialPhysicalPagePool
341		.InitInitialPostArea(args);
342	if (error != B_OK)
343		return error;
344
345	return B_OK;
346}
347
348
349status_t
350X86PagingMethod32Bit::CreateTranslationMap(bool kernel, VMTranslationMap** _map)
351{
352	X86VMTranslationMap32Bit* map = new(std::nothrow) X86VMTranslationMap32Bit;
353	if (map == NULL)
354		return B_NO_MEMORY;
355
356	status_t error = map->Init(kernel);
357	if (error != B_OK) {
358		delete map;
359		return error;
360	}
361
362	*_map = map;
363	return B_OK;
364}
365
366
367status_t
368X86PagingMethod32Bit::MapEarly(kernel_args* args, addr_t virtualAddress,
369	phys_addr_t physicalAddress, uint8 attributes,
370	page_num_t (*get_free_page)(kernel_args*))
371{
372	// XXX horrible back door to map a page quickly regardless of translation
373	// map object, etc. used only during VM setup.
374	// uses a 'page hole' set up in the stage 2 bootloader. The page hole is
375	// created by pointing one of the pgdir entries back at itself, effectively
376	// mapping the contents of all of the 4MB of pagetables into a 4 MB region.
377	// It's only used here, and is later unmapped.
378
379	// check to see if a page table exists for this range
380	int index = VADDR_TO_PDENT(virtualAddress);
381	if ((fPageHolePageDir[index] & X86_PDE_PRESENT) == 0) {
382		phys_addr_t pgtable;
383		page_directory_entry *e;
384		// we need to allocate a pgtable
385		pgtable = get_free_page(args);
386		// pgtable is in pages, convert to physical address
387		pgtable *= B_PAGE_SIZE;
388
389		TRACE("X86PagingMethod32Bit::MapEarly(): asked for free page for "
390			"pgtable. %#" B_PRIxPHYSADDR "\n", pgtable);
391
392		// put it in the pgdir
393		e = &fPageHolePageDir[index];
394		PutPageTableInPageDir(e, pgtable, attributes);
395
396		// zero it out in it's new mapping
397		memset((unsigned int*)((addr_t)fPageHole
398				+ (virtualAddress / B_PAGE_SIZE / 1024) * B_PAGE_SIZE),
399			0, B_PAGE_SIZE);
400	}
401
402	ASSERT_PRINT(
403		(fPageHole[virtualAddress / B_PAGE_SIZE] & X86_PTE_PRESENT) == 0,
404		"virtual address: %#" B_PRIxADDR ", pde: %#" B_PRIx32
405		", existing pte: %#" B_PRIx32, virtualAddress, fPageHolePageDir[index],
406		fPageHole[virtualAddress / B_PAGE_SIZE]);
407
408	// now, fill in the pentry
409	PutPageTableEntryInTable(fPageHole + virtualAddress / B_PAGE_SIZE,
410		physicalAddress, attributes, 0, IS_KERNEL_ADDRESS(virtualAddress));
411
412	return B_OK;
413}
414
415
416bool
417X86PagingMethod32Bit::IsKernelPageAccessible(addr_t virtualAddress,
418	uint32 protection)
419{
420	// We only trust the kernel team's page directory. So switch to it first.
421	// Always set it to make sure the TLBs don't contain obsolete data.
422	uint32 physicalPageDirectory = x86_read_cr3();
423	x86_write_cr3(fKernelPhysicalPageDirectory);
424
425	// get the page directory entry for the address
426	page_directory_entry pageDirectoryEntry;
427	uint32 index = VADDR_TO_PDENT(virtualAddress);
428
429	if (physicalPageDirectory == fKernelPhysicalPageDirectory) {
430		pageDirectoryEntry = fKernelVirtualPageDirectory[index];
431	} else if (fPhysicalPageMapper != NULL) {
432		// map the original page directory and get the entry
433		void* handle;
434		addr_t virtualPageDirectory;
435		status_t error = fPhysicalPageMapper->GetPageDebug(
436			physicalPageDirectory, &virtualPageDirectory, &handle);
437		if (error == B_OK) {
438			pageDirectoryEntry
439				= ((page_directory_entry*)virtualPageDirectory)[index];
440			fPhysicalPageMapper->PutPageDebug(virtualPageDirectory, handle);
441		} else
442			pageDirectoryEntry = 0;
443	} else
444		pageDirectoryEntry = 0;
445
446	// map the page table and get the entry
447	page_table_entry pageTableEntry;
448	index = VADDR_TO_PTENT(virtualAddress);
449
450	if ((pageDirectoryEntry & X86_PDE_PRESENT) != 0
451			&& fPhysicalPageMapper != NULL) {
452		void* handle;
453		addr_t virtualPageTable;
454		status_t error = fPhysicalPageMapper->GetPageDebug(
455			pageDirectoryEntry & X86_PDE_ADDRESS_MASK, &virtualPageTable,
456			&handle);
457		if (error == B_OK) {
458			pageTableEntry = ((page_table_entry*)virtualPageTable)[index];
459			fPhysicalPageMapper->PutPageDebug(virtualPageTable, handle);
460		} else
461			pageTableEntry = 0;
462	} else
463		pageTableEntry = 0;
464
465	// switch back to the original page directory
466	if (physicalPageDirectory != fKernelPhysicalPageDirectory)
467		x86_write_cr3(physicalPageDirectory);
468
469	if ((pageTableEntry & X86_PTE_PRESENT) == 0)
470		return false;
471
472	// present means kernel-readable, so check for writable
473	return (protection & B_KERNEL_WRITE_AREA) == 0
474		|| (pageTableEntry & X86_PTE_WRITABLE) != 0;
475}
476
477
478/*static*/ void
479X86PagingMethod32Bit::PutPageTableInPageDir(page_directory_entry* entry,
480	phys_addr_t pgtablePhysical, uint32 attributes)
481{
482	*entry = (pgtablePhysical & X86_PDE_ADDRESS_MASK)
483		| X86_PDE_PRESENT
484		| X86_PDE_WRITABLE
485		| X86_PDE_USER;
486		// TODO: we ignore the attributes of the page table - for compatibility
487		// with BeOS we allow having user accessible areas in the kernel address
488		// space. This is currently being used by some drivers, mainly for the
489		// frame buffer. Our current real time data implementation makes use of
490		// this fact, too.
491		// We might want to get rid of this possibility one day, especially if
492		// we intend to port it to a platform that does not support this.
493}
494
495
496/*static*/ void
497X86PagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry,
498	phys_addr_t physicalAddress, uint32 attributes, uint32 memoryType,
499	bool globalPage)
500{
501	page_table_entry page = (physicalAddress & X86_PTE_ADDRESS_MASK)
502		| X86_PTE_PRESENT | (globalPage ? X86_PTE_GLOBAL : 0)
503		| MemoryTypeToPageTableEntryFlags(memoryType);
504
505	// if the page is user accessible, it's automatically
506	// accessible in kernel space, too (but with the same
507	// protection)
508	if ((attributes & B_USER_PROTECTION) != 0) {
509		page |= X86_PTE_USER;
510		if ((attributes & B_WRITE_AREA) != 0)
511			page |= X86_PTE_WRITABLE;
512	} else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
513		page |= X86_PTE_WRITABLE;
514
515	// put it in the page table
516	*(volatile page_table_entry*)entry = page;
517}
518
519
520/*static*/ void
521X86PagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables,
522	addr_t address, size_t size)
523{
524	memset(pageTables, 0, B_PAGE_SIZE * (size / (B_PAGE_SIZE * 1024)));
525
526	// put the array of pgtables directly into the kernel pagedir
527	// these will be wired and kept mapped into virtual space to be easy to get
528	// to
529	{
530		addr_t virtualTable = (addr_t)pageTables;
531
532		page_directory_entry* pageHolePageDir
533			= X86PagingMethod32Bit::Method()->PageHolePageDir();
534
535		for (size_t i = 0; i < (size / (B_PAGE_SIZE * 1024));
536				i++, virtualTable += B_PAGE_SIZE) {
537			phys_addr_t physicalTable = 0;
538			_EarlyQuery(virtualTable, &physicalTable);
539			page_directory_entry* entry = &pageHolePageDir[
540				(address / (B_PAGE_SIZE * 1024)) + i];
541			PutPageTableInPageDir(entry, physicalTable,
542				B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA);
543		}
544	}
545}
546
547
548//! TODO: currently assumes this translation map is active
549/*static*/ status_t
550X86PagingMethod32Bit::_EarlyQuery(addr_t virtualAddress,
551	phys_addr_t *_physicalAddress)
552{
553	X86PagingMethod32Bit* method = X86PagingMethod32Bit::Method();
554	int index = VADDR_TO_PDENT(virtualAddress);
555	if ((method->PageHolePageDir()[index] & X86_PDE_PRESENT) == 0) {
556		// no pagetable here
557		return B_ERROR;
558	}
559
560	page_table_entry* entry = method->PageHole() + virtualAddress / B_PAGE_SIZE;
561	if ((*entry & X86_PTE_PRESENT) == 0) {
562		// page mapping not valid
563		return B_ERROR;
564	}
565
566	*_physicalAddress = *entry & X86_PTE_ADDRESS_MASK;
567	return B_OK;
568}
569