1/*
2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk
3 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
4 * Copyright 2002-2010, Axel D��rfler, axeld@pinc-software.de.
5 * Distributed under the terms of the MIT License.
6 *
7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8 * Distributed under the terms of the NewOS License.
9 */
10
11
12#include "paging/64bit/X86VMTranslationMap64Bit.h"
13
14#include <int.h>
15#include <slab/Slab.h>
16#include <thread.h>
17#include <util/AutoLock.h>
18#include <util/ThreadAutoLock.h>
19#include <vm/vm_page.h>
20#include <vm/VMAddressSpace.h>
21#include <vm/VMCache.h>
22
23#include "paging/64bit/X86PagingMethod64Bit.h"
24#include "paging/64bit/X86PagingStructures64Bit.h"
25#include "paging/x86_physical_page_mapper.h"
26
27
28//#define TRACE_X86_VM_TRANSLATION_MAP_64BIT
29#ifdef TRACE_X86_VM_TRANSLATION_MAP_64BIT
30#	define TRACE(x...) dprintf(x)
31#else
32#	define TRACE(x...) ;
33#endif
34
35
36// #pragma mark - X86VMTranslationMap64Bit
37
38
39X86VMTranslationMap64Bit::X86VMTranslationMap64Bit(bool la57)
40	:
41	fPagingStructures(NULL),
42	fLA57(la57)
43{
44}
45
46
47X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()
48{
49	TRACE("X86VMTranslationMap64Bit::~X86VMTranslationMap64Bit()\n");
50
51	if (fPagingStructures == NULL)
52		return;
53
54	if (fPageMapper != NULL) {
55		phys_addr_t address;
56		vm_page* page;
57
58		// Free all structures in the bottom half of the PMLTop (user memory).
59		uint64* virtualPML4 = fPagingStructures->VirtualPMLTop();
60		for (uint32 i = 0; i < 256; i++) {
61			if ((virtualPML4[i] & X86_64_PML4E_PRESENT) == 0)
62				continue;
63
64			uint64* virtualPDPT = (uint64*)fPageMapper->GetPageTableAt(
65				virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK);
66			for (uint32 j = 0; j < 512; j++) {
67				if ((virtualPDPT[j] & X86_64_PDPTE_PRESENT) == 0)
68					continue;
69
70				uint64* virtualPageDir = (uint64*)fPageMapper->GetPageTableAt(
71					virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK);
72				for (uint32 k = 0; k < 512; k++) {
73					if ((virtualPageDir[k] & X86_64_PDE_PRESENT) == 0)
74						continue;
75
76					address = virtualPageDir[k] & X86_64_PDE_ADDRESS_MASK;
77					page = vm_lookup_page(address / B_PAGE_SIZE);
78					if (page == NULL) {
79						panic("page table %u %u %u on invalid page %#"
80							B_PRIxPHYSADDR "\n", i, j, k, address);
81					}
82
83					DEBUG_PAGE_ACCESS_START(page);
84					vm_page_set_state(page, PAGE_STATE_FREE);
85				}
86
87				address = virtualPDPT[j] & X86_64_PDPTE_ADDRESS_MASK;
88				page = vm_lookup_page(address / B_PAGE_SIZE);
89				if (page == NULL) {
90					panic("page directory %u %u on invalid page %#"
91						B_PRIxPHYSADDR "\n", i, j, address);
92				}
93
94				DEBUG_PAGE_ACCESS_START(page);
95				vm_page_set_state(page, PAGE_STATE_FREE);
96			}
97
98			address = virtualPML4[i] & X86_64_PML4E_ADDRESS_MASK;
99			page = vm_lookup_page(address / B_PAGE_SIZE);
100			if (page == NULL) {
101				panic("PDPT %u on invalid page %#" B_PRIxPHYSADDR "\n", i,
102					address);
103			}
104
105			DEBUG_PAGE_ACCESS_START(page);
106			vm_page_set_state(page, PAGE_STATE_FREE);
107		}
108
109		fPageMapper->Delete();
110	}
111
112	fPagingStructures->RemoveReference();
113}
114
115
116status_t
117X86VMTranslationMap64Bit::Init(bool kernel)
118{
119	TRACE("X86VMTranslationMap64Bit::Init()\n");
120
121	X86VMTranslationMap::Init(kernel);
122
123	fPagingStructures = new(std::nothrow) X86PagingStructures64Bit;
124	if (fPagingStructures == NULL)
125		return B_NO_MEMORY;
126
127	X86PagingMethod64Bit* method = X86PagingMethod64Bit::Method();
128
129	if (kernel) {
130		// Get the page mapper.
131		fPageMapper = method->KernelPhysicalPageMapper();
132
133		// Kernel PMLTop is already mapped.
134		fPagingStructures->Init(method->KernelVirtualPMLTop(),
135			method->KernelPhysicalPMLTop());
136	} else {
137		// Allocate a physical page mapper.
138		status_t error = method->PhysicalPageMapper()
139			->CreateTranslationMapPhysicalPageMapper(&fPageMapper);
140		if (error != B_OK)
141			return error;
142
143		// Assuming that only the top 2 PMLTop entries are occupied for the
144		// kernel.
145		STATIC_ASSERT(KERNEL_PMAP_BASE == 0xffffff0000000000);
146		STATIC_ASSERT(KERNEL_BASE == 0xffffff0000000000);
147
148		// Allocate and clear the PMLTop.
149		uint64* virtualPMLTop = (uint64*)memalign(B_PAGE_SIZE, B_PAGE_SIZE);
150		if (virtualPMLTop == NULL)
151			return B_NO_MEMORY;
152		memset(virtualPMLTop, 0, B_PAGE_SIZE);
153
154		// Copy the top 2 PMLTop entries.
155		virtualPMLTop[510] = method->KernelVirtualPMLTop()[510];
156		virtualPMLTop[511] = method->KernelVirtualPMLTop()[511];
157
158		// Look up the PMLTop physical address.
159		phys_addr_t physicalPMLTop;
160		vm_get_page_mapping(VMAddressSpace::KernelID(), (addr_t)virtualPMLTop,
161			&physicalPMLTop);
162
163		// Initialize the paging structures.
164		fPagingStructures->Init(virtualPMLTop, physicalPMLTop);
165	}
166
167	return B_OK;
168}
169
170
171size_t
172X86VMTranslationMap64Bit::MaxPagesNeededToMap(addr_t start, addr_t end) const
173{
174	// If start == 0, the actual base address is not yet known to the caller and
175	// we shall assume the worst case, which is where the start address is the
176	// last page covered by a PDPT or PML4.
177	if (start == 0) {
178		start = (fLA57 ? k64BitPML4TRange : k64BitPDPTRange) - B_PAGE_SIZE;
179		end += start;
180	}
181
182	size_t requiredPML4s = 0;
183	if (fLA57) {
184		requiredPML4s = end / k64BitPML4TRange + 1
185			- start / k64BitPML4TRange;
186	}
187	size_t requiredPDPTs = end / k64BitPDPTRange + 1
188		- start / k64BitPDPTRange;
189	size_t requiredPageDirs = end / k64BitPageDirectoryRange + 1
190		- start / k64BitPageDirectoryRange;
191	size_t requiredPageTables = end / k64BitPageTableRange + 1
192		- start / k64BitPageTableRange;
193
194	return requiredPML4s + requiredPDPTs + requiredPageDirs
195		+ requiredPageTables;
196}
197
198
199status_t
200X86VMTranslationMap64Bit::Map(addr_t virtualAddress, phys_addr_t physicalAddress,
201	uint32 attributes, uint32 memoryType, vm_page_reservation* reservation)
202{
203	TRACE("X86VMTranslationMap64Bit::Map(%#" B_PRIxADDR ", %#" B_PRIxPHYSADDR
204		")\n", virtualAddress, physicalAddress);
205
206	ThreadCPUPinner pinner(thread_get_current_thread());
207
208	// Look up the page table for the virtual address, allocating new tables
209	// if required. Shouldn't fail.
210	uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
211		fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
212		true, reservation, fPageMapper, fMapCount);
213	ASSERT(entry != NULL);
214
215	// The entry should not already exist.
216	ASSERT_PRINT((*entry & X86_64_PTE_PRESENT) == 0,
217		"virtual address: %#" B_PRIxADDR ", existing pte: %#" B_PRIx64,
218		virtualAddress, *entry);
219
220	// Fill in the table entry.
221	X86PagingMethod64Bit::PutPageTableEntryInTable(entry, physicalAddress,
222		attributes, memoryType, fIsKernelMap);
223
224	// Note: We don't need to invalidate the TLB for this address, as previously
225	// the entry was not present and the TLB doesn't cache those entries.
226
227	fMapCount++;
228
229	return 0;
230}
231
232
233status_t
234X86VMTranslationMap64Bit::Unmap(addr_t start, addr_t end)
235{
236	start = ROUNDDOWN(start, B_PAGE_SIZE);
237	if (start >= end)
238		return B_OK;
239
240	TRACE("X86VMTranslationMap64Bit::Unmap(%#" B_PRIxADDR ", %#" B_PRIxADDR
241		")\n", start, end);
242
243	ThreadCPUPinner pinner(thread_get_current_thread());
244
245	do {
246		uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
247			fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
248			NULL, fPageMapper, fMapCount);
249		if (pageTable == NULL) {
250			// Move on to the next page table.
251			start = ROUNDUP(start + 1, k64BitPageTableRange);
252			continue;
253		}
254
255		for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
256				index < k64BitTableEntryCount && start < end;
257				index++, start += B_PAGE_SIZE) {
258			if ((pageTable[index] & X86_64_PTE_PRESENT) == 0)
259				continue;
260
261			TRACE("X86VMTranslationMap64Bit::Unmap(): removing page %#"
262				B_PRIxADDR " (%#" B_PRIxPHYSADDR ")\n", start,
263				pageTable[index] & X86_64_PTE_ADDRESS_MASK);
264
265			uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(
266				&pageTable[index], X86_64_PTE_PRESENT);
267			fMapCount--;
268
269			if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
270				// Note, that we only need to invalidate the address, if the
271				// accessed flags was set, since only then the entry could have
272				// been in any TLB.
273				InvalidatePage(start);
274			}
275		}
276	} while (start != 0 && start < end);
277
278	return B_OK;
279}
280
281
282status_t
283X86VMTranslationMap64Bit::DebugMarkRangePresent(addr_t start, addr_t end,
284	bool markPresent)
285{
286	start = ROUNDDOWN(start, B_PAGE_SIZE);
287	if (start >= end)
288		return B_OK;
289
290	TRACE("X86VMTranslationMap64Bit::DebugMarkRangePresent(%#" B_PRIxADDR
291		", %#" B_PRIxADDR ")\n", start, end);
292
293	ThreadCPUPinner pinner(thread_get_current_thread());
294
295	do {
296		uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
297			fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
298			NULL, fPageMapper, fMapCount);
299		if (pageTable == NULL) {
300			// Move on to the next page table.
301			start = ROUNDUP(start + 1, k64BitPageTableRange);
302			continue;
303		}
304
305		for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
306				index < k64BitTableEntryCount && start < end;
307				index++, start += B_PAGE_SIZE) {
308			if ((pageTable[index] & X86_64_PTE_PRESENT) == 0) {
309				if (!markPresent)
310					continue;
311
312				X86PagingMethod64Bit::SetTableEntryFlags(&pageTable[index],
313					X86_64_PTE_PRESENT);
314			} else {
315				if (markPresent)
316					continue;
317
318				uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(
319					&pageTable[index], X86_64_PTE_PRESENT);
320
321				if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
322					// Note, that we only need to invalidate the address, if the
323					// accessed flags was set, since only then the entry could
324					// have been in any TLB.
325					InvalidatePage(start);
326				}
327			}
328		}
329	} while (start != 0 && start < end);
330
331	return B_OK;
332}
333
334
335status_t
336X86VMTranslationMap64Bit::UnmapPage(VMArea* area, addr_t address,
337	bool updatePageQueue)
338{
339	ASSERT(address % B_PAGE_SIZE == 0);
340
341	TRACE("X86VMTranslationMap64Bit::UnmapPage(%#" B_PRIxADDR ")\n", address);
342
343	ThreadCPUPinner pinner(thread_get_current_thread());
344
345	// Look up the page table for the virtual address.
346	uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
347		fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
348		false, NULL, fPageMapper, fMapCount);
349	if (entry == NULL)
350		return B_ENTRY_NOT_FOUND;
351
352	RecursiveLocker locker(fLock);
353
354	uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
355
356	pinner.Unlock();
357
358	if ((oldEntry & X86_64_PTE_PRESENT) == 0)
359		return B_ENTRY_NOT_FOUND;
360
361	fMapCount--;
362
363	if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
364		// Note, that we only need to invalidate the address, if the
365		// accessed flags was set, since only then the entry could have been
366		// in any TLB.
367		InvalidatePage(address);
368
369		Flush();
370
371		// NOTE: Between clearing the page table entry and Flush() other
372		// processors (actually even this processor with another thread of the
373		// same team) could still access the page in question via their cached
374		// entry. We can obviously lose a modified flag in this case, with the
375		// effect that the page looks unmodified (and might thus be recycled),
376		// but is actually modified.
377		// In most cases this is harmless, but for vm_remove_all_page_mappings()
378		// this is actually a problem.
379		// Interestingly FreeBSD seems to ignore this problem as well
380		// (cf. pmap_remove_all()), unless I've missed something.
381	}
382
383	locker.Detach();
384		// PageUnmapped() will unlock for us
385
386	PageUnmapped(area, (oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE,
387		(oldEntry & X86_64_PTE_ACCESSED) != 0,
388		(oldEntry & X86_64_PTE_DIRTY) != 0, updatePageQueue);
389
390	return B_OK;
391}
392
393
394void
395X86VMTranslationMap64Bit::UnmapPages(VMArea* area, addr_t base, size_t size,
396	bool updatePageQueue)
397{
398	if (size == 0)
399		return;
400
401	addr_t start = base;
402	addr_t end = base + size - 1;
403
404	TRACE("X86VMTranslationMap64Bit::UnmapPages(%p, %#" B_PRIxADDR ", %#"
405		B_PRIxADDR ")\n", area, start, end);
406
407	VMAreaMappings queue;
408
409	RecursiveLocker locker(fLock);
410	ThreadCPUPinner pinner(thread_get_current_thread());
411
412	do {
413		uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
414			fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
415			NULL, fPageMapper, fMapCount);
416		if (pageTable == NULL) {
417			// Move on to the next page table.
418			start = ROUNDUP(start + 1, k64BitPageTableRange);
419			continue;
420		}
421
422		for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
423				index < k64BitTableEntryCount && start < end;
424				index++, start += B_PAGE_SIZE) {
425			uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(
426				&pageTable[index]);
427			if ((oldEntry & X86_64_PTE_PRESENT) == 0)
428				continue;
429
430			fMapCount--;
431
432			if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
433				// Note, that we only need to invalidate the address, if the
434				// accessed flags was set, since only then the entry could have
435				// been in any TLB.
436				InvalidatePage(start);
437			}
438
439			if (area->cache_type != CACHE_TYPE_DEVICE) {
440				// get the page
441				vm_page* page = vm_lookup_page(
442					(oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
443				ASSERT(page != NULL);
444
445				DEBUG_PAGE_ACCESS_START(page);
446
447				// transfer the accessed/dirty flags to the page
448				if ((oldEntry & X86_64_PTE_ACCESSED) != 0)
449					page->accessed = true;
450				if ((oldEntry & X86_64_PTE_DIRTY) != 0)
451					page->modified = true;
452
453				// remove the mapping object/decrement the wired_count of the
454				// page
455				if (area->wiring == B_NO_LOCK) {
456					vm_page_mapping* mapping = NULL;
457					vm_page_mappings::Iterator iterator
458						= page->mappings.GetIterator();
459					while ((mapping = iterator.Next()) != NULL) {
460						if (mapping->area == area)
461							break;
462					}
463
464					ASSERT(mapping != NULL);
465
466					area->mappings.Remove(mapping);
467					page->mappings.Remove(mapping);
468					queue.Add(mapping);
469				} else
470					page->DecrementWiredCount();
471
472				if (!page->IsMapped()) {
473					atomic_add(&gMappedPagesCount, -1);
474
475					if (updatePageQueue) {
476						if (page->Cache()->temporary)
477							vm_page_set_state(page, PAGE_STATE_INACTIVE);
478						else if (page->modified)
479							vm_page_set_state(page, PAGE_STATE_MODIFIED);
480						else
481							vm_page_set_state(page, PAGE_STATE_CACHED);
482					}
483				}
484
485				DEBUG_PAGE_ACCESS_END(page);
486			}
487		}
488
489		Flush();
490			// flush explicitly, since we directly use the lock
491	} while (start != 0 && start < end);
492
493	// TODO: As in UnmapPage() we can lose page dirty flags here. ATM it's not
494	// really critical here, as in all cases this method is used, the unmapped
495	// area range is unmapped for good (resized/cut) and the pages will likely
496	// be freed.
497
498	locker.Unlock();
499
500	// free removed mappings
501	bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
502	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
503		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
504	while (vm_page_mapping* mapping = queue.RemoveHead())
505		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
506}
507
508
509void
510X86VMTranslationMap64Bit::UnmapArea(VMArea* area, bool deletingAddressSpace,
511	bool ignoreTopCachePageFlags)
512{
513	TRACE("X86VMTranslationMap64Bit::UnmapArea(%p)\n", area);
514
515	if (area->cache_type == CACHE_TYPE_DEVICE || area->wiring != B_NO_LOCK) {
516		X86VMTranslationMap64Bit::UnmapPages(area, area->Base(), area->Size(),
517			true);
518		return;
519	}
520
521	bool unmapPages = !deletingAddressSpace || !ignoreTopCachePageFlags;
522
523	RecursiveLocker locker(fLock);
524	ThreadCPUPinner pinner(thread_get_current_thread());
525
526	VMAreaMappings mappings;
527	mappings.MoveFrom(&area->mappings);
528
529	for (VMAreaMappings::Iterator it = mappings.GetIterator();
530			vm_page_mapping* mapping = it.Next();) {
531		vm_page* page = mapping->page;
532		page->mappings.Remove(mapping);
533
534		VMCache* cache = page->Cache();
535
536		bool pageFullyUnmapped = false;
537		if (!page->IsMapped()) {
538			atomic_add(&gMappedPagesCount, -1);
539			pageFullyUnmapped = true;
540		}
541
542		if (unmapPages || cache != area->cache) {
543			addr_t address = area->Base()
544				+ ((page->cache_offset * B_PAGE_SIZE) - area->cache_offset);
545
546			uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
547				fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
548				false, NULL, fPageMapper, fMapCount);
549			if (entry == NULL) {
550				panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
551					"has no page table", page, area, address);
552				continue;
553			}
554
555			uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntry(entry);
556
557			if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
558				panic("page %p has mapping for area %p (%#" B_PRIxADDR "), but "
559					"has no page table entry", page, area, address);
560				continue;
561			}
562
563			// transfer the accessed/dirty flags to the page and invalidate
564			// the mapping, if necessary
565			if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
566				page->accessed = true;
567
568				if (!deletingAddressSpace)
569					InvalidatePage(address);
570			}
571
572			if ((oldEntry & X86_64_PTE_DIRTY) != 0)
573				page->modified = true;
574
575			if (pageFullyUnmapped) {
576				DEBUG_PAGE_ACCESS_START(page);
577
578				if (cache->temporary)
579					vm_page_set_state(page, PAGE_STATE_INACTIVE);
580				else if (page->modified)
581					vm_page_set_state(page, PAGE_STATE_MODIFIED);
582				else
583					vm_page_set_state(page, PAGE_STATE_CACHED);
584
585				DEBUG_PAGE_ACCESS_END(page);
586			}
587		}
588
589		fMapCount--;
590	}
591
592	Flush();
593		// flush explicitely, since we directly use the lock
594
595	locker.Unlock();
596
597	bool isKernelSpace = area->address_space == VMAddressSpace::Kernel();
598	uint32 freeFlags = CACHE_DONT_WAIT_FOR_MEMORY
599		| (isKernelSpace ? CACHE_DONT_LOCK_KERNEL_SPACE : 0);
600	while (vm_page_mapping* mapping = mappings.RemoveHead())
601		object_cache_free(gPageMappingsObjectCache, mapping, freeFlags);
602}
603
604
605status_t
606X86VMTranslationMap64Bit::Query(addr_t virtualAddress,
607	phys_addr_t* _physicalAddress, uint32* _flags)
608{
609	*_flags = 0;
610	*_physicalAddress = 0;
611
612	ThreadCPUPinner pinner(thread_get_current_thread());
613
614	// This function may be called on the physical map area, so we must handle
615	// large pages here. Look up the page directory entry for the virtual
616	// address.
617	uint64* pde = X86PagingMethod64Bit::PageDirectoryEntryForAddress(
618		fPagingStructures->VirtualPMLTop(), virtualAddress, fIsKernelMap,
619		false, NULL, fPageMapper, fMapCount);
620	if (pde == NULL || (*pde & X86_64_PDE_PRESENT) == 0)
621		return B_OK;
622
623	uint64 entry;
624	if ((*pde & X86_64_PDE_LARGE_PAGE) != 0) {
625		entry = *pde;
626		*_physicalAddress = (entry & X86_64_PDE_ADDRESS_MASK)
627			+ (virtualAddress % 0x200000);
628	} else {
629		uint64* virtualPageTable = (uint64*)fPageMapper->GetPageTableAt(
630			*pde & X86_64_PDE_ADDRESS_MASK);
631		entry = virtualPageTable[VADDR_TO_PTE(virtualAddress)];
632		*_physicalAddress = entry & X86_64_PTE_ADDRESS_MASK;
633	}
634
635	// Translate the page state flags.
636	if ((entry & X86_64_PTE_USER) != 0) {
637		*_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_WRITE_AREA : 0)
638			| B_READ_AREA
639			| ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_EXECUTE_AREA : 0);
640	}
641
642	*_flags |= ((entry & X86_64_PTE_WRITABLE) != 0 ? B_KERNEL_WRITE_AREA : 0)
643		| B_KERNEL_READ_AREA
644		| ((entry & X86_64_PTE_NOT_EXECUTABLE) == 0 ? B_KERNEL_EXECUTE_AREA : 0)
645		| ((entry & X86_64_PTE_DIRTY) != 0 ? PAGE_MODIFIED : 0)
646		| ((entry & X86_64_PTE_ACCESSED) != 0 ? PAGE_ACCESSED : 0)
647		| ((entry & X86_64_PTE_PRESENT) != 0 ? PAGE_PRESENT : 0);
648
649	TRACE("X86VMTranslationMap64Bit::Query(%#" B_PRIxADDR ") -> %#"
650		B_PRIxPHYSADDR " %#" B_PRIx32 " (entry: %#" B_PRIx64 ")\n",
651		virtualAddress, *_physicalAddress, *_flags, entry);
652
653	return B_OK;
654}
655
656
657status_t
658X86VMTranslationMap64Bit::QueryInterrupt(addr_t virtualAddress,
659	phys_addr_t* _physicalAddress, uint32* _flags)
660{
661	// With our page mapper, there is no difference in getting a page table
662	// when interrupts are enabled or disabled, so just call Query().
663	return Query(virtualAddress, _physicalAddress, _flags);
664}
665
666
667status_t
668X86VMTranslationMap64Bit::Protect(addr_t start, addr_t end, uint32 attributes,
669	uint32 memoryType)
670{
671	start = ROUNDDOWN(start, B_PAGE_SIZE);
672	if (start >= end)
673		return B_OK;
674
675	TRACE("X86VMTranslationMap64Bit::Protect(%#" B_PRIxADDR ", %#" B_PRIxADDR
676		", %#" B_PRIx32 ")\n", start, end, attributes);
677
678	// compute protection flags
679	uint64 newProtectionFlags = 0;
680	if ((attributes & B_USER_PROTECTION) != 0) {
681		newProtectionFlags = X86_64_PTE_USER;
682		if ((attributes & B_WRITE_AREA) != 0)
683			newProtectionFlags |= X86_64_PTE_WRITABLE;
684		if ((attributes & B_EXECUTE_AREA) == 0
685			&& x86_check_feature(IA32_FEATURE_AMD_EXT_NX, FEATURE_EXT_AMD)) {
686			newProtectionFlags |= X86_64_PTE_NOT_EXECUTABLE;
687		}
688	} else if ((attributes & B_KERNEL_WRITE_AREA) != 0)
689		newProtectionFlags = X86_64_PTE_WRITABLE;
690
691	ThreadCPUPinner pinner(thread_get_current_thread());
692
693	do {
694		uint64* pageTable = X86PagingMethod64Bit::PageTableForAddress(
695			fPagingStructures->VirtualPMLTop(), start, fIsKernelMap, false,
696			NULL, fPageMapper, fMapCount);
697		if (pageTable == NULL) {
698			// Move on to the next page table.
699			start = ROUNDUP(start + 1, k64BitPageTableRange);
700			continue;
701		}
702
703		for (uint32 index = start / B_PAGE_SIZE % k64BitTableEntryCount;
704				index < k64BitTableEntryCount && start < end;
705				index++, start += B_PAGE_SIZE) {
706			uint64 entry = pageTable[index];
707			if ((entry & X86_64_PTE_PRESENT) == 0)
708				continue;
709
710			TRACE("X86VMTranslationMap64Bit::Protect(): protect page %#"
711				B_PRIxADDR "\n", start);
712
713			// set the new protection flags -- we want to do that atomically,
714			// without changing the accessed or dirty flag
715			uint64 oldEntry;
716			while (true) {
717				oldEntry = X86PagingMethod64Bit::TestAndSetTableEntry(
718					&pageTable[index],
719					(entry & ~(X86_64_PTE_PROTECTION_MASK
720							| X86_64_PTE_MEMORY_TYPE_MASK))
721						| newProtectionFlags
722						| X86PagingMethod64Bit::MemoryTypeToPageTableEntryFlags(
723							memoryType),
724					entry);
725				if (oldEntry == entry)
726					break;
727				entry = oldEntry;
728			}
729
730			if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
731				// Note, that we only need to invalidate the address, if the
732				// accessed flag was set, since only then the entry could have
733				// been in any TLB.
734				InvalidatePage(start);
735			}
736		}
737	} while (start != 0 && start < end);
738
739	return B_OK;
740}
741
742
743status_t
744X86VMTranslationMap64Bit::ClearFlags(addr_t address, uint32 flags)
745{
746	TRACE("X86VMTranslationMap64Bit::ClearFlags(%#" B_PRIxADDR ", %#" B_PRIx32
747		")\n", address, flags);
748
749	ThreadCPUPinner pinner(thread_get_current_thread());
750
751	uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
752		fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
753		false, NULL, fPageMapper, fMapCount);
754	if (entry == NULL)
755		return B_OK;
756
757	uint64 flagsToClear = ((flags & PAGE_MODIFIED) ? X86_64_PTE_DIRTY : 0)
758		| ((flags & PAGE_ACCESSED) ? X86_64_PTE_ACCESSED : 0);
759
760	uint64 oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
761		flagsToClear);
762
763	if ((oldEntry & flagsToClear) != 0)
764		InvalidatePage(address);
765
766	return B_OK;
767}
768
769
770bool
771X86VMTranslationMap64Bit::ClearAccessedAndModified(VMArea* area, addr_t address,
772	bool unmapIfUnaccessed, bool& _modified)
773{
774	ASSERT(address % B_PAGE_SIZE == 0);
775
776	TRACE("X86VMTranslationMap64Bit::ClearAccessedAndModified(%#" B_PRIxADDR
777		")\n", address);
778
779	RecursiveLocker locker(fLock);
780	ThreadCPUPinner pinner(thread_get_current_thread());
781
782	uint64* entry = X86PagingMethod64Bit::PageTableEntryForAddress(
783		fPagingStructures->VirtualPMLTop(), address, fIsKernelMap,
784		false, NULL, fPageMapper, fMapCount);
785	if (entry == NULL)
786		return false;
787
788	uint64 oldEntry;
789
790	if (unmapIfUnaccessed) {
791		while (true) {
792			oldEntry = *entry;
793			if ((oldEntry & X86_64_PTE_PRESENT) == 0) {
794				// page mapping not valid
795				return false;
796			}
797
798			if (oldEntry & X86_64_PTE_ACCESSED) {
799				// page was accessed -- just clear the flags
800				oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
801					X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
802				break;
803			}
804
805			// page hasn't been accessed -- unmap it
806			if (X86PagingMethod64Bit::TestAndSetTableEntry(entry, 0, oldEntry)
807					== oldEntry) {
808				break;
809			}
810
811			// something changed -- check again
812		}
813	} else {
814		oldEntry = X86PagingMethod64Bit::ClearTableEntryFlags(entry,
815			X86_64_PTE_ACCESSED | X86_64_PTE_DIRTY);
816	}
817
818	pinner.Unlock();
819
820	_modified = (oldEntry & X86_64_PTE_DIRTY) != 0;
821
822	if ((oldEntry & X86_64_PTE_ACCESSED) != 0) {
823		// Note, that we only need to invalidate the address, if the
824		// accessed flags was set, since only then the entry could have been
825		// in any TLB.
826		InvalidatePage(address);
827		Flush();
828
829		return true;
830	}
831
832	if (!unmapIfUnaccessed)
833		return false;
834
835	// We have unmapped the address. Do the "high level" stuff.
836
837	fMapCount--;
838
839	locker.Detach();
840		// UnaccessedPageUnmapped() will unlock for us
841
842	UnaccessedPageUnmapped(area,
843		(oldEntry & X86_64_PTE_ADDRESS_MASK) / B_PAGE_SIZE);
844
845	return false;
846}
847
848
849X86PagingStructures*
850X86VMTranslationMap64Bit::PagingStructures() const
851{
852	return fPagingStructures;
853}
854