1/*
2 * Copyright 2022 Haiku, Inc. All Rights Reserved.
3 * Distributed under the terms of the MIT License.
4 */
5#include "VMSAv8TranslationMap.h"
6
7#include <util/AutoLock.h>
8#include <util/ThreadAutoLock.h>
9#include <vm/vm_page.h>
10#include <vm/vm_priv.h>
11
12
13static constexpr uint64_t kPteAddrMask = (((1UL << 36) - 1) << 12);
14static constexpr uint64_t kPteAttrMask = ~(kPteAddrMask | 0x3);
15
16static constexpr uint64_t kAttrSWDBM = (1UL << 55);
17static constexpr uint64_t kAttrUXN = (1UL << 54);
18static constexpr uint64_t kAttrPXN = (1UL << 53);
19static constexpr uint64_t kAttrDBM = (1UL << 51);
20static constexpr uint64_t kAttrNG = (1UL << 11);
21static constexpr uint64_t kAttrAF = (1UL << 10);
22static constexpr uint64_t kAttrSH1 = (1UL << 9);
23static constexpr uint64_t kAttrSH0 = (1UL << 8);
24static constexpr uint64_t kAttrAP2 = (1UL << 7);
25static constexpr uint64_t kAttrAP1 = (1UL << 6);
26
27uint32_t VMSAv8TranslationMap::fHwFeature;
28uint64_t VMSAv8TranslationMap::fMair;
29
30
31VMSAv8TranslationMap::VMSAv8TranslationMap(
32	bool kernel, phys_addr_t pageTable, int pageBits, int vaBits, int minBlockLevel)
33	:
34	fIsKernel(kernel),
35	fPageTable(pageTable),
36	fPageBits(pageBits),
37	fVaBits(vaBits),
38	fMinBlockLevel(minBlockLevel)
39{
40	dprintf("VMSAv8TranslationMap\n");
41
42	fInitialLevel = CalcStartLevel(fVaBits, fPageBits);
43}
44
45
46VMSAv8TranslationMap::~VMSAv8TranslationMap()
47{
48	dprintf("~VMSAv8TranslationMap\n");
49
50	// FreeTable(fPageTable, fInitialLevel);
51}
52
53
54int
55VMSAv8TranslationMap::CalcStartLevel(int vaBits, int pageBits)
56{
57	int level = 4;
58
59	int bitsLeft = vaBits - pageBits;
60	while (bitsLeft > 0) {
61		int tableBits = pageBits - 3;
62		bitsLeft -= tableBits;
63		level--;
64	}
65
66	ASSERT(level >= 0);
67
68	return level;
69}
70
71
72bool
73VMSAv8TranslationMap::Lock()
74{
75	recursive_lock_lock(&fLock);
76	return true;
77}
78
79
80void
81VMSAv8TranslationMap::Unlock()
82{
83	if (recursive_lock_get_recursion(&fLock) == 1) {
84		// we're about to release it for the last time
85		Flush();
86	}
87	recursive_lock_unlock(&fLock);
88}
89
90
91addr_t
92VMSAv8TranslationMap::MappedSize() const
93{
94	panic("VMSAv8TranslationMap::MappedSize not implemented");
95	return 0;
96}
97
98
99size_t
100VMSAv8TranslationMap::MaxPagesNeededToMap(addr_t start, addr_t end) const
101{
102	size_t result = 0;
103	size_t size = end - start + 1;
104
105	for (int i = fInitialLevel; i < 3; i++) {
106		int tableBits = fPageBits - 3;
107		int shift = tableBits * (3 - i) + fPageBits;
108		uint64_t entrySize = 1UL << shift;
109
110		result += size / entrySize + 2;
111	}
112
113	return result;
114}
115
116
117uint64_t*
118VMSAv8TranslationMap::TableFromPa(phys_addr_t pa)
119{
120	return reinterpret_cast<uint64_t*>(KERNEL_PMAP_BASE + pa);
121}
122
123
124uint64_t
125VMSAv8TranslationMap::MakeBlock(phys_addr_t pa, int level, uint64_t attr)
126{
127	ASSERT(level >= fMinBlockLevel && level < 4);
128
129	return pa | attr | (level == 3 ? 0x3 : 0x1);
130}
131
132
133void
134VMSAv8TranslationMap::FreeTable(phys_addr_t ptPa, int level)
135{
136	ASSERT(level < 3);
137
138	if (level + 1 < 3) {
139		int tableBits = fPageBits - 3;
140		uint64_t tableSize = 1UL << tableBits;
141
142		uint64_t* pt = TableFromPa(ptPa);
143		for (uint64_t i = 0; i < tableSize; i++) {
144			uint64_t pte = pt[i];
145			if ((pte & 0x3) == 0x3) {
146				FreeTable(pte & kPteAddrMask, level + 1);
147			}
148		}
149	}
150
151	vm_page* page = vm_lookup_page(ptPa >> fPageBits);
152	vm_page_set_state(page, PAGE_STATE_FREE);
153}
154
155
156phys_addr_t
157VMSAv8TranslationMap::MakeTable(
158	phys_addr_t ptPa, int level, int index, vm_page_reservation* reservation)
159{
160	if (level == 3)
161		return 0;
162
163	uint64_t* pte = &TableFromPa(ptPa)[index];
164	vm_page* page = NULL;
165
166retry:
167	uint64_t oldPte = atomic_get64((int64*) pte);
168
169	int type = oldPte & 0x3;
170	if (type == 0x3) {
171		return oldPte & kPteAddrMask;
172	} else if (reservation != NULL) {
173		if (page == NULL)
174			page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
175		phys_addr_t newTablePa = page->physical_page_number << fPageBits;
176
177		if (type == 0x1) {
178			// If we're replacing existing block mapping convert it to pagetable
179			int tableBits = fPageBits - 3;
180			int shift = tableBits * (3 - (level + 1)) + fPageBits;
181			uint64_t entrySize = 1UL << shift;
182			uint64_t tableSize = 1UL << tableBits;
183
184			uint64_t* newTable = TableFromPa(newTablePa);
185			uint64_t addr = oldPte & kPteAddrMask;
186			uint64_t attr = oldPte & kPteAttrMask;
187
188			for (uint64_t i = 0; i < tableSize; i++) {
189				newTable[i] = MakeBlock(addr + i * entrySize, level + 1, attr);
190			}
191		}
192
193		asm("dsb ish");
194
195		// FIXME: this is not enough on real hardware with SMP
196		if ((uint64_t) atomic_test_and_set64((int64*) pte, newTablePa | 0x3, oldPte) != oldPte)
197			goto retry;
198
199		return newTablePa;
200	}
201
202	return 0;
203}
204
205
206void
207VMSAv8TranslationMap::MapRange(phys_addr_t ptPa, int level, addr_t va, phys_addr_t pa, size_t size,
208	VMSAv8TranslationMap::VMAction action, uint64_t attr, vm_page_reservation* reservation)
209{
210	ASSERT(level < 4);
211	ASSERT(ptPa != 0);
212	ASSERT(reservation != NULL || action != VMAction::MAP);
213
214	int tableBits = fPageBits - 3;
215	uint64_t tableMask = (1UL << tableBits) - 1;
216
217	int shift = tableBits * (3 - level) + fPageBits;
218	uint64_t entrySize = 1UL << shift;
219
220	uint64_t entryMask = entrySize - 1;
221	uint64_t nextVa = va;
222	uint64_t end = va + size;
223	int index;
224
225	// Handle misaligned header that straddles entry boundary in next-level table
226	if ((va & entryMask) != 0) {
227		uint64_t aligned = (va & ~entryMask) + entrySize;
228		if (end > aligned) {
229			index = (va >> shift) & tableMask;
230			phys_addr_t table = MakeTable(ptPa, level, index, reservation);
231			MapRange(table, level + 1, va, pa, aligned - va, action, attr, reservation);
232			nextVa = aligned;
233		}
234	}
235
236	// Handle fully aligned and appropriately sized chunks
237	while (nextVa + entrySize <= end) {
238		phys_addr_t targetPa = pa + (nextVa - va);
239		index = (nextVa >> shift) & tableMask;
240
241		bool blockAllowed = false;
242		if (action == VMAction::MAP)
243			blockAllowed = (level >= fMinBlockLevel && (targetPa & entryMask) == 0);
244		if (action == VMAction::SET_ATTR || action == VMAction::CLEAR_FLAGS)
245			blockAllowed = (MakeTable(ptPa, level, index, NULL) == 0);
246		if (action == VMAction::UNMAP)
247			blockAllowed = true;
248
249		if (blockAllowed) {
250			// Everything is aligned, we can make block mapping there
251			uint64_t* pte = &TableFromPa(ptPa)[index];
252
253		retry:
254			uint64_t oldPte = atomic_get64((int64*) pte);
255
256			if (action == VMAction::MAP || (oldPte & 0x1) != 0) {
257				uint64_t newPte = 0;
258				if (action == VMAction::MAP) {
259					newPte = MakeBlock(targetPa, level, attr);
260				} else if (action == VMAction::SET_ATTR) {
261					newPte = MakeBlock(oldPte & kPteAddrMask, level, MoveAttrFlags(attr, oldPte));
262				} else if (action == VMAction::CLEAR_FLAGS) {
263					newPte = MakeBlock(oldPte & kPteAddrMask, level, ClearAttrFlags(oldPte, attr));
264				} else if (action == VMAction::UNMAP) {
265					newPte = 0;
266					tmp_pte = oldPte;
267				}
268
269				// FIXME: this might not be enough on real hardware with SMP for some cases
270				if ((uint64_t) atomic_test_and_set64((int64*) pte, newPte, oldPte) != oldPte)
271					goto retry;
272
273				if (level < 3 && (oldPte & 0x3) == 0x3) {
274					// If we're replacing existing pagetable clean it up
275					FreeTable(oldPte & kPteAddrMask, level);
276				}
277			}
278		} else {
279			// Otherwise handle mapping in next-level table
280			phys_addr_t table = MakeTable(ptPa, level, index, reservation);
281			MapRange(table, level + 1, nextVa, targetPa, entrySize, action, attr, reservation);
282		}
283		nextVa += entrySize;
284	}
285
286	// Handle misaligned tail area (or entirety of small area) in next-level table
287	if (nextVa < end) {
288		index = (nextVa >> shift) & tableMask;
289		phys_addr_t table = MakeTable(ptPa, level, index, reservation);
290		MapRange(
291			table, level + 1, nextVa, pa + (nextVa - va), end - nextVa, action, attr, reservation);
292	}
293}
294
295
296uint8_t
297VMSAv8TranslationMap::MairIndex(uint8_t type)
298{
299	for (int i = 0; i < 8; i++)
300		if (((fMair >> (i * 8)) & 0xff) == type)
301			return i;
302
303	panic("MAIR entry not found");
304	return 0;
305}
306
307
308uint64_t
309VMSAv8TranslationMap::ClearAttrFlags(uint64_t attr, uint32 flags)
310{
311	attr &= kPteAttrMask;
312
313	if ((flags & PAGE_ACCESSED) != 0)
314		attr &= ~kAttrAF;
315
316	if ((flags & PAGE_MODIFIED) != 0 && (attr & kAttrSWDBM) != 0)
317		attr |= kAttrAP2;
318
319	return attr;
320}
321
322
323uint64_t
324VMSAv8TranslationMap::MoveAttrFlags(uint64_t newAttr, uint64_t oldAttr)
325{
326	if ((oldAttr & kAttrAF) != 0)
327		newAttr |= kAttrAF;
328	if (((newAttr & oldAttr) & kAttrSWDBM) != 0 && (oldAttr & kAttrAP2) == 0)
329		newAttr &= ~kAttrAP2;
330
331	return newAttr;
332}
333
334
335uint64_t
336VMSAv8TranslationMap::GetMemoryAttr(uint32 attributes, uint32 memoryType, bool isKernel)
337{
338	uint64_t attr = 0;
339
340	if (!isKernel)
341		attr |= kAttrNG;
342
343	if ((attributes & B_EXECUTE_AREA) == 0)
344		attr |= kAttrUXN;
345	if ((attributes & B_KERNEL_EXECUTE_AREA) == 0)
346		attr |= kAttrPXN;
347
348	if ((attributes & B_READ_AREA) == 0) {
349		attr |= kAttrAP2;
350		if ((attributes & B_KERNEL_WRITE_AREA) != 0)
351			attr |= kAttrSWDBM;
352	} else {
353		attr |= kAttrAP2 | kAttrAP1;
354		if ((attributes & B_WRITE_AREA) != 0)
355			attr |= kAttrSWDBM;
356	}
357
358	if ((fHwFeature & HW_DIRTY) != 0 && (attr & kAttrSWDBM))
359		attr |= kAttrDBM;
360
361	attr |= kAttrSH1 | kAttrSH0;
362
363	attr |= MairIndex(MAIR_NORMAL_WB) << 2;
364
365	return attr;
366}
367
368
369status_t
370VMSAv8TranslationMap::Map(addr_t va, phys_addr_t pa, uint32 attributes, uint32 memoryType,
371	vm_page_reservation* reservation)
372{
373	ThreadCPUPinner pinner(thread_get_current_thread());
374
375	uint64_t pageMask = (1UL << fPageBits) - 1;
376	uint64_t vaMask = (1UL << fVaBits) - 1;
377
378	ASSERT((va & pageMask) == 0);
379	ASSERT((pa & pageMask) == 0);
380	ASSERT(ValidateVa(va));
381
382	uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel);
383
384	if (!fPageTable) {
385		vm_page* page = vm_page_allocate_page(reservation, PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
386		fPageTable = page->physical_page_number << fPageBits;
387	}
388
389	MapRange(
390		fPageTable, fInitialLevel, va & vaMask, pa, B_PAGE_SIZE, VMAction::MAP, attr, reservation);
391
392	return B_OK;
393}
394
395
396status_t
397VMSAv8TranslationMap::Unmap(addr_t start, addr_t end)
398{
399	ThreadCPUPinner pinner(thread_get_current_thread());
400
401	size_t size = end - start + 1;
402
403	uint64_t pageMask = (1UL << fPageBits) - 1;
404	uint64_t vaMask = (1UL << fVaBits) - 1;
405
406	ASSERT((start & pageMask) == 0);
407	ASSERT((size & pageMask) == 0);
408	ASSERT(ValidateVa(start));
409
410	MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::UNMAP, 0, NULL);
411
412	return B_OK;
413}
414
415
416status_t
417VMSAv8TranslationMap::UnmapPage(VMArea* area, addr_t address, bool updatePageQueue)
418{
419
420	ThreadCPUPinner pinner(thread_get_current_thread());
421	RecursiveLocker locker(fLock);
422
423	// TODO: replace this kludge
424
425	phys_addr_t pa;
426	uint64_t pte;
427	if (!WalkTable(fPageTable, fInitialLevel, address, &pa, &pte))
428		return B_ENTRY_NOT_FOUND;
429
430	uint64_t vaMask = (1UL << fVaBits) - 1;
431	MapRange(fPageTable, fInitialLevel, address & vaMask, 0, B_PAGE_SIZE, VMAction::UNMAP, 0, NULL);
432
433	pinner.Unlock();
434	locker.Detach();
435	PageUnmapped(area, pa >> fPageBits, (tmp_pte & kAttrAF) != 0, (tmp_pte & kAttrAP2) == 0,
436		updatePageQueue);
437
438	return B_OK;
439}
440
441
442bool
443VMSAv8TranslationMap::WalkTable(
444	phys_addr_t ptPa, int level, addr_t va, phys_addr_t* pa, uint64_t* rpte)
445{
446	int tableBits = fPageBits - 3;
447	uint64_t tableMask = (1UL << tableBits) - 1;
448
449	int shift = tableBits * (3 - level) + fPageBits;
450	uint64_t entrySize = 1UL << shift;
451	uint64_t entryMask = entrySize - 1;
452
453	int index = (va >> shift) & tableMask;
454
455	uint64_t pte = TableFromPa(ptPa)[index];
456	int type = pte & 0x3;
457
458	if ((type & 0x1) == 0)
459		return false;
460
461	uint64_t addr = pte & kPteAddrMask;
462	if (level < 3) {
463		if (type == 0x3) {
464			return WalkTable(addr, level + 1, va, pa, rpte);
465		} else {
466			*pa = addr | (va & entryMask);
467			*rpte = pte;
468		}
469	} else {
470		ASSERT(type == 0x3);
471		*pa = addr;
472		*rpte = pte;
473	}
474
475	return true;
476}
477
478
479bool
480VMSAv8TranslationMap::ValidateVa(addr_t va)
481{
482	uint64_t vaMask = (1UL << fVaBits) - 1;
483	bool kernelAddr = (va & (1UL << 63)) != 0;
484	if (kernelAddr != fIsKernel)
485		return false;
486	if ((va & ~vaMask) != (fIsKernel ? ~vaMask : 0))
487		return false;
488	return true;
489}
490
491
492status_t
493VMSAv8TranslationMap::Query(addr_t va, phys_addr_t* pa, uint32* flags)
494{
495	ThreadCPUPinner pinner(thread_get_current_thread());
496
497	ASSERT(ValidateVa(va));
498
499	uint64_t pte = 0;
500	bool ret = WalkTable(fPageTable, fInitialLevel, va, pa, &pte);
501
502	uint32 result = 0;
503
504	if (ret) {
505		result |= PAGE_PRESENT;
506
507		if ((pte & kAttrAF) != 0)
508			result |= PAGE_ACCESSED;
509		if ((pte & kAttrAP2) == 0)
510			result |= PAGE_MODIFIED;
511
512		if ((pte & kAttrUXN) == 0)
513			result |= B_EXECUTE_AREA;
514		if ((pte & kAttrPXN) == 0)
515			result |= B_KERNEL_EXECUTE_AREA;
516
517		result |= B_KERNEL_READ_AREA;
518
519		if ((pte & kAttrAP1) != 0)
520			result |= B_READ_AREA;
521
522		if ((pte & kAttrAP2) == 0 || (pte & kAttrSWDBM) != 0) {
523			result |= B_KERNEL_WRITE_AREA;
524
525			if ((pte & kAttrAP1) != 0)
526				result |= B_WRITE_AREA;
527		}
528	}
529
530	*flags = result;
531	return B_OK;
532}
533
534
535status_t
536VMSAv8TranslationMap::QueryInterrupt(
537	addr_t virtualAddress, phys_addr_t* _physicalAddress, uint32* _flags)
538{
539	return Query(virtualAddress, _physicalAddress, _flags);
540}
541
542
543status_t
544VMSAv8TranslationMap::Protect(addr_t start, addr_t end, uint32 attributes, uint32 memoryType)
545{
546	ThreadCPUPinner pinner(thread_get_current_thread());
547
548	size_t size = end - start + 1;
549
550	uint64_t pageMask = (1UL << fPageBits) - 1;
551	uint64_t vaMask = (1UL << fVaBits) - 1;
552
553	ASSERT((start & pageMask) == 0);
554	ASSERT((size & pageMask) == 0);
555	ASSERT(ValidateVa(start));
556
557	uint64_t attr = GetMemoryAttr(attributes, memoryType, fIsKernel);
558	MapRange(fPageTable, fInitialLevel, start & vaMask, 0, size, VMAction::SET_ATTR, attr, NULL);
559
560	return B_OK;
561}
562
563
564status_t
565VMSAv8TranslationMap::ClearFlags(addr_t va, uint32 flags)
566{
567	ThreadCPUPinner pinner(thread_get_current_thread());
568
569	uint64_t pageMask = (1UL << fPageBits) - 1;
570	uint64_t vaMask = (1UL << fVaBits) - 1;
571
572	ASSERT((va & pageMask) == 0);
573	ASSERT(ValidateVa(va));
574
575	MapRange(
576		fPageTable, fInitialLevel, va & vaMask, 0, B_PAGE_SIZE, VMAction::CLEAR_FLAGS, flags, NULL);
577
578	return B_OK;
579}
580
581
582bool
583VMSAv8TranslationMap::ClearAccessedAndModified(
584	VMArea* area, addr_t address, bool unmapIfUnaccessed, bool& _modified)
585{
586	panic("VMSAv8TranslationMap::ClearAccessedAndModified not implemented\n");
587	return B_OK;
588}
589
590
591void
592VMSAv8TranslationMap::Flush()
593{
594	ThreadCPUPinner pinner(thread_get_current_thread());
595
596	arch_cpu_global_TLB_invalidate();
597}
598