lib/aarch64/processor.c

21 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)
23 	return (v + vm->page_size) & ~(vm->page_size - 1);
26 static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva)
28 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
29 	uint64_t mask = (1UL << (vm->va_bits - shift)) - 1;
34 static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)
36 	unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift;
37 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
39 	TEST_ASSERT(vm->pgtable_levels == 4,
40 		"Mode %d does not have 4 page table levels", vm->mode);
45 static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)
47 	unsigned int shift = (vm->page_shift - 3) + vm->page_shift;
48 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
50 	TEST_ASSERT(vm->pgtable_levels >= 3,
51 		"Mode %d does not have >= 3 page table levels", vm->mode);
56 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva)
58 	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
59 	return (gva >> vm->page_shift) & mask;
62 static inline bool use_lpa2_pte_format(struct kvm_vm *vm)
64 	return (vm->page_size == SZ_4K || vm->page_size == SZ_16K) &&
65 	    (vm->pa_bits > 48 || vm->va_bits > 48);
68 static uint64_t addr_pte(struct kvm_vm *vm, uint64_t pa, uint64_t attrs)
72 	if (use_lpa2_pte_format(vm)) {
73 		pte = pa & GENMASK(49, vm->page_shift);
77 		pte = pa & GENMASK(47, vm->page_shift);
78 		if (vm->page_shift == 16)
86 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t pte)
90 	if (use_lpa2_pte_format(vm)) {
91 		pa = pte & GENMASK(49, vm->page_shift);
94 		pa = pte & GENMASK(47, vm->page_shift);
95 		if (vm->page_shift == 16)
102 static uint64_t ptrs_per_pgd(struct kvm_vm *vm)
104 	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
105 	return 1 << (vm->va_bits - shift);
108 static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm)
110 	return 1 << (vm->page_shift - 3);
113 void virt_arch_pgd_alloc(struct kvm_vm *vm)
115 	size_t nr_pages = page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size;
117 	if (vm->pgd_created)
120 	vm->pgd = vm_phy_pages_alloc(vm, nr_pages,
122 				     vm->memslots[MEM_REGION_PT]);
123 	vm->pgd_created = true;
126 static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
132 	TEST_ASSERT((vaddr % vm->page_size) == 0,
134 		"  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
135 	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
136 		(vaddr >> vm->page_shift)),
138 	TEST_ASSERT((paddr % vm->page_size) == 0,
140 		"  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
141 	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
143 		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
144 		paddr, vm->max_gfn, vm->page_size);
146 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8;
148 		*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
150 	switch (vm->pgtable_levels) {
152 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;
154 			*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
157 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;
159 			*ptep = addr_pte(vm, vm_alloc_page_table(vm), 3);
162 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;
168 	*ptep = addr_pte(vm, paddr, (attr_idx << 2) | (1 << 10) | 3);  /* AF */
171 void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
175 	_virt_pg_map(vm, vaddr, paddr, attr_idx);
178 uint64_t *virt_get_pte_hva(struct kvm_vm *vm, vm_vaddr_t gva)
182 	if (!vm->pgd_created)
185 	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8;
189 	switch (vm->pgtable_levels) {
191 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8;
196 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8;
201 		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8;
212 	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
216 vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
218 	uint64_t *ptep = virt_get_pte_hva(vm, gva);
220 	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
223 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
232 	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {
233 		ptep = addr_gpa2hva(vm, pte);
237 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1);
242 void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
244 	int level = 4 - (vm->pgtable_levels - 1);
247 	if (!vm->pgd_created)
250 	for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) {
251 		ptep = addr_gpa2hva(vm, pgd);
255 		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level);
262 	struct kvm_vm *vm = vcpu->vm;
270 		vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred);
286 	switch (vm->mode) {
310 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
313 	ttbr0_el1 = vm->pgd & GENMASK(47, vm->page_shift);
316 	switch (vm->mode) {
321 		ttbr0_el1 |= FIELD_GET(GENMASK(51, 48), vm->pgd) << 2;
340 		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
346 	tcr_el1 |= (64 - vm->va_bits) /* T0SZ */;
347 	if (use_lpa2_pte_format(vm))
373 static struct kvm_vcpu *__aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
378 	struct kvm_vcpu *vcpu = __vm_vcpu_add(vm, vcpu_id);
380 	stack_size = vm->page_size == 4096 ? DEFAULT_STACK_PGS * vm->page_size :
381 					     vm->page_size;
382 	stack_vaddr = __vm_vaddr_alloc(vm, stack_size,
392 struct kvm_vcpu *aarch64_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
395 	struct kvm_vcpu *vcpu = __aarch64_vcpu_add(vm, vcpu_id, init);
402 struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id)
404 	return __aarch64_vcpu_add(vm, vcpu_id, NULL);
494 void vm_init_descriptor_tables(struct kvm_vm *vm)
496 	vm->handlers = __vm_vaddr_alloc(vm, sizeof(struct handlers),
497 					vm->page_size, MEM_REGION_DATA);
499 	*(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
502 void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec,
505 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);
513 void vm_install_exception_handler(struct kvm_vm *vm, int vector,
516 	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);
631 void vm_vaddr_populate_bitmap(struct kvm_vm *vm)
637 	sparsebit_set_num(vm->vpages_valid, 0,
638 			  (1ULL << vm->va_bits) >> vm->page_shift);