1139825Simp// SPDX-License-Identifier: GPL-2.0 248006Skato/* 348006Skato * Copyright (C) 2020 - Google LLC 448006Skato * Author: Quentin Perret <qperret@google.com> 548006Skato */ 648006Skato 748006Skato#include <linux/init.h> 848006Skato#include <linux/kmemleak.h> 948006Skato#include <linux/kvm_host.h> 1048006Skato#include <linux/memblock.h> 1148006Skato#include <linux/mutex.h> 1248006Skato#include <linux/sort.h> 1348006Skato 1448006Skato#include <asm/kvm_pkvm.h> 1548006Skato 1648006Skato#include "hyp_constants.h" 1748006Skato 1848006SkatoDEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized); 1948006Skato 2048006Skatostatic struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory); 2148006Skatostatic unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr); 2248006Skato 2348006Skatophys_addr_t hyp_mem_base; 2448006Skatophys_addr_t hyp_mem_size; 2548006Skato 2648006Skatostatic int cmp_hyp_memblock(const void *p1, const void *p2) 2748006Skato{ 2848006Skato const struct memblock_region *r1 = p1; 2948006Skato const struct memblock_region *r2 = p2; 3048006Skato 3148006Skato return r1->base < r2->base ? -1 : (r1->base > r2->base); 3248006Skato} 3348006Skato 3448006Skatostatic void __init sort_memblock_regions(void) 3548006Skato{ 3648006Skato sort(hyp_memory, 3748006Skato *hyp_memblock_nr_ptr, 3848006Skato sizeof(struct memblock_region), 3948006Skato cmp_hyp_memblock, 4048006Skato NULL); 4148006Skato} 4248006Skato 4348006Skatostatic int __init register_memblock_regions(void) 4448006Skato{ 4548006Skato struct memblock_region *reg; 4648006Skato 4748006Skato for_each_mem_region(reg) { 4848006Skato if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS) 4950477Speter return -ENOMEM; 5048006Skato 5148006Skato hyp_memory[*hyp_memblock_nr_ptr] = *reg; 5248006Skato (*hyp_memblock_nr_ptr)++; 5348006Skato } 5448006Skato sort_memblock_regions(); 5548006Skato 5648006Skato return 0; 5748006Skato} 5848006Skato 5948006Skatovoid __init kvm_hyp_reserve(void) 6048006Skato{ 6148006Skato u64 hyp_mem_pages = 0; 6248006Skato int ret; 6348006Skato 6448006Skato if (!is_hyp_mode_available() || is_kernel_in_hyp_mode()) 65108470Sschweikh return; 6648006Skato 6748006Skato if (kvm_get_mode() != KVM_MODE_PROTECTED) 6848006Skato return; 6948006Skato 7048006Skato ret = register_memblock_regions(); 7148006Skato if (ret) { 7248006Skato *hyp_memblock_nr_ptr = 0; 7348006Skato kvm_err("Failed to register hyp memblocks: %d\n", ret); 7448006Skato return; 7548006Skato } 7648006Skato 7748006Skato hyp_mem_pages += hyp_s1_pgtable_pages(); 7848006Skato hyp_mem_pages += host_s2_pgtable_pages(); 7948006Skato hyp_mem_pages += hyp_vm_table_pages(); 8048006Skato hyp_mem_pages += hyp_vmemmap_pages(STRUCT_HYP_PAGE_SIZE); 8148006Skato hyp_mem_pages += hyp_ffa_proxy_pages(); 8248006Skato 8348006Skato /* 8448006Skato * Try to allocate a PMD-aligned region to reduce TLB pressure once 8548006Skato * this is unmapped from the host stage-2, and fallback to PAGE_SIZE. 8648006Skato */ 8748006Skato hyp_mem_size = hyp_mem_pages << PAGE_SHIFT; 8848006Skato hyp_mem_base = memblock_phys_alloc(ALIGN(hyp_mem_size, PMD_SIZE), 8948006Skato PMD_SIZE); 9048006Skato if (!hyp_mem_base) 9148006Skato hyp_mem_base = memblock_phys_alloc(hyp_mem_size, PAGE_SIZE); 9248006Skato else 9348006Skato hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE); 9448006Skato 9548006Skato if (!hyp_mem_base) { 9648006Skato kvm_err("Failed to reserve hyp memory\n"); 9748006Skato return; 9848006Skato } 9948006Skato 10048006Skato kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20, 10148006Skato hyp_mem_base); 10248006Skato} 10348006Skato 10448006Skatostatic void __pkvm_destroy_hyp_vm(struct kvm *host_kvm) 10548006Skato{ 10648006Skato if (host_kvm->arch.pkvm.handle) { 10761116Snyan WARN_ON(kvm_call_hyp_nvhe(__pkvm_teardown_vm, 10848006Skato host_kvm->arch.pkvm.handle)); 109130174Sphk } 11048006Skato 11148006Skato host_kvm->arch.pkvm.handle = 0; 112116431Sphk free_hyp_memcache(&host_kvm->arch.pkvm.teardown_mc); 11348006Skato} 11480537Snyan 11580537Snyan/* 11680537Snyan * Allocates and donates memory for hypervisor VM structs at EL2. 11780537Snyan * 11880537Snyan * Allocates space for the VM state, which includes the hyp vm as well as 11980537Snyan * the hyp vcpus. 12080537Snyan * 12148006Skato * Stores an opaque handler in the kvm struct for future reference. 12260950Snyan * 12348006Skato * Return 0 on success, negative error code on failure. 12448006Skato */ 12548006Skatostatic int __pkvm_create_hyp_vm(struct kvm *host_kvm) 12648006Skato{ 12748006Skato size_t pgd_sz, hyp_vm_sz, hyp_vcpu_sz; 12848006Skato struct kvm_vcpu *host_vcpu; 12948006Skato pkvm_handle_t handle; 13048006Skato void *pgd, *hyp_vm; 13148006Skato unsigned long idx; 13248006Skato int ret; 13348006Skato 13448006Skato if (host_kvm->created_vcpus < 1) 13548006Skato return -EINVAL; 13648006Skato 13748006Skato pgd_sz = kvm_pgtable_stage2_pgd_size(host_kvm->arch.mmu.vtcr); 13848006Skato 13948006Skato /* 14048006Skato * The PGD pages will be reclaimed using a hyp_memcache which implies 14148006Skato * page granularity. So, use alloc_pages_exact() to get individual 14248006Skato * refcounts. 14348006Skato */ 14448006Skato pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT); 14548006Skato if (!pgd) 14648006Skato return -ENOMEM; 14748006Skato 14848006Skato /* Allocate memory to donate to hyp for vm and vcpu pointers. */ 14948006Skato hyp_vm_sz = PAGE_ALIGN(size_add(PKVM_HYP_VM_SIZE, 15048006Skato size_mul(sizeof(void *), 15180537Snyan host_kvm->created_vcpus))); 15280537Snyan hyp_vm = alloc_pages_exact(hyp_vm_sz, GFP_KERNEL_ACCOUNT); 15380537Snyan if (!hyp_vm) { 15480537Snyan ret = -ENOMEM; 15580537Snyan goto free_pgd; 15648006Skato } 15748006Skato 15848006Skato /* Donate the VM memory to hyp and let hyp initialize it. */ 15948006Skato ret = kvm_call_hyp_nvhe(__pkvm_init_vm, host_kvm, hyp_vm, pgd); 16048006Skato if (ret < 0) 16148006Skato goto free_vm; 16248006Skato 16348006Skato handle = ret; 16448006Skato 16548006Skato host_kvm->arch.pkvm.handle = handle; 16648006Skato 16748006Skato /* Donate memory for the vcpus at hyp and initialize it. */ 168116431Sphk hyp_vcpu_sz = PAGE_ALIGN(PKVM_HYP_VCPU_SIZE); 16948006Skato kvm_for_each_vcpu(idx, host_vcpu, host_kvm) { 17048006Skato void *hyp_vcpu; 17148006Skato 17248006Skato /* Indexing of the vcpus to be sequential starting at 0. */ 17348006Skato if (WARN_ON(host_vcpu->vcpu_idx != idx)) { 17448006Skato ret = -EINVAL; 17548006Skato goto destroy_vm; 17648006Skato } 17780537Snyan 17848006Skato hyp_vcpu = alloc_pages_exact(hyp_vcpu_sz, GFP_KERNEL_ACCOUNT); 17948006Skato if (!hyp_vcpu) { 18048006Skato ret = -ENOMEM; 18148006Skato goto destroy_vm; 18248006Skato } 18348006Skato 18448006Skato ret = kvm_call_hyp_nvhe(__pkvm_init_vcpu, handle, host_vcpu, 18548006Skato hyp_vcpu); 18648006Skato if (ret) { 18748006Skato free_pages_exact(hyp_vcpu, hyp_vcpu_sz); 18848006Skato goto destroy_vm; 18948006Skato } 19048006Skato } 19148006Skato 19248006Skato return 0; 19348006Skato 19448006Skatodestroy_vm: 19548006Skato __pkvm_destroy_hyp_vm(host_kvm); 19648006Skato return ret; 19748006Skatofree_vm: 19848006Skato free_pages_exact(hyp_vm, hyp_vm_sz); 19948006Skatofree_pgd: 20048006Skato free_pages_exact(pgd, pgd_sz); 20148006Skato return ret; 20248006Skato} 20348006Skato 20448006Skatoint pkvm_create_hyp_vm(struct kvm *host_kvm) 20548006Skato{ 20648006Skato int ret = 0; 20748006Skato 20880537Snyan mutex_lock(&host_kvm->arch.config_lock); 20980537Snyan if (!host_kvm->arch.pkvm.handle) 21080537Snyan ret = __pkvm_create_hyp_vm(host_kvm); 21148006Skato mutex_unlock(&host_kvm->arch.config_lock); 21280537Snyan 21380537Snyan return ret; 21480537Snyan} 21580537Snyan 21680537Snyanvoid pkvm_destroy_hyp_vm(struct kvm *host_kvm) 21780537Snyan{ 21848006Skato mutex_lock(&host_kvm->arch.config_lock); 21948006Skato __pkvm_destroy_hyp_vm(host_kvm); 22080537Snyan mutex_unlock(&host_kvm->arch.config_lock); 22180537Snyan} 22280537Snyan 22380537Snyanint pkvm_init_host_vm(struct kvm *host_kvm) 22480537Snyan{ 22580537Snyan mutex_init(&host_kvm->lock); 22680537Snyan return 0; 22780537Snyan} 22848006Skato 22948006Skatostatic void __init _kvm_host_prot_finalize(void *arg) 23048006Skato{ 23148006Skato int *err = arg; 23248006Skato 23348006Skato if (WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize))) 234126080Sphk WRITE_ONCE(*err, -EINVAL); 235126080Sphk} 236111815Sphk 237111815Sphkstatic int __init pkvm_drop_host_privileges(void) 238111815Sphk{ 239111815Sphk int ret = 0; 240111815Sphk 24148006Skato /* 24248006Skato * Flip the static key upfront as that may no longer be possible 24380537Snyan * once the host stage 2 is installed. 24480537Snyan */ 24548006Skato static_branch_enable(&kvm_protected_mode_initialized); 24648006Skato on_each_cpu(_kvm_host_prot_finalize, &ret, 1); 24748006Skato return ret; 24848006Skato} 24948006Skato 25048006Skatostatic int __init finalize_pkvm(void) 25148006Skato{ 25248006Skato int ret; 25348006Skato 25448006Skato if (!is_protected_kvm_enabled() || !is_kvm_arm_initialised()) 25548006Skato return 0; 25648006Skato 25748006Skato /* 25848006Skato * Exclude HYP sections from kmemleak so that they don't get peeked 25948006Skato * at, which would end badly once inaccessible. 26048006Skato */ 26148006Skato kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start); 26248006Skato kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size); 26348006Skato 26448006Skato ret = pkvm_drop_host_privileges(); 26548006Skato if (ret) 26648006Skato pr_err("Failed to finalize Hyp protection: %d\n", ret); 26748006Skato 26848006Skato return ret; 26948006Skato} 27048006Skatodevice_initcall_sync(finalize_pkvm); 27148006Skato