14Srgrimes// SPDX-License-Identifier: GPL-2.0-only 24Srgrimes/* 34Srgrimes * Copyright (C) 2009 SUSE Linux Products GmbH. All rights reserved. 44Srgrimes * 54Srgrimes * Authors: 64Srgrimes * Alexander Graf <agraf@suse.de> 74Srgrimes * Kevin Wolf <mail@kevin-wolf.de> 84Srgrimes */ 94Srgrimes 104Srgrimes#include <linux/kvm_host.h> 114Srgrimes#include <linux/pkeys.h> 124Srgrimes 134Srgrimes#include <asm/kvm_ppc.h> 144Srgrimes#include <asm/kvm_book3s.h> 154Srgrimes#include <asm/book3s/64/mmu-hash.h> 164Srgrimes#include <asm/machdep.h> 174Srgrimes#include <asm/mmu_context.h> 184Srgrimes#include <asm/hw_irq.h> 194Srgrimes#include "trace_pr.h" 204Srgrimes#include "book3s.h" 214Srgrimes 224Srgrimes#define PTE_SIZE 12 234Srgrimes 244Srgrimesvoid kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte) 254Srgrimes{ 264Srgrimes mmu_hash_ops.hpte_invalidate(pte->slot, pte->host_vpn, 274Srgrimes pte->pagesize, pte->pagesize, 284Srgrimes MMU_SEGSIZE_256M, false); 294Srgrimes} 304Srgrimes 314Srgrimes/* We keep 512 gvsid->hvsid entries, mapping the guest ones to the array using 324Srgrimes * a hash, so we don't waste cycles on looping */ 334Srgrimesstatic u16 kvmppc_sid_hash(struct kvm_vcpu *vcpu, u64 gvsid) 34593Srgrimes{ 3550477Speter return (u16)(((gvsid >> (SID_MAP_BITS * 7)) & SID_MAP_MASK) ^ 364Srgrimes ((gvsid >> (SID_MAP_BITS * 6)) & SID_MAP_MASK) ^ 374Srgrimes ((gvsid >> (SID_MAP_BITS * 5)) & SID_MAP_MASK) ^ 3879609Speter ((gvsid >> (SID_MAP_BITS * 4)) & SID_MAP_MASK) ^ 3932929Seivind ((gvsid >> (SID_MAP_BITS * 3)) & SID_MAP_MASK) ^ 4013290Speter ((gvsid >> (SID_MAP_BITS * 2)) & SID_MAP_MASK) ^ 4113225Swollman ((gvsid >> (SID_MAP_BITS * 1)) & SID_MAP_MASK) ^ 422056Swollman ((gvsid >> (SID_MAP_BITS * 0)) & SID_MAP_MASK)); 432056Swollman} 4445720Speter 4511865Sphk 4676166Smarkmstatic struct kvmppc_sid_map *find_sid_vsid(struct kvm_vcpu *vcpu, u64 gvsid) 4733281Sbde{ 4845720Speter struct kvmppc_sid_map *map; 4976166Smarkm u16 sid_map_mask; 5076166Smarkm 5176166Smarkm if (kvmppc_get_msr(vcpu) & MSR_PR) 5211865Sphk gvsid |= VSID_PR; 5345720Speter 5445720Speter sid_map_mask = kvmppc_sid_hash(vcpu, gvsid); 5522093Sbde map = &to_book3s(vcpu)->sid_map[sid_map_mask]; 564478Sbde if (map->valid && (map->guest_vsid == gvsid)) { 5722093Sbde trace_kvm_book3s_slb_found(gvsid, map->host_vsid); 584478Sbde return map; 5976906Sbde } 603816Swollman 6131255Sbde map = &to_book3s(vcpu)->sid_map[SID_MAP_MASK - sid_map_mask]; 6225083Sjdp if (map->valid && (map->guest_vsid == gvsid)) { 6331255Sbde trace_kvm_book3s_slb_found(gvsid, map->host_vsid); 6430805Sbde return map; 6530805Sbde } 6626309Speter 672056Swollman trace_kvm_book3s_slb_fail(sid_map_mask, gvsid); 6830805Sbde return NULL; 6931255Sbde} 704478Sbde 7131255Sbdeint kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *orig_pte, 7245720Speter bool iswrite) 732056Swollman{ 7430805Sbde unsigned long vpn; 753816Swollman kvm_pfn_t hpaddr; 7631255Sbde ulong hash, hpteg; 772056Swollman u64 vsid; 7826373Sdfr int ret; 792056Swollman int rflags = 0x192; 8031255Sbde int vflags = 0; 8160008Swollman int attempt = 0; 824Srgrimes struct kvmppc_sid_map *map; 834Srgrimes int r = 0; 844Srgrimes int hpsize = MMU_PAGE_4K; 854Srgrimes bool writable; 864Srgrimes unsigned long mmu_seq; 8719653Sbde struct kvm *kvm = vcpu->kvm; 8819653Sbde struct hpte_cache *cpte; 8919653Sbde unsigned long gfn = orig_pte->raddr >> PAGE_SHIFT; 9019653Sbde unsigned long pfn; 9149081Scracauer 9219653Sbde /* used to check for invalidations in progress */ 934Srgrimes mmu_seq = kvm->mmu_invalidate_seq; 944Srgrimes smp_rmb(); 955351Sbde 964Srgrimes /* Get host physical address for gpa */ 974Srgrimes pfn = kvmppc_gpa_to_pfn(vcpu, orig_pte->raddr, iswrite, &writable); 9835215Sbde if (is_error_noslot_pfn(pfn)) { 9935215Sbde printk(KERN_INFO "Couldn't get guest page for gpa %lx!\n", 10035215Sbde orig_pte->raddr); 1015351Sbde r = -EINVAL; 1025351Sbde goto out; 10379609Speter } 10479609Speter hpaddr = pfn << PAGE_SHIFT; 1054Srgrimes 1064Srgrimes /* and write the mapping ea -> hpa into the pt */ 1074Srgrimes vcpu->arch.mmu.esid_to_vsid(vcpu, orig_pte->eaddr >> SID_SHIFT, &vsid); 1084Srgrimes map = find_sid_vsid(vcpu, vsid); 1094Srgrimes if (!map) { 1104Srgrimes ret = kvmppc_mmu_map_segment(vcpu, orig_pte->eaddr); 1114Srgrimes WARN_ON(ret < 0); 1124Srgrimes map = find_sid_vsid(vcpu, vsid); 1134Srgrimes } 1144Srgrimes if (!map) { 1154Srgrimes printk(KERN_ERR "KVM: Segment map for 0x%llx (0x%lx) failed\n", 1164Srgrimes vsid, orig_pte->eaddr); 1174Srgrimes WARN_ON(true); 1184Srgrimes r = -EINVAL; 11979609Speter goto out; 12079609Speter } 1214Srgrimes 1224Srgrimes vpn = hpt_vpn(orig_pte->eaddr, map->host_vsid, MMU_SEGSIZE_256M); 1234Srgrimes 1244Srgrimes kvm_set_pfn_accessed(pfn); 1254Srgrimes if (!orig_pte->may_write || !writable) 12679609Speter rflags |= PP_RXRX; 12779609Speter else { 12879609Speter mark_page_dirty(vcpu->kvm, gfn); 12979609Speter kvm_set_pfn_dirty(pfn); 13079609Speter } 13179609Speter 13279609Speter if (!orig_pte->may_execute) 13379609Speter rflags |= HPTE_R_N; 13479609Speter else 13579609Speter kvmppc_mmu_flush_icache(pfn); 13679609Speter 13779609Speter rflags |= pte_to_hpte_pkey_bits(0, HPTE_USE_KERNEL_KEY); 13879609Speter rflags = (rflags & ~HPTE_R_WIMG) | orig_pte->wimg; 13979609Speter 14079609Speter /* 14179609Speter * Use 64K pages if possible; otherwise, on 64K page kernels, 14279609Speter * we need to transfer 4 more bits from guest real to host real addr. 14379609Speter */ 14479609Speter if (vsid & VSID_64K) 14579609Speter hpsize = MMU_PAGE_64K; 14679609Speter else 14779609Speter hpaddr |= orig_pte->raddr & (~0xfffULL & ~PAGE_MASK); 14879609Speter 14979609Speter hash = hpt_hash(vpn, mmu_psize_defs[hpsize].shift, MMU_SEGSIZE_256M); 15079609Speter 15179609Speter cpte = kvmppc_mmu_hpte_cache_next(vcpu); 15279609Speter 15379609Speter spin_lock(&kvm->mmu_lock); 1544Srgrimes if (!cpte || mmu_invalidate_retry(kvm, mmu_seq)) { 1554Srgrimes r = -EAGAIN; 15645720Speter goto out_unlock; 15750181Speter } 15877015Sbde 15977015Sbdemap_again: 16077015Sbde hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 16145720Speter 16245720Speter /* In case we tried normal mapping already, let's nuke old entries */ 16379609Speter if (attempt > 1) 16479609Speter if (mmu_hash_ops.hpte_remove(hpteg) < 0) { 16578260Speter r = -1; 16633281Sbde goto out_unlock; 16733281Sbde } 16845100Sdt 1694Srgrimes ret = mmu_hash_ops.hpte_insert(hpteg, vpn, hpaddr, rflags, vflags, 1705351Sbde hpsize, hpsize, MMU_SEGSIZE_256M); 17111865Sphk 17211865Sphk if (ret == -1) { 17311865Sphk /* If we couldn't map a primary PTE, try a secondary */ 17411865Sphk hash = ~hash; 17511865Sphk vflags ^= HPTE_V_SECONDARY; 17641591Sarchie attempt++; 17741591Sarchie goto map_again; 17841591Sarchie } else if (ret < 0) { 17941591Sarchie r = -EIO; 18026812Speter goto out_unlock; 1814Srgrimes } else { 1824Srgrimes trace_kvm_book3s_64_mmu_map(rflags, hpteg, 1834Srgrimes vpn, hpaddr, orig_pte); 18447226Speter 1854Srgrimes /* 18627567Sfsmp * The mmu_hash_ops code may give us a secondary entry even 1874Srgrimes * though we asked for a primary. Fix up. 1884Srgrimes */ 1894Srgrimes if ((ret & _PTEIDX_SECONDARY) && !(vflags & HPTE_V_SECONDARY)) { 1905351Sbde hash = ~hash; 1914Srgrimes hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); 19225164Speter } 19335302Sbde 19435302Sbde cpte->slot = hpteg + (ret & 7); 19535302Sbde cpte->host_vpn = vpn; 19646548Sbde cpte->pte = *orig_pte; 19735302Sbde cpte->pfn = pfn; 19835302Sbde cpte->pagesize = hpsize; 19935302Sbde 20035302Sbde kvmppc_mmu_hpte_cache_map(vcpu, cpte); 20135302Sbde cpte = NULL; 20235302Sbde } 20335302Sbde 20435302Sbdeout_unlock: 20535302Sbde spin_unlock(&kvm->mmu_lock); 20635302Sbde kvm_release_pfn_clean(pfn); 20735302Sbde if (cpte) 2084Srgrimes kvmppc_mmu_hpte_cache_free(cpte); 2094Srgrimes 2104478Sbdeout: 21135302Sbde return r; 21235302Sbde} 21335302Sbde 21446548Sbdevoid kvmppc_mmu_unmap_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) 21535302Sbde{ 21635302Sbde u64 mask = 0xfffffffffULL; 21735302Sbde u64 vsid; 21835302Sbde 21935302Sbde vcpu->arch.mmu.esid_to_vsid(vcpu, pte->eaddr >> SID_SHIFT, &vsid); 2204Srgrimes if (vsid & VSID_64K) 22127567Sfsmp mask = 0xffffffff0ULL; 2224Srgrimes kvmppc_mmu_pte_vflush(vcpu, pte->vpage, mask); 2234Srgrimes} 22450181Speter 22550181Speterstatic struct kvmppc_sid_map *create_sid_map(struct kvm_vcpu *vcpu, u64 gvsid) 22650181Speter{ 22750181Speter unsigned long vsid_bits = VSID_BITS_65_256M; 22850181Speter struct kvmppc_sid_map *map; 22950181Speter struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu); 23050181Speter u16 sid_map_mask; 23150181Speter static int backwards_map; 23250181Speter 23350181Speter if (kvmppc_get_msr(vcpu) & MSR_PR) 23450181Speter gvsid |= VSID_PR; 23550181Speter 23650181Speter /* We might get collisions that trap in preceding order, so let's 23750181Speter map them differently */ 23877015Sbde 23950181Speter sid_map_mask = kvmppc_sid_hash(vcpu, gvsid); 24077015Sbde if (backwards_map) 24177015Sbde sid_map_mask = SID_MAP_MASK - sid_map_mask; 24277015Sbde 24377015Sbde map = &to_book3s(vcpu)->sid_map[sid_map_mask]; 24477015Sbde 24577015Sbde /* Make sure we're taking the other map next time */ 24677015Sbde backwards_map = !backwards_map; 24777015Sbde 24877015Sbde /* Uh-oh ... out of mappings. Let's flush! */ 24977015Sbde if (vcpu_book3s->proto_vsid_next == vcpu_book3s->proto_vsid_max) { 25077015Sbde vcpu_book3s->proto_vsid_next = vcpu_book3s->proto_vsid_first; 25177015Sbde memset(vcpu_book3s->sid_map, 0, 25277015Sbde sizeof(struct kvmppc_sid_map) * SID_MAP_NUM); 25377015Sbde kvmppc_mmu_pte_flush(vcpu, 0, 0); 25477015Sbde kvmppc_mmu_flush_segments(vcpu); 25577015Sbde } 25677015Sbde 25777015Sbde if (mmu_has_feature(MMU_FTR_68_BIT_VA)) 25877015Sbde vsid_bits = VSID_BITS_256M; 25977015Sbde 26077015Sbde map->host_vsid = vsid_scramble(vcpu_book3s->proto_vsid_next++, 26177015Sbde VSID_MULTIPLIER_256M, vsid_bits); 26277015Sbde 26377015Sbde map->guest_vsid = gvsid; 26477015Sbde map->valid = true; 26577015Sbde 26677015Sbde trace_kvm_book3s_slb_map(sid_map_mask, gvsid, map->host_vsid); 26777015Sbde 26877015Sbde return map; 26977015Sbde} 27077015Sbde 27177015Sbdestatic int kvmppc_mmu_next_segment(struct kvm_vcpu *vcpu, ulong esid) 27277015Sbde{ 27377015Sbde struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 27477015Sbde int i; 27577015Sbde int max_slb_size = 64; 27677015Sbde int found_inval = -1; 27777015Sbde int r; 27877015Sbde 27977015Sbde /* Are we overwriting? */ 28077015Sbde for (i = 0; i < svcpu->slb_max; i++) { 28177015Sbde if (!(svcpu->slb[i].esid & SLB_ESID_V)) 28277015Sbde found_inval = i; 28377015Sbde else if ((svcpu->slb[i].esid & ESID_MASK) == esid) { 2844Srgrimes r = i; 2854Srgrimes goto out; 2864Srgrimes } 2874Srgrimes } 2884Srgrimes 2894Srgrimes /* Found a spare entry that was invalidated before */ 29045720Speter if (found_inval >= 0) { 29145720Speter r = found_inval; 2924Srgrimes goto out; 29346555Speter } 29427567Sfsmp 29547226Speter /* No spare invalid entry, so create one */ 29647226Speter 29745720Speter if (mmu_slb_size < 64) 29827567Sfsmp max_slb_size = mmu_slb_size; 29927567Sfsmp 30027567Sfsmp /* Overflowing -> purge */ 3014Srgrimes if ((svcpu->slb_max) == max_slb_size) 30274903Sjhb kvmppc_mmu_flush_segments(vcpu); 3034Srgrimes 3044Srgrimes r = svcpu->slb_max; 3054Srgrimes svcpu->slb_max++; 3064Srgrimes 3074Srgrimesout: 3084Srgrimes svcpu_put(svcpu); 3094Srgrimes return r; 3104Srgrimes} 3114Srgrimes 3124Srgrimesint kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr) 31347226Speter{ 31447226Speter struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 31547226Speter u64 esid = eaddr >> SID_SHIFT; 31674903Sjhb u64 slb_esid = (eaddr & ESID_MASK) | SLB_ESID_V; 3174Srgrimes u64 slb_vsid = SLB_VSID_USER; 3184Srgrimes u64 gvsid; 3194Srgrimes int slb_index; 3204Srgrimes struct kvmppc_sid_map *map; 32145720Speter int r = 0; 32247226Speter 32312929Sdg slb_index = kvmppc_mmu_next_segment(vcpu, eaddr & ESID_MASK); 32412929Sdg 32565557Sjasone if (vcpu->arch.mmu.esid_to_vsid(vcpu, esid, &gvsid)) { 32665557Sjasone /* Invalidate an entry */ 32765557Sjasone svcpu->slb[slb_index].esid = 0; 32865557Sjasone r = -ENOENT; 32965557Sjasone goto out; 33065557Sjasone } 33174903Sjhb 33245720Speter map = find_sid_vsid(vcpu, gvsid); 33374903Sjhb if (!map) 3344Srgrimes map = create_sid_map(vcpu, gvsid); 3354Srgrimes 3364Srgrimes map->guest_esid = esid; 3374Srgrimes 33874903Sjhb slb_vsid |= (map->host_vsid << 12); 3394Srgrimes slb_vsid &= ~SLB_VSID_KP; 34027567Sfsmp slb_esid |= slb_index; 34127567Sfsmp 3424Srgrimes#ifdef CONFIG_PPC_64K_PAGES 3434Srgrimes /* Set host segment base page size to 64K if possible */ 3444Srgrimes if (gvsid & VSID_64K) 34545720Speter slb_vsid |= mmu_psize_defs[MMU_PAGE_64K].sllp; 34645720Speter#endif 3474Srgrimes 34841591Sarchie svcpu->slb[slb_index].esid = slb_esid; 3495351Sbde svcpu->slb[slb_index].vsid = slb_vsid; 3505351Sbde 35141591Sarchie trace_kvm_book3s_slbmte(slb_vsid, slb_esid); 3525351Sbde 3534Srgrimesout: 3544Srgrimes svcpu_put(svcpu); 3554Srgrimes return r; 3564Srgrimes} 3574Srgrimes 3584Srgrimesvoid kvmppc_mmu_flush_segment(struct kvm_vcpu *vcpu, ulong ea, ulong seg_size) 3594Srgrimes{ 3604Srgrimes struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 3614Srgrimes ulong seg_mask = -seg_size; 3624Srgrimes int i; 3634Srgrimes 3644Srgrimes for (i = 0; i < svcpu->slb_max; i++) { 3654Srgrimes if ((svcpu->slb[i].esid & SLB_ESID_V) && 3664Srgrimes (svcpu->slb[i].esid & seg_mask) == ea) { 3674Srgrimes /* Invalidate this entry */ 3684Srgrimes svcpu->slb[i].esid = 0; 3694Srgrimes } 3704Srgrimes } 3714Srgrimes 3724Srgrimes svcpu_put(svcpu); 3734Srgrimes} 3744Srgrimes 3754Srgrimesvoid kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu) 3764Srgrimes{ 3774Srgrimes struct kvmppc_book3s_shadow_vcpu *svcpu = svcpu_get(vcpu); 3784Srgrimes svcpu->slb_max = 0; 3794Srgrimes svcpu->slb[0].esid = 0; 3804Srgrimes svcpu_put(svcpu); 3814Srgrimes} 3824Srgrimes 3834Srgrimesvoid kvmppc_mmu_destroy_pr(struct kvm_vcpu *vcpu) 3844Srgrimes{ 38527567Sfsmp kvmppc_mmu_hpte_destroy(vcpu); 38646555Speter __destroy_context(to_book3s(vcpu)->context_id[0]); 3876664Sbde} 38827567Sfsmp 38927567Sfsmpint kvmppc_mmu_init_pr(struct kvm_vcpu *vcpu) 39027567Sfsmp{ 39127567Sfsmp struct kvmppc_vcpu_book3s *vcpu3s = to_book3s(vcpu); 39245720Speter int err; 39345720Speter 39427567Sfsmp err = hash__alloc_context_id(); 39545720Speter if (err < 0) 39645720Speter return -1; 39727567Sfsmp vcpu3s->context_id[0] = err; 39827567Sfsmp 3996664Sbde vcpu3s->proto_vsid_max = ((u64)(vcpu3s->context_id[0] + 1) 4006664Sbde << ESID_BITS) - 1; 4016664Sbde vcpu3s->proto_vsid_first = (u64)vcpu3s->context_id[0] << ESID_BITS; 4025351Sbde vcpu3s->proto_vsid_next = vcpu3s->proto_vsid_first; 4034Srgrimes 4044Srgrimes kvmppc_mmu_hpte_init(vcpu); 4054Srgrimes 4064Srgrimes return 0; 4074Srgrimes} 4084Srgrimes