mmu_phyp.c revision 255643
1255643Snwhitehorn/* 2255643Snwhitehorn * Copyright (C) 2010 Andreas Tobler 3255643Snwhitehorn * All rights reserved. 4255643Snwhitehorn * 5255643Snwhitehorn * Redistribution and use in source and binary forms, with or without 6255643Snwhitehorn * modification, are permitted provided that the following conditions 7255643Snwhitehorn * are met: 8255643Snwhitehorn * 1. Redistributions of source code must retain the above copyright 9255643Snwhitehorn * notice, this list of conditions and the following disclaimer. 10255643Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 11255643Snwhitehorn * notice, this list of conditions and the following disclaimer in the 12255643Snwhitehorn * documentation and/or other materials provided with the distribution. 13255643Snwhitehorn * 14255643Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15255643Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16255643Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17255643Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18255643Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19255643Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20255643Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21255643Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22255643Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23255643Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24255643Snwhitehorn */ 25255643Snwhitehorn 26255643Snwhitehorn#include <sys/cdefs.h> 27255643Snwhitehorn__FBSDID("$FreeBSD: head/sys/powerpc/pseries/mmu_phyp.c 255643 2013-09-17 17:37:04Z nwhitehorn $"); 28255643Snwhitehorn 29255643Snwhitehorn#include <sys/param.h> 30255643Snwhitehorn#include <sys/kernel.h> 31255643Snwhitehorn#include <sys/ktr.h> 32255643Snwhitehorn#include <sys/lock.h> 33255643Snwhitehorn#include <sys/msgbuf.h> 34255643Snwhitehorn#include <sys/mutex.h> 35255643Snwhitehorn#include <sys/proc.h> 36255643Snwhitehorn#include <sys/sysctl.h> 37255643Snwhitehorn#include <sys/systm.h> 38255643Snwhitehorn#include <sys/vmmeter.h> 39255643Snwhitehorn 40255643Snwhitehorn#include <dev/ofw/openfirm.h> 41255643Snwhitehorn#include <machine/ofw_machdep.h> 42255643Snwhitehorn 43255643Snwhitehorn#include <vm/vm.h> 44255643Snwhitehorn#include <vm/vm_param.h> 45255643Snwhitehorn#include <vm/vm_kern.h> 46255643Snwhitehorn#include <vm/vm_page.h> 47255643Snwhitehorn#include <vm/vm_map.h> 48255643Snwhitehorn#include <vm/vm_object.h> 49255643Snwhitehorn#include <vm/vm_extern.h> 50255643Snwhitehorn#include <vm/vm_pageout.h> 51255643Snwhitehorn#include <vm/uma.h> 52255643Snwhitehorn 53255643Snwhitehorn#include <powerpc/aim/mmu_oea64.h> 54255643Snwhitehorn 55255643Snwhitehorn#include "mmu_if.h" 56255643Snwhitehorn#include "moea64_if.h" 57255643Snwhitehorn 58255643Snwhitehorn#include "phyp-hvcall.h" 59255643Snwhitehorn 60255643Snwhitehornextern int n_slbs; 61255643Snwhitehorn 62255643Snwhitehorn/* 63255643Snwhitehorn * Kernel MMU interface 64255643Snwhitehorn */ 65255643Snwhitehorn 66255643Snwhitehornstatic void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 67255643Snwhitehorn vm_offset_t kernelend); 68255643Snwhitehornstatic void mphyp_cpu_bootstrap(mmu_t mmup, int ap); 69255643Snwhitehornstatic void mphyp_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 70255643Snwhitehornstatic void mphyp_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 71255643Snwhitehorn uint64_t vpn, u_int64_t ptebit); 72255643Snwhitehornstatic void mphyp_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 73255643Snwhitehorn uint64_t vpn); 74255643Snwhitehornstatic void mphyp_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 75255643Snwhitehorn uint64_t vpn); 76255643Snwhitehornstatic int mphyp_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 77255643Snwhitehornstatic uintptr_t mphyp_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 78255643Snwhitehorn 79255643Snwhitehorn#define VSID_HASH_MASK 0x0000007fffffffffULL 80255643Snwhitehorn 81255643Snwhitehorn 82255643Snwhitehornstatic mmu_method_t mphyp_methods[] = { 83255643Snwhitehorn MMUMETHOD(mmu_bootstrap, mphyp_bootstrap), 84255643Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap), 85255643Snwhitehorn 86255643Snwhitehorn MMUMETHOD(moea64_pte_synch, mphyp_pte_synch), 87255643Snwhitehorn MMUMETHOD(moea64_pte_clear, mphyp_pte_clear), 88255643Snwhitehorn MMUMETHOD(moea64_pte_unset, mphyp_pte_unset), 89255643Snwhitehorn MMUMETHOD(moea64_pte_change, mphyp_pte_change), 90255643Snwhitehorn MMUMETHOD(moea64_pte_insert, mphyp_pte_insert), 91255643Snwhitehorn MMUMETHOD(moea64_pvo_to_pte, mphyp_pvo_to_pte), 92255643Snwhitehorn 93255643Snwhitehorn { 0, 0 } 94255643Snwhitehorn}; 95255643Snwhitehorn 96255643SnwhitehornMMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu); 97255643Snwhitehorn 98255643Snwhitehornstatic void 99255643Snwhitehornmphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 100255643Snwhitehorn{ 101255643Snwhitehorn uint64_t final_pteg_count = 0; 102255643Snwhitehorn char buf[8]; 103255643Snwhitehorn uint32_t prop[2]; 104255643Snwhitehorn uint32_t nptlp, shift = 0, slb_encoding = 0; 105255643Snwhitehorn phandle_t dev, node, root; 106255643Snwhitehorn int idx, len, res; 107255643Snwhitehorn 108255643Snwhitehorn moea64_early_bootstrap(mmup, kernelstart, kernelend); 109255643Snwhitehorn 110255643Snwhitehorn root = OF_peer(0); 111255643Snwhitehorn 112255643Snwhitehorn dev = OF_child(root); 113255643Snwhitehorn while (dev != 0) { 114255643Snwhitehorn res = OF_getprop(dev, "name", buf, sizeof(buf)); 115255643Snwhitehorn if (res > 0 && strcmp(buf, "cpus") == 0) 116255643Snwhitehorn break; 117255643Snwhitehorn dev = OF_peer(dev); 118255643Snwhitehorn } 119255643Snwhitehorn 120255643Snwhitehorn node = OF_child(dev); 121255643Snwhitehorn 122255643Snwhitehorn while (node != 0) { 123255643Snwhitehorn res = OF_getprop(node, "device_type", buf, sizeof(buf)); 124255643Snwhitehorn if (res > 0 && strcmp(buf, "cpu") == 0) 125255643Snwhitehorn break; 126255643Snwhitehorn node = OF_peer(node); 127255643Snwhitehorn } 128255643Snwhitehorn 129255643Snwhitehorn res = OF_getprop(node, "ibm,pft-size", prop, sizeof(prop)); 130255643Snwhitehorn if (res <= 0) 131255643Snwhitehorn panic("mmu_phyp: unknown PFT size"); 132255643Snwhitehorn final_pteg_count = 1 << prop[1]; 133255643Snwhitehorn res = OF_getprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 134255643Snwhitehorn if (res > 0) 135255643Snwhitehorn n_slbs = prop[0]; 136255643Snwhitehorn 137255643Snwhitehorn moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 138255643Snwhitehorn 139255643Snwhitehorn /* 140255643Snwhitehorn * Scan the large page size property for PAPR compatible machines. 141255643Snwhitehorn * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 142255643Snwhitehorn * for the encoding of the property. 143255643Snwhitehorn */ 144255643Snwhitehorn 145255643Snwhitehorn len = OF_getproplen(node, "ibm,segment-page-sizes"); 146255643Snwhitehorn if (len > 0) { 147255643Snwhitehorn /* 148255643Snwhitehorn * We have to use a variable length array on the stack 149255643Snwhitehorn * since we have very limited stack space. 150255643Snwhitehorn */ 151255643Snwhitehorn cell_t arr[len/sizeof(cell_t)]; 152255643Snwhitehorn res = OF_getprop(node, "ibm,segment-page-sizes", &arr, 153255643Snwhitehorn sizeof(arr)); 154255643Snwhitehorn len /= 4; 155255643Snwhitehorn idx = 0; 156255643Snwhitehorn while (len > 0) { 157255643Snwhitehorn shift = arr[idx]; 158255643Snwhitehorn slb_encoding = arr[idx + 1]; 159255643Snwhitehorn nptlp = arr[idx + 2]; 160255643Snwhitehorn idx += 3; 161255643Snwhitehorn len -= 3; 162255643Snwhitehorn while (len > 0 && nptlp) { 163255643Snwhitehorn idx += 2; 164255643Snwhitehorn len -= 2; 165255643Snwhitehorn nptlp--; 166255643Snwhitehorn } 167255643Snwhitehorn } 168255643Snwhitehorn moea64_large_page_shift = shift; 169255643Snwhitehorn moea64_large_page_size = 1 << shift; 170255643Snwhitehorn } 171255643Snwhitehorn 172255643Snwhitehorn moea64_mid_bootstrap(mmup, kernelstart, kernelend); 173255643Snwhitehorn moea64_late_bootstrap(mmup, kernelstart, kernelend); 174255643Snwhitehorn} 175255643Snwhitehorn 176255643Snwhitehornstatic void 177255643Snwhitehornmphyp_cpu_bootstrap(mmu_t mmup, int ap) 178255643Snwhitehorn{ 179255643Snwhitehorn struct slb *slb = PCPU_GET(slb); 180255643Snwhitehorn register_t seg0; 181255643Snwhitehorn int i; 182255643Snwhitehorn 183255643Snwhitehorn /* 184255643Snwhitehorn * Install kernel SLB entries 185255643Snwhitehorn */ 186255643Snwhitehorn 187255643Snwhitehorn __asm __volatile ("slbia"); 188255643Snwhitehorn __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 189255643Snwhitehorn for (i = 0; i < 64; i++) { 190255643Snwhitehorn if (!(slb[i].slbe & SLBE_VALID)) 191255643Snwhitehorn continue; 192255643Snwhitehorn 193255643Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 194255643Snwhitehorn "r"(slb[i].slbv), "r"(slb[i].slbe)); 195255643Snwhitehorn } 196255643Snwhitehorn} 197255643Snwhitehorn 198255643Snwhitehornstatic void 199255643Snwhitehornmphyp_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 200255643Snwhitehorn{ 201255643Snwhitehorn struct lpte pte; 202255643Snwhitehorn uint64_t junk; 203255643Snwhitehorn 204255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pte.pte_hi, &pte.pte_lo, 205255643Snwhitehorn &junk); 206255643Snwhitehorn 207255643Snwhitehorn pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 208255643Snwhitehorn} 209255643Snwhitehorn 210255643Snwhitehornstatic void 211255643Snwhitehornmphyp_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 212255643Snwhitehorn u_int64_t ptebit) 213255643Snwhitehorn{ 214255643Snwhitehorn 215255643Snwhitehorn if (ptebit & LPTE_CHG) 216255643Snwhitehorn phyp_hcall(H_CLEAR_MOD, 0, slot); 217255643Snwhitehorn if (ptebit & LPTE_REF) 218255643Snwhitehorn phyp_hcall(H_CLEAR_REF, 0, slot); 219255643Snwhitehorn} 220255643Snwhitehorn 221255643Snwhitehornstatic void 222255643Snwhitehornmphyp_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 223255643Snwhitehorn{ 224255643Snwhitehorn 225255643Snwhitehorn /* XXX: last argument can check the VPN -- set flag to enable */ 226255643Snwhitehorn phyp_hcall(H_REMOVE, 0, slot, vpn); 227255643Snwhitehorn} 228255643Snwhitehorn 229255643Snwhitehornstatic void 230255643Snwhitehornmphyp_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 231255643Snwhitehorn{ 232255643Snwhitehorn struct lpte evicted; 233255643Snwhitehorn uint64_t index, junk; 234255643Snwhitehorn int64_t result; 235255643Snwhitehorn 236255643Snwhitehorn /* 237255643Snwhitehorn * NB: this is protected by the global table lock, so this two-step 238255643Snwhitehorn * is safe, except for the scratch-page case. No CPUs on which we run 239255643Snwhitehorn * this code should be using scratch pages. 240255643Snwhitehorn */ 241255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 242255643Snwhitehorn ("Locked pages not supported on PHYP")); 243255643Snwhitehorn 244255643Snwhitehorn /* XXX: optimization using H_PROTECT for common case? */ 245255643Snwhitehorn result = phyp_hcall(H_REMOVE, 0, slot, vpn); 246255643Snwhitehorn if (result != H_SUCCESS) 247255643Snwhitehorn panic("mphyp_pte_change() invalidation failure: %ld\n", result); 248255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, slot, pvo_pt->pte_hi, 249255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 250255643Snwhitehorn if (result != H_SUCCESS) 251255643Snwhitehorn panic("mphyp_pte_change() insertion failure: %ld\n", result); 252255643Snwhitehorn} 253255643Snwhitehorn 254255643Snwhitehornstatic __inline int 255255643Snwhitehornmphyp_pte_spillable_ident(u_int ptegidx, struct lpte *to_evict) 256255643Snwhitehorn{ 257255643Snwhitehorn uint64_t slot, junk, k; 258255643Snwhitehorn struct lpte pt; 259255643Snwhitehorn int i, j; 260255643Snwhitehorn 261255643Snwhitehorn /* Start at a random slot */ 262255643Snwhitehorn i = mftb() % 8; 263255643Snwhitehorn k = -1; 264255643Snwhitehorn for (j = 0; j < 8; j++) { 265255643Snwhitehorn slot = (ptegidx << 3) + (i + j) % 8; 266255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, &pt.pte_lo, 267255643Snwhitehorn &junk); 268255643Snwhitehorn 269255643Snwhitehorn if (pt.pte_hi & LPTE_SWBITS) 270255643Snwhitehorn continue; 271255643Snwhitehorn 272255643Snwhitehorn /* This is a candidate, so remember it */ 273255643Snwhitehorn k = slot; 274255643Snwhitehorn 275255643Snwhitehorn /* Try to get a page that has not been used lately */ 276255643Snwhitehorn if (!(pt.pte_lo & LPTE_REF)) { 277255643Snwhitehorn memcpy(to_evict, &pt, sizeof(struct lpte)); 278255643Snwhitehorn return (k); 279255643Snwhitehorn } 280255643Snwhitehorn } 281255643Snwhitehorn 282255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &to_evict->pte_hi, 283255643Snwhitehorn &to_evict->pte_lo, &junk); 284255643Snwhitehorn return (k); 285255643Snwhitehorn} 286255643Snwhitehorn 287255643Snwhitehornstatic int 288255643Snwhitehornmphyp_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 289255643Snwhitehorn{ 290255643Snwhitehorn int64_t result; 291255643Snwhitehorn struct lpte evicted; 292255643Snwhitehorn struct pvo_entry *pvo; 293255643Snwhitehorn uint64_t index, junk; 294255643Snwhitehorn u_int pteg_bktidx; 295255643Snwhitehorn 296255643Snwhitehorn /* Check for locked pages, which we can't support on this system */ 297255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 298255643Snwhitehorn ("Locked pages not supported on PHYP")); 299255643Snwhitehorn 300255643Snwhitehorn /* Initialize PTE */ 301255643Snwhitehorn pvo_pt->pte_hi |= LPTE_VALID; 302255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 303255643Snwhitehorn evicted.pte_hi = 0; 304255643Snwhitehorn 305255643Snwhitehorn /* 306255643Snwhitehorn * First try primary hash. 307255643Snwhitehorn */ 308255643Snwhitehorn pteg_bktidx = ptegidx; 309255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, pvo_pt->pte_hi, 310255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 311255643Snwhitehorn if (result == H_SUCCESS) 312255643Snwhitehorn return (index & 0x07); 313255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 314255643Snwhitehorn "(ptegidx: %#x/%#x, PTE %#lx/%#lx", result, ptegidx, 315255643Snwhitehorn moea64_pteg_count, pvo_pt->pte_hi, pvo_pt->pte_lo)); 316255643Snwhitehorn 317255643Snwhitehorn /* 318255643Snwhitehorn * Next try secondary hash. 319255643Snwhitehorn */ 320255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 321255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 322255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, 323255643Snwhitehorn pvo_pt->pte_hi, pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 324255643Snwhitehorn if (result == H_SUCCESS) 325255643Snwhitehorn return (index & 0x07); 326255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 327255643Snwhitehorn result)); 328255643Snwhitehorn 329255643Snwhitehorn /* 330255643Snwhitehorn * Out of luck. Find a PTE to sacrifice. 331255643Snwhitehorn */ 332255643Snwhitehorn pteg_bktidx = ptegidx; 333255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 334255643Snwhitehorn if (index == -1L) { 335255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 336255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 337255643Snwhitehorn } 338255643Snwhitehorn 339255643Snwhitehorn if (index == -1L) { 340255643Snwhitehorn /* No freeable slots in either PTEG? We're hosed. */ 341255643Snwhitehorn panic("mphyp_pte_insert: overflow"); 342255643Snwhitehorn return (-1); 343255643Snwhitehorn } 344255643Snwhitehorn 345255643Snwhitehorn if (pteg_bktidx == ptegidx) 346255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 347255643Snwhitehorn else 348255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 349255643Snwhitehorn 350255643Snwhitehorn /* 351255643Snwhitehorn * Synchronize the sacrifice PTE with its PVO, then mark both 352255643Snwhitehorn * invalid. The PVO will be reused when/if the VM system comes 353255643Snwhitehorn * here after a fault. 354255643Snwhitehorn */ 355255643Snwhitehorn 356255643Snwhitehorn if (evicted.pte_hi & LPTE_HID) 357255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 358255643Snwhitehorn 359255643Snwhitehorn LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { 360255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) { 361255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 362255643Snwhitehorn ("Invalid PVO for valid PTE!")); 363255643Snwhitehorn phyp_hcall(H_REMOVE, 0, index, 0); 364255643Snwhitehorn PVO_PTEGIDX_CLR(pvo); 365255643Snwhitehorn moea64_pte_overflow++; 366255643Snwhitehorn break; 367255643Snwhitehorn } 368255643Snwhitehorn } 369255643Snwhitehorn 370255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi, 371255643Snwhitehorn ("Unable to find PVO for spilled PTE")); 372255643Snwhitehorn 373255643Snwhitehorn /* 374255643Snwhitehorn * Set the new PTE. 375255643Snwhitehorn */ 376255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pvo_pt->pte_hi, 377255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 378255643Snwhitehorn if (result == H_SUCCESS) 379255643Snwhitehorn return (index & 0x07); 380255643Snwhitehorn 381255643Snwhitehorn panic("Page replacement error: %ld", result); 382255643Snwhitehorn return (-1); 383255643Snwhitehorn} 384255643Snwhitehorn 385255643Snwhitehornstatic __inline u_int 386255643Snwhitehornva_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 387255643Snwhitehorn{ 388255643Snwhitehorn uint64_t hash; 389255643Snwhitehorn int shift; 390255643Snwhitehorn 391255643Snwhitehorn shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 392255643Snwhitehorn hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 393255643Snwhitehorn shift); 394255643Snwhitehorn return (hash & moea64_pteg_mask); 395255643Snwhitehorn} 396255643Snwhitehorn 397255643Snwhitehornstatic uintptr_t 398255643Snwhitehornmphyp_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 399255643Snwhitehorn{ 400255643Snwhitehorn uint64_t vsid; 401255643Snwhitehorn u_int ptegidx; 402255643Snwhitehorn 403255643Snwhitehorn /* If the PTEG index is not set, then there is no page table entry */ 404255643Snwhitehorn if (!PVO_PTEGIDX_ISSET(pvo)) 405255643Snwhitehorn return (-1); 406255643Snwhitehorn 407255643Snwhitehorn vsid = PVO_VSID(pvo); 408255643Snwhitehorn ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 409255643Snwhitehorn 410255643Snwhitehorn /* 411255643Snwhitehorn * We can find the actual pte entry without searching by grabbing 412255643Snwhitehorn * the PTEG index from 3 unused bits in pvo_vaddr and by 413255643Snwhitehorn * noticing the HID bit. 414255643Snwhitehorn */ 415255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 416255643Snwhitehorn ptegidx ^= moea64_pteg_mask; 417255643Snwhitehorn 418255643Snwhitehorn return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 419255643Snwhitehorn} 420255643Snwhitehorn 421