1255643Snwhitehorn/* 2255643Snwhitehorn * Copyright (C) 2010 Andreas Tobler 3255643Snwhitehorn * All rights reserved. 4255643Snwhitehorn * 5255643Snwhitehorn * Redistribution and use in source and binary forms, with or without 6255643Snwhitehorn * modification, are permitted provided that the following conditions 7255643Snwhitehorn * are met: 8255643Snwhitehorn * 1. Redistributions of source code must retain the above copyright 9255643Snwhitehorn * notice, this list of conditions and the following disclaimer. 10255643Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 11255643Snwhitehorn * notice, this list of conditions and the following disclaimer in the 12255643Snwhitehorn * documentation and/or other materials provided with the distribution. 13255643Snwhitehorn * 14255643Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15255643Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16255643Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17255643Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18255643Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19255643Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20255643Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21255643Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22255643Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23255643Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24255643Snwhitehorn */ 25255643Snwhitehorn 26255643Snwhitehorn#include <sys/cdefs.h> 27255643Snwhitehorn__FBSDID("$FreeBSD$"); 28255643Snwhitehorn 29255643Snwhitehorn#include <sys/param.h> 30255643Snwhitehorn#include <sys/kernel.h> 31255643Snwhitehorn#include <sys/ktr.h> 32255643Snwhitehorn#include <sys/lock.h> 33255643Snwhitehorn#include <sys/msgbuf.h> 34255643Snwhitehorn#include <sys/mutex.h> 35255643Snwhitehorn#include <sys/proc.h> 36255643Snwhitehorn#include <sys/sysctl.h> 37255643Snwhitehorn#include <sys/systm.h> 38255643Snwhitehorn#include <sys/vmmeter.h> 39255643Snwhitehorn 40255643Snwhitehorn#include <dev/ofw/openfirm.h> 41255643Snwhitehorn#include <machine/ofw_machdep.h> 42255643Snwhitehorn 43255643Snwhitehorn#include <vm/vm.h> 44255643Snwhitehorn#include <vm/vm_param.h> 45255643Snwhitehorn#include <vm/vm_kern.h> 46255643Snwhitehorn#include <vm/vm_page.h> 47255643Snwhitehorn#include <vm/vm_map.h> 48255643Snwhitehorn#include <vm/vm_object.h> 49255643Snwhitehorn#include <vm/vm_extern.h> 50255643Snwhitehorn#include <vm/vm_pageout.h> 51255643Snwhitehorn#include <vm/uma.h> 52255643Snwhitehorn 53255643Snwhitehorn#include <powerpc/aim/mmu_oea64.h> 54255643Snwhitehorn 55255643Snwhitehorn#include "mmu_if.h" 56255643Snwhitehorn#include "moea64_if.h" 57255643Snwhitehorn 58255643Snwhitehorn#include "phyp-hvcall.h" 59255643Snwhitehorn 60255643Snwhitehornextern int n_slbs; 61255643Snwhitehorn 62255643Snwhitehorn/* 63255643Snwhitehorn * Kernel MMU interface 64255643Snwhitehorn */ 65255643Snwhitehorn 66255643Snwhitehornstatic void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 67255643Snwhitehorn vm_offset_t kernelend); 68255643Snwhitehornstatic void mphyp_cpu_bootstrap(mmu_t mmup, int ap); 69255643Snwhitehornstatic void mphyp_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 70255643Snwhitehornstatic void mphyp_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 71255643Snwhitehorn uint64_t vpn, u_int64_t ptebit); 72255643Snwhitehornstatic void mphyp_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 73255643Snwhitehorn uint64_t vpn); 74255643Snwhitehornstatic void mphyp_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 75255643Snwhitehorn uint64_t vpn); 76255643Snwhitehornstatic int mphyp_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 77255643Snwhitehornstatic uintptr_t mphyp_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 78255643Snwhitehorn 79255643Snwhitehorn#define VSID_HASH_MASK 0x0000007fffffffffULL 80255643Snwhitehorn 81255643Snwhitehorn 82255643Snwhitehornstatic mmu_method_t mphyp_methods[] = { 83255643Snwhitehorn MMUMETHOD(mmu_bootstrap, mphyp_bootstrap), 84255643Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap), 85255643Snwhitehorn 86255643Snwhitehorn MMUMETHOD(moea64_pte_synch, mphyp_pte_synch), 87255643Snwhitehorn MMUMETHOD(moea64_pte_clear, mphyp_pte_clear), 88255643Snwhitehorn MMUMETHOD(moea64_pte_unset, mphyp_pte_unset), 89255643Snwhitehorn MMUMETHOD(moea64_pte_change, mphyp_pte_change), 90255643Snwhitehorn MMUMETHOD(moea64_pte_insert, mphyp_pte_insert), 91255643Snwhitehorn MMUMETHOD(moea64_pvo_to_pte, mphyp_pvo_to_pte), 92255643Snwhitehorn 93255643Snwhitehorn { 0, 0 } 94255643Snwhitehorn}; 95255643Snwhitehorn 96255643SnwhitehornMMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu); 97255643Snwhitehorn 98255643Snwhitehornstatic void 99255643Snwhitehornmphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 100255643Snwhitehorn{ 101255643Snwhitehorn uint64_t final_pteg_count = 0; 102255643Snwhitehorn char buf[8]; 103255643Snwhitehorn uint32_t prop[2]; 104255643Snwhitehorn uint32_t nptlp, shift = 0, slb_encoding = 0; 105255643Snwhitehorn phandle_t dev, node, root; 106255643Snwhitehorn int idx, len, res; 107255643Snwhitehorn 108255643Snwhitehorn moea64_early_bootstrap(mmup, kernelstart, kernelend); 109255643Snwhitehorn 110255643Snwhitehorn root = OF_peer(0); 111255643Snwhitehorn 112255643Snwhitehorn dev = OF_child(root); 113255643Snwhitehorn while (dev != 0) { 114255643Snwhitehorn res = OF_getprop(dev, "name", buf, sizeof(buf)); 115255643Snwhitehorn if (res > 0 && strcmp(buf, "cpus") == 0) 116255643Snwhitehorn break; 117255643Snwhitehorn dev = OF_peer(dev); 118255643Snwhitehorn } 119255643Snwhitehorn 120255643Snwhitehorn node = OF_child(dev); 121255643Snwhitehorn 122255643Snwhitehorn while (node != 0) { 123255643Snwhitehorn res = OF_getprop(node, "device_type", buf, sizeof(buf)); 124255643Snwhitehorn if (res > 0 && strcmp(buf, "cpu") == 0) 125255643Snwhitehorn break; 126255643Snwhitehorn node = OF_peer(node); 127255643Snwhitehorn } 128255643Snwhitehorn 129255643Snwhitehorn res = OF_getprop(node, "ibm,pft-size", prop, sizeof(prop)); 130255643Snwhitehorn if (res <= 0) 131255643Snwhitehorn panic("mmu_phyp: unknown PFT size"); 132255643Snwhitehorn final_pteg_count = 1 << prop[1]; 133255643Snwhitehorn res = OF_getprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 134255643Snwhitehorn if (res > 0) 135255643Snwhitehorn n_slbs = prop[0]; 136255643Snwhitehorn 137255643Snwhitehorn moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 138255643Snwhitehorn 139255643Snwhitehorn /* 140255643Snwhitehorn * Scan the large page size property for PAPR compatible machines. 141255643Snwhitehorn * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 142255643Snwhitehorn * for the encoding of the property. 143255643Snwhitehorn */ 144255643Snwhitehorn 145255643Snwhitehorn len = OF_getproplen(node, "ibm,segment-page-sizes"); 146255643Snwhitehorn if (len > 0) { 147255643Snwhitehorn /* 148255643Snwhitehorn * We have to use a variable length array on the stack 149255643Snwhitehorn * since we have very limited stack space. 150255643Snwhitehorn */ 151255643Snwhitehorn cell_t arr[len/sizeof(cell_t)]; 152255643Snwhitehorn res = OF_getprop(node, "ibm,segment-page-sizes", &arr, 153255643Snwhitehorn sizeof(arr)); 154255643Snwhitehorn len /= 4; 155255643Snwhitehorn idx = 0; 156255643Snwhitehorn while (len > 0) { 157255643Snwhitehorn shift = arr[idx]; 158255643Snwhitehorn slb_encoding = arr[idx + 1]; 159255643Snwhitehorn nptlp = arr[idx + 2]; 160255643Snwhitehorn idx += 3; 161255643Snwhitehorn len -= 3; 162255643Snwhitehorn while (len > 0 && nptlp) { 163255643Snwhitehorn idx += 2; 164255643Snwhitehorn len -= 2; 165255643Snwhitehorn nptlp--; 166255643Snwhitehorn } 167255643Snwhitehorn } 168255643Snwhitehorn moea64_large_page_shift = shift; 169255643Snwhitehorn moea64_large_page_size = 1 << shift; 170255643Snwhitehorn } 171255643Snwhitehorn 172255643Snwhitehorn moea64_mid_bootstrap(mmup, kernelstart, kernelend); 173255643Snwhitehorn moea64_late_bootstrap(mmup, kernelstart, kernelend); 174255643Snwhitehorn} 175255643Snwhitehorn 176255643Snwhitehornstatic void 177255643Snwhitehornmphyp_cpu_bootstrap(mmu_t mmup, int ap) 178255643Snwhitehorn{ 179255643Snwhitehorn struct slb *slb = PCPU_GET(slb); 180255643Snwhitehorn register_t seg0; 181255643Snwhitehorn int i; 182255643Snwhitehorn 183255643Snwhitehorn /* 184255643Snwhitehorn * Install kernel SLB entries 185255643Snwhitehorn */ 186255643Snwhitehorn 187255643Snwhitehorn __asm __volatile ("slbia"); 188255643Snwhitehorn __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 189255643Snwhitehorn for (i = 0; i < 64; i++) { 190255643Snwhitehorn if (!(slb[i].slbe & SLBE_VALID)) 191255643Snwhitehorn continue; 192255643Snwhitehorn 193255643Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 194255643Snwhitehorn "r"(slb[i].slbv), "r"(slb[i].slbe)); 195255643Snwhitehorn } 196255643Snwhitehorn} 197255643Snwhitehorn 198255643Snwhitehornstatic void 199255643Snwhitehornmphyp_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 200255643Snwhitehorn{ 201255643Snwhitehorn struct lpte pte; 202255643Snwhitehorn uint64_t junk; 203255643Snwhitehorn 204255909Snwhitehorn __asm __volatile("ptesync"); 205255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pte.pte_hi, &pte.pte_lo, 206255643Snwhitehorn &junk); 207255643Snwhitehorn 208255643Snwhitehorn pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 209255643Snwhitehorn} 210255643Snwhitehorn 211255643Snwhitehornstatic void 212255643Snwhitehornmphyp_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 213255643Snwhitehorn u_int64_t ptebit) 214255643Snwhitehorn{ 215255643Snwhitehorn 216255643Snwhitehorn if (ptebit & LPTE_CHG) 217255643Snwhitehorn phyp_hcall(H_CLEAR_MOD, 0, slot); 218255643Snwhitehorn if (ptebit & LPTE_REF) 219255643Snwhitehorn phyp_hcall(H_CLEAR_REF, 0, slot); 220255643Snwhitehorn} 221255643Snwhitehorn 222255643Snwhitehornstatic void 223255643Snwhitehornmphyp_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 224255643Snwhitehorn{ 225255909Snwhitehorn struct lpte pte; 226255909Snwhitehorn uint64_t junk; 227255909Snwhitehorn int err; 228255643Snwhitehorn 229255909Snwhitehorn err = phyp_pft_hcall(H_REMOVE, 1UL << 31, slot, 230255909Snwhitehorn pvo_pt->pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 231255909Snwhitehorn &junk); 232255909Snwhitehorn KASSERT(err == H_SUCCESS, ("Error removing page: %d", err)); 233255909Snwhitehorn 234255909Snwhitehorn pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 235255643Snwhitehorn} 236255643Snwhitehorn 237255643Snwhitehornstatic void 238255643Snwhitehornmphyp_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 239255643Snwhitehorn{ 240255643Snwhitehorn struct lpte evicted; 241255643Snwhitehorn uint64_t index, junk; 242255643Snwhitehorn int64_t result; 243255643Snwhitehorn 244255643Snwhitehorn /* 245255643Snwhitehorn * NB: this is protected by the global table lock, so this two-step 246255643Snwhitehorn * is safe, except for the scratch-page case. No CPUs on which we run 247255643Snwhitehorn * this code should be using scratch pages. 248255643Snwhitehorn */ 249255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 250255643Snwhitehorn ("Locked pages not supported on PHYP")); 251255643Snwhitehorn 252255643Snwhitehorn /* XXX: optimization using H_PROTECT for common case? */ 253255909Snwhitehorn mphyp_pte_unset(mmu, slot, pvo_pt, vpn); 254255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, slot, pvo_pt->pte_hi, 255255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 256255643Snwhitehorn if (result != H_SUCCESS) 257255643Snwhitehorn panic("mphyp_pte_change() insertion failure: %ld\n", result); 258255643Snwhitehorn} 259255643Snwhitehorn 260255643Snwhitehornstatic __inline int 261255643Snwhitehornmphyp_pte_spillable_ident(u_int ptegidx, struct lpte *to_evict) 262255643Snwhitehorn{ 263255643Snwhitehorn uint64_t slot, junk, k; 264255643Snwhitehorn struct lpte pt; 265255643Snwhitehorn int i, j; 266255643Snwhitehorn 267255643Snwhitehorn /* Start at a random slot */ 268255643Snwhitehorn i = mftb() % 8; 269255643Snwhitehorn k = -1; 270255643Snwhitehorn for (j = 0; j < 8; j++) { 271255643Snwhitehorn slot = (ptegidx << 3) + (i + j) % 8; 272255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, &pt.pte_lo, 273255643Snwhitehorn &junk); 274255643Snwhitehorn 275255643Snwhitehorn if (pt.pte_hi & LPTE_SWBITS) 276255643Snwhitehorn continue; 277255643Snwhitehorn 278255643Snwhitehorn /* This is a candidate, so remember it */ 279255643Snwhitehorn k = slot; 280255643Snwhitehorn 281255643Snwhitehorn /* Try to get a page that has not been used lately */ 282255643Snwhitehorn if (!(pt.pte_lo & LPTE_REF)) { 283255643Snwhitehorn memcpy(to_evict, &pt, sizeof(struct lpte)); 284255643Snwhitehorn return (k); 285255643Snwhitehorn } 286255643Snwhitehorn } 287255643Snwhitehorn 288255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &to_evict->pte_hi, 289255643Snwhitehorn &to_evict->pte_lo, &junk); 290255643Snwhitehorn return (k); 291255643Snwhitehorn} 292255643Snwhitehorn 293255643Snwhitehornstatic int 294255643Snwhitehornmphyp_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 295255643Snwhitehorn{ 296255643Snwhitehorn int64_t result; 297255643Snwhitehorn struct lpte evicted; 298255643Snwhitehorn struct pvo_entry *pvo; 299255643Snwhitehorn uint64_t index, junk; 300255643Snwhitehorn u_int pteg_bktidx; 301255643Snwhitehorn 302255643Snwhitehorn /* Check for locked pages, which we can't support on this system */ 303255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 304255643Snwhitehorn ("Locked pages not supported on PHYP")); 305255643Snwhitehorn 306255643Snwhitehorn /* Initialize PTE */ 307255643Snwhitehorn pvo_pt->pte_hi |= LPTE_VALID; 308255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 309255643Snwhitehorn evicted.pte_hi = 0; 310255643Snwhitehorn 311255643Snwhitehorn /* 312255643Snwhitehorn * First try primary hash. 313255643Snwhitehorn */ 314255643Snwhitehorn pteg_bktidx = ptegidx; 315255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, pvo_pt->pte_hi, 316255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 317255643Snwhitehorn if (result == H_SUCCESS) 318255643Snwhitehorn return (index & 0x07); 319255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 320255643Snwhitehorn "(ptegidx: %#x/%#x, PTE %#lx/%#lx", result, ptegidx, 321255643Snwhitehorn moea64_pteg_count, pvo_pt->pte_hi, pvo_pt->pte_lo)); 322255643Snwhitehorn 323255643Snwhitehorn /* 324255643Snwhitehorn * Next try secondary hash. 325255643Snwhitehorn */ 326255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 327255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 328255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, 329255643Snwhitehorn pvo_pt->pte_hi, pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 330255643Snwhitehorn if (result == H_SUCCESS) 331255643Snwhitehorn return (index & 0x07); 332255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 333255643Snwhitehorn result)); 334255643Snwhitehorn 335255643Snwhitehorn /* 336255643Snwhitehorn * Out of luck. Find a PTE to sacrifice. 337255643Snwhitehorn */ 338255643Snwhitehorn pteg_bktidx = ptegidx; 339255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 340255643Snwhitehorn if (index == -1L) { 341255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 342255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 343255643Snwhitehorn } 344255643Snwhitehorn 345255643Snwhitehorn if (index == -1L) { 346255643Snwhitehorn /* No freeable slots in either PTEG? We're hosed. */ 347255643Snwhitehorn panic("mphyp_pte_insert: overflow"); 348255643Snwhitehorn return (-1); 349255643Snwhitehorn } 350255643Snwhitehorn 351255643Snwhitehorn if (pteg_bktidx == ptegidx) 352255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 353255643Snwhitehorn else 354255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 355255643Snwhitehorn 356255643Snwhitehorn /* 357255643Snwhitehorn * Synchronize the sacrifice PTE with its PVO, then mark both 358255643Snwhitehorn * invalid. The PVO will be reused when/if the VM system comes 359255643Snwhitehorn * here after a fault. 360255643Snwhitehorn */ 361255643Snwhitehorn 362255643Snwhitehorn if (evicted.pte_hi & LPTE_HID) 363255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 364255643Snwhitehorn 365255643Snwhitehorn LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { 366255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) { 367255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 368255643Snwhitehorn ("Invalid PVO for valid PTE!")); 369255909Snwhitehorn mphyp_pte_unset(mmu, index, &pvo->pvo_pte.lpte, 370255909Snwhitehorn pvo->pvo_vpn); 371255643Snwhitehorn PVO_PTEGIDX_CLR(pvo); 372255643Snwhitehorn moea64_pte_overflow++; 373255643Snwhitehorn break; 374255643Snwhitehorn } 375255643Snwhitehorn } 376255643Snwhitehorn 377255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi, 378255643Snwhitehorn ("Unable to find PVO for spilled PTE")); 379255643Snwhitehorn 380255643Snwhitehorn /* 381255643Snwhitehorn * Set the new PTE. 382255643Snwhitehorn */ 383255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pvo_pt->pte_hi, 384255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 385255643Snwhitehorn if (result == H_SUCCESS) 386255643Snwhitehorn return (index & 0x07); 387255643Snwhitehorn 388255643Snwhitehorn panic("Page replacement error: %ld", result); 389255643Snwhitehorn return (-1); 390255643Snwhitehorn} 391255643Snwhitehorn 392255643Snwhitehornstatic __inline u_int 393255643Snwhitehornva_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 394255643Snwhitehorn{ 395255643Snwhitehorn uint64_t hash; 396255643Snwhitehorn int shift; 397255643Snwhitehorn 398255643Snwhitehorn shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 399255643Snwhitehorn hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 400255643Snwhitehorn shift); 401255643Snwhitehorn return (hash & moea64_pteg_mask); 402255643Snwhitehorn} 403255643Snwhitehorn 404255643Snwhitehornstatic uintptr_t 405255643Snwhitehornmphyp_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 406255643Snwhitehorn{ 407255643Snwhitehorn uint64_t vsid; 408255643Snwhitehorn u_int ptegidx; 409255643Snwhitehorn 410255643Snwhitehorn /* If the PTEG index is not set, then there is no page table entry */ 411255643Snwhitehorn if (!PVO_PTEGIDX_ISSET(pvo)) 412255643Snwhitehorn return (-1); 413255643Snwhitehorn 414255643Snwhitehorn vsid = PVO_VSID(pvo); 415255643Snwhitehorn ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 416255643Snwhitehorn 417255643Snwhitehorn /* 418255643Snwhitehorn * We can find the actual pte entry without searching by grabbing 419255643Snwhitehorn * the PTEG index from 3 unused bits in pvo_vaddr and by 420255643Snwhitehorn * noticing the HID bit. 421255643Snwhitehorn */ 422255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 423255643Snwhitehorn ptegidx ^= moea64_pteg_mask; 424255643Snwhitehorn 425255643Snwhitehorn return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 426255643Snwhitehorn} 427255643Snwhitehorn 428