1255643Snwhitehorn/* 2255643Snwhitehorn * Copyright (C) 2010 Andreas Tobler 3255643Snwhitehorn * All rights reserved. 4255643Snwhitehorn * 5255643Snwhitehorn * Redistribution and use in source and binary forms, with or without 6255643Snwhitehorn * modification, are permitted provided that the following conditions 7255643Snwhitehorn * are met: 8255643Snwhitehorn * 1. Redistributions of source code must retain the above copyright 9255643Snwhitehorn * notice, this list of conditions and the following disclaimer. 10255643Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 11255643Snwhitehorn * notice, this list of conditions and the following disclaimer in the 12255643Snwhitehorn * documentation and/or other materials provided with the distribution. 13255643Snwhitehorn * 14255643Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15255643Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16255643Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17255643Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18255643Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19255643Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20255643Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21255643Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22255643Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23255643Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24255643Snwhitehorn */ 25255643Snwhitehorn 26255643Snwhitehorn#include <sys/cdefs.h> 27255643Snwhitehorn__FBSDID("$FreeBSD$"); 28255643Snwhitehorn 29255643Snwhitehorn#include <sys/param.h> 30255643Snwhitehorn#include <sys/kernel.h> 31255643Snwhitehorn#include <sys/ktr.h> 32255643Snwhitehorn#include <sys/lock.h> 33255643Snwhitehorn#include <sys/msgbuf.h> 34255643Snwhitehorn#include <sys/mutex.h> 35255643Snwhitehorn#include <sys/proc.h> 36255643Snwhitehorn#include <sys/sysctl.h> 37255643Snwhitehorn#include <sys/systm.h> 38255643Snwhitehorn#include <sys/vmmeter.h> 39255643Snwhitehorn 40255643Snwhitehorn#include <dev/ofw/openfirm.h> 41255643Snwhitehorn#include <machine/ofw_machdep.h> 42255643Snwhitehorn 43255643Snwhitehorn#include <vm/vm.h> 44255643Snwhitehorn#include <vm/vm_param.h> 45255643Snwhitehorn#include <vm/vm_kern.h> 46255643Snwhitehorn#include <vm/vm_page.h> 47255643Snwhitehorn#include <vm/vm_map.h> 48255643Snwhitehorn#include <vm/vm_object.h> 49255643Snwhitehorn#include <vm/vm_extern.h> 50255643Snwhitehorn#include <vm/vm_pageout.h> 51255643Snwhitehorn#include <vm/uma.h> 52255643Snwhitehorn 53255643Snwhitehorn#include <powerpc/aim/mmu_oea64.h> 54255643Snwhitehorn 55255643Snwhitehorn#include "mmu_if.h" 56255643Snwhitehorn#include "moea64_if.h" 57255643Snwhitehorn 58255643Snwhitehorn#include "phyp-hvcall.h" 59255643Snwhitehorn 60255643Snwhitehornextern int n_slbs; 61255643Snwhitehorn 62255643Snwhitehorn/* 63255643Snwhitehorn * Kernel MMU interface 64255643Snwhitehorn */ 65255643Snwhitehorn 66255643Snwhitehornstatic void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 67255643Snwhitehorn vm_offset_t kernelend); 68255643Snwhitehornstatic void mphyp_cpu_bootstrap(mmu_t mmup, int ap); 69255643Snwhitehornstatic void mphyp_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 70255643Snwhitehornstatic void mphyp_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 71255643Snwhitehorn uint64_t vpn, u_int64_t ptebit); 72255643Snwhitehornstatic void mphyp_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 73255643Snwhitehorn uint64_t vpn); 74255643Snwhitehornstatic void mphyp_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 75255643Snwhitehorn uint64_t vpn); 76255643Snwhitehornstatic int mphyp_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 77255643Snwhitehornstatic uintptr_t mphyp_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 78255643Snwhitehorn 79255643Snwhitehorn#define VSID_HASH_MASK 0x0000007fffffffffULL 80255643Snwhitehorn 81255643Snwhitehorn 82255643Snwhitehornstatic mmu_method_t mphyp_methods[] = { 83255643Snwhitehorn MMUMETHOD(mmu_bootstrap, mphyp_bootstrap), 84255643Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap), 85255643Snwhitehorn 86255643Snwhitehorn MMUMETHOD(moea64_pte_synch, mphyp_pte_synch), 87255643Snwhitehorn MMUMETHOD(moea64_pte_clear, mphyp_pte_clear), 88255643Snwhitehorn MMUMETHOD(moea64_pte_unset, mphyp_pte_unset), 89255643Snwhitehorn MMUMETHOD(moea64_pte_change, mphyp_pte_change), 90255643Snwhitehorn MMUMETHOD(moea64_pte_insert, mphyp_pte_insert), 91255643Snwhitehorn MMUMETHOD(moea64_pvo_to_pte, mphyp_pvo_to_pte), 92255643Snwhitehorn 93255643Snwhitehorn { 0, 0 } 94255643Snwhitehorn}; 95255643Snwhitehorn 96255643SnwhitehornMMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu); 97255643Snwhitehorn 98255643Snwhitehornstatic void 99255643Snwhitehornmphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 100255643Snwhitehorn{ 101255643Snwhitehorn uint64_t final_pteg_count = 0; 102255643Snwhitehorn char buf[8]; 103255643Snwhitehorn uint32_t prop[2]; 104255643Snwhitehorn uint32_t nptlp, shift = 0, slb_encoding = 0; 105255643Snwhitehorn phandle_t dev, node, root; 106255643Snwhitehorn int idx, len, res; 107255643Snwhitehorn 108255643Snwhitehorn moea64_early_bootstrap(mmup, kernelstart, kernelend); 109255643Snwhitehorn 110255643Snwhitehorn root = OF_peer(0); 111255643Snwhitehorn 112255643Snwhitehorn dev = OF_child(root); 113255643Snwhitehorn while (dev != 0) { 114255643Snwhitehorn res = OF_getprop(dev, "name", buf, sizeof(buf)); 115255643Snwhitehorn if (res > 0 && strcmp(buf, "cpus") == 0) 116255643Snwhitehorn break; 117255643Snwhitehorn dev = OF_peer(dev); 118255643Snwhitehorn } 119255643Snwhitehorn 120255643Snwhitehorn node = OF_child(dev); 121255643Snwhitehorn 122255643Snwhitehorn while (node != 0) { 123255643Snwhitehorn res = OF_getprop(node, "device_type", buf, sizeof(buf)); 124255643Snwhitehorn if (res > 0 && strcmp(buf, "cpu") == 0) 125255643Snwhitehorn break; 126255643Snwhitehorn node = OF_peer(node); 127255643Snwhitehorn } 128255643Snwhitehorn 129255643Snwhitehorn res = OF_getprop(node, "ibm,pft-size", prop, sizeof(prop)); 130255643Snwhitehorn if (res <= 0) 131255643Snwhitehorn panic("mmu_phyp: unknown PFT size"); 132255643Snwhitehorn final_pteg_count = 1 << prop[1]; 133255643Snwhitehorn res = OF_getprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 134255643Snwhitehorn if (res > 0) 135255643Snwhitehorn n_slbs = prop[0]; 136255643Snwhitehorn 137255643Snwhitehorn moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 138255643Snwhitehorn 139255643Snwhitehorn /* 140255643Snwhitehorn * Scan the large page size property for PAPR compatible machines. 141255643Snwhitehorn * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 142255643Snwhitehorn * for the encoding of the property. 143255643Snwhitehorn */ 144255643Snwhitehorn 145255643Snwhitehorn len = OF_getproplen(node, "ibm,segment-page-sizes"); 146255643Snwhitehorn if (len > 0) { 147255643Snwhitehorn /* 148255643Snwhitehorn * We have to use a variable length array on the stack 149255643Snwhitehorn * since we have very limited stack space. 150255643Snwhitehorn */ 151255643Snwhitehorn cell_t arr[len/sizeof(cell_t)]; 152255643Snwhitehorn res = OF_getprop(node, "ibm,segment-page-sizes", &arr, 153255643Snwhitehorn sizeof(arr)); 154255643Snwhitehorn len /= 4; 155255643Snwhitehorn idx = 0; 156255643Snwhitehorn while (len > 0) { 157255643Snwhitehorn shift = arr[idx]; 158255643Snwhitehorn slb_encoding = arr[idx + 1]; 159255643Snwhitehorn nptlp = arr[idx + 2]; 160255643Snwhitehorn idx += 3; 161255643Snwhitehorn len -= 3; 162255643Snwhitehorn while (len > 0 && nptlp) { 163255643Snwhitehorn idx += 2; 164255643Snwhitehorn len -= 2; 165255643Snwhitehorn nptlp--; 166255643Snwhitehorn } 167255643Snwhitehorn } 168259227Sandreast 169259227Sandreast /* For now we allow shift only to be <= 0x18. */ 170259227Sandreast if (shift >= 0x18) 171259227Sandreast shift = 0x18; 172259227Sandreast 173255643Snwhitehorn moea64_large_page_shift = shift; 174255643Snwhitehorn moea64_large_page_size = 1 << shift; 175255643Snwhitehorn } 176255643Snwhitehorn 177255643Snwhitehorn moea64_mid_bootstrap(mmup, kernelstart, kernelend); 178255643Snwhitehorn moea64_late_bootstrap(mmup, kernelstart, kernelend); 179255643Snwhitehorn} 180255643Snwhitehorn 181255643Snwhitehornstatic void 182255643Snwhitehornmphyp_cpu_bootstrap(mmu_t mmup, int ap) 183255643Snwhitehorn{ 184255643Snwhitehorn struct slb *slb = PCPU_GET(slb); 185255643Snwhitehorn register_t seg0; 186255643Snwhitehorn int i; 187255643Snwhitehorn 188255643Snwhitehorn /* 189255643Snwhitehorn * Install kernel SLB entries 190255643Snwhitehorn */ 191255643Snwhitehorn 192255643Snwhitehorn __asm __volatile ("slbia"); 193255643Snwhitehorn __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 194255643Snwhitehorn for (i = 0; i < 64; i++) { 195255643Snwhitehorn if (!(slb[i].slbe & SLBE_VALID)) 196255643Snwhitehorn continue; 197255643Snwhitehorn 198255643Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 199255643Snwhitehorn "r"(slb[i].slbv), "r"(slb[i].slbe)); 200255643Snwhitehorn } 201255643Snwhitehorn} 202255643Snwhitehorn 203255643Snwhitehornstatic void 204255643Snwhitehornmphyp_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 205255643Snwhitehorn{ 206255643Snwhitehorn struct lpte pte; 207255643Snwhitehorn uint64_t junk; 208255643Snwhitehorn 209255909Snwhitehorn __asm __volatile("ptesync"); 210255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pte.pte_hi, &pte.pte_lo, 211255643Snwhitehorn &junk); 212255643Snwhitehorn 213255643Snwhitehorn pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 214255643Snwhitehorn} 215255643Snwhitehorn 216255643Snwhitehornstatic void 217255643Snwhitehornmphyp_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 218255643Snwhitehorn u_int64_t ptebit) 219255643Snwhitehorn{ 220255643Snwhitehorn 221255643Snwhitehorn if (ptebit & LPTE_CHG) 222255643Snwhitehorn phyp_hcall(H_CLEAR_MOD, 0, slot); 223255643Snwhitehorn if (ptebit & LPTE_REF) 224255643Snwhitehorn phyp_hcall(H_CLEAR_REF, 0, slot); 225255643Snwhitehorn} 226255643Snwhitehorn 227255643Snwhitehornstatic void 228255643Snwhitehornmphyp_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 229255643Snwhitehorn{ 230255909Snwhitehorn struct lpte pte; 231255909Snwhitehorn uint64_t junk; 232255909Snwhitehorn int err; 233255643Snwhitehorn 234255909Snwhitehorn err = phyp_pft_hcall(H_REMOVE, 1UL << 31, slot, 235255909Snwhitehorn pvo_pt->pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 236255909Snwhitehorn &junk); 237255909Snwhitehorn KASSERT(err == H_SUCCESS, ("Error removing page: %d", err)); 238255909Snwhitehorn 239255909Snwhitehorn pvo_pt->pte_lo |= pte.pte_lo & (LPTE_CHG | LPTE_REF); 240255643Snwhitehorn} 241255643Snwhitehorn 242255643Snwhitehornstatic void 243255643Snwhitehornmphyp_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 244255643Snwhitehorn{ 245255643Snwhitehorn struct lpte evicted; 246255643Snwhitehorn uint64_t index, junk; 247255643Snwhitehorn int64_t result; 248255643Snwhitehorn 249255643Snwhitehorn /* 250255643Snwhitehorn * NB: this is protected by the global table lock, so this two-step 251255643Snwhitehorn * is safe, except for the scratch-page case. No CPUs on which we run 252255643Snwhitehorn * this code should be using scratch pages. 253255643Snwhitehorn */ 254255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 255255643Snwhitehorn ("Locked pages not supported on PHYP")); 256255643Snwhitehorn 257255643Snwhitehorn /* XXX: optimization using H_PROTECT for common case? */ 258255909Snwhitehorn mphyp_pte_unset(mmu, slot, pvo_pt, vpn); 259255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, slot, pvo_pt->pte_hi, 260255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 261255643Snwhitehorn if (result != H_SUCCESS) 262255643Snwhitehorn panic("mphyp_pte_change() insertion failure: %ld\n", result); 263255643Snwhitehorn} 264255643Snwhitehorn 265255643Snwhitehornstatic __inline int 266255643Snwhitehornmphyp_pte_spillable_ident(u_int ptegidx, struct lpte *to_evict) 267255643Snwhitehorn{ 268255643Snwhitehorn uint64_t slot, junk, k; 269255643Snwhitehorn struct lpte pt; 270255643Snwhitehorn int i, j; 271255643Snwhitehorn 272255643Snwhitehorn /* Start at a random slot */ 273255643Snwhitehorn i = mftb() % 8; 274255643Snwhitehorn k = -1; 275255643Snwhitehorn for (j = 0; j < 8; j++) { 276255643Snwhitehorn slot = (ptegidx << 3) + (i + j) % 8; 277255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, &pt.pte_lo, 278255643Snwhitehorn &junk); 279255643Snwhitehorn 280255643Snwhitehorn if (pt.pte_hi & LPTE_SWBITS) 281255643Snwhitehorn continue; 282255643Snwhitehorn 283255643Snwhitehorn /* This is a candidate, so remember it */ 284255643Snwhitehorn k = slot; 285255643Snwhitehorn 286255643Snwhitehorn /* Try to get a page that has not been used lately */ 287255643Snwhitehorn if (!(pt.pte_lo & LPTE_REF)) { 288255643Snwhitehorn memcpy(to_evict, &pt, sizeof(struct lpte)); 289255643Snwhitehorn return (k); 290255643Snwhitehorn } 291255643Snwhitehorn } 292255643Snwhitehorn 293255643Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &to_evict->pte_hi, 294255643Snwhitehorn &to_evict->pte_lo, &junk); 295255643Snwhitehorn return (k); 296255643Snwhitehorn} 297255643Snwhitehorn 298255643Snwhitehornstatic int 299255643Snwhitehornmphyp_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 300255643Snwhitehorn{ 301255643Snwhitehorn int64_t result; 302255643Snwhitehorn struct lpte evicted; 303255643Snwhitehorn struct pvo_entry *pvo; 304255643Snwhitehorn uint64_t index, junk; 305255643Snwhitehorn u_int pteg_bktidx; 306255643Snwhitehorn 307255643Snwhitehorn /* Check for locked pages, which we can't support on this system */ 308255643Snwhitehorn KASSERT(!(pvo_pt->pte_hi & LPTE_LOCKED), 309255643Snwhitehorn ("Locked pages not supported on PHYP")); 310255643Snwhitehorn 311255643Snwhitehorn /* Initialize PTE */ 312255643Snwhitehorn pvo_pt->pte_hi |= LPTE_VALID; 313255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 314255643Snwhitehorn evicted.pte_hi = 0; 315255643Snwhitehorn 316255643Snwhitehorn /* 317255643Snwhitehorn * First try primary hash. 318255643Snwhitehorn */ 319255643Snwhitehorn pteg_bktidx = ptegidx; 320255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, pvo_pt->pte_hi, 321255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 322255643Snwhitehorn if (result == H_SUCCESS) 323255643Snwhitehorn return (index & 0x07); 324255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 325255643Snwhitehorn "(ptegidx: %#x/%#x, PTE %#lx/%#lx", result, ptegidx, 326255643Snwhitehorn moea64_pteg_count, pvo_pt->pte_hi, pvo_pt->pte_lo)); 327255643Snwhitehorn 328255643Snwhitehorn /* 329255643Snwhitehorn * Next try secondary hash. 330255643Snwhitehorn */ 331255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 332255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 333255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pteg_bktidx << 3, 334255643Snwhitehorn pvo_pt->pte_hi, pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 335255643Snwhitehorn if (result == H_SUCCESS) 336255643Snwhitehorn return (index & 0x07); 337255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 338255643Snwhitehorn result)); 339255643Snwhitehorn 340255643Snwhitehorn /* 341255643Snwhitehorn * Out of luck. Find a PTE to sacrifice. 342255643Snwhitehorn */ 343255643Snwhitehorn pteg_bktidx = ptegidx; 344255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 345255643Snwhitehorn if (index == -1L) { 346255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; 347255643Snwhitehorn index = mphyp_pte_spillable_ident(pteg_bktidx, &evicted); 348255643Snwhitehorn } 349255643Snwhitehorn 350255643Snwhitehorn if (index == -1L) { 351255643Snwhitehorn /* No freeable slots in either PTEG? We're hosed. */ 352255643Snwhitehorn panic("mphyp_pte_insert: overflow"); 353255643Snwhitehorn return (-1); 354255643Snwhitehorn } 355255643Snwhitehorn 356255643Snwhitehorn if (pteg_bktidx == ptegidx) 357255643Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 358255643Snwhitehorn else 359255643Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 360255643Snwhitehorn 361255643Snwhitehorn /* 362255643Snwhitehorn * Synchronize the sacrifice PTE with its PVO, then mark both 363255643Snwhitehorn * invalid. The PVO will be reused when/if the VM system comes 364255643Snwhitehorn * here after a fault. 365255643Snwhitehorn */ 366255643Snwhitehorn 367255643Snwhitehorn if (evicted.pte_hi & LPTE_HID) 368255643Snwhitehorn pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 369255643Snwhitehorn 370255643Snwhitehorn LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) { 371255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi) { 372255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 373255643Snwhitehorn ("Invalid PVO for valid PTE!")); 374255909Snwhitehorn mphyp_pte_unset(mmu, index, &pvo->pvo_pte.lpte, 375255909Snwhitehorn pvo->pvo_vpn); 376255643Snwhitehorn PVO_PTEGIDX_CLR(pvo); 377255643Snwhitehorn moea64_pte_overflow++; 378255643Snwhitehorn break; 379255643Snwhitehorn } 380255643Snwhitehorn } 381255643Snwhitehorn 382255643Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi == evicted.pte_hi, 383255643Snwhitehorn ("Unable to find PVO for spilled PTE")); 384255643Snwhitehorn 385255643Snwhitehorn /* 386255643Snwhitehorn * Set the new PTE. 387255643Snwhitehorn */ 388255643Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pvo_pt->pte_hi, 389255643Snwhitehorn pvo_pt->pte_lo, &index, &evicted.pte_lo, &junk); 390255643Snwhitehorn if (result == H_SUCCESS) 391255643Snwhitehorn return (index & 0x07); 392255643Snwhitehorn 393255643Snwhitehorn panic("Page replacement error: %ld", result); 394255643Snwhitehorn return (-1); 395255643Snwhitehorn} 396255643Snwhitehorn 397255643Snwhitehornstatic __inline u_int 398255643Snwhitehornva_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 399255643Snwhitehorn{ 400255643Snwhitehorn uint64_t hash; 401255643Snwhitehorn int shift; 402255643Snwhitehorn 403255643Snwhitehorn shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 404255643Snwhitehorn hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 405255643Snwhitehorn shift); 406255643Snwhitehorn return (hash & moea64_pteg_mask); 407255643Snwhitehorn} 408255643Snwhitehorn 409255643Snwhitehornstatic uintptr_t 410255643Snwhitehornmphyp_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 411255643Snwhitehorn{ 412255643Snwhitehorn uint64_t vsid; 413255643Snwhitehorn u_int ptegidx; 414255643Snwhitehorn 415255643Snwhitehorn /* If the PTEG index is not set, then there is no page table entry */ 416255643Snwhitehorn if (!PVO_PTEGIDX_ISSET(pvo)) 417255643Snwhitehorn return (-1); 418255643Snwhitehorn 419255643Snwhitehorn vsid = PVO_VSID(pvo); 420255643Snwhitehorn ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 421255643Snwhitehorn 422255643Snwhitehorn /* 423255643Snwhitehorn * We can find the actual pte entry without searching by grabbing 424255643Snwhitehorn * the PTEG index from 3 unused bits in pvo_vaddr and by 425255643Snwhitehorn * noticing the HID bit. 426255643Snwhitehorn */ 427255643Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 428255643Snwhitehorn ptegidx ^= moea64_pteg_mask; 429255643Snwhitehorn 430255643Snwhitehorn return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 431255643Snwhitehorn} 432255643Snwhitehorn 433