1331722Seadler/* 2255643Snwhitehorn * Copyright (C) 2010 Andreas Tobler 3255643Snwhitehorn * All rights reserved. 4255643Snwhitehorn * 5255643Snwhitehorn * Redistribution and use in source and binary forms, with or without 6255643Snwhitehorn * modification, are permitted provided that the following conditions 7255643Snwhitehorn * are met: 8255643Snwhitehorn * 1. Redistributions of source code must retain the above copyright 9255643Snwhitehorn * notice, this list of conditions and the following disclaimer. 10255643Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 11255643Snwhitehorn * notice, this list of conditions and the following disclaimer in the 12255643Snwhitehorn * documentation and/or other materials provided with the distribution. 13255643Snwhitehorn * 14255643Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15255643Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16255643Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17255643Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18255643Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19255643Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20255643Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21255643Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22255643Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23255643Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24255643Snwhitehorn */ 25255643Snwhitehorn 26255643Snwhitehorn#include <sys/cdefs.h> 27255643Snwhitehorn__FBSDID("$FreeBSD$"); 28255643Snwhitehorn 29255643Snwhitehorn#include <sys/param.h> 30255643Snwhitehorn#include <sys/kernel.h> 31255643Snwhitehorn#include <sys/ktr.h> 32255643Snwhitehorn#include <sys/lock.h> 33280147Snwhitehorn#include <sys/rmlock.h> 34255643Snwhitehorn#include <sys/mutex.h> 35255643Snwhitehorn#include <sys/proc.h> 36255643Snwhitehorn#include <sys/sysctl.h> 37255643Snwhitehorn#include <sys/systm.h> 38255643Snwhitehorn#include <sys/vmmeter.h> 39255643Snwhitehorn 40255643Snwhitehorn#include <dev/ofw/openfirm.h> 41255643Snwhitehorn#include <machine/ofw_machdep.h> 42255643Snwhitehorn 43255643Snwhitehorn#include <vm/vm.h> 44255643Snwhitehorn#include <vm/vm_param.h> 45255643Snwhitehorn#include <vm/vm_kern.h> 46255643Snwhitehorn#include <vm/vm_page.h> 47255643Snwhitehorn#include <vm/vm_map.h> 48255643Snwhitehorn#include <vm/vm_object.h> 49255643Snwhitehorn#include <vm/vm_extern.h> 50255643Snwhitehorn#include <vm/vm_pageout.h> 51255643Snwhitehorn#include <vm/uma.h> 52255643Snwhitehorn 53255643Snwhitehorn#include <powerpc/aim/mmu_oea64.h> 54255643Snwhitehorn 55255643Snwhitehorn#include "mmu_if.h" 56255643Snwhitehorn#include "moea64_if.h" 57255643Snwhitehorn 58255643Snwhitehorn#include "phyp-hvcall.h" 59255643Snwhitehorn 60255643Snwhitehornextern int n_slbs; 61255643Snwhitehorn 62280147Snwhitehornstatic struct rmlock mphyp_eviction_lock; 63279252Snwhitehorn 64255643Snwhitehorn/* 65255643Snwhitehorn * Kernel MMU interface 66255643Snwhitehorn */ 67255643Snwhitehorn 68255643Snwhitehornstatic void mphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 69255643Snwhitehorn vm_offset_t kernelend); 70255643Snwhitehornstatic void mphyp_cpu_bootstrap(mmu_t mmup, int ap); 71279252Snwhitehornstatic int64_t mphyp_pte_synch(mmu_t, struct pvo_entry *pvo); 72279252Snwhitehornstatic int64_t mphyp_pte_clear(mmu_t, struct pvo_entry *pvo, uint64_t ptebit); 73279252Snwhitehornstatic int64_t mphyp_pte_unset(mmu_t, struct pvo_entry *pvo); 74279252Snwhitehornstatic int mphyp_pte_insert(mmu_t, struct pvo_entry *pvo); 75255643Snwhitehorn 76255643Snwhitehornstatic mmu_method_t mphyp_methods[] = { 77255643Snwhitehorn MMUMETHOD(mmu_bootstrap, mphyp_bootstrap), 78255643Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, mphyp_cpu_bootstrap), 79255643Snwhitehorn 80255643Snwhitehorn MMUMETHOD(moea64_pte_synch, mphyp_pte_synch), 81255643Snwhitehorn MMUMETHOD(moea64_pte_clear, mphyp_pte_clear), 82255643Snwhitehorn MMUMETHOD(moea64_pte_unset, mphyp_pte_unset), 83255643Snwhitehorn MMUMETHOD(moea64_pte_insert, mphyp_pte_insert), 84255643Snwhitehorn 85279252Snwhitehorn /* XXX: pmap_copy_page, pmap_init_page with H_PAGE_INIT */ 86279252Snwhitehorn 87255643Snwhitehorn { 0, 0 } 88255643Snwhitehorn}; 89255643Snwhitehorn 90255643SnwhitehornMMU_DEF_INHERIT(pseries_mmu, "mmu_phyp", mphyp_methods, 0, oea64_mmu); 91255643Snwhitehorn 92279252Snwhitehornstatic int brokenkvm = 0; 93279252Snwhitehorn 94255643Snwhitehornstatic void 95279252Snwhitehornprint_kvm_bug_warning(void *data) 96279252Snwhitehorn{ 97279252Snwhitehorn 98279252Snwhitehorn if (brokenkvm) 99279252Snwhitehorn printf("WARNING: Running on a broken hypervisor that does " 100279252Snwhitehorn "not support mandatory H_CLEAR_MOD and H_CLEAR_REF " 101279252Snwhitehorn "hypercalls. Performance will be suboptimal.\n"); 102279252Snwhitehorn} 103279252Snwhitehorn 104279252SnwhitehornSYSINIT(kvmbugwarn1, SI_SUB_COPYRIGHT, SI_ORDER_THIRD + 1, 105279252Snwhitehorn print_kvm_bug_warning, NULL); 106279252SnwhitehornSYSINIT(kvmbugwarn2, SI_SUB_LAST, SI_ORDER_THIRD + 1, print_kvm_bug_warning, 107279252Snwhitehorn NULL); 108279252Snwhitehorn 109279252Snwhitehornstatic void 110255643Snwhitehornmphyp_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 111255643Snwhitehorn{ 112255643Snwhitehorn uint64_t final_pteg_count = 0; 113255643Snwhitehorn char buf[8]; 114255643Snwhitehorn uint32_t prop[2]; 115255643Snwhitehorn uint32_t nptlp, shift = 0, slb_encoding = 0; 116278389Snwhitehorn uint32_t lp_size, lp_encoding; 117279940Snwhitehorn struct lpte old; 118279940Snwhitehorn uint64_t vsid; 119255643Snwhitehorn phandle_t dev, node, root; 120255643Snwhitehorn int idx, len, res; 121255643Snwhitehorn 122280147Snwhitehorn rm_init(&mphyp_eviction_lock, "pte eviction"); 123279252Snwhitehorn 124255643Snwhitehorn moea64_early_bootstrap(mmup, kernelstart, kernelend); 125255643Snwhitehorn 126255643Snwhitehorn root = OF_peer(0); 127255643Snwhitehorn 128255643Snwhitehorn dev = OF_child(root); 129255643Snwhitehorn while (dev != 0) { 130255643Snwhitehorn res = OF_getprop(dev, "name", buf, sizeof(buf)); 131255643Snwhitehorn if (res > 0 && strcmp(buf, "cpus") == 0) 132255643Snwhitehorn break; 133255643Snwhitehorn dev = OF_peer(dev); 134255643Snwhitehorn } 135255643Snwhitehorn 136255643Snwhitehorn node = OF_child(dev); 137255643Snwhitehorn 138255643Snwhitehorn while (node != 0) { 139255643Snwhitehorn res = OF_getprop(node, "device_type", buf, sizeof(buf)); 140255643Snwhitehorn if (res > 0 && strcmp(buf, "cpu") == 0) 141255643Snwhitehorn break; 142255643Snwhitehorn node = OF_peer(node); 143255643Snwhitehorn } 144255643Snwhitehorn 145290989Snwhitehorn res = OF_getencprop(node, "ibm,pft-size", prop, sizeof(prop)); 146255643Snwhitehorn if (res <= 0) 147255643Snwhitehorn panic("mmu_phyp: unknown PFT size"); 148255643Snwhitehorn final_pteg_count = 1 << prop[1]; 149290989Snwhitehorn res = OF_getencprop(node, "ibm,slb-size", prop, sizeof(prop[0])); 150255643Snwhitehorn if (res > 0) 151255643Snwhitehorn n_slbs = prop[0]; 152255643Snwhitehorn 153255643Snwhitehorn moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 154255643Snwhitehorn 155279940Snwhitehorn /* Clear any old page table entries */ 156279940Snwhitehorn for (idx = 0; idx < moea64_pteg_count*8; idx++) { 157279940Snwhitehorn phyp_pft_hcall(H_READ, 0, idx, 0, 0, &old.pte_hi, 158279940Snwhitehorn &old.pte_lo, &old.pte_lo); 159279940Snwhitehorn vsid = (old.pte_hi << (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) >> 28; 160279940Snwhitehorn if (vsid == VSID_VRMA || vsid == 0 /* Older VRMA */) 161279940Snwhitehorn continue; 162279940Snwhitehorn 163279940Snwhitehorn if (old.pte_hi & LPTE_VALID) 164279940Snwhitehorn phyp_hcall(H_REMOVE, 0, idx, 0); 165279940Snwhitehorn } 166279940Snwhitehorn 167255643Snwhitehorn /* 168255643Snwhitehorn * Scan the large page size property for PAPR compatible machines. 169255643Snwhitehorn * See PAPR D.5 Changes to Section 5.1.4, 'CPU Node Properties' 170255643Snwhitehorn * for the encoding of the property. 171255643Snwhitehorn */ 172255643Snwhitehorn 173255643Snwhitehorn len = OF_getproplen(node, "ibm,segment-page-sizes"); 174255643Snwhitehorn if (len > 0) { 175255643Snwhitehorn /* 176255643Snwhitehorn * We have to use a variable length array on the stack 177255643Snwhitehorn * since we have very limited stack space. 178255643Snwhitehorn */ 179278389Snwhitehorn pcell_t arr[len/sizeof(cell_t)]; 180278389Snwhitehorn res = OF_getencprop(node, "ibm,segment-page-sizes", arr, 181278389Snwhitehorn sizeof(arr)); 182255643Snwhitehorn len /= 4; 183255643Snwhitehorn idx = 0; 184255643Snwhitehorn while (len > 0) { 185255643Snwhitehorn shift = arr[idx]; 186255643Snwhitehorn slb_encoding = arr[idx + 1]; 187255643Snwhitehorn nptlp = arr[idx + 2]; 188255643Snwhitehorn idx += 3; 189255643Snwhitehorn len -= 3; 190255643Snwhitehorn while (len > 0 && nptlp) { 191278389Snwhitehorn lp_size = arr[idx]; 192278389Snwhitehorn lp_encoding = arr[idx+1]; 193278389Snwhitehorn if (slb_encoding == SLBV_L && lp_encoding == 0) 194278389Snwhitehorn break; 195278389Snwhitehorn 196255643Snwhitehorn idx += 2; 197255643Snwhitehorn len -= 2; 198255643Snwhitehorn nptlp--; 199255643Snwhitehorn } 200278389Snwhitehorn if (nptlp && slb_encoding == SLBV_L && lp_encoding == 0) 201278389Snwhitehorn break; 202255643Snwhitehorn } 203258503Sandreast 204278389Snwhitehorn if (len == 0) 205278389Snwhitehorn panic("Standard large pages (SLB[L] = 1, PTE[LP] = 0) " 206278389Snwhitehorn "not supported by this system. Please enable huge " 207278389Snwhitehorn "page backing if running under PowerKVM."); 208258503Sandreast 209255643Snwhitehorn moea64_large_page_shift = shift; 210278389Snwhitehorn moea64_large_page_size = 1ULL << lp_size; 211255643Snwhitehorn } 212255643Snwhitehorn 213255643Snwhitehorn moea64_mid_bootstrap(mmup, kernelstart, kernelend); 214255643Snwhitehorn moea64_late_bootstrap(mmup, kernelstart, kernelend); 215279252Snwhitehorn 216279252Snwhitehorn /* Test for broken versions of KVM that don't conform to the spec */ 217279252Snwhitehorn if (phyp_hcall(H_CLEAR_MOD, 0, 0) == H_FUNCTION) 218279252Snwhitehorn brokenkvm = 1; 219255643Snwhitehorn} 220255643Snwhitehorn 221255643Snwhitehornstatic void 222255643Snwhitehornmphyp_cpu_bootstrap(mmu_t mmup, int ap) 223255643Snwhitehorn{ 224255643Snwhitehorn struct slb *slb = PCPU_GET(slb); 225255643Snwhitehorn register_t seg0; 226255643Snwhitehorn int i; 227255643Snwhitehorn 228255643Snwhitehorn /* 229255643Snwhitehorn * Install kernel SLB entries 230255643Snwhitehorn */ 231255643Snwhitehorn 232255643Snwhitehorn __asm __volatile ("slbia"); 233255643Snwhitehorn __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 234255643Snwhitehorn for (i = 0; i < 64; i++) { 235255643Snwhitehorn if (!(slb[i].slbe & SLBE_VALID)) 236255643Snwhitehorn continue; 237255643Snwhitehorn 238255643Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 239255643Snwhitehorn "r"(slb[i].slbv), "r"(slb[i].slbe)); 240255643Snwhitehorn } 241255643Snwhitehorn} 242255643Snwhitehorn 243279252Snwhitehornstatic int64_t 244279252Snwhitehornmphyp_pte_synch(mmu_t mmu, struct pvo_entry *pvo) 245255643Snwhitehorn{ 246255643Snwhitehorn struct lpte pte; 247255643Snwhitehorn uint64_t junk; 248255643Snwhitehorn 249255909Snwhitehorn __asm __volatile("ptesync"); 250279252Snwhitehorn phyp_pft_hcall(H_READ, 0, pvo->pvo_pte.slot, 0, 0, &pte.pte_hi, 251279252Snwhitehorn &pte.pte_lo, &junk); 252279252Snwhitehorn if ((pte.pte_hi & LPTE_AVPN_MASK) != 253279252Snwhitehorn ((pvo->pvo_vpn >> (ADDR_API_SHFT64 - ADDR_PIDX_SHFT)) & 254279252Snwhitehorn LPTE_AVPN_MASK)) 255279252Snwhitehorn return (-1); 256279252Snwhitehorn if (!(pte.pte_hi & LPTE_VALID)) 257279252Snwhitehorn return (-1); 258255643Snwhitehorn 259279252Snwhitehorn return (pte.pte_lo & (LPTE_CHG | LPTE_REF)); 260255643Snwhitehorn} 261255643Snwhitehorn 262279252Snwhitehornstatic int64_t 263279252Snwhitehornmphyp_pte_clear(mmu_t mmu, struct pvo_entry *pvo, uint64_t ptebit) 264255643Snwhitehorn{ 265280147Snwhitehorn struct rm_priotracker track; 266279252Snwhitehorn int64_t refchg; 267279252Snwhitehorn uint64_t ptelo, junk; 268279252Snwhitehorn int err; 269255643Snwhitehorn 270279252Snwhitehorn /* 271279252Snwhitehorn * This involves two steps (synch and clear) so we need the entry 272279252Snwhitehorn * not to change in the middle. We are protected against deliberate 273279252Snwhitehorn * unset by virtue of holding the pmap lock. Protection against 274279252Snwhitehorn * incidental unset (page table eviction) comes from holding the 275279252Snwhitehorn * shared eviction lock. 276279252Snwhitehorn */ 277279252Snwhitehorn PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 278280147Snwhitehorn rm_rlock(&mphyp_eviction_lock, &track); 279279252Snwhitehorn 280279252Snwhitehorn refchg = mphyp_pte_synch(mmu, pvo); 281279252Snwhitehorn if (refchg < 0) { 282280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 283279252Snwhitehorn return (refchg); 284279252Snwhitehorn } 285279252Snwhitehorn 286279252Snwhitehorn if (brokenkvm) { 287279252Snwhitehorn /* 288279252Snwhitehorn * No way to clear either bit, which is total madness. 289279252Snwhitehorn * Pessimistically claim that, once modified, it stays so 290279252Snwhitehorn * forever and that it is never referenced. 291279252Snwhitehorn */ 292280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 293279252Snwhitehorn return (refchg & ~LPTE_REF); 294279252Snwhitehorn } 295279252Snwhitehorn 296279252Snwhitehorn if (ptebit & LPTE_CHG) { 297279252Snwhitehorn err = phyp_pft_hcall(H_CLEAR_MOD, 0, pvo->pvo_pte.slot, 0, 0, 298279252Snwhitehorn &ptelo, &junk, &junk); 299279252Snwhitehorn KASSERT(err == H_SUCCESS, 300279252Snwhitehorn ("Error clearing page change bit: %d", err)); 301279252Snwhitehorn refchg |= (ptelo & LPTE_CHG); 302279252Snwhitehorn } 303279252Snwhitehorn if (ptebit & LPTE_REF) { 304279252Snwhitehorn err = phyp_pft_hcall(H_CLEAR_REF, 0, pvo->pvo_pte.slot, 0, 0, 305279252Snwhitehorn &ptelo, &junk, &junk); 306279252Snwhitehorn KASSERT(err == H_SUCCESS, 307279252Snwhitehorn ("Error clearing page reference bit: %d", err)); 308279252Snwhitehorn refchg |= (ptelo & LPTE_REF); 309279252Snwhitehorn } 310279252Snwhitehorn 311280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 312279252Snwhitehorn 313279252Snwhitehorn return (refchg); 314255643Snwhitehorn} 315255643Snwhitehorn 316279252Snwhitehornstatic int64_t 317279252Snwhitehornmphyp_pte_unset(mmu_t mmu, struct pvo_entry *pvo) 318255643Snwhitehorn{ 319255909Snwhitehorn struct lpte pte; 320255909Snwhitehorn uint64_t junk; 321255909Snwhitehorn int err; 322255643Snwhitehorn 323279252Snwhitehorn PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 324255909Snwhitehorn 325279252Snwhitehorn moea64_pte_from_pvo(pvo, &pte); 326255643Snwhitehorn 327279252Snwhitehorn err = phyp_pft_hcall(H_REMOVE, H_AVPN, pvo->pvo_pte.slot, 328279252Snwhitehorn pte.pte_hi & LPTE_AVPN_MASK, 0, &pte.pte_hi, &pte.pte_lo, 329279252Snwhitehorn &junk); 330279252Snwhitehorn KASSERT(err == H_SUCCESS || err == H_NOT_FOUND, 331279252Snwhitehorn ("Error removing page: %d", err)); 332255643Snwhitehorn 333279252Snwhitehorn if (err == H_NOT_FOUND) { 334279252Snwhitehorn moea64_pte_overflow--; 335279252Snwhitehorn return (-1); 336279252Snwhitehorn } 337255643Snwhitehorn 338279252Snwhitehorn return (pte.pte_lo & (LPTE_REF | LPTE_CHG)); 339255643Snwhitehorn} 340255643Snwhitehorn 341279252Snwhitehornstatic uintptr_t 342279252Snwhitehornmphyp_pte_spillable_ident(uintptr_t ptegbase, struct lpte *to_evict) 343255643Snwhitehorn{ 344255643Snwhitehorn uint64_t slot, junk, k; 345255643Snwhitehorn struct lpte pt; 346255643Snwhitehorn int i, j; 347255643Snwhitehorn 348255643Snwhitehorn /* Start at a random slot */ 349255643Snwhitehorn i = mftb() % 8; 350255643Snwhitehorn k = -1; 351255643Snwhitehorn for (j = 0; j < 8; j++) { 352279252Snwhitehorn slot = ptegbase + (i + j) % 8; 353279252Snwhitehorn phyp_pft_hcall(H_READ, 0, slot, 0, 0, &pt.pte_hi, 354279252Snwhitehorn &pt.pte_lo, &junk); 355255643Snwhitehorn 356278456Snwhitehorn if (pt.pte_hi & LPTE_WIRED) 357255643Snwhitehorn continue; 358255643Snwhitehorn 359255643Snwhitehorn /* This is a candidate, so remember it */ 360255643Snwhitehorn k = slot; 361255643Snwhitehorn 362255643Snwhitehorn /* Try to get a page that has not been used lately */ 363279252Snwhitehorn if (!(pt.pte_hi & LPTE_VALID) || !(pt.pte_lo & LPTE_REF)) { 364255643Snwhitehorn memcpy(to_evict, &pt, sizeof(struct lpte)); 365255643Snwhitehorn return (k); 366255643Snwhitehorn } 367255643Snwhitehorn } 368255643Snwhitehorn 369278434Snwhitehorn if (k == -1) 370278434Snwhitehorn return (k); 371278434Snwhitehorn 372278434Snwhitehorn phyp_pft_hcall(H_READ, 0, k, 0, 0, &to_evict->pte_hi, 373255643Snwhitehorn &to_evict->pte_lo, &junk); 374255643Snwhitehorn return (k); 375255643Snwhitehorn} 376255643Snwhitehorn 377255643Snwhitehornstatic int 378279252Snwhitehornmphyp_pte_insert(mmu_t mmu, struct pvo_entry *pvo) 379255643Snwhitehorn{ 380280147Snwhitehorn struct rm_priotracker track; 381255643Snwhitehorn int64_t result; 382279252Snwhitehorn struct lpte evicted, pte; 383279252Snwhitehorn uint64_t index, junk, lastptelo; 384255643Snwhitehorn 385279252Snwhitehorn PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED); 386255643Snwhitehorn 387255643Snwhitehorn /* Initialize PTE */ 388279252Snwhitehorn moea64_pte_from_pvo(pvo, &pte); 389255643Snwhitehorn evicted.pte_hi = 0; 390255643Snwhitehorn 391279252Snwhitehorn /* Make sure further insertion is locked out during evictions */ 392280147Snwhitehorn rm_rlock(&mphyp_eviction_lock, &track); 393279252Snwhitehorn 394255643Snwhitehorn /* 395255643Snwhitehorn * First try primary hash. 396255643Snwhitehorn */ 397279252Snwhitehorn pvo->pvo_pte.slot &= ~7UL; /* Base slot address */ 398279252Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, pte.pte_hi, 399279252Snwhitehorn pte.pte_lo, &index, &evicted.pte_lo, &junk); 400279252Snwhitehorn if (result == H_SUCCESS) { 401280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 402279252Snwhitehorn pvo->pvo_pte.slot = index; 403279252Snwhitehorn return (0); 404279252Snwhitehorn } 405255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Page insertion error: %ld " 406279252Snwhitehorn "(ptegidx: %#zx/%#x, PTE %#lx/%#lx", result, pvo->pvo_pte.slot, 407279252Snwhitehorn moea64_pteg_count, pte.pte_hi, pte.pte_lo)); 408255643Snwhitehorn 409255643Snwhitehorn /* 410255643Snwhitehorn * Next try secondary hash. 411255643Snwhitehorn */ 412279252Snwhitehorn pvo->pvo_vaddr ^= PVO_HID; 413279252Snwhitehorn pte.pte_hi ^= LPTE_HID; 414279252Snwhitehorn pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 415279252Snwhitehorn 416279252Snwhitehorn result = phyp_pft_hcall(H_ENTER, 0, pvo->pvo_pte.slot, 417279252Snwhitehorn pte.pte_hi, pte.pte_lo, &index, &evicted.pte_lo, &junk); 418279252Snwhitehorn if (result == H_SUCCESS) { 419280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 420279252Snwhitehorn pvo->pvo_pte.slot = index; 421279252Snwhitehorn return (0); 422279252Snwhitehorn } 423255643Snwhitehorn KASSERT(result == H_PTEG_FULL, ("Secondary page insertion error: %ld", 424255643Snwhitehorn result)); 425255643Snwhitehorn 426255643Snwhitehorn /* 427255643Snwhitehorn * Out of luck. Find a PTE to sacrifice. 428255643Snwhitehorn */ 429279252Snwhitehorn 430279252Snwhitehorn /* Lock out all insertions for a bit */ 431280147Snwhitehorn rm_runlock(&mphyp_eviction_lock, &track); 432280147Snwhitehorn rm_wlock(&mphyp_eviction_lock); 433279252Snwhitehorn 434279252Snwhitehorn index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 435255643Snwhitehorn if (index == -1L) { 436279252Snwhitehorn /* Try other hash table? */ 437279252Snwhitehorn pvo->pvo_vaddr ^= PVO_HID; 438279252Snwhitehorn pte.pte_hi ^= LPTE_HID; 439279252Snwhitehorn pvo->pvo_pte.slot ^= (moea64_pteg_mask << 3); 440279252Snwhitehorn index = mphyp_pte_spillable_ident(pvo->pvo_pte.slot, &evicted); 441255643Snwhitehorn } 442255643Snwhitehorn 443255643Snwhitehorn if (index == -1L) { 444255643Snwhitehorn /* No freeable slots in either PTEG? We're hosed. */ 445280147Snwhitehorn rm_wunlock(&mphyp_eviction_lock); 446255643Snwhitehorn panic("mphyp_pte_insert: overflow"); 447255643Snwhitehorn return (-1); 448255643Snwhitehorn } 449255643Snwhitehorn 450279252Snwhitehorn /* Victim acquired: update page before waving goodbye */ 451279252Snwhitehorn if (evicted.pte_hi & LPTE_VALID) { 452279252Snwhitehorn result = phyp_pft_hcall(H_REMOVE, H_AVPN, index, 453279252Snwhitehorn evicted.pte_hi & LPTE_AVPN_MASK, 0, &junk, &lastptelo, 454279252Snwhitehorn &junk); 455279252Snwhitehorn moea64_pte_overflow++; 456279252Snwhitehorn KASSERT(result == H_SUCCESS, 457279252Snwhitehorn ("Error evicting page: %d", (int)result)); 458255643Snwhitehorn } 459255643Snwhitehorn 460255643Snwhitehorn /* 461255643Snwhitehorn * Set the new PTE. 462255643Snwhitehorn */ 463279252Snwhitehorn result = phyp_pft_hcall(H_ENTER, H_EXACT, index, pte.pte_hi, 464279252Snwhitehorn pte.pte_lo, &index, &evicted.pte_lo, &junk); 465280147Snwhitehorn rm_wunlock(&mphyp_eviction_lock); /* All clear */ 466279252Snwhitehorn 467279252Snwhitehorn pvo->pvo_pte.slot = index; 468255643Snwhitehorn if (result == H_SUCCESS) 469279252Snwhitehorn return (0); 470255643Snwhitehorn 471255643Snwhitehorn panic("Page replacement error: %ld", result); 472279252Snwhitehorn return (result); 473255643Snwhitehorn} 474255643Snwhitehorn 475