1217044Snwhitehorn/*- 2217044Snwhitehorn * Copyright (C) 2010 Nathan Whitehorn 3217044Snwhitehorn * All rights reserved. 4217044Snwhitehorn * 5217044Snwhitehorn * Redistribution and use in source and binary forms, with or without 6217044Snwhitehorn * modification, are permitted provided that the following conditions 7217044Snwhitehorn * are met: 8217044Snwhitehorn * 1. Redistributions of source code must retain the above copyright 9217044Snwhitehorn * notice, this list of conditions and the following disclaimer. 10217044Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 11217044Snwhitehorn * notice, this list of conditions and the following disclaimer in the 12217044Snwhitehorn * documentation and/or other materials provided with the distribution. 13217044Snwhitehorn * 14217044Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15217044Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16217044Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17217044Snwhitehorn * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 18217044Snwhitehorn * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19217044Snwhitehorn * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 20217044Snwhitehorn * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 21217044Snwhitehorn * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 22217044Snwhitehorn * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 23217044Snwhitehorn * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24217044Snwhitehorn */ 25217044Snwhitehorn 26217044Snwhitehorn#include <sys/cdefs.h> 27217044Snwhitehorn__FBSDID("$FreeBSD: releng/10.3/sys/powerpc/ps3/mmu_ps3.c 247297 2013-02-26 01:00:11Z attilio $"); 28217044Snwhitehorn 29217044Snwhitehorn#include <sys/param.h> 30217044Snwhitehorn#include <sys/kernel.h> 31217044Snwhitehorn#include <sys/ktr.h> 32217044Snwhitehorn#include <sys/lock.h> 33217044Snwhitehorn#include <sys/msgbuf.h> 34217044Snwhitehorn#include <sys/mutex.h> 35217044Snwhitehorn#include <sys/proc.h> 36217044Snwhitehorn#include <sys/sysctl.h> 37217044Snwhitehorn#include <sys/systm.h> 38217044Snwhitehorn#include <sys/vmmeter.h> 39217044Snwhitehorn 40217044Snwhitehorn#include <vm/vm.h> 41217044Snwhitehorn#include <vm/vm_param.h> 42217044Snwhitehorn#include <vm/vm_kern.h> 43217044Snwhitehorn#include <vm/vm_page.h> 44217044Snwhitehorn#include <vm/vm_map.h> 45217044Snwhitehorn#include <vm/vm_object.h> 46217044Snwhitehorn#include <vm/vm_extern.h> 47217044Snwhitehorn#include <vm/vm_pageout.h> 48217044Snwhitehorn#include <vm/uma.h> 49217044Snwhitehorn 50217044Snwhitehorn#include <powerpc/aim/mmu_oea64.h> 51217044Snwhitehorn 52217044Snwhitehorn#include "mmu_if.h" 53217044Snwhitehorn#include "moea64_if.h" 54217044Snwhitehorn#include "ps3-hvcall.h" 55217044Snwhitehorn 56217044Snwhitehorn#define VSID_HASH_MASK 0x0000007fffffffffUL 57217044Snwhitehorn#define PTESYNC() __asm __volatile("ptesync") 58217044Snwhitehorn 59217044Snwhitehornextern int ps3fb_remap(void); 60217044Snwhitehorn 61217044Snwhitehornstatic uint64_t mps3_vas_id; 62217044Snwhitehorn 63217044Snwhitehorn/* 64217044Snwhitehorn * Kernel MMU interface 65217044Snwhitehorn */ 66217044Snwhitehorn 67217044Snwhitehornstatic void mps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, 68217044Snwhitehorn vm_offset_t kernelend); 69217044Snwhitehornstatic void mps3_cpu_bootstrap(mmu_t mmup, int ap); 70217044Snwhitehornstatic void mps3_pte_synch(mmu_t, uintptr_t pt, struct lpte *pvo_pt); 71217044Snwhitehornstatic void mps3_pte_clear(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 72217044Snwhitehorn uint64_t vpn, uint64_t ptebit); 73217044Snwhitehornstatic void mps3_pte_unset(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 74217044Snwhitehorn uint64_t vpn); 75217044Snwhitehornstatic void mps3_pte_change(mmu_t, uintptr_t pt, struct lpte *pvo_pt, 76217044Snwhitehorn uint64_t vpn); 77217044Snwhitehornstatic int mps3_pte_insert(mmu_t, u_int ptegidx, struct lpte *pvo_pt); 78217044Snwhitehornstatic uintptr_t mps3_pvo_to_pte(mmu_t, const struct pvo_entry *pvo); 79217044Snwhitehorn 80217044Snwhitehorn 81217044Snwhitehornstatic mmu_method_t mps3_methods[] = { 82217044Snwhitehorn MMUMETHOD(mmu_bootstrap, mps3_bootstrap), 83217044Snwhitehorn MMUMETHOD(mmu_cpu_bootstrap, mps3_cpu_bootstrap), 84217044Snwhitehorn 85217044Snwhitehorn MMUMETHOD(moea64_pte_synch, mps3_pte_synch), 86217044Snwhitehorn MMUMETHOD(moea64_pte_clear, mps3_pte_clear), 87217044Snwhitehorn MMUMETHOD(moea64_pte_unset, mps3_pte_unset), 88217044Snwhitehorn MMUMETHOD(moea64_pte_change, mps3_pte_change), 89217044Snwhitehorn MMUMETHOD(moea64_pte_insert, mps3_pte_insert), 90217044Snwhitehorn MMUMETHOD(moea64_pvo_to_pte, mps3_pvo_to_pte), 91217044Snwhitehorn 92217044Snwhitehorn { 0, 0 } 93217044Snwhitehorn}; 94217044Snwhitehorn 95217044SnwhitehornMMU_DEF_INHERIT(ps3_mmu, "mmu_ps3", mps3_methods, 0, oea64_mmu); 96217044Snwhitehorn 97217044Snwhitehornstatic void 98217044Snwhitehornmps3_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) 99217044Snwhitehorn{ 100217044Snwhitehorn uint64_t final_pteg_count; 101217044Snwhitehorn 102217044Snwhitehorn moea64_early_bootstrap(mmup, kernelstart, kernelend); 103217044Snwhitehorn 104217044Snwhitehorn lv1_construct_virtual_address_space( 105217044Snwhitehorn 20 /* log_2(moea64_pteg_count) */, 2 /* n page sizes */, 106217044Snwhitehorn (24UL << 56) | (16UL << 48) /* page sizes 16 MB + 64 KB */, 107217044Snwhitehorn &mps3_vas_id, &final_pteg_count 108217044Snwhitehorn ); 109217044Snwhitehorn 110217044Snwhitehorn moea64_pteg_count = final_pteg_count / sizeof(struct lpteg); 111217044Snwhitehorn 112217044Snwhitehorn moea64_mid_bootstrap(mmup, kernelstart, kernelend); 113217044Snwhitehorn moea64_late_bootstrap(mmup, kernelstart, kernelend); 114217044Snwhitehorn} 115217044Snwhitehorn 116217044Snwhitehornstatic void 117217044Snwhitehornmps3_cpu_bootstrap(mmu_t mmup, int ap) 118217044Snwhitehorn{ 119217044Snwhitehorn struct slb *slb = PCPU_GET(slb); 120217044Snwhitehorn register_t seg0; 121217044Snwhitehorn int i; 122217044Snwhitehorn 123217044Snwhitehorn mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); 124217044Snwhitehorn 125217044Snwhitehorn /* 126217044Snwhitehorn * Destroy the loader's address space if we are coming up for 127217044Snwhitehorn * the first time, and redo the FB mapping so we can continue 128217044Snwhitehorn * having a console. 129217044Snwhitehorn */ 130217044Snwhitehorn 131217044Snwhitehorn if (!ap) 132217044Snwhitehorn lv1_destruct_virtual_address_space(0); 133217044Snwhitehorn 134217044Snwhitehorn lv1_select_virtual_address_space(mps3_vas_id); 135217044Snwhitehorn 136217044Snwhitehorn if (!ap) 137217044Snwhitehorn ps3fb_remap(); 138217044Snwhitehorn 139217044Snwhitehorn /* 140217044Snwhitehorn * Install kernel SLB entries 141217044Snwhitehorn */ 142217044Snwhitehorn 143217044Snwhitehorn __asm __volatile ("slbia"); 144217044Snwhitehorn __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0)); 145217044Snwhitehorn for (i = 0; i < 64; i++) { 146217044Snwhitehorn if (!(slb[i].slbe & SLBE_VALID)) 147217044Snwhitehorn continue; 148217044Snwhitehorn 149217044Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 150217044Snwhitehorn "r"(slb[i].slbv), "r"(slb[i].slbe)); 151217044Snwhitehorn } 152217044Snwhitehorn} 153217044Snwhitehorn 154217044Snwhitehornstatic void 155217044Snwhitehornmps3_pte_synch(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt) 156217044Snwhitehorn{ 157217044Snwhitehorn uint64_t halfbucket[4], rcbits; 158217044Snwhitehorn 159217044Snwhitehorn PTESYNC(); 160217044Snwhitehorn lv1_read_htab_entries(mps3_vas_id, slot & ~0x3UL, &halfbucket[0], 161217044Snwhitehorn &halfbucket[1], &halfbucket[2], &halfbucket[3], &rcbits); 162217044Snwhitehorn 163217044Snwhitehorn /* 164217044Snwhitehorn * rcbits contains the low 12 bits of each PTEs 2nd part, 165217044Snwhitehorn * spaced at 16-bit intervals 166217044Snwhitehorn */ 167217044Snwhitehorn 168217044Snwhitehorn KASSERT((halfbucket[slot & 0x3] & LPTE_AVPN_MASK) == 169217044Snwhitehorn (pvo_pt->pte_hi & LPTE_AVPN_MASK), 170217044Snwhitehorn ("PTE upper word %#lx != %#lx\n", 171217044Snwhitehorn halfbucket[slot & 0x3], pvo_pt->pte_hi)); 172217044Snwhitehorn 173217044Snwhitehorn pvo_pt->pte_lo |= (rcbits >> ((3 - (slot & 0x3))*16)) & 174217044Snwhitehorn (LPTE_CHG | LPTE_REF); 175217044Snwhitehorn} 176217044Snwhitehorn 177217044Snwhitehornstatic void 178217044Snwhitehornmps3_pte_clear(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn, 179217044Snwhitehorn u_int64_t ptebit) 180217044Snwhitehorn{ 181217044Snwhitehorn 182217044Snwhitehorn lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, 183217044Snwhitehorn pvo_pt->pte_lo & ~ptebit); 184217044Snwhitehorn} 185217044Snwhitehorn 186217044Snwhitehornstatic void 187217044Snwhitehornmps3_pte_unset(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 188217044Snwhitehorn{ 189217044Snwhitehorn 190217044Snwhitehorn mps3_pte_synch(mmu, slot, pvo_pt); 191217044Snwhitehorn pvo_pt->pte_hi &= ~LPTE_VALID; 192217044Snwhitehorn lv1_write_htab_entry(mps3_vas_id, slot, 0, 0); 193217044Snwhitehorn moea64_pte_valid--; 194217044Snwhitehorn} 195217044Snwhitehorn 196217044Snwhitehornstatic void 197217044Snwhitehornmps3_pte_change(mmu_t mmu, uintptr_t slot, struct lpte *pvo_pt, uint64_t vpn) 198217044Snwhitehorn{ 199217044Snwhitehorn 200217044Snwhitehorn mps3_pte_synch(mmu, slot, pvo_pt); 201217044Snwhitehorn lv1_write_htab_entry(mps3_vas_id, slot, pvo_pt->pte_hi, 202217044Snwhitehorn pvo_pt->pte_lo); 203217044Snwhitehorn} 204217044Snwhitehorn 205217044Snwhitehornstatic int 206217044Snwhitehornmps3_pte_insert(mmu_t mmu, u_int ptegidx, struct lpte *pvo_pt) 207217044Snwhitehorn{ 208217044Snwhitehorn int result; 209217044Snwhitehorn struct lpte evicted; 210217044Snwhitehorn struct pvo_entry *pvo; 211217044Snwhitehorn uint64_t index; 212217044Snwhitehorn 213217044Snwhitehorn pvo_pt->pte_hi |= LPTE_VALID; 214217044Snwhitehorn pvo_pt->pte_hi &= ~LPTE_HID; 215217044Snwhitehorn evicted.pte_hi = 0; 216217044Snwhitehorn PTESYNC(); 217217044Snwhitehorn result = lv1_insert_htab_entry(mps3_vas_id, ptegidx << 3, 218217044Snwhitehorn pvo_pt->pte_hi, pvo_pt->pte_lo, LPTE_LOCKED | LPTE_WIRED, 0, 219217044Snwhitehorn &index, &evicted.pte_hi, &evicted.pte_lo); 220217044Snwhitehorn 221217044Snwhitehorn if (result != 0) { 222217044Snwhitehorn /* No freeable slots in either PTEG? We're hosed. */ 223217044Snwhitehorn panic("mps3_pte_insert: overflow (%d)", result); 224217044Snwhitehorn return (-1); 225217044Snwhitehorn } 226217044Snwhitehorn 227217044Snwhitehorn /* 228217044Snwhitehorn * See where we ended up. 229217044Snwhitehorn */ 230217044Snwhitehorn if (index >> 3 != ptegidx) 231217044Snwhitehorn pvo_pt->pte_hi |= LPTE_HID; 232217044Snwhitehorn 233217044Snwhitehorn moea64_pte_valid++; 234217044Snwhitehorn 235217044Snwhitehorn if (!evicted.pte_hi) 236217044Snwhitehorn return (index & 0x7); 237217044Snwhitehorn 238217044Snwhitehorn /* 239217044Snwhitehorn * Synchronize the sacrifice PTE with its PVO, then mark both 240217044Snwhitehorn * invalid. The PVO will be reused when/if the VM system comes 241217044Snwhitehorn * here after a fault. 242217044Snwhitehorn */ 243217044Snwhitehorn 244217044Snwhitehorn ptegidx = index >> 3; /* Where the sacrifice PTE was found */ 245217044Snwhitehorn if (evicted.pte_hi & LPTE_HID) 246217044Snwhitehorn ptegidx ^= moea64_pteg_mask; /* PTEs indexed by primary */ 247217044Snwhitehorn 248217044Snwhitehorn KASSERT((evicted.pte_hi & (LPTE_WIRED | LPTE_LOCKED)) == 0, 249217044Snwhitehorn ("Evicted a wired PTE")); 250217044Snwhitehorn 251217044Snwhitehorn result = 0; 252217044Snwhitehorn LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { 253217044Snwhitehorn if (!PVO_PTEGIDX_ISSET(pvo)) 254217044Snwhitehorn continue; 255217044Snwhitehorn 256217044Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi == (evicted.pte_hi | LPTE_VALID)) { 257217044Snwhitehorn KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID, 258217044Snwhitehorn ("Invalid PVO for valid PTE!")); 259217044Snwhitehorn pvo->pvo_pte.lpte.pte_hi &= ~LPTE_VALID; 260217044Snwhitehorn pvo->pvo_pte.lpte.pte_lo |= 261217044Snwhitehorn evicted.pte_lo & (LPTE_REF | LPTE_CHG); 262217044Snwhitehorn PVO_PTEGIDX_CLR(pvo); 263217044Snwhitehorn moea64_pte_valid--; 264217044Snwhitehorn moea64_pte_overflow++; 265217044Snwhitehorn result = 1; 266217044Snwhitehorn break; 267217044Snwhitehorn } 268217044Snwhitehorn } 269217044Snwhitehorn 270217044Snwhitehorn KASSERT(result == 1, ("PVO for sacrifice PTE not found")); 271217044Snwhitehorn 272217044Snwhitehorn return (index & 0x7); 273217044Snwhitehorn} 274217044Snwhitehorn 275217044Snwhitehornstatic __inline u_int 276217044Snwhitehornva_to_pteg(uint64_t vsid, vm_offset_t addr, int large) 277217044Snwhitehorn{ 278217044Snwhitehorn uint64_t hash; 279217044Snwhitehorn int shift; 280217044Snwhitehorn 281217044Snwhitehorn shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT; 282217044Snwhitehorn hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >> 283217044Snwhitehorn shift); 284217044Snwhitehorn return (hash & moea64_pteg_mask); 285217044Snwhitehorn} 286217044Snwhitehorn 287217044Snwhitehornuintptr_t 288217044Snwhitehornmps3_pvo_to_pte(mmu_t mmu, const struct pvo_entry *pvo) 289217044Snwhitehorn{ 290217044Snwhitehorn uint64_t vsid; 291217044Snwhitehorn u_int ptegidx; 292217044Snwhitehorn 293217044Snwhitehorn /* If the PTEG index is not set, then there is no page table entry */ 294217044Snwhitehorn if (!PVO_PTEGIDX_ISSET(pvo)) 295217044Snwhitehorn return (-1); 296217044Snwhitehorn 297217044Snwhitehorn vsid = PVO_VSID(pvo); 298217044Snwhitehorn ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); 299217044Snwhitehorn 300217044Snwhitehorn /* 301217044Snwhitehorn * We can find the actual pte entry without searching by grabbing 302217044Snwhitehorn * the PTEG index from 3 unused bits in pvo_vaddr and by 303217044Snwhitehorn * noticing the HID bit. 304217044Snwhitehorn */ 305217044Snwhitehorn if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID) 306217044Snwhitehorn ptegidx ^= moea64_pteg_mask; 307217044Snwhitehorn 308217044Snwhitehorn return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo)); 309217044Snwhitehorn} 310217044Snwhitehorn 311