slb.c revision 230123
1209975Snwhitehorn/*- 2209975Snwhitehorn * Copyright (c) 2010 Nathan Whitehorn 3209975Snwhitehorn * All rights reserved. 4209975Snwhitehorn * 5209975Snwhitehorn * Redistribution and use in source and binary forms, with or without 6209975Snwhitehorn * modification, are permitted provided that the following conditions 7209975Snwhitehorn * are met: 8209975Snwhitehorn * 9209975Snwhitehorn * 1. Redistributions of source code must retain the above copyright 10209975Snwhitehorn * notice, this list of conditions and the following disclaimer. 11209975Snwhitehorn * 2. Redistributions in binary form must reproduce the above copyright 12209975Snwhitehorn * notice, this list of conditions and the following disclaimer in the 13209975Snwhitehorn * documentation and/or other materials provided with the distribution. 14209975Snwhitehorn * 15209975Snwhitehorn * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16209975Snwhitehorn * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17209975Snwhitehorn * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18209975Snwhitehorn * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19209975Snwhitehorn * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20209975Snwhitehorn * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21209975Snwhitehorn * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22209975Snwhitehorn * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23209975Snwhitehorn * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24209975Snwhitehorn * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25209975Snwhitehorn * 26209975Snwhitehorn * $FreeBSD: head/sys/powerpc/aim/slb.c 230123 2012-01-15 00:08:14Z nwhitehorn $ 27209975Snwhitehorn */ 28209975Snwhitehorn 29209975Snwhitehorn#include <sys/param.h> 30209975Snwhitehorn#include <sys/kernel.h> 31209975Snwhitehorn#include <sys/lock.h> 32209975Snwhitehorn#include <sys/mutex.h> 33209975Snwhitehorn#include <sys/proc.h> 34209975Snwhitehorn#include <sys/systm.h> 35209975Snwhitehorn 36209975Snwhitehorn#include <vm/vm.h> 37209975Snwhitehorn#include <vm/pmap.h> 38209975Snwhitehorn#include <vm/uma.h> 39215159Snwhitehorn#include <vm/vm.h> 40209975Snwhitehorn#include <vm/vm_map.h> 41215159Snwhitehorn#include <vm/vm_page.h> 42215159Snwhitehorn#include <vm/vm_pageout.h> 43209975Snwhitehorn 44209975Snwhitehorn#include <machine/md_var.h> 45215159Snwhitehorn#include <machine/platform.h> 46209975Snwhitehorn#include <machine/pmap.h> 47209975Snwhitehorn#include <machine/vmparam.h> 48209975Snwhitehorn 49209975Snwhitehornuintptr_t moea64_get_unique_vsid(void); 50209975Snwhitehornvoid moea64_release_vsid(uint64_t vsid); 51212715Snwhitehornstatic void slb_zone_init(void *); 52209975Snwhitehorn 53222620Snwhitehornstatic uma_zone_t slbt_zone; 54222620Snwhitehornstatic uma_zone_t slb_cache_zone; 55222620Snwhitehornint n_slbs = 64; 56212715Snwhitehorn 57212715SnwhitehornSYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL); 58212715Snwhitehorn 59212715Snwhitehornstruct slbtnode { 60212715Snwhitehorn uint16_t ua_alloc; 61212715Snwhitehorn uint8_t ua_level; 62212715Snwhitehorn /* Only 36 bits needed for full 64-bit address space. */ 63212715Snwhitehorn uint64_t ua_base; 64212715Snwhitehorn union { 65212715Snwhitehorn struct slbtnode *ua_child[16]; 66212715Snwhitehorn struct slb slb_entries[16]; 67212715Snwhitehorn } u; 68209975Snwhitehorn}; 69209975Snwhitehorn 70212715Snwhitehorn/* 71212715Snwhitehorn * For a full 64-bit address space, there are 36 bits in play in an 72212715Snwhitehorn * esid, so 8 levels, with the leaf being at level 0. 73212715Snwhitehorn * 74212715Snwhitehorn * |3333|3322|2222|2222|1111|1111|11 | | | esid 75212715Snwhitehorn * |5432|1098|7654|3210|9876|5432|1098|7654|3210| bits 76212715Snwhitehorn * +----+----+----+----+----+----+----+----+----+-------- 77212715Snwhitehorn * | 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | level 78212715Snwhitehorn */ 79212715Snwhitehorn#define UAD_ROOT_LEVEL 8 80212715Snwhitehorn#define UAD_LEAF_LEVEL 0 81209975Snwhitehorn 82212715Snwhitehornstatic inline int 83212715Snwhitehornesid2idx(uint64_t esid, int level) 84212715Snwhitehorn{ 85212715Snwhitehorn int shift; 86209975Snwhitehorn 87212715Snwhitehorn shift = level * 4; 88212715Snwhitehorn return ((esid >> shift) & 0xF); 89212715Snwhitehorn} 90209975Snwhitehorn 91212715Snwhitehorn/* 92212715Snwhitehorn * The ua_base field should have 0 bits after the first 4*(level+1) 93212715Snwhitehorn * bits; i.e. only 94212715Snwhitehorn */ 95212715Snwhitehorn#define uad_baseok(ua) \ 96212715Snwhitehorn (esid2base(ua->ua_base, ua->ua_level) == ua->ua_base) 97209975Snwhitehorn 98212715Snwhitehorn 99212715Snwhitehornstatic inline uint64_t 100212715Snwhitehornesid2base(uint64_t esid, int level) 101209975Snwhitehorn{ 102212715Snwhitehorn uint64_t mask; 103212715Snwhitehorn int shift; 104209975Snwhitehorn 105212715Snwhitehorn shift = (level + 1) * 4; 106212715Snwhitehorn mask = ~((1ULL << shift) - 1); 107212715Snwhitehorn return (esid & mask); 108212715Snwhitehorn} 109212715Snwhitehorn 110212715Snwhitehorn/* 111212715Snwhitehorn * Allocate a new leaf node for the specified esid/vmhandle from the 112212715Snwhitehorn * parent node. 113212715Snwhitehorn */ 114212715Snwhitehornstatic struct slb * 115212715Snwhitehornmake_new_leaf(uint64_t esid, uint64_t slbv, struct slbtnode *parent) 116212715Snwhitehorn{ 117212715Snwhitehorn struct slbtnode *child; 118212715Snwhitehorn struct slb *retval; 119212715Snwhitehorn int idx; 120212715Snwhitehorn 121212715Snwhitehorn idx = esid2idx(esid, parent->ua_level); 122212715Snwhitehorn KASSERT(parent->u.ua_child[idx] == NULL, ("Child already exists!")); 123212715Snwhitehorn 124212715Snwhitehorn /* unlock and M_WAITOK and loop? */ 125212715Snwhitehorn child = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 126212715Snwhitehorn KASSERT(child != NULL, ("unhandled NULL case")); 127212715Snwhitehorn 128212715Snwhitehorn child->ua_level = UAD_LEAF_LEVEL; 129212715Snwhitehorn child->ua_base = esid2base(esid, child->ua_level); 130212715Snwhitehorn idx = esid2idx(esid, child->ua_level); 131212715Snwhitehorn child->u.slb_entries[idx].slbv = slbv; 132212715Snwhitehorn child->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; 133212715Snwhitehorn setbit(&child->ua_alloc, idx); 134212715Snwhitehorn 135212715Snwhitehorn retval = &child->u.slb_entries[idx]; 136212715Snwhitehorn 137212715Snwhitehorn /* 138212715Snwhitehorn * The above stores must be visible before the next one, so 139212715Snwhitehorn * that a lockless searcher always sees a valid path through 140212715Snwhitehorn * the tree. 141212715Snwhitehorn */ 142212715Snwhitehorn powerpc_sync(); 143212715Snwhitehorn 144212715Snwhitehorn idx = esid2idx(esid, parent->ua_level); 145212715Snwhitehorn parent->u.ua_child[idx] = child; 146212715Snwhitehorn setbit(&parent->ua_alloc, idx); 147212715Snwhitehorn 148212715Snwhitehorn return (retval); 149212715Snwhitehorn} 150212715Snwhitehorn 151212715Snwhitehorn/* 152212715Snwhitehorn * Allocate a new intermediate node to fit between the parent and 153212715Snwhitehorn * esid. 154212715Snwhitehorn */ 155212715Snwhitehornstatic struct slbtnode* 156212715Snwhitehornmake_intermediate(uint64_t esid, struct slbtnode *parent) 157212715Snwhitehorn{ 158212715Snwhitehorn struct slbtnode *child, *inter; 159212715Snwhitehorn int idx, level; 160212715Snwhitehorn 161212715Snwhitehorn idx = esid2idx(esid, parent->ua_level); 162212715Snwhitehorn child = parent->u.ua_child[idx]; 163212715Snwhitehorn KASSERT(esid2base(esid, child->ua_level) != child->ua_base, 164212715Snwhitehorn ("No need for an intermediate node?")); 165212715Snwhitehorn 166212715Snwhitehorn /* 167212715Snwhitehorn * Find the level where the existing child and our new esid 168212715Snwhitehorn * meet. It must be lower than parent->ua_level or we would 169212715Snwhitehorn * have chosen a different index in parent. 170212715Snwhitehorn */ 171212715Snwhitehorn level = child->ua_level + 1; 172212715Snwhitehorn while (esid2base(esid, level) != 173212715Snwhitehorn esid2base(child->ua_base, level)) 174212715Snwhitehorn level++; 175212715Snwhitehorn KASSERT(level < parent->ua_level, 176212715Snwhitehorn ("Found splitting level %d for %09jx and %09jx, " 177212715Snwhitehorn "but it's the same as %p's", 178212715Snwhitehorn level, esid, child->ua_base, parent)); 179212715Snwhitehorn 180212715Snwhitehorn /* unlock and M_WAITOK and loop? */ 181212715Snwhitehorn inter = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 182212715Snwhitehorn KASSERT(inter != NULL, ("unhandled NULL case")); 183212715Snwhitehorn 184212715Snwhitehorn /* Set up intermediate node to point to child ... */ 185212715Snwhitehorn inter->ua_level = level; 186212715Snwhitehorn inter->ua_base = esid2base(esid, inter->ua_level); 187212715Snwhitehorn idx = esid2idx(child->ua_base, inter->ua_level); 188212715Snwhitehorn inter->u.ua_child[idx] = child; 189212715Snwhitehorn setbit(&inter->ua_alloc, idx); 190212715Snwhitehorn powerpc_sync(); 191212715Snwhitehorn 192212715Snwhitehorn /* Set up parent to point to intermediate node ... */ 193212715Snwhitehorn idx = esid2idx(inter->ua_base, parent->ua_level); 194212715Snwhitehorn parent->u.ua_child[idx] = inter; 195212715Snwhitehorn setbit(&parent->ua_alloc, idx); 196212715Snwhitehorn 197212715Snwhitehorn return (inter); 198212715Snwhitehorn} 199212715Snwhitehorn 200212715Snwhitehornuint64_t 201212715Snwhitehornkernel_va_to_slbv(vm_offset_t va) 202212715Snwhitehorn{ 203217451Sandreast uint64_t slbv; 204212715Snwhitehorn 205212715Snwhitehorn /* Set kernel VSID to deterministic value */ 206214574Snwhitehorn slbv = (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)) << SLBV_VSID_SHIFT; 207209975Snwhitehorn 208212715Snwhitehorn /* Figure out if this is a large-page mapping */ 209212715Snwhitehorn if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) { 210212715Snwhitehorn /* 211212715Snwhitehorn * XXX: If we have set up a direct map, assumes 212212715Snwhitehorn * all physical memory is mapped with large pages. 213212715Snwhitehorn */ 214212715Snwhitehorn if (mem_valid(va, 0) == 0) 215212715Snwhitehorn slbv |= SLBV_L; 216209975Snwhitehorn } 217212715Snwhitehorn 218212715Snwhitehorn return (slbv); 219212715Snwhitehorn} 220209975Snwhitehorn 221212715Snwhitehornstruct slb * 222212715Snwhitehornuser_va_to_slb_entry(pmap_t pm, vm_offset_t va) 223212715Snwhitehorn{ 224212715Snwhitehorn uint64_t esid = va >> ADDR_SR_SHFT; 225212715Snwhitehorn struct slbtnode *ua; 226212715Snwhitehorn int idx; 227209975Snwhitehorn 228212715Snwhitehorn ua = pm->pm_slb_tree_root; 229209975Snwhitehorn 230212715Snwhitehorn for (;;) { 231212715Snwhitehorn KASSERT(uad_baseok(ua), ("uad base %016jx level %d bad!", 232212715Snwhitehorn ua->ua_base, ua->ua_level)); 233212715Snwhitehorn idx = esid2idx(esid, ua->ua_level); 234209975Snwhitehorn 235212715Snwhitehorn /* 236212715Snwhitehorn * This code is specific to ppc64 where a load is 237212715Snwhitehorn * atomic, so no need for atomic_load macro. 238212715Snwhitehorn */ 239212715Snwhitehorn if (ua->ua_level == UAD_LEAF_LEVEL) 240212715Snwhitehorn return ((ua->u.slb_entries[idx].slbe & SLBE_VALID) ? 241212715Snwhitehorn &ua->u.slb_entries[idx] : NULL); 242212715Snwhitehorn 243212715Snwhitehorn ua = ua->u.ua_child[idx]; 244212715Snwhitehorn if (ua == NULL || 245212715Snwhitehorn esid2base(esid, ua->ua_level) != ua->ua_base) 246212715Snwhitehorn return (NULL); 247212715Snwhitehorn } 248212715Snwhitehorn 249212715Snwhitehorn return (NULL); 250209975Snwhitehorn} 251209975Snwhitehorn 252209975Snwhitehornuint64_t 253209975Snwhitehornva_to_vsid(pmap_t pm, vm_offset_t va) 254209975Snwhitehorn{ 255212715Snwhitehorn struct slb *entry; 256209975Snwhitehorn 257209975Snwhitehorn /* Shortcut kernel case */ 258210704Snwhitehorn if (pm == kernel_pmap) 259210704Snwhitehorn return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT)); 260209975Snwhitehorn 261209975Snwhitehorn /* 262209975Snwhitehorn * If there is no vsid for this VA, we need to add a new entry 263209975Snwhitehorn * to the PMAP's segment table. 264209975Snwhitehorn */ 265209975Snwhitehorn 266212715Snwhitehorn entry = user_va_to_slb_entry(pm, va); 267212715Snwhitehorn 268212715Snwhitehorn if (entry == NULL) 269212722Snwhitehorn return (allocate_user_vsid(pm, 270212722Snwhitehorn (uintptr_t)va >> ADDR_SR_SHFT, 0)); 271209975Snwhitehorn 272212715Snwhitehorn return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT); 273209975Snwhitehorn} 274209975Snwhitehorn 275209975Snwhitehornuint64_t 276212722Snwhitehornallocate_user_vsid(pmap_t pm, uint64_t esid, int large) 277209975Snwhitehorn{ 278212715Snwhitehorn uint64_t vsid, slbv; 279212715Snwhitehorn struct slbtnode *ua, *next, *inter; 280212715Snwhitehorn struct slb *slb; 281212715Snwhitehorn int idx; 282209975Snwhitehorn 283212715Snwhitehorn KASSERT(pm != kernel_pmap, ("Attempting to allocate a kernel VSID")); 284209975Snwhitehorn 285212715Snwhitehorn PMAP_LOCK_ASSERT(pm, MA_OWNED); 286212715Snwhitehorn vsid = moea64_get_unique_vsid(); 287209975Snwhitehorn 288212715Snwhitehorn slbv = vsid << SLBV_VSID_SHIFT; 289212715Snwhitehorn if (large) 290212715Snwhitehorn slbv |= SLBV_L; 291209975Snwhitehorn 292212715Snwhitehorn ua = pm->pm_slb_tree_root; 293209975Snwhitehorn 294212715Snwhitehorn /* Descend to the correct leaf or NULL pointer. */ 295212715Snwhitehorn for (;;) { 296212715Snwhitehorn KASSERT(uad_baseok(ua), 297212715Snwhitehorn ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level)); 298212715Snwhitehorn idx = esid2idx(esid, ua->ua_level); 299209975Snwhitehorn 300212715Snwhitehorn if (ua->ua_level == UAD_LEAF_LEVEL) { 301212715Snwhitehorn ua->u.slb_entries[idx].slbv = slbv; 302212715Snwhitehorn eieio(); 303212715Snwhitehorn ua->u.slb_entries[idx].slbe = (esid << SLBE_ESID_SHIFT) 304212715Snwhitehorn | SLBE_VALID; 305212715Snwhitehorn setbit(&ua->ua_alloc, idx); 306212715Snwhitehorn slb = &ua->u.slb_entries[idx]; 307212715Snwhitehorn break; 308212715Snwhitehorn } 309209975Snwhitehorn 310212715Snwhitehorn next = ua->u.ua_child[idx]; 311212715Snwhitehorn if (next == NULL) { 312212715Snwhitehorn slb = make_new_leaf(esid, slbv, ua); 313212715Snwhitehorn break; 314212715Snwhitehorn } 315212715Snwhitehorn 316212715Snwhitehorn /* 317212715Snwhitehorn * Check if the next item down has an okay ua_base. 318212715Snwhitehorn * If not, we need to allocate an intermediate node. 319212715Snwhitehorn */ 320212715Snwhitehorn if (esid2base(esid, next->ua_level) != next->ua_base) { 321212715Snwhitehorn inter = make_intermediate(esid, ua); 322212715Snwhitehorn slb = make_new_leaf(esid, slbv, inter); 323212715Snwhitehorn break; 324212715Snwhitehorn } 325212715Snwhitehorn 326212715Snwhitehorn ua = next; 327209975Snwhitehorn } 328209975Snwhitehorn 329209975Snwhitehorn /* 330209975Snwhitehorn * Someone probably wants this soon, and it may be a wired 331209975Snwhitehorn * SLB mapping, so pre-spill this entry. 332209975Snwhitehorn */ 333212715Snwhitehorn eieio(); 334212722Snwhitehorn slb_insert_user(pm, slb); 335209975Snwhitehorn 336209975Snwhitehorn return (vsid); 337209975Snwhitehorn} 338209975Snwhitehorn 339212715Snwhitehornvoid 340212715Snwhitehornfree_vsid(pmap_t pm, uint64_t esid, int large) 341212715Snwhitehorn{ 342212715Snwhitehorn struct slbtnode *ua; 343212715Snwhitehorn int idx; 344212715Snwhitehorn 345212715Snwhitehorn PMAP_LOCK_ASSERT(pm, MA_OWNED); 346212715Snwhitehorn 347212715Snwhitehorn ua = pm->pm_slb_tree_root; 348212715Snwhitehorn /* Descend to the correct leaf. */ 349212715Snwhitehorn for (;;) { 350212715Snwhitehorn KASSERT(uad_baseok(ua), 351212715Snwhitehorn ("uad base %09jx level %d bad!", ua->ua_base, ua->ua_level)); 352212715Snwhitehorn 353212715Snwhitehorn idx = esid2idx(esid, ua->ua_level); 354212715Snwhitehorn if (ua->ua_level == UAD_LEAF_LEVEL) { 355212715Snwhitehorn ua->u.slb_entries[idx].slbv = 0; 356212715Snwhitehorn eieio(); 357212715Snwhitehorn ua->u.slb_entries[idx].slbe = 0; 358212715Snwhitehorn clrbit(&ua->ua_alloc, idx); 359212715Snwhitehorn return; 360212715Snwhitehorn } 361212715Snwhitehorn 362212715Snwhitehorn ua = ua->u.ua_child[idx]; 363212715Snwhitehorn if (ua == NULL || 364212715Snwhitehorn esid2base(esid, ua->ua_level) != ua->ua_base) { 365212715Snwhitehorn /* Perhaps just return instead of assert? */ 366212715Snwhitehorn KASSERT(0, 367212715Snwhitehorn ("Asked to remove an entry that was never inserted!")); 368212715Snwhitehorn return; 369212715Snwhitehorn } 370212715Snwhitehorn } 371212715Snwhitehorn} 372212715Snwhitehorn 373212715Snwhitehornstatic void 374212715Snwhitehornfree_slb_tree_node(struct slbtnode *ua) 375212715Snwhitehorn{ 376212715Snwhitehorn int idx; 377212715Snwhitehorn 378212715Snwhitehorn for (idx = 0; idx < 16; idx++) { 379212715Snwhitehorn if (ua->ua_level != UAD_LEAF_LEVEL) { 380212715Snwhitehorn if (ua->u.ua_child[idx] != NULL) 381212715Snwhitehorn free_slb_tree_node(ua->u.ua_child[idx]); 382212715Snwhitehorn } else { 383212715Snwhitehorn if (ua->u.slb_entries[idx].slbv != 0) 384212715Snwhitehorn moea64_release_vsid(ua->u.slb_entries[idx].slbv 385212715Snwhitehorn >> SLBV_VSID_SHIFT); 386212715Snwhitehorn } 387212715Snwhitehorn } 388212715Snwhitehorn 389212715Snwhitehorn uma_zfree(slbt_zone, ua); 390212715Snwhitehorn} 391212715Snwhitehorn 392212715Snwhitehornvoid 393212715Snwhitehornslb_free_tree(pmap_t pm) 394212715Snwhitehorn{ 395212715Snwhitehorn 396212715Snwhitehorn free_slb_tree_node(pm->pm_slb_tree_root); 397212715Snwhitehorn} 398212715Snwhitehorn 399212715Snwhitehornstruct slbtnode * 400212715Snwhitehornslb_alloc_tree(void) 401212715Snwhitehorn{ 402212715Snwhitehorn struct slbtnode *root; 403212715Snwhitehorn 404212715Snwhitehorn root = uma_zalloc(slbt_zone, M_NOWAIT | M_ZERO); 405212715Snwhitehorn root->ua_level = UAD_ROOT_LEVEL; 406212715Snwhitehorn 407212715Snwhitehorn return (root); 408212715Snwhitehorn} 409212715Snwhitehorn 410209975Snwhitehorn/* Lock entries mapping kernel text and stacks */ 411209975Snwhitehorn 412209975Snwhitehornvoid 413212722Snwhitehornslb_insert_kernel(uint64_t slbe, uint64_t slbv) 414209975Snwhitehorn{ 415212722Snwhitehorn struct slb *slbcache; 416230123Snwhitehorn int i; 417209975Snwhitehorn 418209975Snwhitehorn /* We don't want to be preempted while modifying the kernel map */ 419209975Snwhitehorn critical_enter(); 420209975Snwhitehorn 421212722Snwhitehorn slbcache = PCPU_GET(slb); 422209975Snwhitehorn 423214574Snwhitehorn /* Check for an unused slot, abusing the user slot as a full flag */ 424214574Snwhitehorn if (slbcache[USER_SLB_SLOT].slbe == 0) { 425222620Snwhitehorn for (i = 0; i < n_slbs; i++) { 426222620Snwhitehorn if (i == USER_SLB_SLOT) 427222620Snwhitehorn continue; 428212722Snwhitehorn if (!(slbcache[i].slbe & SLBE_VALID)) 429212722Snwhitehorn goto fillkernslb; 430212722Snwhitehorn } 431209975Snwhitehorn 432222620Snwhitehorn if (i == n_slbs) 433214574Snwhitehorn slbcache[USER_SLB_SLOT].slbe = 1; 434212722Snwhitehorn } 435212722Snwhitehorn 436230123Snwhitehorn i = mftb() % n_slbs; 437230123Snwhitehorn if (i == USER_SLB_SLOT) 438230123Snwhitehorn i = (i+1) % n_slbs; 439209975Snwhitehorn 440212722Snwhitehornfillkernslb: 441222620Snwhitehorn KASSERT(i != USER_SLB_SLOT, 442222620Snwhitehorn ("Filling user SLB slot with a kernel mapping")); 443212722Snwhitehorn slbcache[i].slbv = slbv; 444212722Snwhitehorn slbcache[i].slbe = slbe | (uint64_t)i; 445209975Snwhitehorn 446209975Snwhitehorn /* If it is for this CPU, put it in the SLB right away */ 447212722Snwhitehorn if (pmap_bootstrapped) { 448209975Snwhitehorn /* slbie not required */ 449209975Snwhitehorn __asm __volatile ("slbmte %0, %1" :: 450212722Snwhitehorn "r"(slbcache[i].slbv), "r"(slbcache[i].slbe)); 451209975Snwhitehorn } 452209975Snwhitehorn 453209975Snwhitehorn critical_exit(); 454209975Snwhitehorn} 455209975Snwhitehorn 456212722Snwhitehornvoid 457212722Snwhitehornslb_insert_user(pmap_t pm, struct slb *slb) 458212722Snwhitehorn{ 459212722Snwhitehorn int i; 460209975Snwhitehorn 461212722Snwhitehorn PMAP_LOCK_ASSERT(pm, MA_OWNED); 462212722Snwhitehorn 463222620Snwhitehorn if (pm->pm_slb_len < n_slbs) { 464212722Snwhitehorn i = pm->pm_slb_len; 465212722Snwhitehorn pm->pm_slb_len++; 466212722Snwhitehorn } else { 467222620Snwhitehorn i = mftb() % n_slbs; 468212722Snwhitehorn } 469212722Snwhitehorn 470212722Snwhitehorn /* Note that this replacement is atomic with respect to trap_subr */ 471212722Snwhitehorn pm->pm_slb[i] = slb; 472212722Snwhitehorn} 473212722Snwhitehorn 474215159Snwhitehornstatic void * 475215159Snwhitehornslb_uma_real_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 476215159Snwhitehorn{ 477215159Snwhitehorn static vm_offset_t realmax = 0; 478215159Snwhitehorn void *va; 479215159Snwhitehorn vm_page_t m; 480227568Salc int pflags; 481215159Snwhitehorn 482215159Snwhitehorn if (realmax == 0) 483215159Snwhitehorn realmax = platform_real_maxaddr(); 484215159Snwhitehorn 485215159Snwhitehorn *flags = UMA_SLAB_PRIV; 486227568Salc if ((wait & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) 487227568Salc pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; 488227568Salc else 489227568Salc pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; 490227568Salc if (wait & M_ZERO) 491227568Salc pflags |= VM_ALLOC_ZERO; 492215159Snwhitehorn 493215159Snwhitehorn for (;;) { 494227568Salc m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax, 495227568Salc PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT); 496215159Snwhitehorn if (m == NULL) { 497215159Snwhitehorn if (wait & M_NOWAIT) 498215159Snwhitehorn return (NULL); 499215159Snwhitehorn VM_WAIT; 500215159Snwhitehorn } else 501215159Snwhitehorn break; 502215159Snwhitehorn } 503215159Snwhitehorn 504215159Snwhitehorn va = (void *) VM_PAGE_TO_PHYS(m); 505215159Snwhitehorn 506215159Snwhitehorn if (!hw_direct_map) 507215159Snwhitehorn pmap_kenter((vm_offset_t)va, VM_PAGE_TO_PHYS(m)); 508215159Snwhitehorn 509215159Snwhitehorn if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) 510215159Snwhitehorn bzero(va, PAGE_SIZE); 511215159Snwhitehorn 512215159Snwhitehorn return (va); 513215159Snwhitehorn} 514215159Snwhitehorn 515209975Snwhitehornstatic void 516209975Snwhitehornslb_zone_init(void *dummy) 517209975Snwhitehorn{ 518209975Snwhitehorn 519212715Snwhitehorn slbt_zone = uma_zcreate("SLB tree node", sizeof(struct slbtnode), 520209975Snwhitehorn NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); 521222620Snwhitehorn slb_cache_zone = uma_zcreate("SLB cache", 522222620Snwhitehorn (n_slbs + 1)*sizeof(struct slb *), NULL, NULL, NULL, NULL, 523222620Snwhitehorn UMA_ALIGN_PTR, UMA_ZONE_VM); 524215159Snwhitehorn 525215159Snwhitehorn if (platform_real_maxaddr() != VM_MAX_ADDRESS) { 526215159Snwhitehorn uma_zone_set_allocf(slb_cache_zone, slb_uma_real_alloc); 527215159Snwhitehorn uma_zone_set_allocf(slbt_zone, slb_uma_real_alloc); 528215159Snwhitehorn } 529209975Snwhitehorn} 530209975Snwhitehorn 531212722Snwhitehornstruct slb ** 532209975Snwhitehornslb_alloc_user_cache(void) 533209975Snwhitehorn{ 534209975Snwhitehorn return (uma_zalloc(slb_cache_zone, M_ZERO)); 535209975Snwhitehorn} 536209975Snwhitehorn 537209975Snwhitehornvoid 538212722Snwhitehornslb_free_user_cache(struct slb **slb) 539209975Snwhitehorn{ 540209975Snwhitehorn uma_zfree(slb_cache_zone, slb); 541209975Snwhitehorn} 542