pmap.c revision 97446
1284990Scy/* 2284990Scy * Copyright (c) 1991 Regents of the University of California. 3284990Scy * All rights reserved. 4284990Scy * Copyright (c) 1994 John S. Dyson 5284990Scy * All rights reserved. 6284990Scy * Copyright (c) 1994 David Greenman 7284990Scy * All rights reserved. 8284990Scy * 9284990Scy * This code is derived from software contributed to Berkeley by 10284990Scy * the Systems Programming Group of the University of Utah Computer 11284990Scy * Science Department and William Jolitz of UUNET Technologies Inc. 12284990Scy * 13284990Scy * Redistribution and use in source and binary forms, with or without 14284990Scy * modification, are permitted provided that the following conditions 15284990Scy * are met: 16284990Scy * 1. Redistributions of source code must retain the above copyright 17284990Scy * notice, this list of conditions and the following disclaimer. 18284990Scy * 2. Redistributions in binary form must reproduce the above copyright 19284990Scy * notice, this list of conditions and the following disclaimer in the 20284990Scy * documentation and/or other materials provided with the distribution. 21284990Scy * 3. All advertising materials mentioning features or use of this software 22284990Scy * must display the following acknowledgement: 23284990Scy * This product includes software developed by the University of 24284990Scy * California, Berkeley and its contributors. 25284990Scy * 4. Neither the name of the University nor the names of its contributors 26284990Scy * may be used to endorse or promote products derived from this software 27284990Scy * without specific prior written permission. 28284990Scy * 29284990Scy * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30284990Scy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31284990Scy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32284990Scy * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33284990Scy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34284990Scy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35284990Scy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36284990Scy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37284990Scy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38284990Scy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39284990Scy * SUCH DAMAGE. 40284990Scy * 41284990Scy * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42284990Scy * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 97446 2002-05-29 05:56:05Z jake $ 43284990Scy */ 44284990Scy 45284990Scy/* 46284990Scy * Manages physical address maps. 47284990Scy * 48284990Scy * In addition to hardware address maps, this module is called upon to 49284990Scy * provide software-use-only maps which may or may not be stored in the 50284990Scy * same form as hardware maps. These pseudo-maps are used to store 51284990Scy * intermediate results from copy operations to and from address spaces. 52284990Scy * 53284990Scy * Since the information managed by this module is also stored by the 54284990Scy * logical address mapping module, this module may throw away valid virtual 55284990Scy * to physical mappings at almost any time. However, invalidations of 56284990Scy * mappings must be done as requested. 57284990Scy * 58284990Scy * In order to cope with hardware architectures which make virtual to 59284990Scy * physical map invalidates expensive, this module may delay invalidate 60284990Scy * reduced protection operations until such time as they are actually 61284990Scy * necessary. This module is given full information as to which processors 62284990Scy * are currently using which maps, and to when physical maps must be made 63284990Scy * correct. 64284990Scy */ 65284990Scy 66284990Scy#include "opt_msgbuf.h" 67284990Scy#include "opt_pmap.h" 68284990Scy 69284990Scy#include <sys/param.h> 70284990Scy#include <sys/kernel.h> 71284990Scy#include <sys/ktr.h> 72284990Scy#include <sys/lock.h> 73284990Scy#include <sys/msgbuf.h> 74284990Scy#include <sys/mutex.h> 75284990Scy#include <sys/proc.h> 76284990Scy#include <sys/smp.h> 77284990Scy#include <sys/sysctl.h> 78284990Scy#include <sys/systm.h> 79284990Scy#include <sys/vmmeter.h> 80284990Scy 81284990Scy#include <dev/ofw/openfirm.h> 82284990Scy 83284990Scy#include <vm/vm.h> 84284990Scy#include <vm/vm_param.h> 85284990Scy#include <vm/vm_kern.h> 86284990Scy#include <vm/vm_page.h> 87284990Scy#include <vm/vm_map.h> 88284990Scy#include <vm/vm_object.h> 89284990Scy#include <vm/vm_extern.h> 90284990Scy#include <vm/vm_pageout.h> 91284990Scy#include <vm/vm_pager.h> 92284990Scy#include <vm/uma.h> 93284990Scy 94284990Scy#include <machine/cache.h> 95284990Scy#include <machine/frame.h> 96284990Scy#include <machine/md_var.h> 97284990Scy#include <machine/metadata.h> 98284990Scy#include <machine/pv.h> 99284990Scy#include <machine/smp.h> 100284990Scy#include <machine/tlb.h> 101284990Scy#include <machine/tte.h> 102284990Scy#include <machine/tsb.h> 103284990Scy 104284990Scy#define PMAP_DEBUG 105284990Scy 106284990Scy#ifndef PMAP_SHPGPERPROC 107284990Scy#define PMAP_SHPGPERPROC 200 108284990Scy#endif 109284990Scy 110284990Scystruct mem_region { 111284990Scy vm_offset_t mr_start; 112284990Scy vm_offset_t mr_size; 113284990Scy}; 114284990Scy 115284990Scystruct ofw_map { 116284990Scy vm_offset_t om_start; 117284990Scy vm_offset_t om_size; 118284990Scy u_long om_tte; 119284990Scy}; 120284990Scy 121284990Scy/* 122284990Scy * Virtual and physical address of message buffer. 123284990Scy */ 124284990Scystruct msgbuf *msgbufp; 125284990Scyvm_offset_t msgbuf_phys; 126284990Scy 127284990Scy/* 128284990Scy * Physical addresses of first and last available physical page. 129284990Scy */ 130284990Scyvm_offset_t avail_start; 131284990Scyvm_offset_t avail_end; 132284990Scy 133284990Scyint pmap_pagedaemon_waken; 134284990Scy 135284990Scy/* 136284990Scy * Map of physical memory reagions. 137284990Scy */ 138284990Scyvm_offset_t phys_avail[128]; 139284990Scystatic struct mem_region mra[128]; 140284990Scystatic struct ofw_map translations[128]; 141284990Scystatic int translations_size; 142284990Scy 143284990Scy/* 144284990Scy * First and last available kernel virtual addresses. 145284990Scy */ 146284990Scyvm_offset_t virtual_avail; 147284990Scyvm_offset_t virtual_end; 148284990Scyvm_offset_t kernel_vm_end; 149284990Scy 150284990Scy/* 151284990Scy * Kernel pmap. 152284990Scy */ 153284990Scystruct pmap kernel_pmap_store; 154284990Scy 155284990Scystatic boolean_t pmap_initialized = FALSE; 156284990Scy 157284990Scy/* 158284990Scy * Allocate physical memory for use in pmap_bootstrap. 159284990Scy */ 160284990Scystatic vm_offset_t pmap_bootstrap_alloc(vm_size_t size); 161284990Scy 162284990Scy/* 163284990Scy * If user pmap is processed with pmap_remove and with pmap_remove and the 164284990Scy * resident count drops to 0, there are no more pages to remove, so we 165284990Scy * need not continue. 166284990Scy */ 167284990Scy#define PMAP_REMOVE_DONE(pm) \ 168284990Scy ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 169284990Scy 170284990Scy/* 171284990Scy * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 172284990Scy * and pmap_protect() instead of trying each virtual address. 173284990Scy */ 174284990Scy#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 175284990Scy 176284990Scy/* Callbacks for tsb_foreach. */ 177284990Scystatic tsb_callback_t pmap_remove_tte; 178284990Scystatic tsb_callback_t pmap_protect_tte; 179284990Scy 180284990Scy#ifdef PMAP_STATS 181284990Scystatic long pmap_enter_nupdate; 182284990Scystatic long pmap_enter_nreplace; 183284990Scystatic long pmap_enter_nnew; 184284990Scystatic long pmap_ncache_enter; 185284990Scystatic long pmap_ncache_enter_nc; 186284990Scystatic long pmap_niflush; 187284990Scy 188284990ScySYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, "Statistics"); 189284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nupdate, CTLFLAG_RD, 190284990Scy &pmap_enter_nupdate, 0, "Number of pmap_enter() updates"); 191284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nreplace, CTLFLAG_RD, 192284990Scy &pmap_enter_nreplace, 0, "Number of pmap_enter() replacements"); 193284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nnew, CTLFLAG_RD, 194284990Scy &pmap_enter_nnew, 0, "Number of pmap_enter() additions"); 195284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter, CTLFLAG_RD, 196284990Scy &pmap_ncache_enter, 0, "Number of pmap_cache_enter() calls"); 197284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter_nc, CTLFLAG_RD, 198284990Scy &pmap_ncache_enter_nc, 0, "Number of pmap_cache_enter() nc"); 199284990ScySYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_niflush, CTLFLAG_RD, 200284990Scy &pmap_niflush, 0, "Number of pmap I$ flushes"); 201284990Scy 202284990Scy#define PMAP_STATS_INC(var) atomic_add_long(&var, 1) 203284990Scy#else 204284990Scy#define PMAP_STATS_INC(var) 205284990Scy#endif 206284990Scy 207284990Scy/* 208284990Scy * Quick sort callout for comparing memory regions. 209284990Scy */ 210284990Scystatic int mr_cmp(const void *a, const void *b); 211284990Scystatic int om_cmp(const void *a, const void *b); 212284990Scystatic int 213284990Scymr_cmp(const void *a, const void *b) 214284990Scy{ 215284990Scy const struct mem_region *mra; 216284990Scy const struct mem_region *mrb; 217284990Scy 218284990Scy mra = a; 219284990Scy mrb = b; 220284990Scy if (mra->mr_start < mrb->mr_start) 221284990Scy return (-1); 222284990Scy else if (mra->mr_start > mrb->mr_start) 223284990Scy return (1); 224284990Scy else 225284990Scy return (0); 226284990Scy} 227284990Scystatic int 228284990Scyom_cmp(const void *a, const void *b) 229284990Scy{ 230284990Scy const struct ofw_map *oma; 231284990Scy const struct ofw_map *omb; 232284990Scy 233284990Scy oma = a; 234284990Scy omb = b; 235284990Scy if (oma->om_start < omb->om_start) 236284990Scy return (-1); 237284990Scy else if (oma->om_start > omb->om_start) 238284990Scy return (1); 239284990Scy else 240284990Scy return (0); 241284990Scy} 242284990Scy 243284990Scy/* 244284990Scy * Bootstrap the system enough to run with virtual memory. 245284990Scy */ 246284990Scyvoid 247284990Scypmap_bootstrap(vm_offset_t ekva) 248284990Scy{ 249284990Scy struct pmap *pm; 250284990Scy struct tte *tp; 251284990Scy vm_offset_t off; 252284990Scy vm_offset_t pa; 253284990Scy vm_offset_t va; 254284990Scy vm_size_t physsz; 255284990Scy ihandle_t pmem; 256284990Scy ihandle_t vmem; 257284990Scy int sz; 258284990Scy int i; 259284990Scy int j; 260284990Scy 261284990Scy /* 262284990Scy * Set the start and end of kva. The kernel is loaded at the first 263284990Scy * available 4 meg super page, so round up to the end of the page. 264284990Scy */ 265284990Scy virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 266284990Scy virtual_end = VM_MAX_KERNEL_ADDRESS; 267284990Scy 268284990Scy /* 269284990Scy * Find out what physical memory is available from the prom and 270284990Scy * initialize the phys_avail array. This must be done before 271284990Scy * pmap_bootstrap_alloc is called. 272284990Scy */ 273284990Scy if ((pmem = OF_finddevice("/memory")) == -1) 274284990Scy panic("pmap_bootstrap: finddevice /memory"); 275284990Scy if ((sz = OF_getproplen(pmem, "available")) == -1) 276284990Scy panic("pmap_bootstrap: getproplen /memory/available"); 277284990Scy if (sizeof(phys_avail) < sz) 278284990Scy panic("pmap_bootstrap: phys_avail too small"); 279284990Scy if (sizeof(mra) < sz) 280284990Scy panic("pmap_bootstrap: mra too small"); 281284990Scy bzero(mra, sz); 282284990Scy if (OF_getprop(pmem, "available", mra, sz) == -1) 283284990Scy panic("pmap_bootstrap: getprop /memory/available"); 284284990Scy sz /= sizeof(*mra); 285284990Scy CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 286284990Scy qsort(mra, sz, sizeof (*mra), mr_cmp); 287284990Scy physsz = 0; 288284990Scy for (i = 0, j = 0; i < sz; i++, j += 2) { 289284990Scy CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 290284990Scy mra[i].mr_size); 291284990Scy phys_avail[j] = mra[i].mr_start; 292284990Scy phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 293284990Scy physsz += mra[i].mr_size; 294284990Scy } 295284990Scy physmem = btoc(physsz); 296284990Scy 297284990Scy /* 298284990Scy * Allocate the kernel tsb and lock it in the tlb. 299284990Scy */ 300284990Scy pa = pmap_bootstrap_alloc(KVA_PAGES * PAGE_SIZE_4M); 301284990Scy if (pa & PAGE_MASK_4M) 302284990Scy panic("pmap_bootstrap: tsb unaligned\n"); 303284990Scy tsb_kernel_phys = pa; 304284990Scy tsb_kernel = (struct tte *)virtual_avail; 305284990Scy virtual_avail += KVA_PAGES * PAGE_SIZE_4M; 306284990Scy pmap_map_tsb(); 307284990Scy bzero(tsb_kernel, KVA_PAGES * PAGE_SIZE_4M); 308284990Scy 309284990Scy /* 310284990Scy * Enter fake 8k pages for the 4MB kernel pages, so that 311284990Scy * pmap_kextract() will work for them. 312284990Scy */ 313284990Scy for (i = 0; i < kernel_tlb_slots; i++) { 314284990Scy pa = kernel_tlbs[i].te_pa; 315284990Scy va = kernel_tlbs[i].te_va; 316284990Scy for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 317284990Scy tp = tsb_kvtotte(va + off); 318284990Scy tp->tte_vpn = TV_VPN(va + off); 319284990Scy tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 320284990Scy TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 321284990Scy } 322284990Scy } 323284990Scy 324284990Scy /* 325284990Scy * Allocate a kernel stack with guard page for thread0 and map it into 326284990Scy * the kernel tsb. 327284990Scy */ 328284990Scy pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); 329284990Scy kstack0_phys = pa; 330284990Scy kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE); 331284990Scy virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE; 332284990Scy for (i = 0; i < KSTACK_PAGES; i++) { 333284990Scy pa = kstack0_phys + i * PAGE_SIZE; 334284990Scy va = kstack0 + i * PAGE_SIZE; 335284990Scy tp = tsb_kvtotte(va); 336284990Scy tp->tte_vpn = TV_VPN(va); 337284990Scy tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 338284990Scy TD_CP | TD_CV | TD_P | TD_W; 339284990Scy } 340284990Scy 341284990Scy /* 342284990Scy * Allocate the message buffer. 343284990Scy */ 344284990Scy msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 345284990Scy 346284990Scy /* 347284990Scy * Add the prom mappings to the kernel tsb. 348284990Scy */ 349284990Scy if ((vmem = OF_finddevice("/virtual-memory")) == -1) 350284990Scy panic("pmap_bootstrap: finddevice /virtual-memory"); 351284990Scy if ((sz = OF_getproplen(vmem, "translations")) == -1) 352284990Scy panic("pmap_bootstrap: getproplen translations"); 353284990Scy if (sizeof(translations) < sz) 354284990Scy panic("pmap_bootstrap: translations too small"); 355284990Scy bzero(translations, sz); 356284990Scy if (OF_getprop(vmem, "translations", translations, sz) == -1) 357284990Scy panic("pmap_bootstrap: getprop /virtual-memory/translations"); 358284990Scy sz /= sizeof(*translations); 359284990Scy translations_size = sz; 360284990Scy CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 361284990Scy qsort(translations, sz, sizeof (*translations), om_cmp); 362284990Scy for (i = 0; i < sz; i++) { 363284990Scy CTR3(KTR_PMAP, 364284990Scy "translation: start=%#lx size=%#lx tte=%#lx", 365284990Scy translations[i].om_start, translations[i].om_size, 366284990Scy translations[i].om_tte); 367284990Scy if (translations[i].om_start < 0xf0000000) /* XXX!!! */ 368284990Scy continue; 369284990Scy for (off = 0; off < translations[i].om_size; 370284990Scy off += PAGE_SIZE) { 371284990Scy va = translations[i].om_start + off; 372284990Scy tp = tsb_kvtotte(va); 373284990Scy tp->tte_vpn = TV_VPN(va); 374284990Scy tp->tte_data = translations[i].om_tte + off; 375284990Scy } 376284990Scy } 377284990Scy 378284990Scy /* 379284990Scy * Calculate the first and last available physical addresses. 380284990Scy */ 381284990Scy avail_start = phys_avail[0]; 382284990Scy for (i = 0; phys_avail[i + 2] != 0; i += 2) 383284990Scy ; 384284990Scy avail_end = phys_avail[i + 1]; 385284990Scy Maxmem = sparc64_btop(avail_end); 386284990Scy 387284990Scy /* 388284990Scy * Allocate virtual address space for the message buffer. 389284990Scy */ 390284990Scy msgbufp = (struct msgbuf *)virtual_avail; 391284990Scy virtual_avail += round_page(MSGBUF_SIZE); 392284990Scy 393284990Scy /* 394284990Scy * Initialize the kernel pmap (which is statically allocated). 395284990Scy */ 396284990Scy pm = kernel_pmap; 397284990Scy for (i = 0; i < MAXCPU; i++) 398284990Scy pm->pm_context[i] = TLB_CTX_KERNEL; 399284990Scy pm->pm_active = ~0; 400284990Scy 401284990Scy /* XXX flush all non-locked tlb entries */ 402284990Scy} 403284990Scy 404284990Scyvoid 405284990Scypmap_map_tsb(void) 406284990Scy{ 407284990Scy vm_offset_t va; 408284990Scy vm_offset_t pa; 409284990Scy u_long data; 410284990Scy u_long s; 411284990Scy int i; 412284990Scy 413284990Scy s = intr_disable(); 414284990Scy 415284990Scy /* 416284990Scy * Map the 4mb tsb pages. 417284990Scy */ 418284990Scy for (i = 0; i < KVA_PAGES; i++) { 419284990Scy va = (vm_offset_t)tsb_kernel + i * PAGE_SIZE_4M; 420284990Scy pa = tsb_kernel_phys + i * PAGE_SIZE_4M; 421284990Scy /* XXX - cheetah */ 422284990Scy data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 423284990Scy TD_P | TD_W; 424284990Scy stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 425284990Scy TLB_TAR_CTX(TLB_CTX_KERNEL)); 426284990Scy stxa(0, ASI_DTLB_DATA_IN_REG, data); 427284990Scy membar(Sync); 428284990Scy } 429284990Scy 430284990Scy /* 431284990Scy * Load the tsb registers. 432284990Scy */ 433284990Scy stxa(AA_DMMU_TSB, ASI_DMMU, (vm_offset_t)tsb_kernel); 434284990Scy stxa(AA_IMMU_TSB, ASI_IMMU, (vm_offset_t)tsb_kernel); 435 membar(Sync); 436 flush(tsb_kernel); 437 438 /* 439 * Set the secondary context to be the kernel context (needed for 440 * fp block operations in the kernel and the cache code). 441 */ 442 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 443 membar(Sync); 444 445 intr_restore(s); 446} 447 448/* 449 * Allocate a physical page of memory directly from the phys_avail map. 450 * Can only be called from pmap_bootstrap before avail start and end are 451 * calculated. 452 */ 453static vm_offset_t 454pmap_bootstrap_alloc(vm_size_t size) 455{ 456 vm_offset_t pa; 457 int i; 458 459 size = round_page(size); 460 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 461 if (phys_avail[i + 1] - phys_avail[i] < size) 462 continue; 463 pa = phys_avail[i]; 464 phys_avail[i] += size; 465 return (pa); 466 } 467 panic("pmap_bootstrap_alloc"); 468} 469 470void 471pmap_context_rollover(void) 472{ 473 u_long data; 474 u_long tag; 475 int i; 476 477 mtx_assert(&sched_lock, MA_OWNED); 478 CTR0(KTR_PMAP, "pmap_context_rollover"); 479 for (i = 0; i < 64; i++) { 480 /* XXX - cheetah */ 481 data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); 482 tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); 483 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 484 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 485 stxa_sync(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG, 0); 486 data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); 487 tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); 488 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 489 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 490 stxa_sync(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, 0); 491 } 492 PCPU_SET(tlb_ctx, PCPU_GET(tlb_ctx_min)); 493} 494 495static __inline u_int 496pmap_context_alloc(void) 497{ 498 u_int context; 499 500 mtx_assert(&sched_lock, MA_OWNED); 501 context = PCPU_GET(tlb_ctx); 502 if (context + 1 == PCPU_GET(tlb_ctx_max)) 503 pmap_context_rollover(); 504 else 505 PCPU_SET(tlb_ctx, context + 1); 506 return (context); 507} 508 509/* 510 * Initialize the pmap module. 511 */ 512void 513pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 514{ 515 vm_offset_t addr; 516 vm_size_t size; 517 int result; 518 int i; 519 520 for (i = 0; i < vm_page_array_size; i++) { 521 vm_page_t m; 522 523 m = &vm_page_array[i]; 524 STAILQ_INIT(&m->md.tte_list); 525 } 526 527 for (i = 0; i < translations_size; i++) { 528 addr = translations[i].om_start; 529 size = translations[i].om_size; 530 if (addr < 0xf0000000) /* XXX */ 531 continue; 532 result = vm_map_find(kernel_map, NULL, 0, &addr, size, TRUE, 533 VM_PROT_ALL, VM_PROT_ALL, 0); 534 if (result != KERN_SUCCESS || addr != translations[i].om_start) 535 panic("pmap_init: vm_map_find"); 536 } 537 538 pmap_initialized = TRUE; 539} 540 541/* 542 * Initialize the address space (zone) for the pv_entries. Set a 543 * high water mark so that the system can recover from excessive 544 * numbers of pv entries. 545 */ 546void 547pmap_init2(void) 548{ 549} 550 551/* 552 * Extract the physical page address associated with the given 553 * map/virtual_address pair. 554 */ 555vm_offset_t 556pmap_extract(pmap_t pm, vm_offset_t va) 557{ 558 struct tte *tp; 559 560 if (pm == kernel_pmap) 561 return (pmap_kextract(va)); 562 tp = tsb_tte_lookup(pm, va); 563 if (tp == NULL) 564 return (0); 565 else 566 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 567} 568 569/* 570 * Extract the physical page address associated with the given kernel virtual 571 * address. 572 */ 573vm_offset_t 574pmap_kextract(vm_offset_t va) 575{ 576 struct tte *tp; 577 578 tp = tsb_kvtotte(va); 579 if ((tp->tte_data & TD_V) == 0) 580 return (0); 581 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 582} 583 584int 585pmap_cache_enter(vm_page_t m, vm_offset_t va) 586{ 587 struct tte *tp; 588 int c, i; 589 590 CTR2(KTR_PMAP, "pmap_cache_enter: m=%p va=%#lx", m, va); 591 PMAP_STATS_INC(pmap_ncache_enter); 592 for (i = 0, c = 0; i < DCACHE_COLORS; i++) { 593 if (i != DCACHE_COLOR(va)) 594 c += m->md.colors[i]; 595 } 596 m->md.colors[DCACHE_COLOR(va)]++; 597 if (c == 0) { 598 CTR0(KTR_PMAP, "pmap_cache_enter: cacheable"); 599 return (1); 600 } 601 PMAP_STATS_INC(pmap_ncache_enter_nc); 602 if (c != 1) { 603 CTR0(KTR_PMAP, "pmap_cache_enter: already uncacheable"); 604 return (0); 605 } 606 CTR0(KTR_PMAP, "pmap_cache_enter: marking uncacheable"); 607 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 608 tp->tte_data &= ~TD_CV; 609 tlb_page_demap(TLB_DTLB | TLB_ITLB, TTE_GET_PMAP(tp), 610 TTE_GET_VA(tp)); 611 } 612 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 613 return (0); 614} 615 616void 617pmap_cache_remove(vm_page_t m, vm_offset_t va) 618{ 619 620 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 621 m->md.colors[DCACHE_COLOR(va)]); 622 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 623 ("pmap_cache_remove: no mappings %d <= 0", 624 m->md.colors[DCACHE_COLOR(va)])); 625 m->md.colors[DCACHE_COLOR(va)]--; 626} 627 628/* 629 * Map a wired page into kernel virtual address space. 630 */ 631void 632pmap_kenter(vm_offset_t va, vm_offset_t pa) 633{ 634 struct tte *tp; 635 636 tp = tsb_kvtotte(va); 637 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 638 va, pa, tp, tp->tte_data); 639 tp->tte_vpn = TV_VPN(va); 640 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | 641 TD_CV | TD_P | TD_W; 642} 643 644/* 645 * Map a wired page into kernel virtual address space. This additionally 646 * takes a flag argument wich is or'ed to the TTE data. This is used by 647 * bus_space_map(). 648 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 649 * to flush entries that might still be in the cache, if applicable. 650 */ 651void 652pmap_kenter_flags(vm_offset_t va, vm_offset_t pa, u_long flags) 653{ 654 struct tte *tp; 655 656 tp = tsb_kvtotte(va); 657 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 658 va, pa, tp, tp->tte_data); 659 tp->tte_vpn = TV_VPN(va); 660 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 661} 662 663/* 664 * Make a temporary mapping for a physical address. This is only intended 665 * to be used for panic dumps. 666 */ 667void * 668pmap_kenter_temporary(vm_offset_t pa, int i) 669{ 670 671 TODO; 672} 673 674/* 675 * Remove a wired page from kernel virtual address space. 676 */ 677void 678pmap_kremove(vm_offset_t va) 679{ 680 struct tte *tp; 681 682 tp = tsb_kvtotte(va); 683 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 684 tp->tte_data); 685 tp->tte_data = 0; 686 tp->tte_vpn = 0; 687} 688 689/* 690 * Map a range of physical addresses into kernel virtual address space. 691 * 692 * The value passed in *virt is a suggested virtual address for the mapping. 693 * Architectures which can support a direct-mapped physical to virtual region 694 * can return the appropriate address within that region, leaving '*virt' 695 * unchanged. We cannot and therefore do not; *virt is updated with the 696 * first usable address after the mapped region. 697 */ 698vm_offset_t 699pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) 700{ 701 struct tte *tp; 702 vm_offset_t sva; 703 vm_offset_t va; 704 vm_offset_t pa; 705 706 pa = pa_start; 707 sva = *virt; 708 va = sva; 709 for (; pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) { 710 tp = tsb_kvtotte(va); 711 tp->tte_vpn = TV_VPN(va); 712 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 713 TD_CP | TD_CV | TD_P | TD_W; 714 } 715 tlb_range_demap(kernel_pmap, sva, sva + (pa_end - pa_start) - 1); 716 *virt = va; 717 return (sva); 718} 719 720/* 721 * Map a list of wired pages into kernel virtual address space. This is 722 * intended for temporary mappings which do not need page modification or 723 * references recorded. Existing mappings in the region are overwritten. 724 */ 725void 726pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 727{ 728 vm_offset_t va; 729 int i; 730 731 va = sva; 732 for (i = 0; i < count; i++, va += PAGE_SIZE) 733 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 734 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 735} 736 737/* 738 * As above, but take an additional flags argument and call 739 * pmap_kenter_flags(). 740 */ 741void 742pmap_qenter_flags(vm_offset_t sva, vm_page_t *m, int count, u_long fl) 743{ 744 vm_offset_t va; 745 int i; 746 747 va = sva; 748 for (i = 0; i < count; i++, va += PAGE_SIZE) 749 pmap_kenter_flags(va, VM_PAGE_TO_PHYS(m[i]), fl); 750 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 751} 752 753/* 754 * Remove page mappings from kernel virtual address space. Intended for 755 * temporary mappings entered by pmap_qenter. 756 */ 757void 758pmap_qremove(vm_offset_t sva, int count) 759{ 760 vm_offset_t va; 761 int i; 762 763 va = sva; 764 for (i = 0; i < count; i++, va += PAGE_SIZE) 765 pmap_kremove(va); 766 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 767} 768 769/* 770 * Create the uarea for a new process. 771 * This routine directly affects the fork perf for a process. 772 */ 773void 774pmap_new_proc(struct proc *p) 775{ 776 vm_page_t ma[UAREA_PAGES]; 777 vm_object_t upobj; 778 vm_offset_t up; 779 vm_page_t m; 780 u_int i; 781 782 /* 783 * Allocate object for the upages. 784 */ 785 upobj = p->p_upages_obj; 786 if (upobj == NULL) { 787 upobj = vm_object_allocate(OBJT_DEFAULT, UAREA_PAGES); 788 p->p_upages_obj = upobj; 789 } 790 791 /* 792 * Get a kernel virtual address for the U area for this process. 793 */ 794 up = (vm_offset_t)p->p_uarea; 795 if (up == 0) { 796 up = kmem_alloc_nofault(kernel_map, UAREA_PAGES * PAGE_SIZE); 797 if (up == 0) 798 panic("pmap_new_proc: upage allocation failed"); 799 p->p_uarea = (struct user *)up; 800 } 801 802 for (i = 0; i < UAREA_PAGES; i++) { 803 /* 804 * Get a uarea page. 805 */ 806 m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 807 ma[i] = m; 808 809 /* 810 * Wire the page. 811 */ 812 m->wire_count++; 813 cnt.v_wire_count++; 814 815 vm_page_wakeup(m); 816 vm_page_flag_clear(m, PG_ZERO); 817 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 818 m->valid = VM_PAGE_BITS_ALL; 819 } 820 821 /* 822 * Enter the pages into the kernel address space. 823 */ 824 pmap_qenter(up, ma, UAREA_PAGES); 825} 826 827/* 828 * Dispose the uarea for a process that has exited. 829 * This routine directly impacts the exit perf of a process. 830 */ 831void 832pmap_dispose_proc(struct proc *p) 833{ 834 vm_object_t upobj; 835 vm_offset_t up; 836 vm_page_t m; 837 int i; 838 839 upobj = p->p_upages_obj; 840 up = (vm_offset_t)p->p_uarea; 841 for (i = 0; i < UAREA_PAGES; i++) { 842 m = vm_page_lookup(upobj, i); 843 if (m == NULL) 844 panic("pmap_dispose_proc: upage already missing?"); 845 vm_page_busy(m); 846 vm_page_unwire(m, 0); 847 vm_page_free(m); 848 } 849 pmap_qremove(up, UAREA_PAGES); 850 851 /* 852 * If the process got swapped out some of its UPAGES might have gotten 853 * swapped. Just get rid of the object to clean up the swap use 854 * proactively. NOTE! might block waiting for paging I/O to complete. 855 */ 856 if (upobj->type == OBJT_SWAP) { 857 p->p_upages_obj = NULL; 858 vm_object_deallocate(upobj); 859 } 860} 861 862/* 863 * Allow the uarea for a process to be prejudicially paged out. 864 */ 865void 866pmap_swapout_proc(struct proc *p) 867{ 868 vm_object_t upobj; 869 vm_offset_t up; 870 vm_page_t m; 871 int i; 872 873 upobj = p->p_upages_obj; 874 up = (vm_offset_t)p->p_uarea; 875 for (i = 0; i < UAREA_PAGES; i++) { 876 m = vm_page_lookup(upobj, i); 877 if (m == NULL) 878 panic("pmap_swapout_proc: upage already missing?"); 879 vm_page_dirty(m); 880 vm_page_unwire(m, 0); 881 } 882 pmap_qremove(up, UAREA_PAGES); 883} 884 885/* 886 * Bring the uarea for a specified process back in. 887 */ 888void 889pmap_swapin_proc(struct proc *p) 890{ 891 vm_page_t ma[UAREA_PAGES]; 892 vm_object_t upobj; 893 vm_offset_t up; 894 vm_page_t m; 895 int rv; 896 int i; 897 898 upobj = p->p_upages_obj; 899 up = (vm_offset_t)p->p_uarea; 900 for (i = 0; i < UAREA_PAGES; i++) { 901 m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 902 if (m->valid != VM_PAGE_BITS_ALL) { 903 rv = vm_pager_get_pages(upobj, &m, 1, 0); 904 if (rv != VM_PAGER_OK) 905 panic("pmap_swapin_proc: cannot get upage"); 906 m = vm_page_lookup(upobj, i); 907 m->valid = VM_PAGE_BITS_ALL; 908 } 909 ma[i] = m; 910 vm_page_wire(m); 911 vm_page_wakeup(m); 912 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 913 } 914 pmap_qenter(up, ma, UAREA_PAGES); 915} 916 917/* 918 * Create the kernel stack and pcb for a new thread. 919 * This routine directly affects the fork perf for a process and 920 * create performance for a thread. 921 */ 922void 923pmap_new_thread(struct thread *td) 924{ 925 vm_page_t ma[KSTACK_PAGES]; 926 vm_object_t ksobj; 927 vm_offset_t ks; 928 vm_page_t m; 929 u_int i; 930 931 /* 932 * Allocate object for the kstack, 933 */ 934 ksobj = td->td_kstack_obj; 935 if (ksobj == NULL) { 936 ksobj = vm_object_allocate(OBJT_DEFAULT, KSTACK_PAGES); 937 td->td_kstack_obj = ksobj; 938 } 939 940 /* 941 * Get a kernel virtual address for the kstack for this thread. 942 */ 943 ks = td->td_kstack; 944 if (ks == 0) { 945 ks = kmem_alloc_nofault(kernel_map, 946 (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE); 947 if (ks == 0) 948 panic("pmap_new_thread: kstack allocation failed"); 949 if (KSTACK_GUARD_PAGES != 0) { 950 tlb_page_demap(TLB_DTLB, kernel_pmap, ks); 951 ks += KSTACK_GUARD_PAGES * PAGE_SIZE; 952 } 953 td->td_kstack = ks; 954 } 955 956 for (i = 0; i < KSTACK_PAGES; i++) { 957 /* 958 * Get a kernel stack page. 959 */ 960 m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 961 ma[i] = m; 962 963 /* 964 * Wire the page. 965 */ 966 m->wire_count++; 967 cnt.v_wire_count++; 968 969 vm_page_wakeup(m); 970 vm_page_flag_clear(m, PG_ZERO); 971 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 972 m->valid = VM_PAGE_BITS_ALL; 973 } 974 975 /* 976 * Enter the page into the kernel address space. 977 */ 978 pmap_qenter(ks, ma, KSTACK_PAGES); 979} 980 981/* 982 * Dispose the kernel stack for a thread that has exited. 983 * This routine directly impacts the exit perf of a process and thread. 984 */ 985void 986pmap_dispose_thread(struct thread *td) 987{ 988 vm_object_t ksobj; 989 vm_offset_t ks; 990 vm_page_t m; 991 int i; 992 993 ksobj = td->td_kstack_obj; 994 ks = td->td_kstack; 995 for (i = 0; i < KSTACK_PAGES; i++) { 996 m = vm_page_lookup(ksobj, i); 997 if (m == NULL) 998 panic("pmap_dispose_proc: kstack already missing?"); 999 vm_page_busy(m); 1000 vm_page_unwire(m, 0); 1001 vm_page_free(m); 1002 } 1003 pmap_qremove(ks, KSTACK_PAGES); 1004 1005 /* 1006 * If the thread got swapped out some of its KSTACK might have gotten 1007 * swapped. Just get rid of the object to clean up the swap use 1008 * proactively. NOTE! might block waiting for paging I/O to complete. 1009 */ 1010 if (ksobj->type == OBJT_SWAP) { 1011 td->td_kstack_obj = NULL; 1012 vm_object_deallocate(ksobj); 1013 } 1014} 1015 1016/* 1017 * Allow the kernel stack for a thread to be prejudicially paged out. 1018 */ 1019void 1020pmap_swapout_thread(struct thread *td) 1021{ 1022 vm_object_t ksobj; 1023 vm_offset_t ks; 1024 vm_page_t m; 1025 int i; 1026 1027 ksobj = td->td_kstack_obj; 1028 ks = (vm_offset_t)td->td_kstack; 1029 for (i = 0; i < KSTACK_PAGES; i++) { 1030 m = vm_page_lookup(ksobj, i); 1031 if (m == NULL) 1032 panic("pmap_swapout_thread: kstack already missing?"); 1033 vm_page_dirty(m); 1034 vm_page_unwire(m, 0); 1035 } 1036 pmap_qremove(ks, KSTACK_PAGES); 1037} 1038 1039/* 1040 * Bring the kernel stack for a specified thread back in. 1041 */ 1042void 1043pmap_swapin_thread(struct thread *td) 1044{ 1045 vm_page_t ma[KSTACK_PAGES]; 1046 vm_object_t ksobj; 1047 vm_offset_t ks; 1048 vm_page_t m; 1049 int rv; 1050 int i; 1051 1052 ksobj = td->td_kstack_obj; 1053 ks = td->td_kstack; 1054 for (i = 0; i < KSTACK_PAGES; i++) { 1055 m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 1056 if (m->valid != VM_PAGE_BITS_ALL) { 1057 rv = vm_pager_get_pages(ksobj, &m, 1, 0); 1058 if (rv != VM_PAGER_OK) 1059 panic("pmap_swapin_proc: cannot get kstack"); 1060 m = vm_page_lookup(ksobj, i); 1061 m->valid = VM_PAGE_BITS_ALL; 1062 } 1063 ma[i] = m; 1064 vm_page_wire(m); 1065 vm_page_wakeup(m); 1066 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 1067 } 1068 pmap_qenter(ks, ma, KSTACK_PAGES); 1069} 1070 1071/* 1072 * Initialize the pmap associated with process 0. 1073 */ 1074void 1075pmap_pinit0(pmap_t pm) 1076{ 1077 int i; 1078 1079 for (i = 0; i < MAXCPU; i++) 1080 pm->pm_context[i] = 0; 1081 pm->pm_active = 0; 1082 pm->pm_tsb = NULL; 1083 pm->pm_tsb_obj = NULL; 1084 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1085} 1086 1087/* 1088 * Initialize a preallocated and zeroed pmap structure, uch as one in a 1089 * vmspace structure. 1090 */ 1091void 1092pmap_pinit(pmap_t pm) 1093{ 1094 vm_page_t ma[TSB_PAGES]; 1095 vm_page_t m; 1096 int i; 1097 1098 /* 1099 * Allocate kva space for the tsb. 1100 */ 1101 if (pm->pm_tsb == NULL) { 1102 pm->pm_tsb = (struct tte *)kmem_alloc_pageable(kernel_map, 1103 TSB_BSIZE); 1104 } 1105 1106 /* 1107 * Allocate an object for it. 1108 */ 1109 if (pm->pm_tsb_obj == NULL) 1110 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1111 1112 for (i = 0; i < TSB_PAGES; i++) { 1113 m = vm_page_grab(pm->pm_tsb_obj, i, 1114 VM_ALLOC_RETRY | VM_ALLOC_ZERO); 1115 if ((m->flags & PG_ZERO) == 0) 1116 pmap_zero_page(m); 1117 1118 m->wire_count++; 1119 cnt.v_wire_count++; 1120 1121 vm_page_flag_clear(m, PG_MAPPED | PG_BUSY); 1122 m->valid = VM_PAGE_BITS_ALL; 1123 1124 ma[i] = m; 1125 } 1126 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1127 1128 for (i = 0; i < MAXCPU; i++) 1129 pm->pm_context[i] = -1; 1130 pm->pm_active = 0; 1131 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1132} 1133 1134void 1135pmap_pinit2(pmap_t pmap) 1136{ 1137 /* XXX: Remove this stub when no longer called */ 1138} 1139 1140/* 1141 * Release any resources held by the given physical map. 1142 * Called when a pmap initialized by pmap_pinit is being released. 1143 * Should only be called if the map contains no valid mappings. 1144 */ 1145void 1146pmap_release(pmap_t pm) 1147{ 1148 vm_object_t obj; 1149 vm_page_t m; 1150 1151 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1152 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1153 obj = pm->pm_tsb_obj; 1154 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1155 KASSERT(pmap_resident_count(pm) == 0, 1156 ("pmap_release: resident pages %ld != 0", 1157 pmap_resident_count(pm))); 1158 TAILQ_FOREACH(m, &obj->memq, listq) { 1159 if (vm_page_sleep_busy(m, FALSE, "pmaprl")) 1160 continue; 1161 vm_page_busy(m); 1162 KASSERT(m->hold_count == 0, 1163 ("pmap_release: freeing held tsb page")); 1164 m->wire_count--; 1165 cnt.v_wire_count--; 1166 vm_page_free_zero(m); 1167 } 1168 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1169} 1170 1171/* 1172 * Grow the number of kernel page table entries. Unneeded. 1173 */ 1174void 1175pmap_growkernel(vm_offset_t addr) 1176{ 1177} 1178 1179/* 1180 * This routine is very drastic, but can save the system 1181 * in a pinch. 1182 */ 1183void 1184pmap_collect(void) 1185{ 1186} 1187 1188static int 1189pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1190 vm_offset_t va) 1191{ 1192 vm_page_t m; 1193 1194 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1195 if ((tp->tte_data & TD_WIRED) != 0) 1196 pm->pm_stats.wired_count--; 1197 if ((tp->tte_data & TD_PV) != 0) { 1198 if ((tp->tte_data & TD_W) != 0 && 1199 pmap_track_modified(pm, va)) 1200 vm_page_dirty(m); 1201 if ((tp->tte_data & TD_REF) != 0) 1202 vm_page_flag_set(m, PG_REFERENCED); 1203 pv_remove(pm, m, tp); 1204 pmap_cache_remove(m, va); 1205 } 1206 TTE_ZERO(tp); 1207 if (PMAP_REMOVE_DONE(pm)) 1208 return (0); 1209 return (1); 1210} 1211 1212/* 1213 * Remove the given range of addresses from the specified map. 1214 */ 1215void 1216pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1217{ 1218 struct tte *tp; 1219 vm_offset_t va; 1220 1221 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1222 pm->pm_context[PCPU_GET(cpuid)], start, end); 1223 if (PMAP_REMOVE_DONE(pm)) 1224 return; 1225 if (end - start > PMAP_TSB_THRESH) { 1226 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1227 tlb_context_demap(pm); 1228 } else { 1229 for (va = start; va < end; va += PAGE_SIZE) { 1230 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1231 if (!pmap_remove_tte(pm, NULL, tp, va)) 1232 break; 1233 } 1234 } 1235 tlb_range_demap(pm, start, end - 1); 1236 } 1237} 1238 1239static int 1240pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1241 vm_offset_t va) 1242{ 1243 vm_page_t m; 1244 1245 if ((tp->tte_data & TD_PV) != 0) { 1246 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1247 if ((tp->tte_data & TD_REF) != 0) { 1248 vm_page_flag_set(m, PG_REFERENCED); 1249 tp->tte_data &= ~TD_REF; 1250 } 1251 if ((tp->tte_data & TD_W) != 0 && 1252 pmap_track_modified(pm, va)) { 1253 vm_page_dirty(m); 1254 } 1255 } 1256 tp->tte_data &= ~(TD_W | TD_SW); 1257 return (0); 1258} 1259 1260/* 1261 * Set the physical protection on the specified range of this map as requested. 1262 */ 1263void 1264pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1265{ 1266 vm_offset_t va; 1267 struct tte *tp; 1268 1269 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1270 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1271 1272 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1273 pmap_remove(pm, sva, eva); 1274 return; 1275 } 1276 1277 if (prot & VM_PROT_WRITE) 1278 return; 1279 1280 if (eva - sva > PMAP_TSB_THRESH) { 1281 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1282 tlb_context_demap(pm); 1283 } else { 1284 for (va = sva; va < eva; va += PAGE_SIZE) { 1285 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1286 pmap_protect_tte(pm, NULL, tp, va); 1287 } 1288 tlb_range_demap(pm, sva, eva - 1); 1289 } 1290} 1291 1292/* 1293 * Map the given physical page at the specified virtual address in the 1294 * target pmap with the protection requested. If specified the page 1295 * will be wired down. 1296 */ 1297void 1298pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1299 boolean_t wired) 1300{ 1301 struct tte *tp; 1302 vm_offset_t pa; 1303 u_long data; 1304 1305 pa = VM_PAGE_TO_PHYS(m); 1306 CTR6(KTR_PMAP, 1307 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1308 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1309 1310 /* 1311 * If there is an existing mapping, and the physical address has not 1312 * changed, must be protection or wiring change. 1313 */ 1314 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1315 CTR0(KTR_PMAP, "pmap_enter: update"); 1316 PMAP_STATS_INC(pmap_enter_nupdate); 1317 1318 /* 1319 * Wiring change, just update stats. 1320 */ 1321 if (wired) { 1322 if ((tp->tte_data & TD_WIRED) == 0) { 1323 tp->tte_data |= TD_WIRED; 1324 pm->pm_stats.wired_count++; 1325 } 1326 } else { 1327 if ((tp->tte_data & TD_WIRED) != 0) { 1328 tp->tte_data &= ~TD_WIRED; 1329 pm->pm_stats.wired_count--; 1330 } 1331 } 1332 1333 /* 1334 * Save the old bits and clear the ones we're interested in. 1335 */ 1336 data = tp->tte_data; 1337 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1338 1339 /* 1340 * If we're turning off write permissions, sense modify status. 1341 */ 1342 if ((prot & VM_PROT_WRITE) != 0) { 1343 tp->tte_data |= TD_SW; 1344 if (wired) { 1345 tp->tte_data |= TD_W; 1346 } 1347 } else if ((data & TD_W) != 0 && 1348 pmap_track_modified(pm, va)) { 1349 vm_page_dirty(m); 1350 } 1351 1352 /* 1353 * If we're turning on execute permissions, flush the icache. 1354 */ 1355 if ((prot & VM_PROT_EXECUTE) != 0) { 1356 if ((data & TD_EXEC) == 0) { 1357 PMAP_STATS_INC(pmap_niflush); 1358 icache_page_inval(pa); 1359 } 1360 tp->tte_data |= TD_EXEC; 1361 } 1362 1363 /* 1364 * Delete the old mapping. 1365 */ 1366 tlb_tte_demap(tp, pm); 1367 } else { 1368 /* 1369 * If there is an existing mapping, but its for a different 1370 * phsyical address, delete the old mapping. 1371 */ 1372 if (tp != NULL) { 1373 CTR0(KTR_PMAP, "pmap_enter: replace"); 1374 PMAP_STATS_INC(pmap_enter_nreplace); 1375 pmap_remove_tte(pm, NULL, tp, va); 1376 tlb_page_demap(TLB_DTLB | TLB_ITLB, pm, va); 1377 } else { 1378 CTR0(KTR_PMAP, "pmap_enter: new"); 1379 PMAP_STATS_INC(pmap_enter_nnew); 1380 } 1381 1382 /* 1383 * Now set up the data and install the new mapping. 1384 */ 1385 data = TD_V | TD_8K | TD_PA(pa) | TD_CP; 1386 if (pm == kernel_pmap) 1387 data |= TD_P; 1388 if (prot & VM_PROT_WRITE) 1389 data |= TD_SW; 1390 if (prot & VM_PROT_EXECUTE) { 1391 data |= TD_EXEC; 1392 PMAP_STATS_INC(pmap_niflush); 1393 icache_page_inval(pa); 1394 } 1395 1396 /* 1397 * If its wired update stats. We also don't need reference or 1398 * modify tracking for wired mappings, so set the bits now. 1399 */ 1400 if (wired) { 1401 pm->pm_stats.wired_count++; 1402 data |= TD_REF | TD_WIRED; 1403 if ((prot & VM_PROT_WRITE) != 0) 1404 data |= TD_W; 1405 } 1406 1407 tsb_tte_enter(pm, m, va, data); 1408 } 1409} 1410 1411void 1412pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1413 vm_pindex_t pindex, vm_size_t size, int limit) 1414{ 1415 /* XXX */ 1416} 1417 1418void 1419pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) 1420{ 1421 /* XXX */ 1422} 1423 1424/* 1425 * Change the wiring attribute for a map/virtual-address pair. 1426 * The mapping must already exist in the pmap. 1427 */ 1428void 1429pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1430{ 1431 struct tte *tp; 1432 1433 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1434 if (wired) { 1435 if ((tp->tte_data & TD_WIRED) == 0) 1436 pm->pm_stats.wired_count++; 1437 tp->tte_data |= TD_WIRED; 1438 } else { 1439 if ((tp->tte_data & TD_WIRED) != 0) 1440 pm->pm_stats.wired_count--; 1441 tp->tte_data &= ~TD_WIRED; 1442 } 1443 } 1444} 1445 1446static int 1447pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1448{ 1449 vm_page_t m; 1450 u_long data; 1451 1452 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1453 data = tp->tte_data & 1454 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1455 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1456 tsb_tte_enter(dst_pmap, m, va, data); 1457 } 1458 return (1); 1459} 1460 1461void 1462pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1463 vm_size_t len, vm_offset_t src_addr) 1464{ 1465 struct tte *tp; 1466 vm_offset_t va; 1467 1468 if (dst_addr != src_addr) 1469 return; 1470 if (len > PMAP_TSB_THRESH) { 1471 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1472 pmap_copy_tte); 1473 tlb_context_demap(dst_pmap); 1474 } else { 1475 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1476 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1477 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1478 } 1479 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1480 } 1481} 1482 1483/* 1484 * Zero a page of physical memory by temporarily mapping it into the tlb. 1485 */ 1486void 1487pmap_zero_page(vm_page_t m) 1488{ 1489 vm_offset_t pa; 1490 1491 pa = VM_PAGE_TO_PHYS(m); 1492 CTR1(KTR_PMAP, "pmap_zero_page: pa=%#lx", pa); 1493 dcache_page_inval(pa); 1494 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1495} 1496 1497void 1498pmap_zero_page_area(vm_page_t m, int off, int size) 1499{ 1500 vm_offset_t pa; 1501 1502 pa = VM_PAGE_TO_PHYS(m); 1503 CTR3(KTR_PMAP, "pmap_zero_page_area: pa=%#lx off=%#x size=%#x", 1504 pa, off, size); 1505 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1506 dcache_page_inval(pa); 1507 aszero(ASI_PHYS_USE_EC, pa + off, size); 1508} 1509 1510/* 1511 * Copy a page of physical memory by temporarily mapping it into the tlb. 1512 */ 1513void 1514pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1515{ 1516 vm_offset_t dst; 1517 vm_offset_t src; 1518 1519 dst = VM_PAGE_TO_PHYS(mdst); 1520 src = VM_PAGE_TO_PHYS(msrc); 1521 CTR2(KTR_PMAP, "pmap_copy_page: src=%#lx dst=%#lx", src, dst); 1522 dcache_page_inval(dst); 1523 ascopy(ASI_PHYS_USE_EC, src, dst, PAGE_SIZE); 1524} 1525 1526/* 1527 * Make the specified page pageable (or not). Unneeded. 1528 */ 1529void 1530pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 1531 boolean_t pageable) 1532{ 1533} 1534 1535/* 1536 * Returns true if the pmap's pv is one of the first 1537 * 16 pvs linked to from this page. This count may 1538 * be changed upwards or downwards in the future; it 1539 * is only necessary that true be returned for a small 1540 * subset of pmaps for proper page aging. 1541 */ 1542boolean_t 1543pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1544{ 1545 1546 if (m->flags & PG_FICTITIOUS) 1547 return (FALSE); 1548 return (pv_page_exists(pm, m)); 1549} 1550 1551/* 1552 * Remove all pages from specified address space, this aids process exit 1553 * speeds. This is much faster than pmap_remove n the case of running down 1554 * an entire address space. Only works for the current pmap. 1555 */ 1556void 1557pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1558{ 1559} 1560 1561/* 1562 * Lower the permission for all mappings to a given page. 1563 */ 1564void 1565pmap_page_protect(vm_page_t m, vm_prot_t prot) 1566{ 1567 1568 if ((prot & VM_PROT_WRITE) == 0) { 1569 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1570 pv_bit_clear(m, TD_W | TD_SW); 1571 else 1572 pv_remove_all(m); 1573 } 1574} 1575 1576vm_offset_t 1577pmap_phys_address(int ppn) 1578{ 1579 1580 return (sparc64_ptob(ppn)); 1581} 1582 1583/* 1584 * pmap_ts_referenced: 1585 * 1586 * Return a count of reference bits for a page, clearing those bits. 1587 * It is not necessary for every reference bit to be cleared, but it 1588 * is necessary that 0 only be returned when there are truly no 1589 * reference bits set. 1590 * 1591 * XXX: The exact number of bits to check and clear is a matter that 1592 * should be tested and standardized at some point in the future for 1593 * optimal aging of shared pages. 1594 */ 1595 1596int 1597pmap_ts_referenced(vm_page_t m) 1598{ 1599 1600 if (m->flags & PG_FICTITIOUS) 1601 return (0); 1602 return (pv_bit_count(m, TD_REF)); 1603} 1604 1605boolean_t 1606pmap_is_modified(vm_page_t m) 1607{ 1608 1609 if (m->flags & PG_FICTITIOUS) 1610 return FALSE; 1611 return (pv_bit_test(m, TD_W)); 1612} 1613 1614void 1615pmap_clear_modify(vm_page_t m) 1616{ 1617 1618 if (m->flags & PG_FICTITIOUS) 1619 return; 1620 pv_bit_clear(m, TD_W); 1621} 1622 1623void 1624pmap_clear_reference(vm_page_t m) 1625{ 1626 1627 if (m->flags & PG_FICTITIOUS) 1628 return; 1629 pv_bit_clear(m, TD_REF); 1630} 1631 1632int 1633pmap_mincore(pmap_t pm, vm_offset_t addr) 1634{ 1635 TODO; 1636 return (0); 1637} 1638 1639/* 1640 * Activate a user pmap. The pmap must be activated before its address space 1641 * can be accessed in any way. 1642 */ 1643void 1644pmap_activate(struct thread *td) 1645{ 1646 struct vmspace *vm; 1647 vm_offset_t tsb; 1648 u_long context; 1649 pmap_t pm; 1650 1651 /* 1652 * Load all the data we need up front to encourage the compiler to 1653 * not issue any loads while we have interrupts disable below. 1654 */ 1655 vm = td->td_proc->p_vmspace; 1656 pm = &vm->vm_pmap; 1657 tsb = (vm_offset_t)pm->pm_tsb; 1658 1659 KASSERT(pm->pm_active == 0, ("pmap_activate: pmap already active?")); 1660 KASSERT(pm->pm_context[PCPU_GET(cpuid)] != 0, 1661 ("pmap_activate: activating nucleus context?")); 1662 1663 mtx_lock_spin(&sched_lock); 1664 wrpr(pstate, 0, PSTATE_MMU); 1665 mov(tsb, TSB_REG); 1666 wrpr(pstate, 0, PSTATE_KERNEL); 1667 context = pmap_context_alloc(); 1668 pm->pm_context[PCPU_GET(cpuid)] = context; 1669 pm->pm_active |= PCPU_GET(cpumask); 1670 PCPU_SET(vmspace, vm); 1671 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 1672 membar(Sync); 1673 mtx_unlock_spin(&sched_lock); 1674} 1675 1676vm_offset_t 1677pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 1678{ 1679 1680 return (va); 1681} 1682