pmap.c revision 97449
1259701Sdim/* 2259701Sdim * Copyright (c) 1991 Regents of the University of California. 3259701Sdim * All rights reserved. 4259701Sdim * Copyright (c) 1994 John S. Dyson 5259701Sdim * All rights reserved. 6259701Sdim * Copyright (c) 1994 David Greenman 7259701Sdim * All rights reserved. 8259701Sdim * 9259701Sdim * This code is derived from software contributed to Berkeley by 10259701Sdim * the Systems Programming Group of the University of Utah Computer 11259701Sdim * Science Department and William Jolitz of UUNET Technologies Inc. 12259701Sdim * 13259701Sdim * Redistribution and use in source and binary forms, with or without 14259701Sdim * modification, are permitted provided that the following conditions 15259701Sdim * are met: 16259701Sdim * 1. Redistributions of source code must retain the above copyright 17259701Sdim * notice, this list of conditions and the following disclaimer. 18259701Sdim * 2. Redistributions in binary form must reproduce the above copyright 19259701Sdim * notice, this list of conditions and the following disclaimer in the 20259701Sdim * documentation and/or other materials provided with the distribution. 21259701Sdim * 3. All advertising materials mentioning features or use of this software 22259701Sdim * must display the following acknowledgement: 23259701Sdim * This product includes software developed by the University of 24259701Sdim * California, Berkeley and its contributors. 25259701Sdim * 4. Neither the name of the University nor the names of its contributors 26259701Sdim * may be used to endorse or promote products derived from this software 27259701Sdim * without specific prior written permission. 28259701Sdim * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 97449 2002-05-29 06:12:13Z jake $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this module is called upon to 49 * provide software-use-only maps which may or may not be stored in the 50 * same form as hardware maps. These pseudo-maps are used to store 51 * intermediate results from copy operations to and from address spaces. 52 * 53 * Since the information managed by this module is also stored by the 54 * logical address mapping module, this module may throw away valid virtual 55 * to physical mappings at almost any time. However, invalidations of 56 * mappings must be done as requested. 57 * 58 * In order to cope with hardware architectures which make virtual to 59 * physical map invalidates expensive, this module may delay invalidate 60 * reduced protection operations until such time as they are actually 61 * necessary. This module is given full information as to which processors 62 * are currently using which maps, and to when physical maps must be made 63 * correct. 64 */ 65 66#include "opt_msgbuf.h" 67#include "opt_pmap.h" 68 69#include <sys/param.h> 70#include <sys/kernel.h> 71#include <sys/ktr.h> 72#include <sys/lock.h> 73#include <sys/msgbuf.h> 74#include <sys/mutex.h> 75#include <sys/proc.h> 76#include <sys/smp.h> 77#include <sys/sysctl.h> 78#include <sys/systm.h> 79#include <sys/vmmeter.h> 80 81#include <dev/ofw/openfirm.h> 82 83#include <vm/vm.h> 84#include <vm/vm_param.h> 85#include <vm/vm_kern.h> 86#include <vm/vm_page.h> 87#include <vm/vm_map.h> 88#include <vm/vm_object.h> 89#include <vm/vm_extern.h> 90#include <vm/vm_pageout.h> 91#include <vm/vm_pager.h> 92#include <vm/uma.h> 93 94#include <machine/cache.h> 95#include <machine/frame.h> 96#include <machine/md_var.h> 97#include <machine/metadata.h> 98#include <machine/smp.h> 99#include <machine/tlb.h> 100#include <machine/tte.h> 101#include <machine/tsb.h> 102 103#define PMAP_DEBUG 104 105#ifndef PMAP_SHPGPERPROC 106#define PMAP_SHPGPERPROC 200 107#endif 108 109struct mem_region { 110 vm_offset_t mr_start; 111 vm_offset_t mr_size; 112}; 113 114struct ofw_map { 115 vm_offset_t om_start; 116 vm_offset_t om_size; 117 u_long om_tte; 118}; 119 120/* 121 * Virtual and physical address of message buffer. 122 */ 123struct msgbuf *msgbufp; 124vm_offset_t msgbuf_phys; 125 126/* 127 * Physical addresses of first and last available physical page. 128 */ 129vm_offset_t avail_start; 130vm_offset_t avail_end; 131 132int pmap_pagedaemon_waken; 133 134/* 135 * Map of physical memory reagions. 136 */ 137vm_offset_t phys_avail[128]; 138static struct mem_region mra[128]; 139static struct ofw_map translations[128]; 140static int translations_size; 141 142/* 143 * First and last available kernel virtual addresses. 144 */ 145vm_offset_t virtual_avail; 146vm_offset_t virtual_end; 147vm_offset_t kernel_vm_end; 148 149/* 150 * Kernel pmap. 151 */ 152struct pmap kernel_pmap_store; 153 154static boolean_t pmap_initialized = FALSE; 155 156/* 157 * Allocate physical memory for use in pmap_bootstrap. 158 */ 159static vm_offset_t pmap_bootstrap_alloc(vm_size_t size); 160 161/* 162 * If user pmap is processed with pmap_remove and with pmap_remove and the 163 * resident count drops to 0, there are no more pages to remove, so we 164 * need not continue. 165 */ 166#define PMAP_REMOVE_DONE(pm) \ 167 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 168 169/* 170 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 171 * and pmap_protect() instead of trying each virtual address. 172 */ 173#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 174 175#ifdef PMAP_STATS 176static long pmap_enter_nupdate; 177static long pmap_enter_nreplace; 178static long pmap_enter_nnew; 179static long pmap_ncache_enter; 180static long pmap_ncache_enter_nc; 181static long pmap_niflush; 182 183SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, "Statistics"); 184SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nupdate, CTLFLAG_RD, 185 &pmap_enter_nupdate, 0, "Number of pmap_enter() updates"); 186SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nreplace, CTLFLAG_RD, 187 &pmap_enter_nreplace, 0, "Number of pmap_enter() replacements"); 188SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_enter_nnew, CTLFLAG_RD, 189 &pmap_enter_nnew, 0, "Number of pmap_enter() additions"); 190SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter, CTLFLAG_RD, 191 &pmap_ncache_enter, 0, "Number of pmap_cache_enter() calls"); 192SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_ncache_enter_nc, CTLFLAG_RD, 193 &pmap_ncache_enter_nc, 0, "Number of pmap_cache_enter() nc"); 194SYSCTL_LONG(_debug_pmap_stats, OID_AUTO, pmap_niflush, CTLFLAG_RD, 195 &pmap_niflush, 0, "Number of pmap I$ flushes"); 196 197#define PMAP_STATS_INC(var) atomic_add_long(&var, 1) 198#else 199#define PMAP_STATS_INC(var) 200#endif 201 202/* 203 * Quick sort callout for comparing memory regions. 204 */ 205static int mr_cmp(const void *a, const void *b); 206static int om_cmp(const void *a, const void *b); 207static int 208mr_cmp(const void *a, const void *b) 209{ 210 const struct mem_region *mra; 211 const struct mem_region *mrb; 212 213 mra = a; 214 mrb = b; 215 if (mra->mr_start < mrb->mr_start) 216 return (-1); 217 else if (mra->mr_start > mrb->mr_start) 218 return (1); 219 else 220 return (0); 221} 222static int 223om_cmp(const void *a, const void *b) 224{ 225 const struct ofw_map *oma; 226 const struct ofw_map *omb; 227 228 oma = a; 229 omb = b; 230 if (oma->om_start < omb->om_start) 231 return (-1); 232 else if (oma->om_start > omb->om_start) 233 return (1); 234 else 235 return (0); 236} 237 238/* 239 * Bootstrap the system enough to run with virtual memory. 240 */ 241void 242pmap_bootstrap(vm_offset_t ekva) 243{ 244 struct pmap *pm; 245 struct tte *tp; 246 vm_offset_t off; 247 vm_offset_t pa; 248 vm_offset_t va; 249 vm_size_t physsz; 250 ihandle_t pmem; 251 ihandle_t vmem; 252 int sz; 253 int i; 254 int j; 255 256 /* 257 * Set the start and end of kva. The kernel is loaded at the first 258 * available 4 meg super page, so round up to the end of the page. 259 */ 260 virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 261 virtual_end = VM_MAX_KERNEL_ADDRESS; 262 263 /* 264 * Find out what physical memory is available from the prom and 265 * initialize the phys_avail array. This must be done before 266 * pmap_bootstrap_alloc is called. 267 */ 268 if ((pmem = OF_finddevice("/memory")) == -1) 269 panic("pmap_bootstrap: finddevice /memory"); 270 if ((sz = OF_getproplen(pmem, "available")) == -1) 271 panic("pmap_bootstrap: getproplen /memory/available"); 272 if (sizeof(phys_avail) < sz) 273 panic("pmap_bootstrap: phys_avail too small"); 274 if (sizeof(mra) < sz) 275 panic("pmap_bootstrap: mra too small"); 276 bzero(mra, sz); 277 if (OF_getprop(pmem, "available", mra, sz) == -1) 278 panic("pmap_bootstrap: getprop /memory/available"); 279 sz /= sizeof(*mra); 280 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 281 qsort(mra, sz, sizeof (*mra), mr_cmp); 282 physsz = 0; 283 for (i = 0, j = 0; i < sz; i++, j += 2) { 284 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 285 mra[i].mr_size); 286 phys_avail[j] = mra[i].mr_start; 287 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 288 physsz += mra[i].mr_size; 289 } 290 physmem = btoc(physsz); 291 292 /* 293 * Allocate the kernel tsb and lock it in the tlb. 294 */ 295 pa = pmap_bootstrap_alloc(KVA_PAGES * PAGE_SIZE_4M); 296 if (pa & PAGE_MASK_4M) 297 panic("pmap_bootstrap: tsb unaligned\n"); 298 tsb_kernel_phys = pa; 299 tsb_kernel = (struct tte *)virtual_avail; 300 virtual_avail += KVA_PAGES * PAGE_SIZE_4M; 301 pmap_map_tsb(); 302 bzero(tsb_kernel, KVA_PAGES * PAGE_SIZE_4M); 303 304 /* 305 * Enter fake 8k pages for the 4MB kernel pages, so that 306 * pmap_kextract() will work for them. 307 */ 308 for (i = 0; i < kernel_tlb_slots; i++) { 309 pa = kernel_tlbs[i].te_pa; 310 va = kernel_tlbs[i].te_va; 311 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 312 tp = tsb_kvtotte(va + off); 313 tp->tte_vpn = TV_VPN(va + off); 314 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 315 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 316 } 317 } 318 319 /* 320 * Allocate a kernel stack with guard page for thread0 and map it into 321 * the kernel tsb. 322 */ 323 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); 324 kstack0_phys = pa; 325 kstack0 = virtual_avail + (KSTACK_GUARD_PAGES * PAGE_SIZE); 326 virtual_avail += (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE; 327 for (i = 0; i < KSTACK_PAGES; i++) { 328 pa = kstack0_phys + i * PAGE_SIZE; 329 va = kstack0 + i * PAGE_SIZE; 330 tp = tsb_kvtotte(va); 331 tp->tte_vpn = TV_VPN(va); 332 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 333 TD_CP | TD_CV | TD_P | TD_W; 334 } 335 336 /* 337 * Allocate the message buffer. 338 */ 339 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 340 341 /* 342 * Add the prom mappings to the kernel tsb. 343 */ 344 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 345 panic("pmap_bootstrap: finddevice /virtual-memory"); 346 if ((sz = OF_getproplen(vmem, "translations")) == -1) 347 panic("pmap_bootstrap: getproplen translations"); 348 if (sizeof(translations) < sz) 349 panic("pmap_bootstrap: translations too small"); 350 bzero(translations, sz); 351 if (OF_getprop(vmem, "translations", translations, sz) == -1) 352 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 353 sz /= sizeof(*translations); 354 translations_size = sz; 355 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 356 qsort(translations, sz, sizeof (*translations), om_cmp); 357 for (i = 0; i < sz; i++) { 358 CTR3(KTR_PMAP, 359 "translation: start=%#lx size=%#lx tte=%#lx", 360 translations[i].om_start, translations[i].om_size, 361 translations[i].om_tte); 362 if (translations[i].om_start < 0xf0000000) /* XXX!!! */ 363 continue; 364 for (off = 0; off < translations[i].om_size; 365 off += PAGE_SIZE) { 366 va = translations[i].om_start + off; 367 tp = tsb_kvtotte(va); 368 tp->tte_vpn = TV_VPN(va); 369 tp->tte_data = translations[i].om_tte + off; 370 } 371 } 372 373 /* 374 * Calculate the first and last available physical addresses. 375 */ 376 avail_start = phys_avail[0]; 377 for (i = 0; phys_avail[i + 2] != 0; i += 2) 378 ; 379 avail_end = phys_avail[i + 1]; 380 Maxmem = sparc64_btop(avail_end); 381 382 /* 383 * Allocate virtual address space for the message buffer. 384 */ 385 msgbufp = (struct msgbuf *)virtual_avail; 386 virtual_avail += round_page(MSGBUF_SIZE); 387 388 /* 389 * Initialize the kernel pmap (which is statically allocated). 390 */ 391 pm = kernel_pmap; 392 for (i = 0; i < MAXCPU; i++) 393 pm->pm_context[i] = TLB_CTX_KERNEL; 394 pm->pm_active = ~0; 395 396 /* XXX flush all non-locked tlb entries */ 397} 398 399void 400pmap_map_tsb(void) 401{ 402 vm_offset_t va; 403 vm_offset_t pa; 404 u_long data; 405 u_long s; 406 int i; 407 408 s = intr_disable(); 409 410 /* 411 * Map the 4mb tsb pages. 412 */ 413 for (i = 0; i < KVA_PAGES; i++) { 414 va = (vm_offset_t)tsb_kernel + i * PAGE_SIZE_4M; 415 pa = tsb_kernel_phys + i * PAGE_SIZE_4M; 416 /* XXX - cheetah */ 417 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 418 TD_P | TD_W; 419 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 420 TLB_TAR_CTX(TLB_CTX_KERNEL)); 421 stxa(0, ASI_DTLB_DATA_IN_REG, data); 422 membar(Sync); 423 } 424 425 /* 426 * Load the tsb registers. 427 */ 428 stxa(AA_DMMU_TSB, ASI_DMMU, (vm_offset_t)tsb_kernel); 429 stxa(AA_IMMU_TSB, ASI_IMMU, (vm_offset_t)tsb_kernel); 430 membar(Sync); 431 flush(tsb_kernel); 432 433 /* 434 * Set the secondary context to be the kernel context (needed for 435 * fp block operations in the kernel and the cache code). 436 */ 437 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 438 membar(Sync); 439 440 intr_restore(s); 441} 442 443/* 444 * Allocate a physical page of memory directly from the phys_avail map. 445 * Can only be called from pmap_bootstrap before avail start and end are 446 * calculated. 447 */ 448static vm_offset_t 449pmap_bootstrap_alloc(vm_size_t size) 450{ 451 vm_offset_t pa; 452 int i; 453 454 size = round_page(size); 455 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 456 if (phys_avail[i + 1] - phys_avail[i] < size) 457 continue; 458 pa = phys_avail[i]; 459 phys_avail[i] += size; 460 return (pa); 461 } 462 panic("pmap_bootstrap_alloc"); 463} 464 465void 466pmap_context_rollover(void) 467{ 468 u_long data; 469 u_long tag; 470 int i; 471 472 mtx_assert(&sched_lock, MA_OWNED); 473 CTR0(KTR_PMAP, "pmap_context_rollover"); 474 for (i = 0; i < 64; i++) { 475 /* XXX - cheetah */ 476 data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); 477 tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); 478 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 479 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 480 stxa_sync(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG, 0); 481 data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); 482 tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); 483 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 484 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 485 stxa_sync(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, 0); 486 } 487 PCPU_SET(tlb_ctx, PCPU_GET(tlb_ctx_min)); 488} 489 490static __inline u_int 491pmap_context_alloc(void) 492{ 493 u_int context; 494 495 mtx_assert(&sched_lock, MA_OWNED); 496 context = PCPU_GET(tlb_ctx); 497 if (context + 1 == PCPU_GET(tlb_ctx_max)) 498 pmap_context_rollover(); 499 else 500 PCPU_SET(tlb_ctx, context + 1); 501 return (context); 502} 503 504/* 505 * Initialize the pmap module. 506 */ 507void 508pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 509{ 510 vm_offset_t addr; 511 vm_size_t size; 512 int result; 513 int i; 514 515 for (i = 0; i < vm_page_array_size; i++) { 516 vm_page_t m; 517 518 m = &vm_page_array[i]; 519 STAILQ_INIT(&m->md.tte_list); 520 m->md.flags = 0; 521 } 522 523 for (i = 0; i < translations_size; i++) { 524 addr = translations[i].om_start; 525 size = translations[i].om_size; 526 if (addr < 0xf0000000) /* XXX */ 527 continue; 528 result = vm_map_find(kernel_map, NULL, 0, &addr, size, TRUE, 529 VM_PROT_ALL, VM_PROT_ALL, 0); 530 if (result != KERN_SUCCESS || addr != translations[i].om_start) 531 panic("pmap_init: vm_map_find"); 532 } 533 534 pmap_initialized = TRUE; 535} 536 537/* 538 * Initialize the address space (zone) for the pv_entries. Set a 539 * high water mark so that the system can recover from excessive 540 * numbers of pv entries. 541 */ 542void 543pmap_init2(void) 544{ 545} 546 547/* 548 * Extract the physical page address associated with the given 549 * map/virtual_address pair. 550 */ 551vm_offset_t 552pmap_extract(pmap_t pm, vm_offset_t va) 553{ 554 struct tte *tp; 555 556 if (pm == kernel_pmap) 557 return (pmap_kextract(va)); 558 tp = tsb_tte_lookup(pm, va); 559 if (tp == NULL) 560 return (0); 561 else 562 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 563} 564 565/* 566 * Extract the physical page address associated with the given kernel virtual 567 * address. 568 */ 569vm_offset_t 570pmap_kextract(vm_offset_t va) 571{ 572 struct tte *tp; 573 574 tp = tsb_kvtotte(va); 575 if ((tp->tte_data & TD_V) == 0) 576 return (0); 577 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 578} 579 580int 581pmap_cache_enter(vm_page_t m, vm_offset_t va) 582{ 583 struct tte *tp; 584 int c, i; 585 586 CTR2(KTR_PMAP, "pmap_cache_enter: m=%p va=%#lx", m, va); 587 PMAP_STATS_INC(pmap_ncache_enter); 588 for (i = 0, c = 0; i < DCACHE_COLORS; i++) { 589 if (i != DCACHE_COLOR(va)) 590 c += m->md.colors[i]; 591 } 592 m->md.colors[DCACHE_COLOR(va)]++; 593 if (c == 0) { 594 CTR0(KTR_PMAP, "pmap_cache_enter: cacheable"); 595 return (1); 596 } 597 PMAP_STATS_INC(pmap_ncache_enter_nc); 598 if (c != 1) { 599 CTR0(KTR_PMAP, "pmap_cache_enter: already uncacheable"); 600 return (0); 601 } 602 CTR0(KTR_PMAP, "pmap_cache_enter: marking uncacheable"); 603 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 604 tp->tte_data &= ~TD_CV; 605 tlb_page_demap(TLB_DTLB | TLB_ITLB, TTE_GET_PMAP(tp), 606 TTE_GET_VA(tp)); 607 } 608 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 609 m->md.flags |= PG_UNCACHEABLE; 610 return (0); 611} 612 613void 614pmap_cache_remove(vm_page_t m, vm_offset_t va) 615{ 616 struct tte *tp; 617 int c, i; 618 619 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 620 m->md.colors[DCACHE_COLOR(va)]); 621 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 622 ("pmap_cache_remove: no mappings %d <= 0", 623 m->md.colors[DCACHE_COLOR(va)])); 624 m->md.colors[DCACHE_COLOR(va)]--; 625 for (i = 0, c = 0; i < DCACHE_COLORS; i++) { 626 if (m->md.colors[i] != 0) 627 c++; 628 } 629 if (c != 1 || (m->md.flags & PG_UNCACHEABLE) == 0) 630 return; 631 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 632 tp->tte_data |= TD_CV; 633 tlb_page_demap(TLB_DTLB | TLB_ITLB, TTE_GET_PMAP(tp), 634 TTE_GET_VA(tp)); 635 } 636 m->md.flags &= ~PG_UNCACHEABLE; 637} 638 639/* 640 * Map a wired page into kernel virtual address space. 641 */ 642void 643pmap_kenter(vm_offset_t va, vm_offset_t pa) 644{ 645 vm_offset_t ova; 646 struct tte *tp; 647 vm_page_t om; 648 vm_page_t m; 649 u_long data; 650 651 tp = tsb_kvtotte(va); 652 m = PHYS_TO_VM_PAGE(pa); 653 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 654 va, pa, tp, tp->tte_data); 655 if ((tp->tte_data & TD_V) != 0) { 656 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 657 ova = TTE_GET_VA(tp); 658 STAILQ_REMOVE(&om->md.tte_list, tp, tte, tte_link); 659 pmap_cache_remove(om, ova); 660 if (va != ova) 661 tlb_page_demap(TLB_DTLB, kernel_pmap, ova); 662 } 663 data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; 664 if (pmap_cache_enter(m, va) != 0) 665 data |= TD_CV; 666 tp->tte_vpn = TV_VPN(va); 667 tp->tte_data = data; 668 STAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 669 tp->tte_pmap = kernel_pmap; 670} 671 672/* 673 * Map a wired page into kernel virtual address space. This additionally 674 * takes a flag argument wich is or'ed to the TTE data. This is used by 675 * bus_space_map(). 676 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 677 * to flush entries that might still be in the cache, if applicable. 678 */ 679void 680pmap_kenter_flags(vm_offset_t va, vm_offset_t pa, u_long flags) 681{ 682 struct tte *tp; 683 684 tp = tsb_kvtotte(va); 685 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 686 va, pa, tp, tp->tte_data); 687 tp->tte_vpn = TV_VPN(va); 688 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 689} 690 691/* 692 * Make a temporary mapping for a physical address. This is only intended 693 * to be used for panic dumps. 694 */ 695void * 696pmap_kenter_temporary(vm_offset_t pa, int i) 697{ 698 699 TODO; 700} 701 702/* 703 * Remove a wired page from kernel virtual address space. 704 */ 705void 706pmap_kremove(vm_offset_t va) 707{ 708 struct tte *tp; 709 vm_page_t m; 710 711 tp = tsb_kvtotte(va); 712 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 713 tp->tte_data); 714 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 715 STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); 716 pmap_cache_remove(m, va); 717 TTE_ZERO(tp); 718} 719 720/* 721 * Map a range of physical addresses into kernel virtual address space. 722 * 723 * The value passed in *virt is a suggested virtual address for the mapping. 724 * Architectures which can support a direct-mapped physical to virtual region 725 * can return the appropriate address within that region, leaving '*virt' 726 * unchanged. We cannot and therefore do not; *virt is updated with the 727 * first usable address after the mapped region. 728 */ 729vm_offset_t 730pmap_map(vm_offset_t *virt, vm_offset_t pa_start, vm_offset_t pa_end, int prot) 731{ 732 struct tte *tp; 733 vm_offset_t sva; 734 vm_offset_t va; 735 vm_offset_t pa; 736 737 pa = pa_start; 738 sva = *virt; 739 va = sva; 740 for (; pa < pa_end; pa += PAGE_SIZE, va += PAGE_SIZE) { 741 tp = tsb_kvtotte(va); 742 tp->tte_vpn = TV_VPN(va); 743 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 744 TD_CP | TD_CV | TD_P | TD_W; 745 } 746 tlb_range_demap(kernel_pmap, sva, sva + (pa_end - pa_start) - 1); 747 *virt = va; 748 return (sva); 749} 750 751/* 752 * Map a list of wired pages into kernel virtual address space. This is 753 * intended for temporary mappings which do not need page modification or 754 * references recorded. Existing mappings in the region are overwritten. 755 */ 756void 757pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 758{ 759 vm_offset_t va; 760 int i; 761 762 va = sva; 763 for (i = 0; i < count; i++, va += PAGE_SIZE) 764 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 765 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 766} 767 768/* 769 * As above, but take an additional flags argument and call 770 * pmap_kenter_flags(). 771 */ 772void 773pmap_qenter_flags(vm_offset_t sva, vm_page_t *m, int count, u_long fl) 774{ 775 vm_offset_t va; 776 int i; 777 778 va = sva; 779 for (i = 0; i < count; i++, va += PAGE_SIZE) 780 pmap_kenter_flags(va, VM_PAGE_TO_PHYS(m[i]), fl); 781 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 782} 783 784/* 785 * Remove page mappings from kernel virtual address space. Intended for 786 * temporary mappings entered by pmap_qenter. 787 */ 788void 789pmap_qremove(vm_offset_t sva, int count) 790{ 791 vm_offset_t va; 792 int i; 793 794 va = sva; 795 for (i = 0; i < count; i++, va += PAGE_SIZE) 796 pmap_kremove(va); 797 tlb_range_demap(kernel_pmap, sva, sva + (count * PAGE_SIZE) - 1); 798} 799 800/* 801 * Create the uarea for a new process. 802 * This routine directly affects the fork perf for a process. 803 */ 804void 805pmap_new_proc(struct proc *p) 806{ 807 vm_page_t ma[UAREA_PAGES]; 808 vm_object_t upobj; 809 vm_offset_t up; 810 vm_page_t m; 811 u_int i; 812 813 /* 814 * Allocate object for the upages. 815 */ 816 upobj = p->p_upages_obj; 817 if (upobj == NULL) { 818 upobj = vm_object_allocate(OBJT_DEFAULT, UAREA_PAGES); 819 p->p_upages_obj = upobj; 820 } 821 822 /* 823 * Get a kernel virtual address for the U area for this process. 824 */ 825 up = (vm_offset_t)p->p_uarea; 826 if (up == 0) { 827 up = kmem_alloc_nofault(kernel_map, UAREA_PAGES * PAGE_SIZE); 828 if (up == 0) 829 panic("pmap_new_proc: upage allocation failed"); 830 p->p_uarea = (struct user *)up; 831 } 832 833 for (i = 0; i < UAREA_PAGES; i++) { 834 /* 835 * Get a uarea page. 836 */ 837 m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 838 ma[i] = m; 839 840 /* 841 * Wire the page. 842 */ 843 m->wire_count++; 844 cnt.v_wire_count++; 845 846 vm_page_wakeup(m); 847 vm_page_flag_clear(m, PG_ZERO); 848 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 849 m->valid = VM_PAGE_BITS_ALL; 850 } 851 852 /* 853 * Enter the pages into the kernel address space. 854 */ 855 pmap_qenter(up, ma, UAREA_PAGES); 856} 857 858/* 859 * Dispose the uarea for a process that has exited. 860 * This routine directly impacts the exit perf of a process. 861 */ 862void 863pmap_dispose_proc(struct proc *p) 864{ 865 vm_object_t upobj; 866 vm_offset_t up; 867 vm_page_t m; 868 int i; 869 870 upobj = p->p_upages_obj; 871 up = (vm_offset_t)p->p_uarea; 872 for (i = 0; i < UAREA_PAGES; i++) { 873 m = vm_page_lookup(upobj, i); 874 if (m == NULL) 875 panic("pmap_dispose_proc: upage already missing?"); 876 vm_page_busy(m); 877 vm_page_unwire(m, 0); 878 vm_page_free(m); 879 } 880 pmap_qremove(up, UAREA_PAGES); 881 882 /* 883 * If the process got swapped out some of its UPAGES might have gotten 884 * swapped. Just get rid of the object to clean up the swap use 885 * proactively. NOTE! might block waiting for paging I/O to complete. 886 */ 887 if (upobj->type == OBJT_SWAP) { 888 p->p_upages_obj = NULL; 889 vm_object_deallocate(upobj); 890 } 891} 892 893/* 894 * Allow the uarea for a process to be prejudicially paged out. 895 */ 896void 897pmap_swapout_proc(struct proc *p) 898{ 899 vm_object_t upobj; 900 vm_offset_t up; 901 vm_page_t m; 902 int i; 903 904 upobj = p->p_upages_obj; 905 up = (vm_offset_t)p->p_uarea; 906 for (i = 0; i < UAREA_PAGES; i++) { 907 m = vm_page_lookup(upobj, i); 908 if (m == NULL) 909 panic("pmap_swapout_proc: upage already missing?"); 910 vm_page_dirty(m); 911 vm_page_unwire(m, 0); 912 } 913 pmap_qremove(up, UAREA_PAGES); 914} 915 916/* 917 * Bring the uarea for a specified process back in. 918 */ 919void 920pmap_swapin_proc(struct proc *p) 921{ 922 vm_page_t ma[UAREA_PAGES]; 923 vm_object_t upobj; 924 vm_offset_t up; 925 vm_page_t m; 926 int rv; 927 int i; 928 929 upobj = p->p_upages_obj; 930 up = (vm_offset_t)p->p_uarea; 931 for (i = 0; i < UAREA_PAGES; i++) { 932 m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 933 if (m->valid != VM_PAGE_BITS_ALL) { 934 rv = vm_pager_get_pages(upobj, &m, 1, 0); 935 if (rv != VM_PAGER_OK) 936 panic("pmap_swapin_proc: cannot get upage"); 937 m = vm_page_lookup(upobj, i); 938 m->valid = VM_PAGE_BITS_ALL; 939 } 940 ma[i] = m; 941 vm_page_wire(m); 942 vm_page_wakeup(m); 943 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 944 } 945 pmap_qenter(up, ma, UAREA_PAGES); 946} 947 948/* 949 * Create the kernel stack and pcb for a new thread. 950 * This routine directly affects the fork perf for a process and 951 * create performance for a thread. 952 */ 953void 954pmap_new_thread(struct thread *td) 955{ 956 vm_page_t ma[KSTACK_PAGES]; 957 vm_object_t ksobj; 958 vm_offset_t ks; 959 vm_page_t m; 960 u_int i; 961 962 /* 963 * Allocate object for the kstack, 964 */ 965 ksobj = td->td_kstack_obj; 966 if (ksobj == NULL) { 967 ksobj = vm_object_allocate(OBJT_DEFAULT, KSTACK_PAGES); 968 td->td_kstack_obj = ksobj; 969 } 970 971 /* 972 * Get a kernel virtual address for the kstack for this thread. 973 */ 974 ks = td->td_kstack; 975 if (ks == 0) { 976 ks = kmem_alloc_nofault(kernel_map, 977 (KSTACK_PAGES + KSTACK_GUARD_PAGES) * PAGE_SIZE); 978 if (ks == 0) 979 panic("pmap_new_thread: kstack allocation failed"); 980 if (KSTACK_GUARD_PAGES != 0) { 981 tlb_page_demap(TLB_DTLB, kernel_pmap, ks); 982 ks += KSTACK_GUARD_PAGES * PAGE_SIZE; 983 } 984 td->td_kstack = ks; 985 } 986 987 for (i = 0; i < KSTACK_PAGES; i++) { 988 /* 989 * Get a kernel stack page. 990 */ 991 m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 992 ma[i] = m; 993 994 /* 995 * Wire the page. 996 */ 997 m->wire_count++; 998 cnt.v_wire_count++; 999 1000 vm_page_wakeup(m); 1001 vm_page_flag_clear(m, PG_ZERO); 1002 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 1003 m->valid = VM_PAGE_BITS_ALL; 1004 } 1005 1006 /* 1007 * Enter the page into the kernel address space. 1008 */ 1009 pmap_qenter(ks, ma, KSTACK_PAGES); 1010} 1011 1012/* 1013 * Dispose the kernel stack for a thread that has exited. 1014 * This routine directly impacts the exit perf of a process and thread. 1015 */ 1016void 1017pmap_dispose_thread(struct thread *td) 1018{ 1019 vm_object_t ksobj; 1020 vm_offset_t ks; 1021 vm_page_t m; 1022 int i; 1023 1024 ksobj = td->td_kstack_obj; 1025 ks = td->td_kstack; 1026 for (i = 0; i < KSTACK_PAGES; i++) { 1027 m = vm_page_lookup(ksobj, i); 1028 if (m == NULL) 1029 panic("pmap_dispose_proc: kstack already missing?"); 1030 vm_page_busy(m); 1031 vm_page_unwire(m, 0); 1032 vm_page_free(m); 1033 } 1034 pmap_qremove(ks, KSTACK_PAGES); 1035 1036 /* 1037 * If the thread got swapped out some of its KSTACK might have gotten 1038 * swapped. Just get rid of the object to clean up the swap use 1039 * proactively. NOTE! might block waiting for paging I/O to complete. 1040 */ 1041 if (ksobj->type == OBJT_SWAP) { 1042 td->td_kstack_obj = NULL; 1043 vm_object_deallocate(ksobj); 1044 } 1045} 1046 1047/* 1048 * Allow the kernel stack for a thread to be prejudicially paged out. 1049 */ 1050void 1051pmap_swapout_thread(struct thread *td) 1052{ 1053 vm_object_t ksobj; 1054 vm_offset_t ks; 1055 vm_page_t m; 1056 int i; 1057 1058 ksobj = td->td_kstack_obj; 1059 ks = (vm_offset_t)td->td_kstack; 1060 for (i = 0; i < KSTACK_PAGES; i++) { 1061 m = vm_page_lookup(ksobj, i); 1062 if (m == NULL) 1063 panic("pmap_swapout_thread: kstack already missing?"); 1064 vm_page_dirty(m); 1065 vm_page_unwire(m, 0); 1066 } 1067 pmap_qremove(ks, KSTACK_PAGES); 1068} 1069 1070/* 1071 * Bring the kernel stack for a specified thread back in. 1072 */ 1073void 1074pmap_swapin_thread(struct thread *td) 1075{ 1076 vm_page_t ma[KSTACK_PAGES]; 1077 vm_object_t ksobj; 1078 vm_offset_t ks; 1079 vm_page_t m; 1080 int rv; 1081 int i; 1082 1083 ksobj = td->td_kstack_obj; 1084 ks = td->td_kstack; 1085 for (i = 0; i < KSTACK_PAGES; i++) { 1086 m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 1087 if (m->valid != VM_PAGE_BITS_ALL) { 1088 rv = vm_pager_get_pages(ksobj, &m, 1, 0); 1089 if (rv != VM_PAGER_OK) 1090 panic("pmap_swapin_proc: cannot get kstack"); 1091 m = vm_page_lookup(ksobj, i); 1092 m->valid = VM_PAGE_BITS_ALL; 1093 } 1094 ma[i] = m; 1095 vm_page_wire(m); 1096 vm_page_wakeup(m); 1097 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); 1098 } 1099 pmap_qenter(ks, ma, KSTACK_PAGES); 1100} 1101 1102/* 1103 * Initialize the pmap associated with process 0. 1104 */ 1105void 1106pmap_pinit0(pmap_t pm) 1107{ 1108 int i; 1109 1110 for (i = 0; i < MAXCPU; i++) 1111 pm->pm_context[i] = 0; 1112 pm->pm_active = 0; 1113 pm->pm_tsb = NULL; 1114 pm->pm_tsb_obj = NULL; 1115 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1116} 1117 1118/* 1119 * Initialize a preallocated and zeroed pmap structure, uch as one in a 1120 * vmspace structure. 1121 */ 1122void 1123pmap_pinit(pmap_t pm) 1124{ 1125 vm_page_t ma[TSB_PAGES]; 1126 vm_page_t m; 1127 int i; 1128 1129 /* 1130 * Allocate kva space for the tsb. 1131 */ 1132 if (pm->pm_tsb == NULL) { 1133 pm->pm_tsb = (struct tte *)kmem_alloc_pageable(kernel_map, 1134 TSB_BSIZE); 1135 } 1136 1137 /* 1138 * Allocate an object for it. 1139 */ 1140 if (pm->pm_tsb_obj == NULL) 1141 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1142 1143 for (i = 0; i < TSB_PAGES; i++) { 1144 m = vm_page_grab(pm->pm_tsb_obj, i, 1145 VM_ALLOC_RETRY | VM_ALLOC_ZERO); 1146 if ((m->flags & PG_ZERO) == 0) 1147 pmap_zero_page(m); 1148 1149 m->wire_count++; 1150 cnt.v_wire_count++; 1151 1152 vm_page_flag_clear(m, PG_MAPPED | PG_BUSY); 1153 m->valid = VM_PAGE_BITS_ALL; 1154 1155 ma[i] = m; 1156 } 1157 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1158 1159 for (i = 0; i < MAXCPU; i++) 1160 pm->pm_context[i] = -1; 1161 pm->pm_active = 0; 1162 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1163} 1164 1165void 1166pmap_pinit2(pmap_t pmap) 1167{ 1168 /* XXX: Remove this stub when no longer called */ 1169} 1170 1171/* 1172 * Release any resources held by the given physical map. 1173 * Called when a pmap initialized by pmap_pinit is being released. 1174 * Should only be called if the map contains no valid mappings. 1175 */ 1176void 1177pmap_release(pmap_t pm) 1178{ 1179 vm_object_t obj; 1180 vm_page_t m; 1181 1182 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1183 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1184 obj = pm->pm_tsb_obj; 1185 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1186 KASSERT(pmap_resident_count(pm) == 0, 1187 ("pmap_release: resident pages %ld != 0", 1188 pmap_resident_count(pm))); 1189 TAILQ_FOREACH(m, &obj->memq, listq) { 1190 if (vm_page_sleep_busy(m, FALSE, "pmaprl")) 1191 continue; 1192 vm_page_busy(m); 1193 KASSERT(m->hold_count == 0, 1194 ("pmap_release: freeing held tsb page")); 1195 m->wire_count--; 1196 cnt.v_wire_count--; 1197 vm_page_free_zero(m); 1198 } 1199 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1200} 1201 1202/* 1203 * Grow the number of kernel page table entries. Unneeded. 1204 */ 1205void 1206pmap_growkernel(vm_offset_t addr) 1207{ 1208} 1209 1210/* 1211 * This routine is very drastic, but can save the system 1212 * in a pinch. 1213 */ 1214void 1215pmap_collect(void) 1216{ 1217} 1218 1219int 1220pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1221 vm_offset_t va) 1222{ 1223 vm_page_t m; 1224 1225 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1226 STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); 1227 if ((tp->tte_data & TD_WIRED) != 0) 1228 pm->pm_stats.wired_count--; 1229 if ((tp->tte_data & TD_PV) != 0) { 1230 if ((tp->tte_data & TD_W) != 0 && 1231 pmap_track_modified(pm, va)) 1232 vm_page_dirty(m); 1233 if ((tp->tte_data & TD_REF) != 0) 1234 vm_page_flag_set(m, PG_REFERENCED); 1235 if (STAILQ_EMPTY(&m->md.tte_list)) 1236 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); 1237 pm->pm_stats.resident_count--; 1238 } 1239 pmap_cache_remove(m, va); 1240 TTE_ZERO(tp); 1241 if (PMAP_REMOVE_DONE(pm)) 1242 return (0); 1243 return (1); 1244} 1245 1246/* 1247 * Remove the given range of addresses from the specified map. 1248 */ 1249void 1250pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1251{ 1252 struct tte *tp; 1253 vm_offset_t va; 1254 1255 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1256 pm->pm_context[PCPU_GET(cpuid)], start, end); 1257 if (PMAP_REMOVE_DONE(pm)) 1258 return; 1259 if (end - start > PMAP_TSB_THRESH) { 1260 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1261 tlb_context_demap(pm); 1262 } else { 1263 for (va = start; va < end; va += PAGE_SIZE) { 1264 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1265 if (!pmap_remove_tte(pm, NULL, tp, va)) 1266 break; 1267 } 1268 } 1269 tlb_range_demap(pm, start, end - 1); 1270 } 1271} 1272 1273void 1274pmap_remove_all(vm_page_t m) 1275{ 1276 struct pmap *pm; 1277 struct tte *tpn; 1278 struct tte *tp; 1279 vm_offset_t va; 1280 1281 KASSERT((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0, 1282 ("pv_remove_all: illegal for unmanaged page %#lx", 1283 VM_PAGE_TO_PHYS(m))); 1284 for (tp = STAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1285 tpn = STAILQ_NEXT(tp, tte_link); 1286 if ((tp->tte_data & TD_PV) == 0) 1287 continue; 1288 pm = TTE_GET_PMAP(tp); 1289 va = TTE_GET_VA(tp); 1290 if ((tp->tte_data & TD_WIRED) != 0) 1291 pm->pm_stats.wired_count--; 1292 if ((tp->tte_data & TD_REF) != 0) 1293 vm_page_flag_set(m, PG_REFERENCED); 1294 if ((tp->tte_data & TD_W) != 0 && 1295 pmap_track_modified(pm, va)) 1296 vm_page_dirty(m); 1297 tp->tte_data &= ~TD_V; 1298 tlb_page_demap(TLB_DTLB | TLB_ITLB, pm, va); 1299 STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); 1300 pm->pm_stats.resident_count--; 1301 pmap_cache_remove(m, va); 1302 TTE_ZERO(tp); 1303 } 1304 vm_page_flag_clear(m, PG_MAPPED | PG_WRITEABLE); 1305} 1306 1307int 1308pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1309 vm_offset_t va) 1310{ 1311 vm_page_t m; 1312 1313 if ((tp->tte_data & TD_PV) != 0) { 1314 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1315 if ((tp->tte_data & TD_REF) != 0) { 1316 vm_page_flag_set(m, PG_REFERENCED); 1317 tp->tte_data &= ~TD_REF; 1318 } 1319 if ((tp->tte_data & TD_W) != 0 && 1320 pmap_track_modified(pm, va)) { 1321 vm_page_dirty(m); 1322 } 1323 } 1324 tp->tte_data &= ~(TD_W | TD_SW); 1325 return (0); 1326} 1327 1328/* 1329 * Set the physical protection on the specified range of this map as requested. 1330 */ 1331void 1332pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1333{ 1334 vm_offset_t va; 1335 struct tte *tp; 1336 1337 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1338 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1339 1340 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1341 pmap_remove(pm, sva, eva); 1342 return; 1343 } 1344 1345 if (prot & VM_PROT_WRITE) 1346 return; 1347 1348 if (eva - sva > PMAP_TSB_THRESH) { 1349 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1350 tlb_context_demap(pm); 1351 } else { 1352 for (va = sva; va < eva; va += PAGE_SIZE) { 1353 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1354 pmap_protect_tte(pm, NULL, tp, va); 1355 } 1356 tlb_range_demap(pm, sva, eva - 1); 1357 } 1358} 1359 1360/* 1361 * Map the given physical page at the specified virtual address in the 1362 * target pmap with the protection requested. If specified the page 1363 * will be wired down. 1364 */ 1365void 1366pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1367 boolean_t wired) 1368{ 1369 struct tte *tp; 1370 vm_offset_t pa; 1371 u_long data; 1372 1373 pa = VM_PAGE_TO_PHYS(m); 1374 CTR6(KTR_PMAP, 1375 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1376 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1377 1378 /* 1379 * If there is an existing mapping, and the physical address has not 1380 * changed, must be protection or wiring change. 1381 */ 1382 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1383 CTR0(KTR_PMAP, "pmap_enter: update"); 1384 PMAP_STATS_INC(pmap_enter_nupdate); 1385 1386 /* 1387 * Wiring change, just update stats. 1388 */ 1389 if (wired) { 1390 if ((tp->tte_data & TD_WIRED) == 0) { 1391 tp->tte_data |= TD_WIRED; 1392 pm->pm_stats.wired_count++; 1393 } 1394 } else { 1395 if ((tp->tte_data & TD_WIRED) != 0) { 1396 tp->tte_data &= ~TD_WIRED; 1397 pm->pm_stats.wired_count--; 1398 } 1399 } 1400 1401 /* 1402 * Save the old bits and clear the ones we're interested in. 1403 */ 1404 data = tp->tte_data; 1405 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1406 1407 /* 1408 * If we're turning off write permissions, sense modify status. 1409 */ 1410 if ((prot & VM_PROT_WRITE) != 0) { 1411 tp->tte_data |= TD_SW; 1412 if (wired) { 1413 tp->tte_data |= TD_W; 1414 } 1415 } else if ((data & TD_W) != 0 && 1416 pmap_track_modified(pm, va)) { 1417 vm_page_dirty(m); 1418 } 1419 1420 /* 1421 * If we're turning on execute permissions, flush the icache. 1422 */ 1423 if ((prot & VM_PROT_EXECUTE) != 0) { 1424 if ((data & TD_EXEC) == 0) { 1425 PMAP_STATS_INC(pmap_niflush); 1426 icache_page_inval(pa); 1427 } 1428 tp->tte_data |= TD_EXEC; 1429 } 1430 1431 /* 1432 * Delete the old mapping. 1433 */ 1434 tlb_tte_demap(tp, pm); 1435 } else { 1436 /* 1437 * If there is an existing mapping, but its for a different 1438 * phsyical address, delete the old mapping. 1439 */ 1440 if (tp != NULL) { 1441 CTR0(KTR_PMAP, "pmap_enter: replace"); 1442 PMAP_STATS_INC(pmap_enter_nreplace); 1443 pmap_remove_tte(pm, NULL, tp, va); 1444 tlb_page_demap(TLB_DTLB | TLB_ITLB, pm, va); 1445 } else { 1446 CTR0(KTR_PMAP, "pmap_enter: new"); 1447 PMAP_STATS_INC(pmap_enter_nnew); 1448 } 1449 1450 /* 1451 * Now set up the data and install the new mapping. 1452 */ 1453 data = TD_V | TD_8K | TD_PA(pa) | TD_CP; 1454 if (pm == kernel_pmap) 1455 data |= TD_P; 1456 if (prot & VM_PROT_WRITE) 1457 data |= TD_SW; 1458 if (prot & VM_PROT_EXECUTE) { 1459 data |= TD_EXEC; 1460 PMAP_STATS_INC(pmap_niflush); 1461 icache_page_inval(pa); 1462 } 1463 1464 /* 1465 * If its wired update stats. We also don't need reference or 1466 * modify tracking for wired mappings, so set the bits now. 1467 */ 1468 if (wired) { 1469 pm->pm_stats.wired_count++; 1470 data |= TD_REF | TD_WIRED; 1471 if ((prot & VM_PROT_WRITE) != 0) 1472 data |= TD_W; 1473 } 1474 1475 tsb_tte_enter(pm, m, va, data); 1476 } 1477} 1478 1479void 1480pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1481 vm_pindex_t pindex, vm_size_t size, int limit) 1482{ 1483 /* XXX */ 1484} 1485 1486void 1487pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) 1488{ 1489 /* XXX */ 1490} 1491 1492/* 1493 * Change the wiring attribute for a map/virtual-address pair. 1494 * The mapping must already exist in the pmap. 1495 */ 1496void 1497pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1498{ 1499 struct tte *tp; 1500 1501 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1502 if (wired) { 1503 if ((tp->tte_data & TD_WIRED) == 0) 1504 pm->pm_stats.wired_count++; 1505 tp->tte_data |= TD_WIRED; 1506 } else { 1507 if ((tp->tte_data & TD_WIRED) != 0) 1508 pm->pm_stats.wired_count--; 1509 tp->tte_data &= ~TD_WIRED; 1510 } 1511 } 1512} 1513 1514static int 1515pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1516{ 1517 vm_page_t m; 1518 u_long data; 1519 1520 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1521 data = tp->tte_data & 1522 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1523 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1524 tsb_tte_enter(dst_pmap, m, va, data); 1525 } 1526 return (1); 1527} 1528 1529void 1530pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1531 vm_size_t len, vm_offset_t src_addr) 1532{ 1533 struct tte *tp; 1534 vm_offset_t va; 1535 1536 if (dst_addr != src_addr) 1537 return; 1538 if (len > PMAP_TSB_THRESH) { 1539 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1540 pmap_copy_tte); 1541 tlb_context_demap(dst_pmap); 1542 } else { 1543 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1544 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1545 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1546 } 1547 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1548 } 1549} 1550 1551/* 1552 * Zero a page of physical memory by temporarily mapping it into the tlb. 1553 */ 1554void 1555pmap_zero_page(vm_page_t m) 1556{ 1557 vm_offset_t pa; 1558 1559 pa = VM_PAGE_TO_PHYS(m); 1560 CTR1(KTR_PMAP, "pmap_zero_page: pa=%#lx", pa); 1561 dcache_page_inval(pa); 1562 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1563} 1564 1565void 1566pmap_zero_page_area(vm_page_t m, int off, int size) 1567{ 1568 vm_offset_t pa; 1569 1570 pa = VM_PAGE_TO_PHYS(m); 1571 CTR3(KTR_PMAP, "pmap_zero_page_area: pa=%#lx off=%#x size=%#x", 1572 pa, off, size); 1573 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1574 dcache_page_inval(pa); 1575 aszero(ASI_PHYS_USE_EC, pa + off, size); 1576} 1577 1578/* 1579 * Copy a page of physical memory by temporarily mapping it into the tlb. 1580 */ 1581void 1582pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1583{ 1584 vm_offset_t dst; 1585 vm_offset_t src; 1586 1587 dst = VM_PAGE_TO_PHYS(mdst); 1588 src = VM_PAGE_TO_PHYS(msrc); 1589 CTR2(KTR_PMAP, "pmap_copy_page: src=%#lx dst=%#lx", src, dst); 1590 dcache_page_inval(dst); 1591 ascopy(ASI_PHYS_USE_EC, src, dst, PAGE_SIZE); 1592} 1593 1594/* 1595 * Make the specified page pageable (or not). Unneeded. 1596 */ 1597void 1598pmap_pageable(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, 1599 boolean_t pageable) 1600{ 1601} 1602 1603/* 1604 * Returns true if the pmap's pv is one of the first 1605 * 16 pvs linked to from this page. This count may 1606 * be changed upwards or downwards in the future; it 1607 * is only necessary that true be returned for a small 1608 * subset of pmaps for proper page aging. 1609 */ 1610boolean_t 1611pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1612{ 1613 struct tte *tp; 1614 int loops; 1615 1616 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1617 return (FALSE); 1618 loops = 0; 1619 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1620 if ((tp->tte_data & TD_PV) == 0) 1621 continue; 1622 if (TTE_GET_PMAP(tp) == pm) 1623 return (TRUE); 1624 if (++loops >= 16) 1625 break; 1626 } 1627 return (FALSE); 1628} 1629 1630/* 1631 * Remove all pages from specified address space, this aids process exit 1632 * speeds. This is much faster than pmap_remove n the case of running down 1633 * an entire address space. Only works for the current pmap. 1634 */ 1635void 1636pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1637{ 1638} 1639 1640/* 1641 * Lower the permission for all mappings to a given page. 1642 */ 1643void 1644pmap_page_protect(vm_page_t m, vm_prot_t prot) 1645{ 1646 1647 if ((prot & VM_PROT_WRITE) == 0) { 1648 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1649 pmap_clear_write(m); 1650 else 1651 pmap_remove_all(m); 1652 } 1653} 1654 1655vm_offset_t 1656pmap_phys_address(int ppn) 1657{ 1658 1659 return (sparc64_ptob(ppn)); 1660} 1661 1662/* 1663 * pmap_ts_referenced: 1664 * 1665 * Return a count of reference bits for a page, clearing those bits. 1666 * It is not necessary for every reference bit to be cleared, but it 1667 * is necessary that 0 only be returned when there are truly no 1668 * reference bits set. 1669 * 1670 * XXX: The exact number of bits to check and clear is a matter that 1671 * should be tested and standardized at some point in the future for 1672 * optimal aging of shared pages. 1673 */ 1674 1675int 1676pmap_ts_referenced(vm_page_t m) 1677{ 1678 struct tte *tpf; 1679 struct tte *tpn; 1680 struct tte *tp; 1681 int count; 1682 1683 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1684 return (0); 1685 count = 0; 1686 if ((tp = STAILQ_FIRST(&m->md.tte_list)) != NULL) { 1687 tpf = tp; 1688 do { 1689 tpn = STAILQ_NEXT(tp, tte_link); 1690 STAILQ_REMOVE(&m->md.tte_list, tp, tte, tte_link); 1691 STAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1692 if ((tp->tte_data & TD_PV) == 0 || 1693 !pmap_track_modified(TTE_GET_PMAP(tp), 1694 TTE_GET_VA(tp))) 1695 continue; 1696 if ((tp->tte_data & TD_REF) != 0) { 1697 tp->tte_data &= ~TD_REF; 1698 if (++count > 4) 1699 break; 1700 } 1701 } while ((tp = tpn) != NULL && tp != tpf); 1702 } 1703 return (count); 1704} 1705 1706boolean_t 1707pmap_is_modified(vm_page_t m) 1708{ 1709 struct tte *tp; 1710 1711 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1712 return FALSE; 1713 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1714 if ((tp->tte_data & TD_PV) == 0 || 1715 !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) 1716 continue; 1717 if ((tp->tte_data & TD_W) != 0) 1718 return (TRUE); 1719 } 1720 return (FALSE); 1721} 1722 1723void 1724pmap_clear_modify(vm_page_t m) 1725{ 1726 struct tte *tp; 1727 1728 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1729 return; 1730 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1731 if ((tp->tte_data & TD_PV) == 0) 1732 continue; 1733 if ((tp->tte_data & TD_W) != 0) { 1734 tp->tte_data &= ~TD_W; 1735 tlb_tte_demap(tp, TTE_GET_PMAP(tp)); 1736 } 1737 } 1738} 1739 1740void 1741pmap_clear_reference(vm_page_t m) 1742{ 1743 struct tte *tp; 1744 1745 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1746 return; 1747 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1748 if ((tp->tte_data & TD_PV) == 0) 1749 continue; 1750 if ((tp->tte_data & TD_REF) != 0) { 1751 tp->tte_data &= ~TD_REF; 1752 tlb_tte_demap(tp, TTE_GET_PMAP(tp)); 1753 } 1754 } 1755} 1756 1757void 1758pmap_clear_write(vm_page_t m) 1759{ 1760 struct tte *tp; 1761 1762 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1763 return; 1764 STAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1765 if ((tp->tte_data & TD_PV) == 0) 1766 continue; 1767 if ((tp->tte_data & (TD_SW | TD_W)) != 0) { 1768 if ((tp->tte_data & TD_W) != 0 && 1769 pmap_track_modified(TTE_GET_PMAP(tp), 1770 TTE_GET_VA(tp))) 1771 vm_page_dirty(m); 1772 tp->tte_data &= ~(TD_SW | TD_W); 1773 tlb_tte_demap(tp, TTE_GET_PMAP(tp)); 1774 } 1775 } 1776} 1777 1778int 1779pmap_mincore(pmap_t pm, vm_offset_t addr) 1780{ 1781 TODO; 1782 return (0); 1783} 1784 1785/* 1786 * Activate a user pmap. The pmap must be activated before its address space 1787 * can be accessed in any way. 1788 */ 1789void 1790pmap_activate(struct thread *td) 1791{ 1792 struct vmspace *vm; 1793 vm_offset_t tsb; 1794 u_long context; 1795 pmap_t pm; 1796 1797 /* 1798 * Load all the data we need up front to encourage the compiler to 1799 * not issue any loads while we have interrupts disable below. 1800 */ 1801 vm = td->td_proc->p_vmspace; 1802 pm = &vm->vm_pmap; 1803 tsb = (vm_offset_t)pm->pm_tsb; 1804 1805 KASSERT(pm->pm_active == 0, ("pmap_activate: pmap already active?")); 1806 KASSERT(pm->pm_context[PCPU_GET(cpuid)] != 0, 1807 ("pmap_activate: activating nucleus context?")); 1808 1809 mtx_lock_spin(&sched_lock); 1810 wrpr(pstate, 0, PSTATE_MMU); 1811 mov(tsb, TSB_REG); 1812 wrpr(pstate, 0, PSTATE_KERNEL); 1813 context = pmap_context_alloc(); 1814 pm->pm_context[PCPU_GET(cpuid)] = context; 1815 pm->pm_active |= PCPU_GET(cpumask); 1816 PCPU_SET(vmspace, vm); 1817 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 1818 membar(Sync); 1819 mtx_unlock_spin(&sched_lock); 1820} 1821 1822vm_offset_t 1823pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 1824{ 1825 1826 return (va); 1827} 1828