pmap.c revision 159303
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 159303 2006-06-05 20:35:27Z alc $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this module is called upon to 49 * provide software-use-only maps which may or may not be stored in the 50 * same form as hardware maps. These pseudo-maps are used to store 51 * intermediate results from copy operations to and from address spaces. 52 * 53 * Since the information managed by this module is also stored by the 54 * logical address mapping module, this module may throw away valid virtual 55 * to physical mappings at almost any time. However, invalidations of 56 * mappings must be done as requested. 57 * 58 * In order to cope with hardware architectures which make virtual to 59 * physical map invalidates expensive, this module may delay invalidate 60 * reduced protection operations until such time as they are actually 61 * necessary. This module is given full information as to which processors 62 * are currently using which maps, and to when physical maps must be made 63 * correct. 64 */ 65 66#include "opt_kstack_pages.h" 67#include "opt_msgbuf.h" 68#include "opt_pmap.h" 69 70#include <sys/param.h> 71#include <sys/kernel.h> 72#include <sys/ktr.h> 73#include <sys/lock.h> 74#include <sys/msgbuf.h> 75#include <sys/mutex.h> 76#include <sys/proc.h> 77#include <sys/smp.h> 78#include <sys/sysctl.h> 79#include <sys/systm.h> 80#include <sys/vmmeter.h> 81 82#include <dev/ofw/openfirm.h> 83 84#include <vm/vm.h> 85#include <vm/vm_param.h> 86#include <vm/vm_kern.h> 87#include <vm/vm_page.h> 88#include <vm/vm_map.h> 89#include <vm/vm_object.h> 90#include <vm/vm_extern.h> 91#include <vm/vm_pageout.h> 92#include <vm/vm_pager.h> 93 94#include <machine/cache.h> 95#include <machine/frame.h> 96#include <machine/instr.h> 97#include <machine/md_var.h> 98#include <machine/metadata.h> 99#include <machine/ofw_mem.h> 100#include <machine/smp.h> 101#include <machine/tlb.h> 102#include <machine/tte.h> 103#include <machine/tsb.h> 104 105#define PMAP_DEBUG 106 107#ifndef PMAP_SHPGPERPROC 108#define PMAP_SHPGPERPROC 200 109#endif 110 111/* 112 * Virtual and physical address of message buffer. 113 */ 114struct msgbuf *msgbufp; 115vm_paddr_t msgbuf_phys; 116 117/* 118 * Map of physical memory reagions. 119 */ 120vm_paddr_t phys_avail[128]; 121static struct ofw_mem_region mra[128]; 122struct ofw_mem_region sparc64_memreg[128]; 123int sparc64_nmemreg; 124static struct ofw_map translations[128]; 125static int translations_size; 126 127static vm_offset_t pmap_idle_map; 128static vm_offset_t pmap_temp_map_1; 129static vm_offset_t pmap_temp_map_2; 130 131/* 132 * First and last available kernel virtual addresses. 133 */ 134vm_offset_t virtual_avail; 135vm_offset_t virtual_end; 136vm_offset_t kernel_vm_end; 137 138vm_offset_t vm_max_kernel_address; 139 140/* 141 * Kernel pmap. 142 */ 143struct pmap kernel_pmap_store; 144 145/* 146 * Allocate physical memory for use in pmap_bootstrap. 147 */ 148static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 149 150/* 151 * Map the given physical page at the specified virtual address in the 152 * target pmap with the protection requested. If specified the page 153 * will be wired down. 154 * 155 * The page queues and pmap must be locked. 156 */ 157static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 158 vm_prot_t prot, boolean_t wired); 159 160extern int tl1_immu_miss_patch_1[]; 161extern int tl1_immu_miss_patch_2[]; 162extern int tl1_dmmu_miss_patch_1[]; 163extern int tl1_dmmu_miss_patch_2[]; 164extern int tl1_dmmu_prot_patch_1[]; 165extern int tl1_dmmu_prot_patch_2[]; 166 167/* 168 * If user pmap is processed with pmap_remove and with pmap_remove and the 169 * resident count drops to 0, there are no more pages to remove, so we 170 * need not continue. 171 */ 172#define PMAP_REMOVE_DONE(pm) \ 173 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 174 175/* 176 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 177 * and pmap_protect() instead of trying each virtual address. 178 */ 179#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 180 181SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 182 183PMAP_STATS_VAR(pmap_nenter); 184PMAP_STATS_VAR(pmap_nenter_update); 185PMAP_STATS_VAR(pmap_nenter_replace); 186PMAP_STATS_VAR(pmap_nenter_new); 187PMAP_STATS_VAR(pmap_nkenter); 188PMAP_STATS_VAR(pmap_nkenter_oc); 189PMAP_STATS_VAR(pmap_nkenter_stupid); 190PMAP_STATS_VAR(pmap_nkremove); 191PMAP_STATS_VAR(pmap_nqenter); 192PMAP_STATS_VAR(pmap_nqremove); 193PMAP_STATS_VAR(pmap_ncache_enter); 194PMAP_STATS_VAR(pmap_ncache_enter_c); 195PMAP_STATS_VAR(pmap_ncache_enter_oc); 196PMAP_STATS_VAR(pmap_ncache_enter_cc); 197PMAP_STATS_VAR(pmap_ncache_enter_coc); 198PMAP_STATS_VAR(pmap_ncache_enter_nc); 199PMAP_STATS_VAR(pmap_ncache_enter_cnc); 200PMAP_STATS_VAR(pmap_ncache_remove); 201PMAP_STATS_VAR(pmap_ncache_remove_c); 202PMAP_STATS_VAR(pmap_ncache_remove_oc); 203PMAP_STATS_VAR(pmap_ncache_remove_cc); 204PMAP_STATS_VAR(pmap_ncache_remove_coc); 205PMAP_STATS_VAR(pmap_ncache_remove_nc); 206PMAP_STATS_VAR(pmap_nzero_page); 207PMAP_STATS_VAR(pmap_nzero_page_c); 208PMAP_STATS_VAR(pmap_nzero_page_oc); 209PMAP_STATS_VAR(pmap_nzero_page_nc); 210PMAP_STATS_VAR(pmap_nzero_page_area); 211PMAP_STATS_VAR(pmap_nzero_page_area_c); 212PMAP_STATS_VAR(pmap_nzero_page_area_oc); 213PMAP_STATS_VAR(pmap_nzero_page_area_nc); 214PMAP_STATS_VAR(pmap_nzero_page_idle); 215PMAP_STATS_VAR(pmap_nzero_page_idle_c); 216PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 217PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 218PMAP_STATS_VAR(pmap_ncopy_page); 219PMAP_STATS_VAR(pmap_ncopy_page_c); 220PMAP_STATS_VAR(pmap_ncopy_page_oc); 221PMAP_STATS_VAR(pmap_ncopy_page_nc); 222PMAP_STATS_VAR(pmap_ncopy_page_dc); 223PMAP_STATS_VAR(pmap_ncopy_page_doc); 224PMAP_STATS_VAR(pmap_ncopy_page_sc); 225PMAP_STATS_VAR(pmap_ncopy_page_soc); 226 227PMAP_STATS_VAR(pmap_nnew_thread); 228PMAP_STATS_VAR(pmap_nnew_thread_oc); 229 230/* 231 * Quick sort callout for comparing memory regions. 232 */ 233static int mr_cmp(const void *a, const void *b); 234static int om_cmp(const void *a, const void *b); 235static int 236mr_cmp(const void *a, const void *b) 237{ 238 const struct ofw_mem_region *mra; 239 const struct ofw_mem_region *mrb; 240 241 mra = a; 242 mrb = b; 243 if (mra->mr_start < mrb->mr_start) 244 return (-1); 245 else if (mra->mr_start > mrb->mr_start) 246 return (1); 247 else 248 return (0); 249} 250static int 251om_cmp(const void *a, const void *b) 252{ 253 const struct ofw_map *oma; 254 const struct ofw_map *omb; 255 256 oma = a; 257 omb = b; 258 if (oma->om_start < omb->om_start) 259 return (-1); 260 else if (oma->om_start > omb->om_start) 261 return (1); 262 else 263 return (0); 264} 265 266/* 267 * Bootstrap the system enough to run with virtual memory. 268 */ 269void 270pmap_bootstrap(vm_offset_t ekva) 271{ 272 struct pmap *pm; 273 struct tte *tp; 274 vm_offset_t off; 275 vm_offset_t va; 276 vm_paddr_t pa; 277 vm_size_t physsz; 278 vm_size_t virtsz; 279 ihandle_t pmem; 280 ihandle_t vmem; 281 int sz; 282 int i; 283 int j; 284 285 /* 286 * Find out what physical memory is available from the prom and 287 * initialize the phys_avail array. This must be done before 288 * pmap_bootstrap_alloc is called. 289 */ 290 if ((pmem = OF_finddevice("/memory")) == -1) 291 panic("pmap_bootstrap: finddevice /memory"); 292 if ((sz = OF_getproplen(pmem, "available")) == -1) 293 panic("pmap_bootstrap: getproplen /memory/available"); 294 if (sizeof(phys_avail) < sz) 295 panic("pmap_bootstrap: phys_avail too small"); 296 if (sizeof(mra) < sz) 297 panic("pmap_bootstrap: mra too small"); 298 bzero(mra, sz); 299 if (OF_getprop(pmem, "available", mra, sz) == -1) 300 panic("pmap_bootstrap: getprop /memory/available"); 301 sz /= sizeof(*mra); 302 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 303 qsort(mra, sz, sizeof (*mra), mr_cmp); 304 physsz = 0; 305 getenv_quad("hw.physmem", &physmem); 306 physmem = ctob(physmem); 307 for (i = 0, j = 0; i < sz; i++, j += 2) { 308 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 309 mra[i].mr_size); 310 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 311 if (btoc(physsz) < physmem) { 312 phys_avail[j] = mra[i].mr_start; 313 phys_avail[j + 1] = mra[i].mr_start + 314 (ctob(physmem) - physsz); 315 physsz = ctob(physmem); 316 } 317 break; 318 } 319 phys_avail[j] = mra[i].mr_start; 320 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 321 physsz += mra[i].mr_size; 322 } 323 physmem = btoc(physsz); 324 325 /* 326 * Calculate the size of kernel virtual memory, and the size and mask 327 * for the kernel tsb. 328 */ 329 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 330 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 331 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 332 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 333 334 /* 335 * Allocate the kernel tsb and lock it in the tlb. 336 */ 337 pa = pmap_bootstrap_alloc(tsb_kernel_size); 338 if (pa & PAGE_MASK_4M) 339 panic("pmap_bootstrap: tsb unaligned\n"); 340 tsb_kernel_phys = pa; 341 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 342 pmap_map_tsb(); 343 bzero(tsb_kernel, tsb_kernel_size); 344 345 /* 346 * Allocate and map the message buffer. 347 */ 348 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 349 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys); 350 351 /* 352 * Patch the virtual address and the tsb mask into the trap table. 353 */ 354 355#define SETHI(rd, imm22) \ 356 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 357 EIF_IMM((imm22) >> 10, 22)) 358#define OR_R_I_R(rd, imm13, rs1) \ 359 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 360 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 361 362#define PATCH(addr) do { \ 363 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 364 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 365 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 366 panic("pmap_boostrap: patched instructions have changed"); \ 367 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 368 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 369 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 370 flush(addr); \ 371 flush(addr + 1); \ 372 flush(addr + 2); \ 373} while (0) 374 375 PATCH(tl1_immu_miss_patch_1); 376 PATCH(tl1_immu_miss_patch_2); 377 PATCH(tl1_dmmu_miss_patch_1); 378 PATCH(tl1_dmmu_miss_patch_2); 379 PATCH(tl1_dmmu_prot_patch_1); 380 PATCH(tl1_dmmu_prot_patch_2); 381 382 /* 383 * Enter fake 8k pages for the 4MB kernel pages, so that 384 * pmap_kextract() will work for them. 385 */ 386 for (i = 0; i < kernel_tlb_slots; i++) { 387 pa = kernel_tlbs[i].te_pa; 388 va = kernel_tlbs[i].te_va; 389 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 390 tp = tsb_kvtotte(va + off); 391 tp->tte_vpn = TV_VPN(va + off, TS_8K); 392 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 393 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 394 } 395 } 396 397 /* 398 * Set the start and end of kva. The kernel is loaded at the first 399 * available 4 meg super page, so round up to the end of the page. 400 */ 401 virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 402 virtual_end = vm_max_kernel_address; 403 kernel_vm_end = vm_max_kernel_address; 404 405 /* 406 * Allocate kva space for temporary mappings. 407 */ 408 pmap_idle_map = virtual_avail; 409 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 410 pmap_temp_map_1 = virtual_avail; 411 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 412 pmap_temp_map_2 = virtual_avail; 413 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 414 415 /* 416 * Allocate a kernel stack with guard page for thread0 and map it into 417 * the kernel tsb. We must ensure that the virtual address is coloured 418 * properly, since we're allocating from phys_avail so the memory won't 419 * have an associated vm_page_t. 420 */ 421 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) * 422 PAGE_SIZE); 423 kstack0_phys = pa; 424 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 425 PAGE_SIZE; 426 kstack0 = virtual_avail; 427 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 428 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 429 ("pmap_bootstrap: kstack0 miscoloured")); 430 for (i = 0; i < KSTACK_PAGES; i++) { 431 pa = kstack0_phys + i * PAGE_SIZE; 432 va = kstack0 + i * PAGE_SIZE; 433 tp = tsb_kvtotte(va); 434 tp->tte_vpn = TV_VPN(va, TS_8K); 435 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 436 TD_CP | TD_CV | TD_P | TD_W; 437 } 438 439 /* 440 * Calculate the last available physical address. 441 */ 442 for (i = 0; phys_avail[i + 2] != 0; i += 2) 443 ; 444 Maxmem = sparc64_btop(phys_avail[i + 1]); 445 446 /* 447 * Add the prom mappings to the kernel tsb. 448 */ 449 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 450 panic("pmap_bootstrap: finddevice /virtual-memory"); 451 if ((sz = OF_getproplen(vmem, "translations")) == -1) 452 panic("pmap_bootstrap: getproplen translations"); 453 if (sizeof(translations) < sz) 454 panic("pmap_bootstrap: translations too small"); 455 bzero(translations, sz); 456 if (OF_getprop(vmem, "translations", translations, sz) == -1) 457 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 458 sz /= sizeof(*translations); 459 translations_size = sz; 460 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 461 qsort(translations, sz, sizeof (*translations), om_cmp); 462 for (i = 0; i < sz; i++) { 463 CTR3(KTR_PMAP, 464 "translation: start=%#lx size=%#lx tte=%#lx", 465 translations[i].om_start, translations[i].om_size, 466 translations[i].om_tte); 467 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 468 translations[i].om_start > VM_MAX_PROM_ADDRESS) 469 continue; 470 for (off = 0; off < translations[i].om_size; 471 off += PAGE_SIZE) { 472 va = translations[i].om_start + off; 473 tp = tsb_kvtotte(va); 474 tp->tte_vpn = TV_VPN(va, TS_8K); 475 tp->tte_data = 476 ((translations[i].om_tte & 477 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) + 478 off; 479 } 480 } 481 482 /* 483 * Get the available physical memory ranges from /memory/reg. These 484 * are only used for kernel dumps, but it may not be wise to do prom 485 * calls in that situation. 486 */ 487 if ((sz = OF_getproplen(pmem, "reg")) == -1) 488 panic("pmap_bootstrap: getproplen /memory/reg"); 489 if (sizeof(sparc64_memreg) < sz) 490 panic("pmap_bootstrap: sparc64_memreg too small"); 491 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 492 panic("pmap_bootstrap: getprop /memory/reg"); 493 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 494 495 /* 496 * Initialize the kernel pmap (which is statically allocated). 497 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 498 * but sparc64 start up is not ready to initialize mutexes yet. 499 * It is called in machdep.c. 500 */ 501 pm = kernel_pmap; 502 for (i = 0; i < MAXCPU; i++) 503 pm->pm_context[i] = TLB_CTX_KERNEL; 504 pm->pm_active = ~0; 505 506 /* XXX flush all non-locked tlb entries */ 507} 508 509void 510pmap_map_tsb(void) 511{ 512 vm_offset_t va; 513 vm_paddr_t pa; 514 u_long data; 515 u_long s; 516 int i; 517 518 s = intr_disable(); 519 520 /* 521 * Map the 4mb tsb pages. 522 */ 523 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 524 va = (vm_offset_t)tsb_kernel + i; 525 pa = tsb_kernel_phys + i; 526 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 527 TD_P | TD_W; 528 /* XXX - cheetah */ 529 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 530 TLB_TAR_CTX(TLB_CTX_KERNEL)); 531 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 532 } 533 534 /* 535 * Set the secondary context to be the kernel context (needed for 536 * fp block operations in the kernel and the cache code). 537 */ 538 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 539 membar(Sync); 540 541 intr_restore(s); 542} 543 544/* 545 * Allocate a physical page of memory directly from the phys_avail map. 546 * Can only be called from pmap_bootstrap before avail start and end are 547 * calculated. 548 */ 549static vm_paddr_t 550pmap_bootstrap_alloc(vm_size_t size) 551{ 552 vm_paddr_t pa; 553 int i; 554 555 size = round_page(size); 556 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 557 if (phys_avail[i + 1] - phys_avail[i] < size) 558 continue; 559 pa = phys_avail[i]; 560 phys_avail[i] += size; 561 return (pa); 562 } 563 panic("pmap_bootstrap_alloc"); 564} 565 566/* 567 * Initialize a vm_page's machine-dependent fields. 568 */ 569void 570pmap_page_init(vm_page_t m) 571{ 572 573 TAILQ_INIT(&m->md.tte_list); 574 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 575 m->md.flags = 0; 576 m->md.pmap = NULL; 577} 578 579/* 580 * Initialize the pmap module. 581 */ 582void 583pmap_init(void) 584{ 585 vm_offset_t addr; 586 vm_size_t size; 587 int result; 588 int i; 589 590 for (i = 0; i < translations_size; i++) { 591 addr = translations[i].om_start; 592 size = translations[i].om_size; 593 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 594 continue; 595 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE, 596 VM_PROT_ALL, VM_PROT_ALL, 0); 597 if (result != KERN_SUCCESS || addr != translations[i].om_start) 598 panic("pmap_init: vm_map_find"); 599 } 600} 601 602/* 603 * Extract the physical page address associated with the given 604 * map/virtual_address pair. 605 */ 606vm_paddr_t 607pmap_extract(pmap_t pm, vm_offset_t va) 608{ 609 struct tte *tp; 610 vm_paddr_t pa; 611 612 if (pm == kernel_pmap) 613 return (pmap_kextract(va)); 614 PMAP_LOCK(pm); 615 tp = tsb_tte_lookup(pm, va); 616 if (tp == NULL) 617 pa = 0; 618 else 619 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 620 PMAP_UNLOCK(pm); 621 return (pa); 622} 623 624/* 625 * Atomically extract and hold the physical page with the given 626 * pmap and virtual address pair if that mapping permits the given 627 * protection. 628 */ 629vm_page_t 630pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 631{ 632 struct tte *tp; 633 vm_page_t m; 634 635 m = NULL; 636 vm_page_lock_queues(); 637 if (pm == kernel_pmap) { 638 if (va >= VM_MIN_DIRECT_ADDRESS) { 639 tp = NULL; 640 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 641 vm_page_hold(m); 642 } else { 643 tp = tsb_kvtotte(va); 644 if ((tp->tte_data & TD_V) == 0) 645 tp = NULL; 646 } 647 } else { 648 PMAP_LOCK(pm); 649 tp = tsb_tte_lookup(pm, va); 650 } 651 if (tp != NULL && ((tp->tte_data & TD_SW) || 652 (prot & VM_PROT_WRITE) == 0)) { 653 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 654 vm_page_hold(m); 655 } 656 vm_page_unlock_queues(); 657 if (pm != kernel_pmap) 658 PMAP_UNLOCK(pm); 659 return (m); 660} 661 662/* 663 * Extract the physical page address associated with the given kernel virtual 664 * address. 665 */ 666vm_paddr_t 667pmap_kextract(vm_offset_t va) 668{ 669 struct tte *tp; 670 671 if (va >= VM_MIN_DIRECT_ADDRESS) 672 return (TLB_DIRECT_TO_PHYS(va)); 673 tp = tsb_kvtotte(va); 674 if ((tp->tte_data & TD_V) == 0) 675 return (0); 676 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 677} 678 679int 680pmap_cache_enter(vm_page_t m, vm_offset_t va) 681{ 682 struct tte *tp; 683 int color; 684 685 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 686 KASSERT((m->flags & PG_FICTITIOUS) == 0, 687 ("pmap_cache_enter: fake page")); 688 PMAP_STATS_INC(pmap_ncache_enter); 689 690 /* 691 * Find the color for this virtual address and note the added mapping. 692 */ 693 color = DCACHE_COLOR(va); 694 m->md.colors[color]++; 695 696 /* 697 * If all existing mappings have the same color, the mapping is 698 * cacheable. 699 */ 700 if (m->md.color == color) { 701 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 702 ("pmap_cache_enter: cacheable, mappings of other color")); 703 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 704 PMAP_STATS_INC(pmap_ncache_enter_c); 705 else 706 PMAP_STATS_INC(pmap_ncache_enter_oc); 707 return (1); 708 } 709 710 /* 711 * If there are no mappings of the other color, and the page still has 712 * the wrong color, this must be a new mapping. Change the color to 713 * match the new mapping, which is cacheable. We must flush the page 714 * from the cache now. 715 */ 716 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 717 KASSERT(m->md.colors[color] == 1, 718 ("pmap_cache_enter: changing color, not new mapping")); 719 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 720 m->md.color = color; 721 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 722 PMAP_STATS_INC(pmap_ncache_enter_cc); 723 else 724 PMAP_STATS_INC(pmap_ncache_enter_coc); 725 return (1); 726 } 727 728 /* 729 * If the mapping is already non-cacheable, just return. 730 */ 731 if (m->md.color == -1) { 732 PMAP_STATS_INC(pmap_ncache_enter_nc); 733 return (0); 734 } 735 736 PMAP_STATS_INC(pmap_ncache_enter_cnc); 737 738 /* 739 * Mark all mappings as uncacheable, flush any lines with the other 740 * color out of the dcache, and set the color to none (-1). 741 */ 742 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 743 atomic_clear_long(&tp->tte_data, TD_CV); 744 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 745 } 746 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 747 m->md.color = -1; 748 return (0); 749} 750 751void 752pmap_cache_remove(vm_page_t m, vm_offset_t va) 753{ 754 struct tte *tp; 755 int color; 756 757 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 758 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 759 m->md.colors[DCACHE_COLOR(va)]); 760 KASSERT((m->flags & PG_FICTITIOUS) == 0, 761 ("pmap_cache_remove: fake page")); 762 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 763 ("pmap_cache_remove: no mappings %d <= 0", 764 m->md.colors[DCACHE_COLOR(va)])); 765 PMAP_STATS_INC(pmap_ncache_remove); 766 767 /* 768 * Find the color for this virtual address and note the removal of 769 * the mapping. 770 */ 771 color = DCACHE_COLOR(va); 772 m->md.colors[color]--; 773 774 /* 775 * If the page is cacheable, just return and keep the same color, even 776 * if there are no longer any mappings. 777 */ 778 if (m->md.color != -1) { 779 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 780 PMAP_STATS_INC(pmap_ncache_remove_c); 781 else 782 PMAP_STATS_INC(pmap_ncache_remove_oc); 783 return; 784 } 785 786 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 787 ("pmap_cache_remove: uncacheable, no mappings of other color")); 788 789 /* 790 * If the page is not cacheable (color is -1), and the number of 791 * mappings for this color is not zero, just return. There are 792 * mappings of the other color still, so remain non-cacheable. 793 */ 794 if (m->md.colors[color] != 0) { 795 PMAP_STATS_INC(pmap_ncache_remove_nc); 796 return; 797 } 798 799 /* 800 * The number of mappings for this color is now zero. Recache the 801 * other colored mappings, and change the page color to the other 802 * color. There should be no lines in the data cache for this page, 803 * so flushing should not be needed. 804 */ 805 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 806 atomic_set_long(&tp->tte_data, TD_CV); 807 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 808 } 809 m->md.color = DCACHE_OTHER_COLOR(color); 810 811 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 812 PMAP_STATS_INC(pmap_ncache_remove_cc); 813 else 814 PMAP_STATS_INC(pmap_ncache_remove_coc); 815} 816 817/* 818 * Map a wired page into kernel virtual address space. 819 */ 820void 821pmap_kenter(vm_offset_t va, vm_page_t m) 822{ 823 vm_offset_t ova; 824 struct tte *tp; 825 vm_page_t om; 826 u_long data; 827 828 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 829 PMAP_STATS_INC(pmap_nkenter); 830 tp = tsb_kvtotte(va); 831 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 832 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 833 if (m->pc != DCACHE_COLOR(va)) { 834 CTR6(KTR_CT2, 835 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx", 836 va, VM_PAGE_TO_PHYS(m), m->object, 837 m->object ? m->object->pg_color : -1, 838 m->object ? m->object->type : -1, 839 m->pindex); 840 PMAP_STATS_INC(pmap_nkenter_oc); 841 } 842 if ((tp->tte_data & TD_V) != 0) { 843 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 844 ova = TTE_GET_VA(tp); 845 if (m == om && va == ova) { 846 PMAP_STATS_INC(pmap_nkenter_stupid); 847 return; 848 } 849 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 850 pmap_cache_remove(om, ova); 851 if (va != ova) 852 tlb_page_demap(kernel_pmap, ova); 853 } 854 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 855 TD_P | TD_W; 856 if (pmap_cache_enter(m, va) != 0) 857 data |= TD_CV; 858 tp->tte_vpn = TV_VPN(va, TS_8K); 859 tp->tte_data = data; 860 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 861} 862 863/* 864 * Map a wired page into kernel virtual address space. This additionally 865 * takes a flag argument wich is or'ed to the TTE data. This is used by 866 * bus_space_map(). 867 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 868 * to flush entries that might still be in the cache, if applicable. 869 */ 870void 871pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 872{ 873 struct tte *tp; 874 875 tp = tsb_kvtotte(va); 876 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 877 va, pa, tp, tp->tte_data); 878 tp->tte_vpn = TV_VPN(va, TS_8K); 879 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 880} 881 882/* 883 * Remove a wired page from kernel virtual address space. 884 */ 885void 886pmap_kremove(vm_offset_t va) 887{ 888 struct tte *tp; 889 vm_page_t m; 890 891 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 892 PMAP_STATS_INC(pmap_nkremove); 893 tp = tsb_kvtotte(va); 894 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 895 tp->tte_data); 896 if ((tp->tte_data & TD_V) == 0) 897 return; 898 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 899 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 900 pmap_cache_remove(m, va); 901 TTE_ZERO(tp); 902} 903 904/* 905 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 906 */ 907void 908pmap_kremove_flags(vm_offset_t va) 909{ 910 struct tte *tp; 911 912 tp = tsb_kvtotte(va); 913 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 914 tp->tte_data); 915 TTE_ZERO(tp); 916} 917 918/* 919 * Map a range of physical addresses into kernel virtual address space. 920 * 921 * The value passed in *virt is a suggested virtual address for the mapping. 922 * Architectures which can support a direct-mapped physical to virtual region 923 * can return the appropriate address within that region, leaving '*virt' 924 * unchanged. 925 */ 926vm_offset_t 927pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 928{ 929 930 return (TLB_PHYS_TO_DIRECT(start)); 931} 932 933/* 934 * Map a list of wired pages into kernel virtual address space. This is 935 * intended for temporary mappings which do not need page modification or 936 * references recorded. Existing mappings in the region are overwritten. 937 */ 938void 939pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 940{ 941 vm_offset_t va; 942 int locked; 943 944 PMAP_STATS_INC(pmap_nqenter); 945 va = sva; 946 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 947 vm_page_lock_queues(); 948 while (count-- > 0) { 949 pmap_kenter(va, *m); 950 va += PAGE_SIZE; 951 m++; 952 } 953 if (!locked) 954 vm_page_unlock_queues(); 955 tlb_range_demap(kernel_pmap, sva, va); 956} 957 958/* 959 * Remove page mappings from kernel virtual address space. Intended for 960 * temporary mappings entered by pmap_qenter. 961 */ 962void 963pmap_qremove(vm_offset_t sva, int count) 964{ 965 vm_offset_t va; 966 int locked; 967 968 PMAP_STATS_INC(pmap_nqremove); 969 va = sva; 970 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 971 vm_page_lock_queues(); 972 while (count-- > 0) { 973 pmap_kremove(va); 974 va += PAGE_SIZE; 975 } 976 if (!locked) 977 vm_page_unlock_queues(); 978 tlb_range_demap(kernel_pmap, sva, va); 979} 980 981/* 982 * Initialize the pmap associated with process 0. 983 */ 984void 985pmap_pinit0(pmap_t pm) 986{ 987 int i; 988 989 PMAP_LOCK_INIT(pm); 990 for (i = 0; i < MAXCPU; i++) 991 pm->pm_context[i] = 0; 992 pm->pm_active = 0; 993 pm->pm_tsb = NULL; 994 pm->pm_tsb_obj = NULL; 995 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 996} 997 998/* 999 * Initialize a preallocated and zeroed pmap structure, uch as one in a 1000 * vmspace structure. 1001 */ 1002void 1003pmap_pinit(pmap_t pm) 1004{ 1005 vm_page_t ma[TSB_PAGES]; 1006 vm_page_t m; 1007 int i; 1008 1009 PMAP_LOCK_INIT(pm); 1010 1011 /* 1012 * Allocate kva space for the tsb. 1013 */ 1014 if (pm->pm_tsb == NULL) { 1015 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1016 TSB_BSIZE); 1017 } 1018 1019 /* 1020 * Allocate an object for it. 1021 */ 1022 if (pm->pm_tsb_obj == NULL) 1023 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1024 1025 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1026 for (i = 0; i < TSB_PAGES; i++) { 1027 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1028 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1029 1030 vm_page_lock_queues(); 1031 m->valid = VM_PAGE_BITS_ALL; 1032 m->md.pmap = pm; 1033 vm_page_unlock_queues(); 1034 1035 ma[i] = m; 1036 } 1037 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1038 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1039 1040 for (i = 0; i < MAXCPU; i++) 1041 pm->pm_context[i] = -1; 1042 pm->pm_active = 0; 1043 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1044} 1045 1046/* 1047 * Release any resources held by the given physical map. 1048 * Called when a pmap initialized by pmap_pinit is being released. 1049 * Should only be called if the map contains no valid mappings. 1050 */ 1051void 1052pmap_release(pmap_t pm) 1053{ 1054 vm_object_t obj; 1055 vm_page_t m; 1056 struct pcpu *pc; 1057 1058 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1059 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1060 KASSERT(pmap_resident_count(pm) == 0, 1061 ("pmap_release: resident pages %ld != 0", 1062 pmap_resident_count(pm))); 1063 1064 /* 1065 * After the pmap was freed, it might be reallocated to a new process. 1066 * When switching, this might lead us to wrongly assume that we need 1067 * not switch contexts because old and new pmap pointer are equal. 1068 * Therefore, make sure that this pmap is not referenced by any PCPU 1069 * pointer any more. This could happen in two cases: 1070 * - A process that referenced the pmap is currently exiting on a CPU. 1071 * However, it is guaranteed to not switch in any more after setting 1072 * its state to PRS_ZOMBIE. 1073 * - A process that referenced this pmap ran on a CPU, but we switched 1074 * to a kernel thread, leaving the pmap pointer unchanged. 1075 */ 1076 mtx_lock_spin(&sched_lock); 1077 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 1078 if (pc->pc_pmap == pm) 1079 pc->pc_pmap = NULL; 1080 } 1081 mtx_unlock_spin(&sched_lock); 1082 1083 obj = pm->pm_tsb_obj; 1084 VM_OBJECT_LOCK(obj); 1085 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1086 while (!TAILQ_EMPTY(&obj->memq)) { 1087 m = TAILQ_FIRST(&obj->memq); 1088 vm_page_lock_queues(); 1089 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1090 continue; 1091 KASSERT(m->hold_count == 0, 1092 ("pmap_release: freeing held tsb page")); 1093 m->md.pmap = NULL; 1094 m->wire_count--; 1095 atomic_subtract_int(&cnt.v_wire_count, 1); 1096 vm_page_free_zero(m); 1097 vm_page_unlock_queues(); 1098 } 1099 VM_OBJECT_UNLOCK(obj); 1100 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1101 PMAP_LOCK_DESTROY(pm); 1102} 1103 1104/* 1105 * Grow the number of kernel page table entries. Unneeded. 1106 */ 1107void 1108pmap_growkernel(vm_offset_t addr) 1109{ 1110 1111 panic("pmap_growkernel: can't grow kernel"); 1112} 1113 1114int 1115pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1116 vm_offset_t va) 1117{ 1118 vm_page_t m; 1119 u_long data; 1120 1121 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1122 data = atomic_readandclear_long(&tp->tte_data); 1123 if ((data & TD_FAKE) == 0) { 1124 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1125 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1126 if ((data & TD_WIRED) != 0) 1127 pm->pm_stats.wired_count--; 1128 if ((data & TD_PV) != 0) { 1129 if ((data & TD_W) != 0) 1130 vm_page_dirty(m); 1131 if ((data & TD_REF) != 0) 1132 vm_page_flag_set(m, PG_REFERENCED); 1133 if (TAILQ_EMPTY(&m->md.tte_list)) 1134 vm_page_flag_clear(m, PG_WRITEABLE); 1135 pm->pm_stats.resident_count--; 1136 } 1137 pmap_cache_remove(m, va); 1138 } 1139 TTE_ZERO(tp); 1140 if (PMAP_REMOVE_DONE(pm)) 1141 return (0); 1142 return (1); 1143} 1144 1145/* 1146 * Remove the given range of addresses from the specified map. 1147 */ 1148void 1149pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1150{ 1151 struct tte *tp; 1152 vm_offset_t va; 1153 1154 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1155 pm->pm_context[PCPU_GET(cpuid)], start, end); 1156 if (PMAP_REMOVE_DONE(pm)) 1157 return; 1158 vm_page_lock_queues(); 1159 PMAP_LOCK(pm); 1160 if (end - start > PMAP_TSB_THRESH) { 1161 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1162 tlb_context_demap(pm); 1163 } else { 1164 for (va = start; va < end; va += PAGE_SIZE) { 1165 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1166 if (!pmap_remove_tte(pm, NULL, tp, va)) 1167 break; 1168 } 1169 } 1170 tlb_range_demap(pm, start, end - 1); 1171 } 1172 PMAP_UNLOCK(pm); 1173 vm_page_unlock_queues(); 1174} 1175 1176void 1177pmap_remove_all(vm_page_t m) 1178{ 1179 struct pmap *pm; 1180 struct tte *tpn; 1181 struct tte *tp; 1182 vm_offset_t va; 1183 1184 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1185 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1186 tpn = TAILQ_NEXT(tp, tte_link); 1187 if ((tp->tte_data & TD_PV) == 0) 1188 continue; 1189 pm = TTE_GET_PMAP(tp); 1190 va = TTE_GET_VA(tp); 1191 PMAP_LOCK(pm); 1192 if ((tp->tte_data & TD_WIRED) != 0) 1193 pm->pm_stats.wired_count--; 1194 if ((tp->tte_data & TD_REF) != 0) 1195 vm_page_flag_set(m, PG_REFERENCED); 1196 if ((tp->tte_data & TD_W) != 0) 1197 vm_page_dirty(m); 1198 tp->tte_data &= ~TD_V; 1199 tlb_page_demap(pm, va); 1200 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1201 pm->pm_stats.resident_count--; 1202 pmap_cache_remove(m, va); 1203 TTE_ZERO(tp); 1204 PMAP_UNLOCK(pm); 1205 } 1206 vm_page_flag_clear(m, PG_WRITEABLE); 1207} 1208 1209int 1210pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1211 vm_offset_t va) 1212{ 1213 u_long data; 1214 vm_page_t m; 1215 1216 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1217 if ((data & TD_PV) != 0) { 1218 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1219 if ((data & TD_REF) != 0) 1220 vm_page_flag_set(m, PG_REFERENCED); 1221 if ((data & TD_W) != 0) 1222 vm_page_dirty(m); 1223 } 1224 return (1); 1225} 1226 1227/* 1228 * Set the physical protection on the specified range of this map as requested. 1229 */ 1230void 1231pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1232{ 1233 vm_offset_t va; 1234 struct tte *tp; 1235 1236 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1237 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1238 1239 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1240 pmap_remove(pm, sva, eva); 1241 return; 1242 } 1243 1244 if (prot & VM_PROT_WRITE) 1245 return; 1246 1247 vm_page_lock_queues(); 1248 PMAP_LOCK(pm); 1249 if (eva - sva > PMAP_TSB_THRESH) { 1250 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1251 tlb_context_demap(pm); 1252 } else { 1253 for (va = sva; va < eva; va += PAGE_SIZE) { 1254 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1255 pmap_protect_tte(pm, NULL, tp, va); 1256 } 1257 tlb_range_demap(pm, sva, eva - 1); 1258 } 1259 PMAP_UNLOCK(pm); 1260 vm_page_unlock_queues(); 1261} 1262 1263/* 1264 * Map the given physical page at the specified virtual address in the 1265 * target pmap with the protection requested. If specified the page 1266 * will be wired down. 1267 */ 1268void 1269pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1270 boolean_t wired) 1271{ 1272 1273 vm_page_lock_queues(); 1274 PMAP_LOCK(pm); 1275 pmap_enter_locked(pm, va, m, prot, wired); 1276 vm_page_unlock_queues(); 1277 PMAP_UNLOCK(pm); 1278} 1279 1280/* 1281 * Map the given physical page at the specified virtual address in the 1282 * target pmap with the protection requested. If specified the page 1283 * will be wired down. 1284 * 1285 * The page queues and pmap must be locked. 1286 */ 1287static void 1288pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1289 boolean_t wired) 1290{ 1291 struct tte *tp; 1292 vm_paddr_t pa; 1293 u_long data; 1294 int i; 1295 1296 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1297 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1298 PMAP_STATS_INC(pmap_nenter); 1299 pa = VM_PAGE_TO_PHYS(m); 1300 1301 /* 1302 * If this is a fake page from the device_pager, but it covers actual 1303 * physical memory, convert to the real backing page. 1304 */ 1305 if ((m->flags & PG_FICTITIOUS) != 0) { 1306 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1307 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1308 m = PHYS_TO_VM_PAGE(pa); 1309 break; 1310 } 1311 } 1312 } 1313 1314 CTR6(KTR_PMAP, 1315 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1316 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1317 1318 /* 1319 * If there is an existing mapping, and the physical address has not 1320 * changed, must be protection or wiring change. 1321 */ 1322 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1323 CTR0(KTR_PMAP, "pmap_enter: update"); 1324 PMAP_STATS_INC(pmap_nenter_update); 1325 1326 /* 1327 * Wiring change, just update stats. 1328 */ 1329 if (wired) { 1330 if ((tp->tte_data & TD_WIRED) == 0) { 1331 tp->tte_data |= TD_WIRED; 1332 pm->pm_stats.wired_count++; 1333 } 1334 } else { 1335 if ((tp->tte_data & TD_WIRED) != 0) { 1336 tp->tte_data &= ~TD_WIRED; 1337 pm->pm_stats.wired_count--; 1338 } 1339 } 1340 1341 /* 1342 * Save the old bits and clear the ones we're interested in. 1343 */ 1344 data = tp->tte_data; 1345 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1346 1347 /* 1348 * If we're turning off write permissions, sense modify status. 1349 */ 1350 if ((prot & VM_PROT_WRITE) != 0) { 1351 tp->tte_data |= TD_SW; 1352 if (wired) { 1353 tp->tte_data |= TD_W; 1354 } 1355 } else if ((data & TD_W) != 0) { 1356 vm_page_dirty(m); 1357 } 1358 1359 /* 1360 * If we're turning on execute permissions, flush the icache. 1361 */ 1362 if ((prot & VM_PROT_EXECUTE) != 0) { 1363 if ((data & TD_EXEC) == 0) { 1364 icache_page_inval(pa); 1365 } 1366 tp->tte_data |= TD_EXEC; 1367 } 1368 1369 /* 1370 * Delete the old mapping. 1371 */ 1372 tlb_page_demap(pm, TTE_GET_VA(tp)); 1373 } else { 1374 /* 1375 * If there is an existing mapping, but its for a different 1376 * phsyical address, delete the old mapping. 1377 */ 1378 if (tp != NULL) { 1379 CTR0(KTR_PMAP, "pmap_enter: replace"); 1380 PMAP_STATS_INC(pmap_nenter_replace); 1381 pmap_remove_tte(pm, NULL, tp, va); 1382 tlb_page_demap(pm, va); 1383 } else { 1384 CTR0(KTR_PMAP, "pmap_enter: new"); 1385 PMAP_STATS_INC(pmap_nenter_new); 1386 } 1387 1388 /* 1389 * Now set up the data and install the new mapping. 1390 */ 1391 data = TD_V | TD_8K | TD_PA(pa); 1392 if (pm == kernel_pmap) 1393 data |= TD_P; 1394 if (prot & VM_PROT_WRITE) 1395 data |= TD_SW; 1396 if (prot & VM_PROT_EXECUTE) { 1397 data |= TD_EXEC; 1398 icache_page_inval(pa); 1399 } 1400 1401 /* 1402 * If its wired update stats. We also don't need reference or 1403 * modify tracking for wired mappings, so set the bits now. 1404 */ 1405 if (wired) { 1406 pm->pm_stats.wired_count++; 1407 data |= TD_REF | TD_WIRED; 1408 if ((prot & VM_PROT_WRITE) != 0) 1409 data |= TD_W; 1410 } 1411 1412 tsb_tte_enter(pm, m, va, TS_8K, data); 1413 } 1414} 1415 1416/* 1417 * Maps a sequence of resident pages belonging to the same object. 1418 * The sequence begins with the given page m_start. This page is 1419 * mapped at the given virtual address start. Each subsequent page is 1420 * mapped at a virtual address that is offset from start by the same 1421 * amount as the page is offset from m_start within the object. The 1422 * last page in the sequence is the page with the largest offset from 1423 * m_start that can be mapped at a virtual address less than the given 1424 * virtual address end. Not every virtual page between start and end 1425 * is mapped; only those for which a resident page exists with the 1426 * corresponding offset from m_start are mapped. 1427 */ 1428void 1429pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1430 vm_page_t m_start, vm_prot_t prot) 1431{ 1432 vm_page_t m; 1433 vm_pindex_t diff, psize; 1434 1435 psize = atop(end - start); 1436 m = m_start; 1437 PMAP_LOCK(pm); 1438 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1439 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1440 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1441 m = TAILQ_NEXT(m, listq); 1442 } 1443 PMAP_UNLOCK(pm); 1444} 1445 1446vm_page_t 1447pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1448 vm_page_t mpte) 1449{ 1450 1451 PMAP_LOCK(pm); 1452 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1453 FALSE); 1454 PMAP_UNLOCK(pm); 1455 return (NULL); 1456} 1457 1458void 1459pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1460 vm_pindex_t pindex, vm_size_t size) 1461{ 1462 1463 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1464 KASSERT(object->type == OBJT_DEVICE, 1465 ("pmap_object_init_pt: non-device object")); 1466} 1467 1468/* 1469 * Change the wiring attribute for a map/virtual-address pair. 1470 * The mapping must already exist in the pmap. 1471 */ 1472void 1473pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1474{ 1475 struct tte *tp; 1476 u_long data; 1477 1478 PMAP_LOCK(pm); 1479 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1480 if (wired) { 1481 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1482 if ((data & TD_WIRED) == 0) 1483 pm->pm_stats.wired_count++; 1484 } else { 1485 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1486 if ((data & TD_WIRED) != 0) 1487 pm->pm_stats.wired_count--; 1488 } 1489 } 1490 PMAP_UNLOCK(pm); 1491} 1492 1493static int 1494pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1495{ 1496 vm_page_t m; 1497 u_long data; 1498 1499 if ((tp->tte_data & TD_FAKE) != 0) 1500 return (1); 1501 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1502 data = tp->tte_data & 1503 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1504 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1505 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1506 } 1507 return (1); 1508} 1509 1510void 1511pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1512 vm_size_t len, vm_offset_t src_addr) 1513{ 1514 struct tte *tp; 1515 vm_offset_t va; 1516 1517 if (dst_addr != src_addr) 1518 return; 1519 vm_page_lock_queues(); 1520 if (dst_pmap < src_pmap) { 1521 PMAP_LOCK(dst_pmap); 1522 PMAP_LOCK(src_pmap); 1523 } else { 1524 PMAP_LOCK(src_pmap); 1525 PMAP_LOCK(dst_pmap); 1526 } 1527 if (len > PMAP_TSB_THRESH) { 1528 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1529 pmap_copy_tte); 1530 tlb_context_demap(dst_pmap); 1531 } else { 1532 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1533 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1534 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1535 } 1536 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1537 } 1538 vm_page_unlock_queues(); 1539 PMAP_UNLOCK(src_pmap); 1540 PMAP_UNLOCK(dst_pmap); 1541} 1542 1543void 1544pmap_zero_page(vm_page_t m) 1545{ 1546 struct tte *tp; 1547 vm_offset_t va; 1548 vm_paddr_t pa; 1549 1550 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1551 ("pmap_zero_page: fake page")); 1552 PMAP_STATS_INC(pmap_nzero_page); 1553 pa = VM_PAGE_TO_PHYS(m); 1554 if (m->md.color == -1) { 1555 PMAP_STATS_INC(pmap_nzero_page_nc); 1556 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1557 } else if (m->md.color == DCACHE_COLOR(pa)) { 1558 PMAP_STATS_INC(pmap_nzero_page_c); 1559 va = TLB_PHYS_TO_DIRECT(pa); 1560 cpu_block_zero((void *)va, PAGE_SIZE); 1561 } else { 1562 PMAP_STATS_INC(pmap_nzero_page_oc); 1563 PMAP_LOCK(kernel_pmap); 1564 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1565 tp = tsb_kvtotte(va); 1566 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1567 tp->tte_vpn = TV_VPN(va, TS_8K); 1568 cpu_block_zero((void *)va, PAGE_SIZE); 1569 tlb_page_demap(kernel_pmap, va); 1570 PMAP_UNLOCK(kernel_pmap); 1571 } 1572} 1573 1574void 1575pmap_zero_page_area(vm_page_t m, int off, int size) 1576{ 1577 struct tte *tp; 1578 vm_offset_t va; 1579 vm_paddr_t pa; 1580 1581 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1582 ("pmap_zero_page_area: fake page")); 1583 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1584 PMAP_STATS_INC(pmap_nzero_page_area); 1585 pa = VM_PAGE_TO_PHYS(m); 1586 if (m->md.color == -1) { 1587 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1588 aszero(ASI_PHYS_USE_EC, pa + off, size); 1589 } else if (m->md.color == DCACHE_COLOR(pa)) { 1590 PMAP_STATS_INC(pmap_nzero_page_area_c); 1591 va = TLB_PHYS_TO_DIRECT(pa); 1592 bzero((void *)(va + off), size); 1593 } else { 1594 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1595 PMAP_LOCK(kernel_pmap); 1596 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1597 tp = tsb_kvtotte(va); 1598 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1599 tp->tte_vpn = TV_VPN(va, TS_8K); 1600 bzero((void *)(va + off), size); 1601 tlb_page_demap(kernel_pmap, va); 1602 PMAP_UNLOCK(kernel_pmap); 1603 } 1604} 1605 1606void 1607pmap_zero_page_idle(vm_page_t m) 1608{ 1609 struct tte *tp; 1610 vm_offset_t va; 1611 vm_paddr_t pa; 1612 1613 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1614 ("pmap_zero_page_idle: fake page")); 1615 PMAP_STATS_INC(pmap_nzero_page_idle); 1616 pa = VM_PAGE_TO_PHYS(m); 1617 if (m->md.color == -1) { 1618 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1619 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1620 } else if (m->md.color == DCACHE_COLOR(pa)) { 1621 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1622 va = TLB_PHYS_TO_DIRECT(pa); 1623 cpu_block_zero((void *)va, PAGE_SIZE); 1624 } else { 1625 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1626 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1627 tp = tsb_kvtotte(va); 1628 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1629 tp->tte_vpn = TV_VPN(va, TS_8K); 1630 cpu_block_zero((void *)va, PAGE_SIZE); 1631 tlb_page_demap(kernel_pmap, va); 1632 } 1633} 1634 1635void 1636pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1637{ 1638 vm_offset_t vdst; 1639 vm_offset_t vsrc; 1640 vm_paddr_t pdst; 1641 vm_paddr_t psrc; 1642 struct tte *tp; 1643 1644 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1645 ("pmap_copy_page: fake dst page")); 1646 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1647 ("pmap_copy_page: fake src page")); 1648 PMAP_STATS_INC(pmap_ncopy_page); 1649 pdst = VM_PAGE_TO_PHYS(mdst); 1650 psrc = VM_PAGE_TO_PHYS(msrc); 1651 if (msrc->md.color == -1 && mdst->md.color == -1) { 1652 PMAP_STATS_INC(pmap_ncopy_page_nc); 1653 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1654 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1655 mdst->md.color == DCACHE_COLOR(pdst)) { 1656 PMAP_STATS_INC(pmap_ncopy_page_c); 1657 vdst = TLB_PHYS_TO_DIRECT(pdst); 1658 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1659 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1660 } else if (msrc->md.color == -1) { 1661 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1662 PMAP_STATS_INC(pmap_ncopy_page_dc); 1663 vdst = TLB_PHYS_TO_DIRECT(pdst); 1664 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1665 PAGE_SIZE); 1666 } else { 1667 PMAP_STATS_INC(pmap_ncopy_page_doc); 1668 PMAP_LOCK(kernel_pmap); 1669 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1670 tp = tsb_kvtotte(vdst); 1671 tp->tte_data = 1672 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1673 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1674 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1675 PAGE_SIZE); 1676 tlb_page_demap(kernel_pmap, vdst); 1677 PMAP_UNLOCK(kernel_pmap); 1678 } 1679 } else if (mdst->md.color == -1) { 1680 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1681 PMAP_STATS_INC(pmap_ncopy_page_sc); 1682 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1683 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1684 PAGE_SIZE); 1685 } else { 1686 PMAP_STATS_INC(pmap_ncopy_page_soc); 1687 PMAP_LOCK(kernel_pmap); 1688 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1689 tp = tsb_kvtotte(vsrc); 1690 tp->tte_data = 1691 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1692 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1693 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1694 PAGE_SIZE); 1695 tlb_page_demap(kernel_pmap, vsrc); 1696 PMAP_UNLOCK(kernel_pmap); 1697 } 1698 } else { 1699 PMAP_STATS_INC(pmap_ncopy_page_oc); 1700 PMAP_LOCK(kernel_pmap); 1701 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1702 tp = tsb_kvtotte(vdst); 1703 tp->tte_data = 1704 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1705 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1706 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1707 tp = tsb_kvtotte(vsrc); 1708 tp->tte_data = 1709 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1710 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1711 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1712 tlb_page_demap(kernel_pmap, vdst); 1713 tlb_page_demap(kernel_pmap, vsrc); 1714 PMAP_UNLOCK(kernel_pmap); 1715 } 1716} 1717 1718/* 1719 * Returns true if the pmap's pv is one of the first 1720 * 16 pvs linked to from this page. This count may 1721 * be changed upwards or downwards in the future; it 1722 * is only necessary that true be returned for a small 1723 * subset of pmaps for proper page aging. 1724 */ 1725boolean_t 1726pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1727{ 1728 struct tte *tp; 1729 int loops; 1730 1731 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1732 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1733 return (FALSE); 1734 loops = 0; 1735 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1736 if ((tp->tte_data & TD_PV) == 0) 1737 continue; 1738 if (TTE_GET_PMAP(tp) == pm) 1739 return (TRUE); 1740 if (++loops >= 16) 1741 break; 1742 } 1743 return (FALSE); 1744} 1745 1746/* 1747 * Remove all pages from specified address space, this aids process exit 1748 * speeds. This is much faster than pmap_remove n the case of running down 1749 * an entire address space. Only works for the current pmap. 1750 */ 1751void 1752pmap_remove_pages(pmap_t pm) 1753{ 1754} 1755 1756/* 1757 * Returns TRUE if the given page has a managed mapping. 1758 */ 1759boolean_t 1760pmap_page_is_mapped(vm_page_t m) 1761{ 1762 struct tte *tp; 1763 1764 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1765 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1766 return (FALSE); 1767 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1768 if ((tp->tte_data & TD_PV) != 0) 1769 return (TRUE); 1770 } 1771 return (FALSE); 1772} 1773 1774/* 1775 * Lower the permission for all mappings to a given page. 1776 */ 1777void 1778pmap_page_protect(vm_page_t m, vm_prot_t prot) 1779{ 1780 1781 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1782 ("pmap_page_protect: fake page")); 1783 if ((prot & VM_PROT_WRITE) == 0) { 1784 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1785 pmap_clear_write(m); 1786 else 1787 pmap_remove_all(m); 1788 } 1789} 1790 1791/* 1792 * pmap_ts_referenced: 1793 * 1794 * Return a count of reference bits for a page, clearing those bits. 1795 * It is not necessary for every reference bit to be cleared, but it 1796 * is necessary that 0 only be returned when there are truly no 1797 * reference bits set. 1798 * 1799 * XXX: The exact number of bits to check and clear is a matter that 1800 * should be tested and standardized at some point in the future for 1801 * optimal aging of shared pages. 1802 */ 1803 1804int 1805pmap_ts_referenced(vm_page_t m) 1806{ 1807 struct tte *tpf; 1808 struct tte *tpn; 1809 struct tte *tp; 1810 u_long data; 1811 int count; 1812 1813 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1814 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1815 return (0); 1816 count = 0; 1817 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1818 tpf = tp; 1819 do { 1820 tpn = TAILQ_NEXT(tp, tte_link); 1821 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1822 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1823 if ((tp->tte_data & TD_PV) == 0) 1824 continue; 1825 data = atomic_clear_long(&tp->tte_data, TD_REF); 1826 if ((data & TD_REF) != 0 && ++count > 4) 1827 break; 1828 } while ((tp = tpn) != NULL && tp != tpf); 1829 } 1830 return (count); 1831} 1832 1833boolean_t 1834pmap_is_modified(vm_page_t m) 1835{ 1836 struct tte *tp; 1837 1838 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1839 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1840 return (FALSE); 1841 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1842 if ((tp->tte_data & TD_PV) == 0) 1843 continue; 1844 if ((tp->tte_data & TD_W) != 0) 1845 return (TRUE); 1846 } 1847 return (FALSE); 1848} 1849 1850/* 1851 * pmap_is_prefaultable: 1852 * 1853 * Return whether or not the specified virtual address is elgible 1854 * for prefault. 1855 */ 1856boolean_t 1857pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1858{ 1859 1860 return (FALSE); 1861} 1862 1863void 1864pmap_clear_modify(vm_page_t m) 1865{ 1866 struct tte *tp; 1867 u_long data; 1868 1869 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1870 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1871 return; 1872 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1873 if ((tp->tte_data & TD_PV) == 0) 1874 continue; 1875 data = atomic_clear_long(&tp->tte_data, TD_W); 1876 if ((data & TD_W) != 0) 1877 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1878 } 1879} 1880 1881void 1882pmap_clear_reference(vm_page_t m) 1883{ 1884 struct tte *tp; 1885 u_long data; 1886 1887 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1888 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1889 return; 1890 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1891 if ((tp->tte_data & TD_PV) == 0) 1892 continue; 1893 data = atomic_clear_long(&tp->tte_data, TD_REF); 1894 if ((data & TD_REF) != 0) 1895 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1896 } 1897} 1898 1899void 1900pmap_clear_write(vm_page_t m) 1901{ 1902 struct tte *tp; 1903 u_long data; 1904 1905 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1906 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1907 (m->flags & PG_WRITEABLE) == 0) 1908 return; 1909 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1910 if ((tp->tte_data & TD_PV) == 0) 1911 continue; 1912 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1913 if ((data & TD_W) != 0) { 1914 vm_page_dirty(m); 1915 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1916 } 1917 } 1918 vm_page_flag_clear(m, PG_WRITEABLE); 1919} 1920 1921int 1922pmap_mincore(pmap_t pm, vm_offset_t addr) 1923{ 1924 /* TODO; */ 1925 return (0); 1926} 1927 1928/* 1929 * Activate a user pmap. The pmap must be activated before its address space 1930 * can be accessed in any way. 1931 */ 1932void 1933pmap_activate(struct thread *td) 1934{ 1935 struct vmspace *vm; 1936 struct pmap *pm; 1937 int context; 1938 1939 vm = td->td_proc->p_vmspace; 1940 pm = vmspace_pmap(vm); 1941 1942 mtx_lock_spin(&sched_lock); 1943 1944 context = PCPU_GET(tlb_ctx); 1945 if (context == PCPU_GET(tlb_ctx_max)) { 1946 tlb_flush_user(); 1947 context = PCPU_GET(tlb_ctx_min); 1948 } 1949 PCPU_SET(tlb_ctx, context + 1); 1950 1951 pm->pm_context[PCPU_GET(cpuid)] = context; 1952 pm->pm_active |= PCPU_GET(cpumask); 1953 PCPU_SET(pmap, pm); 1954 1955 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 1956 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 1957 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 1958 membar(Sync); 1959 1960 mtx_unlock_spin(&sched_lock); 1961} 1962 1963vm_offset_t 1964pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 1965{ 1966 1967 return (va); 1968} 1969