pmap.c revision 152630
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 152630 2005-11-20 06:09:49Z alc $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this module is called upon to 49 * provide software-use-only maps which may or may not be stored in the 50 * same form as hardware maps. These pseudo-maps are used to store 51 * intermediate results from copy operations to and from address spaces. 52 * 53 * Since the information managed by this module is also stored by the 54 * logical address mapping module, this module may throw away valid virtual 55 * to physical mappings at almost any time. However, invalidations of 56 * mappings must be done as requested. 57 * 58 * In order to cope with hardware architectures which make virtual to 59 * physical map invalidates expensive, this module may delay invalidate 60 * reduced protection operations until such time as they are actually 61 * necessary. This module is given full information as to which processors 62 * are currently using which maps, and to when physical maps must be made 63 * correct. 64 */ 65 66#include "opt_kstack_pages.h" 67#include "opt_msgbuf.h" 68#include "opt_pmap.h" 69 70#include <sys/param.h> 71#include <sys/kernel.h> 72#include <sys/ktr.h> 73#include <sys/lock.h> 74#include <sys/msgbuf.h> 75#include <sys/mutex.h> 76#include <sys/proc.h> 77#include <sys/smp.h> 78#include <sys/sysctl.h> 79#include <sys/systm.h> 80#include <sys/vmmeter.h> 81 82#include <dev/ofw/openfirm.h> 83 84#include <vm/vm.h> 85#include <vm/vm_param.h> 86#include <vm/vm_kern.h> 87#include <vm/vm_page.h> 88#include <vm/vm_map.h> 89#include <vm/vm_object.h> 90#include <vm/vm_extern.h> 91#include <vm/vm_pageout.h> 92#include <vm/vm_pager.h> 93 94#include <machine/cache.h> 95#include <machine/frame.h> 96#include <machine/instr.h> 97#include <machine/md_var.h> 98#include <machine/metadata.h> 99#include <machine/ofw_mem.h> 100#include <machine/smp.h> 101#include <machine/tlb.h> 102#include <machine/tte.h> 103#include <machine/tsb.h> 104 105#define PMAP_DEBUG 106 107#ifndef PMAP_SHPGPERPROC 108#define PMAP_SHPGPERPROC 200 109#endif 110 111/* 112 * Virtual and physical address of message buffer. 113 */ 114struct msgbuf *msgbufp; 115vm_paddr_t msgbuf_phys; 116 117/* 118 * Map of physical memory reagions. 119 */ 120vm_paddr_t phys_avail[128]; 121static struct ofw_mem_region mra[128]; 122struct ofw_mem_region sparc64_memreg[128]; 123int sparc64_nmemreg; 124static struct ofw_map translations[128]; 125static int translations_size; 126 127static vm_offset_t pmap_idle_map; 128static vm_offset_t pmap_temp_map_1; 129static vm_offset_t pmap_temp_map_2; 130 131/* 132 * First and last available kernel virtual addresses. 133 */ 134vm_offset_t virtual_avail; 135vm_offset_t virtual_end; 136vm_offset_t kernel_vm_end; 137 138vm_offset_t vm_max_kernel_address; 139 140/* 141 * Kernel pmap. 142 */ 143struct pmap kernel_pmap_store; 144 145/* 146 * Allocate physical memory for use in pmap_bootstrap. 147 */ 148static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 149 150extern int tl1_immu_miss_patch_1[]; 151extern int tl1_immu_miss_patch_2[]; 152extern int tl1_dmmu_miss_patch_1[]; 153extern int tl1_dmmu_miss_patch_2[]; 154extern int tl1_dmmu_prot_patch_1[]; 155extern int tl1_dmmu_prot_patch_2[]; 156 157/* 158 * If user pmap is processed with pmap_remove and with pmap_remove and the 159 * resident count drops to 0, there are no more pages to remove, so we 160 * need not continue. 161 */ 162#define PMAP_REMOVE_DONE(pm) \ 163 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 164 165/* 166 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 167 * and pmap_protect() instead of trying each virtual address. 168 */ 169#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 170 171SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 172 173PMAP_STATS_VAR(pmap_nenter); 174PMAP_STATS_VAR(pmap_nenter_update); 175PMAP_STATS_VAR(pmap_nenter_replace); 176PMAP_STATS_VAR(pmap_nenter_new); 177PMAP_STATS_VAR(pmap_nkenter); 178PMAP_STATS_VAR(pmap_nkenter_oc); 179PMAP_STATS_VAR(pmap_nkenter_stupid); 180PMAP_STATS_VAR(pmap_nkremove); 181PMAP_STATS_VAR(pmap_nqenter); 182PMAP_STATS_VAR(pmap_nqremove); 183PMAP_STATS_VAR(pmap_ncache_enter); 184PMAP_STATS_VAR(pmap_ncache_enter_c); 185PMAP_STATS_VAR(pmap_ncache_enter_oc); 186PMAP_STATS_VAR(pmap_ncache_enter_cc); 187PMAP_STATS_VAR(pmap_ncache_enter_coc); 188PMAP_STATS_VAR(pmap_ncache_enter_nc); 189PMAP_STATS_VAR(pmap_ncache_enter_cnc); 190PMAP_STATS_VAR(pmap_ncache_remove); 191PMAP_STATS_VAR(pmap_ncache_remove_c); 192PMAP_STATS_VAR(pmap_ncache_remove_oc); 193PMAP_STATS_VAR(pmap_ncache_remove_cc); 194PMAP_STATS_VAR(pmap_ncache_remove_coc); 195PMAP_STATS_VAR(pmap_ncache_remove_nc); 196PMAP_STATS_VAR(pmap_nzero_page); 197PMAP_STATS_VAR(pmap_nzero_page_c); 198PMAP_STATS_VAR(pmap_nzero_page_oc); 199PMAP_STATS_VAR(pmap_nzero_page_nc); 200PMAP_STATS_VAR(pmap_nzero_page_area); 201PMAP_STATS_VAR(pmap_nzero_page_area_c); 202PMAP_STATS_VAR(pmap_nzero_page_area_oc); 203PMAP_STATS_VAR(pmap_nzero_page_area_nc); 204PMAP_STATS_VAR(pmap_nzero_page_idle); 205PMAP_STATS_VAR(pmap_nzero_page_idle_c); 206PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 207PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 208PMAP_STATS_VAR(pmap_ncopy_page); 209PMAP_STATS_VAR(pmap_ncopy_page_c); 210PMAP_STATS_VAR(pmap_ncopy_page_oc); 211PMAP_STATS_VAR(pmap_ncopy_page_nc); 212PMAP_STATS_VAR(pmap_ncopy_page_dc); 213PMAP_STATS_VAR(pmap_ncopy_page_doc); 214PMAP_STATS_VAR(pmap_ncopy_page_sc); 215PMAP_STATS_VAR(pmap_ncopy_page_soc); 216 217PMAP_STATS_VAR(pmap_nnew_thread); 218PMAP_STATS_VAR(pmap_nnew_thread_oc); 219 220/* 221 * Quick sort callout for comparing memory regions. 222 */ 223static int mr_cmp(const void *a, const void *b); 224static int om_cmp(const void *a, const void *b); 225static int 226mr_cmp(const void *a, const void *b) 227{ 228 const struct ofw_mem_region *mra; 229 const struct ofw_mem_region *mrb; 230 231 mra = a; 232 mrb = b; 233 if (mra->mr_start < mrb->mr_start) 234 return (-1); 235 else if (mra->mr_start > mrb->mr_start) 236 return (1); 237 else 238 return (0); 239} 240static int 241om_cmp(const void *a, const void *b) 242{ 243 const struct ofw_map *oma; 244 const struct ofw_map *omb; 245 246 oma = a; 247 omb = b; 248 if (oma->om_start < omb->om_start) 249 return (-1); 250 else if (oma->om_start > omb->om_start) 251 return (1); 252 else 253 return (0); 254} 255 256/* 257 * Bootstrap the system enough to run with virtual memory. 258 */ 259void 260pmap_bootstrap(vm_offset_t ekva) 261{ 262 struct pmap *pm; 263 struct tte *tp; 264 vm_offset_t off; 265 vm_offset_t va; 266 vm_paddr_t pa; 267 vm_size_t physsz; 268 vm_size_t virtsz; 269 ihandle_t pmem; 270 ihandle_t vmem; 271 int sz; 272 int i; 273 int j; 274 275 /* 276 * Find out what physical memory is available from the prom and 277 * initialize the phys_avail array. This must be done before 278 * pmap_bootstrap_alloc is called. 279 */ 280 if ((pmem = OF_finddevice("/memory")) == -1) 281 panic("pmap_bootstrap: finddevice /memory"); 282 if ((sz = OF_getproplen(pmem, "available")) == -1) 283 panic("pmap_bootstrap: getproplen /memory/available"); 284 if (sizeof(phys_avail) < sz) 285 panic("pmap_bootstrap: phys_avail too small"); 286 if (sizeof(mra) < sz) 287 panic("pmap_bootstrap: mra too small"); 288 bzero(mra, sz); 289 if (OF_getprop(pmem, "available", mra, sz) == -1) 290 panic("pmap_bootstrap: getprop /memory/available"); 291 sz /= sizeof(*mra); 292 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 293 qsort(mra, sz, sizeof (*mra), mr_cmp); 294 physsz = 0; 295 getenv_quad("hw.physmem", &physmem); 296 for (i = 0, j = 0; i < sz; i++, j += 2) { 297 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 298 mra[i].mr_size); 299 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 300 if (btoc(physsz) < physmem) { 301 phys_avail[j] = mra[i].mr_start; 302 phys_avail[j + 1] = mra[i].mr_start + 303 (ctob(physmem) - physsz); 304 physsz = ctob(physmem); 305 } 306 break; 307 } 308 phys_avail[j] = mra[i].mr_start; 309 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 310 physsz += mra[i].mr_size; 311 } 312 physmem = btoc(physsz); 313 314 /* 315 * Calculate the size of kernel virtual memory, and the size and mask 316 * for the kernel tsb. 317 */ 318 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 319 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 320 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 321 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 322 323 /* 324 * Allocate the kernel tsb and lock it in the tlb. 325 */ 326 pa = pmap_bootstrap_alloc(tsb_kernel_size); 327 if (pa & PAGE_MASK_4M) 328 panic("pmap_bootstrap: tsb unaligned\n"); 329 tsb_kernel_phys = pa; 330 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 331 pmap_map_tsb(); 332 bzero(tsb_kernel, tsb_kernel_size); 333 334 /* 335 * Allocate and map the message buffer. 336 */ 337 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 338 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys); 339 340 /* 341 * Patch the virtual address and the tsb mask into the trap table. 342 */ 343 344#define SETHI(rd, imm22) \ 345 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 346 EIF_IMM((imm22) >> 10, 22)) 347#define OR_R_I_R(rd, imm13, rs1) \ 348 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 349 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 350 351#define PATCH(addr) do { \ 352 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 353 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 354 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 355 panic("pmap_boostrap: patched instructions have changed"); \ 356 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 357 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 358 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 359 flush(addr); \ 360 flush(addr + 1); \ 361 flush(addr + 2); \ 362} while (0) 363 364 PATCH(tl1_immu_miss_patch_1); 365 PATCH(tl1_immu_miss_patch_2); 366 PATCH(tl1_dmmu_miss_patch_1); 367 PATCH(tl1_dmmu_miss_patch_2); 368 PATCH(tl1_dmmu_prot_patch_1); 369 PATCH(tl1_dmmu_prot_patch_2); 370 371 /* 372 * Enter fake 8k pages for the 4MB kernel pages, so that 373 * pmap_kextract() will work for them. 374 */ 375 for (i = 0; i < kernel_tlb_slots; i++) { 376 pa = kernel_tlbs[i].te_pa; 377 va = kernel_tlbs[i].te_va; 378 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 379 tp = tsb_kvtotte(va + off); 380 tp->tte_vpn = TV_VPN(va + off, TS_8K); 381 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 382 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 383 } 384 } 385 386 /* 387 * Set the start and end of kva. The kernel is loaded at the first 388 * available 4 meg super page, so round up to the end of the page. 389 */ 390 virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 391 virtual_end = vm_max_kernel_address; 392 kernel_vm_end = vm_max_kernel_address; 393 394 /* 395 * Allocate kva space for temporary mappings. 396 */ 397 pmap_idle_map = virtual_avail; 398 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 399 pmap_temp_map_1 = virtual_avail; 400 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 401 pmap_temp_map_2 = virtual_avail; 402 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 403 404 /* 405 * Allocate a kernel stack with guard page for thread0 and map it into 406 * the kernel tsb. We must ensure that the virtual address is coloured 407 * properly, since we're allocating from phys_avail so the memory won't 408 * have an associated vm_page_t. 409 */ 410 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) * 411 PAGE_SIZE); 412 kstack0_phys = pa; 413 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 414 PAGE_SIZE; 415 kstack0 = virtual_avail; 416 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 417 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 418 ("pmap_bootstrap: kstack0 miscoloured")); 419 for (i = 0; i < KSTACK_PAGES; i++) { 420 pa = kstack0_phys + i * PAGE_SIZE; 421 va = kstack0 + i * PAGE_SIZE; 422 tp = tsb_kvtotte(va); 423 tp->tte_vpn = TV_VPN(va, TS_8K); 424 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 425 TD_CP | TD_CV | TD_P | TD_W; 426 } 427 428 /* 429 * Calculate the last available physical address. 430 */ 431 for (i = 0; phys_avail[i + 2] != 0; i += 2) 432 ; 433 Maxmem = sparc64_btop(phys_avail[i + 1]); 434 435 /* 436 * Add the prom mappings to the kernel tsb. 437 */ 438 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 439 panic("pmap_bootstrap: finddevice /virtual-memory"); 440 if ((sz = OF_getproplen(vmem, "translations")) == -1) 441 panic("pmap_bootstrap: getproplen translations"); 442 if (sizeof(translations) < sz) 443 panic("pmap_bootstrap: translations too small"); 444 bzero(translations, sz); 445 if (OF_getprop(vmem, "translations", translations, sz) == -1) 446 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 447 sz /= sizeof(*translations); 448 translations_size = sz; 449 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 450 qsort(translations, sz, sizeof (*translations), om_cmp); 451 for (i = 0; i < sz; i++) { 452 CTR3(KTR_PMAP, 453 "translation: start=%#lx size=%#lx tte=%#lx", 454 translations[i].om_start, translations[i].om_size, 455 translations[i].om_tte); 456 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 457 translations[i].om_start > VM_MAX_PROM_ADDRESS) 458 continue; 459 for (off = 0; off < translations[i].om_size; 460 off += PAGE_SIZE) { 461 va = translations[i].om_start + off; 462 tp = tsb_kvtotte(va); 463 tp->tte_vpn = TV_VPN(va, TS_8K); 464 tp->tte_data = 465 ((translations[i].om_tte & 466 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) + 467 off; 468 } 469 } 470 471 /* 472 * Get the available physical memory ranges from /memory/reg. These 473 * are only used for kernel dumps, but it may not be wise to do prom 474 * calls in that situation. 475 */ 476 if ((sz = OF_getproplen(pmem, "reg")) == -1) 477 panic("pmap_bootstrap: getproplen /memory/reg"); 478 if (sizeof(sparc64_memreg) < sz) 479 panic("pmap_bootstrap: sparc64_memreg too small"); 480 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 481 panic("pmap_bootstrap: getprop /memory/reg"); 482 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 483 484 /* 485 * Initialize the kernel pmap (which is statically allocated). 486 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 487 * but sparc64 start up is not ready to initialize mutexes yet. 488 * It is called in machdep.c. 489 */ 490 pm = kernel_pmap; 491 for (i = 0; i < MAXCPU; i++) 492 pm->pm_context[i] = TLB_CTX_KERNEL; 493 pm->pm_active = ~0; 494 495 /* XXX flush all non-locked tlb entries */ 496} 497 498void 499pmap_map_tsb(void) 500{ 501 vm_offset_t va; 502 vm_paddr_t pa; 503 u_long data; 504 u_long s; 505 int i; 506 507 s = intr_disable(); 508 509 /* 510 * Map the 4mb tsb pages. 511 */ 512 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 513 va = (vm_offset_t)tsb_kernel + i; 514 pa = tsb_kernel_phys + i; 515 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 516 TD_P | TD_W; 517 /* XXX - cheetah */ 518 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 519 TLB_TAR_CTX(TLB_CTX_KERNEL)); 520 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 521 } 522 523 /* 524 * Set the secondary context to be the kernel context (needed for 525 * fp block operations in the kernel and the cache code). 526 */ 527 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 528 membar(Sync); 529 530 intr_restore(s); 531} 532 533/* 534 * Allocate a physical page of memory directly from the phys_avail map. 535 * Can only be called from pmap_bootstrap before avail start and end are 536 * calculated. 537 */ 538static vm_paddr_t 539pmap_bootstrap_alloc(vm_size_t size) 540{ 541 vm_paddr_t pa; 542 int i; 543 544 size = round_page(size); 545 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 546 if (phys_avail[i + 1] - phys_avail[i] < size) 547 continue; 548 pa = phys_avail[i]; 549 phys_avail[i] += size; 550 return (pa); 551 } 552 panic("pmap_bootstrap_alloc"); 553} 554 555/* 556 * Initialize a vm_page's machine-dependent fields. 557 */ 558void 559pmap_page_init(vm_page_t m) 560{ 561 562 TAILQ_INIT(&m->md.tte_list); 563 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 564 m->md.flags = 0; 565 m->md.pmap = NULL; 566} 567 568/* 569 * Initialize the pmap module. 570 */ 571void 572pmap_init(void) 573{ 574 vm_offset_t addr; 575 vm_size_t size; 576 int result; 577 int i; 578 579 for (i = 0; i < translations_size; i++) { 580 addr = translations[i].om_start; 581 size = translations[i].om_size; 582 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 583 continue; 584 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE, 585 VM_PROT_ALL, VM_PROT_ALL, 0); 586 if (result != KERN_SUCCESS || addr != translations[i].om_start) 587 panic("pmap_init: vm_map_find"); 588 } 589} 590 591/* 592 * Extract the physical page address associated with the given 593 * map/virtual_address pair. 594 */ 595vm_paddr_t 596pmap_extract(pmap_t pm, vm_offset_t va) 597{ 598 struct tte *tp; 599 vm_paddr_t pa; 600 601 if (pm == kernel_pmap) 602 return (pmap_kextract(va)); 603 PMAP_LOCK(pm); 604 tp = tsb_tte_lookup(pm, va); 605 if (tp == NULL) 606 pa = 0; 607 else 608 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 609 PMAP_UNLOCK(pm); 610 return (pa); 611} 612 613/* 614 * Atomically extract and hold the physical page with the given 615 * pmap and virtual address pair if that mapping permits the given 616 * protection. 617 */ 618vm_page_t 619pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 620{ 621 struct tte *tp; 622 vm_page_t m; 623 624 m = NULL; 625 vm_page_lock_queues(); 626 if (pm == kernel_pmap) { 627 if (va >= VM_MIN_DIRECT_ADDRESS) { 628 tp = NULL; 629 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 630 vm_page_hold(m); 631 } else { 632 tp = tsb_kvtotte(va); 633 if ((tp->tte_data & TD_V) == 0) 634 tp = NULL; 635 } 636 } else { 637 PMAP_LOCK(pm); 638 tp = tsb_tte_lookup(pm, va); 639 } 640 if (tp != NULL && ((tp->tte_data & TD_SW) || 641 (prot & VM_PROT_WRITE) == 0)) { 642 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 643 vm_page_hold(m); 644 } 645 vm_page_unlock_queues(); 646 if (pm != kernel_pmap) 647 PMAP_UNLOCK(pm); 648 return (m); 649} 650 651/* 652 * Extract the physical page address associated with the given kernel virtual 653 * address. 654 */ 655vm_paddr_t 656pmap_kextract(vm_offset_t va) 657{ 658 struct tte *tp; 659 660 if (va >= VM_MIN_DIRECT_ADDRESS) 661 return (TLB_DIRECT_TO_PHYS(va)); 662 tp = tsb_kvtotte(va); 663 if ((tp->tte_data & TD_V) == 0) 664 return (0); 665 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 666} 667 668int 669pmap_cache_enter(vm_page_t m, vm_offset_t va) 670{ 671 struct tte *tp; 672 int color; 673 674 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 675 KASSERT((m->flags & PG_FICTITIOUS) == 0, 676 ("pmap_cache_enter: fake page")); 677 PMAP_STATS_INC(pmap_ncache_enter); 678 679 /* 680 * Find the color for this virtual address and note the added mapping. 681 */ 682 color = DCACHE_COLOR(va); 683 m->md.colors[color]++; 684 685 /* 686 * If all existing mappings have the same color, the mapping is 687 * cacheable. 688 */ 689 if (m->md.color == color) { 690 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 691 ("pmap_cache_enter: cacheable, mappings of other color")); 692 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 693 PMAP_STATS_INC(pmap_ncache_enter_c); 694 else 695 PMAP_STATS_INC(pmap_ncache_enter_oc); 696 return (1); 697 } 698 699 /* 700 * If there are no mappings of the other color, and the page still has 701 * the wrong color, this must be a new mapping. Change the color to 702 * match the new mapping, which is cacheable. We must flush the page 703 * from the cache now. 704 */ 705 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 706 KASSERT(m->md.colors[color] == 1, 707 ("pmap_cache_enter: changing color, not new mapping")); 708 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 709 m->md.color = color; 710 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 711 PMAP_STATS_INC(pmap_ncache_enter_cc); 712 else 713 PMAP_STATS_INC(pmap_ncache_enter_coc); 714 return (1); 715 } 716 717 /* 718 * If the mapping is already non-cacheable, just return. 719 */ 720 if (m->md.color == -1) { 721 PMAP_STATS_INC(pmap_ncache_enter_nc); 722 return (0); 723 } 724 725 PMAP_STATS_INC(pmap_ncache_enter_cnc); 726 727 /* 728 * Mark all mappings as uncacheable, flush any lines with the other 729 * color out of the dcache, and set the color to none (-1). 730 */ 731 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 732 atomic_clear_long(&tp->tte_data, TD_CV); 733 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 734 } 735 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 736 m->md.color = -1; 737 return (0); 738} 739 740void 741pmap_cache_remove(vm_page_t m, vm_offset_t va) 742{ 743 struct tte *tp; 744 int color; 745 746 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 747 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 748 m->md.colors[DCACHE_COLOR(va)]); 749 KASSERT((m->flags & PG_FICTITIOUS) == 0, 750 ("pmap_cache_remove: fake page")); 751 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 752 ("pmap_cache_remove: no mappings %d <= 0", 753 m->md.colors[DCACHE_COLOR(va)])); 754 PMAP_STATS_INC(pmap_ncache_remove); 755 756 /* 757 * Find the color for this virtual address and note the removal of 758 * the mapping. 759 */ 760 color = DCACHE_COLOR(va); 761 m->md.colors[color]--; 762 763 /* 764 * If the page is cacheable, just return and keep the same color, even 765 * if there are no longer any mappings. 766 */ 767 if (m->md.color != -1) { 768 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 769 PMAP_STATS_INC(pmap_ncache_remove_c); 770 else 771 PMAP_STATS_INC(pmap_ncache_remove_oc); 772 return; 773 } 774 775 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 776 ("pmap_cache_remove: uncacheable, no mappings of other color")); 777 778 /* 779 * If the page is not cacheable (color is -1), and the number of 780 * mappings for this color is not zero, just return. There are 781 * mappings of the other color still, so remain non-cacheable. 782 */ 783 if (m->md.colors[color] != 0) { 784 PMAP_STATS_INC(pmap_ncache_remove_nc); 785 return; 786 } 787 788 /* 789 * The number of mappings for this color is now zero. Recache the 790 * other colored mappings, and change the page color to the other 791 * color. There should be no lines in the data cache for this page, 792 * so flushing should not be needed. 793 */ 794 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 795 atomic_set_long(&tp->tte_data, TD_CV); 796 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 797 } 798 m->md.color = DCACHE_OTHER_COLOR(color); 799 800 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 801 PMAP_STATS_INC(pmap_ncache_remove_cc); 802 else 803 PMAP_STATS_INC(pmap_ncache_remove_coc); 804} 805 806/* 807 * Map a wired page into kernel virtual address space. 808 */ 809void 810pmap_kenter(vm_offset_t va, vm_page_t m) 811{ 812 vm_offset_t ova; 813 struct tte *tp; 814 vm_page_t om; 815 u_long data; 816 817 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 818 PMAP_STATS_INC(pmap_nkenter); 819 tp = tsb_kvtotte(va); 820 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 821 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 822 if (m->pc != DCACHE_COLOR(va)) { 823 CTR6(KTR_CT2, 824 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx", 825 va, VM_PAGE_TO_PHYS(m), m->object, 826 m->object ? m->object->pg_color : -1, 827 m->object ? m->object->type : -1, 828 m->pindex); 829 PMAP_STATS_INC(pmap_nkenter_oc); 830 } 831 if ((tp->tte_data & TD_V) != 0) { 832 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 833 ova = TTE_GET_VA(tp); 834 if (m == om && va == ova) { 835 PMAP_STATS_INC(pmap_nkenter_stupid); 836 return; 837 } 838 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 839 pmap_cache_remove(om, ova); 840 if (va != ova) 841 tlb_page_demap(kernel_pmap, ova); 842 } 843 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 844 TD_P | TD_W; 845 if (pmap_cache_enter(m, va) != 0) 846 data |= TD_CV; 847 tp->tte_vpn = TV_VPN(va, TS_8K); 848 tp->tte_data = data; 849 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 850} 851 852/* 853 * Map a wired page into kernel virtual address space. This additionally 854 * takes a flag argument wich is or'ed to the TTE data. This is used by 855 * bus_space_map(). 856 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 857 * to flush entries that might still be in the cache, if applicable. 858 */ 859void 860pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 861{ 862 struct tte *tp; 863 864 tp = tsb_kvtotte(va); 865 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 866 va, pa, tp, tp->tte_data); 867 tp->tte_vpn = TV_VPN(va, TS_8K); 868 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 869} 870 871/* 872 * Remove a wired page from kernel virtual address space. 873 */ 874void 875pmap_kremove(vm_offset_t va) 876{ 877 struct tte *tp; 878 vm_page_t m; 879 880 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 881 PMAP_STATS_INC(pmap_nkremove); 882 tp = tsb_kvtotte(va); 883 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 884 tp->tte_data); 885 if ((tp->tte_data & TD_V) == 0) 886 return; 887 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 888 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 889 pmap_cache_remove(m, va); 890 TTE_ZERO(tp); 891} 892 893/* 894 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 895 */ 896void 897pmap_kremove_flags(vm_offset_t va) 898{ 899 struct tte *tp; 900 901 tp = tsb_kvtotte(va); 902 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 903 tp->tte_data); 904 TTE_ZERO(tp); 905} 906 907/* 908 * Map a range of physical addresses into kernel virtual address space. 909 * 910 * The value passed in *virt is a suggested virtual address for the mapping. 911 * Architectures which can support a direct-mapped physical to virtual region 912 * can return the appropriate address within that region, leaving '*virt' 913 * unchanged. 914 */ 915vm_offset_t 916pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 917{ 918 919 return (TLB_PHYS_TO_DIRECT(start)); 920} 921 922/* 923 * Map a list of wired pages into kernel virtual address space. This is 924 * intended for temporary mappings which do not need page modification or 925 * references recorded. Existing mappings in the region are overwritten. 926 */ 927void 928pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 929{ 930 vm_offset_t va; 931 int locked; 932 933 PMAP_STATS_INC(pmap_nqenter); 934 va = sva; 935 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 936 vm_page_lock_queues(); 937 while (count-- > 0) { 938 pmap_kenter(va, *m); 939 va += PAGE_SIZE; 940 m++; 941 } 942 if (!locked) 943 vm_page_unlock_queues(); 944 tlb_range_demap(kernel_pmap, sva, va); 945} 946 947/* 948 * Remove page mappings from kernel virtual address space. Intended for 949 * temporary mappings entered by pmap_qenter. 950 */ 951void 952pmap_qremove(vm_offset_t sva, int count) 953{ 954 vm_offset_t va; 955 int locked; 956 957 PMAP_STATS_INC(pmap_nqremove); 958 va = sva; 959 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 960 vm_page_lock_queues(); 961 while (count-- > 0) { 962 pmap_kremove(va); 963 va += PAGE_SIZE; 964 } 965 if (!locked) 966 vm_page_unlock_queues(); 967 tlb_range_demap(kernel_pmap, sva, va); 968} 969 970/* 971 * Initialize the pmap associated with process 0. 972 */ 973void 974pmap_pinit0(pmap_t pm) 975{ 976 int i; 977 978 PMAP_LOCK_INIT(pm); 979 for (i = 0; i < MAXCPU; i++) 980 pm->pm_context[i] = 0; 981 pm->pm_active = 0; 982 pm->pm_tsb = NULL; 983 pm->pm_tsb_obj = NULL; 984 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 985} 986 987/* 988 * Initialize a preallocated and zeroed pmap structure, uch as one in a 989 * vmspace structure. 990 */ 991void 992pmap_pinit(pmap_t pm) 993{ 994 vm_page_t ma[TSB_PAGES]; 995 vm_page_t m; 996 int i; 997 998 PMAP_LOCK_INIT(pm); 999 1000 /* 1001 * Allocate kva space for the tsb. 1002 */ 1003 if (pm->pm_tsb == NULL) { 1004 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1005 TSB_BSIZE); 1006 } 1007 1008 /* 1009 * Allocate an object for it. 1010 */ 1011 if (pm->pm_tsb_obj == NULL) 1012 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1013 1014 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1015 for (i = 0; i < TSB_PAGES; i++) { 1016 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1017 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1018 1019 vm_page_lock_queues(); 1020 m->valid = VM_PAGE_BITS_ALL; 1021 m->md.pmap = pm; 1022 vm_page_unlock_queues(); 1023 1024 ma[i] = m; 1025 } 1026 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1027 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1028 1029 for (i = 0; i < MAXCPU; i++) 1030 pm->pm_context[i] = -1; 1031 pm->pm_active = 0; 1032 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1033} 1034 1035/* 1036 * Release any resources held by the given physical map. 1037 * Called when a pmap initialized by pmap_pinit is being released. 1038 * Should only be called if the map contains no valid mappings. 1039 */ 1040void 1041pmap_release(pmap_t pm) 1042{ 1043 vm_object_t obj; 1044 vm_page_t m; 1045 struct pcpu *pc; 1046 1047 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1048 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1049 KASSERT(pmap_resident_count(pm) == 0, 1050 ("pmap_release: resident pages %ld != 0", 1051 pmap_resident_count(pm))); 1052 1053 /* 1054 * After the pmap was freed, it might be reallocated to a new process. 1055 * When switching, this might lead us to wrongly assume that we need 1056 * not switch contexts because old and new pmap pointer are equal. 1057 * Therefore, make sure that this pmap is not referenced by any PCPU 1058 * pointer any more. This could happen in two cases: 1059 * - A process that referenced the pmap is currently exiting on a CPU. 1060 * However, it is guaranteed to not switch in any more after setting 1061 * its state to PRS_ZOMBIE. 1062 * - A process that referenced this pmap ran on a CPU, but we switched 1063 * to a kernel thread, leaving the pmap pointer unchanged. 1064 */ 1065 mtx_lock_spin(&sched_lock); 1066 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 1067 if (pc->pc_pmap == pm) 1068 pc->pc_pmap = NULL; 1069 } 1070 mtx_unlock_spin(&sched_lock); 1071 1072 obj = pm->pm_tsb_obj; 1073 VM_OBJECT_LOCK(obj); 1074 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1075 while (!TAILQ_EMPTY(&obj->memq)) { 1076 m = TAILQ_FIRST(&obj->memq); 1077 vm_page_lock_queues(); 1078 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1079 continue; 1080 KASSERT(m->hold_count == 0, 1081 ("pmap_release: freeing held tsb page")); 1082 m->md.pmap = NULL; 1083 m->wire_count--; 1084 atomic_subtract_int(&cnt.v_wire_count, 1); 1085 vm_page_free_zero(m); 1086 vm_page_unlock_queues(); 1087 } 1088 VM_OBJECT_UNLOCK(obj); 1089 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1090 PMAP_LOCK_DESTROY(pm); 1091} 1092 1093/* 1094 * Grow the number of kernel page table entries. Unneeded. 1095 */ 1096void 1097pmap_growkernel(vm_offset_t addr) 1098{ 1099 1100 panic("pmap_growkernel: can't grow kernel"); 1101} 1102 1103int 1104pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1105 vm_offset_t va) 1106{ 1107 vm_page_t m; 1108 u_long data; 1109 1110 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1111 data = atomic_readandclear_long(&tp->tte_data); 1112 if ((data & TD_FAKE) == 0) { 1113 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1114 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1115 if ((data & TD_WIRED) != 0) 1116 pm->pm_stats.wired_count--; 1117 if ((data & TD_PV) != 0) { 1118 if ((data & TD_W) != 0 && pmap_track_modified(pm, va)) 1119 vm_page_dirty(m); 1120 if ((data & TD_REF) != 0) 1121 vm_page_flag_set(m, PG_REFERENCED); 1122 if (TAILQ_EMPTY(&m->md.tte_list)) 1123 vm_page_flag_clear(m, PG_WRITEABLE); 1124 pm->pm_stats.resident_count--; 1125 } 1126 pmap_cache_remove(m, va); 1127 } 1128 TTE_ZERO(tp); 1129 if (PMAP_REMOVE_DONE(pm)) 1130 return (0); 1131 return (1); 1132} 1133 1134/* 1135 * Remove the given range of addresses from the specified map. 1136 */ 1137void 1138pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1139{ 1140 struct tte *tp; 1141 vm_offset_t va; 1142 1143 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1144 pm->pm_context[PCPU_GET(cpuid)], start, end); 1145 if (PMAP_REMOVE_DONE(pm)) 1146 return; 1147 vm_page_lock_queues(); 1148 PMAP_LOCK(pm); 1149 if (end - start > PMAP_TSB_THRESH) { 1150 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1151 tlb_context_demap(pm); 1152 } else { 1153 for (va = start; va < end; va += PAGE_SIZE) { 1154 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1155 if (!pmap_remove_tte(pm, NULL, tp, va)) 1156 break; 1157 } 1158 } 1159 tlb_range_demap(pm, start, end - 1); 1160 } 1161 PMAP_UNLOCK(pm); 1162 vm_page_unlock_queues(); 1163} 1164 1165void 1166pmap_remove_all(vm_page_t m) 1167{ 1168 struct pmap *pm; 1169 struct tte *tpn; 1170 struct tte *tp; 1171 vm_offset_t va; 1172 1173 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1174 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1175 tpn = TAILQ_NEXT(tp, tte_link); 1176 if ((tp->tte_data & TD_PV) == 0) 1177 continue; 1178 pm = TTE_GET_PMAP(tp); 1179 va = TTE_GET_VA(tp); 1180 PMAP_LOCK(pm); 1181 if ((tp->tte_data & TD_WIRED) != 0) 1182 pm->pm_stats.wired_count--; 1183 if ((tp->tte_data & TD_REF) != 0) 1184 vm_page_flag_set(m, PG_REFERENCED); 1185 if ((tp->tte_data & TD_W) != 0 && 1186 pmap_track_modified(pm, va)) 1187 vm_page_dirty(m); 1188 tp->tte_data &= ~TD_V; 1189 tlb_page_demap(pm, va); 1190 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1191 pm->pm_stats.resident_count--; 1192 pmap_cache_remove(m, va); 1193 TTE_ZERO(tp); 1194 PMAP_UNLOCK(pm); 1195 } 1196 vm_page_flag_clear(m, PG_WRITEABLE); 1197} 1198 1199int 1200pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1201 vm_offset_t va) 1202{ 1203 u_long data; 1204 vm_page_t m; 1205 1206 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1207 if ((data & TD_PV) != 0) { 1208 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1209 if ((data & TD_REF) != 0) 1210 vm_page_flag_set(m, PG_REFERENCED); 1211 if ((data & TD_W) != 0 && pmap_track_modified(pm, va)) 1212 vm_page_dirty(m); 1213 } 1214 return (1); 1215} 1216 1217/* 1218 * Set the physical protection on the specified range of this map as requested. 1219 */ 1220void 1221pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1222{ 1223 vm_offset_t va; 1224 struct tte *tp; 1225 1226 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1227 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1228 1229 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1230 pmap_remove(pm, sva, eva); 1231 return; 1232 } 1233 1234 if (prot & VM_PROT_WRITE) 1235 return; 1236 1237 vm_page_lock_queues(); 1238 PMAP_LOCK(pm); 1239 if (eva - sva > PMAP_TSB_THRESH) { 1240 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1241 tlb_context_demap(pm); 1242 } else { 1243 for (va = sva; va < eva; va += PAGE_SIZE) { 1244 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1245 pmap_protect_tte(pm, NULL, tp, va); 1246 } 1247 tlb_range_demap(pm, sva, eva - 1); 1248 } 1249 PMAP_UNLOCK(pm); 1250 vm_page_unlock_queues(); 1251} 1252 1253/* 1254 * Map the given physical page at the specified virtual address in the 1255 * target pmap with the protection requested. If specified the page 1256 * will be wired down. 1257 */ 1258void 1259pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1260 boolean_t wired) 1261{ 1262 struct tte *tp; 1263 vm_paddr_t pa; 1264 u_long data; 1265 int i; 1266 1267 PMAP_STATS_INC(pmap_nenter); 1268 pa = VM_PAGE_TO_PHYS(m); 1269 1270 /* 1271 * If this is a fake page from the device_pager, but it covers actual 1272 * physical memory, convert to the real backing page. 1273 */ 1274 if ((m->flags & PG_FICTITIOUS) != 0) { 1275 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1276 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1277 m = PHYS_TO_VM_PAGE(pa); 1278 break; 1279 } 1280 } 1281 } 1282 1283 CTR6(KTR_PMAP, 1284 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1285 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1286 1287 vm_page_lock_queues(); 1288 PMAP_LOCK(pm); 1289 1290 /* 1291 * If there is an existing mapping, and the physical address has not 1292 * changed, must be protection or wiring change. 1293 */ 1294 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1295 CTR0(KTR_PMAP, "pmap_enter: update"); 1296 PMAP_STATS_INC(pmap_nenter_update); 1297 1298 /* 1299 * Wiring change, just update stats. 1300 */ 1301 if (wired) { 1302 if ((tp->tte_data & TD_WIRED) == 0) { 1303 tp->tte_data |= TD_WIRED; 1304 pm->pm_stats.wired_count++; 1305 } 1306 } else { 1307 if ((tp->tte_data & TD_WIRED) != 0) { 1308 tp->tte_data &= ~TD_WIRED; 1309 pm->pm_stats.wired_count--; 1310 } 1311 } 1312 1313 /* 1314 * Save the old bits and clear the ones we're interested in. 1315 */ 1316 data = tp->tte_data; 1317 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1318 1319 /* 1320 * If we're turning off write permissions, sense modify status. 1321 */ 1322 if ((prot & VM_PROT_WRITE) != 0) { 1323 tp->tte_data |= TD_SW; 1324 if (wired) { 1325 tp->tte_data |= TD_W; 1326 } 1327 } else if ((data & TD_W) != 0 && 1328 pmap_track_modified(pm, va)) { 1329 vm_page_dirty(m); 1330 } 1331 1332 /* 1333 * If we're turning on execute permissions, flush the icache. 1334 */ 1335 if ((prot & VM_PROT_EXECUTE) != 0) { 1336 if ((data & TD_EXEC) == 0) { 1337 icache_page_inval(pa); 1338 } 1339 tp->tte_data |= TD_EXEC; 1340 } 1341 1342 /* 1343 * Delete the old mapping. 1344 */ 1345 tlb_page_demap(pm, TTE_GET_VA(tp)); 1346 } else { 1347 /* 1348 * If there is an existing mapping, but its for a different 1349 * phsyical address, delete the old mapping. 1350 */ 1351 if (tp != NULL) { 1352 CTR0(KTR_PMAP, "pmap_enter: replace"); 1353 PMAP_STATS_INC(pmap_nenter_replace); 1354 pmap_remove_tte(pm, NULL, tp, va); 1355 tlb_page_demap(pm, va); 1356 } else { 1357 CTR0(KTR_PMAP, "pmap_enter: new"); 1358 PMAP_STATS_INC(pmap_nenter_new); 1359 } 1360 1361 /* 1362 * Now set up the data and install the new mapping. 1363 */ 1364 data = TD_V | TD_8K | TD_PA(pa); 1365 if (pm == kernel_pmap) 1366 data |= TD_P; 1367 if (prot & VM_PROT_WRITE) 1368 data |= TD_SW; 1369 if (prot & VM_PROT_EXECUTE) { 1370 data |= TD_EXEC; 1371 icache_page_inval(pa); 1372 } 1373 1374 /* 1375 * If its wired update stats. We also don't need reference or 1376 * modify tracking for wired mappings, so set the bits now. 1377 */ 1378 if (wired) { 1379 pm->pm_stats.wired_count++; 1380 data |= TD_REF | TD_WIRED; 1381 if ((prot & VM_PROT_WRITE) != 0) 1382 data |= TD_W; 1383 } 1384 1385 tsb_tte_enter(pm, m, va, TS_8K, data); 1386 } 1387 vm_page_unlock_queues(); 1388 PMAP_UNLOCK(pm); 1389} 1390 1391vm_page_t 1392pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1393 vm_page_t mpte) 1394{ 1395 1396 vm_page_busy(m); 1397 vm_page_unlock_queues(); 1398 VM_OBJECT_UNLOCK(m->object); 1399 pmap_enter(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1400 VM_OBJECT_LOCK(m->object); 1401 vm_page_lock_queues(); 1402 vm_page_wakeup(m); 1403 return (NULL); 1404} 1405 1406void 1407pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1408 vm_pindex_t pindex, vm_size_t size) 1409{ 1410 1411 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1412 KASSERT(object->type == OBJT_DEVICE, 1413 ("pmap_object_init_pt: non-device object")); 1414} 1415 1416/* 1417 * Change the wiring attribute for a map/virtual-address pair. 1418 * The mapping must already exist in the pmap. 1419 */ 1420void 1421pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1422{ 1423 struct tte *tp; 1424 u_long data; 1425 1426 PMAP_LOCK(pm); 1427 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1428 if (wired) { 1429 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1430 if ((data & TD_WIRED) == 0) 1431 pm->pm_stats.wired_count++; 1432 } else { 1433 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1434 if ((data & TD_WIRED) != 0) 1435 pm->pm_stats.wired_count--; 1436 } 1437 } 1438 PMAP_UNLOCK(pm); 1439} 1440 1441static int 1442pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1443{ 1444 vm_page_t m; 1445 u_long data; 1446 1447 if ((tp->tte_data & TD_FAKE) != 0) 1448 return (1); 1449 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1450 data = tp->tte_data & 1451 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1452 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1453 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1454 } 1455 return (1); 1456} 1457 1458void 1459pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1460 vm_size_t len, vm_offset_t src_addr) 1461{ 1462 struct tte *tp; 1463 vm_offset_t va; 1464 1465 if (dst_addr != src_addr) 1466 return; 1467 vm_page_lock_queues(); 1468 if (dst_pmap < src_pmap) { 1469 PMAP_LOCK(dst_pmap); 1470 PMAP_LOCK(src_pmap); 1471 } else { 1472 PMAP_LOCK(src_pmap); 1473 PMAP_LOCK(dst_pmap); 1474 } 1475 if (len > PMAP_TSB_THRESH) { 1476 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1477 pmap_copy_tte); 1478 tlb_context_demap(dst_pmap); 1479 } else { 1480 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1481 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1482 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1483 } 1484 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1485 } 1486 vm_page_unlock_queues(); 1487 PMAP_UNLOCK(src_pmap); 1488 PMAP_UNLOCK(dst_pmap); 1489} 1490 1491void 1492pmap_zero_page(vm_page_t m) 1493{ 1494 struct tte *tp; 1495 vm_offset_t va; 1496 vm_paddr_t pa; 1497 1498 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1499 ("pmap_zero_page: fake page")); 1500 PMAP_STATS_INC(pmap_nzero_page); 1501 pa = VM_PAGE_TO_PHYS(m); 1502 if (m->md.color == -1) { 1503 PMAP_STATS_INC(pmap_nzero_page_nc); 1504 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1505 } else if (m->md.color == DCACHE_COLOR(pa)) { 1506 PMAP_STATS_INC(pmap_nzero_page_c); 1507 va = TLB_PHYS_TO_DIRECT(pa); 1508 cpu_block_zero((void *)va, PAGE_SIZE); 1509 } else { 1510 PMAP_STATS_INC(pmap_nzero_page_oc); 1511 PMAP_LOCK(kernel_pmap); 1512 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1513 tp = tsb_kvtotte(va); 1514 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1515 tp->tte_vpn = TV_VPN(va, TS_8K); 1516 cpu_block_zero((void *)va, PAGE_SIZE); 1517 tlb_page_demap(kernel_pmap, va); 1518 PMAP_UNLOCK(kernel_pmap); 1519 } 1520} 1521 1522void 1523pmap_zero_page_area(vm_page_t m, int off, int size) 1524{ 1525 struct tte *tp; 1526 vm_offset_t va; 1527 vm_paddr_t pa; 1528 1529 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1530 ("pmap_zero_page_area: fake page")); 1531 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1532 PMAP_STATS_INC(pmap_nzero_page_area); 1533 pa = VM_PAGE_TO_PHYS(m); 1534 if (m->md.color == -1) { 1535 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1536 aszero(ASI_PHYS_USE_EC, pa + off, size); 1537 } else if (m->md.color == DCACHE_COLOR(pa)) { 1538 PMAP_STATS_INC(pmap_nzero_page_area_c); 1539 va = TLB_PHYS_TO_DIRECT(pa); 1540 bzero((void *)(va + off), size); 1541 } else { 1542 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1543 PMAP_LOCK(kernel_pmap); 1544 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1545 tp = tsb_kvtotte(va); 1546 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1547 tp->tte_vpn = TV_VPN(va, TS_8K); 1548 bzero((void *)(va + off), size); 1549 tlb_page_demap(kernel_pmap, va); 1550 PMAP_UNLOCK(kernel_pmap); 1551 } 1552} 1553 1554void 1555pmap_zero_page_idle(vm_page_t m) 1556{ 1557 struct tte *tp; 1558 vm_offset_t va; 1559 vm_paddr_t pa; 1560 1561 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1562 ("pmap_zero_page_idle: fake page")); 1563 PMAP_STATS_INC(pmap_nzero_page_idle); 1564 pa = VM_PAGE_TO_PHYS(m); 1565 if (m->md.color == -1) { 1566 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1567 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1568 } else if (m->md.color == DCACHE_COLOR(pa)) { 1569 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1570 va = TLB_PHYS_TO_DIRECT(pa); 1571 cpu_block_zero((void *)va, PAGE_SIZE); 1572 } else { 1573 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1574 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1575 tp = tsb_kvtotte(va); 1576 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1577 tp->tte_vpn = TV_VPN(va, TS_8K); 1578 cpu_block_zero((void *)va, PAGE_SIZE); 1579 tlb_page_demap(kernel_pmap, va); 1580 } 1581} 1582 1583void 1584pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1585{ 1586 vm_offset_t vdst; 1587 vm_offset_t vsrc; 1588 vm_paddr_t pdst; 1589 vm_paddr_t psrc; 1590 struct tte *tp; 1591 1592 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1593 ("pmap_copy_page: fake dst page")); 1594 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1595 ("pmap_copy_page: fake src page")); 1596 PMAP_STATS_INC(pmap_ncopy_page); 1597 pdst = VM_PAGE_TO_PHYS(mdst); 1598 psrc = VM_PAGE_TO_PHYS(msrc); 1599 if (msrc->md.color == -1 && mdst->md.color == -1) { 1600 PMAP_STATS_INC(pmap_ncopy_page_nc); 1601 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1602 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1603 mdst->md.color == DCACHE_COLOR(pdst)) { 1604 PMAP_STATS_INC(pmap_ncopy_page_c); 1605 vdst = TLB_PHYS_TO_DIRECT(pdst); 1606 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1607 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1608 } else if (msrc->md.color == -1) { 1609 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1610 PMAP_STATS_INC(pmap_ncopy_page_dc); 1611 vdst = TLB_PHYS_TO_DIRECT(pdst); 1612 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1613 PAGE_SIZE); 1614 } else { 1615 PMAP_STATS_INC(pmap_ncopy_page_doc); 1616 PMAP_LOCK(kernel_pmap); 1617 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1618 tp = tsb_kvtotte(vdst); 1619 tp->tte_data = 1620 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1621 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1622 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1623 PAGE_SIZE); 1624 tlb_page_demap(kernel_pmap, vdst); 1625 PMAP_UNLOCK(kernel_pmap); 1626 } 1627 } else if (mdst->md.color == -1) { 1628 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1629 PMAP_STATS_INC(pmap_ncopy_page_sc); 1630 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1631 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1632 PAGE_SIZE); 1633 } else { 1634 PMAP_STATS_INC(pmap_ncopy_page_soc); 1635 PMAP_LOCK(kernel_pmap); 1636 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1637 tp = tsb_kvtotte(vsrc); 1638 tp->tte_data = 1639 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1640 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1641 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1642 PAGE_SIZE); 1643 tlb_page_demap(kernel_pmap, vsrc); 1644 PMAP_UNLOCK(kernel_pmap); 1645 } 1646 } else { 1647 PMAP_STATS_INC(pmap_ncopy_page_oc); 1648 PMAP_LOCK(kernel_pmap); 1649 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1650 tp = tsb_kvtotte(vdst); 1651 tp->tte_data = 1652 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1653 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1654 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1655 tp = tsb_kvtotte(vsrc); 1656 tp->tte_data = 1657 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1658 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1659 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1660 tlb_page_demap(kernel_pmap, vdst); 1661 tlb_page_demap(kernel_pmap, vsrc); 1662 PMAP_UNLOCK(kernel_pmap); 1663 } 1664} 1665 1666/* 1667 * Returns true if the pmap's pv is one of the first 1668 * 16 pvs linked to from this page. This count may 1669 * be changed upwards or downwards in the future; it 1670 * is only necessary that true be returned for a small 1671 * subset of pmaps for proper page aging. 1672 */ 1673boolean_t 1674pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1675{ 1676 struct tte *tp; 1677 int loops; 1678 1679 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1680 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1681 return (FALSE); 1682 loops = 0; 1683 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1684 if ((tp->tte_data & TD_PV) == 0) 1685 continue; 1686 if (TTE_GET_PMAP(tp) == pm) 1687 return (TRUE); 1688 if (++loops >= 16) 1689 break; 1690 } 1691 return (FALSE); 1692} 1693 1694/* 1695 * Remove all pages from specified address space, this aids process exit 1696 * speeds. This is much faster than pmap_remove n the case of running down 1697 * an entire address space. Only works for the current pmap. 1698 */ 1699void 1700pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1701{ 1702} 1703 1704/* 1705 * Returns TRUE if the given page has a managed mapping. 1706 */ 1707boolean_t 1708pmap_page_is_mapped(vm_page_t m) 1709{ 1710 struct tte *tp; 1711 1712 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1713 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1714 return (FALSE); 1715 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1716 if ((tp->tte_data & TD_PV) != 0) 1717 return (TRUE); 1718 } 1719 return (FALSE); 1720} 1721 1722/* 1723 * Lower the permission for all mappings to a given page. 1724 */ 1725void 1726pmap_page_protect(vm_page_t m, vm_prot_t prot) 1727{ 1728 1729 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1730 ("pmap_page_protect: fake page")); 1731 if ((prot & VM_PROT_WRITE) == 0) { 1732 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1733 pmap_clear_write(m); 1734 else 1735 pmap_remove_all(m); 1736 } 1737} 1738 1739/* 1740 * pmap_ts_referenced: 1741 * 1742 * Return a count of reference bits for a page, clearing those bits. 1743 * It is not necessary for every reference bit to be cleared, but it 1744 * is necessary that 0 only be returned when there are truly no 1745 * reference bits set. 1746 * 1747 * XXX: The exact number of bits to check and clear is a matter that 1748 * should be tested and standardized at some point in the future for 1749 * optimal aging of shared pages. 1750 */ 1751 1752int 1753pmap_ts_referenced(vm_page_t m) 1754{ 1755 struct tte *tpf; 1756 struct tte *tpn; 1757 struct tte *tp; 1758 u_long data; 1759 int count; 1760 1761 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1762 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1763 return (0); 1764 count = 0; 1765 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1766 tpf = tp; 1767 do { 1768 tpn = TAILQ_NEXT(tp, tte_link); 1769 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1770 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1771 if ((tp->tte_data & TD_PV) == 0 || 1772 !pmap_track_modified(TTE_GET_PMAP(tp), 1773 TTE_GET_VA(tp))) 1774 continue; 1775 data = atomic_clear_long(&tp->tte_data, TD_REF); 1776 if ((data & TD_REF) != 0 && ++count > 4) 1777 break; 1778 } while ((tp = tpn) != NULL && tp != tpf); 1779 } 1780 return (count); 1781} 1782 1783boolean_t 1784pmap_is_modified(vm_page_t m) 1785{ 1786 struct tte *tp; 1787 1788 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1789 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1790 return (FALSE); 1791 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1792 if ((tp->tte_data & TD_PV) == 0 || 1793 !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) 1794 continue; 1795 if ((tp->tte_data & TD_W) != 0) 1796 return (TRUE); 1797 } 1798 return (FALSE); 1799} 1800 1801/* 1802 * pmap_is_prefaultable: 1803 * 1804 * Return whether or not the specified virtual address is elgible 1805 * for prefault. 1806 */ 1807boolean_t 1808pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1809{ 1810 1811 return (FALSE); 1812} 1813 1814void 1815pmap_clear_modify(vm_page_t m) 1816{ 1817 struct tte *tp; 1818 u_long data; 1819 1820 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1821 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1822 return; 1823 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1824 if ((tp->tte_data & TD_PV) == 0) 1825 continue; 1826 data = atomic_clear_long(&tp->tte_data, TD_W); 1827 if ((data & TD_W) != 0) 1828 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1829 } 1830} 1831 1832void 1833pmap_clear_reference(vm_page_t m) 1834{ 1835 struct tte *tp; 1836 u_long data; 1837 1838 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1839 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1840 return; 1841 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1842 if ((tp->tte_data & TD_PV) == 0) 1843 continue; 1844 data = atomic_clear_long(&tp->tte_data, TD_REF); 1845 if ((data & TD_REF) != 0) 1846 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1847 } 1848} 1849 1850void 1851pmap_clear_write(vm_page_t m) 1852{ 1853 struct tte *tp; 1854 u_long data; 1855 1856 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1857 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1858 (m->flags & PG_WRITEABLE) == 0) 1859 return; 1860 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1861 if ((tp->tte_data & TD_PV) == 0) 1862 continue; 1863 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1864 if ((data & TD_W) != 0) { 1865 if (pmap_track_modified(TTE_GET_PMAP(tp), 1866 TTE_GET_VA(tp))) 1867 vm_page_dirty(m); 1868 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1869 } 1870 } 1871 vm_page_flag_clear(m, PG_WRITEABLE); 1872} 1873 1874int 1875pmap_mincore(pmap_t pm, vm_offset_t addr) 1876{ 1877 /* TODO; */ 1878 return (0); 1879} 1880 1881/* 1882 * Activate a user pmap. The pmap must be activated before its address space 1883 * can be accessed in any way. 1884 */ 1885void 1886pmap_activate(struct thread *td) 1887{ 1888 struct vmspace *vm; 1889 struct pmap *pm; 1890 int context; 1891 1892 vm = td->td_proc->p_vmspace; 1893 pm = vmspace_pmap(vm); 1894 1895 mtx_lock_spin(&sched_lock); 1896 1897 context = PCPU_GET(tlb_ctx); 1898 if (context == PCPU_GET(tlb_ctx_max)) { 1899 tlb_flush_user(); 1900 context = PCPU_GET(tlb_ctx_min); 1901 } 1902 PCPU_SET(tlb_ctx, context + 1); 1903 1904 pm->pm_context[PCPU_GET(cpuid)] = context; 1905 pm->pm_active |= PCPU_GET(cpumask); 1906 PCPU_SET(pmap, pm); 1907 1908 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 1909 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 1910 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 1911 membar(Sync); 1912 1913 mtx_unlock_spin(&sched_lock); 1914} 1915 1916vm_offset_t 1917pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 1918{ 1919 1920 return (va); 1921} 1922