pmap.c revision 170170
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 170170 2007-05-31 22:52:15Z attilio $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this module is called upon to 49 * provide software-use-only maps which may or may not be stored in the 50 * same form as hardware maps. These pseudo-maps are used to store 51 * intermediate results from copy operations to and from address spaces. 52 * 53 * Since the information managed by this module is also stored by the 54 * logical address mapping module, this module may throw away valid virtual 55 * to physical mappings at almost any time. However, invalidations of 56 * mappings must be done as requested. 57 * 58 * In order to cope with hardware architectures which make virtual to 59 * physical map invalidates expensive, this module may delay invalidate 60 * reduced protection operations until such time as they are actually 61 * necessary. This module is given full information as to which processors 62 * are currently using which maps, and to when physical maps must be made 63 * correct. 64 */ 65 66#include "opt_kstack_pages.h" 67#include "opt_msgbuf.h" 68#include "opt_pmap.h" 69 70#include <sys/param.h> 71#include <sys/kernel.h> 72#include <sys/ktr.h> 73#include <sys/lock.h> 74#include <sys/msgbuf.h> 75#include <sys/mutex.h> 76#include <sys/proc.h> 77#include <sys/smp.h> 78#include <sys/sysctl.h> 79#include <sys/systm.h> 80#include <sys/vmmeter.h> 81 82#include <dev/ofw/openfirm.h> 83 84#include <vm/vm.h> 85#include <vm/vm_param.h> 86#include <vm/vm_kern.h> 87#include <vm/vm_page.h> 88#include <vm/vm_map.h> 89#include <vm/vm_object.h> 90#include <vm/vm_extern.h> 91#include <vm/vm_pageout.h> 92#include <vm/vm_pager.h> 93 94#include <machine/cache.h> 95#include <machine/frame.h> 96#include <machine/instr.h> 97#include <machine/md_var.h> 98#include <machine/metadata.h> 99#include <machine/ofw_mem.h> 100#include <machine/smp.h> 101#include <machine/tlb.h> 102#include <machine/tte.h> 103#include <machine/tsb.h> 104 105#define PMAP_DEBUG 106 107#ifndef PMAP_SHPGPERPROC 108#define PMAP_SHPGPERPROC 200 109#endif 110 111/* 112 * Virtual and physical address of message buffer. 113 */ 114struct msgbuf *msgbufp; 115vm_paddr_t msgbuf_phys; 116 117/* 118 * Map of physical memory reagions. 119 */ 120vm_paddr_t phys_avail[128]; 121static struct ofw_mem_region mra[128]; 122struct ofw_mem_region sparc64_memreg[128]; 123int sparc64_nmemreg; 124static struct ofw_map translations[128]; 125static int translations_size; 126 127static vm_offset_t pmap_idle_map; 128static vm_offset_t pmap_temp_map_1; 129static vm_offset_t pmap_temp_map_2; 130 131/* 132 * First and last available kernel virtual addresses. 133 */ 134vm_offset_t virtual_avail; 135vm_offset_t virtual_end; 136vm_offset_t kernel_vm_end; 137 138vm_offset_t vm_max_kernel_address; 139 140/* 141 * Kernel pmap. 142 */ 143struct pmap kernel_pmap_store; 144 145/* 146 * Allocate physical memory for use in pmap_bootstrap. 147 */ 148static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 149 150/* 151 * Map the given physical page at the specified virtual address in the 152 * target pmap with the protection requested. If specified the page 153 * will be wired down. 154 * 155 * The page queues and pmap must be locked. 156 */ 157static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 158 vm_prot_t prot, boolean_t wired); 159 160extern int tl1_immu_miss_patch_1[]; 161extern int tl1_immu_miss_patch_2[]; 162extern int tl1_dmmu_miss_patch_1[]; 163extern int tl1_dmmu_miss_patch_2[]; 164extern int tl1_dmmu_prot_patch_1[]; 165extern int tl1_dmmu_prot_patch_2[]; 166 167/* 168 * If user pmap is processed with pmap_remove and with pmap_remove and the 169 * resident count drops to 0, there are no more pages to remove, so we 170 * need not continue. 171 */ 172#define PMAP_REMOVE_DONE(pm) \ 173 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 174 175/* 176 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 177 * and pmap_protect() instead of trying each virtual address. 178 */ 179#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 180 181SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 182 183PMAP_STATS_VAR(pmap_nenter); 184PMAP_STATS_VAR(pmap_nenter_update); 185PMAP_STATS_VAR(pmap_nenter_replace); 186PMAP_STATS_VAR(pmap_nenter_new); 187PMAP_STATS_VAR(pmap_nkenter); 188PMAP_STATS_VAR(pmap_nkenter_oc); 189PMAP_STATS_VAR(pmap_nkenter_stupid); 190PMAP_STATS_VAR(pmap_nkremove); 191PMAP_STATS_VAR(pmap_nqenter); 192PMAP_STATS_VAR(pmap_nqremove); 193PMAP_STATS_VAR(pmap_ncache_enter); 194PMAP_STATS_VAR(pmap_ncache_enter_c); 195PMAP_STATS_VAR(pmap_ncache_enter_oc); 196PMAP_STATS_VAR(pmap_ncache_enter_cc); 197PMAP_STATS_VAR(pmap_ncache_enter_coc); 198PMAP_STATS_VAR(pmap_ncache_enter_nc); 199PMAP_STATS_VAR(pmap_ncache_enter_cnc); 200PMAP_STATS_VAR(pmap_ncache_remove); 201PMAP_STATS_VAR(pmap_ncache_remove_c); 202PMAP_STATS_VAR(pmap_ncache_remove_oc); 203PMAP_STATS_VAR(pmap_ncache_remove_cc); 204PMAP_STATS_VAR(pmap_ncache_remove_coc); 205PMAP_STATS_VAR(pmap_ncache_remove_nc); 206PMAP_STATS_VAR(pmap_nzero_page); 207PMAP_STATS_VAR(pmap_nzero_page_c); 208PMAP_STATS_VAR(pmap_nzero_page_oc); 209PMAP_STATS_VAR(pmap_nzero_page_nc); 210PMAP_STATS_VAR(pmap_nzero_page_area); 211PMAP_STATS_VAR(pmap_nzero_page_area_c); 212PMAP_STATS_VAR(pmap_nzero_page_area_oc); 213PMAP_STATS_VAR(pmap_nzero_page_area_nc); 214PMAP_STATS_VAR(pmap_nzero_page_idle); 215PMAP_STATS_VAR(pmap_nzero_page_idle_c); 216PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 217PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 218PMAP_STATS_VAR(pmap_ncopy_page); 219PMAP_STATS_VAR(pmap_ncopy_page_c); 220PMAP_STATS_VAR(pmap_ncopy_page_oc); 221PMAP_STATS_VAR(pmap_ncopy_page_nc); 222PMAP_STATS_VAR(pmap_ncopy_page_dc); 223PMAP_STATS_VAR(pmap_ncopy_page_doc); 224PMAP_STATS_VAR(pmap_ncopy_page_sc); 225PMAP_STATS_VAR(pmap_ncopy_page_soc); 226 227PMAP_STATS_VAR(pmap_nnew_thread); 228PMAP_STATS_VAR(pmap_nnew_thread_oc); 229 230/* 231 * Quick sort callout for comparing memory regions. 232 */ 233static int mr_cmp(const void *a, const void *b); 234static int om_cmp(const void *a, const void *b); 235static int 236mr_cmp(const void *a, const void *b) 237{ 238 const struct ofw_mem_region *mra; 239 const struct ofw_mem_region *mrb; 240 241 mra = a; 242 mrb = b; 243 if (mra->mr_start < mrb->mr_start) 244 return (-1); 245 else if (mra->mr_start > mrb->mr_start) 246 return (1); 247 else 248 return (0); 249} 250static int 251om_cmp(const void *a, const void *b) 252{ 253 const struct ofw_map *oma; 254 const struct ofw_map *omb; 255 256 oma = a; 257 omb = b; 258 if (oma->om_start < omb->om_start) 259 return (-1); 260 else if (oma->om_start > omb->om_start) 261 return (1); 262 else 263 return (0); 264} 265 266/* 267 * Bootstrap the system enough to run with virtual memory. 268 */ 269void 270pmap_bootstrap(vm_offset_t ekva) 271{ 272 struct pmap *pm; 273 struct tte *tp; 274 vm_offset_t off; 275 vm_offset_t va; 276 vm_paddr_t pa; 277 vm_size_t physsz; 278 vm_size_t virtsz; 279 ihandle_t pmem; 280 ihandle_t vmem; 281 int sz; 282 int i; 283 int j; 284 285 /* 286 * Find out what physical memory is available from the prom and 287 * initialize the phys_avail array. This must be done before 288 * pmap_bootstrap_alloc is called. 289 */ 290 if ((pmem = OF_finddevice("/memory")) == -1) 291 panic("pmap_bootstrap: finddevice /memory"); 292 if ((sz = OF_getproplen(pmem, "available")) == -1) 293 panic("pmap_bootstrap: getproplen /memory/available"); 294 if (sizeof(phys_avail) < sz) 295 panic("pmap_bootstrap: phys_avail too small"); 296 if (sizeof(mra) < sz) 297 panic("pmap_bootstrap: mra too small"); 298 bzero(mra, sz); 299 if (OF_getprop(pmem, "available", mra, sz) == -1) 300 panic("pmap_bootstrap: getprop /memory/available"); 301 sz /= sizeof(*mra); 302 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 303 qsort(mra, sz, sizeof (*mra), mr_cmp); 304 physsz = 0; 305 getenv_quad("hw.physmem", &physmem); 306 physmem = btoc(physmem); 307 for (i = 0, j = 0; i < sz; i++, j += 2) { 308 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 309 mra[i].mr_size); 310 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 311 if (btoc(physsz) < physmem) { 312 phys_avail[j] = mra[i].mr_start; 313 phys_avail[j + 1] = mra[i].mr_start + 314 (ctob(physmem) - physsz); 315 physsz = ctob(physmem); 316 } 317 break; 318 } 319 phys_avail[j] = mra[i].mr_start; 320 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 321 physsz += mra[i].mr_size; 322 } 323 physmem = btoc(physsz); 324 325 /* 326 * Calculate the size of kernel virtual memory, and the size and mask 327 * for the kernel tsb. 328 */ 329 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 330 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 331 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 332 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 333 334 /* 335 * Allocate the kernel tsb and lock it in the tlb. 336 */ 337 pa = pmap_bootstrap_alloc(tsb_kernel_size); 338 if (pa & PAGE_MASK_4M) 339 panic("pmap_bootstrap: tsb unaligned\n"); 340 tsb_kernel_phys = pa; 341 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 342 pmap_map_tsb(); 343 bzero(tsb_kernel, tsb_kernel_size); 344 345 /* 346 * Allocate and map the message buffer. 347 */ 348 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 349 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys); 350 351 /* 352 * Patch the virtual address and the tsb mask into the trap table. 353 */ 354 355#define SETHI(rd, imm22) \ 356 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 357 EIF_IMM((imm22) >> 10, 22)) 358#define OR_R_I_R(rd, imm13, rs1) \ 359 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 360 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 361 362#define PATCH(addr) do { \ 363 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 364 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 365 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 366 panic("pmap_boostrap: patched instructions have changed"); \ 367 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 368 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 369 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 370 flush(addr); \ 371 flush(addr + 1); \ 372 flush(addr + 2); \ 373} while (0) 374 375 PATCH(tl1_immu_miss_patch_1); 376 PATCH(tl1_immu_miss_patch_2); 377 PATCH(tl1_dmmu_miss_patch_1); 378 PATCH(tl1_dmmu_miss_patch_2); 379 PATCH(tl1_dmmu_prot_patch_1); 380 PATCH(tl1_dmmu_prot_patch_2); 381 382 /* 383 * Enter fake 8k pages for the 4MB kernel pages, so that 384 * pmap_kextract() will work for them. 385 */ 386 for (i = 0; i < kernel_tlb_slots; i++) { 387 pa = kernel_tlbs[i].te_pa; 388 va = kernel_tlbs[i].te_va; 389 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 390 tp = tsb_kvtotte(va + off); 391 tp->tte_vpn = TV_VPN(va + off, TS_8K); 392 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 393 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 394 } 395 } 396 397 /* 398 * Set the start and end of kva. The kernel is loaded at the first 399 * available 4 meg super page, so round up to the end of the page. 400 */ 401 virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 402 virtual_end = vm_max_kernel_address; 403 kernel_vm_end = vm_max_kernel_address; 404 405 /* 406 * Allocate kva space for temporary mappings. 407 */ 408 pmap_idle_map = virtual_avail; 409 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 410 pmap_temp_map_1 = virtual_avail; 411 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 412 pmap_temp_map_2 = virtual_avail; 413 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 414 415 /* 416 * Allocate a kernel stack with guard page for thread0 and map it into 417 * the kernel tsb. We must ensure that the virtual address is coloured 418 * properly, since we're allocating from phys_avail so the memory won't 419 * have an associated vm_page_t. 420 */ 421 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) * 422 PAGE_SIZE); 423 kstack0_phys = pa; 424 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 425 PAGE_SIZE; 426 kstack0 = virtual_avail; 427 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 428 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 429 ("pmap_bootstrap: kstack0 miscoloured")); 430 for (i = 0; i < KSTACK_PAGES; i++) { 431 pa = kstack0_phys + i * PAGE_SIZE; 432 va = kstack0 + i * PAGE_SIZE; 433 tp = tsb_kvtotte(va); 434 tp->tte_vpn = TV_VPN(va, TS_8K); 435 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 436 TD_CP | TD_CV | TD_P | TD_W; 437 } 438 439 /* 440 * Calculate the last available physical address. 441 */ 442 for (i = 0; phys_avail[i + 2] != 0; i += 2) 443 ; 444 Maxmem = sparc64_btop(phys_avail[i + 1]); 445 446 /* 447 * Add the prom mappings to the kernel tsb. 448 */ 449 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 450 panic("pmap_bootstrap: finddevice /virtual-memory"); 451 if ((sz = OF_getproplen(vmem, "translations")) == -1) 452 panic("pmap_bootstrap: getproplen translations"); 453 if (sizeof(translations) < sz) 454 panic("pmap_bootstrap: translations too small"); 455 bzero(translations, sz); 456 if (OF_getprop(vmem, "translations", translations, sz) == -1) 457 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 458 sz /= sizeof(*translations); 459 translations_size = sz; 460 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 461 qsort(translations, sz, sizeof (*translations), om_cmp); 462 for (i = 0; i < sz; i++) { 463 CTR3(KTR_PMAP, 464 "translation: start=%#lx size=%#lx tte=%#lx", 465 translations[i].om_start, translations[i].om_size, 466 translations[i].om_tte); 467 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 468 translations[i].om_start > VM_MAX_PROM_ADDRESS) 469 continue; 470 for (off = 0; off < translations[i].om_size; 471 off += PAGE_SIZE) { 472 va = translations[i].om_start + off; 473 tp = tsb_kvtotte(va); 474 tp->tte_vpn = TV_VPN(va, TS_8K); 475 tp->tte_data = 476 ((translations[i].om_tte & 477 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) + 478 off; 479 } 480 } 481 482 /* 483 * Get the available physical memory ranges from /memory/reg. These 484 * are only used for kernel dumps, but it may not be wise to do prom 485 * calls in that situation. 486 */ 487 if ((sz = OF_getproplen(pmem, "reg")) == -1) 488 panic("pmap_bootstrap: getproplen /memory/reg"); 489 if (sizeof(sparc64_memreg) < sz) 490 panic("pmap_bootstrap: sparc64_memreg too small"); 491 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 492 panic("pmap_bootstrap: getprop /memory/reg"); 493 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 494 495 /* 496 * Initialize the kernel pmap (which is statically allocated). 497 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 498 * but sparc64 start up is not ready to initialize mutexes yet. 499 * It is called in machdep.c. 500 */ 501 pm = kernel_pmap; 502 for (i = 0; i < MAXCPU; i++) 503 pm->pm_context[i] = TLB_CTX_KERNEL; 504 pm->pm_active = ~0; 505 506 /* XXX flush all non-locked tlb entries */ 507} 508 509void 510pmap_map_tsb(void) 511{ 512 vm_offset_t va; 513 vm_paddr_t pa; 514 u_long data; 515 u_long s; 516 int i; 517 518 s = intr_disable(); 519 520 /* 521 * Map the 4mb tsb pages. 522 */ 523 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 524 va = (vm_offset_t)tsb_kernel + i; 525 pa = tsb_kernel_phys + i; 526 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 527 TD_P | TD_W; 528 /* XXX - cheetah */ 529 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 530 TLB_TAR_CTX(TLB_CTX_KERNEL)); 531 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 532 } 533 534 /* 535 * Set the secondary context to be the kernel context (needed for 536 * fp block operations in the kernel and the cache code). 537 */ 538 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 539 membar(Sync); 540 541 intr_restore(s); 542} 543 544/* 545 * Allocate a physical page of memory directly from the phys_avail map. 546 * Can only be called from pmap_bootstrap before avail start and end are 547 * calculated. 548 */ 549static vm_paddr_t 550pmap_bootstrap_alloc(vm_size_t size) 551{ 552 vm_paddr_t pa; 553 int i; 554 555 size = round_page(size); 556 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 557 if (phys_avail[i + 1] - phys_avail[i] < size) 558 continue; 559 pa = phys_avail[i]; 560 phys_avail[i] += size; 561 return (pa); 562 } 563 panic("pmap_bootstrap_alloc"); 564} 565 566/* 567 * Initialize a vm_page's machine-dependent fields. 568 */ 569void 570pmap_page_init(vm_page_t m) 571{ 572 573 TAILQ_INIT(&m->md.tte_list); 574 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 575 m->md.flags = 0; 576 m->md.pmap = NULL; 577} 578 579/* 580 * Initialize the pmap module. 581 */ 582void 583pmap_init(void) 584{ 585 vm_offset_t addr; 586 vm_size_t size; 587 int result; 588 int i; 589 590 for (i = 0; i < translations_size; i++) { 591 addr = translations[i].om_start; 592 size = translations[i].om_size; 593 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 594 continue; 595 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE, 596 VM_PROT_ALL, VM_PROT_ALL, 0); 597 if (result != KERN_SUCCESS || addr != translations[i].om_start) 598 panic("pmap_init: vm_map_find"); 599 } 600} 601 602/* 603 * Extract the physical page address associated with the given 604 * map/virtual_address pair. 605 */ 606vm_paddr_t 607pmap_extract(pmap_t pm, vm_offset_t va) 608{ 609 struct tte *tp; 610 vm_paddr_t pa; 611 612 if (pm == kernel_pmap) 613 return (pmap_kextract(va)); 614 PMAP_LOCK(pm); 615 tp = tsb_tte_lookup(pm, va); 616 if (tp == NULL) 617 pa = 0; 618 else 619 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 620 PMAP_UNLOCK(pm); 621 return (pa); 622} 623 624/* 625 * Atomically extract and hold the physical page with the given 626 * pmap and virtual address pair if that mapping permits the given 627 * protection. 628 */ 629vm_page_t 630pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 631{ 632 struct tte *tp; 633 vm_page_t m; 634 635 m = NULL; 636 vm_page_lock_queues(); 637 if (pm == kernel_pmap) { 638 if (va >= VM_MIN_DIRECT_ADDRESS) { 639 tp = NULL; 640 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 641 vm_page_hold(m); 642 } else { 643 tp = tsb_kvtotte(va); 644 if ((tp->tte_data & TD_V) == 0) 645 tp = NULL; 646 } 647 } else { 648 PMAP_LOCK(pm); 649 tp = tsb_tte_lookup(pm, va); 650 } 651 if (tp != NULL && ((tp->tte_data & TD_SW) || 652 (prot & VM_PROT_WRITE) == 0)) { 653 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 654 vm_page_hold(m); 655 } 656 vm_page_unlock_queues(); 657 if (pm != kernel_pmap) 658 PMAP_UNLOCK(pm); 659 return (m); 660} 661 662/* 663 * Extract the physical page address associated with the given kernel virtual 664 * address. 665 */ 666vm_paddr_t 667pmap_kextract(vm_offset_t va) 668{ 669 struct tte *tp; 670 671 if (va >= VM_MIN_DIRECT_ADDRESS) 672 return (TLB_DIRECT_TO_PHYS(va)); 673 tp = tsb_kvtotte(va); 674 if ((tp->tte_data & TD_V) == 0) 675 return (0); 676 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 677} 678 679int 680pmap_cache_enter(vm_page_t m, vm_offset_t va) 681{ 682 struct tte *tp; 683 int color; 684 685 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 686 KASSERT((m->flags & PG_FICTITIOUS) == 0, 687 ("pmap_cache_enter: fake page")); 688 PMAP_STATS_INC(pmap_ncache_enter); 689 690 /* 691 * Find the color for this virtual address and note the added mapping. 692 */ 693 color = DCACHE_COLOR(va); 694 m->md.colors[color]++; 695 696 /* 697 * If all existing mappings have the same color, the mapping is 698 * cacheable. 699 */ 700 if (m->md.color == color) { 701 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 702 ("pmap_cache_enter: cacheable, mappings of other color")); 703 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 704 PMAP_STATS_INC(pmap_ncache_enter_c); 705 else 706 PMAP_STATS_INC(pmap_ncache_enter_oc); 707 return (1); 708 } 709 710 /* 711 * If there are no mappings of the other color, and the page still has 712 * the wrong color, this must be a new mapping. Change the color to 713 * match the new mapping, which is cacheable. We must flush the page 714 * from the cache now. 715 */ 716 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 717 KASSERT(m->md.colors[color] == 1, 718 ("pmap_cache_enter: changing color, not new mapping")); 719 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 720 m->md.color = color; 721 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 722 PMAP_STATS_INC(pmap_ncache_enter_cc); 723 else 724 PMAP_STATS_INC(pmap_ncache_enter_coc); 725 return (1); 726 } 727 728 /* 729 * If the mapping is already non-cacheable, just return. 730 */ 731 if (m->md.color == -1) { 732 PMAP_STATS_INC(pmap_ncache_enter_nc); 733 return (0); 734 } 735 736 PMAP_STATS_INC(pmap_ncache_enter_cnc); 737 738 /* 739 * Mark all mappings as uncacheable, flush any lines with the other 740 * color out of the dcache, and set the color to none (-1). 741 */ 742 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 743 atomic_clear_long(&tp->tte_data, TD_CV); 744 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 745 } 746 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 747 m->md.color = -1; 748 return (0); 749} 750 751void 752pmap_cache_remove(vm_page_t m, vm_offset_t va) 753{ 754 struct tte *tp; 755 int color; 756 757 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 758 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 759 m->md.colors[DCACHE_COLOR(va)]); 760 KASSERT((m->flags & PG_FICTITIOUS) == 0, 761 ("pmap_cache_remove: fake page")); 762 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 763 ("pmap_cache_remove: no mappings %d <= 0", 764 m->md.colors[DCACHE_COLOR(va)])); 765 PMAP_STATS_INC(pmap_ncache_remove); 766 767 /* 768 * Find the color for this virtual address and note the removal of 769 * the mapping. 770 */ 771 color = DCACHE_COLOR(va); 772 m->md.colors[color]--; 773 774 /* 775 * If the page is cacheable, just return and keep the same color, even 776 * if there are no longer any mappings. 777 */ 778 if (m->md.color != -1) { 779 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 780 PMAP_STATS_INC(pmap_ncache_remove_c); 781 else 782 PMAP_STATS_INC(pmap_ncache_remove_oc); 783 return; 784 } 785 786 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 787 ("pmap_cache_remove: uncacheable, no mappings of other color")); 788 789 /* 790 * If the page is not cacheable (color is -1), and the number of 791 * mappings for this color is not zero, just return. There are 792 * mappings of the other color still, so remain non-cacheable. 793 */ 794 if (m->md.colors[color] != 0) { 795 PMAP_STATS_INC(pmap_ncache_remove_nc); 796 return; 797 } 798 799 /* 800 * The number of mappings for this color is now zero. Recache the 801 * other colored mappings, and change the page color to the other 802 * color. There should be no lines in the data cache for this page, 803 * so flushing should not be needed. 804 */ 805 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 806 atomic_set_long(&tp->tte_data, TD_CV); 807 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 808 } 809 m->md.color = DCACHE_OTHER_COLOR(color); 810 811 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 812 PMAP_STATS_INC(pmap_ncache_remove_cc); 813 else 814 PMAP_STATS_INC(pmap_ncache_remove_coc); 815} 816 817/* 818 * Map a wired page into kernel virtual address space. 819 */ 820void 821pmap_kenter(vm_offset_t va, vm_page_t m) 822{ 823 vm_offset_t ova; 824 struct tte *tp; 825 vm_page_t om; 826 u_long data; 827 828 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 829 PMAP_STATS_INC(pmap_nkenter); 830 tp = tsb_kvtotte(va); 831 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 832 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 833 if (m->pc != DCACHE_COLOR(va)) { 834 CTR6(KTR_CT2, 835 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx", 836 va, VM_PAGE_TO_PHYS(m), m->object, 837 m->object ? m->object->pg_color : -1, 838 m->object ? m->object->type : -1, 839 m->pindex); 840 PMAP_STATS_INC(pmap_nkenter_oc); 841 } 842 if ((tp->tte_data & TD_V) != 0) { 843 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 844 ova = TTE_GET_VA(tp); 845 if (m == om && va == ova) { 846 PMAP_STATS_INC(pmap_nkenter_stupid); 847 return; 848 } 849 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 850 pmap_cache_remove(om, ova); 851 if (va != ova) 852 tlb_page_demap(kernel_pmap, ova); 853 } 854 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 855 TD_P | TD_W; 856 if (pmap_cache_enter(m, va) != 0) 857 data |= TD_CV; 858 tp->tte_vpn = TV_VPN(va, TS_8K); 859 tp->tte_data = data; 860 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 861} 862 863/* 864 * Map a wired page into kernel virtual address space. This additionally 865 * takes a flag argument wich is or'ed to the TTE data. This is used by 866 * bus_space_map(). 867 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 868 * to flush entries that might still be in the cache, if applicable. 869 */ 870void 871pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 872{ 873 struct tte *tp; 874 875 tp = tsb_kvtotte(va); 876 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 877 va, pa, tp, tp->tte_data); 878 tp->tte_vpn = TV_VPN(va, TS_8K); 879 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 880} 881 882/* 883 * Remove a wired page from kernel virtual address space. 884 */ 885void 886pmap_kremove(vm_offset_t va) 887{ 888 struct tte *tp; 889 vm_page_t m; 890 891 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 892 PMAP_STATS_INC(pmap_nkremove); 893 tp = tsb_kvtotte(va); 894 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 895 tp->tte_data); 896 if ((tp->tte_data & TD_V) == 0) 897 return; 898 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 899 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 900 pmap_cache_remove(m, va); 901 TTE_ZERO(tp); 902} 903 904/* 905 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 906 */ 907void 908pmap_kremove_flags(vm_offset_t va) 909{ 910 struct tte *tp; 911 912 tp = tsb_kvtotte(va); 913 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 914 tp->tte_data); 915 TTE_ZERO(tp); 916} 917 918/* 919 * Map a range of physical addresses into kernel virtual address space. 920 * 921 * The value passed in *virt is a suggested virtual address for the mapping. 922 * Architectures which can support a direct-mapped physical to virtual region 923 * can return the appropriate address within that region, leaving '*virt' 924 * unchanged. 925 */ 926vm_offset_t 927pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 928{ 929 930 return (TLB_PHYS_TO_DIRECT(start)); 931} 932 933/* 934 * Map a list of wired pages into kernel virtual address space. This is 935 * intended for temporary mappings which do not need page modification or 936 * references recorded. Existing mappings in the region are overwritten. 937 */ 938void 939pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 940{ 941 vm_offset_t va; 942 int locked; 943 944 PMAP_STATS_INC(pmap_nqenter); 945 va = sva; 946 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 947 vm_page_lock_queues(); 948 while (count-- > 0) { 949 pmap_kenter(va, *m); 950 va += PAGE_SIZE; 951 m++; 952 } 953 if (!locked) 954 vm_page_unlock_queues(); 955 tlb_range_demap(kernel_pmap, sva, va); 956} 957 958/* 959 * Remove page mappings from kernel virtual address space. Intended for 960 * temporary mappings entered by pmap_qenter. 961 */ 962void 963pmap_qremove(vm_offset_t sva, int count) 964{ 965 vm_offset_t va; 966 int locked; 967 968 PMAP_STATS_INC(pmap_nqremove); 969 va = sva; 970 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 971 vm_page_lock_queues(); 972 while (count-- > 0) { 973 pmap_kremove(va); 974 va += PAGE_SIZE; 975 } 976 if (!locked) 977 vm_page_unlock_queues(); 978 tlb_range_demap(kernel_pmap, sva, va); 979} 980 981/* 982 * Initialize the pmap associated with process 0. 983 */ 984void 985pmap_pinit0(pmap_t pm) 986{ 987 int i; 988 989 PMAP_LOCK_INIT(pm); 990 for (i = 0; i < MAXCPU; i++) 991 pm->pm_context[i] = 0; 992 pm->pm_active = 0; 993 pm->pm_tsb = NULL; 994 pm->pm_tsb_obj = NULL; 995 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 996} 997 998/* 999 * Initialize a preallocated and zeroed pmap structure, such as one in a 1000 * vmspace structure. 1001 */ 1002void 1003pmap_pinit(pmap_t pm) 1004{ 1005 vm_page_t ma[TSB_PAGES]; 1006 vm_page_t m; 1007 int i; 1008 1009 PMAP_LOCK_INIT(pm); 1010 1011 /* 1012 * Allocate kva space for the tsb. 1013 */ 1014 if (pm->pm_tsb == NULL) { 1015 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1016 TSB_BSIZE); 1017 } 1018 1019 /* 1020 * Allocate an object for it. 1021 */ 1022 if (pm->pm_tsb_obj == NULL) 1023 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1024 1025 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1026 for (i = 0; i < TSB_PAGES; i++) { 1027 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1028 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1029 m->valid = VM_PAGE_BITS_ALL; 1030 m->md.pmap = pm; 1031 ma[i] = m; 1032 } 1033 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1034 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1035 1036 for (i = 0; i < MAXCPU; i++) 1037 pm->pm_context[i] = -1; 1038 pm->pm_active = 0; 1039 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1040} 1041 1042/* 1043 * Release any resources held by the given physical map. 1044 * Called when a pmap initialized by pmap_pinit is being released. 1045 * Should only be called if the map contains no valid mappings. 1046 */ 1047void 1048pmap_release(pmap_t pm) 1049{ 1050 vm_object_t obj; 1051 vm_page_t m; 1052 struct pcpu *pc; 1053 1054 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1055 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1056 KASSERT(pmap_resident_count(pm) == 0, 1057 ("pmap_release: resident pages %ld != 0", 1058 pmap_resident_count(pm))); 1059 1060 /* 1061 * After the pmap was freed, it might be reallocated to a new process. 1062 * When switching, this might lead us to wrongly assume that we need 1063 * not switch contexts because old and new pmap pointer are equal. 1064 * Therefore, make sure that this pmap is not referenced by any PCPU 1065 * pointer any more. This could happen in two cases: 1066 * - A process that referenced the pmap is currently exiting on a CPU. 1067 * However, it is guaranteed to not switch in any more after setting 1068 * its state to PRS_ZOMBIE. 1069 * - A process that referenced this pmap ran on a CPU, but we switched 1070 * to a kernel thread, leaving the pmap pointer unchanged. 1071 */ 1072 mtx_lock_spin(&sched_lock); 1073 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 1074 if (pc->pc_pmap == pm) 1075 pc->pc_pmap = NULL; 1076 } 1077 mtx_unlock_spin(&sched_lock); 1078 1079 obj = pm->pm_tsb_obj; 1080 VM_OBJECT_LOCK(obj); 1081 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1082 while (!TAILQ_EMPTY(&obj->memq)) { 1083 m = TAILQ_FIRST(&obj->memq); 1084 vm_page_lock_queues(); 1085 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1086 continue; 1087 KASSERT(m->hold_count == 0, 1088 ("pmap_release: freeing held tsb page")); 1089 m->md.pmap = NULL; 1090 m->wire_count--; 1091 atomic_subtract_int(&cnt.v_wire_count, 1); 1092 vm_page_free_zero(m); 1093 vm_page_unlock_queues(); 1094 } 1095 VM_OBJECT_UNLOCK(obj); 1096 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1097 PMAP_LOCK_DESTROY(pm); 1098} 1099 1100/* 1101 * Grow the number of kernel page table entries. Unneeded. 1102 */ 1103void 1104pmap_growkernel(vm_offset_t addr) 1105{ 1106 1107 panic("pmap_growkernel: can't grow kernel"); 1108} 1109 1110int 1111pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1112 vm_offset_t va) 1113{ 1114 vm_page_t m; 1115 u_long data; 1116 1117 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1118 data = atomic_readandclear_long(&tp->tte_data); 1119 if ((data & TD_FAKE) == 0) { 1120 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1121 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1122 if ((data & TD_WIRED) != 0) 1123 pm->pm_stats.wired_count--; 1124 if ((data & TD_PV) != 0) { 1125 if ((data & TD_W) != 0) 1126 vm_page_dirty(m); 1127 if ((data & TD_REF) != 0) 1128 vm_page_flag_set(m, PG_REFERENCED); 1129 if (TAILQ_EMPTY(&m->md.tte_list)) 1130 vm_page_flag_clear(m, PG_WRITEABLE); 1131 pm->pm_stats.resident_count--; 1132 } 1133 pmap_cache_remove(m, va); 1134 } 1135 TTE_ZERO(tp); 1136 if (PMAP_REMOVE_DONE(pm)) 1137 return (0); 1138 return (1); 1139} 1140 1141/* 1142 * Remove the given range of addresses from the specified map. 1143 */ 1144void 1145pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1146{ 1147 struct tte *tp; 1148 vm_offset_t va; 1149 1150 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1151 pm->pm_context[PCPU_GET(cpuid)], start, end); 1152 if (PMAP_REMOVE_DONE(pm)) 1153 return; 1154 vm_page_lock_queues(); 1155 PMAP_LOCK(pm); 1156 if (end - start > PMAP_TSB_THRESH) { 1157 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1158 tlb_context_demap(pm); 1159 } else { 1160 for (va = start; va < end; va += PAGE_SIZE) { 1161 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1162 if (!pmap_remove_tte(pm, NULL, tp, va)) 1163 break; 1164 } 1165 } 1166 tlb_range_demap(pm, start, end - 1); 1167 } 1168 PMAP_UNLOCK(pm); 1169 vm_page_unlock_queues(); 1170} 1171 1172void 1173pmap_remove_all(vm_page_t m) 1174{ 1175 struct pmap *pm; 1176 struct tte *tpn; 1177 struct tte *tp; 1178 vm_offset_t va; 1179 1180 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1181 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1182 tpn = TAILQ_NEXT(tp, tte_link); 1183 if ((tp->tte_data & TD_PV) == 0) 1184 continue; 1185 pm = TTE_GET_PMAP(tp); 1186 va = TTE_GET_VA(tp); 1187 PMAP_LOCK(pm); 1188 if ((tp->tte_data & TD_WIRED) != 0) 1189 pm->pm_stats.wired_count--; 1190 if ((tp->tte_data & TD_REF) != 0) 1191 vm_page_flag_set(m, PG_REFERENCED); 1192 if ((tp->tte_data & TD_W) != 0) 1193 vm_page_dirty(m); 1194 tp->tte_data &= ~TD_V; 1195 tlb_page_demap(pm, va); 1196 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1197 pm->pm_stats.resident_count--; 1198 pmap_cache_remove(m, va); 1199 TTE_ZERO(tp); 1200 PMAP_UNLOCK(pm); 1201 } 1202 vm_page_flag_clear(m, PG_WRITEABLE); 1203} 1204 1205int 1206pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1207 vm_offset_t va) 1208{ 1209 u_long data; 1210 vm_page_t m; 1211 1212 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1213 if ((data & TD_PV) != 0) { 1214 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1215 if ((data & TD_REF) != 0) 1216 vm_page_flag_set(m, PG_REFERENCED); 1217 if ((data & TD_W) != 0) 1218 vm_page_dirty(m); 1219 } 1220 return (1); 1221} 1222 1223/* 1224 * Set the physical protection on the specified range of this map as requested. 1225 */ 1226void 1227pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1228{ 1229 vm_offset_t va; 1230 struct tte *tp; 1231 1232 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1233 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1234 1235 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1236 pmap_remove(pm, sva, eva); 1237 return; 1238 } 1239 1240 if (prot & VM_PROT_WRITE) 1241 return; 1242 1243 vm_page_lock_queues(); 1244 PMAP_LOCK(pm); 1245 if (eva - sva > PMAP_TSB_THRESH) { 1246 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1247 tlb_context_demap(pm); 1248 } else { 1249 for (va = sva; va < eva; va += PAGE_SIZE) { 1250 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1251 pmap_protect_tte(pm, NULL, tp, va); 1252 } 1253 tlb_range_demap(pm, sva, eva - 1); 1254 } 1255 PMAP_UNLOCK(pm); 1256 vm_page_unlock_queues(); 1257} 1258 1259/* 1260 * Map the given physical page at the specified virtual address in the 1261 * target pmap with the protection requested. If specified the page 1262 * will be wired down. 1263 */ 1264void 1265pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1266 boolean_t wired) 1267{ 1268 1269 vm_page_lock_queues(); 1270 PMAP_LOCK(pm); 1271 pmap_enter_locked(pm, va, m, prot, wired); 1272 vm_page_unlock_queues(); 1273 PMAP_UNLOCK(pm); 1274} 1275 1276/* 1277 * Map the given physical page at the specified virtual address in the 1278 * target pmap with the protection requested. If specified the page 1279 * will be wired down. 1280 * 1281 * The page queues and pmap must be locked. 1282 */ 1283static void 1284pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1285 boolean_t wired) 1286{ 1287 struct tte *tp; 1288 vm_paddr_t pa; 1289 u_long data; 1290 int i; 1291 1292 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1293 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1294 PMAP_STATS_INC(pmap_nenter); 1295 pa = VM_PAGE_TO_PHYS(m); 1296 1297 /* 1298 * If this is a fake page from the device_pager, but it covers actual 1299 * physical memory, convert to the real backing page. 1300 */ 1301 if ((m->flags & PG_FICTITIOUS) != 0) { 1302 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1303 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1304 m = PHYS_TO_VM_PAGE(pa); 1305 break; 1306 } 1307 } 1308 } 1309 1310 CTR6(KTR_PMAP, 1311 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1312 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1313 1314 /* 1315 * If there is an existing mapping, and the physical address has not 1316 * changed, must be protection or wiring change. 1317 */ 1318 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1319 CTR0(KTR_PMAP, "pmap_enter: update"); 1320 PMAP_STATS_INC(pmap_nenter_update); 1321 1322 /* 1323 * Wiring change, just update stats. 1324 */ 1325 if (wired) { 1326 if ((tp->tte_data & TD_WIRED) == 0) { 1327 tp->tte_data |= TD_WIRED; 1328 pm->pm_stats.wired_count++; 1329 } 1330 } else { 1331 if ((tp->tte_data & TD_WIRED) != 0) { 1332 tp->tte_data &= ~TD_WIRED; 1333 pm->pm_stats.wired_count--; 1334 } 1335 } 1336 1337 /* 1338 * Save the old bits and clear the ones we're interested in. 1339 */ 1340 data = tp->tte_data; 1341 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1342 1343 /* 1344 * If we're turning off write permissions, sense modify status. 1345 */ 1346 if ((prot & VM_PROT_WRITE) != 0) { 1347 tp->tte_data |= TD_SW; 1348 if (wired) { 1349 tp->tte_data |= TD_W; 1350 } 1351 vm_page_flag_set(m, PG_WRITEABLE); 1352 } else if ((data & TD_W) != 0) { 1353 vm_page_dirty(m); 1354 } 1355 1356 /* 1357 * If we're turning on execute permissions, flush the icache. 1358 */ 1359 if ((prot & VM_PROT_EXECUTE) != 0) { 1360 if ((data & TD_EXEC) == 0) { 1361 icache_page_inval(pa); 1362 } 1363 tp->tte_data |= TD_EXEC; 1364 } 1365 1366 /* 1367 * Delete the old mapping. 1368 */ 1369 tlb_page_demap(pm, TTE_GET_VA(tp)); 1370 } else { 1371 /* 1372 * If there is an existing mapping, but its for a different 1373 * phsyical address, delete the old mapping. 1374 */ 1375 if (tp != NULL) { 1376 CTR0(KTR_PMAP, "pmap_enter: replace"); 1377 PMAP_STATS_INC(pmap_nenter_replace); 1378 pmap_remove_tte(pm, NULL, tp, va); 1379 tlb_page_demap(pm, va); 1380 } else { 1381 CTR0(KTR_PMAP, "pmap_enter: new"); 1382 PMAP_STATS_INC(pmap_nenter_new); 1383 } 1384 1385 /* 1386 * Now set up the data and install the new mapping. 1387 */ 1388 data = TD_V | TD_8K | TD_PA(pa); 1389 if (pm == kernel_pmap) 1390 data |= TD_P; 1391 if ((prot & VM_PROT_WRITE) != 0) { 1392 data |= TD_SW; 1393 vm_page_flag_set(m, PG_WRITEABLE); 1394 } 1395 if (prot & VM_PROT_EXECUTE) { 1396 data |= TD_EXEC; 1397 icache_page_inval(pa); 1398 } 1399 1400 /* 1401 * If its wired update stats. We also don't need reference or 1402 * modify tracking for wired mappings, so set the bits now. 1403 */ 1404 if (wired) { 1405 pm->pm_stats.wired_count++; 1406 data |= TD_REF | TD_WIRED; 1407 if ((prot & VM_PROT_WRITE) != 0) 1408 data |= TD_W; 1409 } 1410 1411 tsb_tte_enter(pm, m, va, TS_8K, data); 1412 } 1413} 1414 1415/* 1416 * Maps a sequence of resident pages belonging to the same object. 1417 * The sequence begins with the given page m_start. This page is 1418 * mapped at the given virtual address start. Each subsequent page is 1419 * mapped at a virtual address that is offset from start by the same 1420 * amount as the page is offset from m_start within the object. The 1421 * last page in the sequence is the page with the largest offset from 1422 * m_start that can be mapped at a virtual address less than the given 1423 * virtual address end. Not every virtual page between start and end 1424 * is mapped; only those for which a resident page exists with the 1425 * corresponding offset from m_start are mapped. 1426 */ 1427void 1428pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1429 vm_page_t m_start, vm_prot_t prot) 1430{ 1431 vm_page_t m; 1432 vm_pindex_t diff, psize; 1433 1434 psize = atop(end - start); 1435 m = m_start; 1436 PMAP_LOCK(pm); 1437 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1438 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1439 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1440 m = TAILQ_NEXT(m, listq); 1441 } 1442 PMAP_UNLOCK(pm); 1443} 1444 1445void 1446pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1447{ 1448 1449 PMAP_LOCK(pm); 1450 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1451 FALSE); 1452 PMAP_UNLOCK(pm); 1453} 1454 1455void 1456pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1457 vm_pindex_t pindex, vm_size_t size) 1458{ 1459 1460 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1461 KASSERT(object->type == OBJT_DEVICE, 1462 ("pmap_object_init_pt: non-device object")); 1463} 1464 1465/* 1466 * Change the wiring attribute for a map/virtual-address pair. 1467 * The mapping must already exist in the pmap. 1468 */ 1469void 1470pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1471{ 1472 struct tte *tp; 1473 u_long data; 1474 1475 PMAP_LOCK(pm); 1476 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1477 if (wired) { 1478 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1479 if ((data & TD_WIRED) == 0) 1480 pm->pm_stats.wired_count++; 1481 } else { 1482 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1483 if ((data & TD_WIRED) != 0) 1484 pm->pm_stats.wired_count--; 1485 } 1486 } 1487 PMAP_UNLOCK(pm); 1488} 1489 1490static int 1491pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1492{ 1493 vm_page_t m; 1494 u_long data; 1495 1496 if ((tp->tte_data & TD_FAKE) != 0) 1497 return (1); 1498 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1499 data = tp->tte_data & 1500 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1501 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1502 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1503 } 1504 return (1); 1505} 1506 1507void 1508pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1509 vm_size_t len, vm_offset_t src_addr) 1510{ 1511 struct tte *tp; 1512 vm_offset_t va; 1513 1514 if (dst_addr != src_addr) 1515 return; 1516 vm_page_lock_queues(); 1517 if (dst_pmap < src_pmap) { 1518 PMAP_LOCK(dst_pmap); 1519 PMAP_LOCK(src_pmap); 1520 } else { 1521 PMAP_LOCK(src_pmap); 1522 PMAP_LOCK(dst_pmap); 1523 } 1524 if (len > PMAP_TSB_THRESH) { 1525 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1526 pmap_copy_tte); 1527 tlb_context_demap(dst_pmap); 1528 } else { 1529 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1530 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1531 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1532 } 1533 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1534 } 1535 vm_page_unlock_queues(); 1536 PMAP_UNLOCK(src_pmap); 1537 PMAP_UNLOCK(dst_pmap); 1538} 1539 1540void 1541pmap_zero_page(vm_page_t m) 1542{ 1543 struct tte *tp; 1544 vm_offset_t va; 1545 vm_paddr_t pa; 1546 1547 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1548 ("pmap_zero_page: fake page")); 1549 PMAP_STATS_INC(pmap_nzero_page); 1550 pa = VM_PAGE_TO_PHYS(m); 1551 if (m->md.color == -1) { 1552 PMAP_STATS_INC(pmap_nzero_page_nc); 1553 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1554 } else if (m->md.color == DCACHE_COLOR(pa)) { 1555 PMAP_STATS_INC(pmap_nzero_page_c); 1556 va = TLB_PHYS_TO_DIRECT(pa); 1557 cpu_block_zero((void *)va, PAGE_SIZE); 1558 } else { 1559 PMAP_STATS_INC(pmap_nzero_page_oc); 1560 PMAP_LOCK(kernel_pmap); 1561 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1562 tp = tsb_kvtotte(va); 1563 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1564 tp->tte_vpn = TV_VPN(va, TS_8K); 1565 cpu_block_zero((void *)va, PAGE_SIZE); 1566 tlb_page_demap(kernel_pmap, va); 1567 PMAP_UNLOCK(kernel_pmap); 1568 } 1569} 1570 1571void 1572pmap_zero_page_area(vm_page_t m, int off, int size) 1573{ 1574 struct tte *tp; 1575 vm_offset_t va; 1576 vm_paddr_t pa; 1577 1578 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1579 ("pmap_zero_page_area: fake page")); 1580 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1581 PMAP_STATS_INC(pmap_nzero_page_area); 1582 pa = VM_PAGE_TO_PHYS(m); 1583 if (m->md.color == -1) { 1584 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1585 aszero(ASI_PHYS_USE_EC, pa + off, size); 1586 } else if (m->md.color == DCACHE_COLOR(pa)) { 1587 PMAP_STATS_INC(pmap_nzero_page_area_c); 1588 va = TLB_PHYS_TO_DIRECT(pa); 1589 bzero((void *)(va + off), size); 1590 } else { 1591 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1592 PMAP_LOCK(kernel_pmap); 1593 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1594 tp = tsb_kvtotte(va); 1595 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1596 tp->tte_vpn = TV_VPN(va, TS_8K); 1597 bzero((void *)(va + off), size); 1598 tlb_page_demap(kernel_pmap, va); 1599 PMAP_UNLOCK(kernel_pmap); 1600 } 1601} 1602 1603void 1604pmap_zero_page_idle(vm_page_t m) 1605{ 1606 struct tte *tp; 1607 vm_offset_t va; 1608 vm_paddr_t pa; 1609 1610 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1611 ("pmap_zero_page_idle: fake page")); 1612 PMAP_STATS_INC(pmap_nzero_page_idle); 1613 pa = VM_PAGE_TO_PHYS(m); 1614 if (m->md.color == -1) { 1615 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1616 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1617 } else if (m->md.color == DCACHE_COLOR(pa)) { 1618 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1619 va = TLB_PHYS_TO_DIRECT(pa); 1620 cpu_block_zero((void *)va, PAGE_SIZE); 1621 } else { 1622 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1623 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1624 tp = tsb_kvtotte(va); 1625 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1626 tp->tte_vpn = TV_VPN(va, TS_8K); 1627 cpu_block_zero((void *)va, PAGE_SIZE); 1628 tlb_page_demap(kernel_pmap, va); 1629 } 1630} 1631 1632void 1633pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1634{ 1635 vm_offset_t vdst; 1636 vm_offset_t vsrc; 1637 vm_paddr_t pdst; 1638 vm_paddr_t psrc; 1639 struct tte *tp; 1640 1641 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1642 ("pmap_copy_page: fake dst page")); 1643 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1644 ("pmap_copy_page: fake src page")); 1645 PMAP_STATS_INC(pmap_ncopy_page); 1646 pdst = VM_PAGE_TO_PHYS(mdst); 1647 psrc = VM_PAGE_TO_PHYS(msrc); 1648 if (msrc->md.color == -1 && mdst->md.color == -1) { 1649 PMAP_STATS_INC(pmap_ncopy_page_nc); 1650 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1651 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1652 mdst->md.color == DCACHE_COLOR(pdst)) { 1653 PMAP_STATS_INC(pmap_ncopy_page_c); 1654 vdst = TLB_PHYS_TO_DIRECT(pdst); 1655 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1656 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1657 } else if (msrc->md.color == -1) { 1658 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1659 PMAP_STATS_INC(pmap_ncopy_page_dc); 1660 vdst = TLB_PHYS_TO_DIRECT(pdst); 1661 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1662 PAGE_SIZE); 1663 } else { 1664 PMAP_STATS_INC(pmap_ncopy_page_doc); 1665 PMAP_LOCK(kernel_pmap); 1666 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1667 tp = tsb_kvtotte(vdst); 1668 tp->tte_data = 1669 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1670 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1671 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1672 PAGE_SIZE); 1673 tlb_page_demap(kernel_pmap, vdst); 1674 PMAP_UNLOCK(kernel_pmap); 1675 } 1676 } else if (mdst->md.color == -1) { 1677 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1678 PMAP_STATS_INC(pmap_ncopy_page_sc); 1679 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1680 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1681 PAGE_SIZE); 1682 } else { 1683 PMAP_STATS_INC(pmap_ncopy_page_soc); 1684 PMAP_LOCK(kernel_pmap); 1685 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1686 tp = tsb_kvtotte(vsrc); 1687 tp->tte_data = 1688 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1689 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1690 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1691 PAGE_SIZE); 1692 tlb_page_demap(kernel_pmap, vsrc); 1693 PMAP_UNLOCK(kernel_pmap); 1694 } 1695 } else { 1696 PMAP_STATS_INC(pmap_ncopy_page_oc); 1697 PMAP_LOCK(kernel_pmap); 1698 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1699 tp = tsb_kvtotte(vdst); 1700 tp->tte_data = 1701 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1702 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1703 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1704 tp = tsb_kvtotte(vsrc); 1705 tp->tte_data = 1706 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1707 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1708 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1709 tlb_page_demap(kernel_pmap, vdst); 1710 tlb_page_demap(kernel_pmap, vsrc); 1711 PMAP_UNLOCK(kernel_pmap); 1712 } 1713} 1714 1715/* 1716 * Returns true if the pmap's pv is one of the first 1717 * 16 pvs linked to from this page. This count may 1718 * be changed upwards or downwards in the future; it 1719 * is only necessary that true be returned for a small 1720 * subset of pmaps for proper page aging. 1721 */ 1722boolean_t 1723pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1724{ 1725 struct tte *tp; 1726 int loops; 1727 1728 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1729 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1730 return (FALSE); 1731 loops = 0; 1732 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1733 if ((tp->tte_data & TD_PV) == 0) 1734 continue; 1735 if (TTE_GET_PMAP(tp) == pm) 1736 return (TRUE); 1737 if (++loops >= 16) 1738 break; 1739 } 1740 return (FALSE); 1741} 1742 1743/* 1744 * Remove all pages from specified address space, this aids process exit 1745 * speeds. This is much faster than pmap_remove n the case of running down 1746 * an entire address space. Only works for the current pmap. 1747 */ 1748void 1749pmap_remove_pages(pmap_t pm) 1750{ 1751} 1752 1753/* 1754 * Returns TRUE if the given page has a managed mapping. 1755 */ 1756boolean_t 1757pmap_page_is_mapped(vm_page_t m) 1758{ 1759 struct tte *tp; 1760 1761 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1762 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1763 return (FALSE); 1764 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1765 if ((tp->tte_data & TD_PV) != 0) 1766 return (TRUE); 1767 } 1768 return (FALSE); 1769} 1770 1771/* 1772 * pmap_ts_referenced: 1773 * 1774 * Return a count of reference bits for a page, clearing those bits. 1775 * It is not necessary for every reference bit to be cleared, but it 1776 * is necessary that 0 only be returned when there are truly no 1777 * reference bits set. 1778 * 1779 * XXX: The exact number of bits to check and clear is a matter that 1780 * should be tested and standardized at some point in the future for 1781 * optimal aging of shared pages. 1782 */ 1783 1784int 1785pmap_ts_referenced(vm_page_t m) 1786{ 1787 struct tte *tpf; 1788 struct tte *tpn; 1789 struct tte *tp; 1790 u_long data; 1791 int count; 1792 1793 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1794 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1795 return (0); 1796 count = 0; 1797 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1798 tpf = tp; 1799 do { 1800 tpn = TAILQ_NEXT(tp, tte_link); 1801 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1802 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1803 if ((tp->tte_data & TD_PV) == 0) 1804 continue; 1805 data = atomic_clear_long(&tp->tte_data, TD_REF); 1806 if ((data & TD_REF) != 0 && ++count > 4) 1807 break; 1808 } while ((tp = tpn) != NULL && tp != tpf); 1809 } 1810 return (count); 1811} 1812 1813boolean_t 1814pmap_is_modified(vm_page_t m) 1815{ 1816 struct tte *tp; 1817 1818 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1819 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1820 return (FALSE); 1821 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1822 if ((tp->tte_data & TD_PV) == 0) 1823 continue; 1824 if ((tp->tte_data & TD_W) != 0) 1825 return (TRUE); 1826 } 1827 return (FALSE); 1828} 1829 1830/* 1831 * pmap_is_prefaultable: 1832 * 1833 * Return whether or not the specified virtual address is elgible 1834 * for prefault. 1835 */ 1836boolean_t 1837pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1838{ 1839 1840 return (FALSE); 1841} 1842 1843void 1844pmap_clear_modify(vm_page_t m) 1845{ 1846 struct tte *tp; 1847 u_long data; 1848 1849 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1850 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1851 return; 1852 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1853 if ((tp->tte_data & TD_PV) == 0) 1854 continue; 1855 data = atomic_clear_long(&tp->tte_data, TD_W); 1856 if ((data & TD_W) != 0) 1857 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1858 } 1859} 1860 1861void 1862pmap_clear_reference(vm_page_t m) 1863{ 1864 struct tte *tp; 1865 u_long data; 1866 1867 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1868 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1869 return; 1870 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1871 if ((tp->tte_data & TD_PV) == 0) 1872 continue; 1873 data = atomic_clear_long(&tp->tte_data, TD_REF); 1874 if ((data & TD_REF) != 0) 1875 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1876 } 1877} 1878 1879void 1880pmap_remove_write(vm_page_t m) 1881{ 1882 struct tte *tp; 1883 u_long data; 1884 1885 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1886 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1887 (m->flags & PG_WRITEABLE) == 0) 1888 return; 1889 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1890 if ((tp->tte_data & TD_PV) == 0) 1891 continue; 1892 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1893 if ((data & TD_W) != 0) { 1894 vm_page_dirty(m); 1895 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1896 } 1897 } 1898 vm_page_flag_clear(m, PG_WRITEABLE); 1899} 1900 1901int 1902pmap_mincore(pmap_t pm, vm_offset_t addr) 1903{ 1904 /* TODO; */ 1905 return (0); 1906} 1907 1908/* 1909 * Activate a user pmap. The pmap must be activated before its address space 1910 * can be accessed in any way. 1911 */ 1912void 1913pmap_activate(struct thread *td) 1914{ 1915 struct vmspace *vm; 1916 struct pmap *pm; 1917 int context; 1918 1919 vm = td->td_proc->p_vmspace; 1920 pm = vmspace_pmap(vm); 1921 1922 mtx_lock_spin(&sched_lock); 1923 1924 context = PCPU_GET(tlb_ctx); 1925 if (context == PCPU_GET(tlb_ctx_max)) { 1926 tlb_flush_user(); 1927 context = PCPU_GET(tlb_ctx_min); 1928 } 1929 PCPU_SET(tlb_ctx, context + 1); 1930 1931 pm->pm_context[PCPU_GET(cpuid)] = context; 1932 pm->pm_active |= PCPU_GET(cpumask); 1933 PCPU_SET(pmap, pm); 1934 1935 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 1936 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 1937 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 1938 membar(Sync); 1939 1940 mtx_unlock_spin(&sched_lock); 1941} 1942 1943vm_offset_t 1944pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 1945{ 1946 1947 return (va); 1948} 1949