pmap.c revision 195840
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 */ 43 44#include <sys/cdefs.h> 45__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/pmap.c 195840 2009-07-24 13:50:29Z jhb $"); 46 47/* 48 * Manages physical address maps. 49 * 50 * In addition to hardware address maps, this module is called upon to 51 * provide software-use-only maps which may or may not be stored in the 52 * same form as hardware maps. These pseudo-maps are used to store 53 * intermediate results from copy operations to and from address spaces. 54 * 55 * Since the information managed by this module is also stored by the 56 * logical address mapping module, this module may throw away valid virtual 57 * to physical mappings at almost any time. However, invalidations of 58 * mappings must be done as requested. 59 * 60 * In order to cope with hardware architectures which make virtual to 61 * physical map invalidates expensive, this module may delay invalidate 62 * reduced protection operations until such time as they are actually 63 * necessary. This module is given full information as to which processors 64 * are currently using which maps, and to when physical maps must be made 65 * correct. 66 */ 67 68#include "opt_kstack_pages.h" 69#include "opt_msgbuf.h" 70#include "opt_pmap.h" 71 72#include <sys/param.h> 73#include <sys/kernel.h> 74#include <sys/ktr.h> 75#include <sys/lock.h> 76#include <sys/msgbuf.h> 77#include <sys/mutex.h> 78#include <sys/proc.h> 79#include <sys/smp.h> 80#include <sys/sysctl.h> 81#include <sys/systm.h> 82#include <sys/vmmeter.h> 83 84#include <dev/ofw/openfirm.h> 85 86#include <vm/vm.h> 87#include <vm/vm_param.h> 88#include <vm/vm_kern.h> 89#include <vm/vm_page.h> 90#include <vm/vm_map.h> 91#include <vm/vm_object.h> 92#include <vm/vm_extern.h> 93#include <vm/vm_pageout.h> 94#include <vm/vm_pager.h> 95 96#include <machine/cache.h> 97#include <machine/frame.h> 98#include <machine/instr.h> 99#include <machine/md_var.h> 100#include <machine/metadata.h> 101#include <machine/ofw_mem.h> 102#include <machine/smp.h> 103#include <machine/tlb.h> 104#include <machine/tte.h> 105#include <machine/tsb.h> 106#include <machine/ver.h> 107 108#define PMAP_DEBUG 109 110#ifndef PMAP_SHPGPERPROC 111#define PMAP_SHPGPERPROC 200 112#endif 113 114/* XXX */ 115#include "opt_sched.h" 116#ifndef SCHED_4BSD 117#error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." 118#endif 119extern struct mtx sched_lock; 120 121/* 122 * Virtual address of message buffer 123 */ 124struct msgbuf *msgbufp; 125 126/* 127 * Map of physical memory reagions 128 */ 129vm_paddr_t phys_avail[128]; 130static struct ofw_mem_region mra[128]; 131struct ofw_mem_region sparc64_memreg[128]; 132int sparc64_nmemreg; 133static struct ofw_map translations[128]; 134static int translations_size; 135 136static vm_offset_t pmap_idle_map; 137static vm_offset_t pmap_temp_map_1; 138static vm_offset_t pmap_temp_map_2; 139 140/* 141 * First and last available kernel virtual addresses 142 */ 143vm_offset_t virtual_avail; 144vm_offset_t virtual_end; 145vm_offset_t kernel_vm_end; 146 147vm_offset_t vm_max_kernel_address; 148 149/* 150 * Kernel pmap 151 */ 152struct pmap kernel_pmap_store; 153 154/* 155 * Allocate physical memory for use in pmap_bootstrap. 156 */ 157static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 158 159/* 160 * Map the given physical page at the specified virtual address in the 161 * target pmap with the protection requested. If specified the page 162 * will be wired down. 163 * 164 * The page queues and pmap must be locked. 165 */ 166static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 167 vm_prot_t prot, boolean_t wired); 168 169extern int tl1_immu_miss_patch_1[]; 170extern int tl1_immu_miss_patch_2[]; 171extern int tl1_dmmu_miss_patch_1[]; 172extern int tl1_dmmu_miss_patch_2[]; 173extern int tl1_dmmu_prot_patch_1[]; 174extern int tl1_dmmu_prot_patch_2[]; 175 176/* 177 * If user pmap is processed with pmap_remove and with pmap_remove and the 178 * resident count drops to 0, there are no more pages to remove, so we 179 * need not continue. 180 */ 181#define PMAP_REMOVE_DONE(pm) \ 182 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 183 184/* 185 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 186 * and pmap_protect() instead of trying each virtual address. 187 */ 188#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 189 190SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 191 192PMAP_STATS_VAR(pmap_nenter); 193PMAP_STATS_VAR(pmap_nenter_update); 194PMAP_STATS_VAR(pmap_nenter_replace); 195PMAP_STATS_VAR(pmap_nenter_new); 196PMAP_STATS_VAR(pmap_nkenter); 197PMAP_STATS_VAR(pmap_nkenter_oc); 198PMAP_STATS_VAR(pmap_nkenter_stupid); 199PMAP_STATS_VAR(pmap_nkremove); 200PMAP_STATS_VAR(pmap_nqenter); 201PMAP_STATS_VAR(pmap_nqremove); 202PMAP_STATS_VAR(pmap_ncache_enter); 203PMAP_STATS_VAR(pmap_ncache_enter_c); 204PMAP_STATS_VAR(pmap_ncache_enter_oc); 205PMAP_STATS_VAR(pmap_ncache_enter_cc); 206PMAP_STATS_VAR(pmap_ncache_enter_coc); 207PMAP_STATS_VAR(pmap_ncache_enter_nc); 208PMAP_STATS_VAR(pmap_ncache_enter_cnc); 209PMAP_STATS_VAR(pmap_ncache_remove); 210PMAP_STATS_VAR(pmap_ncache_remove_c); 211PMAP_STATS_VAR(pmap_ncache_remove_oc); 212PMAP_STATS_VAR(pmap_ncache_remove_cc); 213PMAP_STATS_VAR(pmap_ncache_remove_coc); 214PMAP_STATS_VAR(pmap_ncache_remove_nc); 215PMAP_STATS_VAR(pmap_nzero_page); 216PMAP_STATS_VAR(pmap_nzero_page_c); 217PMAP_STATS_VAR(pmap_nzero_page_oc); 218PMAP_STATS_VAR(pmap_nzero_page_nc); 219PMAP_STATS_VAR(pmap_nzero_page_area); 220PMAP_STATS_VAR(pmap_nzero_page_area_c); 221PMAP_STATS_VAR(pmap_nzero_page_area_oc); 222PMAP_STATS_VAR(pmap_nzero_page_area_nc); 223PMAP_STATS_VAR(pmap_nzero_page_idle); 224PMAP_STATS_VAR(pmap_nzero_page_idle_c); 225PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 226PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 227PMAP_STATS_VAR(pmap_ncopy_page); 228PMAP_STATS_VAR(pmap_ncopy_page_c); 229PMAP_STATS_VAR(pmap_ncopy_page_oc); 230PMAP_STATS_VAR(pmap_ncopy_page_nc); 231PMAP_STATS_VAR(pmap_ncopy_page_dc); 232PMAP_STATS_VAR(pmap_ncopy_page_doc); 233PMAP_STATS_VAR(pmap_ncopy_page_sc); 234PMAP_STATS_VAR(pmap_ncopy_page_soc); 235 236PMAP_STATS_VAR(pmap_nnew_thread); 237PMAP_STATS_VAR(pmap_nnew_thread_oc); 238 239/* 240 * Quick sort callout for comparing memory regions 241 */ 242static int mr_cmp(const void *a, const void *b); 243static int om_cmp(const void *a, const void *b); 244static int 245mr_cmp(const void *a, const void *b) 246{ 247 const struct ofw_mem_region *mra; 248 const struct ofw_mem_region *mrb; 249 250 mra = a; 251 mrb = b; 252 if (mra->mr_start < mrb->mr_start) 253 return (-1); 254 else if (mra->mr_start > mrb->mr_start) 255 return (1); 256 else 257 return (0); 258} 259static int 260om_cmp(const void *a, const void *b) 261{ 262 const struct ofw_map *oma; 263 const struct ofw_map *omb; 264 265 oma = a; 266 omb = b; 267 if (oma->om_start < omb->om_start) 268 return (-1); 269 else if (oma->om_start > omb->om_start) 270 return (1); 271 else 272 return (0); 273} 274 275/* 276 * Bootstrap the system enough to run with virtual memory. 277 */ 278void 279pmap_bootstrap(void) 280{ 281 struct pmap *pm; 282 struct tte *tp; 283 vm_offset_t off; 284 vm_offset_t va; 285 vm_paddr_t pa; 286 vm_size_t physsz; 287 vm_size_t virtsz; 288 phandle_t pmem; 289 phandle_t vmem; 290 int sz; 291 int i; 292 int j; 293 294 /* 295 * Find out what physical memory is available from the PROM and 296 * initialize the phys_avail array. This must be done before 297 * pmap_bootstrap_alloc is called. 298 */ 299 if ((pmem = OF_finddevice("/memory")) == -1) 300 panic("pmap_bootstrap: finddevice /memory"); 301 if ((sz = OF_getproplen(pmem, "available")) == -1) 302 panic("pmap_bootstrap: getproplen /memory/available"); 303 if (sizeof(phys_avail) < sz) 304 panic("pmap_bootstrap: phys_avail too small"); 305 if (sizeof(mra) < sz) 306 panic("pmap_bootstrap: mra too small"); 307 bzero(mra, sz); 308 if (OF_getprop(pmem, "available", mra, sz) == -1) 309 panic("pmap_bootstrap: getprop /memory/available"); 310 sz /= sizeof(*mra); 311 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 312 qsort(mra, sz, sizeof (*mra), mr_cmp); 313 physsz = 0; 314 getenv_quad("hw.physmem", &physmem); 315 physmem = btoc(physmem); 316 for (i = 0, j = 0; i < sz; i++, j += 2) { 317 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 318 mra[i].mr_size); 319 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 320 if (btoc(physsz) < physmem) { 321 phys_avail[j] = mra[i].mr_start; 322 phys_avail[j + 1] = mra[i].mr_start + 323 (ctob(physmem) - physsz); 324 physsz = ctob(physmem); 325 } 326 break; 327 } 328 phys_avail[j] = mra[i].mr_start; 329 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 330 physsz += mra[i].mr_size; 331 } 332 physmem = btoc(physsz); 333 334 /* 335 * Calculate the size of kernel virtual memory, and the size and mask 336 * for the kernel TSB based on the phsyical memory size but limited 337 * by letting the kernel TSB take up no more than half of the dTLB 338 * slots available for locked entries. 339 */ 340 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 341 virtsz = MIN(virtsz, 342 (dtlb_slots / 2 * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); 343 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 344 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 345 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 346 347 if (kernel_tlb_slots + PCPU_PAGES + tsb_kernel_size / PAGE_SIZE_4M + 348 1 /* PROM page */ + 1 /* spare */ > dtlb_slots) 349 panic("pmap_bootstrap: insufficient dTLB entries"); 350 if (kernel_tlb_slots + 1 /* PROM page */ + 1 /* spare */ > itlb_slots) 351 panic("pmap_bootstrap: insufficient iTLB entries"); 352 353 /* 354 * Allocate the kernel TSB and lock it in the TLB. 355 */ 356 pa = pmap_bootstrap_alloc(tsb_kernel_size); 357 if (pa & PAGE_MASK_4M) 358 panic("pmap_bootstrap: tsb unaligned\n"); 359 tsb_kernel_phys = pa; 360 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 361 pmap_map_tsb(); 362 bzero(tsb_kernel, tsb_kernel_size); 363 364 /* 365 * Allocate and map the dynamic per-CPU area for the BSP. 366 */ 367 pa = pmap_bootstrap_alloc(DPCPU_SIZE); 368 dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); 369 370 /* 371 * Allocate and map the message buffer. 372 */ 373 pa = pmap_bootstrap_alloc(MSGBUF_SIZE); 374 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); 375 376 /* 377 * Patch the virtual address and the tsb mask into the trap table. 378 */ 379 380#define SETHI(rd, imm22) \ 381 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 382 EIF_IMM((imm22) >> 10, 22)) 383#define OR_R_I_R(rd, imm13, rs1) \ 384 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 385 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 386 387#define PATCH(addr) do { \ 388 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 389 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 390 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 391 panic("pmap_boostrap: patched instructions have changed"); \ 392 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 393 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 394 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 395 flush(addr); \ 396 flush(addr + 1); \ 397 flush(addr + 2); \ 398} while (0) 399 400 PATCH(tl1_immu_miss_patch_1); 401 PATCH(tl1_immu_miss_patch_2); 402 PATCH(tl1_dmmu_miss_patch_1); 403 PATCH(tl1_dmmu_miss_patch_2); 404 PATCH(tl1_dmmu_prot_patch_1); 405 PATCH(tl1_dmmu_prot_patch_2); 406 407 /* 408 * Enter fake 8k pages for the 4MB kernel pages, so that 409 * pmap_kextract() will work for them. 410 */ 411 for (i = 0; i < kernel_tlb_slots; i++) { 412 pa = kernel_tlbs[i].te_pa; 413 va = kernel_tlbs[i].te_va; 414 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 415 tp = tsb_kvtotte(va + off); 416 tp->tte_vpn = TV_VPN(va + off, TS_8K); 417 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 418 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 419 } 420 } 421 422 /* 423 * Set the start and end of KVA. The kernel is loaded starting 424 * at the first available 4MB super page, so we advance to the 425 * end of the last one used for it. 426 */ 427 virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; 428 virtual_end = vm_max_kernel_address; 429 kernel_vm_end = vm_max_kernel_address; 430 431 /* 432 * Allocate kva space for temporary mappings. 433 */ 434 pmap_idle_map = virtual_avail; 435 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 436 pmap_temp_map_1 = virtual_avail; 437 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 438 pmap_temp_map_2 = virtual_avail; 439 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 440 441 /* 442 * Allocate a kernel stack with guard page for thread0 and map it 443 * into the kernel TSB. We must ensure that the virtual address is 444 * coloured properly, since we're allocating from phys_avail so the 445 * memory won't have an associated vm_page_t. 446 */ 447 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); 448 kstack0_phys = pa; 449 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 450 PAGE_SIZE; 451 kstack0 = virtual_avail; 452 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 453 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 454 ("pmap_bootstrap: kstack0 miscoloured")); 455 for (i = 0; i < KSTACK_PAGES; i++) { 456 pa = kstack0_phys + i * PAGE_SIZE; 457 va = kstack0 + i * PAGE_SIZE; 458 tp = tsb_kvtotte(va); 459 tp->tte_vpn = TV_VPN(va, TS_8K); 460 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 461 TD_CP | TD_CV | TD_P | TD_W; 462 } 463 464 /* 465 * Calculate the last available physical address. 466 */ 467 for (i = 0; phys_avail[i + 2] != 0; i += 2) 468 ; 469 Maxmem = sparc64_btop(phys_avail[i + 1]); 470 471 /* 472 * Add the PROM mappings to the kernel TSB. 473 */ 474 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 475 panic("pmap_bootstrap: finddevice /virtual-memory"); 476 if ((sz = OF_getproplen(vmem, "translations")) == -1) 477 panic("pmap_bootstrap: getproplen translations"); 478 if (sizeof(translations) < sz) 479 panic("pmap_bootstrap: translations too small"); 480 bzero(translations, sz); 481 if (OF_getprop(vmem, "translations", translations, sz) == -1) 482 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 483 sz /= sizeof(*translations); 484 translations_size = sz; 485 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 486 qsort(translations, sz, sizeof (*translations), om_cmp); 487 for (i = 0; i < sz; i++) { 488 CTR3(KTR_PMAP, 489 "translation: start=%#lx size=%#lx tte=%#lx", 490 translations[i].om_start, translations[i].om_size, 491 translations[i].om_tte); 492 if ((translations[i].om_tte & TD_V) == 0) 493 continue; 494 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 495 translations[i].om_start > VM_MAX_PROM_ADDRESS) 496 continue; 497 for (off = 0; off < translations[i].om_size; 498 off += PAGE_SIZE) { 499 va = translations[i].om_start + off; 500 tp = tsb_kvtotte(va); 501 tp->tte_vpn = TV_VPN(va, TS_8K); 502 tp->tte_data = 503 ((translations[i].om_tte & 504 ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | 505 (cpu_impl < CPU_IMPL_ULTRASPARCIII ? 506 (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : 507 (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | 508 (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + 509 off; 510 } 511 } 512 513 /* 514 * Get the available physical memory ranges from /memory/reg. These 515 * are only used for kernel dumps, but it may not be wise to do PROM 516 * calls in that situation. 517 */ 518 if ((sz = OF_getproplen(pmem, "reg")) == -1) 519 panic("pmap_bootstrap: getproplen /memory/reg"); 520 if (sizeof(sparc64_memreg) < sz) 521 panic("pmap_bootstrap: sparc64_memreg too small"); 522 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 523 panic("pmap_bootstrap: getprop /memory/reg"); 524 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 525 526 /* 527 * Initialize the kernel pmap (which is statically allocated). 528 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 529 * but sparc64 start up is not ready to initialize mutexes yet. 530 * It is called in machdep.c. 531 */ 532 pm = kernel_pmap; 533 for (i = 0; i < MAXCPU; i++) 534 pm->pm_context[i] = TLB_CTX_KERNEL; 535 pm->pm_active = ~0; 536 537 /* 538 * Flush all non-locked TLB entries possibly left over by the 539 * firmware. 540 */ 541 tlb_flush_nonlocked(); 542} 543 544void 545pmap_map_tsb(void) 546{ 547 vm_offset_t va; 548 vm_paddr_t pa; 549 u_long data; 550 register_t s; 551 int i; 552 553 s = intr_disable(); 554 555 /* 556 * Map the 4MB TSB pages. 557 */ 558 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 559 va = (vm_offset_t)tsb_kernel + i; 560 pa = tsb_kernel_phys + i; 561 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 562 TD_P | TD_W; 563 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 564 TLB_TAR_CTX(TLB_CTX_KERNEL)); 565 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 566 } 567 568 /* 569 * Set the secondary context to be the kernel context (needed for 570 * FP block operations in the kernel). 571 */ 572 stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & 573 TLB_SCXR_PGSZ_MASK) | TLB_CTX_KERNEL); 574 flush(KERNBASE); 575 576 intr_restore(s); 577} 578 579/* 580 * Allocate a physical page of memory directly from the phys_avail map. 581 * Can only be called from pmap_bootstrap before avail start and end are 582 * calculated. 583 */ 584static vm_paddr_t 585pmap_bootstrap_alloc(vm_size_t size) 586{ 587 vm_paddr_t pa; 588 int i; 589 590 size = roundup(size, PAGE_SIZE * DCACHE_COLORS); 591 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 592 if (phys_avail[i + 1] - phys_avail[i] < size) 593 continue; 594 pa = phys_avail[i]; 595 phys_avail[i] += size; 596 return (pa); 597 } 598 panic("pmap_bootstrap_alloc"); 599} 600 601/* 602 * Initialize a vm_page's machine-dependent fields. 603 */ 604void 605pmap_page_init(vm_page_t m) 606{ 607 608 TAILQ_INIT(&m->md.tte_list); 609 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 610 m->md.flags = 0; 611 m->md.pmap = NULL; 612} 613 614/* 615 * Initialize the pmap module. 616 */ 617void 618pmap_init(void) 619{ 620 vm_offset_t addr; 621 vm_size_t size; 622 int result; 623 int i; 624 625 for (i = 0; i < translations_size; i++) { 626 addr = translations[i].om_start; 627 size = translations[i].om_size; 628 if ((translations[i].om_tte & TD_V) == 0) 629 continue; 630 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 631 continue; 632 result = vm_map_find(kernel_map, NULL, 0, &addr, size, 633 VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 634 if (result != KERN_SUCCESS || addr != translations[i].om_start) 635 panic("pmap_init: vm_map_find"); 636 } 637} 638 639/* 640 * Extract the physical page address associated with the given 641 * map/virtual_address pair. 642 */ 643vm_paddr_t 644pmap_extract(pmap_t pm, vm_offset_t va) 645{ 646 struct tte *tp; 647 vm_paddr_t pa; 648 649 if (pm == kernel_pmap) 650 return (pmap_kextract(va)); 651 PMAP_LOCK(pm); 652 tp = tsb_tte_lookup(pm, va); 653 if (tp == NULL) 654 pa = 0; 655 else 656 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 657 PMAP_UNLOCK(pm); 658 return (pa); 659} 660 661/* 662 * Atomically extract and hold the physical page with the given 663 * pmap and virtual address pair if that mapping permits the given 664 * protection. 665 */ 666vm_page_t 667pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 668{ 669 struct tte *tp; 670 vm_page_t m; 671 672 m = NULL; 673 vm_page_lock_queues(); 674 if (pm == kernel_pmap) { 675 if (va >= VM_MIN_DIRECT_ADDRESS) { 676 tp = NULL; 677 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 678 vm_page_hold(m); 679 } else { 680 tp = tsb_kvtotte(va); 681 if ((tp->tte_data & TD_V) == 0) 682 tp = NULL; 683 } 684 } else { 685 PMAP_LOCK(pm); 686 tp = tsb_tte_lookup(pm, va); 687 } 688 if (tp != NULL && ((tp->tte_data & TD_SW) || 689 (prot & VM_PROT_WRITE) == 0)) { 690 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 691 vm_page_hold(m); 692 } 693 vm_page_unlock_queues(); 694 if (pm != kernel_pmap) 695 PMAP_UNLOCK(pm); 696 return (m); 697} 698 699/* 700 * Extract the physical page address associated with the given kernel virtual 701 * address. 702 */ 703vm_paddr_t 704pmap_kextract(vm_offset_t va) 705{ 706 struct tte *tp; 707 708 if (va >= VM_MIN_DIRECT_ADDRESS) 709 return (TLB_DIRECT_TO_PHYS(va)); 710 tp = tsb_kvtotte(va); 711 if ((tp->tte_data & TD_V) == 0) 712 return (0); 713 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 714} 715 716int 717pmap_cache_enter(vm_page_t m, vm_offset_t va) 718{ 719 struct tte *tp; 720 int color; 721 722 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 723 KASSERT((m->flags & PG_FICTITIOUS) == 0, 724 ("pmap_cache_enter: fake page")); 725 PMAP_STATS_INC(pmap_ncache_enter); 726 727 /* 728 * Find the color for this virtual address and note the added mapping. 729 */ 730 color = DCACHE_COLOR(va); 731 m->md.colors[color]++; 732 733 /* 734 * If all existing mappings have the same color, the mapping is 735 * cacheable. 736 */ 737 if (m->md.color == color) { 738 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 739 ("pmap_cache_enter: cacheable, mappings of other color")); 740 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 741 PMAP_STATS_INC(pmap_ncache_enter_c); 742 else 743 PMAP_STATS_INC(pmap_ncache_enter_oc); 744 return (1); 745 } 746 747 /* 748 * If there are no mappings of the other color, and the page still has 749 * the wrong color, this must be a new mapping. Change the color to 750 * match the new mapping, which is cacheable. We must flush the page 751 * from the cache now. 752 */ 753 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 754 KASSERT(m->md.colors[color] == 1, 755 ("pmap_cache_enter: changing color, not new mapping")); 756 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 757 m->md.color = color; 758 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 759 PMAP_STATS_INC(pmap_ncache_enter_cc); 760 else 761 PMAP_STATS_INC(pmap_ncache_enter_coc); 762 return (1); 763 } 764 765 /* 766 * If the mapping is already non-cacheable, just return. 767 */ 768 if (m->md.color == -1) { 769 PMAP_STATS_INC(pmap_ncache_enter_nc); 770 return (0); 771 } 772 773 PMAP_STATS_INC(pmap_ncache_enter_cnc); 774 775 /* 776 * Mark all mappings as uncacheable, flush any lines with the other 777 * color out of the dcache, and set the color to none (-1). 778 */ 779 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 780 atomic_clear_long(&tp->tte_data, TD_CV); 781 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 782 } 783 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 784 m->md.color = -1; 785 return (0); 786} 787 788void 789pmap_cache_remove(vm_page_t m, vm_offset_t va) 790{ 791 struct tte *tp; 792 int color; 793 794 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 795 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 796 m->md.colors[DCACHE_COLOR(va)]); 797 KASSERT((m->flags & PG_FICTITIOUS) == 0, 798 ("pmap_cache_remove: fake page")); 799 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 800 ("pmap_cache_remove: no mappings %d <= 0", 801 m->md.colors[DCACHE_COLOR(va)])); 802 PMAP_STATS_INC(pmap_ncache_remove); 803 804 /* 805 * Find the color for this virtual address and note the removal of 806 * the mapping. 807 */ 808 color = DCACHE_COLOR(va); 809 m->md.colors[color]--; 810 811 /* 812 * If the page is cacheable, just return and keep the same color, even 813 * if there are no longer any mappings. 814 */ 815 if (m->md.color != -1) { 816 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 817 PMAP_STATS_INC(pmap_ncache_remove_c); 818 else 819 PMAP_STATS_INC(pmap_ncache_remove_oc); 820 return; 821 } 822 823 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 824 ("pmap_cache_remove: uncacheable, no mappings of other color")); 825 826 /* 827 * If the page is not cacheable (color is -1), and the number of 828 * mappings for this color is not zero, just return. There are 829 * mappings of the other color still, so remain non-cacheable. 830 */ 831 if (m->md.colors[color] != 0) { 832 PMAP_STATS_INC(pmap_ncache_remove_nc); 833 return; 834 } 835 836 /* 837 * The number of mappings for this color is now zero. Recache the 838 * other colored mappings, and change the page color to the other 839 * color. There should be no lines in the data cache for this page, 840 * so flushing should not be needed. 841 */ 842 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 843 atomic_set_long(&tp->tte_data, TD_CV); 844 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 845 } 846 m->md.color = DCACHE_OTHER_COLOR(color); 847 848 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 849 PMAP_STATS_INC(pmap_ncache_remove_cc); 850 else 851 PMAP_STATS_INC(pmap_ncache_remove_coc); 852} 853 854/* 855 * Map a wired page into kernel virtual address space. 856 */ 857void 858pmap_kenter(vm_offset_t va, vm_page_t m) 859{ 860 vm_offset_t ova; 861 struct tte *tp; 862 vm_page_t om; 863 u_long data; 864 865 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 866 PMAP_STATS_INC(pmap_nkenter); 867 tp = tsb_kvtotte(va); 868 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 869 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 870 if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { 871 CTR5(KTR_CT2, 872 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", 873 va, VM_PAGE_TO_PHYS(m), m->object, 874 m->object ? m->object->type : -1, 875 m->pindex); 876 PMAP_STATS_INC(pmap_nkenter_oc); 877 } 878 if ((tp->tte_data & TD_V) != 0) { 879 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 880 ova = TTE_GET_VA(tp); 881 if (m == om && va == ova) { 882 PMAP_STATS_INC(pmap_nkenter_stupid); 883 return; 884 } 885 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 886 pmap_cache_remove(om, ova); 887 if (va != ova) 888 tlb_page_demap(kernel_pmap, ova); 889 } 890 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 891 TD_P | TD_W; 892 if (pmap_cache_enter(m, va) != 0) 893 data |= TD_CV; 894 tp->tte_vpn = TV_VPN(va, TS_8K); 895 tp->tte_data = data; 896 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 897} 898 899/* 900 * Map a wired page into kernel virtual address space. This additionally 901 * takes a flag argument wich is or'ed to the TTE data. This is used by 902 * sparc64_bus_mem_map(). 903 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 904 * to flush entries that might still be in the cache, if applicable. 905 */ 906void 907pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 908{ 909 struct tte *tp; 910 911 tp = tsb_kvtotte(va); 912 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 913 va, pa, tp, tp->tte_data); 914 tp->tte_vpn = TV_VPN(va, TS_8K); 915 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 916} 917 918/* 919 * Remove a wired page from kernel virtual address space. 920 */ 921void 922pmap_kremove(vm_offset_t va) 923{ 924 struct tte *tp; 925 vm_page_t m; 926 927 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 928 PMAP_STATS_INC(pmap_nkremove); 929 tp = tsb_kvtotte(va); 930 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 931 tp->tte_data); 932 if ((tp->tte_data & TD_V) == 0) 933 return; 934 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 935 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 936 pmap_cache_remove(m, va); 937 TTE_ZERO(tp); 938} 939 940/* 941 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 942 */ 943void 944pmap_kremove_flags(vm_offset_t va) 945{ 946 struct tte *tp; 947 948 tp = tsb_kvtotte(va); 949 CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, 950 tp->tte_data); 951 TTE_ZERO(tp); 952} 953 954/* 955 * Map a range of physical addresses into kernel virtual address space. 956 * 957 * The value passed in *virt is a suggested virtual address for the mapping. 958 * Architectures which can support a direct-mapped physical to virtual region 959 * can return the appropriate address within that region, leaving '*virt' 960 * unchanged. 961 */ 962vm_offset_t 963pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 964{ 965 966 return (TLB_PHYS_TO_DIRECT(start)); 967} 968 969/* 970 * Map a list of wired pages into kernel virtual address space. This is 971 * intended for temporary mappings which do not need page modification or 972 * references recorded. Existing mappings in the region are overwritten. 973 */ 974void 975pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 976{ 977 vm_offset_t va; 978 int locked; 979 980 PMAP_STATS_INC(pmap_nqenter); 981 va = sva; 982 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 983 vm_page_lock_queues(); 984 while (count-- > 0) { 985 pmap_kenter(va, *m); 986 va += PAGE_SIZE; 987 m++; 988 } 989 if (!locked) 990 vm_page_unlock_queues(); 991 tlb_range_demap(kernel_pmap, sva, va); 992} 993 994/* 995 * Remove page mappings from kernel virtual address space. Intended for 996 * temporary mappings entered by pmap_qenter. 997 */ 998void 999pmap_qremove(vm_offset_t sva, int count) 1000{ 1001 vm_offset_t va; 1002 int locked; 1003 1004 PMAP_STATS_INC(pmap_nqremove); 1005 va = sva; 1006 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 1007 vm_page_lock_queues(); 1008 while (count-- > 0) { 1009 pmap_kremove(va); 1010 va += PAGE_SIZE; 1011 } 1012 if (!locked) 1013 vm_page_unlock_queues(); 1014 tlb_range_demap(kernel_pmap, sva, va); 1015} 1016 1017/* 1018 * Initialize the pmap associated with process 0. 1019 */ 1020void 1021pmap_pinit0(pmap_t pm) 1022{ 1023 int i; 1024 1025 PMAP_LOCK_INIT(pm); 1026 for (i = 0; i < MAXCPU; i++) 1027 pm->pm_context[i] = 0; 1028 pm->pm_active = 0; 1029 pm->pm_tsb = NULL; 1030 pm->pm_tsb_obj = NULL; 1031 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1032} 1033 1034/* 1035 * Initialize a preallocated and zeroed pmap structure, such as one in a 1036 * vmspace structure. 1037 */ 1038int 1039pmap_pinit(pmap_t pm) 1040{ 1041 vm_page_t ma[TSB_PAGES]; 1042 vm_page_t m; 1043 int i; 1044 1045 PMAP_LOCK_INIT(pm); 1046 1047 /* 1048 * Allocate KVA space for the TSB. 1049 */ 1050 if (pm->pm_tsb == NULL) { 1051 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1052 TSB_BSIZE); 1053 if (pm->pm_tsb == NULL) { 1054 PMAP_LOCK_DESTROY(pm); 1055 return (0); 1056 } 1057 } 1058 1059 /* 1060 * Allocate an object for it. 1061 */ 1062 if (pm->pm_tsb_obj == NULL) 1063 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1064 1065 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1066 for (i = 0; i < TSB_PAGES; i++) { 1067 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1068 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1069 m->valid = VM_PAGE_BITS_ALL; 1070 m->md.pmap = pm; 1071 ma[i] = m; 1072 } 1073 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1074 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1075 1076 for (i = 0; i < MAXCPU; i++) 1077 pm->pm_context[i] = -1; 1078 pm->pm_active = 0; 1079 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1080 return (1); 1081} 1082 1083/* 1084 * Release any resources held by the given physical map. 1085 * Called when a pmap initialized by pmap_pinit is being released. 1086 * Should only be called if the map contains no valid mappings. 1087 */ 1088void 1089pmap_release(pmap_t pm) 1090{ 1091 vm_object_t obj; 1092 vm_page_t m; 1093 struct pcpu *pc; 1094 1095 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1096 pm->pm_context[curcpu], pm->pm_tsb); 1097 KASSERT(pmap_resident_count(pm) == 0, 1098 ("pmap_release: resident pages %ld != 0", 1099 pmap_resident_count(pm))); 1100 1101 /* 1102 * After the pmap was freed, it might be reallocated to a new process. 1103 * When switching, this might lead us to wrongly assume that we need 1104 * not switch contexts because old and new pmap pointer are equal. 1105 * Therefore, make sure that this pmap is not referenced by any PCPU 1106 * pointer any more. This could happen in two cases: 1107 * - A process that referenced the pmap is currently exiting on a CPU. 1108 * However, it is guaranteed to not switch in any more after setting 1109 * its state to PRS_ZOMBIE. 1110 * - A process that referenced this pmap ran on a CPU, but we switched 1111 * to a kernel thread, leaving the pmap pointer unchanged. 1112 */ 1113 mtx_lock_spin(&sched_lock); 1114 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 1115 if (pc->pc_pmap == pm) 1116 pc->pc_pmap = NULL; 1117 } 1118 mtx_unlock_spin(&sched_lock); 1119 1120 obj = pm->pm_tsb_obj; 1121 VM_OBJECT_LOCK(obj); 1122 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1123 while (!TAILQ_EMPTY(&obj->memq)) { 1124 m = TAILQ_FIRST(&obj->memq); 1125 vm_page_lock_queues(); 1126 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1127 continue; 1128 KASSERT(m->hold_count == 0, 1129 ("pmap_release: freeing held tsb page")); 1130 m->md.pmap = NULL; 1131 m->wire_count--; 1132 atomic_subtract_int(&cnt.v_wire_count, 1); 1133 vm_page_free_zero(m); 1134 vm_page_unlock_queues(); 1135 } 1136 VM_OBJECT_UNLOCK(obj); 1137 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1138 PMAP_LOCK_DESTROY(pm); 1139} 1140 1141/* 1142 * Grow the number of kernel page table entries. Unneeded. 1143 */ 1144void 1145pmap_growkernel(vm_offset_t addr) 1146{ 1147 1148 panic("pmap_growkernel: can't grow kernel"); 1149} 1150 1151int 1152pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1153 vm_offset_t va) 1154{ 1155 vm_page_t m; 1156 u_long data; 1157 1158 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1159 data = atomic_readandclear_long(&tp->tte_data); 1160 if ((data & TD_FAKE) == 0) { 1161 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1162 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1163 if ((data & TD_WIRED) != 0) 1164 pm->pm_stats.wired_count--; 1165 if ((data & TD_PV) != 0) { 1166 if ((data & TD_W) != 0) 1167 vm_page_dirty(m); 1168 if ((data & TD_REF) != 0) 1169 vm_page_flag_set(m, PG_REFERENCED); 1170 if (TAILQ_EMPTY(&m->md.tte_list)) 1171 vm_page_flag_clear(m, PG_WRITEABLE); 1172 pm->pm_stats.resident_count--; 1173 } 1174 pmap_cache_remove(m, va); 1175 } 1176 TTE_ZERO(tp); 1177 if (PMAP_REMOVE_DONE(pm)) 1178 return (0); 1179 return (1); 1180} 1181 1182/* 1183 * Remove the given range of addresses from the specified map. 1184 */ 1185void 1186pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1187{ 1188 struct tte *tp; 1189 vm_offset_t va; 1190 1191 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1192 pm->pm_context[curcpu], start, end); 1193 if (PMAP_REMOVE_DONE(pm)) 1194 return; 1195 vm_page_lock_queues(); 1196 PMAP_LOCK(pm); 1197 if (end - start > PMAP_TSB_THRESH) { 1198 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1199 tlb_context_demap(pm); 1200 } else { 1201 for (va = start; va < end; va += PAGE_SIZE) { 1202 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1203 if (!pmap_remove_tte(pm, NULL, tp, va)) 1204 break; 1205 } 1206 } 1207 tlb_range_demap(pm, start, end - 1); 1208 } 1209 PMAP_UNLOCK(pm); 1210 vm_page_unlock_queues(); 1211} 1212 1213void 1214pmap_remove_all(vm_page_t m) 1215{ 1216 struct pmap *pm; 1217 struct tte *tpn; 1218 struct tte *tp; 1219 vm_offset_t va; 1220 1221 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1222 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1223 tpn = TAILQ_NEXT(tp, tte_link); 1224 if ((tp->tte_data & TD_PV) == 0) 1225 continue; 1226 pm = TTE_GET_PMAP(tp); 1227 va = TTE_GET_VA(tp); 1228 PMAP_LOCK(pm); 1229 if ((tp->tte_data & TD_WIRED) != 0) 1230 pm->pm_stats.wired_count--; 1231 if ((tp->tte_data & TD_REF) != 0) 1232 vm_page_flag_set(m, PG_REFERENCED); 1233 if ((tp->tte_data & TD_W) != 0) 1234 vm_page_dirty(m); 1235 tp->tte_data &= ~TD_V; 1236 tlb_page_demap(pm, va); 1237 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1238 pm->pm_stats.resident_count--; 1239 pmap_cache_remove(m, va); 1240 TTE_ZERO(tp); 1241 PMAP_UNLOCK(pm); 1242 } 1243 vm_page_flag_clear(m, PG_WRITEABLE); 1244} 1245 1246int 1247pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1248 vm_offset_t va) 1249{ 1250 u_long data; 1251 vm_page_t m; 1252 1253 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1254 if ((data & TD_PV) != 0) { 1255 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1256 if ((data & TD_REF) != 0) 1257 vm_page_flag_set(m, PG_REFERENCED); 1258 if ((data & TD_W) != 0) 1259 vm_page_dirty(m); 1260 } 1261 return (1); 1262} 1263 1264/* 1265 * Set the physical protection on the specified range of this map as requested. 1266 */ 1267void 1268pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1269{ 1270 vm_offset_t va; 1271 struct tte *tp; 1272 1273 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1274 pm->pm_context[curcpu], sva, eva, prot); 1275 1276 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1277 pmap_remove(pm, sva, eva); 1278 return; 1279 } 1280 1281 if (prot & VM_PROT_WRITE) 1282 return; 1283 1284 vm_page_lock_queues(); 1285 PMAP_LOCK(pm); 1286 if (eva - sva > PMAP_TSB_THRESH) { 1287 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1288 tlb_context_demap(pm); 1289 } else { 1290 for (va = sva; va < eva; va += PAGE_SIZE) { 1291 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1292 pmap_protect_tte(pm, NULL, tp, va); 1293 } 1294 tlb_range_demap(pm, sva, eva - 1); 1295 } 1296 PMAP_UNLOCK(pm); 1297 vm_page_unlock_queues(); 1298} 1299 1300/* 1301 * Map the given physical page at the specified virtual address in the 1302 * target pmap with the protection requested. If specified the page 1303 * will be wired down. 1304 */ 1305void 1306pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, 1307 vm_prot_t prot, boolean_t wired) 1308{ 1309 1310 vm_page_lock_queues(); 1311 PMAP_LOCK(pm); 1312 pmap_enter_locked(pm, va, m, prot, wired); 1313 vm_page_unlock_queues(); 1314 PMAP_UNLOCK(pm); 1315} 1316 1317/* 1318 * Map the given physical page at the specified virtual address in the 1319 * target pmap with the protection requested. If specified the page 1320 * will be wired down. 1321 * 1322 * The page queues and pmap must be locked. 1323 */ 1324static void 1325pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1326 boolean_t wired) 1327{ 1328 struct tte *tp; 1329 vm_paddr_t pa; 1330 u_long data; 1331 int i; 1332 1333 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1334 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1335 PMAP_STATS_INC(pmap_nenter); 1336 pa = VM_PAGE_TO_PHYS(m); 1337 1338 /* 1339 * If this is a fake page from the device_pager, but it covers actual 1340 * physical memory, convert to the real backing page. 1341 */ 1342 if ((m->flags & PG_FICTITIOUS) != 0) { 1343 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1344 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1345 m = PHYS_TO_VM_PAGE(pa); 1346 break; 1347 } 1348 } 1349 } 1350 1351 CTR6(KTR_PMAP, 1352 "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1353 pm->pm_context[curcpu], m, va, pa, prot, wired); 1354 1355 /* 1356 * If there is an existing mapping, and the physical address has not 1357 * changed, must be protection or wiring change. 1358 */ 1359 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1360 CTR0(KTR_PMAP, "pmap_enter_locked: update"); 1361 PMAP_STATS_INC(pmap_nenter_update); 1362 1363 /* 1364 * Wiring change, just update stats. 1365 */ 1366 if (wired) { 1367 if ((tp->tte_data & TD_WIRED) == 0) { 1368 tp->tte_data |= TD_WIRED; 1369 pm->pm_stats.wired_count++; 1370 } 1371 } else { 1372 if ((tp->tte_data & TD_WIRED) != 0) { 1373 tp->tte_data &= ~TD_WIRED; 1374 pm->pm_stats.wired_count--; 1375 } 1376 } 1377 1378 /* 1379 * Save the old bits and clear the ones we're interested in. 1380 */ 1381 data = tp->tte_data; 1382 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1383 1384 /* 1385 * If we're turning off write permissions, sense modify status. 1386 */ 1387 if ((prot & VM_PROT_WRITE) != 0) { 1388 tp->tte_data |= TD_SW; 1389 if (wired) { 1390 tp->tte_data |= TD_W; 1391 } 1392 vm_page_flag_set(m, PG_WRITEABLE); 1393 } else if ((data & TD_W) != 0) { 1394 vm_page_dirty(m); 1395 } 1396 1397 /* 1398 * If we're turning on execute permissions, flush the icache. 1399 */ 1400 if ((prot & VM_PROT_EXECUTE) != 0) { 1401 if ((data & TD_EXEC) == 0) { 1402 icache_page_inval(pa); 1403 } 1404 tp->tte_data |= TD_EXEC; 1405 } 1406 1407 /* 1408 * Delete the old mapping. 1409 */ 1410 tlb_page_demap(pm, TTE_GET_VA(tp)); 1411 } else { 1412 /* 1413 * If there is an existing mapping, but its for a different 1414 * phsyical address, delete the old mapping. 1415 */ 1416 if (tp != NULL) { 1417 CTR0(KTR_PMAP, "pmap_enter_locked: replace"); 1418 PMAP_STATS_INC(pmap_nenter_replace); 1419 pmap_remove_tte(pm, NULL, tp, va); 1420 tlb_page_demap(pm, va); 1421 } else { 1422 CTR0(KTR_PMAP, "pmap_enter_locked: new"); 1423 PMAP_STATS_INC(pmap_nenter_new); 1424 } 1425 1426 /* 1427 * Now set up the data and install the new mapping. 1428 */ 1429 data = TD_V | TD_8K | TD_PA(pa); 1430 if (pm == kernel_pmap) 1431 data |= TD_P; 1432 if ((prot & VM_PROT_WRITE) != 0) { 1433 data |= TD_SW; 1434 vm_page_flag_set(m, PG_WRITEABLE); 1435 } 1436 if (prot & VM_PROT_EXECUTE) { 1437 data |= TD_EXEC; 1438 icache_page_inval(pa); 1439 } 1440 1441 /* 1442 * If its wired update stats. We also don't need reference or 1443 * modify tracking for wired mappings, so set the bits now. 1444 */ 1445 if (wired) { 1446 pm->pm_stats.wired_count++; 1447 data |= TD_REF | TD_WIRED; 1448 if ((prot & VM_PROT_WRITE) != 0) 1449 data |= TD_W; 1450 } 1451 1452 tsb_tte_enter(pm, m, va, TS_8K, data); 1453 } 1454} 1455 1456/* 1457 * Maps a sequence of resident pages belonging to the same object. 1458 * The sequence begins with the given page m_start. This page is 1459 * mapped at the given virtual address start. Each subsequent page is 1460 * mapped at a virtual address that is offset from start by the same 1461 * amount as the page is offset from m_start within the object. The 1462 * last page in the sequence is the page with the largest offset from 1463 * m_start that can be mapped at a virtual address less than the given 1464 * virtual address end. Not every virtual page between start and end 1465 * is mapped; only those for which a resident page exists with the 1466 * corresponding offset from m_start are mapped. 1467 */ 1468void 1469pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1470 vm_page_t m_start, vm_prot_t prot) 1471{ 1472 vm_page_t m; 1473 vm_pindex_t diff, psize; 1474 1475 psize = atop(end - start); 1476 m = m_start; 1477 PMAP_LOCK(pm); 1478 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1479 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1480 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1481 m = TAILQ_NEXT(m, listq); 1482 } 1483 PMAP_UNLOCK(pm); 1484} 1485 1486void 1487pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1488{ 1489 1490 PMAP_LOCK(pm); 1491 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1492 FALSE); 1493 PMAP_UNLOCK(pm); 1494} 1495 1496void 1497pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1498 vm_pindex_t pindex, vm_size_t size) 1499{ 1500 1501 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1502 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 1503 ("pmap_object_init_pt: non-device object")); 1504} 1505 1506/* 1507 * Change the wiring attribute for a map/virtual-address pair. 1508 * The mapping must already exist in the pmap. 1509 */ 1510void 1511pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1512{ 1513 struct tte *tp; 1514 u_long data; 1515 1516 PMAP_LOCK(pm); 1517 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1518 if (wired) { 1519 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1520 if ((data & TD_WIRED) == 0) 1521 pm->pm_stats.wired_count++; 1522 } else { 1523 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1524 if ((data & TD_WIRED) != 0) 1525 pm->pm_stats.wired_count--; 1526 } 1527 } 1528 PMAP_UNLOCK(pm); 1529} 1530 1531static int 1532pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1533{ 1534 vm_page_t m; 1535 u_long data; 1536 1537 if ((tp->tte_data & TD_FAKE) != 0) 1538 return (1); 1539 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1540 data = tp->tte_data & 1541 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1542 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1543 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1544 } 1545 return (1); 1546} 1547 1548void 1549pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1550 vm_size_t len, vm_offset_t src_addr) 1551{ 1552 struct tte *tp; 1553 vm_offset_t va; 1554 1555 if (dst_addr != src_addr) 1556 return; 1557 vm_page_lock_queues(); 1558 if (dst_pmap < src_pmap) { 1559 PMAP_LOCK(dst_pmap); 1560 PMAP_LOCK(src_pmap); 1561 } else { 1562 PMAP_LOCK(src_pmap); 1563 PMAP_LOCK(dst_pmap); 1564 } 1565 if (len > PMAP_TSB_THRESH) { 1566 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1567 pmap_copy_tte); 1568 tlb_context_demap(dst_pmap); 1569 } else { 1570 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1571 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1572 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1573 } 1574 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1575 } 1576 vm_page_unlock_queues(); 1577 PMAP_UNLOCK(src_pmap); 1578 PMAP_UNLOCK(dst_pmap); 1579} 1580 1581void 1582pmap_zero_page(vm_page_t m) 1583{ 1584 struct tte *tp; 1585 vm_offset_t va; 1586 vm_paddr_t pa; 1587 1588 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1589 ("pmap_zero_page: fake page")); 1590 PMAP_STATS_INC(pmap_nzero_page); 1591 pa = VM_PAGE_TO_PHYS(m); 1592 if (m->md.color == -1) { 1593 PMAP_STATS_INC(pmap_nzero_page_nc); 1594 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1595 } else if (m->md.color == DCACHE_COLOR(pa)) { 1596 PMAP_STATS_INC(pmap_nzero_page_c); 1597 va = TLB_PHYS_TO_DIRECT(pa); 1598 cpu_block_zero((void *)va, PAGE_SIZE); 1599 } else { 1600 PMAP_STATS_INC(pmap_nzero_page_oc); 1601 PMAP_LOCK(kernel_pmap); 1602 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1603 tp = tsb_kvtotte(va); 1604 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1605 tp->tte_vpn = TV_VPN(va, TS_8K); 1606 cpu_block_zero((void *)va, PAGE_SIZE); 1607 tlb_page_demap(kernel_pmap, va); 1608 PMAP_UNLOCK(kernel_pmap); 1609 } 1610} 1611 1612void 1613pmap_zero_page_area(vm_page_t m, int off, int size) 1614{ 1615 struct tte *tp; 1616 vm_offset_t va; 1617 vm_paddr_t pa; 1618 1619 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1620 ("pmap_zero_page_area: fake page")); 1621 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1622 PMAP_STATS_INC(pmap_nzero_page_area); 1623 pa = VM_PAGE_TO_PHYS(m); 1624 if (m->md.color == -1) { 1625 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1626 aszero(ASI_PHYS_USE_EC, pa + off, size); 1627 } else if (m->md.color == DCACHE_COLOR(pa)) { 1628 PMAP_STATS_INC(pmap_nzero_page_area_c); 1629 va = TLB_PHYS_TO_DIRECT(pa); 1630 bzero((void *)(va + off), size); 1631 } else { 1632 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1633 PMAP_LOCK(kernel_pmap); 1634 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1635 tp = tsb_kvtotte(va); 1636 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1637 tp->tte_vpn = TV_VPN(va, TS_8K); 1638 bzero((void *)(va + off), size); 1639 tlb_page_demap(kernel_pmap, va); 1640 PMAP_UNLOCK(kernel_pmap); 1641 } 1642} 1643 1644void 1645pmap_zero_page_idle(vm_page_t m) 1646{ 1647 struct tte *tp; 1648 vm_offset_t va; 1649 vm_paddr_t pa; 1650 1651 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1652 ("pmap_zero_page_idle: fake page")); 1653 PMAP_STATS_INC(pmap_nzero_page_idle); 1654 pa = VM_PAGE_TO_PHYS(m); 1655 if (m->md.color == -1) { 1656 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1657 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1658 } else if (m->md.color == DCACHE_COLOR(pa)) { 1659 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1660 va = TLB_PHYS_TO_DIRECT(pa); 1661 cpu_block_zero((void *)va, PAGE_SIZE); 1662 } else { 1663 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1664 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1665 tp = tsb_kvtotte(va); 1666 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1667 tp->tte_vpn = TV_VPN(va, TS_8K); 1668 cpu_block_zero((void *)va, PAGE_SIZE); 1669 tlb_page_demap(kernel_pmap, va); 1670 } 1671} 1672 1673void 1674pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1675{ 1676 vm_offset_t vdst; 1677 vm_offset_t vsrc; 1678 vm_paddr_t pdst; 1679 vm_paddr_t psrc; 1680 struct tte *tp; 1681 1682 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1683 ("pmap_copy_page: fake dst page")); 1684 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1685 ("pmap_copy_page: fake src page")); 1686 PMAP_STATS_INC(pmap_ncopy_page); 1687 pdst = VM_PAGE_TO_PHYS(mdst); 1688 psrc = VM_PAGE_TO_PHYS(msrc); 1689 if (msrc->md.color == -1 && mdst->md.color == -1) { 1690 PMAP_STATS_INC(pmap_ncopy_page_nc); 1691 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1692 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1693 mdst->md.color == DCACHE_COLOR(pdst)) { 1694 PMAP_STATS_INC(pmap_ncopy_page_c); 1695 vdst = TLB_PHYS_TO_DIRECT(pdst); 1696 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1697 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1698 } else if (msrc->md.color == -1) { 1699 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1700 PMAP_STATS_INC(pmap_ncopy_page_dc); 1701 vdst = TLB_PHYS_TO_DIRECT(pdst); 1702 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1703 PAGE_SIZE); 1704 } else { 1705 PMAP_STATS_INC(pmap_ncopy_page_doc); 1706 PMAP_LOCK(kernel_pmap); 1707 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1708 tp = tsb_kvtotte(vdst); 1709 tp->tte_data = 1710 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1711 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1712 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1713 PAGE_SIZE); 1714 tlb_page_demap(kernel_pmap, vdst); 1715 PMAP_UNLOCK(kernel_pmap); 1716 } 1717 } else if (mdst->md.color == -1) { 1718 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1719 PMAP_STATS_INC(pmap_ncopy_page_sc); 1720 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1721 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1722 PAGE_SIZE); 1723 } else { 1724 PMAP_STATS_INC(pmap_ncopy_page_soc); 1725 PMAP_LOCK(kernel_pmap); 1726 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1727 tp = tsb_kvtotte(vsrc); 1728 tp->tte_data = 1729 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1730 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1731 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1732 PAGE_SIZE); 1733 tlb_page_demap(kernel_pmap, vsrc); 1734 PMAP_UNLOCK(kernel_pmap); 1735 } 1736 } else { 1737 PMAP_STATS_INC(pmap_ncopy_page_oc); 1738 PMAP_LOCK(kernel_pmap); 1739 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1740 tp = tsb_kvtotte(vdst); 1741 tp->tte_data = 1742 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1743 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1744 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1745 tp = tsb_kvtotte(vsrc); 1746 tp->tte_data = 1747 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1748 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1749 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1750 tlb_page_demap(kernel_pmap, vdst); 1751 tlb_page_demap(kernel_pmap, vsrc); 1752 PMAP_UNLOCK(kernel_pmap); 1753 } 1754} 1755 1756/* 1757 * Returns true if the pmap's pv is one of the first 1758 * 16 pvs linked to from this page. This count may 1759 * be changed upwards or downwards in the future; it 1760 * is only necessary that true be returned for a small 1761 * subset of pmaps for proper page aging. 1762 */ 1763boolean_t 1764pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1765{ 1766 struct tte *tp; 1767 int loops; 1768 1769 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1770 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1771 return (FALSE); 1772 loops = 0; 1773 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1774 if ((tp->tte_data & TD_PV) == 0) 1775 continue; 1776 if (TTE_GET_PMAP(tp) == pm) 1777 return (TRUE); 1778 if (++loops >= 16) 1779 break; 1780 } 1781 return (FALSE); 1782} 1783 1784/* 1785 * Return the number of managed mappings to the given physical page 1786 * that are wired. 1787 */ 1788int 1789pmap_page_wired_mappings(vm_page_t m) 1790{ 1791 struct tte *tp; 1792 int count; 1793 1794 count = 0; 1795 if ((m->flags & PG_FICTITIOUS) != 0) 1796 return (count); 1797 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1798 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1799 if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) 1800 count++; 1801 return (count); 1802} 1803 1804/* 1805 * Remove all pages from specified address space, this aids process exit 1806 * speeds. This is much faster than pmap_remove n the case of running down 1807 * an entire address space. Only works for the current pmap. 1808 */ 1809void 1810pmap_remove_pages(pmap_t pm) 1811{ 1812 1813} 1814 1815/* 1816 * Returns TRUE if the given page has a managed mapping. 1817 */ 1818boolean_t 1819pmap_page_is_mapped(vm_page_t m) 1820{ 1821 struct tte *tp; 1822 1823 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1824 return (FALSE); 1825 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1826 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1827 if ((tp->tte_data & TD_PV) != 0) 1828 return (TRUE); 1829 } 1830 return (FALSE); 1831} 1832 1833/* 1834 * Return a count of reference bits for a page, clearing those bits. 1835 * It is not necessary for every reference bit to be cleared, but it 1836 * is necessary that 0 only be returned when there are truly no 1837 * reference bits set. 1838 * 1839 * XXX: The exact number of bits to check and clear is a matter that 1840 * should be tested and standardized at some point in the future for 1841 * optimal aging of shared pages. 1842 */ 1843int 1844pmap_ts_referenced(vm_page_t m) 1845{ 1846 struct tte *tpf; 1847 struct tte *tpn; 1848 struct tte *tp; 1849 u_long data; 1850 int count; 1851 1852 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1853 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1854 return (0); 1855 count = 0; 1856 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1857 tpf = tp; 1858 do { 1859 tpn = TAILQ_NEXT(tp, tte_link); 1860 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1861 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1862 if ((tp->tte_data & TD_PV) == 0) 1863 continue; 1864 data = atomic_clear_long(&tp->tte_data, TD_REF); 1865 if ((data & TD_REF) != 0 && ++count > 4) 1866 break; 1867 } while ((tp = tpn) != NULL && tp != tpf); 1868 } 1869 return (count); 1870} 1871 1872boolean_t 1873pmap_is_modified(vm_page_t m) 1874{ 1875 struct tte *tp; 1876 1877 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1878 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1879 return (FALSE); 1880 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1881 if ((tp->tte_data & TD_PV) == 0) 1882 continue; 1883 if ((tp->tte_data & TD_W) != 0) 1884 return (TRUE); 1885 } 1886 return (FALSE); 1887} 1888 1889/* 1890 * pmap_is_prefaultable: 1891 * 1892 * Return whether or not the specified virtual address is elgible 1893 * for prefault. 1894 */ 1895boolean_t 1896pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1897{ 1898 1899 return (FALSE); 1900} 1901 1902void 1903pmap_clear_modify(vm_page_t m) 1904{ 1905 struct tte *tp; 1906 u_long data; 1907 1908 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1909 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1910 return; 1911 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1912 if ((tp->tte_data & TD_PV) == 0) 1913 continue; 1914 data = atomic_clear_long(&tp->tte_data, TD_W); 1915 if ((data & TD_W) != 0) 1916 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1917 } 1918} 1919 1920void 1921pmap_clear_reference(vm_page_t m) 1922{ 1923 struct tte *tp; 1924 u_long data; 1925 1926 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1927 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1928 return; 1929 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1930 if ((tp->tte_data & TD_PV) == 0) 1931 continue; 1932 data = atomic_clear_long(&tp->tte_data, TD_REF); 1933 if ((data & TD_REF) != 0) 1934 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1935 } 1936} 1937 1938void 1939pmap_remove_write(vm_page_t m) 1940{ 1941 struct tte *tp; 1942 u_long data; 1943 1944 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1945 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1946 (m->flags & PG_WRITEABLE) == 0) 1947 return; 1948 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1949 if ((tp->tte_data & TD_PV) == 0) 1950 continue; 1951 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1952 if ((data & TD_W) != 0) { 1953 vm_page_dirty(m); 1954 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1955 } 1956 } 1957 vm_page_flag_clear(m, PG_WRITEABLE); 1958} 1959 1960int 1961pmap_mincore(pmap_t pm, vm_offset_t addr) 1962{ 1963 1964 /* TODO; */ 1965 return (0); 1966} 1967 1968/* 1969 * Activate a user pmap. The pmap must be activated before its address space 1970 * can be accessed in any way. 1971 */ 1972void 1973pmap_activate(struct thread *td) 1974{ 1975 struct vmspace *vm; 1976 struct pmap *pm; 1977 int context; 1978 1979 vm = td->td_proc->p_vmspace; 1980 pm = vmspace_pmap(vm); 1981 1982 mtx_lock_spin(&sched_lock); 1983 1984 context = PCPU_GET(tlb_ctx); 1985 if (context == PCPU_GET(tlb_ctx_max)) { 1986 tlb_flush_user(); 1987 context = PCPU_GET(tlb_ctx_min); 1988 } 1989 PCPU_SET(tlb_ctx, context + 1); 1990 1991 pm->pm_context[curcpu] = context; 1992 pm->pm_active |= PCPU_GET(cpumask); 1993 PCPU_SET(pmap, pm); 1994 1995 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 1996 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 1997 stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & 1998 TLB_PCXR_PGSZ_MASK) | context); 1999 flush(KERNBASE); 2000 2001 mtx_unlock_spin(&sched_lock); 2002} 2003 2004/* 2005 * Increase the starting virtual address of the given mapping if a 2006 * different alignment might result in more superpage mappings. 2007 */ 2008void 2009pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2010 vm_offset_t *addr, vm_size_t size) 2011{ 2012 2013} 2014