pmap.c revision 204152
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 */ 43 44#include <sys/cdefs.h> 45__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/pmap.c 204152 2010-02-20 23:24:19Z marius $"); 46 47/* 48 * Manages physical address maps. 49 * 50 * In addition to hardware address maps, this module is called upon to 51 * provide software-use-only maps which may or may not be stored in the 52 * same form as hardware maps. These pseudo-maps are used to store 53 * intermediate results from copy operations to and from address spaces. 54 * 55 * Since the information managed by this module is also stored by the 56 * logical address mapping module, this module may throw away valid virtual 57 * to physical mappings at almost any time. However, invalidations of 58 * mappings must be done as requested. 59 * 60 * In order to cope with hardware architectures which make virtual to 61 * physical map invalidates expensive, this module may delay invalidate 62 * reduced protection operations until such time as they are actually 63 * necessary. This module is given full information as to which processors 64 * are currently using which maps, and to when physical maps must be made 65 * correct. 66 */ 67 68#include "opt_kstack_pages.h" 69#include "opt_msgbuf.h" 70#include "opt_pmap.h" 71 72#include <sys/param.h> 73#include <sys/kernel.h> 74#include <sys/ktr.h> 75#include <sys/lock.h> 76#include <sys/msgbuf.h> 77#include <sys/mutex.h> 78#include <sys/proc.h> 79#include <sys/smp.h> 80#include <sys/sysctl.h> 81#include <sys/systm.h> 82#include <sys/vmmeter.h> 83 84#include <dev/ofw/openfirm.h> 85 86#include <vm/vm.h> 87#include <vm/vm_param.h> 88#include <vm/vm_kern.h> 89#include <vm/vm_page.h> 90#include <vm/vm_map.h> 91#include <vm/vm_object.h> 92#include <vm/vm_extern.h> 93#include <vm/vm_pageout.h> 94#include <vm/vm_pager.h> 95 96#include <machine/cache.h> 97#include <machine/frame.h> 98#include <machine/instr.h> 99#include <machine/md_var.h> 100#include <machine/metadata.h> 101#include <machine/ofw_mem.h> 102#include <machine/smp.h> 103#include <machine/tlb.h> 104#include <machine/tte.h> 105#include <machine/tsb.h> 106#include <machine/ver.h> 107 108#define PMAP_DEBUG 109 110#ifndef PMAP_SHPGPERPROC 111#define PMAP_SHPGPERPROC 200 112#endif 113 114/* XXX */ 115#include "opt_sched.h" 116#ifndef SCHED_4BSD 117#error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." 118#endif 119extern struct mtx sched_lock; 120 121/* 122 * Virtual address of message buffer 123 */ 124struct msgbuf *msgbufp; 125 126/* 127 * Map of physical memory reagions 128 */ 129vm_paddr_t phys_avail[128]; 130static struct ofw_mem_region mra[128]; 131struct ofw_mem_region sparc64_memreg[128]; 132int sparc64_nmemreg; 133static struct ofw_map translations[128]; 134static int translations_size; 135 136static vm_offset_t pmap_idle_map; 137static vm_offset_t pmap_temp_map_1; 138static vm_offset_t pmap_temp_map_2; 139 140/* 141 * First and last available kernel virtual addresses 142 */ 143vm_offset_t virtual_avail; 144vm_offset_t virtual_end; 145vm_offset_t kernel_vm_end; 146 147vm_offset_t vm_max_kernel_address; 148 149/* 150 * Kernel pmap 151 */ 152struct pmap kernel_pmap_store; 153 154/* 155 * Allocate physical memory for use in pmap_bootstrap. 156 */ 157static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 158 159/* 160 * Map the given physical page at the specified virtual address in the 161 * target pmap with the protection requested. If specified the page 162 * will be wired down. 163 * 164 * The page queues and pmap must be locked. 165 */ 166static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 167 vm_prot_t prot, boolean_t wired); 168 169extern int tl1_immu_miss_patch_1[]; 170extern int tl1_immu_miss_patch_2[]; 171extern int tl1_dmmu_miss_patch_1[]; 172extern int tl1_dmmu_miss_patch_2[]; 173extern int tl1_dmmu_prot_patch_1[]; 174extern int tl1_dmmu_prot_patch_2[]; 175 176/* 177 * If user pmap is processed with pmap_remove and with pmap_remove and the 178 * resident count drops to 0, there are no more pages to remove, so we 179 * need not continue. 180 */ 181#define PMAP_REMOVE_DONE(pm) \ 182 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 183 184/* 185 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 186 * and pmap_protect() instead of trying each virtual address. 187 */ 188#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 189 190SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 191 192PMAP_STATS_VAR(pmap_nenter); 193PMAP_STATS_VAR(pmap_nenter_update); 194PMAP_STATS_VAR(pmap_nenter_replace); 195PMAP_STATS_VAR(pmap_nenter_new); 196PMAP_STATS_VAR(pmap_nkenter); 197PMAP_STATS_VAR(pmap_nkenter_oc); 198PMAP_STATS_VAR(pmap_nkenter_stupid); 199PMAP_STATS_VAR(pmap_nkremove); 200PMAP_STATS_VAR(pmap_nqenter); 201PMAP_STATS_VAR(pmap_nqremove); 202PMAP_STATS_VAR(pmap_ncache_enter); 203PMAP_STATS_VAR(pmap_ncache_enter_c); 204PMAP_STATS_VAR(pmap_ncache_enter_oc); 205PMAP_STATS_VAR(pmap_ncache_enter_cc); 206PMAP_STATS_VAR(pmap_ncache_enter_coc); 207PMAP_STATS_VAR(pmap_ncache_enter_nc); 208PMAP_STATS_VAR(pmap_ncache_enter_cnc); 209PMAP_STATS_VAR(pmap_ncache_remove); 210PMAP_STATS_VAR(pmap_ncache_remove_c); 211PMAP_STATS_VAR(pmap_ncache_remove_oc); 212PMAP_STATS_VAR(pmap_ncache_remove_cc); 213PMAP_STATS_VAR(pmap_ncache_remove_coc); 214PMAP_STATS_VAR(pmap_ncache_remove_nc); 215PMAP_STATS_VAR(pmap_nzero_page); 216PMAP_STATS_VAR(pmap_nzero_page_c); 217PMAP_STATS_VAR(pmap_nzero_page_oc); 218PMAP_STATS_VAR(pmap_nzero_page_nc); 219PMAP_STATS_VAR(pmap_nzero_page_area); 220PMAP_STATS_VAR(pmap_nzero_page_area_c); 221PMAP_STATS_VAR(pmap_nzero_page_area_oc); 222PMAP_STATS_VAR(pmap_nzero_page_area_nc); 223PMAP_STATS_VAR(pmap_nzero_page_idle); 224PMAP_STATS_VAR(pmap_nzero_page_idle_c); 225PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 226PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 227PMAP_STATS_VAR(pmap_ncopy_page); 228PMAP_STATS_VAR(pmap_ncopy_page_c); 229PMAP_STATS_VAR(pmap_ncopy_page_oc); 230PMAP_STATS_VAR(pmap_ncopy_page_nc); 231PMAP_STATS_VAR(pmap_ncopy_page_dc); 232PMAP_STATS_VAR(pmap_ncopy_page_doc); 233PMAP_STATS_VAR(pmap_ncopy_page_sc); 234PMAP_STATS_VAR(pmap_ncopy_page_soc); 235 236PMAP_STATS_VAR(pmap_nnew_thread); 237PMAP_STATS_VAR(pmap_nnew_thread_oc); 238 239/* 240 * Quick sort callout for comparing memory regions 241 */ 242static int mr_cmp(const void *a, const void *b); 243static int om_cmp(const void *a, const void *b); 244 245static int 246mr_cmp(const void *a, const void *b) 247{ 248 const struct ofw_mem_region *mra; 249 const struct ofw_mem_region *mrb; 250 251 mra = a; 252 mrb = b; 253 if (mra->mr_start < mrb->mr_start) 254 return (-1); 255 else if (mra->mr_start > mrb->mr_start) 256 return (1); 257 else 258 return (0); 259} 260 261static int 262om_cmp(const void *a, const void *b) 263{ 264 const struct ofw_map *oma; 265 const struct ofw_map *omb; 266 267 oma = a; 268 omb = b; 269 if (oma->om_start < omb->om_start) 270 return (-1); 271 else if (oma->om_start > omb->om_start) 272 return (1); 273 else 274 return (0); 275} 276 277/* 278 * Bootstrap the system enough to run with virtual memory. 279 */ 280void 281pmap_bootstrap(u_int cpu_impl) 282{ 283 struct pmap *pm; 284 struct tte *tp; 285 vm_offset_t off; 286 vm_offset_t va; 287 vm_paddr_t pa; 288 vm_size_t physsz; 289 vm_size_t virtsz; 290 phandle_t pmem; 291 phandle_t vmem; 292 int sz; 293 int i; 294 int j; 295 296 /* 297 * Find out what physical memory is available from the PROM and 298 * initialize the phys_avail array. This must be done before 299 * pmap_bootstrap_alloc is called. 300 */ 301 if ((pmem = OF_finddevice("/memory")) == -1) 302 panic("pmap_bootstrap: finddevice /memory"); 303 if ((sz = OF_getproplen(pmem, "available")) == -1) 304 panic("pmap_bootstrap: getproplen /memory/available"); 305 if (sizeof(phys_avail) < sz) 306 panic("pmap_bootstrap: phys_avail too small"); 307 if (sizeof(mra) < sz) 308 panic("pmap_bootstrap: mra too small"); 309 bzero(mra, sz); 310 if (OF_getprop(pmem, "available", mra, sz) == -1) 311 panic("pmap_bootstrap: getprop /memory/available"); 312 sz /= sizeof(*mra); 313 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 314 qsort(mra, sz, sizeof (*mra), mr_cmp); 315 physsz = 0; 316 getenv_quad("hw.physmem", &physmem); 317 physmem = btoc(physmem); 318 for (i = 0, j = 0; i < sz; i++, j += 2) { 319 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 320 mra[i].mr_size); 321 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 322 if (btoc(physsz) < physmem) { 323 phys_avail[j] = mra[i].mr_start; 324 phys_avail[j + 1] = mra[i].mr_start + 325 (ctob(physmem) - physsz); 326 physsz = ctob(physmem); 327 } 328 break; 329 } 330 phys_avail[j] = mra[i].mr_start; 331 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 332 physsz += mra[i].mr_size; 333 } 334 physmem = btoc(physsz); 335 336 /* 337 * Calculate the size of kernel virtual memory, and the size and mask 338 * for the kernel TSB based on the phsyical memory size but limited 339 * by letting the kernel TSB take up no more than half of the dTLB 340 * slots available for locked entries. 341 */ 342 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 343 virtsz = MIN(virtsz, 344 (dtlb_slots / 2 * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); 345 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 346 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 347 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 348 349 if (kernel_tlb_slots + PCPU_PAGES + tsb_kernel_size / PAGE_SIZE_4M + 350 1 /* PROM page */ + 1 /* spare */ > dtlb_slots) 351 panic("pmap_bootstrap: insufficient dTLB entries"); 352 if (kernel_tlb_slots + 1 /* PROM page */ + 1 /* spare */ > itlb_slots) 353 panic("pmap_bootstrap: insufficient iTLB entries"); 354 355 /* 356 * Allocate the kernel TSB and lock it in the TLB. 357 */ 358 pa = pmap_bootstrap_alloc(tsb_kernel_size); 359 if (pa & PAGE_MASK_4M) 360 panic("pmap_bootstrap: tsb unaligned\n"); 361 tsb_kernel_phys = pa; 362 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 363 pmap_map_tsb(); 364 bzero(tsb_kernel, tsb_kernel_size); 365 366 /* 367 * Allocate and map the dynamic per-CPU area for the BSP. 368 */ 369 pa = pmap_bootstrap_alloc(DPCPU_SIZE); 370 dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); 371 372 /* 373 * Allocate and map the message buffer. 374 */ 375 pa = pmap_bootstrap_alloc(MSGBUF_SIZE); 376 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); 377 378 /* 379 * Patch the virtual address and the tsb mask into the trap table. 380 */ 381 382#define SETHI(rd, imm22) \ 383 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 384 EIF_IMM((imm22) >> 10, 22)) 385#define OR_R_I_R(rd, imm13, rs1) \ 386 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 387 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 388 389#define PATCH(addr) do { \ 390 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 391 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 392 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 393 panic("pmap_boostrap: patched instructions have changed"); \ 394 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 395 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 396 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 397 flush(addr); \ 398 flush(addr + 1); \ 399 flush(addr + 2); \ 400} while (0) 401 402 PATCH(tl1_immu_miss_patch_1); 403 PATCH(tl1_immu_miss_patch_2); 404 PATCH(tl1_dmmu_miss_patch_1); 405 PATCH(tl1_dmmu_miss_patch_2); 406 PATCH(tl1_dmmu_prot_patch_1); 407 PATCH(tl1_dmmu_prot_patch_2); 408 409 /* 410 * Enter fake 8k pages for the 4MB kernel pages, so that 411 * pmap_kextract() will work for them. 412 */ 413 for (i = 0; i < kernel_tlb_slots; i++) { 414 pa = kernel_tlbs[i].te_pa; 415 va = kernel_tlbs[i].te_va; 416 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 417 tp = tsb_kvtotte(va + off); 418 tp->tte_vpn = TV_VPN(va + off, TS_8K); 419 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 420 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 421 } 422 } 423 424 /* 425 * Set the start and end of KVA. The kernel is loaded starting 426 * at the first available 4MB super page, so we advance to the 427 * end of the last one used for it. 428 */ 429 virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; 430 virtual_end = vm_max_kernel_address; 431 kernel_vm_end = vm_max_kernel_address; 432 433 /* 434 * Allocate kva space for temporary mappings. 435 */ 436 pmap_idle_map = virtual_avail; 437 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 438 pmap_temp_map_1 = virtual_avail; 439 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 440 pmap_temp_map_2 = virtual_avail; 441 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 442 443 /* 444 * Allocate a kernel stack with guard page for thread0 and map it 445 * into the kernel TSB. We must ensure that the virtual address is 446 * coloured properly, since we're allocating from phys_avail so the 447 * memory won't have an associated vm_page_t. 448 */ 449 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); 450 kstack0_phys = pa; 451 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 452 PAGE_SIZE; 453 kstack0 = virtual_avail; 454 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 455 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 456 ("pmap_bootstrap: kstack0 miscoloured")); 457 for (i = 0; i < KSTACK_PAGES; i++) { 458 pa = kstack0_phys + i * PAGE_SIZE; 459 va = kstack0 + i * PAGE_SIZE; 460 tp = tsb_kvtotte(va); 461 tp->tte_vpn = TV_VPN(va, TS_8K); 462 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 463 TD_CP | TD_CV | TD_P | TD_W; 464 } 465 466 /* 467 * Calculate the last available physical address. 468 */ 469 for (i = 0; phys_avail[i + 2] != 0; i += 2) 470 ; 471 Maxmem = sparc64_btop(phys_avail[i + 1]); 472 473 /* 474 * Add the PROM mappings to the kernel TSB. 475 */ 476 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 477 panic("pmap_bootstrap: finddevice /virtual-memory"); 478 if ((sz = OF_getproplen(vmem, "translations")) == -1) 479 panic("pmap_bootstrap: getproplen translations"); 480 if (sizeof(translations) < sz) 481 panic("pmap_bootstrap: translations too small"); 482 bzero(translations, sz); 483 if (OF_getprop(vmem, "translations", translations, sz) == -1) 484 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 485 sz /= sizeof(*translations); 486 translations_size = sz; 487 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 488 qsort(translations, sz, sizeof (*translations), om_cmp); 489 for (i = 0; i < sz; i++) { 490 CTR3(KTR_PMAP, 491 "translation: start=%#lx size=%#lx tte=%#lx", 492 translations[i].om_start, translations[i].om_size, 493 translations[i].om_tte); 494 if ((translations[i].om_tte & TD_V) == 0) 495 continue; 496 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 497 translations[i].om_start > VM_MAX_PROM_ADDRESS) 498 continue; 499 for (off = 0; off < translations[i].om_size; 500 off += PAGE_SIZE) { 501 va = translations[i].om_start + off; 502 tp = tsb_kvtotte(va); 503 tp->tte_vpn = TV_VPN(va, TS_8K); 504 tp->tte_data = 505 ((translations[i].om_tte & 506 ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | 507 (cpu_impl < CPU_IMPL_ULTRASPARCIII ? 508 (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : 509 (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | 510 (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + 511 off; 512 } 513 } 514 515 /* 516 * Get the available physical memory ranges from /memory/reg. These 517 * are only used for kernel dumps, but it may not be wise to do PROM 518 * calls in that situation. 519 */ 520 if ((sz = OF_getproplen(pmem, "reg")) == -1) 521 panic("pmap_bootstrap: getproplen /memory/reg"); 522 if (sizeof(sparc64_memreg) < sz) 523 panic("pmap_bootstrap: sparc64_memreg too small"); 524 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 525 panic("pmap_bootstrap: getprop /memory/reg"); 526 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 527 528 /* 529 * Initialize the kernel pmap (which is statically allocated). 530 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 531 * but sparc64 start up is not ready to initialize mutexes yet. 532 * It is called in machdep.c. 533 */ 534 pm = kernel_pmap; 535 for (i = 0; i < MAXCPU; i++) 536 pm->pm_context[i] = TLB_CTX_KERNEL; 537 pm->pm_active = ~0; 538 539 /* 540 * Flush all non-locked TLB entries possibly left over by the 541 * firmware. 542 */ 543 tlb_flush_nonlocked(); 544} 545 546void 547pmap_map_tsb(void) 548{ 549 vm_offset_t va; 550 vm_paddr_t pa; 551 u_long data; 552 register_t s; 553 int i; 554 555 s = intr_disable(); 556 557 /* 558 * Map the 4MB TSB pages. 559 */ 560 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 561 va = (vm_offset_t)tsb_kernel + i; 562 pa = tsb_kernel_phys + i; 563 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 564 TD_P | TD_W; 565 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 566 TLB_TAR_CTX(TLB_CTX_KERNEL)); 567 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 568 } 569 570 /* 571 * Set the secondary context to be the kernel context (needed for 572 * FP block operations in the kernel). 573 */ 574 stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & 575 TLB_SCXR_PGSZ_MASK) | TLB_CTX_KERNEL); 576 flush(KERNBASE); 577 578 intr_restore(s); 579} 580 581/* 582 * Allocate a physical page of memory directly from the phys_avail map. 583 * Can only be called from pmap_bootstrap before avail start and end are 584 * calculated. 585 */ 586static vm_paddr_t 587pmap_bootstrap_alloc(vm_size_t size) 588{ 589 vm_paddr_t pa; 590 int i; 591 592 size = roundup(size, PAGE_SIZE * DCACHE_COLORS); 593 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 594 if (phys_avail[i + 1] - phys_avail[i] < size) 595 continue; 596 pa = phys_avail[i]; 597 phys_avail[i] += size; 598 return (pa); 599 } 600 panic("pmap_bootstrap_alloc"); 601} 602 603/* 604 * Initialize a vm_page's machine-dependent fields. 605 */ 606void 607pmap_page_init(vm_page_t m) 608{ 609 610 TAILQ_INIT(&m->md.tte_list); 611 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 612 m->md.flags = 0; 613 m->md.pmap = NULL; 614} 615 616/* 617 * Initialize the pmap module. 618 */ 619void 620pmap_init(void) 621{ 622 vm_offset_t addr; 623 vm_size_t size; 624 int result; 625 int i; 626 627 for (i = 0; i < translations_size; i++) { 628 addr = translations[i].om_start; 629 size = translations[i].om_size; 630 if ((translations[i].om_tte & TD_V) == 0) 631 continue; 632 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 633 continue; 634 result = vm_map_find(kernel_map, NULL, 0, &addr, size, 635 VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 636 if (result != KERN_SUCCESS || addr != translations[i].om_start) 637 panic("pmap_init: vm_map_find"); 638 } 639} 640 641/* 642 * Extract the physical page address associated with the given 643 * map/virtual_address pair. 644 */ 645vm_paddr_t 646pmap_extract(pmap_t pm, vm_offset_t va) 647{ 648 struct tte *tp; 649 vm_paddr_t pa; 650 651 if (pm == kernel_pmap) 652 return (pmap_kextract(va)); 653 PMAP_LOCK(pm); 654 tp = tsb_tte_lookup(pm, va); 655 if (tp == NULL) 656 pa = 0; 657 else 658 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 659 PMAP_UNLOCK(pm); 660 return (pa); 661} 662 663/* 664 * Atomically extract and hold the physical page with the given 665 * pmap and virtual address pair if that mapping permits the given 666 * protection. 667 */ 668vm_page_t 669pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 670{ 671 struct tte *tp; 672 vm_page_t m; 673 674 m = NULL; 675 vm_page_lock_queues(); 676 if (pm == kernel_pmap) { 677 if (va >= VM_MIN_DIRECT_ADDRESS) { 678 tp = NULL; 679 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 680 vm_page_hold(m); 681 } else { 682 tp = tsb_kvtotte(va); 683 if ((tp->tte_data & TD_V) == 0) 684 tp = NULL; 685 } 686 } else { 687 PMAP_LOCK(pm); 688 tp = tsb_tte_lookup(pm, va); 689 } 690 if (tp != NULL && ((tp->tte_data & TD_SW) || 691 (prot & VM_PROT_WRITE) == 0)) { 692 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 693 vm_page_hold(m); 694 } 695 vm_page_unlock_queues(); 696 if (pm != kernel_pmap) 697 PMAP_UNLOCK(pm); 698 return (m); 699} 700 701/* 702 * Extract the physical page address associated with the given kernel virtual 703 * address. 704 */ 705vm_paddr_t 706pmap_kextract(vm_offset_t va) 707{ 708 struct tte *tp; 709 710 if (va >= VM_MIN_DIRECT_ADDRESS) 711 return (TLB_DIRECT_TO_PHYS(va)); 712 tp = tsb_kvtotte(va); 713 if ((tp->tte_data & TD_V) == 0) 714 return (0); 715 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 716} 717 718int 719pmap_cache_enter(vm_page_t m, vm_offset_t va) 720{ 721 struct tte *tp; 722 int color; 723 724 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 725 KASSERT((m->flags & PG_FICTITIOUS) == 0, 726 ("pmap_cache_enter: fake page")); 727 PMAP_STATS_INC(pmap_ncache_enter); 728 729 /* 730 * Find the color for this virtual address and note the added mapping. 731 */ 732 color = DCACHE_COLOR(va); 733 m->md.colors[color]++; 734 735 /* 736 * If all existing mappings have the same color, the mapping is 737 * cacheable. 738 */ 739 if (m->md.color == color) { 740 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 741 ("pmap_cache_enter: cacheable, mappings of other color")); 742 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 743 PMAP_STATS_INC(pmap_ncache_enter_c); 744 else 745 PMAP_STATS_INC(pmap_ncache_enter_oc); 746 return (1); 747 } 748 749 /* 750 * If there are no mappings of the other color, and the page still has 751 * the wrong color, this must be a new mapping. Change the color to 752 * match the new mapping, which is cacheable. We must flush the page 753 * from the cache now. 754 */ 755 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 756 KASSERT(m->md.colors[color] == 1, 757 ("pmap_cache_enter: changing color, not new mapping")); 758 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 759 m->md.color = color; 760 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 761 PMAP_STATS_INC(pmap_ncache_enter_cc); 762 else 763 PMAP_STATS_INC(pmap_ncache_enter_coc); 764 return (1); 765 } 766 767 /* 768 * If the mapping is already non-cacheable, just return. 769 */ 770 if (m->md.color == -1) { 771 PMAP_STATS_INC(pmap_ncache_enter_nc); 772 return (0); 773 } 774 775 PMAP_STATS_INC(pmap_ncache_enter_cnc); 776 777 /* 778 * Mark all mappings as uncacheable, flush any lines with the other 779 * color out of the dcache, and set the color to none (-1). 780 */ 781 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 782 atomic_clear_long(&tp->tte_data, TD_CV); 783 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 784 } 785 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 786 m->md.color = -1; 787 return (0); 788} 789 790void 791pmap_cache_remove(vm_page_t m, vm_offset_t va) 792{ 793 struct tte *tp; 794 int color; 795 796 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 797 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 798 m->md.colors[DCACHE_COLOR(va)]); 799 KASSERT((m->flags & PG_FICTITIOUS) == 0, 800 ("pmap_cache_remove: fake page")); 801 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 802 ("pmap_cache_remove: no mappings %d <= 0", 803 m->md.colors[DCACHE_COLOR(va)])); 804 PMAP_STATS_INC(pmap_ncache_remove); 805 806 /* 807 * Find the color for this virtual address and note the removal of 808 * the mapping. 809 */ 810 color = DCACHE_COLOR(va); 811 m->md.colors[color]--; 812 813 /* 814 * If the page is cacheable, just return and keep the same color, even 815 * if there are no longer any mappings. 816 */ 817 if (m->md.color != -1) { 818 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 819 PMAP_STATS_INC(pmap_ncache_remove_c); 820 else 821 PMAP_STATS_INC(pmap_ncache_remove_oc); 822 return; 823 } 824 825 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 826 ("pmap_cache_remove: uncacheable, no mappings of other color")); 827 828 /* 829 * If the page is not cacheable (color is -1), and the number of 830 * mappings for this color is not zero, just return. There are 831 * mappings of the other color still, so remain non-cacheable. 832 */ 833 if (m->md.colors[color] != 0) { 834 PMAP_STATS_INC(pmap_ncache_remove_nc); 835 return; 836 } 837 838 /* 839 * The number of mappings for this color is now zero. Recache the 840 * other colored mappings, and change the page color to the other 841 * color. There should be no lines in the data cache for this page, 842 * so flushing should not be needed. 843 */ 844 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 845 atomic_set_long(&tp->tte_data, TD_CV); 846 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 847 } 848 m->md.color = DCACHE_OTHER_COLOR(color); 849 850 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 851 PMAP_STATS_INC(pmap_ncache_remove_cc); 852 else 853 PMAP_STATS_INC(pmap_ncache_remove_coc); 854} 855 856/* 857 * Map a wired page into kernel virtual address space. 858 */ 859void 860pmap_kenter(vm_offset_t va, vm_page_t m) 861{ 862 vm_offset_t ova; 863 struct tte *tp; 864 vm_page_t om; 865 u_long data; 866 867 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 868 PMAP_STATS_INC(pmap_nkenter); 869 tp = tsb_kvtotte(va); 870 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 871 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 872 if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { 873 CTR5(KTR_CT2, 874 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", 875 va, VM_PAGE_TO_PHYS(m), m->object, 876 m->object ? m->object->type : -1, 877 m->pindex); 878 PMAP_STATS_INC(pmap_nkenter_oc); 879 } 880 if ((tp->tte_data & TD_V) != 0) { 881 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 882 ova = TTE_GET_VA(tp); 883 if (m == om && va == ova) { 884 PMAP_STATS_INC(pmap_nkenter_stupid); 885 return; 886 } 887 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 888 pmap_cache_remove(om, ova); 889 if (va != ova) 890 tlb_page_demap(kernel_pmap, ova); 891 } 892 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 893 TD_P | TD_W; 894 if (pmap_cache_enter(m, va) != 0) 895 data |= TD_CV; 896 tp->tte_vpn = TV_VPN(va, TS_8K); 897 tp->tte_data = data; 898 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 899} 900 901/* 902 * Map a wired page into kernel virtual address space. This additionally 903 * takes a flag argument wich is or'ed to the TTE data. This is used by 904 * sparc64_bus_mem_map(). 905 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 906 * to flush entries that might still be in the cache, if applicable. 907 */ 908void 909pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 910{ 911 struct tte *tp; 912 913 tp = tsb_kvtotte(va); 914 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 915 va, pa, tp, tp->tte_data); 916 tp->tte_vpn = TV_VPN(va, TS_8K); 917 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 918} 919 920/* 921 * Remove a wired page from kernel virtual address space. 922 */ 923void 924pmap_kremove(vm_offset_t va) 925{ 926 struct tte *tp; 927 vm_page_t m; 928 929 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 930 PMAP_STATS_INC(pmap_nkremove); 931 tp = tsb_kvtotte(va); 932 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 933 tp->tte_data); 934 if ((tp->tte_data & TD_V) == 0) 935 return; 936 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 937 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 938 pmap_cache_remove(m, va); 939 TTE_ZERO(tp); 940} 941 942/* 943 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 944 */ 945void 946pmap_kremove_flags(vm_offset_t va) 947{ 948 struct tte *tp; 949 950 tp = tsb_kvtotte(va); 951 CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, 952 tp->tte_data); 953 TTE_ZERO(tp); 954} 955 956/* 957 * Map a range of physical addresses into kernel virtual address space. 958 * 959 * The value passed in *virt is a suggested virtual address for the mapping. 960 * Architectures which can support a direct-mapped physical to virtual region 961 * can return the appropriate address within that region, leaving '*virt' 962 * unchanged. 963 */ 964vm_offset_t 965pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 966{ 967 968 return (TLB_PHYS_TO_DIRECT(start)); 969} 970 971/* 972 * Map a list of wired pages into kernel virtual address space. This is 973 * intended for temporary mappings which do not need page modification or 974 * references recorded. Existing mappings in the region are overwritten. 975 */ 976void 977pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 978{ 979 vm_offset_t va; 980 int locked; 981 982 PMAP_STATS_INC(pmap_nqenter); 983 va = sva; 984 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 985 vm_page_lock_queues(); 986 while (count-- > 0) { 987 pmap_kenter(va, *m); 988 va += PAGE_SIZE; 989 m++; 990 } 991 if (!locked) 992 vm_page_unlock_queues(); 993 tlb_range_demap(kernel_pmap, sva, va); 994} 995 996/* 997 * Remove page mappings from kernel virtual address space. Intended for 998 * temporary mappings entered by pmap_qenter. 999 */ 1000void 1001pmap_qremove(vm_offset_t sva, int count) 1002{ 1003 vm_offset_t va; 1004 int locked; 1005 1006 PMAP_STATS_INC(pmap_nqremove); 1007 va = sva; 1008 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 1009 vm_page_lock_queues(); 1010 while (count-- > 0) { 1011 pmap_kremove(va); 1012 va += PAGE_SIZE; 1013 } 1014 if (!locked) 1015 vm_page_unlock_queues(); 1016 tlb_range_demap(kernel_pmap, sva, va); 1017} 1018 1019/* 1020 * Initialize the pmap associated with process 0. 1021 */ 1022void 1023pmap_pinit0(pmap_t pm) 1024{ 1025 int i; 1026 1027 PMAP_LOCK_INIT(pm); 1028 for (i = 0; i < MAXCPU; i++) 1029 pm->pm_context[i] = 0; 1030 pm->pm_active = 0; 1031 pm->pm_tsb = NULL; 1032 pm->pm_tsb_obj = NULL; 1033 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1034} 1035 1036/* 1037 * Initialize a preallocated and zeroed pmap structure, such as one in a 1038 * vmspace structure. 1039 */ 1040int 1041pmap_pinit(pmap_t pm) 1042{ 1043 vm_page_t ma[TSB_PAGES]; 1044 vm_page_t m; 1045 int i; 1046 1047 PMAP_LOCK_INIT(pm); 1048 1049 /* 1050 * Allocate KVA space for the TSB. 1051 */ 1052 if (pm->pm_tsb == NULL) { 1053 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1054 TSB_BSIZE); 1055 if (pm->pm_tsb == NULL) { 1056 PMAP_LOCK_DESTROY(pm); 1057 return (0); 1058 } 1059 } 1060 1061 /* 1062 * Allocate an object for it. 1063 */ 1064 if (pm->pm_tsb_obj == NULL) 1065 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1066 1067 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1068 for (i = 0; i < TSB_PAGES; i++) { 1069 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1070 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1071 m->valid = VM_PAGE_BITS_ALL; 1072 m->md.pmap = pm; 1073 ma[i] = m; 1074 } 1075 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1076 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1077 1078 for (i = 0; i < MAXCPU; i++) 1079 pm->pm_context[i] = -1; 1080 pm->pm_active = 0; 1081 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1082 return (1); 1083} 1084 1085/* 1086 * Release any resources held by the given physical map. 1087 * Called when a pmap initialized by pmap_pinit is being released. 1088 * Should only be called if the map contains no valid mappings. 1089 */ 1090void 1091pmap_release(pmap_t pm) 1092{ 1093 vm_object_t obj; 1094 vm_page_t m; 1095 struct pcpu *pc; 1096 1097 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1098 pm->pm_context[curcpu], pm->pm_tsb); 1099 KASSERT(pmap_resident_count(pm) == 0, 1100 ("pmap_release: resident pages %ld != 0", 1101 pmap_resident_count(pm))); 1102 1103 /* 1104 * After the pmap was freed, it might be reallocated to a new process. 1105 * When switching, this might lead us to wrongly assume that we need 1106 * not switch contexts because old and new pmap pointer are equal. 1107 * Therefore, make sure that this pmap is not referenced by any PCPU 1108 * pointer any more. This could happen in two cases: 1109 * - A process that referenced the pmap is currently exiting on a CPU. 1110 * However, it is guaranteed to not switch in any more after setting 1111 * its state to PRS_ZOMBIE. 1112 * - A process that referenced this pmap ran on a CPU, but we switched 1113 * to a kernel thread, leaving the pmap pointer unchanged. 1114 */ 1115 mtx_lock_spin(&sched_lock); 1116 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) 1117 if (pc->pc_pmap == pm) 1118 pc->pc_pmap = NULL; 1119 mtx_unlock_spin(&sched_lock); 1120 1121 obj = pm->pm_tsb_obj; 1122 VM_OBJECT_LOCK(obj); 1123 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1124 while (!TAILQ_EMPTY(&obj->memq)) { 1125 m = TAILQ_FIRST(&obj->memq); 1126 vm_page_lock_queues(); 1127 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1128 continue; 1129 KASSERT(m->hold_count == 0, 1130 ("pmap_release: freeing held tsb page")); 1131 m->md.pmap = NULL; 1132 m->wire_count--; 1133 atomic_subtract_int(&cnt.v_wire_count, 1); 1134 vm_page_free_zero(m); 1135 vm_page_unlock_queues(); 1136 } 1137 VM_OBJECT_UNLOCK(obj); 1138 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1139 PMAP_LOCK_DESTROY(pm); 1140} 1141 1142/* 1143 * Grow the number of kernel page table entries. Unneeded. 1144 */ 1145void 1146pmap_growkernel(vm_offset_t addr) 1147{ 1148 1149 panic("pmap_growkernel: can't grow kernel"); 1150} 1151 1152int 1153pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1154 vm_offset_t va) 1155{ 1156 vm_page_t m; 1157 u_long data; 1158 1159 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1160 data = atomic_readandclear_long(&tp->tte_data); 1161 if ((data & TD_FAKE) == 0) { 1162 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1163 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1164 if ((data & TD_WIRED) != 0) 1165 pm->pm_stats.wired_count--; 1166 if ((data & TD_PV) != 0) { 1167 if ((data & TD_W) != 0) 1168 vm_page_dirty(m); 1169 if ((data & TD_REF) != 0) 1170 vm_page_flag_set(m, PG_REFERENCED); 1171 if (TAILQ_EMPTY(&m->md.tte_list)) 1172 vm_page_flag_clear(m, PG_WRITEABLE); 1173 pm->pm_stats.resident_count--; 1174 } 1175 pmap_cache_remove(m, va); 1176 } 1177 TTE_ZERO(tp); 1178 if (PMAP_REMOVE_DONE(pm)) 1179 return (0); 1180 return (1); 1181} 1182 1183/* 1184 * Remove the given range of addresses from the specified map. 1185 */ 1186void 1187pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1188{ 1189 struct tte *tp; 1190 vm_offset_t va; 1191 1192 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1193 pm->pm_context[curcpu], start, end); 1194 if (PMAP_REMOVE_DONE(pm)) 1195 return; 1196 vm_page_lock_queues(); 1197 PMAP_LOCK(pm); 1198 if (end - start > PMAP_TSB_THRESH) { 1199 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1200 tlb_context_demap(pm); 1201 } else { 1202 for (va = start; va < end; va += PAGE_SIZE) 1203 if ((tp = tsb_tte_lookup(pm, va)) != NULL && 1204 !pmap_remove_tte(pm, NULL, tp, va)) 1205 break; 1206 tlb_range_demap(pm, start, end - 1); 1207 } 1208 PMAP_UNLOCK(pm); 1209 vm_page_unlock_queues(); 1210} 1211 1212void 1213pmap_remove_all(vm_page_t m) 1214{ 1215 struct pmap *pm; 1216 struct tte *tpn; 1217 struct tte *tp; 1218 vm_offset_t va; 1219 1220 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1221 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1222 tpn = TAILQ_NEXT(tp, tte_link); 1223 if ((tp->tte_data & TD_PV) == 0) 1224 continue; 1225 pm = TTE_GET_PMAP(tp); 1226 va = TTE_GET_VA(tp); 1227 PMAP_LOCK(pm); 1228 if ((tp->tte_data & TD_WIRED) != 0) 1229 pm->pm_stats.wired_count--; 1230 if ((tp->tte_data & TD_REF) != 0) 1231 vm_page_flag_set(m, PG_REFERENCED); 1232 if ((tp->tte_data & TD_W) != 0) 1233 vm_page_dirty(m); 1234 tp->tte_data &= ~TD_V; 1235 tlb_page_demap(pm, va); 1236 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1237 pm->pm_stats.resident_count--; 1238 pmap_cache_remove(m, va); 1239 TTE_ZERO(tp); 1240 PMAP_UNLOCK(pm); 1241 } 1242 vm_page_flag_clear(m, PG_WRITEABLE); 1243} 1244 1245int 1246pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1247 vm_offset_t va) 1248{ 1249 u_long data; 1250 vm_page_t m; 1251 1252 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1253 if ((data & TD_PV) != 0) { 1254 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1255 if ((data & TD_REF) != 0) 1256 vm_page_flag_set(m, PG_REFERENCED); 1257 if ((data & TD_W) != 0) 1258 vm_page_dirty(m); 1259 } 1260 return (1); 1261} 1262 1263/* 1264 * Set the physical protection on the specified range of this map as requested. 1265 */ 1266void 1267pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1268{ 1269 vm_offset_t va; 1270 struct tte *tp; 1271 1272 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1273 pm->pm_context[curcpu], sva, eva, prot); 1274 1275 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1276 pmap_remove(pm, sva, eva); 1277 return; 1278 } 1279 1280 if (prot & VM_PROT_WRITE) 1281 return; 1282 1283 vm_page_lock_queues(); 1284 PMAP_LOCK(pm); 1285 if (eva - sva > PMAP_TSB_THRESH) { 1286 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1287 tlb_context_demap(pm); 1288 } else { 1289 for (va = sva; va < eva; va += PAGE_SIZE) 1290 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1291 pmap_protect_tte(pm, NULL, tp, va); 1292 tlb_range_demap(pm, sva, eva - 1); 1293 } 1294 PMAP_UNLOCK(pm); 1295 vm_page_unlock_queues(); 1296} 1297 1298/* 1299 * Map the given physical page at the specified virtual address in the 1300 * target pmap with the protection requested. If specified the page 1301 * will be wired down. 1302 */ 1303void 1304pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, 1305 vm_prot_t prot, boolean_t wired) 1306{ 1307 1308 vm_page_lock_queues(); 1309 PMAP_LOCK(pm); 1310 pmap_enter_locked(pm, va, m, prot, wired); 1311 vm_page_unlock_queues(); 1312 PMAP_UNLOCK(pm); 1313} 1314 1315/* 1316 * Map the given physical page at the specified virtual address in the 1317 * target pmap with the protection requested. If specified the page 1318 * will be wired down. 1319 * 1320 * The page queues and pmap must be locked. 1321 */ 1322static void 1323pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1324 boolean_t wired) 1325{ 1326 struct tte *tp; 1327 vm_paddr_t pa; 1328 u_long data; 1329 int i; 1330 1331 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1332 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1333 PMAP_STATS_INC(pmap_nenter); 1334 pa = VM_PAGE_TO_PHYS(m); 1335 1336 /* 1337 * If this is a fake page from the device_pager, but it covers actual 1338 * physical memory, convert to the real backing page. 1339 */ 1340 if ((m->flags & PG_FICTITIOUS) != 0) { 1341 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1342 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1343 m = PHYS_TO_VM_PAGE(pa); 1344 break; 1345 } 1346 } 1347 } 1348 1349 CTR6(KTR_PMAP, 1350 "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1351 pm->pm_context[curcpu], m, va, pa, prot, wired); 1352 1353 /* 1354 * If there is an existing mapping, and the physical address has not 1355 * changed, must be protection or wiring change. 1356 */ 1357 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1358 CTR0(KTR_PMAP, "pmap_enter_locked: update"); 1359 PMAP_STATS_INC(pmap_nenter_update); 1360 1361 /* 1362 * Wiring change, just update stats. 1363 */ 1364 if (wired) { 1365 if ((tp->tte_data & TD_WIRED) == 0) { 1366 tp->tte_data |= TD_WIRED; 1367 pm->pm_stats.wired_count++; 1368 } 1369 } else { 1370 if ((tp->tte_data & TD_WIRED) != 0) { 1371 tp->tte_data &= ~TD_WIRED; 1372 pm->pm_stats.wired_count--; 1373 } 1374 } 1375 1376 /* 1377 * Save the old bits and clear the ones we're interested in. 1378 */ 1379 data = tp->tte_data; 1380 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1381 1382 /* 1383 * If we're turning off write permissions, sense modify status. 1384 */ 1385 if ((prot & VM_PROT_WRITE) != 0) { 1386 tp->tte_data |= TD_SW; 1387 if (wired) 1388 tp->tte_data |= TD_W; 1389 vm_page_flag_set(m, PG_WRITEABLE); 1390 } else if ((data & TD_W) != 0) 1391 vm_page_dirty(m); 1392 1393 /* 1394 * If we're turning on execute permissions, flush the icache. 1395 */ 1396 if ((prot & VM_PROT_EXECUTE) != 0) { 1397 if ((data & TD_EXEC) == 0) 1398 icache_page_inval(pa); 1399 tp->tte_data |= TD_EXEC; 1400 } 1401 1402 /* 1403 * Delete the old mapping. 1404 */ 1405 tlb_page_demap(pm, TTE_GET_VA(tp)); 1406 } else { 1407 /* 1408 * If there is an existing mapping, but its for a different 1409 * phsyical address, delete the old mapping. 1410 */ 1411 if (tp != NULL) { 1412 CTR0(KTR_PMAP, "pmap_enter_locked: replace"); 1413 PMAP_STATS_INC(pmap_nenter_replace); 1414 pmap_remove_tte(pm, NULL, tp, va); 1415 tlb_page_demap(pm, va); 1416 } else { 1417 CTR0(KTR_PMAP, "pmap_enter_locked: new"); 1418 PMAP_STATS_INC(pmap_nenter_new); 1419 } 1420 1421 /* 1422 * Now set up the data and install the new mapping. 1423 */ 1424 data = TD_V | TD_8K | TD_PA(pa); 1425 if (pm == kernel_pmap) 1426 data |= TD_P; 1427 if ((prot & VM_PROT_WRITE) != 0) { 1428 data |= TD_SW; 1429 vm_page_flag_set(m, PG_WRITEABLE); 1430 } 1431 if (prot & VM_PROT_EXECUTE) { 1432 data |= TD_EXEC; 1433 icache_page_inval(pa); 1434 } 1435 1436 /* 1437 * If its wired update stats. We also don't need reference or 1438 * modify tracking for wired mappings, so set the bits now. 1439 */ 1440 if (wired) { 1441 pm->pm_stats.wired_count++; 1442 data |= TD_REF | TD_WIRED; 1443 if ((prot & VM_PROT_WRITE) != 0) 1444 data |= TD_W; 1445 } 1446 1447 tsb_tte_enter(pm, m, va, TS_8K, data); 1448 } 1449} 1450 1451/* 1452 * Maps a sequence of resident pages belonging to the same object. 1453 * The sequence begins with the given page m_start. This page is 1454 * mapped at the given virtual address start. Each subsequent page is 1455 * mapped at a virtual address that is offset from start by the same 1456 * amount as the page is offset from m_start within the object. The 1457 * last page in the sequence is the page with the largest offset from 1458 * m_start that can be mapped at a virtual address less than the given 1459 * virtual address end. Not every virtual page between start and end 1460 * is mapped; only those for which a resident page exists with the 1461 * corresponding offset from m_start are mapped. 1462 */ 1463void 1464pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1465 vm_page_t m_start, vm_prot_t prot) 1466{ 1467 vm_page_t m; 1468 vm_pindex_t diff, psize; 1469 1470 psize = atop(end - start); 1471 m = m_start; 1472 PMAP_LOCK(pm); 1473 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1474 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1475 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1476 m = TAILQ_NEXT(m, listq); 1477 } 1478 PMAP_UNLOCK(pm); 1479} 1480 1481void 1482pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1483{ 1484 1485 PMAP_LOCK(pm); 1486 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1487 FALSE); 1488 PMAP_UNLOCK(pm); 1489} 1490 1491void 1492pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1493 vm_pindex_t pindex, vm_size_t size) 1494{ 1495 1496 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1497 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 1498 ("pmap_object_init_pt: non-device object")); 1499} 1500 1501/* 1502 * Change the wiring attribute for a map/virtual-address pair. 1503 * The mapping must already exist in the pmap. 1504 */ 1505void 1506pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1507{ 1508 struct tte *tp; 1509 u_long data; 1510 1511 PMAP_LOCK(pm); 1512 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1513 if (wired) { 1514 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1515 if ((data & TD_WIRED) == 0) 1516 pm->pm_stats.wired_count++; 1517 } else { 1518 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1519 if ((data & TD_WIRED) != 0) 1520 pm->pm_stats.wired_count--; 1521 } 1522 } 1523 PMAP_UNLOCK(pm); 1524} 1525 1526static int 1527pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, 1528 vm_offset_t va) 1529{ 1530 vm_page_t m; 1531 u_long data; 1532 1533 if ((tp->tte_data & TD_FAKE) != 0) 1534 return (1); 1535 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1536 data = tp->tte_data & 1537 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1538 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1539 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1540 } 1541 return (1); 1542} 1543 1544void 1545pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1546 vm_size_t len, vm_offset_t src_addr) 1547{ 1548 struct tte *tp; 1549 vm_offset_t va; 1550 1551 if (dst_addr != src_addr) 1552 return; 1553 vm_page_lock_queues(); 1554 if (dst_pmap < src_pmap) { 1555 PMAP_LOCK(dst_pmap); 1556 PMAP_LOCK(src_pmap); 1557 } else { 1558 PMAP_LOCK(src_pmap); 1559 PMAP_LOCK(dst_pmap); 1560 } 1561 if (len > PMAP_TSB_THRESH) { 1562 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1563 pmap_copy_tte); 1564 tlb_context_demap(dst_pmap); 1565 } else { 1566 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) 1567 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1568 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1569 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1570 } 1571 vm_page_unlock_queues(); 1572 PMAP_UNLOCK(src_pmap); 1573 PMAP_UNLOCK(dst_pmap); 1574} 1575 1576void 1577pmap_zero_page(vm_page_t m) 1578{ 1579 struct tte *tp; 1580 vm_offset_t va; 1581 vm_paddr_t pa; 1582 1583 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1584 ("pmap_zero_page: fake page")); 1585 PMAP_STATS_INC(pmap_nzero_page); 1586 pa = VM_PAGE_TO_PHYS(m); 1587 if (m->md.color == -1) { 1588 PMAP_STATS_INC(pmap_nzero_page_nc); 1589 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1590 } else if (m->md.color == DCACHE_COLOR(pa)) { 1591 PMAP_STATS_INC(pmap_nzero_page_c); 1592 va = TLB_PHYS_TO_DIRECT(pa); 1593 cpu_block_zero((void *)va, PAGE_SIZE); 1594 } else { 1595 PMAP_STATS_INC(pmap_nzero_page_oc); 1596 PMAP_LOCK(kernel_pmap); 1597 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1598 tp = tsb_kvtotte(va); 1599 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1600 tp->tte_vpn = TV_VPN(va, TS_8K); 1601 cpu_block_zero((void *)va, PAGE_SIZE); 1602 tlb_page_demap(kernel_pmap, va); 1603 PMAP_UNLOCK(kernel_pmap); 1604 } 1605} 1606 1607void 1608pmap_zero_page_area(vm_page_t m, int off, int size) 1609{ 1610 struct tte *tp; 1611 vm_offset_t va; 1612 vm_paddr_t pa; 1613 1614 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1615 ("pmap_zero_page_area: fake page")); 1616 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1617 PMAP_STATS_INC(pmap_nzero_page_area); 1618 pa = VM_PAGE_TO_PHYS(m); 1619 if (m->md.color == -1) { 1620 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1621 aszero(ASI_PHYS_USE_EC, pa + off, size); 1622 } else if (m->md.color == DCACHE_COLOR(pa)) { 1623 PMAP_STATS_INC(pmap_nzero_page_area_c); 1624 va = TLB_PHYS_TO_DIRECT(pa); 1625 bzero((void *)(va + off), size); 1626 } else { 1627 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1628 PMAP_LOCK(kernel_pmap); 1629 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1630 tp = tsb_kvtotte(va); 1631 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1632 tp->tte_vpn = TV_VPN(va, TS_8K); 1633 bzero((void *)(va + off), size); 1634 tlb_page_demap(kernel_pmap, va); 1635 PMAP_UNLOCK(kernel_pmap); 1636 } 1637} 1638 1639void 1640pmap_zero_page_idle(vm_page_t m) 1641{ 1642 struct tte *tp; 1643 vm_offset_t va; 1644 vm_paddr_t pa; 1645 1646 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1647 ("pmap_zero_page_idle: fake page")); 1648 PMAP_STATS_INC(pmap_nzero_page_idle); 1649 pa = VM_PAGE_TO_PHYS(m); 1650 if (m->md.color == -1) { 1651 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1652 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1653 } else if (m->md.color == DCACHE_COLOR(pa)) { 1654 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1655 va = TLB_PHYS_TO_DIRECT(pa); 1656 cpu_block_zero((void *)va, PAGE_SIZE); 1657 } else { 1658 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1659 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1660 tp = tsb_kvtotte(va); 1661 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1662 tp->tte_vpn = TV_VPN(va, TS_8K); 1663 cpu_block_zero((void *)va, PAGE_SIZE); 1664 tlb_page_demap(kernel_pmap, va); 1665 } 1666} 1667 1668void 1669pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1670{ 1671 vm_offset_t vdst; 1672 vm_offset_t vsrc; 1673 vm_paddr_t pdst; 1674 vm_paddr_t psrc; 1675 struct tte *tp; 1676 1677 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1678 ("pmap_copy_page: fake dst page")); 1679 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1680 ("pmap_copy_page: fake src page")); 1681 PMAP_STATS_INC(pmap_ncopy_page); 1682 pdst = VM_PAGE_TO_PHYS(mdst); 1683 psrc = VM_PAGE_TO_PHYS(msrc); 1684 if (msrc->md.color == -1 && mdst->md.color == -1) { 1685 PMAP_STATS_INC(pmap_ncopy_page_nc); 1686 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1687 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1688 mdst->md.color == DCACHE_COLOR(pdst)) { 1689 PMAP_STATS_INC(pmap_ncopy_page_c); 1690 vdst = TLB_PHYS_TO_DIRECT(pdst); 1691 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1692 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1693 } else if (msrc->md.color == -1) { 1694 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1695 PMAP_STATS_INC(pmap_ncopy_page_dc); 1696 vdst = TLB_PHYS_TO_DIRECT(pdst); 1697 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1698 PAGE_SIZE); 1699 } else { 1700 PMAP_STATS_INC(pmap_ncopy_page_doc); 1701 PMAP_LOCK(kernel_pmap); 1702 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1703 tp = tsb_kvtotte(vdst); 1704 tp->tte_data = 1705 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1706 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1707 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1708 PAGE_SIZE); 1709 tlb_page_demap(kernel_pmap, vdst); 1710 PMAP_UNLOCK(kernel_pmap); 1711 } 1712 } else if (mdst->md.color == -1) { 1713 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1714 PMAP_STATS_INC(pmap_ncopy_page_sc); 1715 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1716 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1717 PAGE_SIZE); 1718 } else { 1719 PMAP_STATS_INC(pmap_ncopy_page_soc); 1720 PMAP_LOCK(kernel_pmap); 1721 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1722 tp = tsb_kvtotte(vsrc); 1723 tp->tte_data = 1724 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1725 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1726 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1727 PAGE_SIZE); 1728 tlb_page_demap(kernel_pmap, vsrc); 1729 PMAP_UNLOCK(kernel_pmap); 1730 } 1731 } else { 1732 PMAP_STATS_INC(pmap_ncopy_page_oc); 1733 PMAP_LOCK(kernel_pmap); 1734 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1735 tp = tsb_kvtotte(vdst); 1736 tp->tte_data = 1737 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1738 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1739 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1740 tp = tsb_kvtotte(vsrc); 1741 tp->tte_data = 1742 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1743 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1744 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1745 tlb_page_demap(kernel_pmap, vdst); 1746 tlb_page_demap(kernel_pmap, vsrc); 1747 PMAP_UNLOCK(kernel_pmap); 1748 } 1749} 1750 1751/* 1752 * Returns true if the pmap's pv is one of the first 1753 * 16 pvs linked to from this page. This count may 1754 * be changed upwards or downwards in the future; it 1755 * is only necessary that true be returned for a small 1756 * subset of pmaps for proper page aging. 1757 */ 1758boolean_t 1759pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1760{ 1761 struct tte *tp; 1762 int loops; 1763 1764 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1765 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1766 return (FALSE); 1767 loops = 0; 1768 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1769 if ((tp->tte_data & TD_PV) == 0) 1770 continue; 1771 if (TTE_GET_PMAP(tp) == pm) 1772 return (TRUE); 1773 if (++loops >= 16) 1774 break; 1775 } 1776 return (FALSE); 1777} 1778 1779/* 1780 * Return the number of managed mappings to the given physical page 1781 * that are wired. 1782 */ 1783int 1784pmap_page_wired_mappings(vm_page_t m) 1785{ 1786 struct tte *tp; 1787 int count; 1788 1789 count = 0; 1790 if ((m->flags & PG_FICTITIOUS) != 0) 1791 return (count); 1792 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1793 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1794 if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) 1795 count++; 1796 return (count); 1797} 1798 1799/* 1800 * Remove all pages from specified address space, this aids process exit 1801 * speeds. This is much faster than pmap_remove n the case of running down 1802 * an entire address space. Only works for the current pmap. 1803 */ 1804void 1805pmap_remove_pages(pmap_t pm) 1806{ 1807 1808} 1809 1810/* 1811 * Returns TRUE if the given page has a managed mapping. 1812 */ 1813boolean_t 1814pmap_page_is_mapped(vm_page_t m) 1815{ 1816 struct tte *tp; 1817 1818 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1819 return (FALSE); 1820 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1821 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1822 if ((tp->tte_data & TD_PV) != 0) 1823 return (TRUE); 1824 return (FALSE); 1825} 1826 1827/* 1828 * Return a count of reference bits for a page, clearing those bits. 1829 * It is not necessary for every reference bit to be cleared, but it 1830 * is necessary that 0 only be returned when there are truly no 1831 * reference bits set. 1832 * 1833 * XXX: The exact number of bits to check and clear is a matter that 1834 * should be tested and standardized at some point in the future for 1835 * optimal aging of shared pages. 1836 */ 1837int 1838pmap_ts_referenced(vm_page_t m) 1839{ 1840 struct tte *tpf; 1841 struct tte *tpn; 1842 struct tte *tp; 1843 u_long data; 1844 int count; 1845 1846 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1847 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1848 return (0); 1849 count = 0; 1850 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1851 tpf = tp; 1852 do { 1853 tpn = TAILQ_NEXT(tp, tte_link); 1854 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1855 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1856 if ((tp->tte_data & TD_PV) == 0) 1857 continue; 1858 data = atomic_clear_long(&tp->tte_data, TD_REF); 1859 if ((data & TD_REF) != 0 && ++count > 4) 1860 break; 1861 } while ((tp = tpn) != NULL && tp != tpf); 1862 } 1863 return (count); 1864} 1865 1866boolean_t 1867pmap_is_modified(vm_page_t m) 1868{ 1869 struct tte *tp; 1870 1871 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1872 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1873 return (FALSE); 1874 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1875 if ((tp->tte_data & TD_PV) == 0) 1876 continue; 1877 if ((tp->tte_data & TD_W) != 0) 1878 return (TRUE); 1879 } 1880 return (FALSE); 1881} 1882 1883/* 1884 * pmap_is_prefaultable: 1885 * 1886 * Return whether or not the specified virtual address is elgible 1887 * for prefault. 1888 */ 1889boolean_t 1890pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1891{ 1892 1893 return (FALSE); 1894} 1895 1896void 1897pmap_clear_modify(vm_page_t m) 1898{ 1899 struct tte *tp; 1900 u_long data; 1901 1902 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1903 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1904 return; 1905 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1906 if ((tp->tte_data & TD_PV) == 0) 1907 continue; 1908 data = atomic_clear_long(&tp->tte_data, TD_W); 1909 if ((data & TD_W) != 0) 1910 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1911 } 1912} 1913 1914void 1915pmap_clear_reference(vm_page_t m) 1916{ 1917 struct tte *tp; 1918 u_long data; 1919 1920 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1921 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1922 return; 1923 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1924 if ((tp->tte_data & TD_PV) == 0) 1925 continue; 1926 data = atomic_clear_long(&tp->tte_data, TD_REF); 1927 if ((data & TD_REF) != 0) 1928 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1929 } 1930} 1931 1932void 1933pmap_remove_write(vm_page_t m) 1934{ 1935 struct tte *tp; 1936 u_long data; 1937 1938 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1939 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1940 (m->flags & PG_WRITEABLE) == 0) 1941 return; 1942 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1943 if ((tp->tte_data & TD_PV) == 0) 1944 continue; 1945 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1946 if ((data & TD_W) != 0) { 1947 vm_page_dirty(m); 1948 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1949 } 1950 } 1951 vm_page_flag_clear(m, PG_WRITEABLE); 1952} 1953 1954int 1955pmap_mincore(pmap_t pm, vm_offset_t addr) 1956{ 1957 1958 /* TODO; */ 1959 return (0); 1960} 1961 1962/* 1963 * Activate a user pmap. The pmap must be activated before its address space 1964 * can be accessed in any way. 1965 */ 1966void 1967pmap_activate(struct thread *td) 1968{ 1969 struct vmspace *vm; 1970 struct pmap *pm; 1971 int context; 1972 1973 vm = td->td_proc->p_vmspace; 1974 pm = vmspace_pmap(vm); 1975 1976 mtx_lock_spin(&sched_lock); 1977 1978 context = PCPU_GET(tlb_ctx); 1979 if (context == PCPU_GET(tlb_ctx_max)) { 1980 tlb_flush_user(); 1981 context = PCPU_GET(tlb_ctx_min); 1982 } 1983 PCPU_SET(tlb_ctx, context + 1); 1984 1985 pm->pm_context[curcpu] = context; 1986 pm->pm_active |= PCPU_GET(cpumask); 1987 PCPU_SET(pmap, pm); 1988 1989 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 1990 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 1991 stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & 1992 TLB_PCXR_PGSZ_MASK) | context); 1993 flush(KERNBASE); 1994 1995 mtx_unlock_spin(&sched_lock); 1996} 1997 1998void 1999pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 2000{ 2001 2002} 2003 2004/* 2005 * Increase the starting virtual address of the given mapping if a 2006 * different alignment might result in more superpage mappings. 2007 */ 2008void 2009pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2010 vm_offset_t *addr, vm_size_t size) 2011{ 2012 2013} 2014