pmap.c revision 209048
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 */ 43 44#include <sys/cdefs.h> 45__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/pmap.c 209048 2010-06-11 15:49:39Z alc $"); 46 47/* 48 * Manages physical address maps. 49 * 50 * In addition to hardware address maps, this module is called upon to 51 * provide software-use-only maps which may or may not be stored in the 52 * same form as hardware maps. These pseudo-maps are used to store 53 * intermediate results from copy operations to and from address spaces. 54 * 55 * Since the information managed by this module is also stored by the 56 * logical address mapping module, this module may throw away valid virtual 57 * to physical mappings at almost any time. However, invalidations of 58 * mappings must be done as requested. 59 * 60 * In order to cope with hardware architectures which make virtual to 61 * physical map invalidates expensive, this module may delay invalidate 62 * reduced protection operations until such time as they are actually 63 * necessary. This module is given full information as to which processors 64 * are currently using which maps, and to when physical maps must be made 65 * correct. 66 */ 67 68#include "opt_kstack_pages.h" 69#include "opt_msgbuf.h" 70#include "opt_pmap.h" 71 72#include <sys/param.h> 73#include <sys/kernel.h> 74#include <sys/ktr.h> 75#include <sys/lock.h> 76#include <sys/msgbuf.h> 77#include <sys/mutex.h> 78#include <sys/proc.h> 79#include <sys/smp.h> 80#include <sys/sysctl.h> 81#include <sys/systm.h> 82#include <sys/vmmeter.h> 83 84#include <dev/ofw/openfirm.h> 85 86#include <vm/vm.h> 87#include <vm/vm_param.h> 88#include <vm/vm_kern.h> 89#include <vm/vm_page.h> 90#include <vm/vm_map.h> 91#include <vm/vm_object.h> 92#include <vm/vm_extern.h> 93#include <vm/vm_pageout.h> 94#include <vm/vm_pager.h> 95 96#include <machine/cache.h> 97#include <machine/frame.h> 98#include <machine/instr.h> 99#include <machine/md_var.h> 100#include <machine/metadata.h> 101#include <machine/ofw_mem.h> 102#include <machine/smp.h> 103#include <machine/tlb.h> 104#include <machine/tte.h> 105#include <machine/tsb.h> 106#include <machine/ver.h> 107 108#define PMAP_DEBUG 109 110#ifndef PMAP_SHPGPERPROC 111#define PMAP_SHPGPERPROC 200 112#endif 113 114/* XXX */ 115#include "opt_sched.h" 116#ifndef SCHED_4BSD 117#error "sparc64 only works with SCHED_4BSD which uses a global scheduler lock." 118#endif 119extern struct mtx sched_lock; 120 121/* 122 * Virtual address of message buffer 123 */ 124struct msgbuf *msgbufp; 125 126/* 127 * Map of physical memory reagions 128 */ 129vm_paddr_t phys_avail[128]; 130static struct ofw_mem_region mra[128]; 131struct ofw_mem_region sparc64_memreg[128]; 132int sparc64_nmemreg; 133static struct ofw_map translations[128]; 134static int translations_size; 135 136static vm_offset_t pmap_idle_map; 137static vm_offset_t pmap_temp_map_1; 138static vm_offset_t pmap_temp_map_2; 139 140/* 141 * First and last available kernel virtual addresses 142 */ 143vm_offset_t virtual_avail; 144vm_offset_t virtual_end; 145vm_offset_t kernel_vm_end; 146 147vm_offset_t vm_max_kernel_address; 148 149/* 150 * Kernel pmap 151 */ 152struct pmap kernel_pmap_store; 153 154/* 155 * Allocate physical memory for use in pmap_bootstrap. 156 */ 157static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size); 158 159/* 160 * Map the given physical page at the specified virtual address in the 161 * target pmap with the protection requested. If specified the page 162 * will be wired down. 163 * 164 * The page queues and pmap must be locked. 165 */ 166static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 167 vm_prot_t prot, boolean_t wired); 168 169extern int tl1_immu_miss_patch_1[]; 170extern int tl1_immu_miss_patch_2[]; 171extern int tl1_dmmu_miss_patch_1[]; 172extern int tl1_dmmu_miss_patch_2[]; 173extern int tl1_dmmu_prot_patch_1[]; 174extern int tl1_dmmu_prot_patch_2[]; 175 176/* 177 * If user pmap is processed with pmap_remove and with pmap_remove and the 178 * resident count drops to 0, there are no more pages to remove, so we 179 * need not continue. 180 */ 181#define PMAP_REMOVE_DONE(pm) \ 182 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 183 184/* 185 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 186 * and pmap_protect() instead of trying each virtual address. 187 */ 188#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 189 190SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 191 192PMAP_STATS_VAR(pmap_nenter); 193PMAP_STATS_VAR(pmap_nenter_update); 194PMAP_STATS_VAR(pmap_nenter_replace); 195PMAP_STATS_VAR(pmap_nenter_new); 196PMAP_STATS_VAR(pmap_nkenter); 197PMAP_STATS_VAR(pmap_nkenter_oc); 198PMAP_STATS_VAR(pmap_nkenter_stupid); 199PMAP_STATS_VAR(pmap_nkremove); 200PMAP_STATS_VAR(pmap_nqenter); 201PMAP_STATS_VAR(pmap_nqremove); 202PMAP_STATS_VAR(pmap_ncache_enter); 203PMAP_STATS_VAR(pmap_ncache_enter_c); 204PMAP_STATS_VAR(pmap_ncache_enter_oc); 205PMAP_STATS_VAR(pmap_ncache_enter_cc); 206PMAP_STATS_VAR(pmap_ncache_enter_coc); 207PMAP_STATS_VAR(pmap_ncache_enter_nc); 208PMAP_STATS_VAR(pmap_ncache_enter_cnc); 209PMAP_STATS_VAR(pmap_ncache_remove); 210PMAP_STATS_VAR(pmap_ncache_remove_c); 211PMAP_STATS_VAR(pmap_ncache_remove_oc); 212PMAP_STATS_VAR(pmap_ncache_remove_cc); 213PMAP_STATS_VAR(pmap_ncache_remove_coc); 214PMAP_STATS_VAR(pmap_ncache_remove_nc); 215PMAP_STATS_VAR(pmap_nzero_page); 216PMAP_STATS_VAR(pmap_nzero_page_c); 217PMAP_STATS_VAR(pmap_nzero_page_oc); 218PMAP_STATS_VAR(pmap_nzero_page_nc); 219PMAP_STATS_VAR(pmap_nzero_page_area); 220PMAP_STATS_VAR(pmap_nzero_page_area_c); 221PMAP_STATS_VAR(pmap_nzero_page_area_oc); 222PMAP_STATS_VAR(pmap_nzero_page_area_nc); 223PMAP_STATS_VAR(pmap_nzero_page_idle); 224PMAP_STATS_VAR(pmap_nzero_page_idle_c); 225PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 226PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 227PMAP_STATS_VAR(pmap_ncopy_page); 228PMAP_STATS_VAR(pmap_ncopy_page_c); 229PMAP_STATS_VAR(pmap_ncopy_page_oc); 230PMAP_STATS_VAR(pmap_ncopy_page_nc); 231PMAP_STATS_VAR(pmap_ncopy_page_dc); 232PMAP_STATS_VAR(pmap_ncopy_page_doc); 233PMAP_STATS_VAR(pmap_ncopy_page_sc); 234PMAP_STATS_VAR(pmap_ncopy_page_soc); 235 236PMAP_STATS_VAR(pmap_nnew_thread); 237PMAP_STATS_VAR(pmap_nnew_thread_oc); 238 239static inline u_long dtlb_get_data(u_int slot); 240 241/* 242 * Quick sort callout for comparing memory regions 243 */ 244static int mr_cmp(const void *a, const void *b); 245static int om_cmp(const void *a, const void *b); 246 247static int 248mr_cmp(const void *a, const void *b) 249{ 250 const struct ofw_mem_region *mra; 251 const struct ofw_mem_region *mrb; 252 253 mra = a; 254 mrb = b; 255 if (mra->mr_start < mrb->mr_start) 256 return (-1); 257 else if (mra->mr_start > mrb->mr_start) 258 return (1); 259 else 260 return (0); 261} 262 263static int 264om_cmp(const void *a, const void *b) 265{ 266 const struct ofw_map *oma; 267 const struct ofw_map *omb; 268 269 oma = a; 270 omb = b; 271 if (oma->om_start < omb->om_start) 272 return (-1); 273 else if (oma->om_start > omb->om_start) 274 return (1); 275 else 276 return (0); 277} 278 279static inline u_long 280dtlb_get_data(u_int slot) 281{ 282 283 /* 284 * We read ASI_DTLB_DATA_ACCESS_REG twice in order to work 285 * around errata of USIII and beyond. 286 */ 287 (void)ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG); 288 return (ldxa(TLB_DAR_SLOT(slot), ASI_DTLB_DATA_ACCESS_REG)); 289} 290 291/* 292 * Bootstrap the system enough to run with virtual memory. 293 */ 294void 295pmap_bootstrap(u_int cpu_impl) 296{ 297 struct pmap *pm; 298 struct tte *tp; 299 vm_offset_t off; 300 vm_offset_t va; 301 vm_paddr_t pa; 302 vm_size_t physsz; 303 vm_size_t virtsz; 304 u_long data; 305 phandle_t pmem; 306 phandle_t vmem; 307 u_int dtlb_slots_avail; 308 int i; 309 int j; 310 int sz; 311 312 /* 313 * Find out what physical memory is available from the PROM and 314 * initialize the phys_avail array. This must be done before 315 * pmap_bootstrap_alloc is called. 316 */ 317 if ((pmem = OF_finddevice("/memory")) == -1) 318 panic("pmap_bootstrap: finddevice /memory"); 319 if ((sz = OF_getproplen(pmem, "available")) == -1) 320 panic("pmap_bootstrap: getproplen /memory/available"); 321 if (sizeof(phys_avail) < sz) 322 panic("pmap_bootstrap: phys_avail too small"); 323 if (sizeof(mra) < sz) 324 panic("pmap_bootstrap: mra too small"); 325 bzero(mra, sz); 326 if (OF_getprop(pmem, "available", mra, sz) == -1) 327 panic("pmap_bootstrap: getprop /memory/available"); 328 sz /= sizeof(*mra); 329 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 330 qsort(mra, sz, sizeof (*mra), mr_cmp); 331 physsz = 0; 332 getenv_quad("hw.physmem", &physmem); 333 physmem = btoc(physmem); 334 for (i = 0, j = 0; i < sz; i++, j += 2) { 335 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 336 mra[i].mr_size); 337 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 338 if (btoc(physsz) < physmem) { 339 phys_avail[j] = mra[i].mr_start; 340 phys_avail[j + 1] = mra[i].mr_start + 341 (ctob(physmem) - physsz); 342 physsz = ctob(physmem); 343 } 344 break; 345 } 346 phys_avail[j] = mra[i].mr_start; 347 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 348 physsz += mra[i].mr_size; 349 } 350 physmem = btoc(physsz); 351 352 /* 353 * Calculate the size of kernel virtual memory, and the size and mask 354 * for the kernel TSB based on the phsyical memory size but limited 355 * by the amount of dTLB slots available for locked entries (given 356 * that for spitfire-class CPUs all of the dt64 slots can hold locked 357 * entries but there is no large dTLB for unlocked ones, we don't use 358 * more than half of it for locked entries). 359 */ 360 dtlb_slots_avail = 0; 361 for (i = 0; i < dtlb_slots; i++) { 362 data = dtlb_get_data(i); 363 if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) 364 dtlb_slots_avail++; 365 } 366#ifdef SMP 367 dtlb_slots_avail -= PCPU_PAGES; 368#endif 369 if (cpu_impl >= CPU_IMPL_ULTRASPARCI && 370 cpu_impl < CPU_IMPL_ULTRASPARCIII) 371 dtlb_slots_avail /= 2; 372 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 373 virtsz = MIN(virtsz, 374 (dtlb_slots_avail * PAGE_SIZE_4M) << (PAGE_SHIFT - TTE_SHIFT)); 375 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 376 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 377 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 378 379 /* 380 * Allocate the kernel TSB and lock it in the TLB. 381 */ 382 pa = pmap_bootstrap_alloc(tsb_kernel_size); 383 if (pa & PAGE_MASK_4M) 384 panic("pmap_bootstrap: tsb unaligned\n"); 385 tsb_kernel_phys = pa; 386 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 387 pmap_map_tsb(); 388 bzero(tsb_kernel, tsb_kernel_size); 389 390 /* 391 * Allocate and map the dynamic per-CPU area for the BSP. 392 */ 393 pa = pmap_bootstrap_alloc(DPCPU_SIZE); 394 dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); 395 396 /* 397 * Allocate and map the message buffer. 398 */ 399 pa = pmap_bootstrap_alloc(MSGBUF_SIZE); 400 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); 401 402 /* 403 * Patch the virtual address and the tsb mask into the trap table. 404 */ 405 406#define SETHI(rd, imm22) \ 407 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 408 EIF_IMM((imm22) >> 10, 22)) 409#define OR_R_I_R(rd, imm13, rs1) \ 410 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 411 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 412 413#define PATCH(addr) do { \ 414 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 415 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 416 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 417 panic("pmap_boostrap: patched instructions have changed"); \ 418 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 419 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 420 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 421 flush(addr); \ 422 flush(addr + 1); \ 423 flush(addr + 2); \ 424} while (0) 425 426 PATCH(tl1_immu_miss_patch_1); 427 PATCH(tl1_immu_miss_patch_2); 428 PATCH(tl1_dmmu_miss_patch_1); 429 PATCH(tl1_dmmu_miss_patch_2); 430 PATCH(tl1_dmmu_prot_patch_1); 431 PATCH(tl1_dmmu_prot_patch_2); 432 433 /* 434 * Enter fake 8k pages for the 4MB kernel pages, so that 435 * pmap_kextract() will work for them. 436 */ 437 for (i = 0; i < kernel_tlb_slots; i++) { 438 pa = kernel_tlbs[i].te_pa; 439 va = kernel_tlbs[i].te_va; 440 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 441 tp = tsb_kvtotte(va + off); 442 tp->tte_vpn = TV_VPN(va + off, TS_8K); 443 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 444 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 445 } 446 } 447 448 /* 449 * Set the start and end of KVA. The kernel is loaded starting 450 * at the first available 4MB super page, so we advance to the 451 * end of the last one used for it. 452 */ 453 virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; 454 virtual_end = vm_max_kernel_address; 455 kernel_vm_end = vm_max_kernel_address; 456 457 /* 458 * Allocate kva space for temporary mappings. 459 */ 460 pmap_idle_map = virtual_avail; 461 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 462 pmap_temp_map_1 = virtual_avail; 463 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 464 pmap_temp_map_2 = virtual_avail; 465 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 466 467 /* 468 * Allocate a kernel stack with guard page for thread0 and map it 469 * into the kernel TSB. We must ensure that the virtual address is 470 * coloured properly, since we're allocating from phys_avail so the 471 * memory won't have an associated vm_page_t. 472 */ 473 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE); 474 kstack0_phys = pa; 475 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 476 PAGE_SIZE; 477 kstack0 = virtual_avail; 478 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 479 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 480 ("pmap_bootstrap: kstack0 miscoloured")); 481 for (i = 0; i < KSTACK_PAGES; i++) { 482 pa = kstack0_phys + i * PAGE_SIZE; 483 va = kstack0 + i * PAGE_SIZE; 484 tp = tsb_kvtotte(va); 485 tp->tte_vpn = TV_VPN(va, TS_8K); 486 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 487 TD_CP | TD_CV | TD_P | TD_W; 488 } 489 490 /* 491 * Calculate the last available physical address. 492 */ 493 for (i = 0; phys_avail[i + 2] != 0; i += 2) 494 ; 495 Maxmem = sparc64_btop(phys_avail[i + 1]); 496 497 /* 498 * Add the PROM mappings to the kernel TSB. 499 */ 500 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 501 panic("pmap_bootstrap: finddevice /virtual-memory"); 502 if ((sz = OF_getproplen(vmem, "translations")) == -1) 503 panic("pmap_bootstrap: getproplen translations"); 504 if (sizeof(translations) < sz) 505 panic("pmap_bootstrap: translations too small"); 506 bzero(translations, sz); 507 if (OF_getprop(vmem, "translations", translations, sz) == -1) 508 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 509 sz /= sizeof(*translations); 510 translations_size = sz; 511 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 512 qsort(translations, sz, sizeof (*translations), om_cmp); 513 for (i = 0; i < sz; i++) { 514 CTR3(KTR_PMAP, 515 "translation: start=%#lx size=%#lx tte=%#lx", 516 translations[i].om_start, translations[i].om_size, 517 translations[i].om_tte); 518 if ((translations[i].om_tte & TD_V) == 0) 519 continue; 520 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 521 translations[i].om_start > VM_MAX_PROM_ADDRESS) 522 continue; 523 for (off = 0; off < translations[i].om_size; 524 off += PAGE_SIZE) { 525 va = translations[i].om_start + off; 526 tp = tsb_kvtotte(va); 527 tp->tte_vpn = TV_VPN(va, TS_8K); 528 tp->tte_data = 529 ((translations[i].om_tte & 530 ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | 531 (cpu_impl >= CPU_IMPL_ULTRASPARCI && 532 cpu_impl < CPU_IMPL_ULTRASPARCIII ? 533 (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : 534 (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | 535 (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + 536 off; 537 } 538 } 539 540 /* 541 * Get the available physical memory ranges from /memory/reg. These 542 * are only used for kernel dumps, but it may not be wise to do PROM 543 * calls in that situation. 544 */ 545 if ((sz = OF_getproplen(pmem, "reg")) == -1) 546 panic("pmap_bootstrap: getproplen /memory/reg"); 547 if (sizeof(sparc64_memreg) < sz) 548 panic("pmap_bootstrap: sparc64_memreg too small"); 549 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 550 panic("pmap_bootstrap: getprop /memory/reg"); 551 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 552 553 /* 554 * Initialize the kernel pmap (which is statically allocated). 555 * NOTE: PMAP_LOCK_INIT() is needed as part of the initialization 556 * but sparc64 start up is not ready to initialize mutexes yet. 557 * It is called in machdep.c. 558 */ 559 pm = kernel_pmap; 560 for (i = 0; i < MAXCPU; i++) 561 pm->pm_context[i] = TLB_CTX_KERNEL; 562 pm->pm_active = ~0; 563 564 /* 565 * Flush all non-locked TLB entries possibly left over by the 566 * firmware. 567 */ 568 tlb_flush_nonlocked(); 569} 570 571void 572pmap_map_tsb(void) 573{ 574 vm_offset_t va; 575 vm_paddr_t pa; 576 u_long data; 577 register_t s; 578 int i; 579 580 s = intr_disable(); 581 582 /* 583 * Map the 4MB TSB pages. 584 */ 585 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 586 va = (vm_offset_t)tsb_kernel + i; 587 pa = tsb_kernel_phys + i; 588 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 589 TD_P | TD_W; 590 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 591 TLB_TAR_CTX(TLB_CTX_KERNEL)); 592 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 593 } 594 595 /* 596 * Set the secondary context to be the kernel context (needed for 597 * FP block operations in the kernel). 598 */ 599 stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & 600 TLB_CXR_PGSZ_MASK) | TLB_CTX_KERNEL); 601 flush(KERNBASE); 602 603 intr_restore(s); 604} 605 606/* 607 * Allocate a physical page of memory directly from the phys_avail map. 608 * Can only be called from pmap_bootstrap before avail start and end are 609 * calculated. 610 */ 611static vm_paddr_t 612pmap_bootstrap_alloc(vm_size_t size) 613{ 614 vm_paddr_t pa; 615 int i; 616 617 size = roundup(size, PAGE_SIZE * DCACHE_COLORS); 618 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 619 if (phys_avail[i + 1] - phys_avail[i] < size) 620 continue; 621 pa = phys_avail[i]; 622 phys_avail[i] += size; 623 return (pa); 624 } 625 panic("pmap_bootstrap_alloc"); 626} 627 628/* 629 * Initialize a vm_page's machine-dependent fields. 630 */ 631void 632pmap_page_init(vm_page_t m) 633{ 634 635 TAILQ_INIT(&m->md.tte_list); 636 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 637 m->md.flags = 0; 638 m->md.pmap = NULL; 639} 640 641/* 642 * Initialize the pmap module. 643 */ 644void 645pmap_init(void) 646{ 647 vm_offset_t addr; 648 vm_size_t size; 649 int result; 650 int i; 651 652 for (i = 0; i < translations_size; i++) { 653 addr = translations[i].om_start; 654 size = translations[i].om_size; 655 if ((translations[i].om_tte & TD_V) == 0) 656 continue; 657 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 658 continue; 659 result = vm_map_find(kernel_map, NULL, 0, &addr, size, 660 VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 661 if (result != KERN_SUCCESS || addr != translations[i].om_start) 662 panic("pmap_init: vm_map_find"); 663 } 664} 665 666/* 667 * Extract the physical page address associated with the given 668 * map/virtual_address pair. 669 */ 670vm_paddr_t 671pmap_extract(pmap_t pm, vm_offset_t va) 672{ 673 struct tte *tp; 674 vm_paddr_t pa; 675 676 if (pm == kernel_pmap) 677 return (pmap_kextract(va)); 678 PMAP_LOCK(pm); 679 tp = tsb_tte_lookup(pm, va); 680 if (tp == NULL) 681 pa = 0; 682 else 683 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 684 PMAP_UNLOCK(pm); 685 return (pa); 686} 687 688/* 689 * Atomically extract and hold the physical page with the given 690 * pmap and virtual address pair if that mapping permits the given 691 * protection. 692 */ 693vm_page_t 694pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 695{ 696 struct tte *tp; 697 vm_page_t m; 698 vm_paddr_t pa; 699 700 m = NULL; 701 pa = 0; 702 PMAP_LOCK(pm); 703retry: 704 if (pm == kernel_pmap) { 705 if (va >= VM_MIN_DIRECT_ADDRESS) { 706 tp = NULL; 707 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 708 (void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va), 709 &pa); 710 vm_page_hold(m); 711 } else { 712 tp = tsb_kvtotte(va); 713 if ((tp->tte_data & TD_V) == 0) 714 tp = NULL; 715 } 716 } else 717 tp = tsb_tte_lookup(pm, va); 718 if (tp != NULL && ((tp->tte_data & TD_SW) || 719 (prot & VM_PROT_WRITE) == 0)) { 720 if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa)) 721 goto retry; 722 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 723 vm_page_hold(m); 724 } 725 PA_UNLOCK_COND(pa); 726 PMAP_UNLOCK(pm); 727 return (m); 728} 729 730/* 731 * Extract the physical page address associated with the given kernel virtual 732 * address. 733 */ 734vm_paddr_t 735pmap_kextract(vm_offset_t va) 736{ 737 struct tte *tp; 738 739 if (va >= VM_MIN_DIRECT_ADDRESS) 740 return (TLB_DIRECT_TO_PHYS(va)); 741 tp = tsb_kvtotte(va); 742 if ((tp->tte_data & TD_V) == 0) 743 return (0); 744 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 745} 746 747int 748pmap_cache_enter(vm_page_t m, vm_offset_t va) 749{ 750 struct tte *tp; 751 int color; 752 753 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 754 KASSERT((m->flags & PG_FICTITIOUS) == 0, 755 ("pmap_cache_enter: fake page")); 756 PMAP_STATS_INC(pmap_ncache_enter); 757 758 /* 759 * Find the color for this virtual address and note the added mapping. 760 */ 761 color = DCACHE_COLOR(va); 762 m->md.colors[color]++; 763 764 /* 765 * If all existing mappings have the same color, the mapping is 766 * cacheable. 767 */ 768 if (m->md.color == color) { 769 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 770 ("pmap_cache_enter: cacheable, mappings of other color")); 771 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 772 PMAP_STATS_INC(pmap_ncache_enter_c); 773 else 774 PMAP_STATS_INC(pmap_ncache_enter_oc); 775 return (1); 776 } 777 778 /* 779 * If there are no mappings of the other color, and the page still has 780 * the wrong color, this must be a new mapping. Change the color to 781 * match the new mapping, which is cacheable. We must flush the page 782 * from the cache now. 783 */ 784 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 785 KASSERT(m->md.colors[color] == 1, 786 ("pmap_cache_enter: changing color, not new mapping")); 787 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 788 m->md.color = color; 789 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 790 PMAP_STATS_INC(pmap_ncache_enter_cc); 791 else 792 PMAP_STATS_INC(pmap_ncache_enter_coc); 793 return (1); 794 } 795 796 /* 797 * If the mapping is already non-cacheable, just return. 798 */ 799 if (m->md.color == -1) { 800 PMAP_STATS_INC(pmap_ncache_enter_nc); 801 return (0); 802 } 803 804 PMAP_STATS_INC(pmap_ncache_enter_cnc); 805 806 /* 807 * Mark all mappings as uncacheable, flush any lines with the other 808 * color out of the dcache, and set the color to none (-1). 809 */ 810 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 811 atomic_clear_long(&tp->tte_data, TD_CV); 812 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 813 } 814 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 815 m->md.color = -1; 816 return (0); 817} 818 819void 820pmap_cache_remove(vm_page_t m, vm_offset_t va) 821{ 822 struct tte *tp; 823 int color; 824 825 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 826 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 827 m->md.colors[DCACHE_COLOR(va)]); 828 KASSERT((m->flags & PG_FICTITIOUS) == 0, 829 ("pmap_cache_remove: fake page")); 830 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 831 ("pmap_cache_remove: no mappings %d <= 0", 832 m->md.colors[DCACHE_COLOR(va)])); 833 PMAP_STATS_INC(pmap_ncache_remove); 834 835 /* 836 * Find the color for this virtual address and note the removal of 837 * the mapping. 838 */ 839 color = DCACHE_COLOR(va); 840 m->md.colors[color]--; 841 842 /* 843 * If the page is cacheable, just return and keep the same color, even 844 * if there are no longer any mappings. 845 */ 846 if (m->md.color != -1) { 847 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 848 PMAP_STATS_INC(pmap_ncache_remove_c); 849 else 850 PMAP_STATS_INC(pmap_ncache_remove_oc); 851 return; 852 } 853 854 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 855 ("pmap_cache_remove: uncacheable, no mappings of other color")); 856 857 /* 858 * If the page is not cacheable (color is -1), and the number of 859 * mappings for this color is not zero, just return. There are 860 * mappings of the other color still, so remain non-cacheable. 861 */ 862 if (m->md.colors[color] != 0) { 863 PMAP_STATS_INC(pmap_ncache_remove_nc); 864 return; 865 } 866 867 /* 868 * The number of mappings for this color is now zero. Recache the 869 * other colored mappings, and change the page color to the other 870 * color. There should be no lines in the data cache for this page, 871 * so flushing should not be needed. 872 */ 873 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 874 atomic_set_long(&tp->tte_data, TD_CV); 875 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 876 } 877 m->md.color = DCACHE_OTHER_COLOR(color); 878 879 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 880 PMAP_STATS_INC(pmap_ncache_remove_cc); 881 else 882 PMAP_STATS_INC(pmap_ncache_remove_coc); 883} 884 885/* 886 * Map a wired page into kernel virtual address space. 887 */ 888void 889pmap_kenter(vm_offset_t va, vm_page_t m) 890{ 891 vm_offset_t ova; 892 struct tte *tp; 893 vm_page_t om; 894 u_long data; 895 896 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 897 PMAP_STATS_INC(pmap_nkenter); 898 tp = tsb_kvtotte(va); 899 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 900 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 901 if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { 902 CTR5(KTR_CT2, 903 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", 904 va, VM_PAGE_TO_PHYS(m), m->object, 905 m->object ? m->object->type : -1, 906 m->pindex); 907 PMAP_STATS_INC(pmap_nkenter_oc); 908 } 909 if ((tp->tte_data & TD_V) != 0) { 910 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 911 ova = TTE_GET_VA(tp); 912 if (m == om && va == ova) { 913 PMAP_STATS_INC(pmap_nkenter_stupid); 914 return; 915 } 916 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 917 pmap_cache_remove(om, ova); 918 if (va != ova) 919 tlb_page_demap(kernel_pmap, ova); 920 } 921 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 922 TD_P | TD_W; 923 if (pmap_cache_enter(m, va) != 0) 924 data |= TD_CV; 925 tp->tte_vpn = TV_VPN(va, TS_8K); 926 tp->tte_data = data; 927 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 928} 929 930/* 931 * Map a wired page into kernel virtual address space. This additionally 932 * takes a flag argument wich is or'ed to the TTE data. This is used by 933 * sparc64_bus_mem_map(). 934 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 935 * to flush entries that might still be in the cache, if applicable. 936 */ 937void 938pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 939{ 940 struct tte *tp; 941 942 tp = tsb_kvtotte(va); 943 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 944 va, pa, tp, tp->tte_data); 945 tp->tte_vpn = TV_VPN(va, TS_8K); 946 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 947} 948 949/* 950 * Remove a wired page from kernel virtual address space. 951 */ 952void 953pmap_kremove(vm_offset_t va) 954{ 955 struct tte *tp; 956 vm_page_t m; 957 958 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 959 PMAP_STATS_INC(pmap_nkremove); 960 tp = tsb_kvtotte(va); 961 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 962 tp->tte_data); 963 if ((tp->tte_data & TD_V) == 0) 964 return; 965 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 966 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 967 pmap_cache_remove(m, va); 968 TTE_ZERO(tp); 969} 970 971/* 972 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 973 */ 974void 975pmap_kremove_flags(vm_offset_t va) 976{ 977 struct tte *tp; 978 979 tp = tsb_kvtotte(va); 980 CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, 981 tp->tte_data); 982 TTE_ZERO(tp); 983} 984 985/* 986 * Map a range of physical addresses into kernel virtual address space. 987 * 988 * The value passed in *virt is a suggested virtual address for the mapping. 989 * Architectures which can support a direct-mapped physical to virtual region 990 * can return the appropriate address within that region, leaving '*virt' 991 * unchanged. 992 */ 993vm_offset_t 994pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 995{ 996 997 return (TLB_PHYS_TO_DIRECT(start)); 998} 999 1000/* 1001 * Map a list of wired pages into kernel virtual address space. This is 1002 * intended for temporary mappings which do not need page modification or 1003 * references recorded. Existing mappings in the region are overwritten. 1004 */ 1005void 1006pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 1007{ 1008 vm_offset_t va; 1009 int locked; 1010 1011 PMAP_STATS_INC(pmap_nqenter); 1012 va = sva; 1013 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 1014 vm_page_lock_queues(); 1015 while (count-- > 0) { 1016 pmap_kenter(va, *m); 1017 va += PAGE_SIZE; 1018 m++; 1019 } 1020 if (!locked) 1021 vm_page_unlock_queues(); 1022 tlb_range_demap(kernel_pmap, sva, va); 1023} 1024 1025/* 1026 * Remove page mappings from kernel virtual address space. Intended for 1027 * temporary mappings entered by pmap_qenter. 1028 */ 1029void 1030pmap_qremove(vm_offset_t sva, int count) 1031{ 1032 vm_offset_t va; 1033 int locked; 1034 1035 PMAP_STATS_INC(pmap_nqremove); 1036 va = sva; 1037 if (!(locked = mtx_owned(&vm_page_queue_mtx))) 1038 vm_page_lock_queues(); 1039 while (count-- > 0) { 1040 pmap_kremove(va); 1041 va += PAGE_SIZE; 1042 } 1043 if (!locked) 1044 vm_page_unlock_queues(); 1045 tlb_range_demap(kernel_pmap, sva, va); 1046} 1047 1048/* 1049 * Initialize the pmap associated with process 0. 1050 */ 1051void 1052pmap_pinit0(pmap_t pm) 1053{ 1054 int i; 1055 1056 PMAP_LOCK_INIT(pm); 1057 for (i = 0; i < MAXCPU; i++) 1058 pm->pm_context[i] = 0; 1059 pm->pm_active = 0; 1060 pm->pm_tsb = NULL; 1061 pm->pm_tsb_obj = NULL; 1062 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1063} 1064 1065/* 1066 * Initialize a preallocated and zeroed pmap structure, such as one in a 1067 * vmspace structure. 1068 */ 1069int 1070pmap_pinit(pmap_t pm) 1071{ 1072 vm_page_t ma[TSB_PAGES]; 1073 vm_page_t m; 1074 int i; 1075 1076 PMAP_LOCK_INIT(pm); 1077 1078 /* 1079 * Allocate KVA space for the TSB. 1080 */ 1081 if (pm->pm_tsb == NULL) { 1082 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1083 TSB_BSIZE); 1084 if (pm->pm_tsb == NULL) { 1085 PMAP_LOCK_DESTROY(pm); 1086 return (0); 1087 } 1088 } 1089 1090 /* 1091 * Allocate an object for it. 1092 */ 1093 if (pm->pm_tsb_obj == NULL) 1094 pm->pm_tsb_obj = vm_object_allocate(OBJT_PHYS, TSB_PAGES); 1095 1096 VM_OBJECT_LOCK(pm->pm_tsb_obj); 1097 for (i = 0; i < TSB_PAGES; i++) { 1098 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1099 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1100 m->valid = VM_PAGE_BITS_ALL; 1101 m->md.pmap = pm; 1102 ma[i] = m; 1103 } 1104 VM_OBJECT_UNLOCK(pm->pm_tsb_obj); 1105 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1106 1107 for (i = 0; i < MAXCPU; i++) 1108 pm->pm_context[i] = -1; 1109 pm->pm_active = 0; 1110 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1111 return (1); 1112} 1113 1114/* 1115 * Release any resources held by the given physical map. 1116 * Called when a pmap initialized by pmap_pinit is being released. 1117 * Should only be called if the map contains no valid mappings. 1118 */ 1119void 1120pmap_release(pmap_t pm) 1121{ 1122 vm_object_t obj; 1123 vm_page_t m; 1124 struct pcpu *pc; 1125 1126 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1127 pm->pm_context[curcpu], pm->pm_tsb); 1128 KASSERT(pmap_resident_count(pm) == 0, 1129 ("pmap_release: resident pages %ld != 0", 1130 pmap_resident_count(pm))); 1131 1132 /* 1133 * After the pmap was freed, it might be reallocated to a new process. 1134 * When switching, this might lead us to wrongly assume that we need 1135 * not switch contexts because old and new pmap pointer are equal. 1136 * Therefore, make sure that this pmap is not referenced by any PCPU 1137 * pointer any more. This could happen in two cases: 1138 * - A process that referenced the pmap is currently exiting on a CPU. 1139 * However, it is guaranteed to not switch in any more after setting 1140 * its state to PRS_ZOMBIE. 1141 * - A process that referenced this pmap ran on a CPU, but we switched 1142 * to a kernel thread, leaving the pmap pointer unchanged. 1143 */ 1144 mtx_lock_spin(&sched_lock); 1145 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) 1146 if (pc->pc_pmap == pm) 1147 pc->pc_pmap = NULL; 1148 mtx_unlock_spin(&sched_lock); 1149 1150 obj = pm->pm_tsb_obj; 1151 VM_OBJECT_LOCK(obj); 1152 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1153 while (!TAILQ_EMPTY(&obj->memq)) { 1154 m = TAILQ_FIRST(&obj->memq); 1155 m->md.pmap = NULL; 1156 m->wire_count--; 1157 atomic_subtract_int(&cnt.v_wire_count, 1); 1158 vm_page_free_zero(m); 1159 } 1160 VM_OBJECT_UNLOCK(obj); 1161 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1162 PMAP_LOCK_DESTROY(pm); 1163} 1164 1165/* 1166 * Grow the number of kernel page table entries. Unneeded. 1167 */ 1168void 1169pmap_growkernel(vm_offset_t addr) 1170{ 1171 1172 panic("pmap_growkernel: can't grow kernel"); 1173} 1174 1175int 1176pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1177 vm_offset_t va) 1178{ 1179 vm_page_t m; 1180 u_long data; 1181 1182 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1183 data = atomic_readandclear_long(&tp->tte_data); 1184 if ((data & TD_FAKE) == 0) { 1185 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1186 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1187 if ((data & TD_WIRED) != 0) 1188 pm->pm_stats.wired_count--; 1189 if ((data & TD_PV) != 0) { 1190 if ((data & TD_W) != 0) 1191 vm_page_dirty(m); 1192 if ((data & TD_REF) != 0) 1193 vm_page_flag_set(m, PG_REFERENCED); 1194 if (TAILQ_EMPTY(&m->md.tte_list)) 1195 vm_page_flag_clear(m, PG_WRITEABLE); 1196 pm->pm_stats.resident_count--; 1197 } 1198 pmap_cache_remove(m, va); 1199 } 1200 TTE_ZERO(tp); 1201 if (PMAP_REMOVE_DONE(pm)) 1202 return (0); 1203 return (1); 1204} 1205 1206/* 1207 * Remove the given range of addresses from the specified map. 1208 */ 1209void 1210pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1211{ 1212 struct tte *tp; 1213 vm_offset_t va; 1214 1215 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1216 pm->pm_context[curcpu], start, end); 1217 if (PMAP_REMOVE_DONE(pm)) 1218 return; 1219 vm_page_lock_queues(); 1220 PMAP_LOCK(pm); 1221 if (end - start > PMAP_TSB_THRESH) { 1222 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1223 tlb_context_demap(pm); 1224 } else { 1225 for (va = start; va < end; va += PAGE_SIZE) 1226 if ((tp = tsb_tte_lookup(pm, va)) != NULL && 1227 !pmap_remove_tte(pm, NULL, tp, va)) 1228 break; 1229 tlb_range_demap(pm, start, end - 1); 1230 } 1231 PMAP_UNLOCK(pm); 1232 vm_page_unlock_queues(); 1233} 1234 1235void 1236pmap_remove_all(vm_page_t m) 1237{ 1238 struct pmap *pm; 1239 struct tte *tpn; 1240 struct tte *tp; 1241 vm_offset_t va; 1242 1243 vm_page_lock_queues(); 1244 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1245 tpn = TAILQ_NEXT(tp, tte_link); 1246 if ((tp->tte_data & TD_PV) == 0) 1247 continue; 1248 pm = TTE_GET_PMAP(tp); 1249 va = TTE_GET_VA(tp); 1250 PMAP_LOCK(pm); 1251 if ((tp->tte_data & TD_WIRED) != 0) 1252 pm->pm_stats.wired_count--; 1253 if ((tp->tte_data & TD_REF) != 0) 1254 vm_page_flag_set(m, PG_REFERENCED); 1255 if ((tp->tte_data & TD_W) != 0) 1256 vm_page_dirty(m); 1257 tp->tte_data &= ~TD_V; 1258 tlb_page_demap(pm, va); 1259 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1260 pm->pm_stats.resident_count--; 1261 pmap_cache_remove(m, va); 1262 TTE_ZERO(tp); 1263 PMAP_UNLOCK(pm); 1264 } 1265 vm_page_flag_clear(m, PG_WRITEABLE); 1266 vm_page_unlock_queues(); 1267} 1268 1269int 1270pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1271 vm_offset_t va) 1272{ 1273 u_long data; 1274 vm_page_t m; 1275 1276 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1277 if ((data & (TD_PV | TD_W)) == (TD_PV | TD_W)) { 1278 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1279 vm_page_dirty(m); 1280 } 1281 return (1); 1282} 1283 1284/* 1285 * Set the physical protection on the specified range of this map as requested. 1286 */ 1287void 1288pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1289{ 1290 vm_offset_t va; 1291 struct tte *tp; 1292 1293 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1294 pm->pm_context[curcpu], sva, eva, prot); 1295 1296 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1297 pmap_remove(pm, sva, eva); 1298 return; 1299 } 1300 1301 if (prot & VM_PROT_WRITE) 1302 return; 1303 1304 vm_page_lock_queues(); 1305 PMAP_LOCK(pm); 1306 if (eva - sva > PMAP_TSB_THRESH) { 1307 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1308 tlb_context_demap(pm); 1309 } else { 1310 for (va = sva; va < eva; va += PAGE_SIZE) 1311 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1312 pmap_protect_tte(pm, NULL, tp, va); 1313 tlb_range_demap(pm, sva, eva - 1); 1314 } 1315 PMAP_UNLOCK(pm); 1316 vm_page_unlock_queues(); 1317} 1318 1319/* 1320 * Map the given physical page at the specified virtual address in the 1321 * target pmap with the protection requested. If specified the page 1322 * will be wired down. 1323 */ 1324void 1325pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, 1326 vm_prot_t prot, boolean_t wired) 1327{ 1328 1329 vm_page_lock_queues(); 1330 PMAP_LOCK(pm); 1331 pmap_enter_locked(pm, va, m, prot, wired); 1332 vm_page_unlock_queues(); 1333 PMAP_UNLOCK(pm); 1334} 1335 1336/* 1337 * Map the given physical page at the specified virtual address in the 1338 * target pmap with the protection requested. If specified the page 1339 * will be wired down. 1340 * 1341 * The page queues and pmap must be locked. 1342 */ 1343static void 1344pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1345 boolean_t wired) 1346{ 1347 struct tte *tp; 1348 vm_paddr_t pa; 1349 u_long data; 1350 int i; 1351 1352 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1353 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1354 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1355 (m->oflags & VPO_BUSY) != 0 || VM_OBJECT_LOCKED(m->object), 1356 ("pmap_enter_locked: page %p is not busy", m)); 1357 PMAP_STATS_INC(pmap_nenter); 1358 pa = VM_PAGE_TO_PHYS(m); 1359 1360 /* 1361 * If this is a fake page from the device_pager, but it covers actual 1362 * physical memory, convert to the real backing page. 1363 */ 1364 if ((m->flags & PG_FICTITIOUS) != 0) { 1365 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1366 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1367 m = PHYS_TO_VM_PAGE(pa); 1368 break; 1369 } 1370 } 1371 } 1372 1373 CTR6(KTR_PMAP, 1374 "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1375 pm->pm_context[curcpu], m, va, pa, prot, wired); 1376 1377 /* 1378 * If there is an existing mapping, and the physical address has not 1379 * changed, must be protection or wiring change. 1380 */ 1381 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1382 CTR0(KTR_PMAP, "pmap_enter_locked: update"); 1383 PMAP_STATS_INC(pmap_nenter_update); 1384 1385 /* 1386 * Wiring change, just update stats. 1387 */ 1388 if (wired) { 1389 if ((tp->tte_data & TD_WIRED) == 0) { 1390 tp->tte_data |= TD_WIRED; 1391 pm->pm_stats.wired_count++; 1392 } 1393 } else { 1394 if ((tp->tte_data & TD_WIRED) != 0) { 1395 tp->tte_data &= ~TD_WIRED; 1396 pm->pm_stats.wired_count--; 1397 } 1398 } 1399 1400 /* 1401 * Save the old bits and clear the ones we're interested in. 1402 */ 1403 data = tp->tte_data; 1404 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1405 1406 /* 1407 * If we're turning off write permissions, sense modify status. 1408 */ 1409 if ((prot & VM_PROT_WRITE) != 0) { 1410 tp->tte_data |= TD_SW; 1411 if (wired) 1412 tp->tte_data |= TD_W; 1413 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) 1414 vm_page_flag_set(m, PG_WRITEABLE); 1415 } else if ((data & TD_W) != 0) 1416 vm_page_dirty(m); 1417 1418 /* 1419 * If we're turning on execute permissions, flush the icache. 1420 */ 1421 if ((prot & VM_PROT_EXECUTE) != 0) { 1422 if ((data & TD_EXEC) == 0) 1423 icache_page_inval(pa); 1424 tp->tte_data |= TD_EXEC; 1425 } 1426 1427 /* 1428 * Delete the old mapping. 1429 */ 1430 tlb_page_demap(pm, TTE_GET_VA(tp)); 1431 } else { 1432 /* 1433 * If there is an existing mapping, but its for a different 1434 * physical address, delete the old mapping. 1435 */ 1436 if (tp != NULL) { 1437 CTR0(KTR_PMAP, "pmap_enter_locked: replace"); 1438 PMAP_STATS_INC(pmap_nenter_replace); 1439 pmap_remove_tte(pm, NULL, tp, va); 1440 tlb_page_demap(pm, va); 1441 } else { 1442 CTR0(KTR_PMAP, "pmap_enter_locked: new"); 1443 PMAP_STATS_INC(pmap_nenter_new); 1444 } 1445 1446 /* 1447 * Now set up the data and install the new mapping. 1448 */ 1449 data = TD_V | TD_8K | TD_PA(pa); 1450 if (pm == kernel_pmap) 1451 data |= TD_P; 1452 if ((prot & VM_PROT_WRITE) != 0) { 1453 data |= TD_SW; 1454 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0) 1455 vm_page_flag_set(m, PG_WRITEABLE); 1456 } 1457 if (prot & VM_PROT_EXECUTE) { 1458 data |= TD_EXEC; 1459 icache_page_inval(pa); 1460 } 1461 1462 /* 1463 * If its wired update stats. We also don't need reference or 1464 * modify tracking for wired mappings, so set the bits now. 1465 */ 1466 if (wired) { 1467 pm->pm_stats.wired_count++; 1468 data |= TD_REF | TD_WIRED; 1469 if ((prot & VM_PROT_WRITE) != 0) 1470 data |= TD_W; 1471 } 1472 1473 tsb_tte_enter(pm, m, va, TS_8K, data); 1474 } 1475} 1476 1477/* 1478 * Maps a sequence of resident pages belonging to the same object. 1479 * The sequence begins with the given page m_start. This page is 1480 * mapped at the given virtual address start. Each subsequent page is 1481 * mapped at a virtual address that is offset from start by the same 1482 * amount as the page is offset from m_start within the object. The 1483 * last page in the sequence is the page with the largest offset from 1484 * m_start that can be mapped at a virtual address less than the given 1485 * virtual address end. Not every virtual page between start and end 1486 * is mapped; only those for which a resident page exists with the 1487 * corresponding offset from m_start are mapped. 1488 */ 1489void 1490pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1491 vm_page_t m_start, vm_prot_t prot) 1492{ 1493 vm_page_t m; 1494 vm_pindex_t diff, psize; 1495 1496 psize = atop(end - start); 1497 m = m_start; 1498 vm_page_lock_queues(); 1499 PMAP_LOCK(pm); 1500 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1501 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1502 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1503 m = TAILQ_NEXT(m, listq); 1504 } 1505 vm_page_unlock_queues(); 1506 PMAP_UNLOCK(pm); 1507} 1508 1509void 1510pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1511{ 1512 1513 vm_page_lock_queues(); 1514 PMAP_LOCK(pm); 1515 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1516 FALSE); 1517 vm_page_unlock_queues(); 1518 PMAP_UNLOCK(pm); 1519} 1520 1521void 1522pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1523 vm_pindex_t pindex, vm_size_t size) 1524{ 1525 1526 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1527 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 1528 ("pmap_object_init_pt: non-device object")); 1529} 1530 1531/* 1532 * Change the wiring attribute for a map/virtual-address pair. 1533 * The mapping must already exist in the pmap. 1534 */ 1535void 1536pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1537{ 1538 struct tte *tp; 1539 u_long data; 1540 1541 PMAP_LOCK(pm); 1542 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1543 if (wired) { 1544 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1545 if ((data & TD_WIRED) == 0) 1546 pm->pm_stats.wired_count++; 1547 } else { 1548 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1549 if ((data & TD_WIRED) != 0) 1550 pm->pm_stats.wired_count--; 1551 } 1552 } 1553 PMAP_UNLOCK(pm); 1554} 1555 1556static int 1557pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, 1558 vm_offset_t va) 1559{ 1560 vm_page_t m; 1561 u_long data; 1562 1563 if ((tp->tte_data & TD_FAKE) != 0) 1564 return (1); 1565 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1566 data = tp->tte_data & 1567 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1568 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1569 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1570 } 1571 return (1); 1572} 1573 1574void 1575pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1576 vm_size_t len, vm_offset_t src_addr) 1577{ 1578 struct tte *tp; 1579 vm_offset_t va; 1580 1581 if (dst_addr != src_addr) 1582 return; 1583 vm_page_lock_queues(); 1584 if (dst_pmap < src_pmap) { 1585 PMAP_LOCK(dst_pmap); 1586 PMAP_LOCK(src_pmap); 1587 } else { 1588 PMAP_LOCK(src_pmap); 1589 PMAP_LOCK(dst_pmap); 1590 } 1591 if (len > PMAP_TSB_THRESH) { 1592 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1593 pmap_copy_tte); 1594 tlb_context_demap(dst_pmap); 1595 } else { 1596 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) 1597 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1598 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1599 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1600 } 1601 vm_page_unlock_queues(); 1602 PMAP_UNLOCK(src_pmap); 1603 PMAP_UNLOCK(dst_pmap); 1604} 1605 1606void 1607pmap_zero_page(vm_page_t m) 1608{ 1609 struct tte *tp; 1610 vm_offset_t va; 1611 vm_paddr_t pa; 1612 1613 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1614 ("pmap_zero_page: fake page")); 1615 PMAP_STATS_INC(pmap_nzero_page); 1616 pa = VM_PAGE_TO_PHYS(m); 1617 if (m->md.color == -1) { 1618 PMAP_STATS_INC(pmap_nzero_page_nc); 1619 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1620 } else if (m->md.color == DCACHE_COLOR(pa)) { 1621 PMAP_STATS_INC(pmap_nzero_page_c); 1622 va = TLB_PHYS_TO_DIRECT(pa); 1623 cpu_block_zero((void *)va, PAGE_SIZE); 1624 } else { 1625 PMAP_STATS_INC(pmap_nzero_page_oc); 1626 PMAP_LOCK(kernel_pmap); 1627 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1628 tp = tsb_kvtotte(va); 1629 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1630 tp->tte_vpn = TV_VPN(va, TS_8K); 1631 cpu_block_zero((void *)va, PAGE_SIZE); 1632 tlb_page_demap(kernel_pmap, va); 1633 PMAP_UNLOCK(kernel_pmap); 1634 } 1635} 1636 1637void 1638pmap_zero_page_area(vm_page_t m, int off, int size) 1639{ 1640 struct tte *tp; 1641 vm_offset_t va; 1642 vm_paddr_t pa; 1643 1644 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1645 ("pmap_zero_page_area: fake page")); 1646 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1647 PMAP_STATS_INC(pmap_nzero_page_area); 1648 pa = VM_PAGE_TO_PHYS(m); 1649 if (m->md.color == -1) { 1650 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1651 aszero(ASI_PHYS_USE_EC, pa + off, size); 1652 } else if (m->md.color == DCACHE_COLOR(pa)) { 1653 PMAP_STATS_INC(pmap_nzero_page_area_c); 1654 va = TLB_PHYS_TO_DIRECT(pa); 1655 bzero((void *)(va + off), size); 1656 } else { 1657 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1658 PMAP_LOCK(kernel_pmap); 1659 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1660 tp = tsb_kvtotte(va); 1661 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1662 tp->tte_vpn = TV_VPN(va, TS_8K); 1663 bzero((void *)(va + off), size); 1664 tlb_page_demap(kernel_pmap, va); 1665 PMAP_UNLOCK(kernel_pmap); 1666 } 1667} 1668 1669void 1670pmap_zero_page_idle(vm_page_t m) 1671{ 1672 struct tte *tp; 1673 vm_offset_t va; 1674 vm_paddr_t pa; 1675 1676 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1677 ("pmap_zero_page_idle: fake page")); 1678 PMAP_STATS_INC(pmap_nzero_page_idle); 1679 pa = VM_PAGE_TO_PHYS(m); 1680 if (m->md.color == -1) { 1681 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1682 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1683 } else if (m->md.color == DCACHE_COLOR(pa)) { 1684 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1685 va = TLB_PHYS_TO_DIRECT(pa); 1686 cpu_block_zero((void *)va, PAGE_SIZE); 1687 } else { 1688 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1689 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1690 tp = tsb_kvtotte(va); 1691 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1692 tp->tte_vpn = TV_VPN(va, TS_8K); 1693 cpu_block_zero((void *)va, PAGE_SIZE); 1694 tlb_page_demap(kernel_pmap, va); 1695 } 1696} 1697 1698void 1699pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1700{ 1701 vm_offset_t vdst; 1702 vm_offset_t vsrc; 1703 vm_paddr_t pdst; 1704 vm_paddr_t psrc; 1705 struct tte *tp; 1706 1707 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1708 ("pmap_copy_page: fake dst page")); 1709 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1710 ("pmap_copy_page: fake src page")); 1711 PMAP_STATS_INC(pmap_ncopy_page); 1712 pdst = VM_PAGE_TO_PHYS(mdst); 1713 psrc = VM_PAGE_TO_PHYS(msrc); 1714 if (msrc->md.color == -1 && mdst->md.color == -1) { 1715 PMAP_STATS_INC(pmap_ncopy_page_nc); 1716 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1717 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1718 mdst->md.color == DCACHE_COLOR(pdst)) { 1719 PMAP_STATS_INC(pmap_ncopy_page_c); 1720 vdst = TLB_PHYS_TO_DIRECT(pdst); 1721 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1722 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1723 } else if (msrc->md.color == -1) { 1724 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1725 PMAP_STATS_INC(pmap_ncopy_page_dc); 1726 vdst = TLB_PHYS_TO_DIRECT(pdst); 1727 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1728 PAGE_SIZE); 1729 } else { 1730 PMAP_STATS_INC(pmap_ncopy_page_doc); 1731 PMAP_LOCK(kernel_pmap); 1732 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1733 tp = tsb_kvtotte(vdst); 1734 tp->tte_data = 1735 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1736 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1737 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1738 PAGE_SIZE); 1739 tlb_page_demap(kernel_pmap, vdst); 1740 PMAP_UNLOCK(kernel_pmap); 1741 } 1742 } else if (mdst->md.color == -1) { 1743 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1744 PMAP_STATS_INC(pmap_ncopy_page_sc); 1745 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1746 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1747 PAGE_SIZE); 1748 } else { 1749 PMAP_STATS_INC(pmap_ncopy_page_soc); 1750 PMAP_LOCK(kernel_pmap); 1751 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1752 tp = tsb_kvtotte(vsrc); 1753 tp->tte_data = 1754 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1755 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1756 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1757 PAGE_SIZE); 1758 tlb_page_demap(kernel_pmap, vsrc); 1759 PMAP_UNLOCK(kernel_pmap); 1760 } 1761 } else { 1762 PMAP_STATS_INC(pmap_ncopy_page_oc); 1763 PMAP_LOCK(kernel_pmap); 1764 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1765 tp = tsb_kvtotte(vdst); 1766 tp->tte_data = 1767 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1768 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1769 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1770 tp = tsb_kvtotte(vsrc); 1771 tp->tte_data = 1772 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1773 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1774 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1775 tlb_page_demap(kernel_pmap, vdst); 1776 tlb_page_demap(kernel_pmap, vsrc); 1777 PMAP_UNLOCK(kernel_pmap); 1778 } 1779} 1780 1781/* 1782 * Returns true if the pmap's pv is one of the first 1783 * 16 pvs linked to from this page. This count may 1784 * be changed upwards or downwards in the future; it 1785 * is only necessary that true be returned for a small 1786 * subset of pmaps for proper page aging. 1787 */ 1788boolean_t 1789pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1790{ 1791 struct tte *tp; 1792 int loops; 1793 boolean_t rv; 1794 1795 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1796 ("pmap_page_exists_quick: page %p is not managed", m)); 1797 loops = 0; 1798 rv = FALSE; 1799 vm_page_lock_queues(); 1800 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1801 if ((tp->tte_data & TD_PV) == 0) 1802 continue; 1803 if (TTE_GET_PMAP(tp) == pm) { 1804 rv = TRUE; 1805 break; 1806 } 1807 if (++loops >= 16) 1808 break; 1809 } 1810 vm_page_unlock_queues(); 1811 return (rv); 1812} 1813 1814/* 1815 * Return the number of managed mappings to the given physical page 1816 * that are wired. 1817 */ 1818int 1819pmap_page_wired_mappings(vm_page_t m) 1820{ 1821 struct tte *tp; 1822 int count; 1823 1824 count = 0; 1825 if ((m->flags & PG_FICTITIOUS) != 0) 1826 return (count); 1827 vm_page_lock_queues(); 1828 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1829 if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) 1830 count++; 1831 vm_page_unlock_queues(); 1832 return (count); 1833} 1834 1835/* 1836 * Remove all pages from specified address space, this aids process exit 1837 * speeds. This is much faster than pmap_remove n the case of running down 1838 * an entire address space. Only works for the current pmap. 1839 */ 1840void 1841pmap_remove_pages(pmap_t pm) 1842{ 1843 1844} 1845 1846/* 1847 * Returns TRUE if the given page has a managed mapping. 1848 */ 1849boolean_t 1850pmap_page_is_mapped(vm_page_t m) 1851{ 1852 struct tte *tp; 1853 boolean_t rv; 1854 1855 rv = FALSE; 1856 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1857 return (rv); 1858 vm_page_lock_queues(); 1859 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1860 if ((tp->tte_data & TD_PV) != 0) { 1861 rv = TRUE; 1862 break; 1863 } 1864 vm_page_unlock_queues(); 1865 return (rv); 1866} 1867 1868/* 1869 * Return a count of reference bits for a page, clearing those bits. 1870 * It is not necessary for every reference bit to be cleared, but it 1871 * is necessary that 0 only be returned when there are truly no 1872 * reference bits set. 1873 * 1874 * XXX: The exact number of bits to check and clear is a matter that 1875 * should be tested and standardized at some point in the future for 1876 * optimal aging of shared pages. 1877 */ 1878int 1879pmap_ts_referenced(vm_page_t m) 1880{ 1881 struct tte *tpf; 1882 struct tte *tpn; 1883 struct tte *tp; 1884 u_long data; 1885 int count; 1886 1887 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1888 ("pmap_ts_referenced: page %p is not managed", m)); 1889 count = 0; 1890 vm_page_lock_queues(); 1891 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1892 tpf = tp; 1893 do { 1894 tpn = TAILQ_NEXT(tp, tte_link); 1895 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1896 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1897 if ((tp->tte_data & TD_PV) == 0) 1898 continue; 1899 data = atomic_clear_long(&tp->tte_data, TD_REF); 1900 if ((data & TD_REF) != 0 && ++count > 4) 1901 break; 1902 } while ((tp = tpn) != NULL && tp != tpf); 1903 } 1904 vm_page_unlock_queues(); 1905 return (count); 1906} 1907 1908boolean_t 1909pmap_is_modified(vm_page_t m) 1910{ 1911 struct tte *tp; 1912 boolean_t rv; 1913 1914 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1915 ("pmap_is_modified: page %p is not managed", m)); 1916 rv = FALSE; 1917 1918 /* 1919 * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be 1920 * concurrently set while the object is locked. Thus, if PG_WRITEABLE 1921 * is clear, no TTEs can have TD_W set. 1922 */ 1923 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1924 if ((m->oflags & VPO_BUSY) == 0 && 1925 (m->flags & PG_WRITEABLE) == 0) 1926 return (rv); 1927 vm_page_lock_queues(); 1928 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1929 if ((tp->tte_data & TD_PV) == 0) 1930 continue; 1931 if ((tp->tte_data & TD_W) != 0) { 1932 rv = TRUE; 1933 break; 1934 } 1935 } 1936 vm_page_unlock_queues(); 1937 return (rv); 1938} 1939 1940/* 1941 * pmap_is_prefaultable: 1942 * 1943 * Return whether or not the specified virtual address is elgible 1944 * for prefault. 1945 */ 1946boolean_t 1947pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 1948{ 1949 1950 return (FALSE); 1951} 1952 1953/* 1954 * Return whether or not the specified physical page was referenced 1955 * in any physical maps. 1956 */ 1957boolean_t 1958pmap_is_referenced(vm_page_t m) 1959{ 1960 struct tte *tp; 1961 boolean_t rv; 1962 1963 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1964 ("pmap_is_referenced: page %p is not managed", m)); 1965 rv = FALSE; 1966 vm_page_lock_queues(); 1967 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1968 if ((tp->tte_data & TD_PV) == 0) 1969 continue; 1970 if ((tp->tte_data & TD_REF) != 0) { 1971 rv = TRUE; 1972 break; 1973 } 1974 } 1975 vm_page_unlock_queues(); 1976 return (rv); 1977} 1978 1979void 1980pmap_clear_modify(vm_page_t m) 1981{ 1982 struct tte *tp; 1983 u_long data; 1984 1985 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 1986 ("pmap_clear_modify: page %p is not managed", m)); 1987 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1988 KASSERT((m->oflags & VPO_BUSY) == 0, 1989 ("pmap_clear_modify: page %p is busy", m)); 1990 1991 /* 1992 * If the page is not PG_WRITEABLE, then no TTEs can have TD_W set. 1993 * If the object containing the page is locked and the page is not 1994 * VPO_BUSY, then PG_WRITEABLE cannot be concurrently set. 1995 */ 1996 if ((m->flags & PG_WRITEABLE) == 0) 1997 return; 1998 vm_page_lock_queues(); 1999 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2000 if ((tp->tte_data & TD_PV) == 0) 2001 continue; 2002 data = atomic_clear_long(&tp->tte_data, TD_W); 2003 if ((data & TD_W) != 0) 2004 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2005 } 2006 vm_page_unlock_queues(); 2007} 2008 2009void 2010pmap_clear_reference(vm_page_t m) 2011{ 2012 struct tte *tp; 2013 u_long data; 2014 2015 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2016 ("pmap_clear_reference: page %p is not managed", m)); 2017 vm_page_lock_queues(); 2018 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2019 if ((tp->tte_data & TD_PV) == 0) 2020 continue; 2021 data = atomic_clear_long(&tp->tte_data, TD_REF); 2022 if ((data & TD_REF) != 0) 2023 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2024 } 2025 vm_page_unlock_queues(); 2026} 2027 2028void 2029pmap_remove_write(vm_page_t m) 2030{ 2031 struct tte *tp; 2032 u_long data; 2033 2034 KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, 2035 ("pmap_remove_write: page %p is not managed", m)); 2036 2037 /* 2038 * If the page is not VPO_BUSY, then PG_WRITEABLE cannot be set by 2039 * another thread while the object is locked. Thus, if PG_WRITEABLE 2040 * is clear, no page table entries need updating. 2041 */ 2042 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 2043 if ((m->oflags & VPO_BUSY) == 0 && 2044 (m->flags & PG_WRITEABLE) == 0) 2045 return; 2046 vm_page_lock_queues(); 2047 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2048 if ((tp->tte_data & TD_PV) == 0) 2049 continue; 2050 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 2051 if ((data & TD_W) != 0) { 2052 vm_page_dirty(m); 2053 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2054 } 2055 } 2056 vm_page_flag_clear(m, PG_WRITEABLE); 2057 vm_page_unlock_queues(); 2058} 2059 2060int 2061pmap_mincore(pmap_t pm, vm_offset_t addr, vm_paddr_t *locked_pa) 2062{ 2063 2064 /* TODO; */ 2065 return (0); 2066} 2067 2068/* 2069 * Activate a user pmap. The pmap must be activated before its address space 2070 * can be accessed in any way. 2071 */ 2072void 2073pmap_activate(struct thread *td) 2074{ 2075 struct vmspace *vm; 2076 struct pmap *pm; 2077 int context; 2078 2079 vm = td->td_proc->p_vmspace; 2080 pm = vmspace_pmap(vm); 2081 2082 mtx_lock_spin(&sched_lock); 2083 2084 context = PCPU_GET(tlb_ctx); 2085 if (context == PCPU_GET(tlb_ctx_max)) { 2086 tlb_flush_user(); 2087 context = PCPU_GET(tlb_ctx_min); 2088 } 2089 PCPU_SET(tlb_ctx, context + 1); 2090 2091 pm->pm_context[curcpu] = context; 2092 pm->pm_active |= PCPU_GET(cpumask); 2093 PCPU_SET(pmap, pm); 2094 2095 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 2096 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 2097 stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & 2098 TLB_CXR_PGSZ_MASK) | context); 2099 flush(KERNBASE); 2100 2101 mtx_unlock_spin(&sched_lock); 2102} 2103 2104void 2105pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 2106{ 2107 2108} 2109 2110/* 2111 * Increase the starting virtual address of the given mapping if a 2112 * different alignment might result in more superpage mappings. 2113 */ 2114void 2115pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2116 vm_offset_t *addr, vm_size_t size) 2117{ 2118 2119} 2120