pmap.c revision 112879
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. All advertising materials mentioning features or use of this software 22 * must display the following acknowledgement: 23 * This product includes software developed by the University of 24 * California, Berkeley and its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 42 * $FreeBSD: head/sys/sparc64/sparc64/pmap.c 112879 2003-03-31 19:56:55Z jake $ 43 */ 44 45/* 46 * Manages physical address maps. 47 * 48 * In addition to hardware address maps, this module is called upon to 49 * provide software-use-only maps which may or may not be stored in the 50 * same form as hardware maps. These pseudo-maps are used to store 51 * intermediate results from copy operations to and from address spaces. 52 * 53 * Since the information managed by this module is also stored by the 54 * logical address mapping module, this module may throw away valid virtual 55 * to physical mappings at almost any time. However, invalidations of 56 * mappings must be done as requested. 57 * 58 * In order to cope with hardware architectures which make virtual to 59 * physical map invalidates expensive, this module may delay invalidate 60 * reduced protection operations until such time as they are actually 61 * necessary. This module is given full information as to which processors 62 * are currently using which maps, and to when physical maps must be made 63 * correct. 64 */ 65 66#include "opt_msgbuf.h" 67#include "opt_pmap.h" 68 69#include <sys/param.h> 70#include <sys/kernel.h> 71#include <sys/ktr.h> 72#include <sys/lock.h> 73#include <sys/msgbuf.h> 74#include <sys/mutex.h> 75#include <sys/proc.h> 76#include <sys/smp.h> 77#include <sys/sysctl.h> 78#include <sys/systm.h> 79#include <sys/vmmeter.h> 80 81#include <dev/ofw/openfirm.h> 82 83#include <vm/vm.h> 84#include <vm/vm_param.h> 85#include <vm/vm_kern.h> 86#include <vm/vm_page.h> 87#include <vm/vm_map.h> 88#include <vm/vm_object.h> 89#include <vm/vm_extern.h> 90#include <vm/vm_pageout.h> 91#include <vm/vm_pager.h> 92 93#include <machine/cache.h> 94#include <machine/frame.h> 95#include <machine/instr.h> 96#include <machine/md_var.h> 97#include <machine/metadata.h> 98#include <machine/ofw_mem.h> 99#include <machine/smp.h> 100#include <machine/tlb.h> 101#include <machine/tte.h> 102#include <machine/tsb.h> 103 104#define PMAP_DEBUG 105 106#ifndef PMAP_SHPGPERPROC 107#define PMAP_SHPGPERPROC 200 108#endif 109 110/* 111 * Virtual and physical address of message buffer. 112 */ 113struct msgbuf *msgbufp; 114vm_offset_t msgbuf_phys; 115 116/* 117 * Physical addresses of first and last available physical page. 118 */ 119vm_offset_t avail_start; 120vm_offset_t avail_end; 121 122int pmap_pagedaemon_waken; 123 124/* 125 * Map of physical memory reagions. 126 */ 127vm_offset_t phys_avail[128]; 128static struct ofw_mem_region mra[128]; 129struct ofw_mem_region sparc64_memreg[128]; 130int sparc64_nmemreg; 131static struct ofw_map translations[128]; 132static int translations_size; 133 134static vm_offset_t pmap_idle_map; 135static vm_offset_t pmap_temp_map_1; 136static vm_offset_t pmap_temp_map_2; 137 138/* 139 * First and last available kernel virtual addresses. 140 */ 141vm_offset_t virtual_avail; 142vm_offset_t virtual_end; 143vm_offset_t kernel_vm_end; 144 145vm_offset_t vm_max_kernel_address; 146 147/* 148 * Kernel pmap. 149 */ 150struct pmap kernel_pmap_store; 151 152/* 153 * Allocate physical memory for use in pmap_bootstrap. 154 */ 155static vm_offset_t pmap_bootstrap_alloc(vm_size_t size); 156 157extern int tl1_immu_miss_patch_1[]; 158extern int tl1_immu_miss_patch_2[]; 159extern int tl1_dmmu_miss_patch_1[]; 160extern int tl1_dmmu_miss_patch_2[]; 161extern int tl1_dmmu_prot_patch_1[]; 162extern int tl1_dmmu_prot_patch_2[]; 163 164/* 165 * If user pmap is processed with pmap_remove and with pmap_remove and the 166 * resident count drops to 0, there are no more pages to remove, so we 167 * need not continue. 168 */ 169#define PMAP_REMOVE_DONE(pm) \ 170 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 171 172/* 173 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 174 * and pmap_protect() instead of trying each virtual address. 175 */ 176#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 177 178SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 179 180PMAP_STATS_VAR(pmap_nenter); 181PMAP_STATS_VAR(pmap_nenter_update); 182PMAP_STATS_VAR(pmap_nenter_replace); 183PMAP_STATS_VAR(pmap_nenter_new); 184PMAP_STATS_VAR(pmap_nkenter); 185PMAP_STATS_VAR(pmap_nkenter_oc); 186PMAP_STATS_VAR(pmap_nkenter_stupid); 187PMAP_STATS_VAR(pmap_nkremove); 188PMAP_STATS_VAR(pmap_nqenter); 189PMAP_STATS_VAR(pmap_nqremove); 190PMAP_STATS_VAR(pmap_ncache_enter); 191PMAP_STATS_VAR(pmap_ncache_enter_c); 192PMAP_STATS_VAR(pmap_ncache_enter_oc); 193PMAP_STATS_VAR(pmap_ncache_enter_cc); 194PMAP_STATS_VAR(pmap_ncache_enter_coc); 195PMAP_STATS_VAR(pmap_ncache_enter_nc); 196PMAP_STATS_VAR(pmap_ncache_enter_cnc); 197PMAP_STATS_VAR(pmap_ncache_remove); 198PMAP_STATS_VAR(pmap_ncache_remove_c); 199PMAP_STATS_VAR(pmap_ncache_remove_oc); 200PMAP_STATS_VAR(pmap_ncache_remove_cc); 201PMAP_STATS_VAR(pmap_ncache_remove_coc); 202PMAP_STATS_VAR(pmap_ncache_remove_nc); 203PMAP_STATS_VAR(pmap_niflush); 204PMAP_STATS_VAR(pmap_nzero_page); 205PMAP_STATS_VAR(pmap_nzero_page_c); 206PMAP_STATS_VAR(pmap_nzero_page_oc); 207PMAP_STATS_VAR(pmap_nzero_page_nc); 208PMAP_STATS_VAR(pmap_nzero_page_area); 209PMAP_STATS_VAR(pmap_nzero_page_area_c); 210PMAP_STATS_VAR(pmap_nzero_page_area_oc); 211PMAP_STATS_VAR(pmap_nzero_page_area_nc); 212PMAP_STATS_VAR(pmap_nzero_page_idle); 213PMAP_STATS_VAR(pmap_nzero_page_idle_c); 214PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 215PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 216PMAP_STATS_VAR(pmap_ncopy_page); 217PMAP_STATS_VAR(pmap_ncopy_page_c); 218PMAP_STATS_VAR(pmap_ncopy_page_oc); 219PMAP_STATS_VAR(pmap_ncopy_page_nc); 220PMAP_STATS_VAR(pmap_ncopy_page_dc); 221PMAP_STATS_VAR(pmap_ncopy_page_doc); 222PMAP_STATS_VAR(pmap_ncopy_page_sc); 223PMAP_STATS_VAR(pmap_ncopy_page_soc); 224 225PMAP_STATS_VAR(pmap_nnew_thread); 226PMAP_STATS_VAR(pmap_nnew_thread_oc); 227 228/* 229 * Quick sort callout for comparing memory regions. 230 */ 231static int mr_cmp(const void *a, const void *b); 232static int om_cmp(const void *a, const void *b); 233static int 234mr_cmp(const void *a, const void *b) 235{ 236 const struct ofw_mem_region *mra; 237 const struct ofw_mem_region *mrb; 238 239 mra = a; 240 mrb = b; 241 if (mra->mr_start < mrb->mr_start) 242 return (-1); 243 else if (mra->mr_start > mrb->mr_start) 244 return (1); 245 else 246 return (0); 247} 248static int 249om_cmp(const void *a, const void *b) 250{ 251 const struct ofw_map *oma; 252 const struct ofw_map *omb; 253 254 oma = a; 255 omb = b; 256 if (oma->om_start < omb->om_start) 257 return (-1); 258 else if (oma->om_start > omb->om_start) 259 return (1); 260 else 261 return (0); 262} 263 264/* 265 * Bootstrap the system enough to run with virtual memory. 266 */ 267void 268pmap_bootstrap(vm_offset_t ekva) 269{ 270 struct pmap *pm; 271 struct tte *tp; 272 vm_offset_t off; 273 vm_offset_t pa; 274 vm_offset_t va; 275 vm_size_t physsz; 276 vm_size_t virtsz; 277 ihandle_t pmem; 278 ihandle_t vmem; 279 int sz; 280 int i; 281 int j; 282 283 /* 284 * Find out what physical memory is available from the prom and 285 * initialize the phys_avail array. This must be done before 286 * pmap_bootstrap_alloc is called. 287 */ 288 if ((pmem = OF_finddevice("/memory")) == -1) 289 panic("pmap_bootstrap: finddevice /memory"); 290 if ((sz = OF_getproplen(pmem, "available")) == -1) 291 panic("pmap_bootstrap: getproplen /memory/available"); 292 if (sizeof(phys_avail) < sz) 293 panic("pmap_bootstrap: phys_avail too small"); 294 if (sizeof(mra) < sz) 295 panic("pmap_bootstrap: mra too small"); 296 bzero(mra, sz); 297 if (OF_getprop(pmem, "available", mra, sz) == -1) 298 panic("pmap_bootstrap: getprop /memory/available"); 299 sz /= sizeof(*mra); 300 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 301 qsort(mra, sz, sizeof (*mra), mr_cmp); 302 physsz = 0; 303 getenv_quad("hw.physmem", &physmem); 304 for (i = 0, j = 0; i < sz; i++, j += 2) { 305 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 306 mra[i].mr_size); 307 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 308 if (btoc(physsz) < physmem) { 309 phys_avail[j] = mra[i].mr_start; 310 phys_avail[j + 1] = mra[i].mr_start + 311 (ctob(physmem) - physsz); 312 physsz = ctob(physmem); 313 } 314 break; 315 } 316 phys_avail[j] = mra[i].mr_start; 317 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 318 physsz += mra[i].mr_size; 319 } 320 physmem = btoc(physsz); 321 322 /* 323 * Calculate the size of kernel virtual memory, and the size and mask 324 * for the kernel tsb. 325 */ 326 virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); 327 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 328 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 329 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 330 331 /* 332 * Allocate the kernel tsb and lock it in the tlb. 333 */ 334 pa = pmap_bootstrap_alloc(tsb_kernel_size); 335 if (pa & PAGE_MASK_4M) 336 panic("pmap_bootstrap: tsb unaligned\n"); 337 tsb_kernel_phys = pa; 338 tsb_kernel = (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 339 pmap_map_tsb(); 340 bzero(tsb_kernel, tsb_kernel_size); 341 342 /* 343 * Allocate and map the message buffer. 344 */ 345 msgbuf_phys = pmap_bootstrap_alloc(MSGBUF_SIZE); 346 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(msgbuf_phys); 347 348 /* 349 * Patch the virtual address and the tsb mask into the trap table. 350 */ 351 352#define SETHI(rd, imm22) \ 353 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 354 EIF_IMM((imm22) >> 10, 22)) 355#define OR_R_I_R(rd, imm13, rs1) \ 356 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 357 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 358 359#define PATCH(addr) do { \ 360 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 361 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, IF_F3_RS1(addr[1])) || \ 362 addr[2] != SETHI(IF_F2_RD(addr[2]), 0x0)) \ 363 panic("pmap_boostrap: patched instructions have changed"); \ 364 addr[0] |= EIF_IMM((tsb_kernel_mask) >> 10, 22); \ 365 addr[1] |= EIF_IMM(tsb_kernel_mask, 10); \ 366 addr[2] |= EIF_IMM(((vm_offset_t)tsb_kernel) >> 10, 22); \ 367 flush(addr); \ 368 flush(addr + 1); \ 369 flush(addr + 2); \ 370} while (0) 371 372 PATCH(tl1_immu_miss_patch_1); 373 PATCH(tl1_immu_miss_patch_2); 374 PATCH(tl1_dmmu_miss_patch_1); 375 PATCH(tl1_dmmu_miss_patch_2); 376 PATCH(tl1_dmmu_prot_patch_1); 377 PATCH(tl1_dmmu_prot_patch_2); 378 379 /* 380 * Enter fake 8k pages for the 4MB kernel pages, so that 381 * pmap_kextract() will work for them. 382 */ 383 for (i = 0; i < kernel_tlb_slots; i++) { 384 pa = kernel_tlbs[i].te_pa; 385 va = kernel_tlbs[i].te_va; 386 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 387 tp = tsb_kvtotte(va + off); 388 tp->tte_vpn = TV_VPN(va + off, TS_8K); 389 tp->tte_data = TD_V | TD_8K | TD_PA(pa + off) | 390 TD_REF | TD_SW | TD_CP | TD_CV | TD_P | TD_W; 391 } 392 } 393 394 /* 395 * Set the start and end of kva. The kernel is loaded at the first 396 * available 4 meg super page, so round up to the end of the page. 397 */ 398 virtual_avail = roundup2(ekva, PAGE_SIZE_4M); 399 virtual_end = vm_max_kernel_address; 400 kernel_vm_end = vm_max_kernel_address; 401 402 /* 403 * Allocate kva space for temporary mappings. 404 */ 405 pmap_idle_map = virtual_avail; 406 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 407 pmap_temp_map_1 = virtual_avail; 408 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 409 pmap_temp_map_2 = virtual_avail; 410 virtual_avail += PAGE_SIZE * DCACHE_COLORS; 411 412 /* 413 * Allocate a kernel stack with guard page for thread0 and map it into 414 * the kernel tsb. We must ensure that the virtual address is coloured 415 * properly, since we're allocating from phys_avail so the memory won't 416 * have an associated vm_page_t. 417 */ 418 pa = pmap_bootstrap_alloc(roundup(KSTACK_PAGES, DCACHE_COLORS) * 419 PAGE_SIZE); 420 kstack0_phys = pa; 421 virtual_avail += roundup(KSTACK_GUARD_PAGES, DCACHE_COLORS) * 422 PAGE_SIZE; 423 kstack0 = virtual_avail; 424 virtual_avail += roundup(KSTACK_PAGES, DCACHE_COLORS) * PAGE_SIZE; 425 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 426 ("pmap_bootstrap: kstack0 miscoloured")); 427 for (i = 0; i < KSTACK_PAGES; i++) { 428 pa = kstack0_phys + i * PAGE_SIZE; 429 va = kstack0 + i * PAGE_SIZE; 430 tp = tsb_kvtotte(va); 431 tp->tte_vpn = TV_VPN(va, TS_8K); 432 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | 433 TD_CP | TD_CV | TD_P | TD_W; 434 } 435 436 /* 437 * Calculate the first and last available physical addresses. 438 */ 439 avail_start = phys_avail[0]; 440 for (i = 0; phys_avail[i + 2] != 0; i += 2) 441 ; 442 avail_end = phys_avail[i + 1]; 443 Maxmem = sparc64_btop(avail_end); 444 445 /* 446 * Add the prom mappings to the kernel tsb. 447 */ 448 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 449 panic("pmap_bootstrap: finddevice /virtual-memory"); 450 if ((sz = OF_getproplen(vmem, "translations")) == -1) 451 panic("pmap_bootstrap: getproplen translations"); 452 if (sizeof(translations) < sz) 453 panic("pmap_bootstrap: translations too small"); 454 bzero(translations, sz); 455 if (OF_getprop(vmem, "translations", translations, sz) == -1) 456 panic("pmap_bootstrap: getprop /virtual-memory/translations"); 457 sz /= sizeof(*translations); 458 translations_size = sz; 459 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 460 qsort(translations, sz, sizeof (*translations), om_cmp); 461 for (i = 0; i < sz; i++) { 462 CTR3(KTR_PMAP, 463 "translation: start=%#lx size=%#lx tte=%#lx", 464 translations[i].om_start, translations[i].om_size, 465 translations[i].om_tte); 466 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 467 translations[i].om_start > VM_MAX_PROM_ADDRESS) 468 continue; 469 for (off = 0; off < translations[i].om_size; 470 off += PAGE_SIZE) { 471 va = translations[i].om_start + off; 472 tp = tsb_kvtotte(va); 473 tp->tte_vpn = TV_VPN(va, TS_8K); 474 tp->tte_data = 475 ((translations[i].om_tte & 476 ~(TD_SOFT_MASK << TD_SOFT_SHIFT)) | TD_EXEC) + 477 off; 478 } 479 } 480 481 /* 482 * Get the available physical memory ranges from /memory/reg. These 483 * are only used for kernel dumps, but it may not be wise to do prom 484 * calls in that situation. 485 */ 486 if ((sz = OF_getproplen(pmem, "reg")) == -1) 487 panic("pmap_bootstrap: getproplen /memory/reg"); 488 if (sizeof(sparc64_memreg) < sz) 489 panic("pmap_bootstrap: sparc64_memreg too small"); 490 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 491 panic("pmap_bootstrap: getprop /memory/reg"); 492 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 493 494 /* 495 * Initialize the kernel pmap (which is statically allocated). 496 */ 497 pm = kernel_pmap; 498 for (i = 0; i < MAXCPU; i++) 499 pm->pm_context[i] = TLB_CTX_KERNEL; 500 pm->pm_active = ~0; 501 502 /* XXX flush all non-locked tlb entries */ 503} 504 505void 506pmap_map_tsb(void) 507{ 508 vm_offset_t va; 509 vm_offset_t pa; 510 u_long data; 511 u_long s; 512 int i; 513 514 s = intr_disable(); 515 516 /* 517 * Map the 4mb tsb pages. 518 */ 519 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 520 va = (vm_offset_t)tsb_kernel + i; 521 pa = tsb_kernel_phys + i; 522 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 523 TD_P | TD_W; 524 /* XXX - cheetah */ 525 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 526 TLB_TAR_CTX(TLB_CTX_KERNEL)); 527 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 528 } 529 530 /* 531 * Set the secondary context to be the kernel context (needed for 532 * fp block operations in the kernel and the cache code). 533 */ 534 stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); 535 membar(Sync); 536 537 intr_restore(s); 538} 539 540/* 541 * Allocate a physical page of memory directly from the phys_avail map. 542 * Can only be called from pmap_bootstrap before avail start and end are 543 * calculated. 544 */ 545static vm_offset_t 546pmap_bootstrap_alloc(vm_size_t size) 547{ 548 vm_offset_t pa; 549 int i; 550 551 size = round_page(size); 552 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 553 if (phys_avail[i + 1] - phys_avail[i] < size) 554 continue; 555 pa = phys_avail[i]; 556 phys_avail[i] += size; 557 return (pa); 558 } 559 panic("pmap_bootstrap_alloc"); 560} 561 562void 563pmap_context_rollover(void) 564{ 565 u_long data; 566 u_long tag; 567 int i; 568 569 mtx_assert(&sched_lock, MA_OWNED); 570 CTR0(KTR_PMAP, "pmap_context_rollover"); 571 for (i = 0; i < tlb_dtlb_entries; i++) { 572 /* XXX - cheetah */ 573 data = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG); 574 tag = ldxa(TLB_DAR_SLOT(i), ASI_DTLB_TAG_READ_REG); 575 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 576 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 577 stxa_sync(TLB_DAR_SLOT(i), ASI_DTLB_DATA_ACCESS_REG, 0); 578 data = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG); 579 tag = ldxa(TLB_DAR_SLOT(i), ASI_ITLB_TAG_READ_REG); 580 if ((data & TD_V) != 0 && (data & TD_L) == 0 && 581 TLB_TAR_CTX(tag) != TLB_CTX_KERNEL) 582 stxa_sync(TLB_DAR_SLOT(i), ASI_ITLB_DATA_ACCESS_REG, 0); 583 } 584 PCPU_SET(tlb_ctx, PCPU_GET(tlb_ctx_min)); 585} 586 587static __inline u_int 588pmap_context_alloc(void) 589{ 590 u_int context; 591 592 mtx_assert(&sched_lock, MA_OWNED); 593 context = PCPU_GET(tlb_ctx); 594 if (context + 1 == PCPU_GET(tlb_ctx_max)) 595 pmap_context_rollover(); 596 else 597 PCPU_SET(tlb_ctx, context + 1); 598 return (context); 599} 600 601/* 602 * Initialize the pmap module. 603 */ 604void 605pmap_init(vm_offset_t phys_start, vm_offset_t phys_end) 606{ 607 vm_offset_t addr; 608 vm_size_t size; 609 int result; 610 int i; 611 612 for (i = 0; i < vm_page_array_size; i++) { 613 vm_page_t m; 614 615 m = &vm_page_array[i]; 616 TAILQ_INIT(&m->md.tte_list); 617 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 618 m->md.flags = 0; 619 m->md.pmap = NULL; 620 } 621 622 for (i = 0; i < translations_size; i++) { 623 addr = translations[i].om_start; 624 size = translations[i].om_size; 625 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 626 continue; 627 result = vm_map_find(kernel_map, NULL, 0, &addr, size, FALSE, 628 VM_PROT_ALL, VM_PROT_ALL, 0); 629 if (result != KERN_SUCCESS || addr != translations[i].om_start) 630 panic("pmap_init: vm_map_find"); 631 } 632} 633 634/* 635 * Initialize the address space (zone) for the pv_entries. Set a 636 * high water mark so that the system can recover from excessive 637 * numbers of pv entries. 638 */ 639void 640pmap_init2(void) 641{ 642} 643 644/* 645 * Extract the physical page address associated with the given 646 * map/virtual_address pair. 647 */ 648vm_offset_t 649pmap_extract(pmap_t pm, vm_offset_t va) 650{ 651 struct tte *tp; 652 653 if (pm == kernel_pmap) 654 return (pmap_kextract(va)); 655 tp = tsb_tte_lookup(pm, va); 656 if (tp == NULL) 657 return (0); 658 else 659 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 660} 661 662/* 663 * Extract the physical page address associated with the given kernel virtual 664 * address. 665 */ 666vm_offset_t 667pmap_kextract(vm_offset_t va) 668{ 669 struct tte *tp; 670 671 if (va >= VM_MIN_DIRECT_ADDRESS) 672 return (TLB_DIRECT_TO_PHYS(va)); 673 tp = tsb_kvtotte(va); 674 if ((tp->tte_data & TD_V) == 0) 675 return (0); 676 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 677} 678 679int 680pmap_cache_enter(vm_page_t m, vm_offset_t va) 681{ 682 struct tte *tp; 683 int color; 684 685 KASSERT((m->flags & PG_FICTITIOUS) == 0, 686 ("pmap_cache_enter: fake page")); 687 PMAP_STATS_INC(pmap_ncache_enter); 688 689 /* 690 * Find the color for this virtual address and note the added mapping. 691 */ 692 color = DCACHE_COLOR(va); 693 m->md.colors[color]++; 694 695 /* 696 * If all existing mappings have the same color, the mapping is 697 * cacheable. 698 */ 699 if (m->md.color == color) { 700 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 701 ("pmap_cache_enter: cacheable, mappings of other color")); 702 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 703 PMAP_STATS_INC(pmap_ncache_enter_c); 704 else 705 PMAP_STATS_INC(pmap_ncache_enter_oc); 706 return (1); 707 } 708 709 /* 710 * If there are no mappings of the other color, and the page still has 711 * the wrong color, this must be a new mapping. Change the color to 712 * match the new mapping, which is cacheable. We must flush the page 713 * from the cache now. 714 */ 715 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 716 KASSERT(m->md.colors[color] == 1, 717 ("pmap_cache_enter: changing color, not new mapping")); 718 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 719 m->md.color = color; 720 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 721 PMAP_STATS_INC(pmap_ncache_enter_cc); 722 else 723 PMAP_STATS_INC(pmap_ncache_enter_coc); 724 return (1); 725 } 726 727 /* 728 * If the mapping is already non-cacheable, just return. 729 */ 730 if (m->md.color == -1) { 731 PMAP_STATS_INC(pmap_ncache_enter_nc); 732 return (0); 733 } 734 735 PMAP_STATS_INC(pmap_ncache_enter_cnc); 736 737 /* 738 * Mark all mappings as uncacheable, flush any lines with the other 739 * color out of the dcache, and set the color to none (-1). 740 */ 741 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 742 atomic_clear_long(&tp->tte_data, TD_CV); 743 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 744 } 745 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 746 m->md.color = -1; 747 return (0); 748} 749 750void 751pmap_cache_remove(vm_page_t m, vm_offset_t va) 752{ 753 struct tte *tp; 754 int color; 755 756 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 757 m->md.colors[DCACHE_COLOR(va)]); 758 KASSERT((m->flags & PG_FICTITIOUS) == 0, 759 ("pmap_cache_remove: fake page")); 760 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 761 ("pmap_cache_remove: no mappings %d <= 0", 762 m->md.colors[DCACHE_COLOR(va)])); 763 PMAP_STATS_INC(pmap_ncache_remove); 764 765 /* 766 * Find the color for this virtual address and note the removal of 767 * the mapping. 768 */ 769 color = DCACHE_COLOR(va); 770 m->md.colors[color]--; 771 772 /* 773 * If the page is cacheable, just return and keep the same color, even 774 * if there are no longer any mappings. 775 */ 776 if (m->md.color != -1) { 777 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 778 PMAP_STATS_INC(pmap_ncache_remove_c); 779 else 780 PMAP_STATS_INC(pmap_ncache_remove_oc); 781 return; 782 } 783 784 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 785 ("pmap_cache_remove: uncacheable, no mappings of other color")); 786 787 /* 788 * If the page is not cacheable (color is -1), and the number of 789 * mappings for this color is not zero, just return. There are 790 * mappings of the other color still, so remain non-cacheable. 791 */ 792 if (m->md.colors[color] != 0) { 793 PMAP_STATS_INC(pmap_ncache_remove_nc); 794 return; 795 } 796 797 /* 798 * The number of mappings for this color is now zero. Recache the 799 * other colored mappings, and change the page color to the other 800 * color. There should be no lines in the data cache for this page, 801 * so flushing should not be needed. 802 */ 803 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 804 atomic_set_long(&tp->tte_data, TD_CV); 805 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 806 } 807 m->md.color = DCACHE_OTHER_COLOR(color); 808 809 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 810 PMAP_STATS_INC(pmap_ncache_remove_cc); 811 else 812 PMAP_STATS_INC(pmap_ncache_remove_coc); 813} 814 815/* 816 * Map a wired page into kernel virtual address space. 817 */ 818void 819pmap_kenter(vm_offset_t va, vm_offset_t pa) 820{ 821 vm_offset_t opa; 822 vm_offset_t ova; 823 struct tte *tp; 824 vm_page_t om; 825 vm_page_t m; 826 u_long data; 827 828 PMAP_STATS_INC(pmap_nkenter); 829 tp = tsb_kvtotte(va); 830 m = PHYS_TO_VM_PAGE(pa); 831 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 832 va, pa, tp, tp->tte_data); 833 if (m->pc != DCACHE_COLOR(va)) { 834 CTR6(KTR_CT2, 835 "pmap_kenter: off colour va=%#lx pa=%#lx o=%p oc=%#lx ot=%d pi=%#lx", 836 va, pa, m->object, 837 m->object ? m->object->pg_color : -1, 838 m->object ? m->object->type : -1, 839 m->pindex); 840 PMAP_STATS_INC(pmap_nkenter_oc); 841 } 842 if ((tp->tte_data & TD_V) != 0) { 843 opa = TTE_GET_PA(tp); 844 ova = TTE_GET_VA(tp); 845 if (pa == opa && va == ova) { 846 PMAP_STATS_INC(pmap_nkenter_stupid); 847 return; 848 } 849 om = PHYS_TO_VM_PAGE(opa); 850 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 851 pmap_cache_remove(om, ova); 852 if (va != ova) 853 tlb_page_demap(kernel_pmap, ova); 854 } 855 data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | TD_P | TD_W; 856 if (pmap_cache_enter(m, va) != 0) 857 data |= TD_CV; 858 tp->tte_vpn = TV_VPN(va, TS_8K); 859 tp->tte_data = data; 860 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 861} 862 863/* 864 * Map a wired page into kernel virtual address space. This additionally 865 * takes a flag argument wich is or'ed to the TTE data. This is used by 866 * bus_space_map(). 867 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 868 * to flush entries that might still be in the cache, if applicable. 869 */ 870void 871pmap_kenter_flags(vm_offset_t va, vm_offset_t pa, u_long flags) 872{ 873 struct tte *tp; 874 875 tp = tsb_kvtotte(va); 876 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 877 va, pa, tp, tp->tte_data); 878 tp->tte_vpn = TV_VPN(va, TS_8K); 879 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 880} 881 882/* 883 * Remove a wired page from kernel virtual address space. 884 */ 885void 886pmap_kremove(vm_offset_t va) 887{ 888 struct tte *tp; 889 vm_page_t m; 890 891 PMAP_STATS_INC(pmap_nkremove); 892 tp = tsb_kvtotte(va); 893 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 894 tp->tte_data); 895 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 896 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 897 pmap_cache_remove(m, va); 898 TTE_ZERO(tp); 899} 900 901/* 902 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 903 */ 904void 905pmap_kremove_flags(vm_offset_t va) 906{ 907 struct tte *tp; 908 909 tp = tsb_kvtotte(va); 910 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 911 tp->tte_data); 912 TTE_ZERO(tp); 913} 914 915/* 916 * Map a range of physical addresses into kernel virtual address space. 917 * 918 * The value passed in *virt is a suggested virtual address for the mapping. 919 * Architectures which can support a direct-mapped physical to virtual region 920 * can return the appropriate address within that region, leaving '*virt' 921 * unchanged. 922 */ 923vm_offset_t 924pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 925{ 926 927 return (TLB_PHYS_TO_DIRECT(start)); 928} 929 930/* 931 * Map a list of wired pages into kernel virtual address space. This is 932 * intended for temporary mappings which do not need page modification or 933 * references recorded. Existing mappings in the region are overwritten. 934 */ 935void 936pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 937{ 938 vm_offset_t va; 939 940 PMAP_STATS_INC(pmap_nqenter); 941 va = sva; 942 while (count-- > 0) { 943 pmap_kenter(va, VM_PAGE_TO_PHYS(*m)); 944 va += PAGE_SIZE; 945 m++; 946 } 947 tlb_range_demap(kernel_pmap, sva, va); 948} 949 950/* 951 * Remove page mappings from kernel virtual address space. Intended for 952 * temporary mappings entered by pmap_qenter. 953 */ 954void 955pmap_qremove(vm_offset_t sva, int count) 956{ 957 vm_offset_t va; 958 959 PMAP_STATS_INC(pmap_nqremove); 960 va = sva; 961 while (count-- > 0) { 962 pmap_kremove(va); 963 va += PAGE_SIZE; 964 } 965 tlb_range_demap(kernel_pmap, sva, va); 966} 967 968#ifndef KSTACK_MAX_PAGES 969#define KSTACK_MAX_PAGES 32 970#endif 971 972/* 973 * Create the kernel stack and pcb for a new thread. 974 * This routine directly affects the fork perf for a process and 975 * create performance for a thread. 976 */ 977void 978pmap_new_thread(struct thread *td, int pages) 979{ 980 vm_page_t ma[KSTACK_MAX_PAGES]; 981 vm_object_t ksobj; 982 vm_offset_t ks; 983 vm_page_t m; 984 u_int i; 985 986 PMAP_STATS_INC(pmap_nnew_thread); 987 /* Bounds check */ 988 if (pages <= 1) 989 pages = KSTACK_PAGES; 990 else if (pages > KSTACK_MAX_PAGES) 991 pages = KSTACK_MAX_PAGES; 992 993 /* 994 * Allocate object for the kstack, 995 */ 996 ksobj = vm_object_allocate(OBJT_DEFAULT, pages); 997 td->td_kstack_obj = ksobj; 998 999 /* 1000 * Get a kernel virtual address for the kstack for this thread. 1001 */ 1002 ks = kmem_alloc_nofault(kernel_map, 1003 (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); 1004 if (ks == 0) 1005 panic("pmap_new_thread: kstack allocation failed"); 1006 if (KSTACK_GUARD_PAGES != 0) { 1007 tlb_page_demap(kernel_pmap, ks); 1008 ks += KSTACK_GUARD_PAGES * PAGE_SIZE; 1009 } 1010 td->td_kstack = ks; 1011 1012 /* 1013 * Knowing the number of pages allocated is useful when you 1014 * want to deallocate them. 1015 */ 1016 td->td_kstack_pages = pages; 1017 1018 for (i = 0; i < pages; i++) { 1019 /* 1020 * Get a kernel stack page. 1021 */ 1022 m = vm_page_grab(ksobj, i, 1023 VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED); 1024 ma[i] = m; 1025 if (DCACHE_COLOR(ks + (i * PAGE_SIZE)) != 1026 DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 1027 PMAP_STATS_INC(pmap_nnew_thread_oc); 1028 1029 vm_page_lock_queues(); 1030 vm_page_wakeup(m); 1031 vm_page_flag_clear(m, PG_ZERO); 1032 m->valid = VM_PAGE_BITS_ALL; 1033 vm_page_unlock_queues(); 1034 } 1035 1036 /* 1037 * Enter the page into the kernel address space. 1038 */ 1039 pmap_qenter(ks, ma, pages); 1040} 1041 1042/* 1043 * Dispose the kernel stack for a thread that has exited. 1044 * This routine directly impacts the exit perf of a process and thread. 1045 */ 1046void 1047pmap_dispose_thread(struct thread *td) 1048{ 1049 vm_object_t ksobj; 1050 vm_offset_t ks; 1051 vm_page_t m; 1052 int i; 1053 int pages; 1054 1055 pages = td->td_kstack_pages; 1056 ksobj = td->td_kstack_obj; 1057 ks = td->td_kstack; 1058 for (i = 0; i < pages ; i++) { 1059 m = vm_page_lookup(ksobj, i); 1060 if (m == NULL) 1061 panic("pmap_dispose_thread: kstack already missing?"); 1062 vm_page_lock_queues(); 1063 vm_page_busy(m); 1064 vm_page_unwire(m, 0); 1065 vm_page_free(m); 1066 vm_page_unlock_queues(); 1067 } 1068 pmap_qremove(ks, pages); 1069 kmem_free(kernel_map, ks - (KSTACK_GUARD_PAGES * PAGE_SIZE), 1070 (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); 1071 vm_object_deallocate(ksobj); 1072} 1073 1074/* 1075 * Set up a variable sized alternate kstack. 1076 */ 1077void 1078pmap_new_altkstack(struct thread *td, int pages) 1079{ 1080 /* shuffle the original stack */ 1081 td->td_altkstack_obj = td->td_kstack_obj; 1082 td->td_altkstack = td->td_kstack; 1083 td->td_altkstack_pages = td->td_kstack_pages; 1084 1085 pmap_new_thread(td, pages); 1086} 1087 1088void 1089pmap_dispose_altkstack(struct thread *td) 1090{ 1091 pmap_dispose_thread(td); 1092 1093 /* restore the original kstack */ 1094 td->td_kstack = td->td_altkstack; 1095 td->td_kstack_obj = td->td_altkstack_obj; 1096 td->td_kstack_pages = td->td_altkstack_pages; 1097 td->td_altkstack = 0; 1098 td->td_altkstack_obj = NULL; 1099 td->td_altkstack_pages = 0; 1100} 1101 1102/* 1103 * Allow the kernel stack for a thread to be prejudicially paged out. 1104 */ 1105void 1106pmap_swapout_thread(struct thread *td) 1107{ 1108 vm_object_t ksobj; 1109 vm_offset_t ks; 1110 vm_page_t m; 1111 int i; 1112 int pages; 1113 1114 pages = td->td_kstack_pages; 1115 ksobj = td->td_kstack_obj; 1116 ks = (vm_offset_t)td->td_kstack; 1117 for (i = 0; i < pages; i++) { 1118 m = vm_page_lookup(ksobj, i); 1119 if (m == NULL) 1120 panic("pmap_swapout_thread: kstack already missing?"); 1121 vm_page_lock_queues(); 1122 vm_page_dirty(m); 1123 vm_page_unwire(m, 0); 1124 vm_page_unlock_queues(); 1125 } 1126 pmap_qremove(ks, pages); 1127} 1128 1129/* 1130 * Bring the kernel stack for a specified thread back in. 1131 */ 1132void 1133pmap_swapin_thread(struct thread *td) 1134{ 1135 vm_page_t ma[KSTACK_MAX_PAGES]; 1136 vm_object_t ksobj; 1137 vm_offset_t ks; 1138 vm_page_t m; 1139 int rv; 1140 int i; 1141 int pages; 1142 1143 pages = td->td_kstack_pages; 1144 ksobj = td->td_kstack_obj; 1145 ks = td->td_kstack; 1146 for (i = 0; i < pages; i++) { 1147 m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 1148 if (m->valid != VM_PAGE_BITS_ALL) { 1149 rv = vm_pager_get_pages(ksobj, &m, 1, 0); 1150 if (rv != VM_PAGER_OK) 1151 panic("pmap_swapin_thread: cannot get kstack"); 1152 m = vm_page_lookup(ksobj, i); 1153 m->valid = VM_PAGE_BITS_ALL; 1154 } 1155 ma[i] = m; 1156 vm_page_lock_queues(); 1157 vm_page_wire(m); 1158 vm_page_wakeup(m); 1159 vm_page_unlock_queues(); 1160 } 1161 pmap_qenter(ks, ma, pages); 1162} 1163 1164/* 1165 * Initialize the pmap associated with process 0. 1166 */ 1167void 1168pmap_pinit0(pmap_t pm) 1169{ 1170 int i; 1171 1172 for (i = 0; i < MAXCPU; i++) 1173 pm->pm_context[i] = 0; 1174 pm->pm_active = 0; 1175 pm->pm_tsb = NULL; 1176 pm->pm_tsb_obj = NULL; 1177 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1178} 1179 1180/* 1181 * Initialize a preallocated and zeroed pmap structure, uch as one in a 1182 * vmspace structure. 1183 */ 1184void 1185pmap_pinit(pmap_t pm) 1186{ 1187 vm_page_t ma[TSB_PAGES]; 1188 vm_page_t m; 1189 int i; 1190 1191 /* 1192 * Allocate kva space for the tsb. 1193 */ 1194 if (pm->pm_tsb == NULL) { 1195 pm->pm_tsb = (struct tte *)kmem_alloc_pageable(kernel_map, 1196 TSB_BSIZE); 1197 } 1198 1199 /* 1200 * Allocate an object for it. 1201 */ 1202 if (pm->pm_tsb_obj == NULL) 1203 pm->pm_tsb_obj = vm_object_allocate(OBJT_DEFAULT, TSB_PAGES); 1204 1205 for (i = 0; i < TSB_PAGES; i++) { 1206 m = vm_page_grab(pm->pm_tsb_obj, i, 1207 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1208 if ((m->flags & PG_ZERO) == 0) 1209 pmap_zero_page(m); 1210 1211 vm_page_lock_queues(); 1212 vm_page_flag_clear(m, PG_BUSY); 1213 m->valid = VM_PAGE_BITS_ALL; 1214 m->md.pmap = pm; 1215 vm_page_unlock_queues(); 1216 1217 ma[i] = m; 1218 } 1219 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1220 1221 for (i = 0; i < MAXCPU; i++) 1222 pm->pm_context[i] = -1; 1223 pm->pm_active = 0; 1224 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1225} 1226 1227void 1228pmap_pinit2(pmap_t pmap) 1229{ 1230 /* XXX: Remove this stub when no longer called */ 1231} 1232 1233/* 1234 * Release any resources held by the given physical map. 1235 * Called when a pmap initialized by pmap_pinit is being released. 1236 * Should only be called if the map contains no valid mappings. 1237 */ 1238void 1239pmap_release(pmap_t pm) 1240{ 1241 vm_object_t obj; 1242 vm_page_t m; 1243 1244 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1245 pm->pm_context[PCPU_GET(cpuid)], pm->pm_tsb); 1246 obj = pm->pm_tsb_obj; 1247 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1248 KASSERT(pmap_resident_count(pm) == 0, 1249 ("pmap_release: resident pages %ld != 0", 1250 pmap_resident_count(pm))); 1251 while (!TAILQ_EMPTY(&obj->memq)) { 1252 m = TAILQ_FIRST(&obj->memq); 1253 vm_page_lock_queues(); 1254 if (vm_page_sleep_if_busy(m, FALSE, "pmaprl")) 1255 continue; 1256 vm_page_busy(m); 1257 KASSERT(m->hold_count == 0, 1258 ("pmap_release: freeing held tsb page")); 1259 m->md.pmap = NULL; 1260 m->wire_count--; 1261 cnt.v_wire_count--; 1262 vm_page_free_zero(m); 1263 vm_page_unlock_queues(); 1264 } 1265 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1266} 1267 1268/* 1269 * Grow the number of kernel page table entries. Unneeded. 1270 */ 1271void 1272pmap_growkernel(vm_offset_t addr) 1273{ 1274 1275 panic("pmap_growkernel: can't grow kernel"); 1276} 1277 1278int 1279pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1280 vm_offset_t va) 1281{ 1282 vm_page_t m; 1283 u_long data; 1284 1285 data = atomic_readandclear_long(&tp->tte_data); 1286 if ((data & TD_FAKE) == 0) { 1287 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1288 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1289 if ((data & TD_WIRED) != 0) 1290 pm->pm_stats.wired_count--; 1291 if ((data & TD_PV) != 0) { 1292 if ((data & TD_W) != 0 && pmap_track_modified(pm, va)) 1293 vm_page_dirty(m); 1294 if ((data & TD_REF) != 0) 1295 vm_page_flag_set(m, PG_REFERENCED); 1296 if (TAILQ_EMPTY(&m->md.tte_list)) 1297 vm_page_flag_clear(m, PG_WRITEABLE); 1298 pm->pm_stats.resident_count--; 1299 } 1300 pmap_cache_remove(m, va); 1301 } 1302 TTE_ZERO(tp); 1303 if (PMAP_REMOVE_DONE(pm)) 1304 return (0); 1305 return (1); 1306} 1307 1308/* 1309 * Remove the given range of addresses from the specified map. 1310 */ 1311void 1312pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1313{ 1314 struct tte *tp; 1315 vm_offset_t va; 1316 1317 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1318 pm->pm_context[PCPU_GET(cpuid)], start, end); 1319 if (PMAP_REMOVE_DONE(pm)) 1320 return; 1321 if (end - start > PMAP_TSB_THRESH) { 1322 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1323 tlb_context_demap(pm); 1324 } else { 1325 for (va = start; va < end; va += PAGE_SIZE) { 1326 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1327 if (!pmap_remove_tte(pm, NULL, tp, va)) 1328 break; 1329 } 1330 } 1331 tlb_range_demap(pm, start, end - 1); 1332 } 1333} 1334 1335void 1336pmap_remove_all(vm_page_t m) 1337{ 1338 struct pmap *pm; 1339 struct tte *tpn; 1340 struct tte *tp; 1341 vm_offset_t va; 1342 1343 KASSERT((m->flags & (PG_FICTITIOUS|PG_UNMANAGED)) == 0, 1344 ("pmap_remove_all: illegal for unmanaged/fake page %#lx", 1345 VM_PAGE_TO_PHYS(m))); 1346 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1347 tpn = TAILQ_NEXT(tp, tte_link); 1348 if ((tp->tte_data & TD_PV) == 0) 1349 continue; 1350 pm = TTE_GET_PMAP(tp); 1351 va = TTE_GET_VA(tp); 1352 if ((tp->tte_data & TD_WIRED) != 0) 1353 pm->pm_stats.wired_count--; 1354 if ((tp->tte_data & TD_REF) != 0) 1355 vm_page_flag_set(m, PG_REFERENCED); 1356 if ((tp->tte_data & TD_W) != 0 && 1357 pmap_track_modified(pm, va)) 1358 vm_page_dirty(m); 1359 tp->tte_data &= ~TD_V; 1360 tlb_page_demap(pm, va); 1361 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1362 pm->pm_stats.resident_count--; 1363 pmap_cache_remove(m, va); 1364 TTE_ZERO(tp); 1365 } 1366 vm_page_flag_clear(m, PG_WRITEABLE); 1367} 1368 1369int 1370pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1371 vm_offset_t va) 1372{ 1373 u_long data; 1374 vm_page_t m; 1375 1376 data = atomic_clear_long(&tp->tte_data, TD_REF | TD_SW | TD_W); 1377 if ((data & TD_PV) != 0) { 1378 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1379 if ((data & TD_REF) != 0) 1380 vm_page_flag_set(m, PG_REFERENCED); 1381 if ((data & TD_W) != 0 && pmap_track_modified(pm, va)) 1382 vm_page_dirty(m); 1383 } 1384 return (0); 1385} 1386 1387/* 1388 * Set the physical protection on the specified range of this map as requested. 1389 */ 1390void 1391pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1392{ 1393 vm_offset_t va; 1394 struct tte *tp; 1395 1396 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1397 pm->pm_context[PCPU_GET(cpuid)], sva, eva, prot); 1398 1399 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1400 pmap_remove(pm, sva, eva); 1401 return; 1402 } 1403 1404 if (prot & VM_PROT_WRITE) 1405 return; 1406 1407 if (eva - sva > PMAP_TSB_THRESH) { 1408 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1409 tlb_context_demap(pm); 1410 } else { 1411 for (va = sva; va < eva; va += PAGE_SIZE) { 1412 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1413 pmap_protect_tte(pm, NULL, tp, va); 1414 } 1415 tlb_range_demap(pm, sva, eva - 1); 1416 } 1417} 1418 1419/* 1420 * Map the given physical page at the specified virtual address in the 1421 * target pmap with the protection requested. If specified the page 1422 * will be wired down. 1423 */ 1424void 1425pmap_enter(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1426 boolean_t wired) 1427{ 1428 struct tte *tp; 1429 vm_offset_t pa; 1430 u_long data; 1431 int i; 1432 1433 PMAP_STATS_INC(pmap_nenter); 1434 pa = VM_PAGE_TO_PHYS(m); 1435 1436 /* 1437 * If this is a fake page from the device_pager, but it covers actual 1438 * physical memory, convert to the real backing page. 1439 */ 1440 if ((m->flags & PG_FICTITIOUS) != 0) { 1441 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 1442 if (pa >= phys_avail[i] && pa <= phys_avail[i + 1]) { 1443 m = PHYS_TO_VM_PAGE(pa); 1444 break; 1445 } 1446 } 1447 } 1448 1449 CTR6(KTR_PMAP, 1450 "pmap_enter: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1451 pm->pm_context[PCPU_GET(cpuid)], m, va, pa, prot, wired); 1452 1453 /* 1454 * If there is an existing mapping, and the physical address has not 1455 * changed, must be protection or wiring change. 1456 */ 1457 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1458 CTR0(KTR_PMAP, "pmap_enter: update"); 1459 PMAP_STATS_INC(pmap_nenter_update); 1460 1461 /* 1462 * Wiring change, just update stats. 1463 */ 1464 if (wired) { 1465 if ((tp->tte_data & TD_WIRED) == 0) { 1466 tp->tte_data |= TD_WIRED; 1467 pm->pm_stats.wired_count++; 1468 } 1469 } else { 1470 if ((tp->tte_data & TD_WIRED) != 0) { 1471 tp->tte_data &= ~TD_WIRED; 1472 pm->pm_stats.wired_count--; 1473 } 1474 } 1475 1476 /* 1477 * Save the old bits and clear the ones we're interested in. 1478 */ 1479 data = tp->tte_data; 1480 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1481 1482 /* 1483 * If we're turning off write permissions, sense modify status. 1484 */ 1485 if ((prot & VM_PROT_WRITE) != 0) { 1486 tp->tte_data |= TD_SW; 1487 if (wired) { 1488 tp->tte_data |= TD_W; 1489 } 1490 } else if ((data & TD_W) != 0 && 1491 pmap_track_modified(pm, va)) { 1492 vm_page_dirty(m); 1493 } 1494 1495 /* 1496 * If we're turning on execute permissions, flush the icache. 1497 */ 1498 if ((prot & VM_PROT_EXECUTE) != 0) { 1499 if ((data & TD_EXEC) == 0) { 1500 PMAP_STATS_INC(pmap_niflush); 1501 icache_page_inval(pa); 1502 } 1503 tp->tte_data |= TD_EXEC; 1504 } 1505 1506 /* 1507 * Delete the old mapping. 1508 */ 1509 tlb_page_demap(pm, TTE_GET_VA(tp)); 1510 } else { 1511 /* 1512 * If there is an existing mapping, but its for a different 1513 * phsyical address, delete the old mapping. 1514 */ 1515 if (tp != NULL) { 1516 CTR0(KTR_PMAP, "pmap_enter: replace"); 1517 PMAP_STATS_INC(pmap_nenter_replace); 1518 vm_page_lock_queues(); 1519 pmap_remove_tte(pm, NULL, tp, va); 1520 vm_page_unlock_queues(); 1521 tlb_page_demap(pm, va); 1522 } else { 1523 CTR0(KTR_PMAP, "pmap_enter: new"); 1524 PMAP_STATS_INC(pmap_nenter_new); 1525 } 1526 1527 /* 1528 * Now set up the data and install the new mapping. 1529 */ 1530 data = TD_V | TD_8K | TD_PA(pa); 1531 if (pm == kernel_pmap) 1532 data |= TD_P; 1533 if (prot & VM_PROT_WRITE) 1534 data |= TD_SW; 1535 if (prot & VM_PROT_EXECUTE) { 1536 data |= TD_EXEC; 1537 PMAP_STATS_INC(pmap_niflush); 1538 icache_page_inval(pa); 1539 } 1540 1541 /* 1542 * If its wired update stats. We also don't need reference or 1543 * modify tracking for wired mappings, so set the bits now. 1544 */ 1545 if (wired) { 1546 pm->pm_stats.wired_count++; 1547 data |= TD_REF | TD_WIRED; 1548 if ((prot & VM_PROT_WRITE) != 0) 1549 data |= TD_W; 1550 } 1551 1552 tsb_tte_enter(pm, m, va, TS_8K, data); 1553 } 1554} 1555 1556void 1557pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1558 vm_pindex_t pindex, vm_size_t size, int limit) 1559{ 1560 /* XXX */ 1561} 1562 1563void 1564pmap_prefault(pmap_t pm, vm_offset_t va, vm_map_entry_t entry) 1565{ 1566 /* XXX */ 1567} 1568 1569/* 1570 * Change the wiring attribute for a map/virtual-address pair. 1571 * The mapping must already exist in the pmap. 1572 */ 1573void 1574pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1575{ 1576 struct tte *tp; 1577 u_long data; 1578 1579 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1580 if (wired) { 1581 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1582 if ((data & TD_WIRED) == 0) 1583 pm->pm_stats.wired_count++; 1584 } else { 1585 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1586 if ((data & TD_WIRED) != 0) 1587 pm->pm_stats.wired_count--; 1588 } 1589 } 1590} 1591 1592static int 1593pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, vm_offset_t va) 1594{ 1595 vm_page_t m; 1596 u_long data; 1597 1598 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1599 data = tp->tte_data & 1600 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1601 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1602 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1603 } 1604 return (1); 1605} 1606 1607void 1608pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1609 vm_size_t len, vm_offset_t src_addr) 1610{ 1611 struct tte *tp; 1612 vm_offset_t va; 1613 1614 if (dst_addr != src_addr) 1615 return; 1616 if (len > PMAP_TSB_THRESH) { 1617 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1618 pmap_copy_tte); 1619 tlb_context_demap(dst_pmap); 1620 } else { 1621 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) { 1622 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1623 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1624 } 1625 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1626 } 1627} 1628 1629void 1630pmap_zero_page(vm_page_t m) 1631{ 1632 vm_offset_t pa; 1633 vm_offset_t va; 1634 struct tte *tp; 1635 1636 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1637 ("pmap_zero_page: fake page")); 1638 PMAP_STATS_INC(pmap_nzero_page); 1639 pa = VM_PAGE_TO_PHYS(m); 1640 if (m->md.color == -1) { 1641 PMAP_STATS_INC(pmap_nzero_page_nc); 1642 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1643 } else if (m->md.color == DCACHE_COLOR(pa)) { 1644 PMAP_STATS_INC(pmap_nzero_page_c); 1645 va = TLB_PHYS_TO_DIRECT(pa); 1646 bzero((void *)va, PAGE_SIZE); 1647 } else { 1648 PMAP_STATS_INC(pmap_nzero_page_oc); 1649 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1650 tp = tsb_kvtotte(va); 1651 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1652 tp->tte_vpn = TV_VPN(va, TS_8K); 1653 bzero((void *)va, PAGE_SIZE); 1654 tlb_page_demap(kernel_pmap, va); 1655 } 1656} 1657 1658void 1659pmap_zero_page_area(vm_page_t m, int off, int size) 1660{ 1661 vm_offset_t pa; 1662 vm_offset_t va; 1663 struct tte *tp; 1664 1665 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1666 ("pmap_zero_page_area: fake page")); 1667 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1668 PMAP_STATS_INC(pmap_nzero_page_area); 1669 pa = VM_PAGE_TO_PHYS(m); 1670 if (m->md.color == -1) { 1671 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1672 aszero(ASI_PHYS_USE_EC, pa + off, size); 1673 } else if (m->md.color == DCACHE_COLOR(pa)) { 1674 PMAP_STATS_INC(pmap_nzero_page_area_c); 1675 va = TLB_PHYS_TO_DIRECT(pa); 1676 bzero((void *)(va + off), size); 1677 } else { 1678 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1679 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1680 tp = tsb_kvtotte(va); 1681 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1682 tp->tte_vpn = TV_VPN(va, TS_8K); 1683 bzero((void *)(va + off), size); 1684 tlb_page_demap(kernel_pmap, va); 1685 } 1686} 1687 1688void 1689pmap_zero_page_idle(vm_page_t m) 1690{ 1691 vm_offset_t pa; 1692 vm_offset_t va; 1693 struct tte *tp; 1694 1695 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1696 ("pmap_zero_page_idle: fake page")); 1697 PMAP_STATS_INC(pmap_nzero_page_idle); 1698 pa = VM_PAGE_TO_PHYS(m); 1699 if (m->md.color == -1) { 1700 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1701 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1702 } else if (m->md.color == DCACHE_COLOR(pa)) { 1703 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1704 va = TLB_PHYS_TO_DIRECT(pa); 1705 bzero((void *)va, PAGE_SIZE); 1706 } else { 1707 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1708 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1709 tp = tsb_kvtotte(va); 1710 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1711 tp->tte_vpn = TV_VPN(va, TS_8K); 1712 bzero((void *)va, PAGE_SIZE); 1713 tlb_page_demap(kernel_pmap, va); 1714 } 1715} 1716 1717void 1718pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1719{ 1720 vm_offset_t pdst; 1721 vm_offset_t psrc; 1722 vm_offset_t vdst; 1723 vm_offset_t vsrc; 1724 struct tte *tp; 1725 1726 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1727 ("pmap_copy_page: fake dst page")); 1728 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1729 ("pmap_copy_page: fake src page")); 1730 PMAP_STATS_INC(pmap_ncopy_page); 1731 pdst = VM_PAGE_TO_PHYS(mdst); 1732 psrc = VM_PAGE_TO_PHYS(msrc); 1733 if (msrc->md.color == -1 && mdst->md.color == -1) { 1734 PMAP_STATS_INC(pmap_ncopy_page_nc); 1735 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1736 } else if (msrc->md.color == DCACHE_COLOR(psrc) && 1737 mdst->md.color == DCACHE_COLOR(pdst)) { 1738 PMAP_STATS_INC(pmap_ncopy_page_c); 1739 vdst = TLB_PHYS_TO_DIRECT(pdst); 1740 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1741 bcopy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1742 } else if (msrc->md.color == -1) { 1743 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1744 PMAP_STATS_INC(pmap_ncopy_page_dc); 1745 vdst = TLB_PHYS_TO_DIRECT(pdst); 1746 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1747 PAGE_SIZE); 1748 } else { 1749 PMAP_STATS_INC(pmap_ncopy_page_doc); 1750 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1751 tp = tsb_kvtotte(vdst); 1752 tp->tte_data = 1753 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1754 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1755 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1756 PAGE_SIZE); 1757 tlb_page_demap(kernel_pmap, vdst); 1758 } 1759 } else if (mdst->md.color == -1) { 1760 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1761 PMAP_STATS_INC(pmap_ncopy_page_sc); 1762 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1763 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1764 PAGE_SIZE); 1765 } else { 1766 PMAP_STATS_INC(pmap_ncopy_page_soc); 1767 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1768 tp = tsb_kvtotte(vsrc); 1769 tp->tte_data = 1770 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1771 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1772 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1773 PAGE_SIZE); 1774 tlb_page_demap(kernel_pmap, vsrc); 1775 } 1776 } else { 1777 PMAP_STATS_INC(pmap_ncopy_page_oc); 1778 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1779 tp = tsb_kvtotte(vdst); 1780 tp->tte_data = 1781 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1782 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1783 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1784 tp = tsb_kvtotte(vsrc); 1785 tp->tte_data = 1786 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1787 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1788 bcopy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1789 tlb_page_demap(kernel_pmap, vdst); 1790 tlb_page_demap(kernel_pmap, vsrc); 1791 } 1792} 1793 1794/* 1795 * Returns true if the pmap's pv is one of the first 1796 * 16 pvs linked to from this page. This count may 1797 * be changed upwards or downwards in the future; it 1798 * is only necessary that true be returned for a small 1799 * subset of pmaps for proper page aging. 1800 */ 1801boolean_t 1802pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1803{ 1804 struct tte *tp; 1805 int loops; 1806 1807 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1808 return (FALSE); 1809 loops = 0; 1810 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1811 if ((tp->tte_data & TD_PV) == 0) 1812 continue; 1813 if (TTE_GET_PMAP(tp) == pm) 1814 return (TRUE); 1815 if (++loops >= 16) 1816 break; 1817 } 1818 return (FALSE); 1819} 1820 1821/* 1822 * Remove all pages from specified address space, this aids process exit 1823 * speeds. This is much faster than pmap_remove n the case of running down 1824 * an entire address space. Only works for the current pmap. 1825 */ 1826void 1827pmap_remove_pages(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 1828{ 1829} 1830 1831/* 1832 * Lower the permission for all mappings to a given page. 1833 */ 1834void 1835pmap_page_protect(vm_page_t m, vm_prot_t prot) 1836{ 1837 1838 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1839 ("pmap_page_protect: fake page")); 1840 if ((prot & VM_PROT_WRITE) == 0) { 1841 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) 1842 pmap_clear_write(m); 1843 else 1844 pmap_remove_all(m); 1845 } 1846} 1847 1848/* 1849 * pmap_ts_referenced: 1850 * 1851 * Return a count of reference bits for a page, clearing those bits. 1852 * It is not necessary for every reference bit to be cleared, but it 1853 * is necessary that 0 only be returned when there are truly no 1854 * reference bits set. 1855 * 1856 * XXX: The exact number of bits to check and clear is a matter that 1857 * should be tested and standardized at some point in the future for 1858 * optimal aging of shared pages. 1859 */ 1860 1861int 1862pmap_ts_referenced(vm_page_t m) 1863{ 1864 struct tte *tpf; 1865 struct tte *tpn; 1866 struct tte *tp; 1867 u_long data; 1868 int count; 1869 1870 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1871 return (0); 1872 count = 0; 1873 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 1874 tpf = tp; 1875 do { 1876 tpn = TAILQ_NEXT(tp, tte_link); 1877 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1878 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1879 if ((tp->tte_data & TD_PV) == 0 || 1880 !pmap_track_modified(TTE_GET_PMAP(tp), 1881 TTE_GET_VA(tp))) 1882 continue; 1883 data = atomic_clear_long(&tp->tte_data, TD_REF); 1884 if ((data & TD_REF) != 0 && ++count > 4) 1885 break; 1886 } while ((tp = tpn) != NULL && tp != tpf); 1887 } 1888 return (count); 1889} 1890 1891boolean_t 1892pmap_is_modified(vm_page_t m) 1893{ 1894 struct tte *tp; 1895 1896 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1897 return FALSE; 1898 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1899 if ((tp->tte_data & TD_PV) == 0 || 1900 !pmap_track_modified(TTE_GET_PMAP(tp), TTE_GET_VA(tp))) 1901 continue; 1902 if ((tp->tte_data & TD_W) != 0) 1903 return (TRUE); 1904 } 1905 return (FALSE); 1906} 1907 1908void 1909pmap_clear_modify(vm_page_t m) 1910{ 1911 struct tte *tp; 1912 u_long data; 1913 1914 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1915 return; 1916 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1917 if ((tp->tte_data & TD_PV) == 0) 1918 continue; 1919 data = atomic_clear_long(&tp->tte_data, TD_W); 1920 if ((data & TD_W) != 0) 1921 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1922 } 1923} 1924 1925void 1926pmap_clear_reference(vm_page_t m) 1927{ 1928 struct tte *tp; 1929 u_long data; 1930 1931 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) 1932 return; 1933 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1934 if ((tp->tte_data & TD_PV) == 0) 1935 continue; 1936 data = atomic_clear_long(&tp->tte_data, TD_REF); 1937 if ((data & TD_REF) != 0) 1938 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1939 } 1940} 1941 1942void 1943pmap_clear_write(vm_page_t m) 1944{ 1945 struct tte *tp; 1946 u_long data; 1947 1948 if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 || 1949 (m->flags & PG_WRITEABLE) == 0) 1950 return; 1951 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1952 if ((tp->tte_data & TD_PV) == 0) 1953 continue; 1954 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1955 if ((data & TD_W) != 0) { 1956 if (pmap_track_modified(TTE_GET_PMAP(tp), 1957 TTE_GET_VA(tp))) 1958 vm_page_dirty(m); 1959 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1960 } 1961 } 1962 vm_page_flag_clear(m, PG_WRITEABLE); 1963} 1964 1965int 1966pmap_mincore(pmap_t pm, vm_offset_t addr) 1967{ 1968 /* TODO; */ 1969 return (0); 1970} 1971 1972/* 1973 * Activate a user pmap. The pmap must be activated before its address space 1974 * can be accessed in any way. 1975 */ 1976void 1977pmap_activate(struct thread *td) 1978{ 1979 struct vmspace *vm; 1980 vm_offset_t tsb; 1981 u_long context; 1982 pmap_t pm; 1983 1984 vm = td->td_proc->p_vmspace; 1985 pm = &vm->vm_pmap; 1986 tsb = (vm_offset_t)pm->pm_tsb; 1987 1988 KASSERT(pm->pm_active == 0, ("pmap_activate: pmap already active?")); 1989 KASSERT(pm->pm_context[PCPU_GET(cpuid)] != 0, 1990 ("pmap_activate: activating nucleus context?")); 1991 1992 mtx_lock_spin(&sched_lock); 1993 stxa(AA_DMMU_TSB, ASI_DMMU, tsb); 1994 stxa(AA_IMMU_TSB, ASI_IMMU, tsb); 1995 membar(Sync); 1996 context = pmap_context_alloc(); 1997 pm->pm_context[PCPU_GET(cpuid)] = context; 1998 pm->pm_active |= PCPU_GET(cpumask); 1999 PCPU_SET(vmspace, vm); 2000 stxa(AA_DMMU_PCXR, ASI_DMMU, context); 2001 membar(Sync); 2002 mtx_unlock_spin(&sched_lock); 2003} 2004 2005vm_offset_t 2006pmap_addr_hint(vm_object_t object, vm_offset_t va, vm_size_t size) 2007{ 2008 2009 return (va); 2010} 2011