pmap.c revision 250884
1/*- 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * Copyright (c) 1994 John S. Dyson 5 * All rights reserved. 6 * Copyright (c) 1994 David Greenman 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * the Systems Programming Group of the University of Utah Computer 11 * Science Department and William Jolitz of UUNET Technologies Inc. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 4. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 * 37 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 38 */ 39 40#include <sys/cdefs.h> 41__FBSDID("$FreeBSD: head/sys/sparc64/sparc64/pmap.c 250884 2013-05-21 20:38:19Z attilio $"); 42 43/* 44 * Manages physical address maps. 45 * 46 * Since the information managed by this module is also stored by the 47 * logical address mapping module, this module may throw away valid virtual 48 * to physical mappings at almost any time. However, invalidations of 49 * mappings must be done as requested. 50 * 51 * In order to cope with hardware architectures which make virtual to 52 * physical map invalidates expensive, this module may delay invalidate 53 * reduced protection operations until such time as they are actually 54 * necessary. This module is given full information as to which processors 55 * are currently using which maps, and to when physical maps must be made 56 * correct. 57 */ 58 59#include "opt_kstack_pages.h" 60#include "opt_pmap.h" 61 62#include <sys/param.h> 63#include <sys/kernel.h> 64#include <sys/ktr.h> 65#include <sys/lock.h> 66#include <sys/msgbuf.h> 67#include <sys/mutex.h> 68#include <sys/proc.h> 69#include <sys/rwlock.h> 70#include <sys/smp.h> 71#include <sys/sysctl.h> 72#include <sys/systm.h> 73#include <sys/vmmeter.h> 74 75#include <dev/ofw/openfirm.h> 76 77#include <vm/vm.h> 78#include <vm/vm_param.h> 79#include <vm/vm_kern.h> 80#include <vm/vm_page.h> 81#include <vm/vm_map.h> 82#include <vm/vm_object.h> 83#include <vm/vm_extern.h> 84#include <vm/vm_pageout.h> 85#include <vm/vm_pager.h> 86#include <vm/vm_phys.h> 87 88#include <machine/cache.h> 89#include <machine/frame.h> 90#include <machine/instr.h> 91#include <machine/md_var.h> 92#include <machine/metadata.h> 93#include <machine/ofw_mem.h> 94#include <machine/smp.h> 95#include <machine/tlb.h> 96#include <machine/tte.h> 97#include <machine/tsb.h> 98#include <machine/ver.h> 99 100/* 101 * Virtual address of message buffer 102 */ 103struct msgbuf *msgbufp; 104 105/* 106 * Map of physical memory reagions 107 */ 108vm_paddr_t phys_avail[128]; 109static struct ofw_mem_region mra[128]; 110struct ofw_mem_region sparc64_memreg[128]; 111int sparc64_nmemreg; 112static struct ofw_map translations[128]; 113static int translations_size; 114 115static vm_offset_t pmap_idle_map; 116static vm_offset_t pmap_temp_map_1; 117static vm_offset_t pmap_temp_map_2; 118 119/* 120 * First and last available kernel virtual addresses 121 */ 122vm_offset_t virtual_avail; 123vm_offset_t virtual_end; 124vm_offset_t kernel_vm_end; 125 126vm_offset_t vm_max_kernel_address; 127 128/* 129 * Kernel pmap 130 */ 131struct pmap kernel_pmap_store; 132 133struct rwlock_padalign tte_list_global_lock; 134 135/* 136 * Allocate physical memory for use in pmap_bootstrap. 137 */ 138static vm_paddr_t pmap_bootstrap_alloc(vm_size_t size, uint32_t colors); 139 140static void pmap_bootstrap_set_tte(struct tte *tp, u_long vpn, u_long data); 141static void pmap_cache_remove(vm_page_t m, vm_offset_t va); 142static int pmap_protect_tte(struct pmap *pm1, struct pmap *pm2, 143 struct tte *tp, vm_offset_t va); 144 145/* 146 * Map the given physical page at the specified virtual address in the 147 * target pmap with the protection requested. If specified the page 148 * will be wired down. 149 * 150 * The page queues and pmap must be locked. 151 */ 152static void pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, 153 vm_prot_t prot, boolean_t wired); 154 155extern int tl1_dmmu_miss_direct_patch_tsb_phys_1[]; 156extern int tl1_dmmu_miss_direct_patch_tsb_phys_end_1[]; 157extern int tl1_dmmu_miss_patch_asi_1[]; 158extern int tl1_dmmu_miss_patch_quad_ldd_1[]; 159extern int tl1_dmmu_miss_patch_tsb_1[]; 160extern int tl1_dmmu_miss_patch_tsb_2[]; 161extern int tl1_dmmu_miss_patch_tsb_mask_1[]; 162extern int tl1_dmmu_miss_patch_tsb_mask_2[]; 163extern int tl1_dmmu_prot_patch_asi_1[]; 164extern int tl1_dmmu_prot_patch_quad_ldd_1[]; 165extern int tl1_dmmu_prot_patch_tsb_1[]; 166extern int tl1_dmmu_prot_patch_tsb_2[]; 167extern int tl1_dmmu_prot_patch_tsb_mask_1[]; 168extern int tl1_dmmu_prot_patch_tsb_mask_2[]; 169extern int tl1_immu_miss_patch_asi_1[]; 170extern int tl1_immu_miss_patch_quad_ldd_1[]; 171extern int tl1_immu_miss_patch_tsb_1[]; 172extern int tl1_immu_miss_patch_tsb_2[]; 173extern int tl1_immu_miss_patch_tsb_mask_1[]; 174extern int tl1_immu_miss_patch_tsb_mask_2[]; 175 176/* 177 * If user pmap is processed with pmap_remove and with pmap_remove and the 178 * resident count drops to 0, there are no more pages to remove, so we 179 * need not continue. 180 */ 181#define PMAP_REMOVE_DONE(pm) \ 182 ((pm) != kernel_pmap && (pm)->pm_stats.resident_count == 0) 183 184/* 185 * The threshold (in bytes) above which tsb_foreach() is used in pmap_remove() 186 * and pmap_protect() instead of trying each virtual address. 187 */ 188#define PMAP_TSB_THRESH ((TSB_SIZE / 2) * PAGE_SIZE) 189 190SYSCTL_NODE(_debug, OID_AUTO, pmap_stats, CTLFLAG_RD, 0, ""); 191 192PMAP_STATS_VAR(pmap_nenter); 193PMAP_STATS_VAR(pmap_nenter_update); 194PMAP_STATS_VAR(pmap_nenter_replace); 195PMAP_STATS_VAR(pmap_nenter_new); 196PMAP_STATS_VAR(pmap_nkenter); 197PMAP_STATS_VAR(pmap_nkenter_oc); 198PMAP_STATS_VAR(pmap_nkenter_stupid); 199PMAP_STATS_VAR(pmap_nkremove); 200PMAP_STATS_VAR(pmap_nqenter); 201PMAP_STATS_VAR(pmap_nqremove); 202PMAP_STATS_VAR(pmap_ncache_enter); 203PMAP_STATS_VAR(pmap_ncache_enter_c); 204PMAP_STATS_VAR(pmap_ncache_enter_oc); 205PMAP_STATS_VAR(pmap_ncache_enter_cc); 206PMAP_STATS_VAR(pmap_ncache_enter_coc); 207PMAP_STATS_VAR(pmap_ncache_enter_nc); 208PMAP_STATS_VAR(pmap_ncache_enter_cnc); 209PMAP_STATS_VAR(pmap_ncache_remove); 210PMAP_STATS_VAR(pmap_ncache_remove_c); 211PMAP_STATS_VAR(pmap_ncache_remove_oc); 212PMAP_STATS_VAR(pmap_ncache_remove_cc); 213PMAP_STATS_VAR(pmap_ncache_remove_coc); 214PMAP_STATS_VAR(pmap_ncache_remove_nc); 215PMAP_STATS_VAR(pmap_nzero_page); 216PMAP_STATS_VAR(pmap_nzero_page_c); 217PMAP_STATS_VAR(pmap_nzero_page_oc); 218PMAP_STATS_VAR(pmap_nzero_page_nc); 219PMAP_STATS_VAR(pmap_nzero_page_area); 220PMAP_STATS_VAR(pmap_nzero_page_area_c); 221PMAP_STATS_VAR(pmap_nzero_page_area_oc); 222PMAP_STATS_VAR(pmap_nzero_page_area_nc); 223PMAP_STATS_VAR(pmap_nzero_page_idle); 224PMAP_STATS_VAR(pmap_nzero_page_idle_c); 225PMAP_STATS_VAR(pmap_nzero_page_idle_oc); 226PMAP_STATS_VAR(pmap_nzero_page_idle_nc); 227PMAP_STATS_VAR(pmap_ncopy_page); 228PMAP_STATS_VAR(pmap_ncopy_page_c); 229PMAP_STATS_VAR(pmap_ncopy_page_oc); 230PMAP_STATS_VAR(pmap_ncopy_page_nc); 231PMAP_STATS_VAR(pmap_ncopy_page_dc); 232PMAP_STATS_VAR(pmap_ncopy_page_doc); 233PMAP_STATS_VAR(pmap_ncopy_page_sc); 234PMAP_STATS_VAR(pmap_ncopy_page_soc); 235 236PMAP_STATS_VAR(pmap_nnew_thread); 237PMAP_STATS_VAR(pmap_nnew_thread_oc); 238 239static inline u_long dtlb_get_data(u_int tlb, u_int slot); 240 241/* 242 * Quick sort callout for comparing memory regions 243 */ 244static int mr_cmp(const void *a, const void *b); 245static int om_cmp(const void *a, const void *b); 246 247static int 248mr_cmp(const void *a, const void *b) 249{ 250 const struct ofw_mem_region *mra; 251 const struct ofw_mem_region *mrb; 252 253 mra = a; 254 mrb = b; 255 if (mra->mr_start < mrb->mr_start) 256 return (-1); 257 else if (mra->mr_start > mrb->mr_start) 258 return (1); 259 else 260 return (0); 261} 262 263static int 264om_cmp(const void *a, const void *b) 265{ 266 const struct ofw_map *oma; 267 const struct ofw_map *omb; 268 269 oma = a; 270 omb = b; 271 if (oma->om_start < omb->om_start) 272 return (-1); 273 else if (oma->om_start > omb->om_start) 274 return (1); 275 else 276 return (0); 277} 278 279static inline u_long 280dtlb_get_data(u_int tlb, u_int slot) 281{ 282 u_long data; 283 register_t s; 284 285 slot = TLB_DAR_SLOT(tlb, slot); 286 /* 287 * We read ASI_DTLB_DATA_ACCESS_REG twice back-to-back in order to 288 * work around errata of USIII and beyond. 289 */ 290 s = intr_disable(); 291 (void)ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 292 data = ldxa(slot, ASI_DTLB_DATA_ACCESS_REG); 293 intr_restore(s); 294 return (data); 295} 296 297/* 298 * Bootstrap the system enough to run with virtual memory. 299 */ 300void 301pmap_bootstrap(u_int cpu_impl) 302{ 303 struct pmap *pm; 304 struct tte *tp; 305 vm_offset_t off; 306 vm_offset_t va; 307 vm_paddr_t pa; 308 vm_size_t physsz; 309 vm_size_t virtsz; 310 u_long data; 311 u_long vpn; 312 phandle_t pmem; 313 phandle_t vmem; 314 u_int dtlb_slots_avail; 315 int i; 316 int j; 317 int sz; 318 uint32_t asi; 319 uint32_t colors; 320 uint32_t ldd; 321 322 /* 323 * Set the kernel context. 324 */ 325 pmap_set_kctx(); 326 327 colors = dcache_color_ignore != 0 ? 1 : DCACHE_COLORS; 328 329 /* 330 * Find out what physical memory is available from the PROM and 331 * initialize the phys_avail array. This must be done before 332 * pmap_bootstrap_alloc is called. 333 */ 334 if ((pmem = OF_finddevice("/memory")) == -1) 335 OF_panic("%s: finddevice /memory", __func__); 336 if ((sz = OF_getproplen(pmem, "available")) == -1) 337 OF_panic("%s: getproplen /memory/available", __func__); 338 if (sizeof(phys_avail) < sz) 339 OF_panic("%s: phys_avail too small", __func__); 340 if (sizeof(mra) < sz) 341 OF_panic("%s: mra too small", __func__); 342 bzero(mra, sz); 343 if (OF_getprop(pmem, "available", mra, sz) == -1) 344 OF_panic("%s: getprop /memory/available", __func__); 345 sz /= sizeof(*mra); 346 CTR0(KTR_PMAP, "pmap_bootstrap: physical memory"); 347 qsort(mra, sz, sizeof (*mra), mr_cmp); 348 physsz = 0; 349 getenv_quad("hw.physmem", &physmem); 350 physmem = btoc(physmem); 351 for (i = 0, j = 0; i < sz; i++, j += 2) { 352 CTR2(KTR_PMAP, "start=%#lx size=%#lx", mra[i].mr_start, 353 mra[i].mr_size); 354 if (physmem != 0 && btoc(physsz + mra[i].mr_size) >= physmem) { 355 if (btoc(physsz) < physmem) { 356 phys_avail[j] = mra[i].mr_start; 357 phys_avail[j + 1] = mra[i].mr_start + 358 (ctob(physmem) - physsz); 359 physsz = ctob(physmem); 360 } 361 break; 362 } 363 phys_avail[j] = mra[i].mr_start; 364 phys_avail[j + 1] = mra[i].mr_start + mra[i].mr_size; 365 physsz += mra[i].mr_size; 366 } 367 physmem = btoc(physsz); 368 369 /* 370 * Calculate the size of kernel virtual memory, and the size and mask 371 * for the kernel TSB based on the phsyical memory size but limited 372 * by the amount of dTLB slots available for locked entries if we have 373 * to lock the TSB in the TLB (given that for spitfire-class CPUs all 374 * of the dt64 slots can hold locked entries but there is no large 375 * dTLB for unlocked ones, we don't use more than half of it for the 376 * TSB). 377 * Note that for reasons unknown OpenSolaris doesn't take advantage of 378 * ASI_ATOMIC_QUAD_LDD_PHYS on UltraSPARC-III. However, given that no 379 * public documentation is available for these, the latter just might 380 * not support it, yet. 381 */ 382 if (cpu_impl == CPU_IMPL_SPARC64V || 383 cpu_impl >= CPU_IMPL_ULTRASPARCIIIp) { 384 tsb_kernel_ldd_phys = 1; 385 virtsz = roundup(5 / 3 * physsz, PAGE_SIZE_4M << 386 (PAGE_SHIFT - TTE_SHIFT)); 387 } else { 388 dtlb_slots_avail = 0; 389 for (i = 0; i < dtlb_slots; i++) { 390 data = dtlb_get_data(cpu_impl == 391 CPU_IMPL_ULTRASPARCIII ? TLB_DAR_T16 : 392 TLB_DAR_T32, i); 393 if ((data & (TD_V | TD_L)) != (TD_V | TD_L)) 394 dtlb_slots_avail++; 395 } 396#ifdef SMP 397 dtlb_slots_avail -= PCPU_PAGES; 398#endif 399 if (cpu_impl >= CPU_IMPL_ULTRASPARCI && 400 cpu_impl < CPU_IMPL_ULTRASPARCIII) 401 dtlb_slots_avail /= 2; 402 virtsz = roundup(physsz, PAGE_SIZE_4M << 403 (PAGE_SHIFT - TTE_SHIFT)); 404 virtsz = MIN(virtsz, (dtlb_slots_avail * PAGE_SIZE_4M) << 405 (PAGE_SHIFT - TTE_SHIFT)); 406 } 407 vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; 408 tsb_kernel_size = virtsz >> (PAGE_SHIFT - TTE_SHIFT); 409 tsb_kernel_mask = (tsb_kernel_size >> TTE_SHIFT) - 1; 410 411 /* 412 * Allocate the kernel TSB and lock it in the TLB if necessary. 413 */ 414 pa = pmap_bootstrap_alloc(tsb_kernel_size, colors); 415 if (pa & PAGE_MASK_4M) 416 OF_panic("%s: TSB unaligned", __func__); 417 tsb_kernel_phys = pa; 418 if (tsb_kernel_ldd_phys == 0) { 419 tsb_kernel = 420 (struct tte *)(VM_MIN_KERNEL_ADDRESS - tsb_kernel_size); 421 pmap_map_tsb(); 422 bzero(tsb_kernel, tsb_kernel_size); 423 } else { 424 tsb_kernel = 425 (struct tte *)TLB_PHYS_TO_DIRECT(tsb_kernel_phys); 426 aszero(ASI_PHYS_USE_EC, tsb_kernel_phys, tsb_kernel_size); 427 } 428 429 /* 430 * Allocate and map the dynamic per-CPU area for the BSP. 431 */ 432 pa = pmap_bootstrap_alloc(DPCPU_SIZE, colors); 433 dpcpu0 = (void *)TLB_PHYS_TO_DIRECT(pa); 434 435 /* 436 * Allocate and map the message buffer. 437 */ 438 pa = pmap_bootstrap_alloc(msgbufsize, colors); 439 msgbufp = (struct msgbuf *)TLB_PHYS_TO_DIRECT(pa); 440 441 /* 442 * Patch the TSB addresses and mask as well as the ASIs used to load 443 * it into the trap table. 444 */ 445 446#define LDDA_R_I_R(rd, imm_asi, rs1, rs2) \ 447 (EIF_OP(IOP_LDST) | EIF_F3_RD(rd) | EIF_F3_OP3(INS3_LDDA) | \ 448 EIF_F3_RS1(rs1) | EIF_F3_I(0) | EIF_F3_IMM_ASI(imm_asi) | \ 449 EIF_F3_RS2(rs2)) 450#define OR_R_I_R(rd, imm13, rs1) \ 451 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_OR) | \ 452 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 453#define SETHI(rd, imm22) \ 454 (EIF_OP(IOP_FORM2) | EIF_F2_RD(rd) | EIF_F2_OP2(INS0_SETHI) | \ 455 EIF_IMM((imm22) >> 10, 22)) 456#define WR_R_I(rd, imm13, rs1) \ 457 (EIF_OP(IOP_MISC) | EIF_F3_RD(rd) | EIF_F3_OP3(INS2_WR) | \ 458 EIF_F3_RS1(rs1) | EIF_F3_I(1) | EIF_IMM(imm13, 13)) 459 460#define PATCH_ASI(addr, asi) do { \ 461 if (addr[0] != WR_R_I(IF_F3_RD(addr[0]), 0x0, \ 462 IF_F3_RS1(addr[0]))) \ 463 OF_panic("%s: patched instructions have changed", \ 464 __func__); \ 465 addr[0] |= EIF_IMM((asi), 13); \ 466 flush(addr); \ 467} while (0) 468 469#define PATCH_LDD(addr, asi) do { \ 470 if (addr[0] != LDDA_R_I_R(IF_F3_RD(addr[0]), 0x0, \ 471 IF_F3_RS1(addr[0]), IF_F3_RS2(addr[0]))) \ 472 OF_panic("%s: patched instructions have changed", \ 473 __func__); \ 474 addr[0] |= EIF_F3_IMM_ASI(asi); \ 475 flush(addr); \ 476} while (0) 477 478#define PATCH_TSB(addr, val) do { \ 479 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 480 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, \ 481 IF_F3_RS1(addr[1])) || \ 482 addr[3] != SETHI(IF_F2_RD(addr[3]), 0x0)) \ 483 OF_panic("%s: patched instructions have changed", \ 484 __func__); \ 485 addr[0] |= EIF_IMM((val) >> 42, 22); \ 486 addr[1] |= EIF_IMM((val) >> 32, 10); \ 487 addr[3] |= EIF_IMM((val) >> 10, 22); \ 488 flush(addr); \ 489 flush(addr + 1); \ 490 flush(addr + 3); \ 491} while (0) 492 493#define PATCH_TSB_MASK(addr, val) do { \ 494 if (addr[0] != SETHI(IF_F2_RD(addr[0]), 0x0) || \ 495 addr[1] != OR_R_I_R(IF_F3_RD(addr[1]), 0x0, \ 496 IF_F3_RS1(addr[1]))) \ 497 OF_panic("%s: patched instructions have changed", \ 498 __func__); \ 499 addr[0] |= EIF_IMM((val) >> 10, 22); \ 500 addr[1] |= EIF_IMM((val), 10); \ 501 flush(addr); \ 502 flush(addr + 1); \ 503} while (0) 504 505 if (tsb_kernel_ldd_phys == 0) { 506 asi = ASI_N; 507 ldd = ASI_NUCLEUS_QUAD_LDD; 508 off = (vm_offset_t)tsb_kernel; 509 } else { 510 asi = ASI_PHYS_USE_EC; 511 ldd = ASI_ATOMIC_QUAD_LDD_PHYS; 512 off = (vm_offset_t)tsb_kernel_phys; 513 } 514 PATCH_TSB(tl1_dmmu_miss_direct_patch_tsb_phys_1, tsb_kernel_phys); 515 PATCH_TSB(tl1_dmmu_miss_direct_patch_tsb_phys_end_1, 516 tsb_kernel_phys + tsb_kernel_size - 1); 517 PATCH_ASI(tl1_dmmu_miss_patch_asi_1, asi); 518 PATCH_LDD(tl1_dmmu_miss_patch_quad_ldd_1, ldd); 519 PATCH_TSB(tl1_dmmu_miss_patch_tsb_1, off); 520 PATCH_TSB(tl1_dmmu_miss_patch_tsb_2, off); 521 PATCH_TSB_MASK(tl1_dmmu_miss_patch_tsb_mask_1, tsb_kernel_mask); 522 PATCH_TSB_MASK(tl1_dmmu_miss_patch_tsb_mask_2, tsb_kernel_mask); 523 PATCH_ASI(tl1_dmmu_prot_patch_asi_1, asi); 524 PATCH_LDD(tl1_dmmu_prot_patch_quad_ldd_1, ldd); 525 PATCH_TSB(tl1_dmmu_prot_patch_tsb_1, off); 526 PATCH_TSB(tl1_dmmu_prot_patch_tsb_2, off); 527 PATCH_TSB_MASK(tl1_dmmu_prot_patch_tsb_mask_1, tsb_kernel_mask); 528 PATCH_TSB_MASK(tl1_dmmu_prot_patch_tsb_mask_2, tsb_kernel_mask); 529 PATCH_ASI(tl1_immu_miss_patch_asi_1, asi); 530 PATCH_LDD(tl1_immu_miss_patch_quad_ldd_1, ldd); 531 PATCH_TSB(tl1_immu_miss_patch_tsb_1, off); 532 PATCH_TSB(tl1_immu_miss_patch_tsb_2, off); 533 PATCH_TSB_MASK(tl1_immu_miss_patch_tsb_mask_1, tsb_kernel_mask); 534 PATCH_TSB_MASK(tl1_immu_miss_patch_tsb_mask_2, tsb_kernel_mask); 535 536 /* 537 * Enter fake 8k pages for the 4MB kernel pages, so that 538 * pmap_kextract() will work for them. 539 */ 540 for (i = 0; i < kernel_tlb_slots; i++) { 541 pa = kernel_tlbs[i].te_pa; 542 va = kernel_tlbs[i].te_va; 543 for (off = 0; off < PAGE_SIZE_4M; off += PAGE_SIZE) { 544 tp = tsb_kvtotte(va + off); 545 vpn = TV_VPN(va + off, TS_8K); 546 data = TD_V | TD_8K | TD_PA(pa + off) | TD_REF | 547 TD_SW | TD_CP | TD_CV | TD_P | TD_W; 548 pmap_bootstrap_set_tte(tp, vpn, data); 549 } 550 } 551 552 /* 553 * Set the start and end of KVA. The kernel is loaded starting 554 * at the first available 4MB super page, so we advance to the 555 * end of the last one used for it. 556 */ 557 virtual_avail = KERNBASE + kernel_tlb_slots * PAGE_SIZE_4M; 558 virtual_end = vm_max_kernel_address; 559 kernel_vm_end = vm_max_kernel_address; 560 561 /* 562 * Allocate kva space for temporary mappings. 563 */ 564 pmap_idle_map = virtual_avail; 565 virtual_avail += PAGE_SIZE * colors; 566 pmap_temp_map_1 = virtual_avail; 567 virtual_avail += PAGE_SIZE * colors; 568 pmap_temp_map_2 = virtual_avail; 569 virtual_avail += PAGE_SIZE * colors; 570 571 /* 572 * Allocate a kernel stack with guard page for thread0 and map it 573 * into the kernel TSB. We must ensure that the virtual address is 574 * colored properly for corresponding CPUs, since we're allocating 575 * from phys_avail so the memory won't have an associated vm_page_t. 576 */ 577 pa = pmap_bootstrap_alloc(KSTACK_PAGES * PAGE_SIZE, colors); 578 kstack0_phys = pa; 579 virtual_avail += roundup(KSTACK_GUARD_PAGES, colors) * PAGE_SIZE; 580 kstack0 = virtual_avail; 581 virtual_avail += roundup(KSTACK_PAGES, colors) * PAGE_SIZE; 582 if (dcache_color_ignore == 0) 583 KASSERT(DCACHE_COLOR(kstack0) == DCACHE_COLOR(kstack0_phys), 584 ("pmap_bootstrap: kstack0 miscolored")); 585 for (i = 0; i < KSTACK_PAGES; i++) { 586 pa = kstack0_phys + i * PAGE_SIZE; 587 va = kstack0 + i * PAGE_SIZE; 588 tp = tsb_kvtotte(va); 589 vpn = TV_VPN(va, TS_8K); 590 data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_SW | TD_CP | 591 TD_CV | TD_P | TD_W; 592 pmap_bootstrap_set_tte(tp, vpn, data); 593 } 594 595 /* 596 * Calculate the last available physical address. 597 */ 598 for (i = 0; phys_avail[i + 2] != 0; i += 2) 599 ; 600 Maxmem = sparc64_btop(phys_avail[i + 1]); 601 602 /* 603 * Add the PROM mappings to the kernel TSB. 604 */ 605 if ((vmem = OF_finddevice("/virtual-memory")) == -1) 606 OF_panic("%s: finddevice /virtual-memory", __func__); 607 if ((sz = OF_getproplen(vmem, "translations")) == -1) 608 OF_panic("%s: getproplen translations", __func__); 609 if (sizeof(translations) < sz) 610 OF_panic("%s: translations too small", __func__); 611 bzero(translations, sz); 612 if (OF_getprop(vmem, "translations", translations, sz) == -1) 613 OF_panic("%s: getprop /virtual-memory/translations", 614 __func__); 615 sz /= sizeof(*translations); 616 translations_size = sz; 617 CTR0(KTR_PMAP, "pmap_bootstrap: translations"); 618 qsort(translations, sz, sizeof (*translations), om_cmp); 619 for (i = 0; i < sz; i++) { 620 CTR3(KTR_PMAP, 621 "translation: start=%#lx size=%#lx tte=%#lx", 622 translations[i].om_start, translations[i].om_size, 623 translations[i].om_tte); 624 if ((translations[i].om_tte & TD_V) == 0) 625 continue; 626 if (translations[i].om_start < VM_MIN_PROM_ADDRESS || 627 translations[i].om_start > VM_MAX_PROM_ADDRESS) 628 continue; 629 for (off = 0; off < translations[i].om_size; 630 off += PAGE_SIZE) { 631 va = translations[i].om_start + off; 632 tp = tsb_kvtotte(va); 633 vpn = TV_VPN(va, TS_8K); 634 data = ((translations[i].om_tte & 635 ~((TD_SOFT2_MASK << TD_SOFT2_SHIFT) | 636 (cpu_impl >= CPU_IMPL_ULTRASPARCI && 637 cpu_impl < CPU_IMPL_ULTRASPARCIII ? 638 (TD_DIAG_SF_MASK << TD_DIAG_SF_SHIFT) : 639 (TD_RSVD_CH_MASK << TD_RSVD_CH_SHIFT)) | 640 (TD_SOFT_MASK << TD_SOFT_SHIFT))) | TD_EXEC) + 641 off; 642 pmap_bootstrap_set_tte(tp, vpn, data); 643 } 644 } 645 646 /* 647 * Get the available physical memory ranges from /memory/reg. These 648 * are only used for kernel dumps, but it may not be wise to do PROM 649 * calls in that situation. 650 */ 651 if ((sz = OF_getproplen(pmem, "reg")) == -1) 652 OF_panic("%s: getproplen /memory/reg", __func__); 653 if (sizeof(sparc64_memreg) < sz) 654 OF_panic("%s: sparc64_memreg too small", __func__); 655 if (OF_getprop(pmem, "reg", sparc64_memreg, sz) == -1) 656 OF_panic("%s: getprop /memory/reg", __func__); 657 sparc64_nmemreg = sz / sizeof(*sparc64_memreg); 658 659 /* 660 * Initialize the kernel pmap (which is statically allocated). 661 */ 662 pm = kernel_pmap; 663 PMAP_LOCK_INIT(pm); 664 for (i = 0; i < MAXCPU; i++) 665 pm->pm_context[i] = TLB_CTX_KERNEL; 666 CPU_FILL(&pm->pm_active); 667 668 /* 669 * Initialize the global tte list lock, which is more commonly 670 * known as the pmap pv global lock. 671 */ 672 rw_init(&tte_list_global_lock, "pmap pv global"); 673 674 /* 675 * Flush all non-locked TLB entries possibly left over by the 676 * firmware. 677 */ 678 tlb_flush_nonlocked(); 679} 680 681/* 682 * Map the 4MB kernel TSB pages. 683 */ 684void 685pmap_map_tsb(void) 686{ 687 vm_offset_t va; 688 vm_paddr_t pa; 689 u_long data; 690 int i; 691 692 for (i = 0; i < tsb_kernel_size; i += PAGE_SIZE_4M) { 693 va = (vm_offset_t)tsb_kernel + i; 694 pa = tsb_kernel_phys + i; 695 data = TD_V | TD_4M | TD_PA(pa) | TD_L | TD_CP | TD_CV | 696 TD_P | TD_W; 697 stxa(AA_DMMU_TAR, ASI_DMMU, TLB_TAR_VA(va) | 698 TLB_TAR_CTX(TLB_CTX_KERNEL)); 699 stxa_sync(0, ASI_DTLB_DATA_IN_REG, data); 700 } 701} 702 703/* 704 * Set the secondary context to be the kernel context (needed for FP block 705 * operations in the kernel). 706 */ 707void 708pmap_set_kctx(void) 709{ 710 711 stxa(AA_DMMU_SCXR, ASI_DMMU, (ldxa(AA_DMMU_SCXR, ASI_DMMU) & 712 TLB_CXR_PGSZ_MASK) | TLB_CTX_KERNEL); 713 flush(KERNBASE); 714} 715 716/* 717 * Allocate a physical page of memory directly from the phys_avail map. 718 * Can only be called from pmap_bootstrap before avail start and end are 719 * calculated. 720 */ 721static vm_paddr_t 722pmap_bootstrap_alloc(vm_size_t size, uint32_t colors) 723{ 724 vm_paddr_t pa; 725 int i; 726 727 size = roundup(size, PAGE_SIZE * colors); 728 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 729 if (phys_avail[i + 1] - phys_avail[i] < size) 730 continue; 731 pa = phys_avail[i]; 732 phys_avail[i] += size; 733 return (pa); 734 } 735 OF_panic("%s: no suitable region found", __func__); 736} 737 738/* 739 * Set a TTE. This function is intended as a helper when tsb_kernel is 740 * direct-mapped but we haven't taken over the trap table, yet, as it's the 741 * case when we are taking advantage of ASI_ATOMIC_QUAD_LDD_PHYS to access 742 * the kernel TSB. 743 */ 744void 745pmap_bootstrap_set_tte(struct tte *tp, u_long vpn, u_long data) 746{ 747 748 if (tsb_kernel_ldd_phys == 0) { 749 tp->tte_vpn = vpn; 750 tp->tte_data = data; 751 } else { 752 stxa((vm_paddr_t)tp + offsetof(struct tte, tte_vpn), 753 ASI_PHYS_USE_EC, vpn); 754 stxa((vm_paddr_t)tp + offsetof(struct tte, tte_data), 755 ASI_PHYS_USE_EC, data); 756 } 757} 758 759/* 760 * Initialize a vm_page's machine-dependent fields. 761 */ 762void 763pmap_page_init(vm_page_t m) 764{ 765 766 TAILQ_INIT(&m->md.tte_list); 767 m->md.color = DCACHE_COLOR(VM_PAGE_TO_PHYS(m)); 768 m->md.flags = 0; 769 m->md.pmap = NULL; 770} 771 772/* 773 * Initialize the pmap module. 774 */ 775void 776pmap_init(void) 777{ 778 vm_offset_t addr; 779 vm_size_t size; 780 int result; 781 int i; 782 783 for (i = 0; i < translations_size; i++) { 784 addr = translations[i].om_start; 785 size = translations[i].om_size; 786 if ((translations[i].om_tte & TD_V) == 0) 787 continue; 788 if (addr < VM_MIN_PROM_ADDRESS || addr > VM_MAX_PROM_ADDRESS) 789 continue; 790 result = vm_map_find(kernel_map, NULL, 0, &addr, size, 791 VMFS_NO_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 792 if (result != KERN_SUCCESS || addr != translations[i].om_start) 793 panic("pmap_init: vm_map_find"); 794 } 795} 796 797/* 798 * Extract the physical page address associated with the given 799 * map/virtual_address pair. 800 */ 801vm_paddr_t 802pmap_extract(pmap_t pm, vm_offset_t va) 803{ 804 struct tte *tp; 805 vm_paddr_t pa; 806 807 if (pm == kernel_pmap) 808 return (pmap_kextract(va)); 809 PMAP_LOCK(pm); 810 tp = tsb_tte_lookup(pm, va); 811 if (tp == NULL) 812 pa = 0; 813 else 814 pa = TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp)); 815 PMAP_UNLOCK(pm); 816 return (pa); 817} 818 819/* 820 * Atomically extract and hold the physical page with the given 821 * pmap and virtual address pair if that mapping permits the given 822 * protection. 823 */ 824vm_page_t 825pmap_extract_and_hold(pmap_t pm, vm_offset_t va, vm_prot_t prot) 826{ 827 struct tte *tp; 828 vm_page_t m; 829 vm_paddr_t pa; 830 831 m = NULL; 832 pa = 0; 833 PMAP_LOCK(pm); 834retry: 835 if (pm == kernel_pmap) { 836 if (va >= VM_MIN_DIRECT_ADDRESS) { 837 tp = NULL; 838 m = PHYS_TO_VM_PAGE(TLB_DIRECT_TO_PHYS(va)); 839 (void)vm_page_pa_tryrelock(pm, TLB_DIRECT_TO_PHYS(va), 840 &pa); 841 vm_page_hold(m); 842 } else { 843 tp = tsb_kvtotte(va); 844 if ((tp->tte_data & TD_V) == 0) 845 tp = NULL; 846 } 847 } else 848 tp = tsb_tte_lookup(pm, va); 849 if (tp != NULL && ((tp->tte_data & TD_SW) || 850 (prot & VM_PROT_WRITE) == 0)) { 851 if (vm_page_pa_tryrelock(pm, TTE_GET_PA(tp), &pa)) 852 goto retry; 853 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 854 vm_page_hold(m); 855 } 856 PA_UNLOCK_COND(pa); 857 PMAP_UNLOCK(pm); 858 return (m); 859} 860 861/* 862 * Extract the physical page address associated with the given kernel virtual 863 * address. 864 */ 865vm_paddr_t 866pmap_kextract(vm_offset_t va) 867{ 868 struct tte *tp; 869 870 if (va >= VM_MIN_DIRECT_ADDRESS) 871 return (TLB_DIRECT_TO_PHYS(va)); 872 tp = tsb_kvtotte(va); 873 if ((tp->tte_data & TD_V) == 0) 874 return (0); 875 return (TTE_GET_PA(tp) | (va & TTE_GET_PAGE_MASK(tp))); 876} 877 878int 879pmap_cache_enter(vm_page_t m, vm_offset_t va) 880{ 881 struct tte *tp; 882 int color; 883 884 rw_assert(&tte_list_global_lock, RA_WLOCKED); 885 KASSERT((m->flags & PG_FICTITIOUS) == 0, 886 ("pmap_cache_enter: fake page")); 887 PMAP_STATS_INC(pmap_ncache_enter); 888 889 if (dcache_color_ignore != 0) 890 return (1); 891 892 /* 893 * Find the color for this virtual address and note the added mapping. 894 */ 895 color = DCACHE_COLOR(va); 896 m->md.colors[color]++; 897 898 /* 899 * If all existing mappings have the same color, the mapping is 900 * cacheable. 901 */ 902 if (m->md.color == color) { 903 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] == 0, 904 ("pmap_cache_enter: cacheable, mappings of other color")); 905 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 906 PMAP_STATS_INC(pmap_ncache_enter_c); 907 else 908 PMAP_STATS_INC(pmap_ncache_enter_oc); 909 return (1); 910 } 911 912 /* 913 * If there are no mappings of the other color, and the page still has 914 * the wrong color, this must be a new mapping. Change the color to 915 * match the new mapping, which is cacheable. We must flush the page 916 * from the cache now. 917 */ 918 if (m->md.colors[DCACHE_OTHER_COLOR(color)] == 0) { 919 KASSERT(m->md.colors[color] == 1, 920 ("pmap_cache_enter: changing color, not new mapping")); 921 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 922 m->md.color = color; 923 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 924 PMAP_STATS_INC(pmap_ncache_enter_cc); 925 else 926 PMAP_STATS_INC(pmap_ncache_enter_coc); 927 return (1); 928 } 929 930 /* 931 * If the mapping is already non-cacheable, just return. 932 */ 933 if (m->md.color == -1) { 934 PMAP_STATS_INC(pmap_ncache_enter_nc); 935 return (0); 936 } 937 938 PMAP_STATS_INC(pmap_ncache_enter_cnc); 939 940 /* 941 * Mark all mappings as uncacheable, flush any lines with the other 942 * color out of the dcache, and set the color to none (-1). 943 */ 944 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 945 atomic_clear_long(&tp->tte_data, TD_CV); 946 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 947 } 948 dcache_page_inval(VM_PAGE_TO_PHYS(m)); 949 m->md.color = -1; 950 return (0); 951} 952 953static void 954pmap_cache_remove(vm_page_t m, vm_offset_t va) 955{ 956 struct tte *tp; 957 int color; 958 959 rw_assert(&tte_list_global_lock, RA_WLOCKED); 960 CTR3(KTR_PMAP, "pmap_cache_remove: m=%p va=%#lx c=%d", m, va, 961 m->md.colors[DCACHE_COLOR(va)]); 962 KASSERT((m->flags & PG_FICTITIOUS) == 0, 963 ("pmap_cache_remove: fake page")); 964 PMAP_STATS_INC(pmap_ncache_remove); 965 966 if (dcache_color_ignore != 0) 967 return; 968 969 KASSERT(m->md.colors[DCACHE_COLOR(va)] > 0, 970 ("pmap_cache_remove: no mappings %d <= 0", 971 m->md.colors[DCACHE_COLOR(va)])); 972 973 /* 974 * Find the color for this virtual address and note the removal of 975 * the mapping. 976 */ 977 color = DCACHE_COLOR(va); 978 m->md.colors[color]--; 979 980 /* 981 * If the page is cacheable, just return and keep the same color, even 982 * if there are no longer any mappings. 983 */ 984 if (m->md.color != -1) { 985 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 986 PMAP_STATS_INC(pmap_ncache_remove_c); 987 else 988 PMAP_STATS_INC(pmap_ncache_remove_oc); 989 return; 990 } 991 992 KASSERT(m->md.colors[DCACHE_OTHER_COLOR(color)] != 0, 993 ("pmap_cache_remove: uncacheable, no mappings of other color")); 994 995 /* 996 * If the page is not cacheable (color is -1), and the number of 997 * mappings for this color is not zero, just return. There are 998 * mappings of the other color still, so remain non-cacheable. 999 */ 1000 if (m->md.colors[color] != 0) { 1001 PMAP_STATS_INC(pmap_ncache_remove_nc); 1002 return; 1003 } 1004 1005 /* 1006 * The number of mappings for this color is now zero. Recache the 1007 * other colored mappings, and change the page color to the other 1008 * color. There should be no lines in the data cache for this page, 1009 * so flushing should not be needed. 1010 */ 1011 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1012 atomic_set_long(&tp->tte_data, TD_CV); 1013 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 1014 } 1015 m->md.color = DCACHE_OTHER_COLOR(color); 1016 1017 if (m->md.color == DCACHE_COLOR(VM_PAGE_TO_PHYS(m))) 1018 PMAP_STATS_INC(pmap_ncache_remove_cc); 1019 else 1020 PMAP_STATS_INC(pmap_ncache_remove_coc); 1021} 1022 1023/* 1024 * Map a wired page into kernel virtual address space. 1025 */ 1026void 1027pmap_kenter(vm_offset_t va, vm_page_t m) 1028{ 1029 vm_offset_t ova; 1030 struct tte *tp; 1031 vm_page_t om; 1032 u_long data; 1033 1034 rw_assert(&tte_list_global_lock, RA_WLOCKED); 1035 PMAP_STATS_INC(pmap_nkenter); 1036 tp = tsb_kvtotte(va); 1037 CTR4(KTR_PMAP, "pmap_kenter: va=%#lx pa=%#lx tp=%p data=%#lx", 1038 va, VM_PAGE_TO_PHYS(m), tp, tp->tte_data); 1039 if (DCACHE_COLOR(VM_PAGE_TO_PHYS(m)) != DCACHE_COLOR(va)) { 1040 CTR5(KTR_SPARE2, 1041 "pmap_kenter: off color va=%#lx pa=%#lx o=%p ot=%d pi=%#lx", 1042 va, VM_PAGE_TO_PHYS(m), m->object, 1043 m->object ? m->object->type : -1, 1044 m->pindex); 1045 PMAP_STATS_INC(pmap_nkenter_oc); 1046 } 1047 if ((tp->tte_data & TD_V) != 0) { 1048 om = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1049 ova = TTE_GET_VA(tp); 1050 if (m == om && va == ova) { 1051 PMAP_STATS_INC(pmap_nkenter_stupid); 1052 return; 1053 } 1054 TAILQ_REMOVE(&om->md.tte_list, tp, tte_link); 1055 pmap_cache_remove(om, ova); 1056 if (va != ova) 1057 tlb_page_demap(kernel_pmap, ova); 1058 } 1059 data = TD_V | TD_8K | VM_PAGE_TO_PHYS(m) | TD_REF | TD_SW | TD_CP | 1060 TD_P | TD_W; 1061 if (pmap_cache_enter(m, va) != 0) 1062 data |= TD_CV; 1063 tp->tte_vpn = TV_VPN(va, TS_8K); 1064 tp->tte_data = data; 1065 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 1066} 1067 1068/* 1069 * Map a wired page into kernel virtual address space. This additionally 1070 * takes a flag argument which is or'ed to the TTE data. This is used by 1071 * sparc64_bus_mem_map(). 1072 * NOTE: if the mapping is non-cacheable, it's the caller's responsibility 1073 * to flush entries that might still be in the cache, if applicable. 1074 */ 1075void 1076pmap_kenter_flags(vm_offset_t va, vm_paddr_t pa, u_long flags) 1077{ 1078 struct tte *tp; 1079 1080 tp = tsb_kvtotte(va); 1081 CTR4(KTR_PMAP, "pmap_kenter_flags: va=%#lx pa=%#lx tp=%p data=%#lx", 1082 va, pa, tp, tp->tte_data); 1083 tp->tte_vpn = TV_VPN(va, TS_8K); 1084 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_REF | TD_P | flags; 1085} 1086 1087/* 1088 * Remove a wired page from kernel virtual address space. 1089 */ 1090void 1091pmap_kremove(vm_offset_t va) 1092{ 1093 struct tte *tp; 1094 vm_page_t m; 1095 1096 rw_assert(&tte_list_global_lock, RA_WLOCKED); 1097 PMAP_STATS_INC(pmap_nkremove); 1098 tp = tsb_kvtotte(va); 1099 CTR3(KTR_PMAP, "pmap_kremove: va=%#lx tp=%p data=%#lx", va, tp, 1100 tp->tte_data); 1101 if ((tp->tte_data & TD_V) == 0) 1102 return; 1103 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1104 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1105 pmap_cache_remove(m, va); 1106 TTE_ZERO(tp); 1107} 1108 1109/* 1110 * Inverse of pmap_kenter_flags, used by bus_space_unmap(). 1111 */ 1112void 1113pmap_kremove_flags(vm_offset_t va) 1114{ 1115 struct tte *tp; 1116 1117 tp = tsb_kvtotte(va); 1118 CTR3(KTR_PMAP, "pmap_kremove_flags: va=%#lx tp=%p data=%#lx", va, tp, 1119 tp->tte_data); 1120 TTE_ZERO(tp); 1121} 1122 1123/* 1124 * Map a range of physical addresses into kernel virtual address space. 1125 * 1126 * The value passed in *virt is a suggested virtual address for the mapping. 1127 * Architectures which can support a direct-mapped physical to virtual region 1128 * can return the appropriate address within that region, leaving '*virt' 1129 * unchanged. 1130 */ 1131vm_offset_t 1132pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) 1133{ 1134 1135 return (TLB_PHYS_TO_DIRECT(start)); 1136} 1137 1138/* 1139 * Map a list of wired pages into kernel virtual address space. This is 1140 * intended for temporary mappings which do not need page modification or 1141 * references recorded. Existing mappings in the region are overwritten. 1142 */ 1143void 1144pmap_qenter(vm_offset_t sva, vm_page_t *m, int count) 1145{ 1146 vm_offset_t va; 1147 1148 PMAP_STATS_INC(pmap_nqenter); 1149 va = sva; 1150 rw_wlock(&tte_list_global_lock); 1151 while (count-- > 0) { 1152 pmap_kenter(va, *m); 1153 va += PAGE_SIZE; 1154 m++; 1155 } 1156 rw_wunlock(&tte_list_global_lock); 1157 tlb_range_demap(kernel_pmap, sva, va); 1158} 1159 1160/* 1161 * Remove page mappings from kernel virtual address space. Intended for 1162 * temporary mappings entered by pmap_qenter. 1163 */ 1164void 1165pmap_qremove(vm_offset_t sva, int count) 1166{ 1167 vm_offset_t va; 1168 1169 PMAP_STATS_INC(pmap_nqremove); 1170 va = sva; 1171 rw_wlock(&tte_list_global_lock); 1172 while (count-- > 0) { 1173 pmap_kremove(va); 1174 va += PAGE_SIZE; 1175 } 1176 rw_wunlock(&tte_list_global_lock); 1177 tlb_range_demap(kernel_pmap, sva, va); 1178} 1179 1180/* 1181 * Initialize the pmap associated with process 0. 1182 */ 1183void 1184pmap_pinit0(pmap_t pm) 1185{ 1186 int i; 1187 1188 PMAP_LOCK_INIT(pm); 1189 for (i = 0; i < MAXCPU; i++) 1190 pm->pm_context[i] = TLB_CTX_KERNEL; 1191 CPU_ZERO(&pm->pm_active); 1192 pm->pm_tsb = NULL; 1193 pm->pm_tsb_obj = NULL; 1194 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1195} 1196 1197/* 1198 * Initialize a preallocated and zeroed pmap structure, such as one in a 1199 * vmspace structure. 1200 */ 1201int 1202pmap_pinit(pmap_t pm) 1203{ 1204 vm_page_t ma[TSB_PAGES]; 1205 vm_page_t m; 1206 int i; 1207 1208 PMAP_LOCK_INIT(pm); 1209 1210 /* 1211 * Allocate KVA space for the TSB. 1212 */ 1213 if (pm->pm_tsb == NULL) { 1214 pm->pm_tsb = (struct tte *)kmem_alloc_nofault(kernel_map, 1215 TSB_BSIZE); 1216 if (pm->pm_tsb == NULL) { 1217 PMAP_LOCK_DESTROY(pm); 1218 return (0); 1219 } 1220 } 1221 1222 /* 1223 * Allocate an object for it. 1224 */ 1225 if (pm->pm_tsb_obj == NULL) 1226 pm->pm_tsb_obj = vm_object_allocate(OBJT_PHYS, TSB_PAGES); 1227 1228 for (i = 0; i < MAXCPU; i++) 1229 pm->pm_context[i] = -1; 1230 CPU_ZERO(&pm->pm_active); 1231 1232 VM_OBJECT_WLOCK(pm->pm_tsb_obj); 1233 for (i = 0; i < TSB_PAGES; i++) { 1234 m = vm_page_grab(pm->pm_tsb_obj, i, VM_ALLOC_NOBUSY | 1235 VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 1236 m->valid = VM_PAGE_BITS_ALL; 1237 m->md.pmap = pm; 1238 ma[i] = m; 1239 } 1240 VM_OBJECT_WUNLOCK(pm->pm_tsb_obj); 1241 pmap_qenter((vm_offset_t)pm->pm_tsb, ma, TSB_PAGES); 1242 1243 bzero(&pm->pm_stats, sizeof(pm->pm_stats)); 1244 return (1); 1245} 1246 1247/* 1248 * Release any resources held by the given physical map. 1249 * Called when a pmap initialized by pmap_pinit is being released. 1250 * Should only be called if the map contains no valid mappings. 1251 */ 1252void 1253pmap_release(pmap_t pm) 1254{ 1255 vm_object_t obj; 1256 vm_page_t m; 1257#ifdef SMP 1258 struct pcpu *pc; 1259#endif 1260 1261 CTR2(KTR_PMAP, "pmap_release: ctx=%#x tsb=%p", 1262 pm->pm_context[curcpu], pm->pm_tsb); 1263 KASSERT(pmap_resident_count(pm) == 0, 1264 ("pmap_release: resident pages %ld != 0", 1265 pmap_resident_count(pm))); 1266 1267 /* 1268 * After the pmap was freed, it might be reallocated to a new process. 1269 * When switching, this might lead us to wrongly assume that we need 1270 * not switch contexts because old and new pmap pointer are equal. 1271 * Therefore, make sure that this pmap is not referenced by any PCPU 1272 * pointer any more. This could happen in two cases: 1273 * - A process that referenced the pmap is currently exiting on a CPU. 1274 * However, it is guaranteed to not switch in any more after setting 1275 * its state to PRS_ZOMBIE. 1276 * - A process that referenced this pmap ran on a CPU, but we switched 1277 * to a kernel thread, leaving the pmap pointer unchanged. 1278 */ 1279#ifdef SMP 1280 sched_pin(); 1281 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 1282 atomic_cmpset_rel_ptr((uintptr_t *)&pc->pc_pmap, 1283 (uintptr_t)pm, (uintptr_t)NULL); 1284 sched_unpin(); 1285#else 1286 critical_enter(); 1287 if (PCPU_GET(pmap) == pm) 1288 PCPU_SET(pmap, NULL); 1289 critical_exit(); 1290#endif 1291 1292 pmap_qremove((vm_offset_t)pm->pm_tsb, TSB_PAGES); 1293 obj = pm->pm_tsb_obj; 1294 VM_OBJECT_WLOCK(obj); 1295 KASSERT(obj->ref_count == 1, ("pmap_release: tsbobj ref count != 1")); 1296 while (!TAILQ_EMPTY(&obj->memq)) { 1297 m = TAILQ_FIRST(&obj->memq); 1298 m->md.pmap = NULL; 1299 m->wire_count--; 1300 atomic_subtract_int(&cnt.v_wire_count, 1); 1301 vm_page_free_zero(m); 1302 } 1303 VM_OBJECT_WUNLOCK(obj); 1304 PMAP_LOCK_DESTROY(pm); 1305} 1306 1307/* 1308 * Grow the number of kernel page table entries. Unneeded. 1309 */ 1310void 1311pmap_growkernel(vm_offset_t addr) 1312{ 1313 1314 panic("pmap_growkernel: can't grow kernel"); 1315} 1316 1317int 1318pmap_remove_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1319 vm_offset_t va) 1320{ 1321 vm_page_t m; 1322 u_long data; 1323 1324 rw_assert(&tte_list_global_lock, RA_WLOCKED); 1325 data = atomic_readandclear_long(&tp->tte_data); 1326 if ((data & TD_FAKE) == 0) { 1327 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1328 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1329 if ((data & TD_WIRED) != 0) 1330 pm->pm_stats.wired_count--; 1331 if ((data & TD_PV) != 0) { 1332 if ((data & TD_W) != 0) 1333 vm_page_dirty(m); 1334 if ((data & TD_REF) != 0) 1335 vm_page_aflag_set(m, PGA_REFERENCED); 1336 if (TAILQ_EMPTY(&m->md.tte_list)) 1337 vm_page_aflag_clear(m, PGA_WRITEABLE); 1338 pm->pm_stats.resident_count--; 1339 } 1340 pmap_cache_remove(m, va); 1341 } 1342 TTE_ZERO(tp); 1343 if (PMAP_REMOVE_DONE(pm)) 1344 return (0); 1345 return (1); 1346} 1347 1348/* 1349 * Remove the given range of addresses from the specified map. 1350 */ 1351void 1352pmap_remove(pmap_t pm, vm_offset_t start, vm_offset_t end) 1353{ 1354 struct tte *tp; 1355 vm_offset_t va; 1356 1357 CTR3(KTR_PMAP, "pmap_remove: ctx=%#lx start=%#lx end=%#lx", 1358 pm->pm_context[curcpu], start, end); 1359 if (PMAP_REMOVE_DONE(pm)) 1360 return; 1361 rw_wlock(&tte_list_global_lock); 1362 PMAP_LOCK(pm); 1363 if (end - start > PMAP_TSB_THRESH) { 1364 tsb_foreach(pm, NULL, start, end, pmap_remove_tte); 1365 tlb_context_demap(pm); 1366 } else { 1367 for (va = start; va < end; va += PAGE_SIZE) 1368 if ((tp = tsb_tte_lookup(pm, va)) != NULL && 1369 !pmap_remove_tte(pm, NULL, tp, va)) 1370 break; 1371 tlb_range_demap(pm, start, end - 1); 1372 } 1373 PMAP_UNLOCK(pm); 1374 rw_wunlock(&tte_list_global_lock); 1375} 1376 1377void 1378pmap_remove_all(vm_page_t m) 1379{ 1380 struct pmap *pm; 1381 struct tte *tpn; 1382 struct tte *tp; 1383 vm_offset_t va; 1384 1385 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1386 ("pmap_remove_all: page %p is not managed", m)); 1387 rw_wlock(&tte_list_global_lock); 1388 for (tp = TAILQ_FIRST(&m->md.tte_list); tp != NULL; tp = tpn) { 1389 tpn = TAILQ_NEXT(tp, tte_link); 1390 if ((tp->tte_data & TD_PV) == 0) 1391 continue; 1392 pm = TTE_GET_PMAP(tp); 1393 va = TTE_GET_VA(tp); 1394 PMAP_LOCK(pm); 1395 if ((tp->tte_data & TD_WIRED) != 0) 1396 pm->pm_stats.wired_count--; 1397 if ((tp->tte_data & TD_REF) != 0) 1398 vm_page_aflag_set(m, PGA_REFERENCED); 1399 if ((tp->tte_data & TD_W) != 0) 1400 vm_page_dirty(m); 1401 tp->tte_data &= ~TD_V; 1402 tlb_page_demap(pm, va); 1403 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 1404 pm->pm_stats.resident_count--; 1405 pmap_cache_remove(m, va); 1406 TTE_ZERO(tp); 1407 PMAP_UNLOCK(pm); 1408 } 1409 vm_page_aflag_clear(m, PGA_WRITEABLE); 1410 rw_wunlock(&tte_list_global_lock); 1411} 1412 1413static int 1414pmap_protect_tte(struct pmap *pm, struct pmap *pm2, struct tte *tp, 1415 vm_offset_t va) 1416{ 1417 u_long data; 1418 vm_page_t m; 1419 1420 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1421 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 1422 if ((data & (TD_PV | TD_W)) == (TD_PV | TD_W)) { 1423 m = PHYS_TO_VM_PAGE(TD_PA(data)); 1424 vm_page_dirty(m); 1425 } 1426 return (1); 1427} 1428 1429/* 1430 * Set the physical protection on the specified range of this map as requested. 1431 */ 1432void 1433pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 1434{ 1435 vm_offset_t va; 1436 struct tte *tp; 1437 1438 CTR4(KTR_PMAP, "pmap_protect: ctx=%#lx sva=%#lx eva=%#lx prot=%#lx", 1439 pm->pm_context[curcpu], sva, eva, prot); 1440 1441 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1442 pmap_remove(pm, sva, eva); 1443 return; 1444 } 1445 1446 if (prot & VM_PROT_WRITE) 1447 return; 1448 1449 PMAP_LOCK(pm); 1450 if (eva - sva > PMAP_TSB_THRESH) { 1451 tsb_foreach(pm, NULL, sva, eva, pmap_protect_tte); 1452 tlb_context_demap(pm); 1453 } else { 1454 for (va = sva; va < eva; va += PAGE_SIZE) 1455 if ((tp = tsb_tte_lookup(pm, va)) != NULL) 1456 pmap_protect_tte(pm, NULL, tp, va); 1457 tlb_range_demap(pm, sva, eva - 1); 1458 } 1459 PMAP_UNLOCK(pm); 1460} 1461 1462/* 1463 * Map the given physical page at the specified virtual address in the 1464 * target pmap with the protection requested. If specified the page 1465 * will be wired down. 1466 */ 1467void 1468pmap_enter(pmap_t pm, vm_offset_t va, vm_prot_t access, vm_page_t m, 1469 vm_prot_t prot, boolean_t wired) 1470{ 1471 1472 rw_wlock(&tte_list_global_lock); 1473 PMAP_LOCK(pm); 1474 pmap_enter_locked(pm, va, m, prot, wired); 1475 rw_wunlock(&tte_list_global_lock); 1476 PMAP_UNLOCK(pm); 1477} 1478 1479/* 1480 * Map the given physical page at the specified virtual address in the 1481 * target pmap with the protection requested. If specified the page 1482 * will be wired down. 1483 * 1484 * The page queues and pmap must be locked. 1485 */ 1486static void 1487pmap_enter_locked(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot, 1488 boolean_t wired) 1489{ 1490 struct tte *tp; 1491 vm_paddr_t pa; 1492 vm_page_t real; 1493 u_long data; 1494 1495 rw_assert(&tte_list_global_lock, RA_WLOCKED); 1496 PMAP_LOCK_ASSERT(pm, MA_OWNED); 1497 if ((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) == 0) 1498 VM_OBJECT_ASSERT_LOCKED(m->object); 1499 PMAP_STATS_INC(pmap_nenter); 1500 pa = VM_PAGE_TO_PHYS(m); 1501 1502 /* 1503 * If this is a fake page from the device_pager, but it covers actual 1504 * physical memory, convert to the real backing page. 1505 */ 1506 if ((m->flags & PG_FICTITIOUS) != 0) { 1507 real = vm_phys_paddr_to_vm_page(pa); 1508 if (real != NULL) 1509 m = real; 1510 } 1511 1512 CTR6(KTR_PMAP, 1513 "pmap_enter_locked: ctx=%p m=%p va=%#lx pa=%#lx prot=%#x wired=%d", 1514 pm->pm_context[curcpu], m, va, pa, prot, wired); 1515 1516 /* 1517 * If there is an existing mapping, and the physical address has not 1518 * changed, must be protection or wiring change. 1519 */ 1520 if ((tp = tsb_tte_lookup(pm, va)) != NULL && TTE_GET_PA(tp) == pa) { 1521 CTR0(KTR_PMAP, "pmap_enter_locked: update"); 1522 PMAP_STATS_INC(pmap_nenter_update); 1523 1524 /* 1525 * Wiring change, just update stats. 1526 */ 1527 if (wired) { 1528 if ((tp->tte_data & TD_WIRED) == 0) { 1529 tp->tte_data |= TD_WIRED; 1530 pm->pm_stats.wired_count++; 1531 } 1532 } else { 1533 if ((tp->tte_data & TD_WIRED) != 0) { 1534 tp->tte_data &= ~TD_WIRED; 1535 pm->pm_stats.wired_count--; 1536 } 1537 } 1538 1539 /* 1540 * Save the old bits and clear the ones we're interested in. 1541 */ 1542 data = tp->tte_data; 1543 tp->tte_data &= ~(TD_EXEC | TD_SW | TD_W); 1544 1545 /* 1546 * If we're turning off write permissions, sense modify status. 1547 */ 1548 if ((prot & VM_PROT_WRITE) != 0) { 1549 tp->tte_data |= TD_SW; 1550 if (wired) 1551 tp->tte_data |= TD_W; 1552 if ((m->oflags & VPO_UNMANAGED) == 0) 1553 vm_page_aflag_set(m, PGA_WRITEABLE); 1554 } else if ((data & TD_W) != 0) 1555 vm_page_dirty(m); 1556 1557 /* 1558 * If we're turning on execute permissions, flush the icache. 1559 */ 1560 if ((prot & VM_PROT_EXECUTE) != 0) { 1561 if ((data & TD_EXEC) == 0) 1562 icache_page_inval(pa); 1563 tp->tte_data |= TD_EXEC; 1564 } 1565 1566 /* 1567 * Delete the old mapping. 1568 */ 1569 tlb_page_demap(pm, TTE_GET_VA(tp)); 1570 } else { 1571 /* 1572 * If there is an existing mapping, but its for a different 1573 * physical address, delete the old mapping. 1574 */ 1575 if (tp != NULL) { 1576 CTR0(KTR_PMAP, "pmap_enter_locked: replace"); 1577 PMAP_STATS_INC(pmap_nenter_replace); 1578 pmap_remove_tte(pm, NULL, tp, va); 1579 tlb_page_demap(pm, va); 1580 } else { 1581 CTR0(KTR_PMAP, "pmap_enter_locked: new"); 1582 PMAP_STATS_INC(pmap_nenter_new); 1583 } 1584 1585 /* 1586 * Now set up the data and install the new mapping. 1587 */ 1588 data = TD_V | TD_8K | TD_PA(pa); 1589 if (pm == kernel_pmap) 1590 data |= TD_P; 1591 if ((prot & VM_PROT_WRITE) != 0) { 1592 data |= TD_SW; 1593 if ((m->oflags & VPO_UNMANAGED) == 0) 1594 vm_page_aflag_set(m, PGA_WRITEABLE); 1595 } 1596 if (prot & VM_PROT_EXECUTE) { 1597 data |= TD_EXEC; 1598 icache_page_inval(pa); 1599 } 1600 1601 /* 1602 * If its wired update stats. We also don't need reference or 1603 * modify tracking for wired mappings, so set the bits now. 1604 */ 1605 if (wired) { 1606 pm->pm_stats.wired_count++; 1607 data |= TD_REF | TD_WIRED; 1608 if ((prot & VM_PROT_WRITE) != 0) 1609 data |= TD_W; 1610 } 1611 1612 tsb_tte_enter(pm, m, va, TS_8K, data); 1613 } 1614} 1615 1616/* 1617 * Maps a sequence of resident pages belonging to the same object. 1618 * The sequence begins with the given page m_start. This page is 1619 * mapped at the given virtual address start. Each subsequent page is 1620 * mapped at a virtual address that is offset from start by the same 1621 * amount as the page is offset from m_start within the object. The 1622 * last page in the sequence is the page with the largest offset from 1623 * m_start that can be mapped at a virtual address less than the given 1624 * virtual address end. Not every virtual page between start and end 1625 * is mapped; only those for which a resident page exists with the 1626 * corresponding offset from m_start are mapped. 1627 */ 1628void 1629pmap_enter_object(pmap_t pm, vm_offset_t start, vm_offset_t end, 1630 vm_page_t m_start, vm_prot_t prot) 1631{ 1632 vm_page_t m; 1633 vm_pindex_t diff, psize; 1634 1635 VM_OBJECT_ASSERT_LOCKED(m_start->object); 1636 1637 psize = atop(end - start); 1638 m = m_start; 1639 rw_wlock(&tte_list_global_lock); 1640 PMAP_LOCK(pm); 1641 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 1642 pmap_enter_locked(pm, start + ptoa(diff), m, prot & 1643 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 1644 m = TAILQ_NEXT(m, listq); 1645 } 1646 rw_wunlock(&tte_list_global_lock); 1647 PMAP_UNLOCK(pm); 1648} 1649 1650void 1651pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_prot_t prot) 1652{ 1653 1654 rw_wlock(&tte_list_global_lock); 1655 PMAP_LOCK(pm); 1656 pmap_enter_locked(pm, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 1657 FALSE); 1658 rw_wunlock(&tte_list_global_lock); 1659 PMAP_UNLOCK(pm); 1660} 1661 1662void 1663pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, 1664 vm_pindex_t pindex, vm_size_t size) 1665{ 1666 1667 VM_OBJECT_ASSERT_WLOCKED(object); 1668 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, 1669 ("pmap_object_init_pt: non-device object")); 1670} 1671 1672/* 1673 * Change the wiring attribute for a map/virtual-address pair. 1674 * The mapping must already exist in the pmap. 1675 */ 1676void 1677pmap_change_wiring(pmap_t pm, vm_offset_t va, boolean_t wired) 1678{ 1679 struct tte *tp; 1680 u_long data; 1681 1682 PMAP_LOCK(pm); 1683 if ((tp = tsb_tte_lookup(pm, va)) != NULL) { 1684 if (wired) { 1685 data = atomic_set_long(&tp->tte_data, TD_WIRED); 1686 if ((data & TD_WIRED) == 0) 1687 pm->pm_stats.wired_count++; 1688 } else { 1689 data = atomic_clear_long(&tp->tte_data, TD_WIRED); 1690 if ((data & TD_WIRED) != 0) 1691 pm->pm_stats.wired_count--; 1692 } 1693 } 1694 PMAP_UNLOCK(pm); 1695} 1696 1697static int 1698pmap_copy_tte(pmap_t src_pmap, pmap_t dst_pmap, struct tte *tp, 1699 vm_offset_t va) 1700{ 1701 vm_page_t m; 1702 u_long data; 1703 1704 if ((tp->tte_data & TD_FAKE) != 0) 1705 return (1); 1706 if (tsb_tte_lookup(dst_pmap, va) == NULL) { 1707 data = tp->tte_data & 1708 ~(TD_PV | TD_REF | TD_SW | TD_CV | TD_W); 1709 m = PHYS_TO_VM_PAGE(TTE_GET_PA(tp)); 1710 tsb_tte_enter(dst_pmap, m, va, TS_8K, data); 1711 } 1712 return (1); 1713} 1714 1715void 1716pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 1717 vm_size_t len, vm_offset_t src_addr) 1718{ 1719 struct tte *tp; 1720 vm_offset_t va; 1721 1722 if (dst_addr != src_addr) 1723 return; 1724 rw_wlock(&tte_list_global_lock); 1725 if (dst_pmap < src_pmap) { 1726 PMAP_LOCK(dst_pmap); 1727 PMAP_LOCK(src_pmap); 1728 } else { 1729 PMAP_LOCK(src_pmap); 1730 PMAP_LOCK(dst_pmap); 1731 } 1732 if (len > PMAP_TSB_THRESH) { 1733 tsb_foreach(src_pmap, dst_pmap, src_addr, src_addr + len, 1734 pmap_copy_tte); 1735 tlb_context_demap(dst_pmap); 1736 } else { 1737 for (va = src_addr; va < src_addr + len; va += PAGE_SIZE) 1738 if ((tp = tsb_tte_lookup(src_pmap, va)) != NULL) 1739 pmap_copy_tte(src_pmap, dst_pmap, tp, va); 1740 tlb_range_demap(dst_pmap, src_addr, src_addr + len - 1); 1741 } 1742 rw_wunlock(&tte_list_global_lock); 1743 PMAP_UNLOCK(src_pmap); 1744 PMAP_UNLOCK(dst_pmap); 1745} 1746 1747void 1748pmap_zero_page(vm_page_t m) 1749{ 1750 struct tte *tp; 1751 vm_offset_t va; 1752 vm_paddr_t pa; 1753 1754 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1755 ("pmap_zero_page: fake page")); 1756 PMAP_STATS_INC(pmap_nzero_page); 1757 pa = VM_PAGE_TO_PHYS(m); 1758 if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { 1759 PMAP_STATS_INC(pmap_nzero_page_c); 1760 va = TLB_PHYS_TO_DIRECT(pa); 1761 cpu_block_zero((void *)va, PAGE_SIZE); 1762 } else if (m->md.color == -1) { 1763 PMAP_STATS_INC(pmap_nzero_page_nc); 1764 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1765 } else { 1766 PMAP_STATS_INC(pmap_nzero_page_oc); 1767 PMAP_LOCK(kernel_pmap); 1768 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1769 tp = tsb_kvtotte(va); 1770 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1771 tp->tte_vpn = TV_VPN(va, TS_8K); 1772 cpu_block_zero((void *)va, PAGE_SIZE); 1773 tlb_page_demap(kernel_pmap, va); 1774 PMAP_UNLOCK(kernel_pmap); 1775 } 1776} 1777 1778void 1779pmap_zero_page_area(vm_page_t m, int off, int size) 1780{ 1781 struct tte *tp; 1782 vm_offset_t va; 1783 vm_paddr_t pa; 1784 1785 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1786 ("pmap_zero_page_area: fake page")); 1787 KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); 1788 PMAP_STATS_INC(pmap_nzero_page_area); 1789 pa = VM_PAGE_TO_PHYS(m); 1790 if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { 1791 PMAP_STATS_INC(pmap_nzero_page_area_c); 1792 va = TLB_PHYS_TO_DIRECT(pa); 1793 bzero((void *)(va + off), size); 1794 } else if (m->md.color == -1) { 1795 PMAP_STATS_INC(pmap_nzero_page_area_nc); 1796 aszero(ASI_PHYS_USE_EC, pa + off, size); 1797 } else { 1798 PMAP_STATS_INC(pmap_nzero_page_area_oc); 1799 PMAP_LOCK(kernel_pmap); 1800 va = pmap_temp_map_1 + (m->md.color * PAGE_SIZE); 1801 tp = tsb_kvtotte(va); 1802 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1803 tp->tte_vpn = TV_VPN(va, TS_8K); 1804 bzero((void *)(va + off), size); 1805 tlb_page_demap(kernel_pmap, va); 1806 PMAP_UNLOCK(kernel_pmap); 1807 } 1808} 1809 1810void 1811pmap_zero_page_idle(vm_page_t m) 1812{ 1813 struct tte *tp; 1814 vm_offset_t va; 1815 vm_paddr_t pa; 1816 1817 KASSERT((m->flags & PG_FICTITIOUS) == 0, 1818 ("pmap_zero_page_idle: fake page")); 1819 PMAP_STATS_INC(pmap_nzero_page_idle); 1820 pa = VM_PAGE_TO_PHYS(m); 1821 if (dcache_color_ignore != 0 || m->md.color == DCACHE_COLOR(pa)) { 1822 PMAP_STATS_INC(pmap_nzero_page_idle_c); 1823 va = TLB_PHYS_TO_DIRECT(pa); 1824 cpu_block_zero((void *)va, PAGE_SIZE); 1825 } else if (m->md.color == -1) { 1826 PMAP_STATS_INC(pmap_nzero_page_idle_nc); 1827 aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); 1828 } else { 1829 PMAP_STATS_INC(pmap_nzero_page_idle_oc); 1830 va = pmap_idle_map + (m->md.color * PAGE_SIZE); 1831 tp = tsb_kvtotte(va); 1832 tp->tte_data = TD_V | TD_8K | TD_PA(pa) | TD_CP | TD_CV | TD_W; 1833 tp->tte_vpn = TV_VPN(va, TS_8K); 1834 cpu_block_zero((void *)va, PAGE_SIZE); 1835 tlb_page_demap(kernel_pmap, va); 1836 } 1837} 1838 1839void 1840pmap_copy_page(vm_page_t msrc, vm_page_t mdst) 1841{ 1842 vm_offset_t vdst; 1843 vm_offset_t vsrc; 1844 vm_paddr_t pdst; 1845 vm_paddr_t psrc; 1846 struct tte *tp; 1847 1848 KASSERT((mdst->flags & PG_FICTITIOUS) == 0, 1849 ("pmap_copy_page: fake dst page")); 1850 KASSERT((msrc->flags & PG_FICTITIOUS) == 0, 1851 ("pmap_copy_page: fake src page")); 1852 PMAP_STATS_INC(pmap_ncopy_page); 1853 pdst = VM_PAGE_TO_PHYS(mdst); 1854 psrc = VM_PAGE_TO_PHYS(msrc); 1855 if (dcache_color_ignore != 0 || 1856 (msrc->md.color == DCACHE_COLOR(psrc) && 1857 mdst->md.color == DCACHE_COLOR(pdst))) { 1858 PMAP_STATS_INC(pmap_ncopy_page_c); 1859 vdst = TLB_PHYS_TO_DIRECT(pdst); 1860 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1861 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1862 } else if (msrc->md.color == -1 && mdst->md.color == -1) { 1863 PMAP_STATS_INC(pmap_ncopy_page_nc); 1864 ascopy(ASI_PHYS_USE_EC, psrc, pdst, PAGE_SIZE); 1865 } else if (msrc->md.color == -1) { 1866 if (mdst->md.color == DCACHE_COLOR(pdst)) { 1867 PMAP_STATS_INC(pmap_ncopy_page_dc); 1868 vdst = TLB_PHYS_TO_DIRECT(pdst); 1869 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1870 PAGE_SIZE); 1871 } else { 1872 PMAP_STATS_INC(pmap_ncopy_page_doc); 1873 PMAP_LOCK(kernel_pmap); 1874 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1875 tp = tsb_kvtotte(vdst); 1876 tp->tte_data = 1877 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1878 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1879 ascopyfrom(ASI_PHYS_USE_EC, psrc, (void *)vdst, 1880 PAGE_SIZE); 1881 tlb_page_demap(kernel_pmap, vdst); 1882 PMAP_UNLOCK(kernel_pmap); 1883 } 1884 } else if (mdst->md.color == -1) { 1885 if (msrc->md.color == DCACHE_COLOR(psrc)) { 1886 PMAP_STATS_INC(pmap_ncopy_page_sc); 1887 vsrc = TLB_PHYS_TO_DIRECT(psrc); 1888 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1889 PAGE_SIZE); 1890 } else { 1891 PMAP_STATS_INC(pmap_ncopy_page_soc); 1892 PMAP_LOCK(kernel_pmap); 1893 vsrc = pmap_temp_map_1 + (msrc->md.color * PAGE_SIZE); 1894 tp = tsb_kvtotte(vsrc); 1895 tp->tte_data = 1896 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1897 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1898 ascopyto((void *)vsrc, ASI_PHYS_USE_EC, pdst, 1899 PAGE_SIZE); 1900 tlb_page_demap(kernel_pmap, vsrc); 1901 PMAP_UNLOCK(kernel_pmap); 1902 } 1903 } else { 1904 PMAP_STATS_INC(pmap_ncopy_page_oc); 1905 PMAP_LOCK(kernel_pmap); 1906 vdst = pmap_temp_map_1 + (mdst->md.color * PAGE_SIZE); 1907 tp = tsb_kvtotte(vdst); 1908 tp->tte_data = 1909 TD_V | TD_8K | TD_PA(pdst) | TD_CP | TD_CV | TD_W; 1910 tp->tte_vpn = TV_VPN(vdst, TS_8K); 1911 vsrc = pmap_temp_map_2 + (msrc->md.color * PAGE_SIZE); 1912 tp = tsb_kvtotte(vsrc); 1913 tp->tte_data = 1914 TD_V | TD_8K | TD_PA(psrc) | TD_CP | TD_CV | TD_W; 1915 tp->tte_vpn = TV_VPN(vsrc, TS_8K); 1916 cpu_block_copy((void *)vsrc, (void *)vdst, PAGE_SIZE); 1917 tlb_page_demap(kernel_pmap, vdst); 1918 tlb_page_demap(kernel_pmap, vsrc); 1919 PMAP_UNLOCK(kernel_pmap); 1920 } 1921} 1922 1923int unmapped_buf_allowed; 1924 1925void 1926pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], 1927 vm_offset_t b_offset, int xfersize) 1928{ 1929 1930 panic("pmap_copy_pages: not implemented"); 1931} 1932 1933/* 1934 * Returns true if the pmap's pv is one of the first 1935 * 16 pvs linked to from this page. This count may 1936 * be changed upwards or downwards in the future; it 1937 * is only necessary that true be returned for a small 1938 * subset of pmaps for proper page aging. 1939 */ 1940boolean_t 1941pmap_page_exists_quick(pmap_t pm, vm_page_t m) 1942{ 1943 struct tte *tp; 1944 int loops; 1945 boolean_t rv; 1946 1947 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 1948 ("pmap_page_exists_quick: page %p is not managed", m)); 1949 loops = 0; 1950 rv = FALSE; 1951 rw_wlock(&tte_list_global_lock); 1952 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 1953 if ((tp->tte_data & TD_PV) == 0) 1954 continue; 1955 if (TTE_GET_PMAP(tp) == pm) { 1956 rv = TRUE; 1957 break; 1958 } 1959 if (++loops >= 16) 1960 break; 1961 } 1962 rw_wunlock(&tte_list_global_lock); 1963 return (rv); 1964} 1965 1966/* 1967 * Return the number of managed mappings to the given physical page 1968 * that are wired. 1969 */ 1970int 1971pmap_page_wired_mappings(vm_page_t m) 1972{ 1973 struct tte *tp; 1974 int count; 1975 1976 count = 0; 1977 if ((m->oflags & VPO_UNMANAGED) != 0) 1978 return (count); 1979 rw_wlock(&tte_list_global_lock); 1980 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 1981 if ((tp->tte_data & (TD_PV | TD_WIRED)) == (TD_PV | TD_WIRED)) 1982 count++; 1983 rw_wunlock(&tte_list_global_lock); 1984 return (count); 1985} 1986 1987/* 1988 * Remove all pages from specified address space, this aids process exit 1989 * speeds. This is much faster than pmap_remove in the case of running down 1990 * an entire address space. Only works for the current pmap. 1991 */ 1992void 1993pmap_remove_pages(pmap_t pm) 1994{ 1995 1996} 1997 1998/* 1999 * Returns TRUE if the given page has a managed mapping. 2000 */ 2001boolean_t 2002pmap_page_is_mapped(vm_page_t m) 2003{ 2004 struct tte *tp; 2005 boolean_t rv; 2006 2007 rv = FALSE; 2008 if ((m->oflags & VPO_UNMANAGED) != 0) 2009 return (rv); 2010 rw_wlock(&tte_list_global_lock); 2011 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) 2012 if ((tp->tte_data & TD_PV) != 0) { 2013 rv = TRUE; 2014 break; 2015 } 2016 rw_wunlock(&tte_list_global_lock); 2017 return (rv); 2018} 2019 2020/* 2021 * Return a count of reference bits for a page, clearing those bits. 2022 * It is not necessary for every reference bit to be cleared, but it 2023 * is necessary that 0 only be returned when there are truly no 2024 * reference bits set. 2025 * 2026 * XXX: The exact number of bits to check and clear is a matter that 2027 * should be tested and standardized at some point in the future for 2028 * optimal aging of shared pages. 2029 */ 2030int 2031pmap_ts_referenced(vm_page_t m) 2032{ 2033 struct tte *tpf; 2034 struct tte *tpn; 2035 struct tte *tp; 2036 u_long data; 2037 int count; 2038 2039 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2040 ("pmap_ts_referenced: page %p is not managed", m)); 2041 count = 0; 2042 rw_wlock(&tte_list_global_lock); 2043 if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { 2044 tpf = tp; 2045 do { 2046 tpn = TAILQ_NEXT(tp, tte_link); 2047 TAILQ_REMOVE(&m->md.tte_list, tp, tte_link); 2048 TAILQ_INSERT_TAIL(&m->md.tte_list, tp, tte_link); 2049 if ((tp->tte_data & TD_PV) == 0) 2050 continue; 2051 data = atomic_clear_long(&tp->tte_data, TD_REF); 2052 if ((data & TD_REF) != 0 && ++count > 4) 2053 break; 2054 } while ((tp = tpn) != NULL && tp != tpf); 2055 } 2056 rw_wunlock(&tte_list_global_lock); 2057 return (count); 2058} 2059 2060boolean_t 2061pmap_is_modified(vm_page_t m) 2062{ 2063 struct tte *tp; 2064 boolean_t rv; 2065 2066 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2067 ("pmap_is_modified: page %p is not managed", m)); 2068 rv = FALSE; 2069 2070 /* 2071 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be 2072 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2073 * is clear, no TTEs can have TD_W set. 2074 */ 2075 VM_OBJECT_ASSERT_WLOCKED(m->object); 2076 if ((m->oflags & VPO_BUSY) == 0 && 2077 (m->aflags & PGA_WRITEABLE) == 0) 2078 return (rv); 2079 rw_wlock(&tte_list_global_lock); 2080 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2081 if ((tp->tte_data & TD_PV) == 0) 2082 continue; 2083 if ((tp->tte_data & TD_W) != 0) { 2084 rv = TRUE; 2085 break; 2086 } 2087 } 2088 rw_wunlock(&tte_list_global_lock); 2089 return (rv); 2090} 2091 2092/* 2093 * pmap_is_prefaultable: 2094 * 2095 * Return whether or not the specified virtual address is elgible 2096 * for prefault. 2097 */ 2098boolean_t 2099pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2100{ 2101 boolean_t rv; 2102 2103 PMAP_LOCK(pmap); 2104 rv = tsb_tte_lookup(pmap, addr) == NULL; 2105 PMAP_UNLOCK(pmap); 2106 return (rv); 2107} 2108 2109/* 2110 * Return whether or not the specified physical page was referenced 2111 * in any physical maps. 2112 */ 2113boolean_t 2114pmap_is_referenced(vm_page_t m) 2115{ 2116 struct tte *tp; 2117 boolean_t rv; 2118 2119 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2120 ("pmap_is_referenced: page %p is not managed", m)); 2121 rv = FALSE; 2122 rw_wlock(&tte_list_global_lock); 2123 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2124 if ((tp->tte_data & TD_PV) == 0) 2125 continue; 2126 if ((tp->tte_data & TD_REF) != 0) { 2127 rv = TRUE; 2128 break; 2129 } 2130 } 2131 rw_wunlock(&tte_list_global_lock); 2132 return (rv); 2133} 2134 2135void 2136pmap_clear_modify(vm_page_t m) 2137{ 2138 struct tte *tp; 2139 u_long data; 2140 2141 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2142 ("pmap_clear_modify: page %p is not managed", m)); 2143 VM_OBJECT_ASSERT_WLOCKED(m->object); 2144 KASSERT((m->oflags & VPO_BUSY) == 0, 2145 ("pmap_clear_modify: page %p is busy", m)); 2146 2147 /* 2148 * If the page is not PGA_WRITEABLE, then no TTEs can have TD_W set. 2149 * If the object containing the page is locked and the page is not 2150 * VPO_BUSY, then PGA_WRITEABLE cannot be concurrently set. 2151 */ 2152 if ((m->aflags & PGA_WRITEABLE) == 0) 2153 return; 2154 rw_wlock(&tte_list_global_lock); 2155 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2156 if ((tp->tte_data & TD_PV) == 0) 2157 continue; 2158 data = atomic_clear_long(&tp->tte_data, TD_W); 2159 if ((data & TD_W) != 0) 2160 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2161 } 2162 rw_wunlock(&tte_list_global_lock); 2163} 2164 2165void 2166pmap_clear_reference(vm_page_t m) 2167{ 2168 struct tte *tp; 2169 u_long data; 2170 2171 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2172 ("pmap_clear_reference: page %p is not managed", m)); 2173 rw_wlock(&tte_list_global_lock); 2174 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2175 if ((tp->tte_data & TD_PV) == 0) 2176 continue; 2177 data = atomic_clear_long(&tp->tte_data, TD_REF); 2178 if ((data & TD_REF) != 0) 2179 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2180 } 2181 rw_wunlock(&tte_list_global_lock); 2182} 2183 2184void 2185pmap_remove_write(vm_page_t m) 2186{ 2187 struct tte *tp; 2188 u_long data; 2189 2190 KASSERT((m->oflags & VPO_UNMANAGED) == 0, 2191 ("pmap_remove_write: page %p is not managed", m)); 2192 2193 /* 2194 * If the page is not VPO_BUSY, then PGA_WRITEABLE cannot be set by 2195 * another thread while the object is locked. Thus, if PGA_WRITEABLE 2196 * is clear, no page table entries need updating. 2197 */ 2198 VM_OBJECT_ASSERT_WLOCKED(m->object); 2199 if ((m->oflags & VPO_BUSY) == 0 && 2200 (m->aflags & PGA_WRITEABLE) == 0) 2201 return; 2202 rw_wlock(&tte_list_global_lock); 2203 TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { 2204 if ((tp->tte_data & TD_PV) == 0) 2205 continue; 2206 data = atomic_clear_long(&tp->tte_data, TD_SW | TD_W); 2207 if ((data & TD_W) != 0) { 2208 vm_page_dirty(m); 2209 tlb_page_demap(TTE_GET_PMAP(tp), TTE_GET_VA(tp)); 2210 } 2211 } 2212 vm_page_aflag_clear(m, PGA_WRITEABLE); 2213 rw_wunlock(&tte_list_global_lock); 2214} 2215 2216int 2217pmap_mincore(pmap_t pm, vm_offset_t addr, vm_paddr_t *locked_pa) 2218{ 2219 2220 /* TODO; */ 2221 return (0); 2222} 2223 2224/* 2225 * Activate a user pmap. The pmap must be activated before its address space 2226 * can be accessed in any way. 2227 */ 2228void 2229pmap_activate(struct thread *td) 2230{ 2231 struct vmspace *vm; 2232 struct pmap *pm; 2233 int context; 2234 2235 critical_enter(); 2236 vm = td->td_proc->p_vmspace; 2237 pm = vmspace_pmap(vm); 2238 2239 context = PCPU_GET(tlb_ctx); 2240 if (context == PCPU_GET(tlb_ctx_max)) { 2241 tlb_flush_user(); 2242 context = PCPU_GET(tlb_ctx_min); 2243 } 2244 PCPU_SET(tlb_ctx, context + 1); 2245 2246 pm->pm_context[curcpu] = context; 2247#ifdef SMP 2248 CPU_SET_ATOMIC(PCPU_GET(cpuid), &pm->pm_active); 2249 atomic_store_ptr((uintptr_t *)PCPU_PTR(pmap), (uintptr_t)pm); 2250#else 2251 CPU_SET(PCPU_GET(cpuid), &pm->pm_active); 2252 PCPU_SET(pmap, pm); 2253#endif 2254 2255 stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); 2256 stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); 2257 stxa(AA_DMMU_PCXR, ASI_DMMU, (ldxa(AA_DMMU_PCXR, ASI_DMMU) & 2258 TLB_CXR_PGSZ_MASK) | context); 2259 flush(KERNBASE); 2260 critical_exit(); 2261} 2262 2263void 2264pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) 2265{ 2266 2267} 2268 2269/* 2270 * Increase the starting virtual address of the given mapping if a 2271 * different alignment might result in more superpage mappings. 2272 */ 2273void 2274pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, 2275 vm_offset_t *addr, vm_size_t size) 2276{ 2277 2278} 2279