pmap-v4.c revision 169763
1/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ 2/*- 3 * Copyright 2004 Olivier Houchard. 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38/*- 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65/*- 66 * Copyright (c) 1999 The NetBSD Foundation, Inc. 67 * All rights reserved. 68 * 69 * This code is derived from software contributed to The NetBSD Foundation 70 * by Charles M. Hannum. 71 * 72 * Redistribution and use in source and binary forms, with or without 73 * modification, are permitted provided that the following conditions 74 * are met: 75 * 1. Redistributions of source code must retain the above copyright 76 * notice, this list of conditions and the following disclaimer. 77 * 2. Redistributions in binary form must reproduce the above copyright 78 * notice, this list of conditions and the following disclaimer in the 79 * documentation and/or other materials provided with the distribution. 80 * 3. All advertising materials mentioning features or use of this software 81 * must display the following acknowledgement: 82 * This product includes software developed by the NetBSD 83 * Foundation, Inc. and its contributors. 84 * 4. Neither the name of The NetBSD Foundation nor the names of its 85 * contributors may be used to endorse or promote products derived 86 * from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 89 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 90 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 91 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 92 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 93 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 94 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 95 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 96 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 97 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 98 * POSSIBILITY OF SUCH DAMAGE. 99 */ 100 101/*- 102 * Copyright (c) 1994-1998 Mark Brinicombe. 103 * Copyright (c) 1994 Brini. 104 * All rights reserved. 105 * 106 * This code is derived from software written for Brini by Mark Brinicombe 107 * 108 * Redistribution and use in source and binary forms, with or without 109 * modification, are permitted provided that the following conditions 110 * are met: 111 * 1. Redistributions of source code must retain the above copyright 112 * notice, this list of conditions and the following disclaimer. 113 * 2. Redistributions in binary form must reproduce the above copyright 114 * notice, this list of conditions and the following disclaimer in the 115 * documentation and/or other materials provided with the distribution. 116 * 3. All advertising materials mentioning features or use of this software 117 * must display the following acknowledgement: 118 * This product includes software developed by Mark Brinicombe. 119 * 4. The name of the author may not be used to endorse or promote products 120 * derived from this software without specific prior written permission. 121 * 122 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 123 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 124 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 125 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 126 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 127 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 128 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 129 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 130 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 131 * 132 * RiscBSD kernel project 133 * 134 * pmap.c 135 * 136 * Machine dependant vm stuff 137 * 138 * Created : 20/09/94 139 */ 140 141/* 142 * Special compilation symbols 143 * PMAP_DEBUG - Build in pmap_debug_level code 144 */ 145/* Include header files */ 146 147#include "opt_vm.h" 148 149#include <sys/cdefs.h> 150__FBSDID("$FreeBSD: head/sys/arm/arm/pmap.c 169763 2007-05-19 13:21:41Z cognet $"); 151#include <sys/param.h> 152#include <sys/systm.h> 153#include <sys/kernel.h> 154#include <sys/proc.h> 155#include <sys/malloc.h> 156#include <sys/msgbuf.h> 157#include <sys/vmmeter.h> 158#include <sys/mman.h> 159#include <sys/smp.h> 160#include <sys/sched.h> 161 162#include <vm/vm.h> 163#include <vm/uma.h> 164#include <vm/pmap.h> 165#include <vm/vm_kern.h> 166#include <vm/vm_object.h> 167#include <vm/vm_map.h> 168#include <vm/vm_page.h> 169#include <vm/vm_pageout.h> 170#include <vm/vm_extern.h> 171#include <sys/lock.h> 172#include <sys/mutex.h> 173#include <machine/md_var.h> 174#include <machine/vmparam.h> 175#include <machine/cpu.h> 176#include <machine/cpufunc.h> 177#include <machine/pcb.h> 178 179#ifdef PMAP_DEBUG 180#define PDEBUG(_lev_,_stat_) \ 181 if (pmap_debug_level >= (_lev_)) \ 182 ((_stat_)) 183#define dprintf printf 184 185int pmap_debug_level = 0; 186#define PMAP_INLINE 187#else /* PMAP_DEBUG */ 188#define PDEBUG(_lev_,_stat_) /* Nothing */ 189#define dprintf(x, arg...) 190#define PMAP_INLINE __inline 191#endif /* PMAP_DEBUG */ 192 193extern struct pv_addr systempage; 194/* 195 * Internal function prototypes 196 */ 197static void pmap_free_pv_entry (pv_entry_t); 198static pv_entry_t pmap_get_pv_entry(void); 199 200static void pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, 201 vm_prot_t, boolean_t, int); 202static void pmap_vac_me_harder(struct vm_page *, pmap_t, 203 vm_offset_t); 204static void pmap_vac_me_kpmap(struct vm_page *, pmap_t, 205 vm_offset_t); 206static void pmap_vac_me_user(struct vm_page *, pmap_t, vm_offset_t); 207static void pmap_alloc_l1(pmap_t); 208static void pmap_free_l1(pmap_t); 209static void pmap_use_l1(pmap_t); 210 211static int pmap_clearbit(struct vm_page *, u_int); 212 213static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); 214static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); 215static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 216static vm_offset_t kernel_pt_lookup(vm_paddr_t); 217 218static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); 219 220vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 221vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 222vm_offset_t pmap_curmaxkvaddr; 223vm_paddr_t kernel_l1pa; 224 225extern void *end; 226vm_offset_t kernel_vm_end = 0; 227 228struct pmap kernel_pmap_store; 229pmap_t kernel_pmap; 230 231static pt_entry_t *csrc_pte, *cdst_pte; 232static vm_offset_t csrcp, cdstp; 233static struct mtx cmtx; 234 235static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 236/* 237 * These routines are called when the CPU type is identified to set up 238 * the PTE prototypes, cache modes, etc. 239 * 240 * The variables are always here, just in case LKMs need to reference 241 * them (though, they shouldn't). 242 */ 243 244pt_entry_t pte_l1_s_cache_mode; 245pt_entry_t pte_l1_s_cache_mode_pt; 246pt_entry_t pte_l1_s_cache_mask; 247 248pt_entry_t pte_l2_l_cache_mode; 249pt_entry_t pte_l2_l_cache_mode_pt; 250pt_entry_t pte_l2_l_cache_mask; 251 252pt_entry_t pte_l2_s_cache_mode; 253pt_entry_t pte_l2_s_cache_mode_pt; 254pt_entry_t pte_l2_s_cache_mask; 255 256pt_entry_t pte_l2_s_prot_u; 257pt_entry_t pte_l2_s_prot_w; 258pt_entry_t pte_l2_s_prot_mask; 259 260pt_entry_t pte_l1_s_proto; 261pt_entry_t pte_l1_c_proto; 262pt_entry_t pte_l2_s_proto; 263 264void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); 265void (*pmap_zero_page_func)(vm_paddr_t, int, int); 266/* 267 * Which pmap is currently 'live' in the cache 268 * 269 * XXXSCW: Fix for SMP ... 270 */ 271union pmap_cache_state *pmap_cache_state; 272 273/* static pt_entry_t *msgbufmap;*/ 274struct msgbuf *msgbufp = 0; 275 276extern void bcopy_page(vm_offset_t, vm_offset_t); 277extern void bzero_page(vm_offset_t); 278 279extern vm_offset_t alloc_firstaddr; 280 281char *_tmppt; 282 283/* 284 * Metadata for L1 translation tables. 285 */ 286struct l1_ttable { 287 /* Entry on the L1 Table list */ 288 SLIST_ENTRY(l1_ttable) l1_link; 289 290 /* Entry on the L1 Least Recently Used list */ 291 TAILQ_ENTRY(l1_ttable) l1_lru; 292 293 /* Track how many domains are allocated from this L1 */ 294 volatile u_int l1_domain_use_count; 295 296 /* 297 * A free-list of domain numbers for this L1. 298 * We avoid using ffs() and a bitmap to track domains since ffs() 299 * is slow on ARM. 300 */ 301 u_int8_t l1_domain_first; 302 u_int8_t l1_domain_free[PMAP_DOMAINS]; 303 304 /* Physical address of this L1 page table */ 305 vm_paddr_t l1_physaddr; 306 307 /* KVA of this L1 page table */ 308 pd_entry_t *l1_kva; 309}; 310 311/* 312 * Convert a virtual address into its L1 table index. That is, the 313 * index used to locate the L2 descriptor table pointer in an L1 table. 314 * This is basically used to index l1->l1_kva[]. 315 * 316 * Each L2 descriptor table represents 1MB of VA space. 317 */ 318#define L1_IDX(va) (((vm_offset_t)(va)) >> L1_S_SHIFT) 319 320/* 321 * L1 Page Tables are tracked using a Least Recently Used list. 322 * - New L1s are allocated from the HEAD. 323 * - Freed L1s are added to the TAIl. 324 * - Recently accessed L1s (where an 'access' is some change to one of 325 * the userland pmaps which owns this L1) are moved to the TAIL. 326 */ 327static TAILQ_HEAD(, l1_ttable) l1_lru_list; 328/* 329 * A list of all L1 tables 330 */ 331static SLIST_HEAD(, l1_ttable) l1_list; 332static struct mtx l1_lru_lock; 333 334/* 335 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 336 * 337 * This is normally 16MB worth L2 page descriptors for any given pmap. 338 * Reference counts are maintained for L2 descriptors so they can be 339 * freed when empty. 340 */ 341struct l2_dtable { 342 /* The number of L2 page descriptors allocated to this l2_dtable */ 343 u_int l2_occupancy; 344 345 /* List of L2 page descriptors */ 346 struct l2_bucket { 347 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 348 vm_paddr_t l2b_phys; /* Physical address of same */ 349 u_short l2b_l1idx; /* This L2 table's L1 index */ 350 u_short l2b_occupancy; /* How many active descriptors */ 351 } l2_bucket[L2_BUCKET_SIZE]; 352}; 353 354/* pmap_kenter_internal flags */ 355#define KENTER_CACHE 0x1 356#define KENTER_USER 0x2 357 358/* 359 * Given an L1 table index, calculate the corresponding l2_dtable index 360 * and bucket index within the l2_dtable. 361 */ 362#define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 363 (L2_SIZE - 1)) 364#define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 365 366/* 367 * Given a virtual address, this macro returns the 368 * virtual address required to drop into the next L2 bucket. 369 */ 370#define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 371 372/* 373 * L2 allocation. 374 */ 375#define pmap_alloc_l2_dtable() \ 376 (void*)uma_zalloc(l2table_zone, M_NOWAIT|M_USE_RESERVE) 377#define pmap_free_l2_dtable(l2) \ 378 uma_zfree(l2table_zone, l2) 379 380/* 381 * We try to map the page tables write-through, if possible. However, not 382 * all CPUs have a write-through cache mode, so on those we have to sync 383 * the cache when we frob page tables. 384 * 385 * We try to evaluate this at compile time, if possible. However, it's 386 * not always possible to do that, hence this run-time var. 387 */ 388int pmap_needs_pte_sync; 389 390/* 391 * Macro to determine if a mapping might be resident in the 392 * instruction cache and/or TLB 393 */ 394#define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 395 396/* 397 * Macro to determine if a mapping might be resident in the 398 * data cache and/or TLB 399 */ 400#define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 401 402#ifndef PMAP_SHPGPERPROC 403#define PMAP_SHPGPERPROC 200 404#endif 405 406#define pmap_is_current(pm) ((pm) == pmap_kernel() || \ 407 curproc->p_vmspace->vm_map.pmap == (pm)) 408static uma_zone_t pvzone; 409uma_zone_t l2zone; 410static uma_zone_t l2table_zone; 411static vm_offset_t pmap_kernel_l2dtable_kva; 412static vm_offset_t pmap_kernel_l2ptp_kva; 413static vm_paddr_t pmap_kernel_l2ptp_phys; 414static struct vm_object pvzone_obj; 415static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; 416 417/* 418 * This list exists for the benefit of pmap_map_chunk(). It keeps track 419 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 420 * find them as necessary. 421 * 422 * Note that the data on this list MUST remain valid after initarm() returns, 423 * as pmap_bootstrap() uses it to contruct L2 table metadata. 424 */ 425SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 426 427static void 428pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 429{ 430 int i; 431 432 l1->l1_kva = l1pt; 433 l1->l1_domain_use_count = 0; 434 l1->l1_domain_first = 1; 435 436 for (i = 0; i < PMAP_DOMAINS; i++) 437 l1->l1_domain_free[i] = i + 2; 438 439 /* 440 * Copy the kernel's L1 entries to each new L1. 441 */ 442 if (l1pt != pmap_kernel()->pm_l1->l1_kva) 443 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 444 445 if ((l1->l1_physaddr = pmap_extract(pmap_kernel(), (vm_offset_t)l1pt)) == 0) 446 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 447 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 448 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 449} 450 451static vm_offset_t 452kernel_pt_lookup(vm_paddr_t pa) 453{ 454 struct pv_addr *pv; 455 456 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 457 if (pv->pv_pa == pa) 458 return (pv->pv_va); 459 } 460 return (0); 461} 462 463#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 464void 465pmap_pte_init_generic(void) 466{ 467 468 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 469 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 470 471 pte_l2_l_cache_mode = L2_B|L2_C; 472 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 473 474 pte_l2_s_cache_mode = L2_B|L2_C; 475 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 476 477 /* 478 * If we have a write-through cache, set B and C. If 479 * we have a write-back cache, then we assume setting 480 * only C will make those pages write-through. 481 */ 482 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { 483 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 484 pte_l2_l_cache_mode_pt = L2_B|L2_C; 485 pte_l2_s_cache_mode_pt = L2_B|L2_C; 486 } else { 487 pte_l1_s_cache_mode_pt = L1_S_C; 488 pte_l2_l_cache_mode_pt = L2_C; 489 pte_l2_s_cache_mode_pt = L2_C; 490 } 491 492 pte_l2_s_prot_u = L2_S_PROT_U_generic; 493 pte_l2_s_prot_w = L2_S_PROT_W_generic; 494 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 495 496 pte_l1_s_proto = L1_S_PROTO_generic; 497 pte_l1_c_proto = L1_C_PROTO_generic; 498 pte_l2_s_proto = L2_S_PROTO_generic; 499 500 pmap_copy_page_func = pmap_copy_page_generic; 501 pmap_zero_page_func = pmap_zero_page_generic; 502} 503 504#if defined(CPU_ARM8) 505void 506pmap_pte_init_arm8(void) 507{ 508 509 /* 510 * ARM8 is compatible with generic, but we need to use 511 * the page tables uncached. 512 */ 513 pmap_pte_init_generic(); 514 515 pte_l1_s_cache_mode_pt = 0; 516 pte_l2_l_cache_mode_pt = 0; 517 pte_l2_s_cache_mode_pt = 0; 518} 519#endif /* CPU_ARM8 */ 520 521#if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 522void 523pmap_pte_init_arm9(void) 524{ 525 526 /* 527 * ARM9 is compatible with generic, but we want to use 528 * write-through caching for now. 529 */ 530 pmap_pte_init_generic(); 531 532 pte_l1_s_cache_mode = L1_S_C; 533 pte_l2_l_cache_mode = L2_C; 534 pte_l2_s_cache_mode = L2_C; 535 536 pte_l1_s_cache_mode_pt = L1_S_C; 537 pte_l2_l_cache_mode_pt = L2_C; 538 pte_l2_s_cache_mode_pt = L2_C; 539} 540#endif /* CPU_ARM9 */ 541#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 542 543#if defined(CPU_ARM10) 544void 545pmap_pte_init_arm10(void) 546{ 547 548 /* 549 * ARM10 is compatible with generic, but we want to use 550 * write-through caching for now. 551 */ 552 pmap_pte_init_generic(); 553 554 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 555 pte_l2_l_cache_mode = L2_B | L2_C; 556 pte_l2_s_cache_mode = L2_B | L2_C; 557 558 pte_l1_s_cache_mode_pt = L1_S_C; 559 pte_l2_l_cache_mode_pt = L2_C; 560 pte_l2_s_cache_mode_pt = L2_C; 561 562} 563#endif /* CPU_ARM10 */ 564 565#if ARM_MMU_SA1 == 1 566void 567pmap_pte_init_sa1(void) 568{ 569 570 /* 571 * The StrongARM SA-1 cache does not have a write-through 572 * mode. So, do the generic initialization, then reset 573 * the page table cache mode to B=1,C=1, and note that 574 * the PTEs need to be sync'd. 575 */ 576 pmap_pte_init_generic(); 577 578 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 579 pte_l2_l_cache_mode_pt = L2_B|L2_C; 580 pte_l2_s_cache_mode_pt = L2_B|L2_C; 581 582 pmap_needs_pte_sync = 1; 583} 584#endif /* ARM_MMU_SA1 == 1*/ 585 586#if ARM_MMU_XSCALE == 1 587#if (ARM_NMMUS > 1) || defined (CPU_XSCALE_CORE3) 588static u_int xscale_use_minidata; 589#endif 590 591void 592pmap_pte_init_xscale(void) 593{ 594 uint32_t auxctl; 595 int write_through = 0; 596 597 pte_l1_s_cache_mode = L1_S_B|L1_S_C|L1_S_XSCALE_P; 598 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 599 600 pte_l2_l_cache_mode = L2_B|L2_C; 601 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 602 603 pte_l2_s_cache_mode = L2_B|L2_C; 604 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 605 606 pte_l1_s_cache_mode_pt = L1_S_C; 607 pte_l2_l_cache_mode_pt = L2_C; 608 pte_l2_s_cache_mode_pt = L2_C; 609#ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 610 /* 611 * The XScale core has an enhanced mode where writes that 612 * miss the cache cause a cache line to be allocated. This 613 * is significantly faster than the traditional, write-through 614 * behavior of this case. 615 */ 616#ifndef CPU_XSCALE_CORE3 617 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 618 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 619 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 620#endif 621#endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 622#ifdef XSCALE_CACHE_WRITE_THROUGH 623 /* 624 * Some versions of the XScale core have various bugs in 625 * their cache units, the work-around for which is to run 626 * the cache in write-through mode. Unfortunately, this 627 * has a major (negative) impact on performance. So, we 628 * go ahead and run fast-and-loose, in the hopes that we 629 * don't line up the planets in a way that will trip the 630 * bugs. 631 * 632 * However, we give you the option to be slow-but-correct. 633 */ 634 write_through = 1; 635#elif defined(XSCALE_CACHE_WRITE_BACK) 636 /* force write back cache mode */ 637 write_through = 0; 638#elif defined(CPU_XSCALE_PXA2X0) 639 /* 640 * Intel PXA2[15]0 processors are known to have a bug in 641 * write-back cache on revision 4 and earlier (stepping 642 * A[01] and B[012]). Fixed for C0 and later. 643 */ 644 { 645 uint32_t id, type; 646 647 id = cpufunc_id(); 648 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 649 650 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 651 if ((id & CPU_ID_REVISION_MASK) < 5) { 652 /* write through for stepping A0-1 and B0-2 */ 653 write_through = 1; 654 } 655 } 656 } 657#endif /* XSCALE_CACHE_WRITE_THROUGH */ 658 659 if (write_through) { 660 pte_l1_s_cache_mode = L1_S_C; 661 pte_l2_l_cache_mode = L2_C; 662 pte_l2_s_cache_mode = L2_C; 663 } 664 665#if (ARM_NMMUS > 1) 666 xscale_use_minidata = 1; 667#endif 668 669 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 670 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 671 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 672 673 pte_l1_s_proto = L1_S_PROTO_xscale; 674 pte_l1_c_proto = L1_C_PROTO_xscale; 675 pte_l2_s_proto = L2_S_PROTO_xscale; 676 677#ifdef CPU_XSCALE_CORE3 678 pmap_copy_page_func = pmap_copy_page_generic; 679 pmap_zero_page_func = pmap_zero_page_generic; 680 xscale_use_minidata = 0; 681 pte_l1_s_cache_mode_pt = pte_l2_l_cache_mode_pt = 682 pte_l2_s_cache_mode_pt = 0; 683#else 684 pmap_copy_page_func = pmap_copy_page_xscale; 685 pmap_zero_page_func = pmap_zero_page_xscale; 686#endif 687 688 /* 689 * Disable ECC protection of page table access, for now. 690 */ 691 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 692 auxctl &= ~XSCALE_AUXCTL_P; 693 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 694} 695 696/* 697 * xscale_setup_minidata: 698 * 699 * Set up the mini-data cache clean area. We require the 700 * caller to allocate the right amount of physically and 701 * virtually contiguous space. 702 */ 703extern vm_offset_t xscale_minidata_clean_addr; 704extern vm_size_t xscale_minidata_clean_size; /* already initialized */ 705void 706xscale_setup_minidata(vm_offset_t l1pt, vm_offset_t va, vm_paddr_t pa) 707{ 708 pd_entry_t *pde = (pd_entry_t *) l1pt; 709 pt_entry_t *pte; 710 vm_size_t size; 711 uint32_t auxctl; 712 713 xscale_minidata_clean_addr = va; 714 715 /* Round it to page size. */ 716 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 717 718 for (; size != 0; 719 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 720 pte = (pt_entry_t *) kernel_pt_lookup( 721 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 722 if (pte == NULL) 723 panic("xscale_setup_minidata: can't find L2 table for " 724 "VA 0x%08x", (u_int32_t) va); 725 pte[l2pte_index(va)] = 726 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 727 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 728 } 729 730 /* 731 * Configure the mini-data cache for write-back with 732 * read/write-allocate. 733 * 734 * NOTE: In order to reconfigure the mini-data cache, we must 735 * make sure it contains no valid data! In order to do that, 736 * we must issue a global data cache invalidate command! 737 * 738 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 739 * THIS IS VERY IMPORTANT! 740 */ 741 742 /* Invalidate data and mini-data. */ 743 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 744 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 745 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 746 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 747} 748#endif 749 750/* 751 * Allocate an L1 translation table for the specified pmap. 752 * This is called at pmap creation time. 753 */ 754static void 755pmap_alloc_l1(pmap_t pm) 756{ 757 struct l1_ttable *l1; 758 u_int8_t domain; 759 760 /* 761 * Remove the L1 at the head of the LRU list 762 */ 763 mtx_lock(&l1_lru_lock); 764 l1 = TAILQ_FIRST(&l1_lru_list); 765 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 766 767 /* 768 * Pick the first available domain number, and update 769 * the link to the next number. 770 */ 771 domain = l1->l1_domain_first; 772 l1->l1_domain_first = l1->l1_domain_free[domain]; 773 774 /* 775 * If there are still free domain numbers in this L1, 776 * put it back on the TAIL of the LRU list. 777 */ 778 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 779 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 780 781 mtx_unlock(&l1_lru_lock); 782 783 /* 784 * Fix up the relevant bits in the pmap structure 785 */ 786 pm->pm_l1 = l1; 787 pm->pm_domain = domain; 788} 789 790/* 791 * Free an L1 translation table. 792 * This is called at pmap destruction time. 793 */ 794static void 795pmap_free_l1(pmap_t pm) 796{ 797 struct l1_ttable *l1 = pm->pm_l1; 798 799 mtx_lock(&l1_lru_lock); 800 801 /* 802 * If this L1 is currently on the LRU list, remove it. 803 */ 804 if (l1->l1_domain_use_count < PMAP_DOMAINS) 805 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 806 807 /* 808 * Free up the domain number which was allocated to the pmap 809 */ 810 l1->l1_domain_free[pm->pm_domain] = l1->l1_domain_first; 811 l1->l1_domain_first = pm->pm_domain; 812 l1->l1_domain_use_count--; 813 814 /* 815 * The L1 now must have at least 1 free domain, so add 816 * it back to the LRU list. If the use count is zero, 817 * put it at the head of the list, otherwise it goes 818 * to the tail. 819 */ 820 if (l1->l1_domain_use_count == 0) { 821 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 822 } else 823 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 824 825 mtx_unlock(&l1_lru_lock); 826} 827 828static PMAP_INLINE void 829pmap_use_l1(pmap_t pm) 830{ 831 struct l1_ttable *l1; 832 833 /* 834 * Do nothing if we're in interrupt context. 835 * Access to an L1 by the kernel pmap must not affect 836 * the LRU list. 837 */ 838 if (pm == pmap_kernel()) 839 return; 840 841 l1 = pm->pm_l1; 842 843 /* 844 * If the L1 is not currently on the LRU list, just return 845 */ 846 if (l1->l1_domain_use_count == PMAP_DOMAINS) 847 return; 848 849 mtx_lock(&l1_lru_lock); 850 851 /* 852 * Check the use count again, now that we've acquired the lock 853 */ 854 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 855 mtx_unlock(&l1_lru_lock); 856 return; 857 } 858 859 /* 860 * Move the L1 to the back of the LRU list 861 */ 862 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 863 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 864 865 mtx_unlock(&l1_lru_lock); 866} 867 868 869/* 870 * Returns a pointer to the L2 bucket associated with the specified pmap 871 * and VA, or NULL if no L2 bucket exists for the address. 872 */ 873static PMAP_INLINE struct l2_bucket * 874pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) 875{ 876 struct l2_dtable *l2; 877 struct l2_bucket *l2b; 878 u_short l1idx; 879 880 l1idx = L1_IDX(va); 881 882 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 883 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 884 return (NULL); 885 886 return (l2b); 887} 888 889/* 890 * Returns a pointer to the L2 bucket associated with the specified pmap 891 * and VA. 892 * 893 * If no L2 bucket exists, perform the necessary allocations to put an L2 894 * bucket/page table in place. 895 * 896 * Note that if a new L2 bucket/page was allocated, the caller *must* 897 * increment the bucket occupancy counter appropriately *before* 898 * releasing the pmap's lock to ensure no other thread or cpu deallocates 899 * the bucket/page in the meantime. 900 */ 901static struct l2_bucket * 902pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) 903{ 904 struct l2_dtable *l2; 905 struct l2_bucket *l2b; 906 u_short l1idx; 907 908 l1idx = L1_IDX(va); 909 910 PMAP_ASSERT_LOCKED(pm); 911 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 912 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 913 /* 914 * No mapping at this address, as there is 915 * no entry in the L1 table. 916 * Need to allocate a new l2_dtable. 917 */ 918again_l2table: 919 PMAP_UNLOCK(pm); 920 vm_page_unlock_queues(); 921 if ((l2 = pmap_alloc_l2_dtable()) == NULL) { 922 vm_page_lock_queues(); 923 PMAP_LOCK(pm); 924 return (NULL); 925 } 926 vm_page_lock_queues(); 927 PMAP_LOCK(pm); 928 if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { 929 PMAP_UNLOCK(pm); 930 vm_page_unlock_queues(); 931 uma_zfree(l2table_zone, l2); 932 vm_page_lock_queues(); 933 PMAP_LOCK(pm); 934 l2 = pm->pm_l2[L2_IDX(l1idx)]; 935 if (l2 == NULL) 936 goto again_l2table; 937 /* 938 * Someone already allocated the l2_dtable while 939 * we were doing the same. 940 */ 941 } else { 942 bzero(l2, sizeof(*l2)); 943 /* 944 * Link it into the parent pmap 945 */ 946 pm->pm_l2[L2_IDX(l1idx)] = l2; 947 } 948 } 949 950 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 951 952 /* 953 * Fetch pointer to the L2 page table associated with the address. 954 */ 955 if (l2b->l2b_kva == NULL) { 956 pt_entry_t *ptep; 957 958 /* 959 * No L2 page table has been allocated. Chances are, this 960 * is because we just allocated the l2_dtable, above. 961 */ 962again_ptep: 963 PMAP_UNLOCK(pm); 964 vm_page_unlock_queues(); 965 ptep = (void*)uma_zalloc(l2zone, M_NOWAIT|M_USE_RESERVE); 966 vm_page_lock_queues(); 967 PMAP_LOCK(pm); 968 if (l2b->l2b_kva != 0) { 969 /* We lost the race. */ 970 PMAP_UNLOCK(pm); 971 vm_page_unlock_queues(); 972 uma_zfree(l2zone, ptep); 973 vm_page_lock_queues(); 974 PMAP_LOCK(pm); 975 if (l2b->l2b_kva == 0) 976 goto again_ptep; 977 return (l2b); 978 } 979 l2b->l2b_phys = vtophys(ptep); 980 if (ptep == NULL) { 981 /* 982 * Oops, no more L2 page tables available at this 983 * time. We may need to deallocate the l2_dtable 984 * if we allocated a new one above. 985 */ 986 if (l2->l2_occupancy == 0) { 987 pm->pm_l2[L2_IDX(l1idx)] = NULL; 988 pmap_free_l2_dtable(l2); 989 } 990 return (NULL); 991 } 992 993 l2->l2_occupancy++; 994 l2b->l2b_kva = ptep; 995 l2b->l2b_l1idx = l1idx; 996 } 997 998 return (l2b); 999} 1000 1001static PMAP_INLINE void 1002#ifndef PMAP_INCLUDE_PTE_SYNC 1003pmap_free_l2_ptp(pt_entry_t *l2) 1004#else 1005pmap_free_l2_ptp(boolean_t need_sync, pt_entry_t *l2) 1006#endif 1007{ 1008#ifdef PMAP_INCLUDE_PTE_SYNC 1009 /* 1010 * Note: With a write-back cache, we may need to sync this 1011 * L2 table before re-using it. 1012 * This is because it may have belonged to a non-current 1013 * pmap, in which case the cache syncs would have been 1014 * skipped when the pages were being unmapped. If the 1015 * L2 table were then to be immediately re-allocated to 1016 * the *current* pmap, it may well contain stale mappings 1017 * which have not yet been cleared by a cache write-back 1018 * and so would still be visible to the mmu. 1019 */ 1020 if (need_sync) 1021 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1022#endif 1023 uma_zfree(l2zone, l2); 1024} 1025/* 1026 * One or more mappings in the specified L2 descriptor table have just been 1027 * invalidated. 1028 * 1029 * Garbage collect the metadata and descriptor table itself if necessary. 1030 * 1031 * The pmap lock must be acquired when this is called (not necessary 1032 * for the kernel pmap). 1033 */ 1034static void 1035pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1036{ 1037 struct l2_dtable *l2; 1038 pd_entry_t *pl1pd, l1pd; 1039 pt_entry_t *ptep; 1040 u_short l1idx; 1041 1042 1043 /* 1044 * Update the bucket's reference count according to how many 1045 * PTEs the caller has just invalidated. 1046 */ 1047 l2b->l2b_occupancy -= count; 1048 1049 /* 1050 * Note: 1051 * 1052 * Level 2 page tables allocated to the kernel pmap are never freed 1053 * as that would require checking all Level 1 page tables and 1054 * removing any references to the Level 2 page table. See also the 1055 * comment elsewhere about never freeing bootstrap L2 descriptors. 1056 * 1057 * We make do with just invalidating the mapping in the L2 table. 1058 * 1059 * This isn't really a big deal in practice and, in fact, leads 1060 * to a performance win over time as we don't need to continually 1061 * alloc/free. 1062 */ 1063 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1064 return; 1065 1066 /* 1067 * There are no more valid mappings in this level 2 page table. 1068 * Go ahead and NULL-out the pointer in the bucket, then 1069 * free the page table. 1070 */ 1071 l1idx = l2b->l2b_l1idx; 1072 ptep = l2b->l2b_kva; 1073 l2b->l2b_kva = NULL; 1074 1075 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1076 1077 /* 1078 * If the L1 slot matches the pmap's domain 1079 * number, then invalidate it. 1080 */ 1081 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 1082 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 1083 *pl1pd = 0; 1084 PTE_SYNC(pl1pd); 1085 } 1086 1087 /* 1088 * Release the L2 descriptor table back to the pool cache. 1089 */ 1090#ifndef PMAP_INCLUDE_PTE_SYNC 1091 pmap_free_l2_ptp(ptep); 1092#else 1093 pmap_free_l2_ptp(!pmap_is_current(pm), ptep); 1094#endif 1095 1096 /* 1097 * Update the reference count in the associated l2_dtable 1098 */ 1099 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1100 if (--l2->l2_occupancy > 0) 1101 return; 1102 1103 /* 1104 * There are no more valid mappings in any of the Level 1 1105 * slots managed by this l2_dtable. Go ahead and NULL-out 1106 * the pointer in the parent pmap and free the l2_dtable. 1107 */ 1108 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1109 pmap_free_l2_dtable(l2); 1110} 1111 1112/* 1113 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1114 * structures. 1115 */ 1116static int 1117pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) 1118{ 1119#ifndef PMAP_INCLUDE_PTE_SYNC 1120 struct l2_bucket *l2b; 1121 pt_entry_t *ptep, pte; 1122#ifdef ARM_USE_SMALL_ALLOC 1123 pd_entry_t *pde; 1124#endif 1125 vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; 1126 1127 /* 1128 * The mappings for these page tables were initially made using 1129 * pmap_kenter() by the pool subsystem. Therefore, the cache- 1130 * mode will not be right for page table mappings. To avoid 1131 * polluting the pmap_kenter() code with a special case for 1132 * page tables, we simply fix up the cache-mode here if it's not 1133 * correct. 1134 */ 1135#ifdef ARM_USE_SMALL_ALLOC 1136 pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)]; 1137 if (!l1pte_section_p(*pde)) { 1138#endif 1139 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1140 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1141 pte = *ptep; 1142 1143 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1144 /* 1145 * Page tables must have the cache-mode set to 1146 * Write-Thru. 1147 */ 1148 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1149 PTE_SYNC(ptep); 1150 cpu_tlb_flushD_SE(va); 1151 cpu_cpwait(); 1152 } 1153#ifdef ARM_USE_SMALL_ALLOC 1154 } 1155#endif 1156#endif 1157 memset(mem, 0, L2_TABLE_SIZE_REAL); 1158 PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1159 return (0); 1160} 1161 1162/* 1163 * A bunch of routines to conditionally flush the caches/TLB depending 1164 * on whether the specified pmap actually needs to be flushed at any 1165 * given time. 1166 */ 1167static PMAP_INLINE void 1168pmap_tlb_flushID_SE(pmap_t pm, vm_offset_t va) 1169{ 1170 1171 if (pmap_is_current(pm)) 1172 cpu_tlb_flushID_SE(va); 1173} 1174 1175static PMAP_INLINE void 1176pmap_tlb_flushD_SE(pmap_t pm, vm_offset_t va) 1177{ 1178 1179 if (pmap_is_current(pm)) 1180 cpu_tlb_flushD_SE(va); 1181} 1182 1183static PMAP_INLINE void 1184pmap_tlb_flushID(pmap_t pm) 1185{ 1186 1187 if (pmap_is_current(pm)) 1188 cpu_tlb_flushID(); 1189} 1190static PMAP_INLINE void 1191pmap_tlb_flushD(pmap_t pm) 1192{ 1193 1194 if (pmap_is_current(pm)) 1195 cpu_tlb_flushD(); 1196} 1197 1198static PMAP_INLINE void 1199pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) 1200{ 1201 1202 if (pmap_is_current(pm)) 1203 cpu_idcache_wbinv_range(va, len); 1204} 1205 1206static PMAP_INLINE void 1207pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, 1208 boolean_t do_inv, boolean_t rd_only) 1209{ 1210 1211 if (pmap_is_current(pm)) { 1212 if (do_inv) { 1213 if (rd_only) 1214 cpu_dcache_inv_range(va, len); 1215 else 1216 cpu_dcache_wbinv_range(va, len); 1217 } else 1218 if (!rd_only) 1219 cpu_dcache_wb_range(va, len); 1220 } 1221} 1222 1223static PMAP_INLINE void 1224pmap_idcache_wbinv_all(pmap_t pm) 1225{ 1226 1227 if (pmap_is_current(pm)) 1228 cpu_idcache_wbinv_all(); 1229} 1230 1231static PMAP_INLINE void 1232pmap_dcache_wbinv_all(pmap_t pm) 1233{ 1234 1235 if (pmap_is_current(pm)) 1236 cpu_dcache_wbinv_all(); 1237} 1238 1239/* 1240 * PTE_SYNC_CURRENT: 1241 * 1242 * Make sure the pte is written out to RAM. 1243 * We need to do this for one of two cases: 1244 * - We're dealing with the kernel pmap 1245 * - There is no pmap active in the cache/tlb. 1246 * - The specified pmap is 'active' in the cache/tlb. 1247 */ 1248#ifdef PMAP_INCLUDE_PTE_SYNC 1249#define PTE_SYNC_CURRENT(pm, ptep) \ 1250do { \ 1251 if (PMAP_NEEDS_PTE_SYNC && \ 1252 pmap_is_current(pm)) \ 1253 PTE_SYNC(ptep); \ 1254} while (/*CONSTCOND*/0) 1255#else 1256#define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ 1257#endif 1258 1259/* 1260 * Since we have a virtually indexed cache, we may need to inhibit caching if 1261 * there is more than one mapping and at least one of them is writable. 1262 * Since we purge the cache on every context switch, we only need to check for 1263 * other mappings within the same pmap, or kernel_pmap. 1264 * This function is also called when a page is unmapped, to possibly reenable 1265 * caching on any remaining mappings. 1266 * 1267 * The code implements the following logic, where: 1268 * 1269 * KW = # of kernel read/write pages 1270 * KR = # of kernel read only pages 1271 * UW = # of user read/write pages 1272 * UR = # of user read only pages 1273 * 1274 * KC = kernel mapping is cacheable 1275 * UC = user mapping is cacheable 1276 * 1277 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1278 * +--------------------------------------------- 1279 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1280 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1281 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1282 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1283 */ 1284 1285static const int pmap_vac_flags[4][4] = { 1286 {-1, 0, 0, PVF_KNC}, 1287 {0, 0, PVF_NC, PVF_NC}, 1288 {0, PVF_NC, PVF_NC, PVF_NC}, 1289 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1290}; 1291 1292static PMAP_INLINE int 1293pmap_get_vac_flags(const struct vm_page *pg) 1294{ 1295 int kidx, uidx; 1296 1297 kidx = 0; 1298 if (pg->md.kro_mappings || pg->md.krw_mappings > 1) 1299 kidx |= 1; 1300 if (pg->md.krw_mappings) 1301 kidx |= 2; 1302 1303 uidx = 0; 1304 if (pg->md.uro_mappings || pg->md.urw_mappings > 1) 1305 uidx |= 1; 1306 if (pg->md.urw_mappings) 1307 uidx |= 2; 1308 1309 return (pmap_vac_flags[uidx][kidx]); 1310} 1311 1312static __inline void 1313pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1314{ 1315 int nattr; 1316 1317 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1318 nattr = pmap_get_vac_flags(pg); 1319 1320 if (nattr < 0) { 1321 pg->md.pvh_attrs &= ~PVF_NC; 1322 return; 1323 } 1324 1325 if (nattr == 0 && (pg->md.pvh_attrs & PVF_NC) == 0) { 1326 return; 1327 } 1328 1329 if (pm == pmap_kernel()) 1330 pmap_vac_me_kpmap(pg, pm, va); 1331 else 1332 pmap_vac_me_user(pg, pm, va); 1333 1334 pg->md.pvh_attrs = (pg->md.pvh_attrs & ~PVF_NC) | nattr; 1335} 1336 1337static void 1338pmap_vac_me_kpmap(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1339{ 1340 u_int u_cacheable, u_entries; 1341 struct pv_entry *pv; 1342 pmap_t last_pmap = pm; 1343 1344 /* 1345 * Pass one, see if there are both kernel and user pmaps for 1346 * this page. Calculate whether there are user-writable or 1347 * kernel-writable pages. 1348 */ 1349 u_cacheable = 0; 1350 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1351 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1352 u_cacheable++; 1353 } 1354 1355 u_entries = pg->md.urw_mappings + pg->md.uro_mappings; 1356 1357 /* 1358 * We know we have just been updating a kernel entry, so if 1359 * all user pages are already cacheable, then there is nothing 1360 * further to do. 1361 */ 1362 if (pg->md.k_mappings == 0 && u_cacheable == u_entries) 1363 return; 1364 1365 if (u_entries) { 1366 /* 1367 * Scan over the list again, for each entry, if it 1368 * might not be set correctly, call pmap_vac_me_user 1369 * to recalculate the settings. 1370 */ 1371 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1372 /* 1373 * We know kernel mappings will get set 1374 * correctly in other calls. We also know 1375 * that if the pmap is the same as last_pmap 1376 * then we've just handled this entry. 1377 */ 1378 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1379 continue; 1380 1381 /* 1382 * If there are kernel entries and this page 1383 * is writable but non-cacheable, then we can 1384 * skip this entry also. 1385 */ 1386 if (pg->md.k_mappings && 1387 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1388 (PVF_NC | PVF_WRITE)) 1389 continue; 1390 1391 /* 1392 * Similarly if there are no kernel-writable 1393 * entries and the page is already 1394 * read-only/cacheable. 1395 */ 1396 if (pg->md.krw_mappings == 0 && 1397 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1398 continue; 1399 1400 /* 1401 * For some of the remaining cases, we know 1402 * that we must recalculate, but for others we 1403 * can't tell if they are correct or not, so 1404 * we recalculate anyway. 1405 */ 1406 pmap_vac_me_user(pg, (last_pmap = pv->pv_pmap), 0); 1407 } 1408 1409 if (pg->md.k_mappings == 0) 1410 return; 1411 } 1412 1413 pmap_vac_me_user(pg, pm, va); 1414} 1415 1416static void 1417pmap_vac_me_user(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1418{ 1419 pmap_t kpmap = pmap_kernel(); 1420 struct pv_entry *pv, *npv; 1421 struct l2_bucket *l2b; 1422 pt_entry_t *ptep, pte; 1423 u_int entries = 0; 1424 u_int writable = 0; 1425 u_int cacheable_entries = 0; 1426 u_int kern_cacheable = 0; 1427 u_int other_writable = 0; 1428 1429 /* 1430 * Count mappings and writable mappings in this pmap. 1431 * Include kernel mappings as part of our own. 1432 * Keep a pointer to the first one. 1433 */ 1434 npv = TAILQ_FIRST(&pg->md.pv_list); 1435 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1436 /* Count mappings in the same pmap */ 1437 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1438 if (entries++ == 0) 1439 npv = pv; 1440 1441 /* Cacheable mappings */ 1442 if ((pv->pv_flags & PVF_NC) == 0) { 1443 cacheable_entries++; 1444 if (kpmap == pv->pv_pmap) 1445 kern_cacheable++; 1446 } 1447 1448 /* Writable mappings */ 1449 if (pv->pv_flags & PVF_WRITE) 1450 ++writable; 1451 } else 1452 if (pv->pv_flags & PVF_WRITE) 1453 other_writable = 1; 1454 } 1455 1456 /* 1457 * Enable or disable caching as necessary. 1458 * Note: the first entry might be part of the kernel pmap, 1459 * so we can't assume this is indicative of the state of the 1460 * other (maybe non-kpmap) entries. 1461 */ 1462 if ((entries > 1 && writable) || 1463 (entries > 0 && pm == kpmap && other_writable)) { 1464 if (cacheable_entries == 0) 1465 return; 1466 1467 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1468 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1469 (pv->pv_flags & PVF_NC)) 1470 continue; 1471 1472 pv->pv_flags |= PVF_NC; 1473 1474 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1475 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1476 pte = *ptep & ~L2_S_CACHE_MASK; 1477 1478 if ((va != pv->pv_va || pm != pv->pv_pmap) && 1479 l2pte_valid(pte)) { 1480 if (PV_BEEN_EXECD(pv->pv_flags)) { 1481 pmap_idcache_wbinv_range(pv->pv_pmap, 1482 pv->pv_va, PAGE_SIZE); 1483 pmap_tlb_flushID_SE(pv->pv_pmap, 1484 pv->pv_va); 1485 } else 1486 if (PV_BEEN_REFD(pv->pv_flags)) { 1487 pmap_dcache_wb_range(pv->pv_pmap, 1488 pv->pv_va, PAGE_SIZE, TRUE, 1489 (pv->pv_flags & PVF_WRITE) == 0); 1490 pmap_tlb_flushD_SE(pv->pv_pmap, 1491 pv->pv_va); 1492 } 1493 } 1494 1495 *ptep = pte; 1496 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1497 } 1498 cpu_cpwait(); 1499 } else 1500 if (entries > cacheable_entries) { 1501 /* 1502 * Turn cacheing back on for some pages. If it is a kernel 1503 * page, only do so if there are no other writable pages. 1504 */ 1505 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1506 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1507 (kpmap != pv->pv_pmap || other_writable))) 1508 continue; 1509 1510 pv->pv_flags &= ~PVF_NC; 1511 1512 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1513 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1514 pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; 1515 1516 if (l2pte_valid(pte)) { 1517 if (PV_BEEN_EXECD(pv->pv_flags)) { 1518 pmap_tlb_flushID_SE(pv->pv_pmap, 1519 pv->pv_va); 1520 } else 1521 if (PV_BEEN_REFD(pv->pv_flags)) { 1522 pmap_tlb_flushD_SE(pv->pv_pmap, 1523 pv->pv_va); 1524 } 1525 } 1526 1527 *ptep = pte; 1528 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1529 } 1530 } 1531} 1532 1533/* 1534 * Modify pte bits for all ptes corresponding to the given physical address. 1535 * We use `maskbits' rather than `clearbits' because we're always passing 1536 * constants and the latter would require an extra inversion at run-time. 1537 */ 1538static int 1539pmap_clearbit(struct vm_page *pg, u_int maskbits) 1540{ 1541 struct l2_bucket *l2b; 1542 struct pv_entry *pv; 1543 pt_entry_t *ptep, npte, opte; 1544 pmap_t pm; 1545 vm_offset_t va; 1546 u_int oflags; 1547 int count = 0; 1548 1549 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1550 1551 /* 1552 * Clear saved attributes (modify, reference) 1553 */ 1554 pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 1555 1556 if (TAILQ_EMPTY(&pg->md.pv_list)) { 1557 return (0); 1558 } 1559 1560 /* 1561 * Loop over all current mappings setting/clearing as appropos 1562 */ 1563 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1564 va = pv->pv_va; 1565 pm = pv->pv_pmap; 1566 oflags = pv->pv_flags; 1567 pv->pv_flags &= ~maskbits; 1568 1569 PMAP_LOCK(pm); 1570 1571 l2b = pmap_get_l2_bucket(pm, va); 1572 1573 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1574 npte = opte = *ptep; 1575 1576 if (maskbits & (PVF_WRITE|PVF_MOD)) { 1577 if ((pv->pv_flags & PVF_NC)) { 1578 /* 1579 * Entry is not cacheable: 1580 * 1581 * Don't turn caching on again if this is a 1582 * modified emulation. This would be 1583 * inconsitent with the settings created by 1584 * pmap_vac_me_harder(). Otherwise, it's safe 1585 * to re-enable cacheing. 1586 * 1587 * There's no need to call pmap_vac_me_harder() 1588 * here: all pages are losing their write 1589 * permission. 1590 */ 1591 if (maskbits & PVF_WRITE) { 1592 npte |= pte_l2_s_cache_mode; 1593 pv->pv_flags &= ~PVF_NC; 1594 } 1595 } else 1596 if (opte & L2_S_PROT_W) { 1597 vm_page_dirty(pg); 1598 /* 1599 * Entry is writable/cacheable: check if pmap 1600 * is current if it is flush it, otherwise it 1601 * won't be in the cache 1602 */ 1603 if (PV_BEEN_EXECD(oflags)) 1604 pmap_idcache_wbinv_range(pm, pv->pv_va, 1605 PAGE_SIZE); 1606 else 1607 if (PV_BEEN_REFD(oflags)) 1608 pmap_dcache_wb_range(pm, pv->pv_va, 1609 PAGE_SIZE, 1610 (maskbits & PVF_REF) ? TRUE : FALSE, 1611 FALSE); 1612 } 1613 1614 /* make the pte read only */ 1615 npte &= ~L2_S_PROT_W; 1616 1617 if (maskbits & PVF_WRITE) { 1618 /* 1619 * Keep alias accounting up to date 1620 */ 1621 if (pv->pv_pmap == pmap_kernel()) { 1622 if (oflags & PVF_WRITE) { 1623 pg->md.krw_mappings--; 1624 pg->md.kro_mappings++; 1625 } 1626 } else 1627 if (oflags & PVF_WRITE) { 1628 pg->md.urw_mappings--; 1629 pg->md.uro_mappings++; 1630 } 1631 } 1632 } 1633 1634 if (maskbits & PVF_REF) { 1635 if ((pv->pv_flags & PVF_NC) == 0 && 1636 (maskbits & (PVF_WRITE|PVF_MOD)) == 0) { 1637 /* 1638 * Check npte here; we may have already 1639 * done the wbinv above, and the validity 1640 * of the PTE is the same for opte and 1641 * npte. 1642 */ 1643 if (npte & L2_S_PROT_W) { 1644 if (PV_BEEN_EXECD(oflags)) 1645 pmap_idcache_wbinv_range(pm, 1646 pv->pv_va, PAGE_SIZE); 1647 else 1648 if (PV_BEEN_REFD(oflags)) 1649 pmap_dcache_wb_range(pm, 1650 pv->pv_va, PAGE_SIZE, 1651 TRUE, FALSE); 1652 } else 1653 if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) { 1654 /* XXXJRT need idcache_inv_range */ 1655 if (PV_BEEN_EXECD(oflags)) 1656 pmap_idcache_wbinv_range(pm, 1657 pv->pv_va, PAGE_SIZE); 1658 else 1659 if (PV_BEEN_REFD(oflags)) 1660 pmap_dcache_wb_range(pm, 1661 pv->pv_va, PAGE_SIZE, 1662 TRUE, TRUE); 1663 } 1664 } 1665 1666 /* 1667 * Make the PTE invalid so that we will take a 1668 * page fault the next time the mapping is 1669 * referenced. 1670 */ 1671 npte &= ~L2_TYPE_MASK; 1672 npte |= L2_TYPE_INV; 1673 } 1674 1675 if (npte != opte) { 1676 count++; 1677 *ptep = npte; 1678 PTE_SYNC(ptep); 1679 /* Flush the TLB entry if a current pmap. */ 1680 if (PV_BEEN_EXECD(oflags)) 1681 pmap_tlb_flushID_SE(pm, pv->pv_va); 1682 else 1683 if (PV_BEEN_REFD(oflags)) 1684 pmap_tlb_flushD_SE(pm, pv->pv_va); 1685 } 1686 1687 PMAP_UNLOCK(pm); 1688 1689 } 1690 1691 if (maskbits & PVF_WRITE) 1692 vm_page_flag_clear(pg, PG_WRITEABLE); 1693 return (count); 1694} 1695 1696/* 1697 * main pv_entry manipulation functions: 1698 * pmap_enter_pv: enter a mapping onto a vm_page list 1699 * pmap_remove_pv: remove a mappiing from a vm_page list 1700 * 1701 * NOTE: pmap_enter_pv expects to lock the pvh itself 1702 * pmap_remove_pv expects te caller to lock the pvh before calling 1703 */ 1704 1705/* 1706 * pmap_enter_pv: enter a mapping onto a vm_page lst 1707 * 1708 * => caller should hold the proper lock on pmap_main_lock 1709 * => caller should have pmap locked 1710 * => we will gain the lock on the vm_page and allocate the new pv_entry 1711 * => caller should adjust ptp's wire_count before calling 1712 * => caller should not adjust pmap's wire_count 1713 */ 1714static void 1715pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 1716 vm_offset_t va, u_int flags) 1717{ 1718 1719 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1720 PMAP_ASSERT_LOCKED(pm); 1721 pve->pv_pmap = pm; 1722 pve->pv_va = va; 1723 pve->pv_flags = flags; 1724 1725 TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); 1726 TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); 1727 pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 1728 if (pm == pmap_kernel()) { 1729 if (flags & PVF_WRITE) 1730 pg->md.krw_mappings++; 1731 else 1732 pg->md.kro_mappings++; 1733 } 1734 if (flags & PVF_WRITE) 1735 pg->md.urw_mappings++; 1736 else 1737 pg->md.uro_mappings++; 1738 pg->md.pv_list_count++; 1739 if (pve->pv_flags & PVF_WIRED) 1740 ++pm->pm_stats.wired_count; 1741 vm_page_flag_set(pg, PG_REFERENCED); 1742} 1743 1744/* 1745 * 1746 * pmap_find_pv: Find a pv entry 1747 * 1748 * => caller should hold lock on vm_page 1749 */ 1750static PMAP_INLINE struct pv_entry * 1751pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1752{ 1753 struct pv_entry *pv; 1754 1755 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1756 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) 1757 if (pm == pv->pv_pmap && va == pv->pv_va) 1758 break; 1759 return (pv); 1760} 1761 1762/* 1763 * vector_page_setprot: 1764 * 1765 * Manipulate the protection of the vector page. 1766 */ 1767void 1768vector_page_setprot(int prot) 1769{ 1770 struct l2_bucket *l2b; 1771 pt_entry_t *ptep; 1772 1773 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 1774 1775 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 1776 1777 *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 1778 PTE_SYNC(ptep); 1779 cpu_tlb_flushD_SE(vector_page); 1780 cpu_cpwait(); 1781} 1782 1783/* 1784 * pmap_remove_pv: try to remove a mapping from a pv_list 1785 * 1786 * => caller should hold proper lock on pmap_main_lock 1787 * => pmap should be locked 1788 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1789 * => caller should adjust ptp's wire_count and free PTP if needed 1790 * => caller should NOT adjust pmap's wire_count 1791 * => we return the removed pve 1792 */ 1793 1794static void 1795pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) 1796{ 1797 1798 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1799 PMAP_ASSERT_LOCKED(pm); 1800 TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); 1801 TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); 1802 if (pve->pv_flags & PVF_WIRED) 1803 --pm->pm_stats.wired_count; 1804 pg->md.pv_list_count--; 1805 if (pg->md.pvh_attrs & PVF_MOD) 1806 vm_page_dirty(pg); 1807 if (pm == pmap_kernel()) { 1808 if (pve->pv_flags & PVF_WRITE) 1809 pg->md.krw_mappings--; 1810 else 1811 pg->md.kro_mappings--; 1812 } else 1813 if (pve->pv_flags & PVF_WRITE) 1814 pg->md.urw_mappings--; 1815 else 1816 pg->md.uro_mappings--; 1817 if (TAILQ_FIRST(&pg->md.pv_list) == NULL || 1818 (pg->md.krw_mappings == 0 && pg->md.urw_mappings == 0)) { 1819 pg->md.pvh_attrs &= ~PVF_MOD; 1820 if (TAILQ_FIRST(&pg->md.pv_list) == NULL) 1821 pg->md.pvh_attrs &= ~PVF_REF; 1822 vm_page_flag_clear(pg, PG_WRITEABLE); 1823 } 1824 if (TAILQ_FIRST(&pg->md.pv_list)) 1825 vm_page_flag_set(pg, PG_REFERENCED); 1826 if (pve->pv_flags & PVF_WRITE) 1827 pmap_vac_me_harder(pg, pm, 0); 1828} 1829 1830static struct pv_entry * 1831pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1832{ 1833 struct pv_entry *pve; 1834 1835 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1836 pve = TAILQ_FIRST(&pg->md.pv_list); 1837 1838 while (pve) { 1839 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 1840 pmap_nuke_pv(pg, pm, pve); 1841 break; 1842 } 1843 pve = TAILQ_NEXT(pve, pv_list); 1844 } 1845 1846 return(pve); /* return removed pve */ 1847} 1848/* 1849 * 1850 * pmap_modify_pv: Update pv flags 1851 * 1852 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1853 * => caller should NOT adjust pmap's wire_count 1854 * => caller must call pmap_vac_me_harder() if writable status of a page 1855 * may have changed. 1856 * => we return the old flags 1857 * 1858 * Modify a physical-virtual mapping in the pv table 1859 */ 1860static u_int 1861pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, 1862 u_int clr_mask, u_int set_mask) 1863{ 1864 struct pv_entry *npv; 1865 u_int flags, oflags; 1866 1867 PMAP_ASSERT_LOCKED(pm); 1868 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1869 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1870 return (0); 1871 1872 /* 1873 * There is at least one VA mapping this page. 1874 */ 1875 1876 if (clr_mask & (PVF_REF | PVF_MOD)) 1877 pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1878 1879 oflags = npv->pv_flags; 1880 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1881 1882 if ((flags ^ oflags) & PVF_WIRED) { 1883 if (flags & PVF_WIRED) 1884 ++pm->pm_stats.wired_count; 1885 else 1886 --pm->pm_stats.wired_count; 1887 } 1888 1889 if ((flags ^ oflags) & PVF_WRITE) { 1890 if (pm == pmap_kernel()) { 1891 if (flags & PVF_WRITE) { 1892 pg->md.krw_mappings++; 1893 pg->md.kro_mappings--; 1894 } else { 1895 pg->md.kro_mappings++; 1896 pg->md.krw_mappings--; 1897 } 1898 } else 1899 if (flags & PVF_WRITE) { 1900 pg->md.urw_mappings++; 1901 pg->md.uro_mappings--; 1902 } else { 1903 pg->md.uro_mappings++; 1904 pg->md.urw_mappings--; 1905 } 1906 if (pg->md.krw_mappings == 0 && pg->md.urw_mappings == 0) { 1907 pg->md.pvh_attrs &= ~PVF_MOD; 1908 vm_page_flag_clear(pg, PG_WRITEABLE); 1909 } 1910 pmap_vac_me_harder(pg, pm, 0); 1911 } 1912 1913 return (oflags); 1914} 1915 1916/* Function to set the debug level of the pmap code */ 1917#ifdef PMAP_DEBUG 1918void 1919pmap_debug(int level) 1920{ 1921 pmap_debug_level = level; 1922 dprintf("pmap_debug: level=%d\n", pmap_debug_level); 1923} 1924#endif /* PMAP_DEBUG */ 1925 1926void 1927pmap_pinit0(struct pmap *pmap) 1928{ 1929 PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); 1930 1931 dprintf("pmap_pinit0: pmap = %08x, pm_pdir = %08x\n", 1932 (u_int32_t) pmap, (u_int32_t) pmap->pm_pdir); 1933 bcopy(kernel_pmap, pmap, sizeof(*pmap)); 1934 bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); 1935 PMAP_LOCK_INIT(pmap); 1936} 1937 1938/* 1939 * Initialize a vm_page's machine-dependent fields. 1940 */ 1941void 1942pmap_page_init(vm_page_t m) 1943{ 1944 1945 TAILQ_INIT(&m->md.pv_list); 1946 m->md.pv_list_count = 0; 1947} 1948 1949/* 1950 * Initialize the pmap module. 1951 * Called by vm_init, to initialize any structures that the pmap 1952 * system needs to map virtual memory. 1953 */ 1954void 1955pmap_init(void) 1956{ 1957 int shpgperproc = PMAP_SHPGPERPROC; 1958 1959 PDEBUG(1, printf("pmap_init: phys_start = %08x\n")); 1960 1961 /* 1962 * init the pv free list 1963 */ 1964 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 1965 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1966 /* 1967 * Now it is safe to enable pv_table recording. 1968 */ 1969 PDEBUG(1, printf("pmap_init: done!\n")); 1970 1971 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 1972 1973 pv_entry_max = shpgperproc * maxproc + VMCNT_GET(page_count); 1974 pv_entry_high_water = 9 * (pv_entry_max / 10); 1975 l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, 1976 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1977 l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), 1978 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1979 UMA_ZONE_VM | UMA_ZONE_NOFREE); 1980 1981 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 1982 1983} 1984 1985int 1986pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) 1987{ 1988 struct l2_dtable *l2; 1989 struct l2_bucket *l2b; 1990 pd_entry_t *pl1pd, l1pd; 1991 pt_entry_t *ptep, pte; 1992 vm_paddr_t pa; 1993 u_int l1idx; 1994 int rv = 0; 1995 1996 l1idx = L1_IDX(va); 1997 vm_page_lock_queues(); 1998 PMAP_LOCK(pm); 1999 2000 /* 2001 * If there is no l2_dtable for this address, then the process 2002 * has no business accessing it. 2003 * 2004 * Note: This will catch userland processes trying to access 2005 * kernel addresses. 2006 */ 2007 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2008 if (l2 == NULL) 2009 goto out; 2010 2011 /* 2012 * Likewise if there is no L2 descriptor table 2013 */ 2014 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2015 if (l2b->l2b_kva == NULL) 2016 goto out; 2017 2018 /* 2019 * Check the PTE itself. 2020 */ 2021 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2022 pte = *ptep; 2023 if (pte == 0) 2024 goto out; 2025 2026 /* 2027 * Catch a userland access to the vector page mapped at 0x0 2028 */ 2029 if (user && (pte & L2_S_PROT_U) == 0) 2030 goto out; 2031 if (va == vector_page) 2032 goto out; 2033 2034 pa = l2pte_pa(pte); 2035 2036 if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { 2037 /* 2038 * This looks like a good candidate for "page modified" 2039 * emulation... 2040 */ 2041 struct pv_entry *pv; 2042 struct vm_page *pg; 2043 2044 /* Extract the physical address of the page */ 2045 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 2046 goto out; 2047 } 2048 /* Get the current flags for this page. */ 2049 2050 pv = pmap_find_pv(pg, pm, va); 2051 if (pv == NULL) { 2052 goto out; 2053 } 2054 2055 /* 2056 * Do the flags say this page is writable? If not then it 2057 * is a genuine write fault. If yes then the write fault is 2058 * our fault as we did not reflect the write access in the 2059 * PTE. Now we know a write has occurred we can correct this 2060 * and also set the modified bit 2061 */ 2062 if ((pv->pv_flags & PVF_WRITE) == 0) { 2063 goto out; 2064 } 2065 2066 pg->md.pvh_attrs |= PVF_REF | PVF_MOD; 2067 vm_page_dirty(pg); 2068 pv->pv_flags |= PVF_REF | PVF_MOD; 2069 2070 /* 2071 * Re-enable write permissions for the page. No need to call 2072 * pmap_vac_me_harder(), since this is just a 2073 * modified-emulation fault, and the PVF_WRITE bit isn't 2074 * changing. We've already set the cacheable bits based on 2075 * the assumption that we can write to this page. 2076 */ 2077 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 2078 PTE_SYNC(ptep); 2079 rv = 1; 2080 } else 2081 if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 2082 /* 2083 * This looks like a good candidate for "page referenced" 2084 * emulation. 2085 */ 2086 struct pv_entry *pv; 2087 struct vm_page *pg; 2088 2089 /* Extract the physical address of the page */ 2090 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2091 goto out; 2092 /* Get the current flags for this page. */ 2093 2094 pv = pmap_find_pv(pg, pm, va); 2095 if (pv == NULL) 2096 goto out; 2097 2098 pg->md.pvh_attrs |= PVF_REF; 2099 pv->pv_flags |= PVF_REF; 2100 2101 2102 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 2103 PTE_SYNC(ptep); 2104 rv = 1; 2105 } 2106 2107 /* 2108 * We know there is a valid mapping here, so simply 2109 * fix up the L1 if necessary. 2110 */ 2111 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2112 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 2113 if (*pl1pd != l1pd) { 2114 *pl1pd = l1pd; 2115 PTE_SYNC(pl1pd); 2116 rv = 1; 2117 } 2118 2119#ifdef CPU_SA110 2120 /* 2121 * There are bugs in the rev K SA110. This is a check for one 2122 * of them. 2123 */ 2124 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 2125 curcpu()->ci_arm_cpurev < 3) { 2126 /* Always current pmap */ 2127 if (l2pte_valid(pte)) { 2128 extern int kernel_debug; 2129 if (kernel_debug & 1) { 2130 struct proc *p = curlwp->l_proc; 2131 printf("prefetch_abort: page is already " 2132 "mapped - pte=%p *pte=%08x\n", ptep, pte); 2133 printf("prefetch_abort: pc=%08lx proc=%p " 2134 "process=%s\n", va, p, p->p_comm); 2135 printf("prefetch_abort: far=%08x fs=%x\n", 2136 cpu_faultaddress(), cpu_faultstatus()); 2137 } 2138#ifdef DDB 2139 if (kernel_debug & 2) 2140 Debugger(); 2141#endif 2142 rv = 1; 2143 } 2144 } 2145#endif /* CPU_SA110 */ 2146 2147#ifdef DEBUG 2148 /* 2149 * If 'rv == 0' at this point, it generally indicates that there is a 2150 * stale TLB entry for the faulting address. This happens when two or 2151 * more processes are sharing an L1. Since we don't flush the TLB on 2152 * a context switch between such processes, we can take domain faults 2153 * for mappings which exist at the same VA in both processes. EVEN IF 2154 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 2155 * example. 2156 * 2157 * This is extremely likely to happen if pmap_enter() updated the L1 2158 * entry for a recently entered mapping. In this case, the TLB is 2159 * flushed for the new mapping, but there may still be TLB entries for 2160 * other mappings belonging to other processes in the 1MB range 2161 * covered by the L1 entry. 2162 * 2163 * Since 'rv == 0', we know that the L1 already contains the correct 2164 * value, so the fault must be due to a stale TLB entry. 2165 * 2166 * Since we always need to flush the TLB anyway in the case where we 2167 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 2168 * stale TLB entries dynamically. 2169 * 2170 * However, the above condition can ONLY happen if the current L1 is 2171 * being shared. If it happens when the L1 is unshared, it indicates 2172 * that other parts of the pmap are not doing their job WRT managing 2173 * the TLB. 2174 */ 2175 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 2176 extern int last_fault_code; 2177 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 2178 pm, va, ftype); 2179 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 2180 l2, l2b, ptep, pl1pd); 2181 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 2182 pte, l1pd, last_fault_code); 2183#ifdef DDB 2184 Debugger(); 2185#endif 2186 } 2187#endif 2188 2189 cpu_tlb_flushID_SE(va); 2190 cpu_cpwait(); 2191 2192 rv = 1; 2193 2194out: 2195 vm_page_unlock_queues(); 2196 PMAP_UNLOCK(pm); 2197 return (rv); 2198} 2199 2200void 2201pmap_postinit(void) 2202{ 2203 struct l2_bucket *l2b; 2204 struct l1_ttable *l1; 2205 pd_entry_t *pl1pt; 2206 pt_entry_t *ptep, pte; 2207 vm_offset_t va, eva; 2208 u_int loop, needed; 2209 2210 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 2211 needed -= 1; 2212 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 2213 2214 for (loop = 0; loop < needed; loop++, l1++) { 2215 /* Allocate a L1 page table */ 2216 va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 2217 0xffffffff, L1_TABLE_SIZE, 0); 2218 2219 if (va == 0) 2220 panic("Cannot allocate L1 KVM"); 2221 2222 eva = va + L1_TABLE_SIZE; 2223 pl1pt = (pd_entry_t *)va; 2224 2225 while (va < eva) { 2226 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2227 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2228 pte = *ptep; 2229 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 2230 *ptep = pte; 2231 PTE_SYNC(ptep); 2232 cpu_tlb_flushD_SE(va); 2233 2234 va += PAGE_SIZE; 2235 } 2236 pmap_init_l1(l1, pl1pt); 2237 } 2238 2239 2240#ifdef DEBUG 2241 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 2242 needed); 2243#endif 2244} 2245 2246/* 2247 * This is used to stuff certain critical values into the PCB where they 2248 * can be accessed quickly from cpu_switch() et al. 2249 */ 2250void 2251pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 2252{ 2253 struct l2_bucket *l2b; 2254 2255 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 2256 pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 2257 (DOMAIN_CLIENT << (pm->pm_domain * 2)); 2258 2259 if (vector_page < KERNBASE) { 2260 pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 2261 l2b = pmap_get_l2_bucket(pm, vector_page); 2262 pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | 2263 L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); 2264 } else 2265 pcb->pcb_pl1vec = NULL; 2266} 2267 2268void 2269pmap_activate(struct thread *td) 2270{ 2271 pmap_t pm; 2272 struct pcb *pcb; 2273 2274 pm = vmspace_pmap(td->td_proc->p_vmspace); 2275 pcb = td->td_pcb; 2276 2277 critical_enter(); 2278 pmap_set_pcb_pagedir(pm, pcb); 2279 2280 if (td == curthread) { 2281 u_int cur_dacr, cur_ttb; 2282 2283 __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 2284 __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); 2285 2286 cur_ttb &= ~(L1_TABLE_SIZE - 1); 2287 2288 if (cur_ttb == (u_int)pcb->pcb_pagedir && 2289 cur_dacr == pcb->pcb_dacr) { 2290 /* 2291 * No need to switch address spaces. 2292 */ 2293 critical_exit(); 2294 return; 2295 } 2296 2297 2298 /* 2299 * We MUST, I repeat, MUST fix up the L1 entry corresponding 2300 * to 'vector_page' in the incoming L1 table before switching 2301 * to it otherwise subsequent interrupts/exceptions (including 2302 * domain faults!) will jump into hyperspace. 2303 */ 2304 if (pcb->pcb_pl1vec) { 2305 2306 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 2307 /* 2308 * Don't need to PTE_SYNC() at this point since 2309 * cpu_setttb() is about to flush both the cache 2310 * and the TLB. 2311 */ 2312 } 2313 2314 cpu_domains(pcb->pcb_dacr); 2315 cpu_setttb(pcb->pcb_pagedir); 2316 } 2317 critical_exit(); 2318} 2319 2320static int 2321pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) 2322{ 2323 pd_entry_t *pdep, pde; 2324 pt_entry_t *ptep, pte; 2325 vm_offset_t pa; 2326 int rv = 0; 2327 2328 /* 2329 * Make sure the descriptor itself has the correct cache mode 2330 */ 2331 pdep = &kl1[L1_IDX(va)]; 2332 pde = *pdep; 2333 2334 if (l1pte_section_p(pde)) { 2335 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 2336 *pdep = (pde & ~L1_S_CACHE_MASK) | 2337 pte_l1_s_cache_mode_pt; 2338 PTE_SYNC(pdep); 2339 cpu_dcache_wbinv_range((vm_offset_t)pdep, 2340 sizeof(*pdep)); 2341 rv = 1; 2342 } 2343 } else { 2344 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2345 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2346 if (ptep == NULL) 2347 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 2348 2349 ptep = &ptep[l2pte_index(va)]; 2350 pte = *ptep; 2351 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 2352 *ptep = (pte & ~L2_S_CACHE_MASK) | 2353 pte_l2_s_cache_mode_pt; 2354 PTE_SYNC(ptep); 2355 cpu_dcache_wbinv_range((vm_offset_t)ptep, 2356 sizeof(*ptep)); 2357 rv = 1; 2358 } 2359 } 2360 2361 return (rv); 2362} 2363 2364static void 2365pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, 2366 pt_entry_t **ptep) 2367{ 2368 vm_offset_t va = *availp; 2369 struct l2_bucket *l2b; 2370 2371 if (ptep) { 2372 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2373 if (l2b == NULL) 2374 panic("pmap_alloc_specials: no l2b for 0x%x", va); 2375 2376 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 2377 } 2378 2379 *vap = va; 2380 *availp = va + (PAGE_SIZE * pages); 2381} 2382 2383/* 2384 * Bootstrap the system enough to run with virtual memory. 2385 * 2386 * On the arm this is called after mapping has already been enabled 2387 * and just syncs the pmap module with what has already been done. 2388 * [We can't call it easily with mapping off since the kernel is not 2389 * mapped with PA == VA, hence we would have to relocate every address 2390 * from the linked base (virtual) address "KERNBASE" to the actual 2391 * (physical) address starting relative to 0] 2392 */ 2393#define PMAP_STATIC_L2_SIZE 16 2394#ifdef ARM_USE_SMALL_ALLOC 2395extern struct mtx smallalloc_mtx; 2396#endif 2397 2398void 2399pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt) 2400{ 2401 static struct l1_ttable static_l1; 2402 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 2403 struct l1_ttable *l1 = &static_l1; 2404 struct l2_dtable *l2; 2405 struct l2_bucket *l2b; 2406 pd_entry_t pde; 2407 pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; 2408 pt_entry_t *ptep; 2409 vm_paddr_t pa; 2410 vm_offset_t va; 2411 vm_size_t size; 2412 int l1idx, l2idx, l2next = 0; 2413 2414 PDEBUG(1, printf("firstaddr = %08x, loadaddr = %08x\n", 2415 firstaddr, loadaddr)); 2416 2417 virtual_avail = firstaddr; 2418 kernel_pmap = &kernel_pmap_store; 2419 kernel_pmap->pm_l1 = l1; 2420 kernel_l1pa = l1pt->pv_pa; 2421 2422 /* 2423 * Scan the L1 translation table created by initarm() and create 2424 * the required metadata for all valid mappings found in it. 2425 */ 2426 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 2427 pde = kernel_l1pt[l1idx]; 2428 2429 /* 2430 * We're only interested in Coarse mappings. 2431 * pmap_extract() can deal with section mappings without 2432 * recourse to checking L2 metadata. 2433 */ 2434 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 2435 continue; 2436 2437 /* 2438 * Lookup the KVA of this L2 descriptor table 2439 */ 2440 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2441 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2442 2443 if (ptep == NULL) { 2444 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 2445 (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); 2446 } 2447 2448 /* 2449 * Fetch the associated L2 metadata structure. 2450 * Allocate a new one if necessary. 2451 */ 2452 if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { 2453 if (l2next == PMAP_STATIC_L2_SIZE) 2454 panic("pmap_bootstrap: out of static L2s"); 2455 kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = 2456 &static_l2[l2next++]; 2457 } 2458 2459 /* 2460 * One more L1 slot tracked... 2461 */ 2462 l2->l2_occupancy++; 2463 2464 /* 2465 * Fill in the details of the L2 descriptor in the 2466 * appropriate bucket. 2467 */ 2468 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2469 l2b->l2b_kva = ptep; 2470 l2b->l2b_phys = pa; 2471 l2b->l2b_l1idx = l1idx; 2472 2473 /* 2474 * Establish an initial occupancy count for this descriptor 2475 */ 2476 for (l2idx = 0; 2477 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 2478 l2idx++) { 2479 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 2480 l2b->l2b_occupancy++; 2481 } 2482 } 2483 2484 /* 2485 * Make sure the descriptor itself has the correct cache mode. 2486 * If not, fix it, but whine about the problem. Port-meisters 2487 * should consider this a clue to fix up their initarm() 2488 * function. :) 2489 */ 2490 if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { 2491 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2492 "L2 pte @ %p\n", ptep); 2493 } 2494 } 2495 2496 2497 /* 2498 * Ensure the primary (kernel) L1 has the correct cache mode for 2499 * a page table. Bitch if it is not correctly set. 2500 */ 2501 for (va = (vm_offset_t)kernel_l1pt; 2502 va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 2503 if (pmap_set_pt_cache_mode(kernel_l1pt, va)) 2504 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2505 "primary L1 @ 0x%x\n", va); 2506 } 2507 2508 cpu_dcache_wbinv_all(); 2509 cpu_tlb_flushID(); 2510 cpu_cpwait(); 2511 2512 PMAP_LOCK_INIT(kernel_pmap); 2513 kernel_pmap->pm_active = -1; 2514 kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; 2515 TAILQ_INIT(&kernel_pmap->pm_pvlist); 2516 2517 /* 2518 * Reserve some special page table entries/VA space for temporary 2519 * mapping of pages. 2520 */ 2521#define SYSMAP(c, p, v, n) \ 2522 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 2523 2524 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 2525 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); 2526 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 2527 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); 2528 size = ((lastaddr - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 2529 pmap_alloc_specials(&virtual_avail, 2530 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 2531 &pmap_kernel_l2ptp_kva, NULL); 2532 2533 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 2534 pmap_alloc_specials(&virtual_avail, 2535 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 2536 &pmap_kernel_l2dtable_kva, NULL); 2537 2538 pmap_alloc_specials(&virtual_avail, 2539 1, (vm_offset_t*)&_tmppt, NULL); 2540 SLIST_INIT(&l1_list); 2541 TAILQ_INIT(&l1_lru_list); 2542 mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); 2543 pmap_init_l1(l1, kernel_l1pt); 2544 cpu_dcache_wbinv_all(); 2545 2546 virtual_avail = round_page(virtual_avail); 2547 virtual_end = lastaddr; 2548 kernel_vm_end = pmap_curmaxkvaddr; 2549 arm_nocache_startaddr = lastaddr; 2550 mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); 2551 2552#ifdef ARM_USE_SMALL_ALLOC 2553 mtx_init(&smallalloc_mtx, "Small alloc page list", NULL, MTX_DEF); 2554 arm_init_smallalloc(); 2555#endif 2556 pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb); 2557} 2558 2559/*************************************************** 2560 * Pmap allocation/deallocation routines. 2561 ***************************************************/ 2562 2563/* 2564 * Release any resources held by the given physical map. 2565 * Called when a pmap initialized by pmap_pinit is being released. 2566 * Should only be called if the map contains no valid mappings. 2567 */ 2568void 2569pmap_release(pmap_t pmap) 2570{ 2571 struct pcb *pcb; 2572 2573 pmap_idcache_wbinv_all(pmap); 2574 pmap_tlb_flushID(pmap); 2575 cpu_cpwait(); 2576 if (vector_page < KERNBASE) { 2577 struct pcb *curpcb = PCPU_GET(curpcb); 2578 pcb = thread0.td_pcb; 2579 if (pmap_is_current(pmap)) { 2580 /* 2581 * Frob the L1 entry corresponding to the vector 2582 * page so that it contains the kernel pmap's domain 2583 * number. This will ensure pmap_remove() does not 2584 * pull the current vector page out from under us. 2585 */ 2586 critical_enter(); 2587 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 2588 cpu_domains(pcb->pcb_dacr); 2589 cpu_setttb(pcb->pcb_pagedir); 2590 critical_exit(); 2591 } 2592 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 2593 /* 2594 * Make sure cpu_switch(), et al, DTRT. This is safe to do 2595 * since this process has no remaining mappings of its own. 2596 */ 2597 curpcb->pcb_pl1vec = pcb->pcb_pl1vec; 2598 curpcb->pcb_l1vec = pcb->pcb_l1vec; 2599 curpcb->pcb_dacr = pcb->pcb_dacr; 2600 curpcb->pcb_pagedir = pcb->pcb_pagedir; 2601 2602 } 2603 pmap_free_l1(pmap); 2604 PMAP_LOCK_DESTROY(pmap); 2605 2606 dprintf("pmap_release()\n"); 2607} 2608 2609 2610 2611/* 2612 * Helper function for pmap_grow_l2_bucket() 2613 */ 2614static __inline int 2615pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) 2616{ 2617 struct l2_bucket *l2b; 2618 pt_entry_t *ptep; 2619 vm_paddr_t pa; 2620 struct vm_page *pg; 2621 2622 pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 2623 if (pg == NULL) 2624 return (1); 2625 pa = VM_PAGE_TO_PHYS(pg); 2626 2627 if (pap) 2628 *pap = pa; 2629 2630 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2631 2632 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2633 *ptep = L2_S_PROTO | pa | cache_mode | 2634 L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); 2635 PTE_SYNC(ptep); 2636 return (0); 2637} 2638 2639/* 2640 * This is the same as pmap_alloc_l2_bucket(), except that it is only 2641 * used by pmap_growkernel(). 2642 */ 2643static __inline struct l2_bucket * 2644pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) 2645{ 2646 struct l2_dtable *l2; 2647 struct l2_bucket *l2b; 2648 struct l1_ttable *l1; 2649 pd_entry_t *pl1pd; 2650 u_short l1idx; 2651 vm_offset_t nva; 2652 2653 l1idx = L1_IDX(va); 2654 2655 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 2656 /* 2657 * No mapping at this address, as there is 2658 * no entry in the L1 table. 2659 * Need to allocate a new l2_dtable. 2660 */ 2661 nva = pmap_kernel_l2dtable_kva; 2662 if ((nva & PAGE_MASK) == 0) { 2663 /* 2664 * Need to allocate a backing page 2665 */ 2666 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 2667 return (NULL); 2668 } 2669 2670 l2 = (struct l2_dtable *)nva; 2671 nva += sizeof(struct l2_dtable); 2672 2673 if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & 2674 PAGE_MASK)) { 2675 /* 2676 * The new l2_dtable straddles a page boundary. 2677 * Map in another page to cover it. 2678 */ 2679 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 2680 return (NULL); 2681 } 2682 2683 pmap_kernel_l2dtable_kva = nva; 2684 2685 /* 2686 * Link it into the parent pmap 2687 */ 2688 pm->pm_l2[L2_IDX(l1idx)] = l2; 2689 memset(l2, 0, sizeof(*l2)); 2690 } 2691 2692 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2693 2694 /* 2695 * Fetch pointer to the L2 page table associated with the address. 2696 */ 2697 if (l2b->l2b_kva == NULL) { 2698 pt_entry_t *ptep; 2699 2700 /* 2701 * No L2 page table has been allocated. Chances are, this 2702 * is because we just allocated the l2_dtable, above. 2703 */ 2704 nva = pmap_kernel_l2ptp_kva; 2705 ptep = (pt_entry_t *)nva; 2706 if ((nva & PAGE_MASK) == 0) { 2707 /* 2708 * Need to allocate a backing page 2709 */ 2710 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 2711 &pmap_kernel_l2ptp_phys)) 2712 return (NULL); 2713 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 2714 } 2715 memset(ptep, 0, L2_TABLE_SIZE_REAL); 2716 l2->l2_occupancy++; 2717 l2b->l2b_kva = ptep; 2718 l2b->l2b_l1idx = l1idx; 2719 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 2720 2721 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 2722 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 2723 } 2724 2725 /* Distribute new L1 entry to all other L1s */ 2726 SLIST_FOREACH(l1, &l1_list, l1_link) { 2727 pl1pd = &l1->l1_kva[L1_IDX(va)]; 2728 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 2729 L1_C_PROTO; 2730 PTE_SYNC(pl1pd); 2731 } 2732 2733 return (l2b); 2734} 2735 2736 2737/* 2738 * grow the number of kernel page table entries, if needed 2739 */ 2740void 2741pmap_growkernel(vm_offset_t addr) 2742{ 2743 pmap_t kpm = pmap_kernel(); 2744 2745 if (addr <= pmap_curmaxkvaddr) 2746 return; /* we are OK */ 2747 2748 /* 2749 * whoops! we need to add kernel PTPs 2750 */ 2751 2752 /* Map 1MB at a time */ 2753 for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) 2754 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 2755 2756 /* 2757 * flush out the cache, expensive but growkernel will happen so 2758 * rarely 2759 */ 2760 cpu_dcache_wbinv_all(); 2761 cpu_tlb_flushD(); 2762 cpu_cpwait(); 2763 kernel_vm_end = pmap_curmaxkvaddr; 2764 2765} 2766 2767 2768/* 2769 * Remove all pages from specified address space 2770 * this aids process exit speeds. Also, this code 2771 * is special cased for current process only, but 2772 * can have the more generic (and slightly slower) 2773 * mode enabled. This is much faster than pmap_remove 2774 * in the case of running down an entire address space. 2775 */ 2776void 2777pmap_remove_pages(pmap_t pmap) 2778{ 2779 struct pv_entry *pv, *npv; 2780 struct l2_bucket *l2b = NULL; 2781 vm_page_t m; 2782 pt_entry_t *pt; 2783 2784 vm_page_lock_queues(); 2785 PMAP_LOCK(pmap); 2786 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2787 if (pv->pv_flags & PVF_WIRED) { 2788 /* The page is wired, cannot remove it now. */ 2789 npv = TAILQ_NEXT(pv, pv_plist); 2790 continue; 2791 } 2792 pmap->pm_stats.resident_count--; 2793 l2b = pmap_get_l2_bucket(pmap, pv->pv_va); 2794 KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); 2795 pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2796 m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); 2797#ifdef ARM_USE_SMALL_ALLOC 2798 KASSERT((vm_offset_t)m >= alloc_firstaddr, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); 2799#else 2800 KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); 2801#endif 2802 *pt = 0; 2803 PTE_SYNC(pt); 2804 npv = TAILQ_NEXT(pv, pv_plist); 2805 pmap_nuke_pv(m, pmap, pv); 2806 if (TAILQ_EMPTY(&m->md.pv_list)) 2807 vm_page_flag_clear(m, PG_WRITEABLE); 2808 pmap_free_pv_entry(pv); 2809 pmap_free_l2_bucket(pmap, l2b, 1); 2810 } 2811 vm_page_unlock_queues(); 2812 cpu_idcache_wbinv_all(); 2813 cpu_tlb_flushID(); 2814 cpu_cpwait(); 2815 PMAP_UNLOCK(pmap); 2816} 2817 2818 2819/*************************************************** 2820 * Low level mapping routines..... 2821 ***************************************************/ 2822 2823/* Map a section into the KVA. */ 2824 2825void 2826pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) 2827{ 2828 pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, 2829 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 2830 struct l1_ttable *l1; 2831 2832 KASSERT(((va | pa) & L1_S_OFFSET) == 0, 2833 ("Not a valid section mapping")); 2834 if (flags & SECTION_CACHE) 2835 pd |= pte_l1_s_cache_mode; 2836 else if (flags & SECTION_PT) 2837 pd |= pte_l1_s_cache_mode_pt; 2838 SLIST_FOREACH(l1, &l1_list, l1_link) { 2839 l1->l1_kva[L1_IDX(va)] = pd; 2840 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 2841 } 2842} 2843 2844/* 2845 * add a wired page to the kva 2846 * note that in order for the mapping to take effect -- you 2847 * should do a invltlb after doing the pmap_kenter... 2848 */ 2849static PMAP_INLINE void 2850pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) 2851{ 2852 struct l2_bucket *l2b; 2853 pt_entry_t *pte; 2854 pt_entry_t opte; 2855 PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", 2856 (uint32_t) va, (uint32_t) pa)); 2857 2858 2859 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2860 if (l2b == NULL) 2861 l2b = pmap_grow_l2_bucket(pmap_kernel(), va); 2862 KASSERT(l2b != NULL, ("No L2 Bucket")); 2863 pte = &l2b->l2b_kva[l2pte_index(va)]; 2864 opte = *pte; 2865 PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", 2866 (uint32_t) pte, opte, *pte)); 2867 if (l2pte_valid(opte)) { 2868 cpu_dcache_wbinv_range(va, PAGE_SIZE); 2869 cpu_tlb_flushD_SE(va); 2870 cpu_cpwait(); 2871 } else { 2872 if (opte == 0) 2873 l2b->l2b_occupancy++; 2874 } 2875 *pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, 2876 VM_PROT_READ | VM_PROT_WRITE); 2877 if (flags & KENTER_CACHE) 2878 *pte |= pte_l2_s_cache_mode; 2879 if (flags & KENTER_USER) 2880 *pte |= L2_S_PROT_U; 2881 PTE_SYNC(pte); 2882} 2883 2884void 2885pmap_kenter(vm_offset_t va, vm_paddr_t pa) 2886{ 2887 pmap_kenter_internal(va, pa, KENTER_CACHE); 2888} 2889 2890void 2891pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) 2892{ 2893 2894 pmap_kenter_internal(va, pa, 0); 2895} 2896 2897void 2898pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) 2899{ 2900 2901 pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); 2902 /* 2903 * Call pmap_fault_fixup now, to make sure we'll have no exception 2904 * at the first use of the new address, or bad things will happen, 2905 * as we use one of these addresses in the exception handlers. 2906 */ 2907 pmap_fault_fixup(pmap_kernel(), va, VM_PROT_READ|VM_PROT_WRITE, 1); 2908} 2909 2910/* 2911 * remove a page rom the kernel pagetables 2912 */ 2913void 2914pmap_kremove(vm_offset_t va) 2915{ 2916 struct l2_bucket *l2b; 2917 pt_entry_t *pte, opte; 2918 2919 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2920 if (!l2b) 2921 return; 2922 KASSERT(l2b != NULL, ("No L2 Bucket")); 2923 pte = &l2b->l2b_kva[l2pte_index(va)]; 2924 opte = *pte; 2925 if (l2pte_valid(opte)) { 2926 cpu_dcache_wbinv_range(va, PAGE_SIZE); 2927 cpu_tlb_flushD_SE(va); 2928 cpu_cpwait(); 2929 *pte = 0; 2930 } 2931} 2932 2933 2934/* 2935 * Used to map a range of physical addresses into kernel 2936 * virtual address space. 2937 * 2938 * The value passed in '*virt' is a suggested virtual address for 2939 * the mapping. Architectures which can support a direct-mapped 2940 * physical to virtual region can return the appropriate address 2941 * within that region, leaving '*virt' unchanged. Other 2942 * architectures should map the pages starting at '*virt' and 2943 * update '*virt' with the first usable address after the mapped 2944 * region. 2945 */ 2946vm_offset_t 2947pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 2948{ 2949#ifdef ARM_USE_SMALL_ALLOC 2950 return (arm_ptovirt(start)); 2951#else 2952 vm_offset_t sva = *virt; 2953 vm_offset_t va = sva; 2954 2955 PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " 2956 "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, 2957 prot)); 2958 2959 while (start < end) { 2960 pmap_kenter(va, start); 2961 va += PAGE_SIZE; 2962 start += PAGE_SIZE; 2963 } 2964 *virt = va; 2965 return (sva); 2966#endif 2967} 2968 2969static void 2970pmap_wb_page(vm_page_t m) 2971{ 2972 struct pv_entry *pv; 2973 2974 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 2975 pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, FALSE, 2976 (pv->pv_flags & PVF_WRITE) == 0); 2977} 2978 2979static void 2980pmap_inv_page(vm_page_t m) 2981{ 2982 struct pv_entry *pv; 2983 2984 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 2985 pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE); 2986} 2987/* 2988 * Add a list of wired pages to the kva 2989 * this routine is only used for temporary 2990 * kernel mappings that do not need to have 2991 * page modification or references recorded. 2992 * Note that old mappings are simply written 2993 * over. The page *must* be wired. 2994 */ 2995void 2996pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 2997{ 2998 int i; 2999 3000 for (i = 0; i < count; i++) { 3001 pmap_wb_page(m[i]); 3002 pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), 3003 KENTER_CACHE); 3004 va += PAGE_SIZE; 3005 } 3006} 3007 3008 3009/* 3010 * this routine jerks page mappings from the 3011 * kernel -- it is meant only for temporary mappings. 3012 */ 3013void 3014pmap_qremove(vm_offset_t va, int count) 3015{ 3016 vm_paddr_t pa; 3017 int i; 3018 3019 for (i = 0; i < count; i++) { 3020 pa = vtophys(va); 3021 if (pa) { 3022 pmap_inv_page(PHYS_TO_VM_PAGE(pa)); 3023 pmap_kremove(va); 3024 } 3025 va += PAGE_SIZE; 3026 } 3027} 3028 3029 3030/* 3031 * pmap_object_init_pt preloads the ptes for a given object 3032 * into the specified pmap. This eliminates the blast of soft 3033 * faults on process startup and immediately after an mmap. 3034 */ 3035void 3036pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 3037 vm_pindex_t pindex, vm_size_t size) 3038{ 3039 3040 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 3041 KASSERT(object->type == OBJT_DEVICE, 3042 ("pmap_object_init_pt: non-device object")); 3043} 3044 3045 3046/* 3047 * pmap_is_prefaultable: 3048 * 3049 * Return whether or not the specified virtual address is elgible 3050 * for prefault. 3051 */ 3052boolean_t 3053pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3054{ 3055 pd_entry_t *pde; 3056 pt_entry_t *pte; 3057 3058 if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) 3059 return (FALSE); 3060 KASSERT(pte != NULL, ("Valid mapping but no pte ?")); 3061 if (*pte == 0) 3062 return (TRUE); 3063 return (FALSE); 3064} 3065 3066/* 3067 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 3068 * Returns TRUE if the mapping exists, else FALSE. 3069 * 3070 * NOTE: This function is only used by a couple of arm-specific modules. 3071 * It is not safe to take any pmap locks here, since we could be right 3072 * in the middle of debugging the pmap anyway... 3073 * 3074 * It is possible for this routine to return FALSE even though a valid 3075 * mapping does exist. This is because we don't lock, so the metadata 3076 * state may be inconsistent. 3077 * 3078 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 3079 * a "section" mapping. 3080 */ 3081boolean_t 3082pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) 3083{ 3084 struct l2_dtable *l2; 3085 pd_entry_t *pl1pd, l1pd; 3086 pt_entry_t *ptep; 3087 u_short l1idx; 3088 3089 if (pm->pm_l1 == NULL) 3090 return (FALSE); 3091 3092 l1idx = L1_IDX(va); 3093 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3094 l1pd = *pl1pd; 3095 3096 if (l1pte_section_p(l1pd)) { 3097 *ptp = NULL; 3098 return (TRUE); 3099 } 3100 3101 if (pm->pm_l2 == NULL) 3102 return (FALSE); 3103 3104 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3105 3106 if (l2 == NULL || 3107 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3108 return (FALSE); 3109 } 3110 3111 *ptp = &ptep[l2pte_index(va)]; 3112 return (TRUE); 3113} 3114 3115/* 3116 * Routine: pmap_remove_all 3117 * Function: 3118 * Removes this physical page from 3119 * all physical maps in which it resides. 3120 * Reflects back modify bits to the pager. 3121 * 3122 * Notes: 3123 * Original versions of this routine were very 3124 * inefficient because they iteratively called 3125 * pmap_remove (slow...) 3126 */ 3127void 3128pmap_remove_all(vm_page_t m) 3129{ 3130 pv_entry_t pv; 3131 pt_entry_t *ptep, pte; 3132 struct l2_bucket *l2b; 3133 boolean_t flush = FALSE; 3134 pmap_t curpm; 3135 int flags = 0; 3136 3137#if defined(PMAP_DEBUG) 3138 /* 3139 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 3140 * pages! 3141 */ 3142 if (m->flags & PG_FICTITIOUS) { 3143 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%x", VM_PAGE_TO_PHYS(m)); 3144 } 3145#endif 3146 3147 if (TAILQ_EMPTY(&m->md.pv_list)) 3148 return; 3149 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 3150 curpm = vmspace_pmap(curproc->p_vmspace); 3151 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 3152 if (flush == FALSE && (pv->pv_pmap == curpm || 3153 pv->pv_pmap == pmap_kernel())) 3154 flush = TRUE; 3155 PMAP_LOCK(pv->pv_pmap); 3156 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 3157 KASSERT(l2b != NULL, ("No l2 bucket")); 3158 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 3159 pte = *ptep; 3160 *ptep = 0; 3161 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 3162 pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); 3163 if (pv->pv_flags & PVF_WIRED) 3164 pv->pv_pmap->pm_stats.wired_count--; 3165 pv->pv_pmap->pm_stats.resident_count--; 3166 flags |= pv->pv_flags; 3167 pmap_nuke_pv(m, pv->pv_pmap, pv); 3168 PMAP_UNLOCK(pv->pv_pmap); 3169 pmap_free_pv_entry(pv); 3170 } 3171 3172 if (flush) { 3173 if (PV_BEEN_EXECD(flags)) 3174 pmap_tlb_flushID(curpm); 3175 else 3176 pmap_tlb_flushD(curpm); 3177 } 3178 vm_page_flag_clear(m, PG_WRITEABLE); 3179} 3180 3181 3182/* 3183 * Set the physical protection on the 3184 * specified range of this map as requested. 3185 */ 3186void 3187pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 3188{ 3189 struct l2_bucket *l2b; 3190 pt_entry_t *ptep, pte; 3191 vm_offset_t next_bucket; 3192 u_int flags; 3193 int flush; 3194 3195 if ((prot & VM_PROT_READ) == 0) { 3196 pmap_remove(pm, sva, eva); 3197 return; 3198 } 3199 3200 if (prot & VM_PROT_WRITE) { 3201 /* 3202 * If this is a read->write transition, just ignore it and let 3203 * vm_fault() take care of it later. 3204 */ 3205 return; 3206 } 3207 3208 vm_page_lock_queues(); 3209 PMAP_LOCK(pm); 3210 3211 /* 3212 * OK, at this point, we know we're doing write-protect operation. 3213 * If the pmap is active, write-back the range. 3214 */ 3215 pmap_dcache_wb_range(pm, sva, eva - sva, FALSE, FALSE); 3216 3217 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 3218 flags = 0; 3219 3220 while (sva < eva) { 3221 next_bucket = L2_NEXT_BUCKET(sva); 3222 if (next_bucket > eva) 3223 next_bucket = eva; 3224 3225 l2b = pmap_get_l2_bucket(pm, sva); 3226 if (l2b == NULL) { 3227 sva = next_bucket; 3228 continue; 3229 } 3230 3231 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3232 3233 while (sva < next_bucket) { 3234 if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { 3235 struct vm_page *pg; 3236 u_int f; 3237 3238 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3239 pte &= ~L2_S_PROT_W; 3240 *ptep = pte; 3241 PTE_SYNC(ptep); 3242 3243 if (pg != NULL) { 3244 f = pmap_modify_pv(pg, pm, sva, 3245 PVF_WRITE, 0); 3246 pmap_vac_me_harder(pg, pm, sva); 3247 vm_page_dirty(pg); 3248 } else 3249 f = PVF_REF | PVF_EXEC; 3250 3251 if (flush >= 0) { 3252 flush++; 3253 flags |= f; 3254 } else 3255 if (PV_BEEN_EXECD(f)) 3256 pmap_tlb_flushID_SE(pm, sva); 3257 else 3258 if (PV_BEEN_REFD(f)) 3259 pmap_tlb_flushD_SE(pm, sva); 3260 } 3261 3262 sva += PAGE_SIZE; 3263 ptep++; 3264 } 3265 } 3266 3267 3268 if (flush) { 3269 if (PV_BEEN_EXECD(flags)) 3270 pmap_tlb_flushID(pm); 3271 else 3272 if (PV_BEEN_REFD(flags)) 3273 pmap_tlb_flushD(pm); 3274 } 3275 vm_page_unlock_queues(); 3276 3277 PMAP_UNLOCK(pm); 3278} 3279 3280 3281/* 3282 * Insert the given physical page (p) at 3283 * the specified virtual address (v) in the 3284 * target physical map with the protection requested. 3285 * 3286 * If specified, the page will be wired down, meaning 3287 * that the related pte can not be reclaimed. 3288 * 3289 * NB: This is the only routine which MAY NOT lazy-evaluate 3290 * or lose information. That is, this routine must actually 3291 * insert this page into the given map NOW. 3292 */ 3293 3294void 3295pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3296 boolean_t wired) 3297{ 3298 3299 vm_page_lock_queues(); 3300 PMAP_LOCK(pmap); 3301 pmap_enter_locked(pmap, va, m, prot, wired, M_WAITOK); 3302 vm_page_unlock_queues(); 3303 PMAP_UNLOCK(pmap); 3304} 3305 3306/* 3307 * The page queues and pmap must be locked. 3308 */ 3309static void 3310pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3311 boolean_t wired, int flags) 3312{ 3313 struct l2_bucket *l2b = NULL; 3314 struct vm_page *opg; 3315 struct pv_entry *pve = NULL; 3316 pt_entry_t *ptep, npte, opte; 3317 u_int nflags; 3318 u_int oflags; 3319 vm_paddr_t pa; 3320 3321 PMAP_ASSERT_LOCKED(pmap); 3322 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 3323 if (va == vector_page) { 3324 pa = systempage.pv_pa; 3325 m = NULL; 3326 } else 3327 pa = VM_PAGE_TO_PHYS(m); 3328 nflags = 0; 3329 if (prot & VM_PROT_WRITE) 3330 nflags |= PVF_WRITE; 3331 if (prot & VM_PROT_EXECUTE) 3332 nflags |= PVF_EXEC; 3333 if (wired) 3334 nflags |= PVF_WIRED; 3335 PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " 3336 "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); 3337 3338 if (pmap == pmap_kernel()) { 3339 l2b = pmap_get_l2_bucket(pmap, va); 3340 if (l2b == NULL) 3341 l2b = pmap_grow_l2_bucket(pmap, va); 3342 } else { 3343do_l2b_alloc: 3344 l2b = pmap_alloc_l2_bucket(pmap, va); 3345 if (l2b == NULL) { 3346 if (flags & M_WAITOK) { 3347 PMAP_UNLOCK(pmap); 3348 vm_page_unlock_queues(); 3349 VM_WAIT; 3350 vm_page_lock_queues(); 3351 PMAP_LOCK(pmap); 3352 goto do_l2b_alloc; 3353 } 3354 return; 3355 } 3356 } 3357 3358 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3359 3360 opte = *ptep; 3361 npte = pa; 3362 oflags = 0; 3363 if (opte) { 3364 /* 3365 * There is already a mapping at this address. 3366 * If the physical address is different, lookup the 3367 * vm_page. 3368 */ 3369 if (l2pte_pa(opte) != pa) 3370 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3371 else 3372 opg = m; 3373 } else 3374 opg = NULL; 3375 3376 if ((prot & (VM_PROT_ALL)) || 3377 (!m || m->md.pvh_attrs & PVF_REF)) { 3378 /* 3379 * - The access type indicates that we don't need 3380 * to do referenced emulation. 3381 * OR 3382 * - The physical page has already been referenced 3383 * so no need to re-do referenced emulation here. 3384 */ 3385 npte |= L2_S_PROTO; 3386 3387 nflags |= PVF_REF; 3388 3389 if (m && ((prot & VM_PROT_WRITE) != 0 || 3390 (m->md.pvh_attrs & PVF_MOD))) { 3391 /* 3392 * This is a writable mapping, and the 3393 * page's mod state indicates it has 3394 * already been modified. Make it 3395 * writable from the outset. 3396 */ 3397 nflags |= PVF_MOD; 3398 if (!(m->md.pvh_attrs & PVF_MOD)) 3399 vm_page_dirty(m); 3400 } 3401 if (m && opte) 3402 vm_page_flag_set(m, PG_REFERENCED); 3403 } else { 3404 /* 3405 * Need to do page referenced emulation. 3406 */ 3407 npte |= L2_TYPE_INV; 3408 } 3409 3410 if (prot & VM_PROT_WRITE) { 3411 npte |= L2_S_PROT_W; 3412 if (m != NULL) 3413 vm_page_flag_set(m, PG_WRITEABLE); 3414 } 3415 npte |= pte_l2_s_cache_mode; 3416 if (m && m == opg) { 3417 /* 3418 * We're changing the attrs of an existing mapping. 3419 */ 3420 oflags = pmap_modify_pv(m, pmap, va, 3421 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3422 PVF_MOD | PVF_REF, nflags); 3423 3424 /* 3425 * We may need to flush the cache if we're 3426 * doing rw-ro... 3427 */ 3428 if (pmap_is_current(pmap) && 3429 (oflags & PVF_NC) == 0 && 3430 (opte & L2_S_PROT_W) != 0 && 3431 (prot & VM_PROT_WRITE) == 0) 3432 cpu_dcache_wb_range(va, PAGE_SIZE); 3433 } else { 3434 /* 3435 * New mapping, or changing the backing page 3436 * of an existing mapping. 3437 */ 3438 if (opg) { 3439 /* 3440 * Replacing an existing mapping with a new one. 3441 * It is part of our managed memory so we 3442 * must remove it from the PV list 3443 */ 3444 pve = pmap_remove_pv(opg, pmap, va); 3445 if (m && (m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) && 3446 pve) 3447 pmap_free_pv_entry(pve); 3448 else if (!pve && 3449 !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) 3450 pve = pmap_get_pv_entry(); 3451 KASSERT(pve != NULL || m->flags & (PG_UNMANAGED | 3452 PG_FICTITIOUS), ("No pv")); 3453 oflags = pve->pv_flags; 3454 3455 /* 3456 * If the old mapping was valid (ref/mod 3457 * emulation creates 'invalid' mappings 3458 * initially) then make sure to frob 3459 * the cache. 3460 */ 3461 if ((oflags & PVF_NC) == 0 && 3462 l2pte_valid(opte)) { 3463 if (PV_BEEN_EXECD(oflags)) { 3464 pmap_idcache_wbinv_range(pmap, va, 3465 PAGE_SIZE); 3466 } else 3467 if (PV_BEEN_REFD(oflags)) { 3468 pmap_dcache_wb_range(pmap, va, 3469 PAGE_SIZE, TRUE, 3470 (oflags & PVF_WRITE) == 0); 3471 } 3472 } 3473 } else if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) 3474 if ((pve = pmap_get_pv_entry()) == NULL) { 3475 panic("pmap_enter: no pv entries"); 3476 } 3477 if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) { 3478 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 3479 ("pmap_enter: managed mapping within the clean submap")); 3480 pmap_enter_pv(m, pve, pmap, va, nflags); 3481 } 3482 } 3483 /* 3484 * Make sure userland mappings get the right permissions 3485 */ 3486 if (pmap != pmap_kernel() && va != vector_page) { 3487 npte |= L2_S_PROT_U; 3488 } 3489 3490 /* 3491 * Keep the stats up to date 3492 */ 3493 if (opte == 0) { 3494 l2b->l2b_occupancy++; 3495 pmap->pm_stats.resident_count++; 3496 } 3497 3498 3499 /* 3500 * If this is just a wiring change, the two PTEs will be 3501 * identical, so there's no need to update the page table. 3502 */ 3503 if (npte != opte) { 3504 boolean_t is_cached = pmap_is_current(pmap); 3505 3506 *ptep = npte; 3507 if (is_cached) { 3508 /* 3509 * We only need to frob the cache/tlb if this pmap 3510 * is current 3511 */ 3512 PTE_SYNC(ptep); 3513 if (L1_IDX(va) != L1_IDX(vector_page) && 3514 l2pte_valid(npte)) { 3515 /* 3516 * This mapping is likely to be accessed as 3517 * soon as we return to userland. Fix up the 3518 * L1 entry to avoid taking another 3519 * page/domain fault. 3520 */ 3521 pd_entry_t *pl1pd, l1pd; 3522 3523 pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; 3524 l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | 3525 L1_C_PROTO; 3526 if (*pl1pd != l1pd) { 3527 *pl1pd = l1pd; 3528 PTE_SYNC(pl1pd); 3529 } 3530 } 3531 } 3532 3533 if (PV_BEEN_EXECD(oflags)) 3534 pmap_tlb_flushID_SE(pmap, va); 3535 else if (PV_BEEN_REFD(oflags)) 3536 pmap_tlb_flushD_SE(pmap, va); 3537 3538 3539 if (m) 3540 pmap_vac_me_harder(m, pmap, va); 3541 } 3542} 3543 3544/* 3545 * Maps a sequence of resident pages belonging to the same object. 3546 * The sequence begins with the given page m_start. This page is 3547 * mapped at the given virtual address start. Each subsequent page is 3548 * mapped at a virtual address that is offset from start by the same 3549 * amount as the page is offset from m_start within the object. The 3550 * last page in the sequence is the page with the largest offset from 3551 * m_start that can be mapped at a virtual address less than the given 3552 * virtual address end. Not every virtual page between start and end 3553 * is mapped; only those for which a resident page exists with the 3554 * corresponding offset from m_start are mapped. 3555 */ 3556void 3557pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 3558 vm_page_t m_start, vm_prot_t prot) 3559{ 3560 vm_page_t m; 3561 vm_pindex_t diff, psize; 3562 3563 psize = atop(end - start); 3564 m = m_start; 3565 PMAP_LOCK(pmap); 3566 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3567 pmap_enter_locked(pmap, start + ptoa(diff), m, prot & 3568 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE, M_NOWAIT); 3569 m = TAILQ_NEXT(m, listq); 3570 } 3571 PMAP_UNLOCK(pmap); 3572} 3573 3574/* 3575 * this code makes some *MAJOR* assumptions: 3576 * 1. Current pmap & pmap exists. 3577 * 2. Not wired. 3578 * 3. Read access. 3579 * 4. No page table pages. 3580 * but is *MUCH* faster than pmap_enter... 3581 */ 3582 3583void 3584pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) 3585{ 3586 3587 PMAP_LOCK(pmap); 3588 pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 3589 FALSE, M_NOWAIT); 3590 PMAP_UNLOCK(pmap); 3591} 3592 3593/* 3594 * Routine: pmap_change_wiring 3595 * Function: Change the wiring attribute for a map/virtual-address 3596 * pair. 3597 * In/out conditions: 3598 * The mapping must already exist in the pmap. 3599 */ 3600void 3601pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 3602{ 3603 struct l2_bucket *l2b; 3604 pt_entry_t *ptep, pte; 3605 vm_page_t pg; 3606 3607 vm_page_lock_queues(); 3608 PMAP_LOCK(pmap); 3609 l2b = pmap_get_l2_bucket(pmap, va); 3610 KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); 3611 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3612 pte = *ptep; 3613 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3614 if (pg) 3615 pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); 3616 vm_page_unlock_queues(); 3617 PMAP_UNLOCK(pmap); 3618} 3619 3620 3621/* 3622 * Copy the range specified by src_addr/len 3623 * from the source map to the range dst_addr/len 3624 * in the destination map. 3625 * 3626 * This routine is only advisory and need not do anything. 3627 */ 3628void 3629pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 3630 vm_size_t len, vm_offset_t src_addr) 3631{ 3632} 3633 3634 3635/* 3636 * Routine: pmap_extract 3637 * Function: 3638 * Extract the physical page address associated 3639 * with the given map/virtual_address pair. 3640 */ 3641vm_paddr_t 3642pmap_extract(pmap_t pm, vm_offset_t va) 3643{ 3644 struct l2_dtable *l2; 3645 pd_entry_t l1pd; 3646 pt_entry_t *ptep, pte; 3647 vm_paddr_t pa; 3648 u_int l1idx; 3649 l1idx = L1_IDX(va); 3650 3651 PMAP_LOCK(pm); 3652 l1pd = pm->pm_l1->l1_kva[l1idx]; 3653 if (l1pte_section_p(l1pd)) { 3654 /* 3655 * These should only happen for pmap_kernel() 3656 */ 3657 KASSERT(pm == pmap_kernel(), ("huh")); 3658 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3659 } else { 3660 /* 3661 * Note that we can't rely on the validity of the L1 3662 * descriptor as an indication that a mapping exists. 3663 * We have to look it up in the L2 dtable. 3664 */ 3665 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3666 3667 if (l2 == NULL || 3668 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3669 PMAP_UNLOCK(pm); 3670 return (0); 3671 } 3672 3673 ptep = &ptep[l2pte_index(va)]; 3674 pte = *ptep; 3675 3676 if (pte == 0) { 3677 PMAP_UNLOCK(pm); 3678 return (0); 3679 } 3680 3681 switch (pte & L2_TYPE_MASK) { 3682 case L2_TYPE_L: 3683 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3684 break; 3685 3686 default: 3687 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3688 break; 3689 } 3690 } 3691 3692 PMAP_UNLOCK(pm); 3693 return (pa); 3694} 3695 3696/* 3697 * Atomically extract and hold the physical page with the given 3698 * pmap and virtual address pair if that mapping permits the given 3699 * protection. 3700 * 3701 */ 3702vm_page_t 3703pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3704{ 3705 struct l2_dtable *l2; 3706 pd_entry_t l1pd; 3707 pt_entry_t *ptep, pte; 3708 vm_paddr_t pa; 3709 vm_page_t m = NULL; 3710 u_int l1idx; 3711 l1idx = L1_IDX(va); 3712 3713 vm_page_lock_queues(); 3714 PMAP_LOCK(pmap); 3715 l1pd = pmap->pm_l1->l1_kva[l1idx]; 3716 if (l1pte_section_p(l1pd)) { 3717 /* 3718 * These should only happen for pmap_kernel() 3719 */ 3720 KASSERT(pmap == pmap_kernel(), ("huh")); 3721 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3722 if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { 3723 m = PHYS_TO_VM_PAGE(pa); 3724 vm_page_hold(m); 3725 } 3726 3727 } else { 3728 /* 3729 * Note that we can't rely on the validity of the L1 3730 * descriptor as an indication that a mapping exists. 3731 * We have to look it up in the L2 dtable. 3732 */ 3733 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 3734 3735 if (l2 == NULL || 3736 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3737 PMAP_UNLOCK(pmap); 3738 vm_page_unlock_queues(); 3739 return (NULL); 3740 } 3741 3742 ptep = &ptep[l2pte_index(va)]; 3743 pte = *ptep; 3744 3745 if (pte == 0) { 3746 PMAP_UNLOCK(pmap); 3747 vm_page_unlock_queues(); 3748 return (NULL); 3749 } 3750 if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { 3751 switch (pte & L2_TYPE_MASK) { 3752 case L2_TYPE_L: 3753 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3754 break; 3755 3756 default: 3757 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3758 break; 3759 } 3760 m = PHYS_TO_VM_PAGE(pa); 3761 vm_page_hold(m); 3762 } 3763 } 3764 3765 PMAP_UNLOCK(pmap); 3766 vm_page_unlock_queues(); 3767 return (m); 3768} 3769 3770/* 3771 * Initialize a preallocated and zeroed pmap structure, 3772 * such as one in a vmspace structure. 3773 */ 3774 3775void 3776pmap_pinit(pmap_t pmap) 3777{ 3778 PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); 3779 3780 PMAP_LOCK_INIT(pmap); 3781 pmap_alloc_l1(pmap); 3782 bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); 3783 3784 pmap->pm_count = 1; 3785 pmap->pm_active = 0; 3786 3787 TAILQ_INIT(&pmap->pm_pvlist); 3788 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 3789 pmap->pm_stats.resident_count = 1; 3790 if (vector_page < KERNBASE) { 3791 pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa), 3792 VM_PROT_READ, 1); 3793 } 3794} 3795 3796 3797/*************************************************** 3798 * page management routines. 3799 ***************************************************/ 3800 3801 3802static void 3803pmap_free_pv_entry(pv_entry_t pv) 3804{ 3805 pv_entry_count--; 3806 uma_zfree(pvzone, pv); 3807} 3808 3809 3810/* 3811 * get a new pv_entry, allocating a block from the system 3812 * when needed. 3813 * the memory allocation is performed bypassing the malloc code 3814 * because of the possibility of allocations at interrupt time. 3815 */ 3816static pv_entry_t 3817pmap_get_pv_entry(void) 3818{ 3819 pv_entry_t ret_value; 3820 3821 pv_entry_count++; 3822 if (pv_entry_count > pv_entry_high_water) 3823 pagedaemon_wakeup(); 3824 ret_value = uma_zalloc(pvzone, M_NOWAIT); 3825 return ret_value; 3826} 3827 3828 3829/* 3830 * Remove the given range of addresses from the specified map. 3831 * 3832 * It is assumed that the start and end are properly 3833 * rounded to the page size. 3834 */ 3835#define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3836void 3837pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 3838{ 3839 struct l2_bucket *l2b; 3840 vm_offset_t next_bucket; 3841 pt_entry_t *ptep; 3842 u_int cleanlist_idx, total, cnt; 3843 struct { 3844 vm_offset_t va; 3845 pt_entry_t *pte; 3846 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3847 u_int mappings, is_exec, is_refd; 3848 int flushall = 0; 3849 3850 3851 /* 3852 * we lock in the pmap => pv_head direction 3853 */ 3854 3855 vm_page_lock_queues(); 3856 PMAP_LOCK(pm); 3857 if (!pmap_is_current(pm)) { 3858 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3859 } else 3860 cleanlist_idx = 0; 3861 3862 total = 0; 3863 while (sva < eva) { 3864 /* 3865 * Do one L2 bucket's worth at a time. 3866 */ 3867 next_bucket = L2_NEXT_BUCKET(sva); 3868 if (next_bucket > eva) 3869 next_bucket = eva; 3870 3871 l2b = pmap_get_l2_bucket(pm, sva); 3872 if (l2b == NULL) { 3873 sva = next_bucket; 3874 continue; 3875 } 3876 3877 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3878 mappings = 0; 3879 3880 while (sva < next_bucket) { 3881 struct vm_page *pg; 3882 pt_entry_t pte; 3883 vm_paddr_t pa; 3884 3885 pte = *ptep; 3886 3887 if (pte == 0) { 3888 /* 3889 * Nothing here, move along 3890 */ 3891 sva += PAGE_SIZE; 3892 ptep++; 3893 continue; 3894 } 3895 3896 pm->pm_stats.resident_count--; 3897 pa = l2pte_pa(pte); 3898 is_exec = 0; 3899 is_refd = 1; 3900 3901 /* 3902 * Update flags. In a number of circumstances, 3903 * we could cluster a lot of these and do a 3904 * number of sequential pages in one go. 3905 */ 3906 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3907 struct pv_entry *pve; 3908 3909 pve = pmap_remove_pv(pg, pm, sva); 3910 if (pve) { 3911 is_exec = PV_BEEN_EXECD(pve->pv_flags); 3912 is_refd = PV_BEEN_REFD(pve->pv_flags); 3913 pmap_free_pv_entry(pve); 3914 } 3915 } 3916 3917 if (!l2pte_valid(pte)) { 3918 *ptep = 0; 3919 PTE_SYNC_CURRENT(pm, ptep); 3920 sva += PAGE_SIZE; 3921 ptep++; 3922 mappings++; 3923 continue; 3924 } 3925 3926 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3927 /* Add to the clean list. */ 3928 cleanlist[cleanlist_idx].pte = ptep; 3929 cleanlist[cleanlist_idx].va = 3930 sva | (is_exec & 1); 3931 cleanlist_idx++; 3932 } else 3933 if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3934 /* Nuke everything if needed. */ 3935 pmap_idcache_wbinv_all(pm); 3936 pmap_tlb_flushID(pm); 3937 3938 /* 3939 * Roll back the previous PTE list, 3940 * and zero out the current PTE. 3941 */ 3942 for (cnt = 0; 3943 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3944 *cleanlist[cnt].pte = 0; 3945 } 3946 *ptep = 0; 3947 PTE_SYNC(ptep); 3948 cleanlist_idx++; 3949 flushall = 1; 3950 } else { 3951 *ptep = 0; 3952 PTE_SYNC(ptep); 3953 if (is_exec) 3954 pmap_tlb_flushID_SE(pm, sva); 3955 else 3956 if (is_refd) 3957 pmap_tlb_flushD_SE(pm, sva); 3958 } 3959 3960 sva += PAGE_SIZE; 3961 ptep++; 3962 mappings++; 3963 } 3964 3965 /* 3966 * Deal with any left overs 3967 */ 3968 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3969 total += cleanlist_idx; 3970 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3971 vm_offset_t clva = 3972 cleanlist[cnt].va & ~1; 3973 if (cleanlist[cnt].va & 1) { 3974 pmap_idcache_wbinv_range(pm, 3975 clva, PAGE_SIZE); 3976 pmap_tlb_flushID_SE(pm, clva); 3977 } else { 3978 pmap_dcache_wb_range(pm, 3979 clva, PAGE_SIZE, TRUE, 3980 FALSE); 3981 pmap_tlb_flushD_SE(pm, clva); 3982 } 3983 *cleanlist[cnt].pte = 0; 3984 PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte); 3985 } 3986 3987 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3988 cleanlist_idx = 0; 3989 else { 3990 /* 3991 * We are removing so much entries it's just 3992 * easier to flush the whole cache. 3993 */ 3994 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3995 pmap_idcache_wbinv_all(pm); 3996 flushall = 1; 3997 } 3998 } 3999 4000 pmap_free_l2_bucket(pm, l2b, mappings); 4001 } 4002 4003 vm_page_unlock_queues(); 4004 if (flushall) 4005 cpu_tlb_flushID(); 4006 PMAP_UNLOCK(pm); 4007} 4008 4009 4010 4011 4012/* 4013 * pmap_zero_page() 4014 * 4015 * Zero a given physical page by mapping it at a page hook point. 4016 * In doing the zero page op, the page we zero is mapped cachable, as with 4017 * StrongARM accesses to non-cached pages are non-burst making writing 4018 * _any_ bulk data very slow. 4019 */ 4020#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 || defined(CPU_XSCALE_CORE3) 4021void 4022pmap_zero_page_generic(vm_paddr_t phys, int off, int size) 4023{ 4024#ifdef ARM_USE_SMALL_ALLOC 4025 char *dstpg; 4026#endif 4027 4028#ifdef DEBUG 4029 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4030 4031 if (pg->md.pvh_list != NULL) 4032 panic("pmap_zero_page: page has mappings"); 4033#endif 4034 4035 if (_arm_bzero && 4036 _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) 4037 return; 4038 4039#ifdef ARM_USE_SMALL_ALLOC 4040 dstpg = (char *)arm_ptovirt(phys); 4041 if (off || size != PAGE_SIZE) { 4042 bzero(dstpg + off, size); 4043 cpu_dcache_wbinv_range((vm_offset_t)(dstpg + off), size); 4044 } else { 4045 bzero_page((vm_offset_t)dstpg); 4046 cpu_dcache_wbinv_range((vm_offset_t)dstpg, PAGE_SIZE); 4047 } 4048#else 4049 4050 mtx_lock(&cmtx); 4051 /* 4052 * Hook in the page, zero it, and purge the cache for that 4053 * zeroed page. Invalidate the TLB as needed. 4054 */ 4055 *cdst_pte = L2_S_PROTO | phys | 4056 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4057 PTE_SYNC(cdst_pte); 4058 cpu_tlb_flushD_SE(cdstp); 4059 cpu_cpwait(); 4060 if (off || size != PAGE_SIZE) { 4061 bzero((void *)(cdstp + off), size); 4062 cpu_dcache_wbinv_range(cdstp + off, size); 4063 } else { 4064 bzero_page(cdstp); 4065 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4066 } 4067 mtx_unlock(&cmtx); 4068#endif 4069} 4070#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 4071 4072#if ARM_MMU_XSCALE == 1 4073void 4074pmap_zero_page_xscale(vm_paddr_t phys, int off, int size) 4075{ 4076 if (_arm_bzero && 4077 _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) 4078 return; 4079 mtx_lock(&cmtx); 4080 /* 4081 * Hook in the page, zero it, and purge the cache for that 4082 * zeroed page. Invalidate the TLB as needed. 4083 */ 4084 *cdst_pte = L2_S_PROTO | phys | 4085 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4086 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4087 PTE_SYNC(cdst_pte); 4088 cpu_tlb_flushD_SE(cdstp); 4089 cpu_cpwait(); 4090 if (off || size != PAGE_SIZE) 4091 bzero((void *)(cdstp + off), size); 4092 else 4093 bzero_page(cdstp); 4094 mtx_unlock(&cmtx); 4095 xscale_cache_clean_minidata(); 4096} 4097 4098/* 4099 * Change the PTEs for the specified kernel mappings such that they 4100 * will use the mini data cache instead of the main data cache. 4101 */ 4102void 4103pmap_use_minicache(vm_offset_t va, vm_size_t size) 4104{ 4105 struct l2_bucket *l2b; 4106 pt_entry_t *ptep, *sptep, pte; 4107 vm_offset_t next_bucket, eva; 4108 4109#if (ARM_NMMUS > 1) || defined(CPU_XSCALE_CORE3) 4110 if (xscale_use_minidata == 0) 4111 return; 4112#endif 4113 4114 eva = va + size; 4115 4116 while (va < eva) { 4117 next_bucket = L2_NEXT_BUCKET(va); 4118 if (next_bucket > eva) 4119 next_bucket = eva; 4120 4121 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 4122 4123 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 4124 4125 while (va < next_bucket) { 4126 pte = *ptep; 4127 if (!l2pte_minidata(pte)) { 4128 cpu_dcache_wbinv_range(va, PAGE_SIZE); 4129 cpu_tlb_flushD_SE(va); 4130 *ptep = pte & ~L2_B; 4131 } 4132 ptep++; 4133 va += PAGE_SIZE; 4134 } 4135 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 4136 } 4137 cpu_cpwait(); 4138} 4139#endif /* ARM_MMU_XSCALE == 1 */ 4140 4141/* 4142 * pmap_zero_page zeros the specified hardware page by mapping 4143 * the page into KVM and using bzero to clear its contents. 4144 */ 4145void 4146pmap_zero_page(vm_page_t m) 4147{ 4148 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), 0, PAGE_SIZE); 4149} 4150 4151 4152/* 4153 * pmap_zero_page_area zeros the specified hardware page by mapping 4154 * the page into KVM and using bzero to clear its contents. 4155 * 4156 * off and size may not cover an area beyond a single hardware page. 4157 */ 4158void 4159pmap_zero_page_area(vm_page_t m, int off, int size) 4160{ 4161 4162 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), off, size); 4163} 4164 4165 4166/* 4167 * pmap_zero_page_idle zeros the specified hardware page by mapping 4168 * the page into KVM and using bzero to clear its contents. This 4169 * is intended to be called from the vm_pagezero process only and 4170 * outside of Giant. 4171 */ 4172void 4173pmap_zero_page_idle(vm_page_t m) 4174{ 4175 4176 pmap_zero_page(m); 4177} 4178 4179#if 0 4180/* 4181 * pmap_clean_page() 4182 * 4183 * This is a local function used to work out the best strategy to clean 4184 * a single page referenced by its entry in the PV table. It's used by 4185 * pmap_copy_page, pmap_zero page and maybe some others later on. 4186 * 4187 * Its policy is effectively: 4188 * o If there are no mappings, we don't bother doing anything with the cache. 4189 * o If there is one mapping, we clean just that page. 4190 * o If there are multiple mappings, we clean the entire cache. 4191 * 4192 * So that some functions can be further optimised, it returns 0 if it didn't 4193 * clean the entire cache, or 1 if it did. 4194 * 4195 * XXX One bug in this routine is that if the pv_entry has a single page 4196 * mapped at 0x00000000 a whole cache clean will be performed rather than 4197 * just the 1 page. Since this should not occur in everyday use and if it does 4198 * it will just result in not the most efficient clean for the page. 4199 */ 4200static int 4201pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 4202{ 4203 pmap_t pm, pm_to_clean = NULL; 4204 struct pv_entry *npv; 4205 u_int cache_needs_cleaning = 0; 4206 u_int flags = 0; 4207 vm_offset_t page_to_clean = 0; 4208 4209 if (pv == NULL) { 4210 /* nothing mapped in so nothing to flush */ 4211 return (0); 4212 } 4213 4214 /* 4215 * Since we flush the cache each time we change to a different 4216 * user vmspace, we only need to flush the page if it is in the 4217 * current pmap. 4218 */ 4219 if (curthread) 4220 pm = vmspace_pmap(curproc->p_vmspace); 4221 else 4222 pm = pmap_kernel(); 4223 4224 for (npv = pv; npv; npv = TAILQ_NEXT(npv, pv_list)) { 4225 if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) { 4226 flags |= npv->pv_flags; 4227 /* 4228 * The page is mapped non-cacheable in 4229 * this map. No need to flush the cache. 4230 */ 4231 if (npv->pv_flags & PVF_NC) { 4232#ifdef DIAGNOSTIC 4233 if (cache_needs_cleaning) 4234 panic("pmap_clean_page: " 4235 "cache inconsistency"); 4236#endif 4237 break; 4238 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 4239 continue; 4240 if (cache_needs_cleaning) { 4241 page_to_clean = 0; 4242 break; 4243 } else { 4244 page_to_clean = npv->pv_va; 4245 pm_to_clean = npv->pv_pmap; 4246 } 4247 cache_needs_cleaning = 1; 4248 } 4249 } 4250 if (page_to_clean) { 4251 if (PV_BEEN_EXECD(flags)) 4252 pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, 4253 PAGE_SIZE); 4254 else 4255 pmap_dcache_wb_range(pm_to_clean, page_to_clean, 4256 PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); 4257 } else if (cache_needs_cleaning) { 4258 if (PV_BEEN_EXECD(flags)) 4259 pmap_idcache_wbinv_all(pm); 4260 else 4261 pmap_dcache_wbinv_all(pm); 4262 return (1); 4263 } 4264 return (0); 4265} 4266#endif 4267 4268/* 4269 * pmap_copy_page copies the specified (machine independent) 4270 * page by mapping the page into virtual memory and using 4271 * bcopy to copy the page, one machine dependent page at a 4272 * time. 4273 */ 4274 4275/* 4276 * pmap_copy_page() 4277 * 4278 * Copy one physical page into another, by mapping the pages into 4279 * hook points. The same comment regarding cachability as in 4280 * pmap_zero_page also applies here. 4281 */ 4282#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 || defined (CPU_XSCALE_CORE3) 4283void 4284pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) 4285{ 4286#if 0 4287 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4288#endif 4289#ifdef DEBUG 4290 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4291 4292 if (dst_pg->md.pvh_list != NULL) 4293 panic("pmap_copy_page: dst page has mappings"); 4294#endif 4295 4296 4297 /* 4298 * Clean the source page. Hold the source page's lock for 4299 * the duration of the copy so that no other mappings can 4300 * be created while we have a potentially aliased mapping. 4301 */ 4302#if 0 4303 /* 4304 * XXX: Not needed while we call cpu_dcache_wbinv_all() in 4305 * pmap_copy_page(). 4306 */ 4307 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4308#endif 4309 /* 4310 * Map the pages into the page hook points, copy them, and purge 4311 * the cache for the appropriate page. Invalidate the TLB 4312 * as required. 4313 */ 4314 mtx_lock(&cmtx); 4315 *csrc_pte = L2_S_PROTO | src | 4316 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 4317 PTE_SYNC(csrc_pte); 4318 *cdst_pte = L2_S_PROTO | dst | 4319 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4320 PTE_SYNC(cdst_pte); 4321 cpu_tlb_flushD_SE(csrcp); 4322 cpu_tlb_flushD_SE(cdstp); 4323 cpu_cpwait(); 4324 bcopy_page(csrcp, cdstp); 4325 mtx_unlock(&cmtx); 4326 cpu_dcache_inv_range(csrcp, PAGE_SIZE); 4327 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4328} 4329#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 4330 4331#if ARM_MMU_XSCALE == 1 4332void 4333pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) 4334{ 4335#if 0 4336 /* XXX: Only needed for pmap_clean_page(), which is commented out. */ 4337 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4338#endif 4339#ifdef DEBUG 4340 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4341 4342 if (dst_pg->md.pvh_list != NULL) 4343 panic("pmap_copy_page: dst page has mappings"); 4344#endif 4345 4346 4347 /* 4348 * Clean the source page. Hold the source page's lock for 4349 * the duration of the copy so that no other mappings can 4350 * be created while we have a potentially aliased mapping. 4351 */ 4352#if 0 4353 /* 4354 * XXX: Not needed while we call cpu_dcache_wbinv_all() in 4355 * pmap_copy_page(). 4356 */ 4357 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4358#endif 4359 /* 4360 * Map the pages into the page hook points, copy them, and purge 4361 * the cache for the appropriate page. Invalidate the TLB 4362 * as required. 4363 */ 4364 mtx_lock(&cmtx); 4365 *csrc_pte = L2_S_PROTO | src | 4366 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4367 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4368 PTE_SYNC(csrc_pte); 4369 *cdst_pte = L2_S_PROTO | dst | 4370 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4371 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4372 PTE_SYNC(cdst_pte); 4373 cpu_tlb_flushD_SE(csrcp); 4374 cpu_tlb_flushD_SE(cdstp); 4375 cpu_cpwait(); 4376 bcopy_page(csrcp, cdstp); 4377 mtx_unlock(&cmtx); 4378 xscale_cache_clean_minidata(); 4379} 4380#endif /* ARM_MMU_XSCALE == 1 */ 4381 4382void 4383pmap_copy_page(vm_page_t src, vm_page_t dst) 4384{ 4385#ifdef ARM_USE_SMALL_ALLOC 4386 vm_offset_t srcpg, dstpg; 4387#endif 4388 4389 cpu_dcache_wbinv_all(); 4390 if (_arm_memcpy && 4391 _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), 4392 (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) 4393 return; 4394#ifdef ARM_USE_SMALL_ALLOC 4395 srcpg = arm_ptovirt(VM_PAGE_TO_PHYS(src)); 4396 dstpg = arm_ptovirt(VM_PAGE_TO_PHYS(dst)); 4397 bcopy_page(srcpg, dstpg); 4398 cpu_dcache_wbinv_range(dstpg, PAGE_SIZE); 4399#else 4400 pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); 4401#endif 4402} 4403 4404 4405 4406 4407/* 4408 * this routine returns true if a physical page resides 4409 * in the given pmap. 4410 */ 4411boolean_t 4412pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 4413{ 4414 pv_entry_t pv; 4415 int loops = 0; 4416 4417 if (m->flags & PG_FICTITIOUS) 4418 return (FALSE); 4419 4420 /* 4421 * Not found, check current mappings returning immediately 4422 */ 4423 for (pv = TAILQ_FIRST(&m->md.pv_list); 4424 pv; 4425 pv = TAILQ_NEXT(pv, pv_list)) { 4426 if (pv->pv_pmap == pmap) { 4427 return (TRUE); 4428 } 4429 loops++; 4430 if (loops >= 16) 4431 break; 4432 } 4433 return (FALSE); 4434} 4435 4436 4437/* 4438 * pmap_ts_referenced: 4439 * 4440 * Return the count of reference bits for a page, clearing all of them. 4441 */ 4442int 4443pmap_ts_referenced(vm_page_t m) 4444{ 4445 4446 if (m->flags & PG_FICTITIOUS) 4447 return (0); 4448 return (pmap_clearbit(m, PVF_REF)); 4449} 4450 4451 4452boolean_t 4453pmap_is_modified(vm_page_t m) 4454{ 4455 4456 if (m->md.pvh_attrs & PVF_MOD) 4457 return (TRUE); 4458 4459 return(FALSE); 4460} 4461 4462 4463/* 4464 * Clear the modify bits on the specified physical page. 4465 */ 4466void 4467pmap_clear_modify(vm_page_t m) 4468{ 4469 4470 if (m->md.pvh_attrs & PVF_MOD) 4471 pmap_clearbit(m, PVF_MOD); 4472} 4473 4474 4475/* 4476 * pmap_clear_reference: 4477 * 4478 * Clear the reference bit on the specified physical page. 4479 */ 4480void 4481pmap_clear_reference(vm_page_t m) 4482{ 4483 4484 if (m->md.pvh_attrs & PVF_REF) 4485 pmap_clearbit(m, PVF_REF); 4486} 4487 4488 4489/* 4490 * Clear the write and modified bits in each of the given page's mappings. 4491 */ 4492void 4493pmap_remove_write(vm_page_t m) 4494{ 4495 4496 if (m->flags & PG_WRITEABLE) 4497 pmap_clearbit(m, PVF_WRITE); 4498} 4499 4500 4501/* 4502 * perform the pmap work for mincore 4503 */ 4504int 4505pmap_mincore(pmap_t pmap, vm_offset_t addr) 4506{ 4507 printf("pmap_mincore()\n"); 4508 4509 return (0); 4510} 4511 4512 4513vm_offset_t 4514pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 4515{ 4516 4517 return(addr); 4518} 4519 4520 4521/* 4522 * Map a set of physical memory pages into the kernel virtual 4523 * address space. Return a pointer to where it is mapped. This 4524 * routine is intended to be used for mapping device memory, 4525 * NOT real memory. 4526 */ 4527void * 4528pmap_mapdev(vm_offset_t pa, vm_size_t size) 4529{ 4530 vm_offset_t va, tmpva, offset; 4531 4532 offset = pa & PAGE_MASK; 4533 size = roundup(size, PAGE_SIZE); 4534 4535 GIANT_REQUIRED; 4536 4537 va = kmem_alloc_nofault(kernel_map, size); 4538 if (!va) 4539 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 4540 for (tmpva = va; size > 0;) { 4541 pmap_kenter_internal(tmpva, pa, 0); 4542 size -= PAGE_SIZE; 4543 tmpva += PAGE_SIZE; 4544 pa += PAGE_SIZE; 4545 } 4546 4547 return ((void *)(va + offset)); 4548} 4549 4550#define BOOTSTRAP_DEBUG 4551 4552/* 4553 * pmap_map_section: 4554 * 4555 * Create a single section mapping. 4556 */ 4557void 4558pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4559 int prot, int cache) 4560{ 4561 pd_entry_t *pde = (pd_entry_t *) l1pt; 4562 pd_entry_t fl; 4563 4564 KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); 4565 4566 switch (cache) { 4567 case PTE_NOCACHE: 4568 default: 4569 fl = 0; 4570 break; 4571 4572 case PTE_CACHE: 4573 fl = pte_l1_s_cache_mode; 4574 break; 4575 4576 case PTE_PAGETABLE: 4577 fl = pte_l1_s_cache_mode_pt; 4578 break; 4579 } 4580 4581 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4582 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 4583 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4584 4585} 4586 4587/* 4588 * pmap_link_l2pt: 4589 * 4590 * Link the L2 page table specified by l2pv.pv_pa into the L1 4591 * page table at the slot for "va". 4592 */ 4593void 4594pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) 4595{ 4596 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 4597 u_int slot = va >> L1_S_SHIFT; 4598 4599 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 4600 4601#ifdef VERBOSE_INIT_ARM 4602 printf("pmap_link_l2pt: pa=0x%x va=0x%x\n", l2pv->pv_pa, l2pv->pv_va); 4603#endif 4604 4605 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 4606 4607 PTE_SYNC(&pde[slot]); 4608 4609 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 4610 4611 4612} 4613 4614/* 4615 * pmap_map_entry 4616 * 4617 * Create a single page mapping. 4618 */ 4619void 4620pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, 4621 int cache) 4622{ 4623 pd_entry_t *pde = (pd_entry_t *) l1pt; 4624 pt_entry_t fl; 4625 pt_entry_t *pte; 4626 4627 KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); 4628 4629 switch (cache) { 4630 case PTE_NOCACHE: 4631 default: 4632 fl = 0; 4633 break; 4634 4635 case PTE_CACHE: 4636 fl = pte_l2_s_cache_mode; 4637 break; 4638 4639 case PTE_PAGETABLE: 4640 fl = pte_l2_s_cache_mode_pt; 4641 break; 4642 } 4643 4644 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4645 panic("pmap_map_entry: no L2 table for VA 0x%08x", va); 4646 4647 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4648 4649 if (pte == NULL) 4650 panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); 4651 4652 pte[l2pte_index(va)] = 4653 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 4654 PTE_SYNC(&pte[l2pte_index(va)]); 4655} 4656 4657/* 4658 * pmap_map_chunk: 4659 * 4660 * Map a chunk of memory using the most efficient mappings 4661 * possible (section. large page, small page) into the 4662 * provided L1 and L2 tables at the specified virtual address. 4663 */ 4664vm_size_t 4665pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4666 vm_size_t size, int prot, int cache) 4667{ 4668 pd_entry_t *pde = (pd_entry_t *) l1pt; 4669 pt_entry_t *pte, f1, f2s, f2l; 4670 vm_size_t resid; 4671 int i; 4672 4673 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 4674 4675 if (l1pt == 0) 4676 panic("pmap_map_chunk: no L1 table provided"); 4677 4678#ifdef VERBOSE_INIT_ARM 4679 printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " 4680 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 4681#endif 4682 4683 switch (cache) { 4684 case PTE_NOCACHE: 4685 default: 4686 f1 = 0; 4687 f2l = 0; 4688 f2s = 0; 4689 break; 4690 4691 case PTE_CACHE: 4692 f1 = pte_l1_s_cache_mode; 4693 f2l = pte_l2_l_cache_mode; 4694 f2s = pte_l2_s_cache_mode; 4695 break; 4696 4697 case PTE_PAGETABLE: 4698 f1 = pte_l1_s_cache_mode_pt; 4699 f2l = pte_l2_l_cache_mode_pt; 4700 f2s = pte_l2_s_cache_mode_pt; 4701 break; 4702 } 4703 4704 size = resid; 4705 4706 while (resid > 0) { 4707 /* See if we can use a section mapping. */ 4708 if (L1_S_MAPPABLE_P(va, pa, resid)) { 4709#ifdef VERBOSE_INIT_ARM 4710 printf("S"); 4711#endif 4712 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4713 L1_S_PROT(PTE_KERNEL, prot) | f1 | 4714 L1_S_DOM(PMAP_DOMAIN_KERNEL); 4715 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4716 va += L1_S_SIZE; 4717 pa += L1_S_SIZE; 4718 resid -= L1_S_SIZE; 4719 continue; 4720 } 4721 4722 /* 4723 * Ok, we're going to use an L2 table. Make sure 4724 * one is actually in the corresponding L1 slot 4725 * for the current VA. 4726 */ 4727 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4728 panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); 4729 4730 pte = (pt_entry_t *) kernel_pt_lookup( 4731 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4732 if (pte == NULL) 4733 panic("pmap_map_chunk: can't find L2 table for VA" 4734 "0x%08x", va); 4735 /* See if we can use a L2 large page mapping. */ 4736 if (L2_L_MAPPABLE_P(va, pa, resid)) { 4737#ifdef VERBOSE_INIT_ARM 4738 printf("L"); 4739#endif 4740 for (i = 0; i < 16; i++) { 4741 pte[l2pte_index(va) + i] = 4742 L2_L_PROTO | pa | 4743 L2_L_PROT(PTE_KERNEL, prot) | f2l; 4744 PTE_SYNC(&pte[l2pte_index(va) + i]); 4745 } 4746 va += L2_L_SIZE; 4747 pa += L2_L_SIZE; 4748 resid -= L2_L_SIZE; 4749 continue; 4750 } 4751 4752 /* Use a small page mapping. */ 4753#ifdef VERBOSE_INIT_ARM 4754 printf("P"); 4755#endif 4756 pte[l2pte_index(va)] = 4757 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 4758 PTE_SYNC(&pte[l2pte_index(va)]); 4759 va += PAGE_SIZE; 4760 pa += PAGE_SIZE; 4761 resid -= PAGE_SIZE; 4762 } 4763#ifdef VERBOSE_INIT_ARM 4764 printf("\n"); 4765#endif 4766 return (size); 4767 4768} 4769 4770/********************** Static device map routines ***************************/ 4771 4772static const struct pmap_devmap *pmap_devmap_table; 4773 4774/* 4775 * Register the devmap table. This is provided in case early console 4776 * initialization needs to register mappings created by bootstrap code 4777 * before pmap_devmap_bootstrap() is called. 4778 */ 4779void 4780pmap_devmap_register(const struct pmap_devmap *table) 4781{ 4782 4783 pmap_devmap_table = table; 4784} 4785 4786/* 4787 * Map all of the static regions in the devmap table, and remember 4788 * the devmap table so other parts of the kernel can look up entries 4789 * later. 4790 */ 4791void 4792pmap_devmap_bootstrap(vm_offset_t l1pt, const struct pmap_devmap *table) 4793{ 4794 int i; 4795 4796 pmap_devmap_table = table; 4797 4798 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4799#ifdef VERBOSE_INIT_ARM 4800 printf("devmap: %08x -> %08x @ %08x\n", 4801 pmap_devmap_table[i].pd_pa, 4802 pmap_devmap_table[i].pd_pa + 4803 pmap_devmap_table[i].pd_size - 1, 4804 pmap_devmap_table[i].pd_va); 4805#endif 4806 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 4807 pmap_devmap_table[i].pd_pa, 4808 pmap_devmap_table[i].pd_size, 4809 pmap_devmap_table[i].pd_prot, 4810 pmap_devmap_table[i].pd_cache); 4811 } 4812} 4813 4814const struct pmap_devmap * 4815pmap_devmap_find_pa(vm_paddr_t pa, vm_size_t size) 4816{ 4817 int i; 4818 4819 if (pmap_devmap_table == NULL) 4820 return (NULL); 4821 4822 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4823 if (pa >= pmap_devmap_table[i].pd_pa && 4824 pa + size <= pmap_devmap_table[i].pd_pa + 4825 pmap_devmap_table[i].pd_size) 4826 return (&pmap_devmap_table[i]); 4827 } 4828 4829 return (NULL); 4830} 4831 4832const struct pmap_devmap * 4833pmap_devmap_find_va(vm_offset_t va, vm_size_t size) 4834{ 4835 int i; 4836 4837 if (pmap_devmap_table == NULL) 4838 return (NULL); 4839 4840 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4841 if (va >= pmap_devmap_table[i].pd_va && 4842 va + size <= pmap_devmap_table[i].pd_va + 4843 pmap_devmap_table[i].pd_size) 4844 return (&pmap_devmap_table[i]); 4845 } 4846 4847 return (NULL); 4848} 4849 4850