pmap-v4.c revision 132560
1/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ 2/* 3 * Copyright 2004 Olivier Houchard. 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38/* 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65/*- 66 * Copyright (c) 1999 The NetBSD Foundation, Inc. 67 * All rights reserved. 68 * 69 * This code is derived from software contributed to The NetBSD Foundation 70 * by Charles M. Hannum. 71 * 72 * Redistribution and use in source and binary forms, with or without 73 * modification, are permitted provided that the following conditions 74 * are met: 75 * 1. Redistributions of source code must retain the above copyright 76 * notice, this list of conditions and the following disclaimer. 77 * 2. Redistributions in binary form must reproduce the above copyright 78 * notice, this list of conditions and the following disclaimer in the 79 * documentation and/or other materials provided with the distribution. 80 * 3. All advertising materials mentioning features or use of this software 81 * must display the following acknowledgement: 82 * This product includes software developed by the NetBSD 83 * Foundation, Inc. and its contributors. 84 * 4. Neither the name of The NetBSD Foundation nor the names of its 85 * contributors may be used to endorse or promote products derived 86 * from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 89 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 90 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 91 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 92 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 93 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 94 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 95 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 96 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 97 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 98 * POSSIBILITY OF SUCH DAMAGE. 99 */ 100 101/* 102 * Copyright (c) 1994-1998 Mark Brinicombe. 103 * Copyright (c) 1994 Brini. 104 * All rights reserved. 105 * 106 * This code is derived from software written for Brini by Mark Brinicombe 107 * 108 * Redistribution and use in source and binary forms, with or without 109 * modification, are permitted provided that the following conditions 110 * are met: 111 * 1. Redistributions of source code must retain the above copyright 112 * notice, this list of conditions and the following disclaimer. 113 * 2. Redistributions in binary form must reproduce the above copyright 114 * notice, this list of conditions and the following disclaimer in the 115 * documentation and/or other materials provided with the distribution. 116 * 3. All advertising materials mentioning features or use of this software 117 * must display the following acknowledgement: 118 * This product includes software developed by Mark Brinicombe. 119 * 4. The name of the author may not be used to endorse or promote products 120 * derived from this software without specific prior written permission. 121 * 122 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 123 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 124 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 125 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 126 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 127 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 128 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 129 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 130 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 131 * 132 * RiscBSD kernel project 133 * 134 * pmap.c 135 * 136 * Machine dependant vm stuff 137 * 138 * Created : 20/09/94 139 */ 140 141/* 142 * Special compilation symbols 143 * PMAP_DEBUG - Build in pmap_debug_level code 144 */ 145/* Include header files */ 146#include <sys/cdefs.h> 147__FBSDID("$FreeBSD: head/sys/arm/arm/pmap.c 132560 2004-07-22 23:04:41Z alc $"); 148#include <sys/param.h> 149#include <sys/systm.h> 150#include <sys/kernel.h> 151#include <sys/proc.h> 152#include <sys/malloc.h> 153#include <sys/msgbuf.h> 154#include <sys/vmmeter.h> 155#include <sys/mman.h> 156#include <sys/smp.h> 157#include <sys/sx.h> 158#include <sys/sched.h> 159 160#include <vm/vm.h> 161#include <vm/uma.h> 162#include <vm/pmap.h> 163#include <vm/vm_kern.h> 164#include <vm/vm_object.h> 165#include <vm/vm_map.h> 166#include <vm/vm_page.h> 167#include <vm/vm_pageout.h> 168#include <vm/vm_extern.h> 169#include <sys/lock.h> 170#include <sys/mutex.h> 171#include <machine/md_var.h> 172#include <machine/vmparam.h> 173#include <machine/cpu.h> 174#include <machine/cpufunc.h> 175#include <machine/pcb.h> 176 177#ifdef PMAP_DEBUG 178#define PDEBUG(_lev_,_stat_) \ 179 if (pmap_debug_level >= (_lev_)) \ 180 ((_stat_)) 181#define dprintf printf 182 183int pmap_debug_level = 0; 184#define PMAP_INLINE 185#else /* PMAP_DEBUG */ 186#define PDEBUG(_lev_,_stat_) /* Nothing */ 187#define dprintf(x, arg...) 188#define PMAP_INLINE 189#endif /* PMAP_DEBUG */ 190 191/* 192 * Get PDEs and PTEs for user/kernel address space 193 */ 194#define pdir_pde(m, v) (m[(vm_offset_t)(v) >> PDR_SHIFT]) 195 196#define pmap_pte_prot(m, p) (protection_codes[p]) 197static int protection_codes[8]; 198 199extern struct pv_addr systempage; 200/* 201 * Internal function prototypes 202 */ 203static PMAP_INLINE void pmap_invalidate_page (pmap_t, vm_offset_t); 204#if 0 205static PMAP_INLINE void pmap_invalidate_tlb (pmap_t, vm_offset_t); 206#endif 207static PMAP_INLINE void pmap_invalidate_tlb_all (pmap_t); 208static PMAP_INLINE void pmap_changebit (vm_page_t, int, boolean_t); 209static PMAP_INLINE int pmap_track_modified(vm_offset_t); 210static pt_entry_t * pmap_pte (pmap_t, vm_offset_t); 211static int pmap_unuse_pt (pmap_t, vm_offset_t, vm_page_t); 212static PMAP_INLINE void pmap_free_pv_entry (pv_entry_t); 213static PMAP_INLINE int pmap_unwire_pte_hold(pmap_t, vm_page_t); 214static void arm_protection_init(void); 215static pv_entry_t pmap_get_pv_entry(void); 216 217static void pmap_vac_me_harder(struct vm_page *, pmap_t, 218 vm_offset_t); 219static void pmap_vac_me_kpmap(struct vm_page *, pmap_t, 220 vm_offset_t); 221static void pmap_vac_me_user(struct vm_page *, pmap_t, vm_offset_t); 222static void pmap_alloc_l1(pmap_t); 223static void pmap_free_l1(pmap_t); 224static void pmap_use_l1(pmap_t); 225static PMAP_INLINE boolean_t pmap_is_current(pmap_t); 226static PMAP_INLINE boolean_t pmap_is_cached(pmap_t); 227 228static void pmap_clearbit(struct vm_page *, u_int); 229 230static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); 231static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); 232static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 233static vm_offset_t kernel_pt_lookup(vm_paddr_t); 234 235static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); 236 237vm_offset_t avail_end; /* PA of last available physical page */ 238vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 239vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 240 241extern void *end; 242vm_offset_t kernel_vm_end = 0; 243 244struct pmap kernel_pmap_store; 245pmap_t kernel_pmap; 246 247static pt_entry_t *csrc_pte, *cdst_pte; 248static vm_offset_t csrcp, cdstp; 249static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 250/* 251 * These routines are called when the CPU type is identified to set up 252 * the PTE prototypes, cache modes, etc. 253 * 254 * The variables are always here, just in case LKMs need to reference 255 * them (though, they shouldn't). 256 */ 257 258pt_entry_t pte_l1_s_cache_mode; 259pt_entry_t pte_l1_s_cache_mode_pt; 260pt_entry_t pte_l1_s_cache_mask; 261 262pt_entry_t pte_l2_l_cache_mode; 263pt_entry_t pte_l2_l_cache_mode_pt; 264pt_entry_t pte_l2_l_cache_mask; 265 266pt_entry_t pte_l2_s_cache_mode; 267pt_entry_t pte_l2_s_cache_mode_pt; 268pt_entry_t pte_l2_s_cache_mask; 269 270pt_entry_t pte_l2_s_prot_u; 271pt_entry_t pte_l2_s_prot_w; 272pt_entry_t pte_l2_s_prot_mask; 273 274pt_entry_t pte_l1_s_proto; 275pt_entry_t pte_l1_c_proto; 276pt_entry_t pte_l2_s_proto; 277 278void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); 279void (*pmap_zero_page_func)(vm_paddr_t, int, int); 280/* 281 * Which pmap is currently 'live' in the cache 282 * 283 * XXXSCW: Fix for SMP ... 284 */ 285union pmap_cache_state *pmap_cache_state; 286 287LIST_HEAD(pmaplist, pmap); 288struct pmaplist allpmaps; 289 290static boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 291 292/* static pt_entry_t *msgbufmap;*/ 293struct msgbuf *msgbufp = 0; 294 295extern void bcopy_page(vm_offset_t, vm_offset_t); 296extern void bzero_page(vm_offset_t); 297/* 298 * Metadata for L1 translation tables. 299 */ 300struct l1_ttable { 301 /* Entry on the L1 Table list */ 302 SLIST_ENTRY(l1_ttable) l1_link; 303 304 /* Entry on the L1 Least Recently Used list */ 305 TAILQ_ENTRY(l1_ttable) l1_lru; 306 307 /* Track how many domains are allocated from this L1 */ 308 volatile u_int l1_domain_use_count; 309 310 /* 311 * A free-list of domain numbers for this L1. 312 * We avoid using ffs() and a bitmap to track domains since ffs() 313 * is slow on ARM. 314 */ 315 u_int8_t l1_domain_first; 316 u_int8_t l1_domain_free[PMAP_DOMAINS]; 317 318 /* Physical address of this L1 page table */ 319 vm_paddr_t l1_physaddr; 320 321 /* KVA of this L1 page table */ 322 pd_entry_t *l1_kva; 323}; 324 325/* 326 * Convert a virtual address into its L1 table index. That is, the 327 * index used to locate the L2 descriptor table pointer in an L1 table. 328 * This is basically used to index l1->l1_kva[]. 329 * 330 * Each L2 descriptor table represents 1MB of VA space. 331 */ 332#define L1_IDX(va) (((vm_offset_t)(va)) >> L1_S_SHIFT) 333 334/* 335 * L1 Page Tables are tracked using a Least Recently Used list. 336 * - New L1s are allocated from the HEAD. 337 * - Freed L1s are added to the TAIl. 338 * - Recently accessed L1s (where an 'access' is some change to one of 339 * the userland pmaps which owns this L1) are moved to the TAIL. 340 */ 341static TAILQ_HEAD(, l1_ttable) l1_lru_list; 342static struct mtx l1_lru_lock; 343 344/* 345 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 346 * 347 * This is normally 16MB worth L2 page descriptors for any given pmap. 348 * Reference counts are maintained for L2 descriptors so they can be 349 * freed when empty. 350 */ 351struct l2_dtable { 352 /* The number of L2 page descriptors allocated to this l2_dtable */ 353 u_int l2_occupancy; 354 355 /* List of L2 page descriptors */ 356 struct l2_bucket { 357 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 358 vm_paddr_t l2b_phys; /* Physical address of same */ 359 u_short l2b_l1idx; /* This L2 table's L1 index */ 360 u_short l2b_occupancy; /* How many active descriptors */ 361 } l2_bucket[L2_BUCKET_SIZE]; 362}; 363 364/* 365 * Given an L1 table index, calculate the corresponding l2_dtable index 366 * and bucket index within the l2_dtable. 367 */ 368#define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 369 (L2_SIZE - 1)) 370#define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 371 372/* 373 * Given a virtual address, this macro returns the 374 * virtual address required to drop into the next L2 bucket. 375 */ 376#define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 377 378/* 379 * L2 allocation. 380 */ 381#define pmap_alloc_l2_dtable() \ 382 (void*)uma_zalloc(l2table_zone, M_NOWAIT) 383#define pmap_free_l2_dtable(l2) \ 384 uma_zfree(l2table_zone, l2) 385 386/* 387 * We try to map the page tables write-through, if possible. However, not 388 * all CPUs have a write-through cache mode, so on those we have to sync 389 * the cache when we frob page tables. 390 * 391 * We try to evaluate this at compile time, if possible. However, it's 392 * not always possible to do that, hence this run-time var. 393 */ 394int pmap_needs_pte_sync; 395 396/* 397 * Macro to determine if a mapping might be resident in the 398 * instruction cache and/or TLB 399 */ 400#define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 401 402/* 403 * Macro to determine if a mapping might be resident in the 404 * data cache and/or TLB 405 */ 406#define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 407 408/* 409 * Cache enable bits in PTE to use on pages that are cacheable. 410 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 411 * can chose between write-through and write-back cacheing. 412 */ 413pt_entry_t pte_cache_mode = (PT_C | PT_B); 414 415/* 416 * Data for the pv entry allocation mechanism 417 */ 418#define MINPV 1024 419 420#ifndef PMAP_SHPGPERPROC 421#define PMAP_SHPGPERPROC 200 422#endif 423 424static uma_zone_t pvzone; 425static uma_zone_t l2zone; 426static uma_zone_t l2table_zone; 427static struct vm_object pvzone_obj; 428static struct vm_object l2zone_obj; 429static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; 430int pmap_pagedaemon_waken = 0; 431 432void pmap_deactivate(struct thread *); 433 434void 435pmap_deactivate(struct thread *td) 436{ 437} 438/* 439 * This list exists for the benefit of pmap_map_chunk(). It keeps track 440 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 441 * find them as necessary. 442 * 443 * Note that the data on this list MUST remain valid after initarm() returns, 444 * as pmap_bootstrap() uses it to contruct L2 table metadata. 445 */ 446SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 447 448static void 449pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 450{ 451 int i; 452 453 l1->l1_kva = l1pt; 454 l1->l1_domain_use_count = 0; 455 l1->l1_domain_first = 0; 456 457 for (i = 0; i < PMAP_DOMAINS; i++) 458 l1->l1_domain_free[i] = i + 1; 459 460 /* 461 * Copy the kernel's L1 entries to each new L1. 462 */ 463 if (pmap_initialized) 464 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 465 466 if ((l1->l1_physaddr = pmap_extract(pmap_kernel(), (vm_offset_t)l1pt)) == 0) 467 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 468 if (l1->l1_physaddr & (L1_TABLE_SIZE - 1)) 469 panic("fuck\n"); 470 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 471} 472 473static vm_offset_t 474kernel_pt_lookup(vm_paddr_t pa) 475{ 476 struct pv_addr *pv; 477 478 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 479#ifndef ARM32_NEW_VM_LAYOUT 480 if (pv->pv_pa == (pa & ~PAGE_MASK)) { 481 return (pv->pv_va | (pa & PAGE_MASK)); 482 } 483#else 484 if (pv->pv_pa == pa) 485 return (pv->pv_va); 486#endif 487 } 488 return (0); 489} 490 491#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 492void 493pmap_pte_init_generic(void) 494{ 495 496 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 497 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 498 499 pte_l2_l_cache_mode = L2_B|L2_C; 500 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 501 502 pte_l2_s_cache_mode = L2_B|L2_C; 503 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 504 505 /* 506 * If we have a write-through cache, set B and C. If 507 * we have a write-back cache, then we assume setting 508 * only C will make those pages write-through. 509 */ 510 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { 511 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 512 pte_l2_l_cache_mode_pt = L2_B|L2_C; 513 pte_l2_s_cache_mode_pt = L2_B|L2_C; 514 } else { 515 pte_l1_s_cache_mode_pt = L1_S_C; 516 pte_l2_l_cache_mode_pt = L2_C; 517 pte_l2_s_cache_mode_pt = L2_C; 518 } 519 520 pte_l2_s_prot_u = L2_S_PROT_U_generic; 521 pte_l2_s_prot_w = L2_S_PROT_W_generic; 522 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 523 524 pte_l1_s_proto = L1_S_PROTO_generic; 525 pte_l1_c_proto = L1_C_PROTO_generic; 526 pte_l2_s_proto = L2_S_PROTO_generic; 527 528 pmap_copy_page_func = pmap_copy_page_generic; 529 pmap_zero_page_func = pmap_zero_page_generic; 530} 531 532#if defined(CPU_ARM8) 533void 534pmap_pte_init_arm8(void) 535{ 536 537 /* 538 * ARM8 is compatible with generic, but we need to use 539 * the page tables uncached. 540 */ 541 pmap_pte_init_generic(); 542 543 pte_l1_s_cache_mode_pt = 0; 544 pte_l2_l_cache_mode_pt = 0; 545 pte_l2_s_cache_mode_pt = 0; 546} 547#endif /* CPU_ARM8 */ 548 549#if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 550void 551pmap_pte_init_arm9(void) 552{ 553 554 /* 555 * ARM9 is compatible with generic, but we want to use 556 * write-through caching for now. 557 */ 558 pmap_pte_init_generic(); 559 560 pte_l1_s_cache_mode = L1_S_C; 561 pte_l2_l_cache_mode = L2_C; 562 pte_l2_s_cache_mode = L2_C; 563 564 pte_l1_s_cache_mode_pt = L1_S_C; 565 pte_l2_l_cache_mode_pt = L2_C; 566 pte_l2_s_cache_mode_pt = L2_C; 567} 568#endif /* CPU_ARM9 */ 569#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 570 571#if defined(CPU_ARM10) 572void 573pmap_pte_init_arm10(void) 574{ 575 576 /* 577 * ARM10 is compatible with generic, but we want to use 578 * write-through caching for now. 579 */ 580 pmap_pte_init_generic(); 581 582 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 583 pte_l2_l_cache_mode = L2_B | L2_C; 584 pte_l2_s_cache_mode = L2_B | L2_C; 585 586 pte_l1_s_cache_mode_pt = L1_S_C; 587 pte_l2_l_cache_mode_pt = L2_C; 588 pte_l2_s_cache_mode_pt = L2_C; 589 590} 591#endif /* CPU_ARM10 */ 592 593#if ARM_MMU_SA1 == 1 594void 595pmap_pte_init_sa1(void) 596{ 597 598 /* 599 * The StrongARM SA-1 cache does not have a write-through 600 * mode. So, do the generic initialization, then reset 601 * the page table cache mode to B=1,C=1, and note that 602 * the PTEs need to be sync'd. 603 */ 604 pmap_pte_init_generic(); 605 606 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 607 pte_l2_l_cache_mode_pt = L2_B|L2_C; 608 pte_l2_s_cache_mode_pt = L2_B|L2_C; 609 610 pmap_needs_pte_sync = 1; 611} 612#endif /* ARM_MMU_SA1 == 1*/ 613 614#if ARM_MMU_XSCALE == 1 615#if (ARM_NMMUS > 1) 616static u_int xscale_use_minidata; 617#endif 618 619void 620pmap_pte_init_xscale(void) 621{ 622 uint32_t auxctl; 623 int write_through = 0; 624 625 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 626 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 627 628 pte_l2_l_cache_mode = L2_B|L2_C; 629 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 630 631 pte_l2_s_cache_mode = L2_B|L2_C; 632 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 633 634 pte_l1_s_cache_mode_pt = L1_S_C; 635 pte_l2_l_cache_mode_pt = L2_C; 636 pte_l2_s_cache_mode_pt = L2_C; 637 638#ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 639 /* 640 * The XScale core has an enhanced mode where writes that 641 * miss the cache cause a cache line to be allocated. This 642 * is significantly faster than the traditional, write-through 643 * behavior of this case. 644 */ 645 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 646 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 647 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 648#endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 649 650#ifdef XSCALE_CACHE_WRITE_THROUGH 651 /* 652 * Some versions of the XScale core have various bugs in 653 * their cache units, the work-around for which is to run 654 * the cache in write-through mode. Unfortunately, this 655 * has a major (negative) impact on performance. So, we 656 * go ahead and run fast-and-loose, in the hopes that we 657 * don't line up the planets in a way that will trip the 658 * bugs. 659 * 660 * However, we give you the option to be slow-but-correct. 661 */ 662 write_through = 1; 663#elif defined(XSCALE_CACHE_WRITE_BACK) 664 /* force write back cache mode */ 665 write_through = 0; 666#elif defined(CPU_XSCALE_PXA2X0) 667 /* 668 * Intel PXA2[15]0 processors are known to have a bug in 669 * write-back cache on revision 4 and earlier (stepping 670 * A[01] and B[012]). Fixed for C0 and later. 671 */ 672 { 673 uint32_t id, type; 674 675 id = cpufunc_id(); 676 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 677 678 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 679 if ((id & CPU_ID_REVISION_MASK) < 5) { 680 /* write through for stepping A0-1 and B0-2 */ 681 write_through = 1; 682 } 683 } 684 } 685#endif /* XSCALE_CACHE_WRITE_THROUGH */ 686 687 if (write_through) { 688 pte_l1_s_cache_mode = L1_S_C; 689 pte_l2_l_cache_mode = L2_C; 690 pte_l2_s_cache_mode = L2_C; 691 } 692 693#if (ARM_NMMUS > 1) 694 xscale_use_minidata = 1; 695#endif 696 697 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 698 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 699 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 700 701 pte_l1_s_proto = L1_S_PROTO_xscale; 702 pte_l1_c_proto = L1_C_PROTO_xscale; 703 pte_l2_s_proto = L2_S_PROTO_xscale; 704 705 pmap_copy_page_func = pmap_copy_page_xscale; 706 pmap_zero_page_func = pmap_zero_page_xscale; 707 708 /* 709 * Disable ECC protection of page table access, for now. 710 */ 711 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 712 auxctl &= ~XSCALE_AUXCTL_P; 713 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 714} 715 716/* 717 * xscale_setup_minidata: 718 * 719 * Set up the mini-data cache clean area. We require the 720 * caller to allocate the right amount of physically and 721 * virtually contiguous space. 722 */ 723extern vm_offset_t xscale_minidata_clean_addr; 724extern vm_size_t xscale_minidata_clean_size; /* already initialized */ 725void 726xscale_setup_minidata(vm_offset_t l1pt, vm_offset_t va, vm_paddr_t pa) 727{ 728 pd_entry_t *pde = (pd_entry_t *) l1pt; 729 pt_entry_t *pte; 730 vm_size_t size; 731 uint32_t auxctl; 732 733 xscale_minidata_clean_addr = va; 734 735 /* Round it to page size. */ 736 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 737 738 for (; size != 0; 739 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 740#ifndef ARM32_NEW_VM_LAYOUT 741 pte = (pt_entry_t *) 742 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 743#else 744 pte = (pt_entry_t *) kernel_pt_lookup( 745 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 746#endif 747 if (pte == NULL) 748 panic("xscale_setup_minidata: can't find L2 table for " 749 "VA 0x%08x", (u_int32_t) va); 750#ifndef ARM32_NEW_VM_LAYOUT 751 pte[(va >> PAGE_SHIFT) & 0x3ff] = 752#else 753 pte[l2pte_index(va)] = 754#endif 755 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 756 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 757 } 758 759 /* 760 * Configure the mini-data cache for write-back with 761 * read/write-allocate. 762 * 763 * NOTE: In order to reconfigure the mini-data cache, we must 764 * make sure it contains no valid data! In order to do that, 765 * we must issue a global data cache invalidate command! 766 * 767 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 768 * THIS IS VERY IMPORTANT! 769 */ 770 771 /* Invalidate data and mini-data. */ 772 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 773 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 774 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 775 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 776} 777#endif 778 779/* 780 * Allocate an L1 translation table for the specified pmap. 781 * This is called at pmap creation time. 782 */ 783static void 784pmap_alloc_l1(pmap_t pm) 785{ 786 struct l1_ttable *l1; 787 u_int8_t domain; 788 789 /* 790 * Remove the L1 at the head of the LRU list 791 */ 792 mtx_lock(&l1_lru_lock); 793 l1 = TAILQ_FIRST(&l1_lru_list); 794 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 795 796 /* 797 * Pick the first available domain number, and update 798 * the link to the next number. 799 */ 800 domain = l1->l1_domain_first; 801 l1->l1_domain_first = l1->l1_domain_free[domain]; 802 803 /* 804 * If there are still free domain numbers in this L1, 805 * put it back on the TAIL of the LRU list. 806 */ 807 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 808 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 809 810 mtx_unlock(&l1_lru_lock); 811 812 /* 813 * Fix up the relevant bits in the pmap structure 814 */ 815 pm->pm_l1 = l1; 816 pm->pm_domain = domain; 817} 818 819/* 820 * Free an L1 translation table. 821 * This is called at pmap destruction time. 822 */ 823static void 824pmap_free_l1(pmap_t pm) 825{ 826 struct l1_ttable *l1 = pm->pm_l1; 827 828 mtx_lock(&l1_lru_lock); 829 830 /* 831 * If this L1 is currently on the LRU list, remove it. 832 */ 833 if (l1->l1_domain_use_count < PMAP_DOMAINS) 834 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 835 836 /* 837 * Free up the domain number which was allocated to the pmap 838 */ 839 l1->l1_domain_free[pm->pm_domain] = l1->l1_domain_first; 840 l1->l1_domain_first = pm->pm_domain; 841 l1->l1_domain_use_count--; 842 843 /* 844 * The L1 now must have at least 1 free domain, so add 845 * it back to the LRU list. If the use count is zero, 846 * put it at the head of the list, otherwise it goes 847 * to the tail. 848 */ 849 if (l1->l1_domain_use_count == 0) { 850 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 851 } else 852 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 853 854 mtx_unlock(&l1_lru_lock); 855} 856 857static PMAP_INLINE void 858pmap_use_l1(pmap_t pm) 859{ 860 struct l1_ttable *l1; 861 862 /* 863 * Do nothing if we're in interrupt context. 864 * Access to an L1 by the kernel pmap must not affect 865 * the LRU list. 866 */ 867 if (pm == pmap_kernel()) 868 return; 869 870 l1 = pm->pm_l1; 871 872 /* 873 * If the L1 is not currently on the LRU list, just return 874 */ 875 if (l1->l1_domain_use_count == PMAP_DOMAINS) 876 return; 877 878 mtx_lock(&l1_lru_lock); 879 880 /* 881 * Check the use count again, now that we've acquired the lock 882 */ 883 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 884 mtx_unlock(&l1_lru_lock); 885 return; 886 } 887 888 /* 889 * Move the L1 to the back of the LRU list 890 */ 891 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 892 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 893 894 mtx_unlock(&l1_lru_lock); 895} 896 897 898/* 899 * Returns a pointer to the L2 bucket associated with the specified pmap 900 * and VA, or NULL if no L2 bucket exists for the address. 901 */ 902static PMAP_INLINE struct l2_bucket * 903pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) 904{ 905 struct l2_dtable *l2; 906 struct l2_bucket *l2b; 907 u_short l1idx; 908 909 l1idx = L1_IDX(va); 910 911 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 912 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 913 return (NULL); 914 915 return (l2b); 916} 917 918/* 919 * Returns a pointer to the L2 bucket associated with the specified pmap 920 * and VA. 921 * 922 * If no L2 bucket exists, perform the necessary allocations to put an L2 923 * bucket/page table in place. 924 * 925 * Note that if a new L2 bucket/page was allocated, the caller *must* 926 * increment the bucket occupancy counter appropriately *before* 927 * releasing the pmap's lock to ensure no other thread or cpu deallocates 928 * the bucket/page in the meantime. 929 */ 930static struct l2_bucket * 931pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) 932{ 933 struct l2_dtable *l2; 934 struct l2_bucket *l2b; 935 u_short l1idx; 936 937 l1idx = L1_IDX(va); 938 939 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 940 /* 941 * No mapping at this address, as there is 942 * no entry in the L1 table. 943 * Need to allocate a new l2_dtable. 944 */ 945 if ((l2 = pmap_alloc_l2_dtable()) == NULL) { 946 return (NULL); 947 } 948 bzero(l2, sizeof(*l2)); 949 /* 950 * Link it into the parent pmap 951 */ 952 pm->pm_l2[L2_IDX(l1idx)] = l2; 953 bzero(l2, sizeof( struct l2_dtable)); 954 } 955 956 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 957 958 /* 959 * Fetch pointer to the L2 page table associated with the address. 960 */ 961 if (l2b->l2b_kva == NULL) { 962 pt_entry_t *ptep; 963 964 /* 965 * No L2 page table has been allocated. Chances are, this 966 * is because we just allocated the l2_dtable, above. 967 */ 968 ptep = (void*)uma_zalloc(l2zone, M_NOWAIT); 969 l2b->l2b_phys = vtophys(ptep); 970 if (ptep == NULL) { 971 /* 972 * Oops, no more L2 page tables available at this 973 * time. We may need to deallocate the l2_dtable 974 * if we allocated a new one above. 975 */ 976 if (l2->l2_occupancy == 0) { 977 pm->pm_l2[L2_IDX(l1idx)] = NULL; 978 pmap_free_l2_dtable(l2); 979 } 980 return (NULL); 981 } 982 983 l2->l2_occupancy++; 984 l2b->l2b_kva = ptep; 985 l2b->l2b_l1idx = l1idx; 986 } 987 988 return (l2b); 989} 990 991static PMAP_INLINE void 992#ifndef PMAP_INCLUDE_PTE_SYNC 993pmap_free_l2_ptp(pt_entry_t *l2) 994#else 995pmap_free_l2_ptp(boolean_t need_sync, pt_entry_t *l2) 996#endif 997{ 998#ifdef PMAP_INCLUDE_PTE_SYNC 999 /* 1000 * Note: With a write-back cache, we may need to sync this 1001 * L2 table before re-using it. 1002 * This is because it may have belonged to a non-current 1003 * pmap, in which case the cache syncs would have been 1004 * skipped when the pages were being unmapped. If the 1005 * L2 table were then to be immediately re-allocated to 1006 * the *current* pmap, it may well contain stale mappings 1007 * which have not yet been cleared by a cache write-back 1008 * and so would still be visible to the mmu. 1009 */ 1010 if (need_sync) 1011 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1012#endif 1013 uma_zfree(l2zone, l2); 1014} 1015/* 1016 * One or more mappings in the specified L2 descriptor table have just been 1017 * invalidated. 1018 * 1019 * Garbage collect the metadata and descriptor table itself if necessary. 1020 * 1021 * The pmap lock must be acquired when this is called (not necessary 1022 * for the kernel pmap). 1023 */ 1024static void 1025pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1026{ 1027 struct l2_dtable *l2; 1028 pd_entry_t *pl1pd, l1pd; 1029 pt_entry_t *ptep; 1030 u_short l1idx; 1031 1032 1033 /* 1034 * Update the bucket's reference count according to how many 1035 * PTEs the caller has just invalidated. 1036 */ 1037 l2b->l2b_occupancy -= count; 1038 1039 /* 1040 * Note: 1041 * 1042 * Level 2 page tables allocated to the kernel pmap are never freed 1043 * as that would require checking all Level 1 page tables and 1044 * removing any references to the Level 2 page table. See also the 1045 * comment elsewhere about never freeing bootstrap L2 descriptors. 1046 * 1047 * We make do with just invalidating the mapping in the L2 table. 1048 * 1049 * This isn't really a big deal in practice and, in fact, leads 1050 * to a performance win over time as we don't need to continually 1051 * alloc/free. 1052 */ 1053 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1054 return; 1055 1056 /* 1057 * There are no more valid mappings in this level 2 page table. 1058 * Go ahead and NULL-out the pointer in the bucket, then 1059 * free the page table. 1060 */ 1061 l1idx = l2b->l2b_l1idx; 1062 ptep = l2b->l2b_kva; 1063 l2b->l2b_kva = NULL; 1064 1065 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1066 1067 /* 1068 * If the L1 slot matches the pmap's domain 1069 * number, then invalidate it. 1070 */ 1071 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 1072 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 1073 *pl1pd = 0; 1074 PTE_SYNC(pl1pd); 1075 } 1076 1077 /* 1078 * Release the L2 descriptor table back to the pool cache. 1079 */ 1080#ifndef PMAP_INCLUDE_PTE_SYNC 1081 pmap_free_l2_ptp(ptep); 1082#else 1083 pmap_free_l2_ptp(!pmap_is_cached(pm), ptep); 1084#endif 1085 1086 /* 1087 * Update the reference count in the associated l2_dtable 1088 */ 1089 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1090 if (--l2->l2_occupancy > 0) 1091 return; 1092 1093 /* 1094 * There are no more valid mappings in any of the Level 1 1095 * slots managed by this l2_dtable. Go ahead and NULL-out 1096 * the pointer in the parent pmap and free the l2_dtable. 1097 */ 1098 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1099 pmap_free_l2_dtable(l2); 1100} 1101 1102/* 1103 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1104 * structures. 1105 */ 1106static void 1107pmap_l2ptp_ctor(void *mem, int size, void *arg) 1108{ 1109#ifndef PMAP_INCLUDE_PTE_SYNC 1110 struct l2_bucket *l2b; 1111 pt_entry_t *ptep, pte; 1112 vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; 1113 1114 /* 1115 * The mappings for these page tables were initially made using 1116 * pmap_kenter_pa() by the pool subsystem. Therefore, the cache- 1117 * mode will not be right for page table mappings. To avoid 1118 * polluting the pmap_kenter_pa() code with a special case for 1119 * page tables, we simply fix up the cache-mode here if it's not 1120 * correct. 1121 */ 1122 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1123 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1124 pte = *ptep; 1125 1126 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1127 /* 1128 * Page tables must have the cache-mode set to Write-Thru. 1129 */ 1130 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1131 PTE_SYNC(ptep); 1132 cpu_tlb_flushD_SE(va); 1133 cpu_cpwait(); 1134 } 1135#endif 1136 1137 memset(mem, 0, L2_TABLE_SIZE_REAL); 1138 PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1139} 1140 1141/* 1142 * A bunch of routines to conditionally flush the caches/TLB depending 1143 * on whether the specified pmap actually needs to be flushed at any 1144 * given time. 1145 */ 1146static PMAP_INLINE void 1147pmap_tlb_flushID_SE(pmap_t pm, vm_offset_t va) 1148{ 1149 1150 if (pm->pm_cstate.cs_tlb_id) 1151 cpu_tlb_flushID_SE(va); 1152} 1153 1154static PMAP_INLINE void 1155pmap_tlb_flushD_SE(pmap_t pm, vm_offset_t va) 1156{ 1157 1158 if (pm->pm_cstate.cs_tlb_d) 1159 cpu_tlb_flushD_SE(va); 1160} 1161 1162static PMAP_INLINE void 1163pmap_tlb_flushID(pmap_t pm) 1164{ 1165 1166 if (pm->pm_cstate.cs_tlb_id) { 1167 cpu_tlb_flushID(); 1168 pm->pm_cstate.cs_tlb = 0; 1169 } 1170} 1171static PMAP_INLINE void 1172pmap_tlb_flushD(pmap_t pm) 1173{ 1174 1175 if (pm->pm_cstate.cs_tlb_d) { 1176 cpu_tlb_flushD(); 1177 pm->pm_cstate.cs_tlb_d = 0; 1178 } 1179} 1180 1181static PMAP_INLINE void 1182pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) 1183{ 1184 1185 if (pm->pm_cstate.cs_cache_id) 1186 cpu_idcache_wbinv_range(va, len); 1187} 1188 1189static PMAP_INLINE void 1190pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, 1191 boolean_t do_inv, boolean_t rd_only) 1192{ 1193 1194 if (pm->pm_cstate.cs_cache_d) { 1195 if (do_inv) { 1196 if (rd_only) 1197 cpu_dcache_inv_range(va, len); 1198 else 1199 cpu_dcache_wbinv_range(va, len); 1200 } else 1201 if (!rd_only) 1202 cpu_dcache_wb_range(va, len); 1203 } 1204} 1205 1206static PMAP_INLINE void 1207pmap_idcache_wbinv_all(pmap_t pm) 1208{ 1209 1210 if (pm->pm_cstate.cs_cache_id) { 1211 cpu_idcache_wbinv_all(); 1212 pm->pm_cstate.cs_cache = 0; 1213 } 1214} 1215 1216static PMAP_INLINE void 1217pmap_dcache_wbinv_all(pmap_t pm) 1218{ 1219 1220 if (pm->pm_cstate.cs_cache_d) { 1221 cpu_dcache_wbinv_all(); 1222 pm->pm_cstate.cs_cache_d = 0; 1223 } 1224} 1225 1226static PMAP_INLINE boolean_t 1227pmap_is_current(pmap_t pm) 1228{ 1229 1230 if (pm == pmap_kernel() || 1231 (curproc && curproc->p_vmspace->vm_map.pmap == pm)) 1232 return (TRUE); 1233 1234 return (FALSE); 1235} 1236 1237static PMAP_INLINE boolean_t 1238pmap_is_cached(pmap_t pm) 1239{ 1240 1241 if (pm == pmap_kernel() || pmap_cache_state == NULL || 1242 pmap_cache_state == &pm->pm_cstate) 1243 return (TRUE); 1244 1245 return (FALSE); 1246} 1247 1248/* 1249 * PTE_SYNC_CURRENT: 1250 * 1251 * Make sure the pte is written out to RAM. 1252 * We need to do this for one of two cases: 1253 * - We're dealing with the kernel pmap 1254 * - There is no pmap active in the cache/tlb. 1255 * - The specified pmap is 'active' in the cache/tlb. 1256 */ 1257#ifdef PMAP_INCLUDE_PTE_SYNC 1258#define PTE_SYNC_CURRENT(pm, ptep) \ 1259do { \ 1260 if (PMAP_NEEDS_PTE_SYNC && \ 1261 pmap_is_cached(pm)) \ 1262 PTE_SYNC(ptep); \ 1263} while (/*CONSTCOND*/0) 1264#else 1265#define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ 1266#endif 1267 1268/* 1269 * Since we have a virtually indexed cache, we may need to inhibit caching if 1270 * there is more than one mapping and at least one of them is writable. 1271 * Since we purge the cache on every context switch, we only need to check for 1272 * other mappings within the same pmap, or kernel_pmap. 1273 * This function is also called when a page is unmapped, to possibly reenable 1274 * caching on any remaining mappings. 1275 * 1276 * The code implements the following logic, where: 1277 * 1278 * KW = # of kernel read/write pages 1279 * KR = # of kernel read only pages 1280 * UW = # of user read/write pages 1281 * UR = # of user read only pages 1282 * 1283 * KC = kernel mapping is cacheable 1284 * UC = user mapping is cacheable 1285 * 1286 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1287 * +--------------------------------------------- 1288 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1289 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1290 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1291 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1292 */ 1293 1294static const int pmap_vac_flags[4][4] = { 1295 {-1, 0, 0, PVF_KNC}, 1296 {0, 0, PVF_NC, PVF_NC}, 1297 {0, PVF_NC, PVF_NC, PVF_NC}, 1298 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1299}; 1300 1301static PMAP_INLINE int 1302pmap_get_vac_flags(const struct vm_page *pg) 1303{ 1304 int kidx, uidx; 1305 1306 kidx = 0; 1307 if (pg->md.kro_mappings || pg->md.krw_mappings > 1) 1308 kidx |= 1; 1309 if (pg->md.krw_mappings) 1310 kidx |= 2; 1311 1312 uidx = 0; 1313 if (pg->md.uro_mappings || pg->md.urw_mappings > 1) 1314 uidx |= 1; 1315 if (pg->md.urw_mappings) 1316 uidx |= 2; 1317 1318 return (pmap_vac_flags[uidx][kidx]); 1319} 1320 1321static __inline void 1322pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1323{ 1324 int nattr; 1325 1326 nattr = pmap_get_vac_flags(pg); 1327 1328 if (nattr < 0) { 1329 pg->md.pvh_attrs &= ~PVF_NC; 1330 return; 1331 } 1332 1333 if (nattr == 0 && (pg->md.pvh_attrs & PVF_NC) == 0) { 1334 return; 1335 } 1336 1337 if (pm == pmap_kernel()) 1338 pmap_vac_me_kpmap(pg, pm, va); 1339 else 1340 pmap_vac_me_user(pg, pm, va); 1341 1342 pg->md.pvh_attrs = (pg->md.pvh_attrs & ~PVF_NC) | nattr; 1343} 1344 1345static void 1346pmap_vac_me_kpmap(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1347{ 1348 u_int u_cacheable, u_entries; 1349 struct pv_entry *pv; 1350 pmap_t last_pmap = pm; 1351 1352 /* 1353 * Pass one, see if there are both kernel and user pmaps for 1354 * this page. Calculate whether there are user-writable or 1355 * kernel-writable pages. 1356 */ 1357 u_cacheable = 0; 1358 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1359 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1360 u_cacheable++; 1361 } 1362 1363 u_entries = pg->md.urw_mappings + pg->md.uro_mappings; 1364 1365 /* 1366 * We know we have just been updating a kernel entry, so if 1367 * all user pages are already cacheable, then there is nothing 1368 * further to do. 1369 */ 1370 if (pg->md.k_mappings == 0 && u_cacheable == u_entries) 1371 return; 1372 1373 if (u_entries) { 1374 /* 1375 * Scan over the list again, for each entry, if it 1376 * might not be set correctly, call pmap_vac_me_user 1377 * to recalculate the settings. 1378 */ 1379 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1380 /* 1381 * We know kernel mappings will get set 1382 * correctly in other calls. We also know 1383 * that if the pmap is the same as last_pmap 1384 * then we've just handled this entry. 1385 */ 1386 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1387 continue; 1388 1389 /* 1390 * If there are kernel entries and this page 1391 * is writable but non-cacheable, then we can 1392 * skip this entry also. 1393 */ 1394 if (pg->md.k_mappings && 1395 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1396 (PVF_NC | PVF_WRITE)) 1397 continue; 1398 1399 /* 1400 * Similarly if there are no kernel-writable 1401 * entries and the page is already 1402 * read-only/cacheable. 1403 */ 1404 if (pg->md.krw_mappings == 0 && 1405 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1406 continue; 1407 1408 /* 1409 * For some of the remaining cases, we know 1410 * that we must recalculate, but for others we 1411 * can't tell if they are correct or not, so 1412 * we recalculate anyway. 1413 */ 1414 pmap_vac_me_user(pg, (last_pmap = pv->pv_pmap), 0); 1415 } 1416 1417 if (pg->md.k_mappings == 0) 1418 return; 1419 } 1420 1421 pmap_vac_me_user(pg, pm, va); 1422} 1423 1424static void 1425pmap_vac_me_user(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1426{ 1427 pmap_t kpmap = pmap_kernel(); 1428 struct pv_entry *pv, *npv; 1429 struct l2_bucket *l2b; 1430 pt_entry_t *ptep, pte; 1431 u_int entries = 0; 1432 u_int writable = 0; 1433 u_int cacheable_entries = 0; 1434 u_int kern_cacheable = 0; 1435 u_int other_writable = 0; 1436 1437 /* 1438 * Count mappings and writable mappings in this pmap. 1439 * Include kernel mappings as part of our own. 1440 * Keep a pointer to the first one. 1441 */ 1442 npv = TAILQ_FIRST(&pg->md.pv_list); 1443 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1444 /* Count mappings in the same pmap */ 1445 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1446 if (entries++ == 0) 1447 npv = pv; 1448 1449 /* Cacheable mappings */ 1450 if ((pv->pv_flags & PVF_NC) == 0) { 1451 cacheable_entries++; 1452 if (kpmap == pv->pv_pmap) 1453 kern_cacheable++; 1454 } 1455 1456 /* Writable mappings */ 1457 if (pv->pv_flags & PVF_WRITE) 1458 ++writable; 1459 } else 1460 if (pv->pv_flags & PVF_WRITE) 1461 other_writable = 1; 1462 } 1463 1464 /* 1465 * Enable or disable caching as necessary. 1466 * Note: the first entry might be part of the kernel pmap, 1467 * so we can't assume this is indicative of the state of the 1468 * other (maybe non-kpmap) entries. 1469 */ 1470 if ((entries > 1 && writable) || 1471 (entries > 0 && pm == kpmap && other_writable)) { 1472 if (cacheable_entries == 0) 1473 return; 1474 1475 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1476 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1477 (pv->pv_flags & PVF_NC)) 1478 continue; 1479 1480 pv->pv_flags |= PVF_NC; 1481 1482 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1483 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1484 pte = *ptep & ~L2_S_CACHE_MASK; 1485 1486 if ((va != pv->pv_va || pm != pv->pv_pmap) && 1487 l2pte_valid(pte)) { 1488 if (PV_BEEN_EXECD(pv->pv_flags)) { 1489 pmap_idcache_wbinv_range(pv->pv_pmap, 1490 pv->pv_va, PAGE_SIZE); 1491 pmap_tlb_flushID_SE(pv->pv_pmap, 1492 pv->pv_va); 1493 } else 1494 if (PV_BEEN_REFD(pv->pv_flags)) { 1495 pmap_dcache_wb_range(pv->pv_pmap, 1496 pv->pv_va, PAGE_SIZE, TRUE, 1497 (pv->pv_flags & PVF_WRITE) == 0); 1498 pmap_tlb_flushD_SE(pv->pv_pmap, 1499 pv->pv_va); 1500 } 1501 } 1502 1503 *ptep = pte; 1504 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1505 } 1506 cpu_cpwait(); 1507 } else 1508 if (entries > cacheable_entries) { 1509 /* 1510 * Turn cacheing back on for some pages. If it is a kernel 1511 * page, only do so if there are no other writable pages. 1512 */ 1513 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1514 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1515 (kpmap != pv->pv_pmap || other_writable))) 1516 continue; 1517 1518 pv->pv_flags &= ~PVF_NC; 1519 1520 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1521 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1522 pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; 1523 1524 if (l2pte_valid(pte)) { 1525 if (PV_BEEN_EXECD(pv->pv_flags)) { 1526 pmap_tlb_flushID_SE(pv->pv_pmap, 1527 pv->pv_va); 1528 } else 1529 if (PV_BEEN_REFD(pv->pv_flags)) { 1530 pmap_tlb_flushD_SE(pv->pv_pmap, 1531 pv->pv_va); 1532 } 1533 } 1534 1535 *ptep = pte; 1536 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1537 } 1538 } 1539} 1540 1541/* 1542 * Modify pte bits for all ptes corresponding to the given physical address. 1543 * We use `maskbits' rather than `clearbits' because we're always passing 1544 * constants and the latter would require an extra inversion at run-time. 1545 */ 1546static void 1547pmap_clearbit(struct vm_page *pg, u_int maskbits) 1548{ 1549 struct l2_bucket *l2b; 1550 struct pv_entry *pv; 1551 pt_entry_t *ptep, npte, opte; 1552 pmap_t pm; 1553 vm_offset_t va; 1554 u_int oflags; 1555 1556#if 0 1557 PMAP_HEAD_TO_MAP_LOCK(); 1558 simple_lock(&pg->mdpage.pvh_slock); 1559#endif 1560 1561 /* 1562 * Clear saved attributes (modify, reference) 1563 */ 1564 pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 1565 1566 if (TAILQ_EMPTY(&pg->md.pv_list)) { 1567#if 0 1568 simple_unlock(&pg->mdpage.pvh_slock); 1569 PMAP_HEAD_TO_MAP_UNLOCK(); 1570#endif 1571 return; 1572 } 1573 1574 /* 1575 * Loop over all current mappings setting/clearing as appropos 1576 */ 1577 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1578 va = pv->pv_va; 1579 pm = pv->pv_pmap; 1580 oflags = pv->pv_flags; 1581 pv->pv_flags &= ~maskbits; 1582 1583#if 0 1584 pmap_acquire_pmap_lock(pm); 1585#endif 1586 1587 l2b = pmap_get_l2_bucket(pm, va); 1588 1589 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1590 npte = opte = *ptep; 1591 1592 if (maskbits & (PVF_WRITE|PVF_MOD)) { 1593 if ((pv->pv_flags & PVF_NC)) { 1594 /* 1595 * Entry is not cacheable: 1596 * 1597 * Don't turn caching on again if this is a 1598 * modified emulation. This would be 1599 * inconsitent with the settings created by 1600 * pmap_vac_me_harder(). Otherwise, it's safe 1601 * to re-enable cacheing. 1602 * 1603 * There's no need to call pmap_vac_me_harder() 1604 * here: all pages are losing their write 1605 * permission. 1606 */ 1607 if (maskbits & PVF_WRITE) { 1608 npte |= pte_l2_s_cache_mode; 1609 pv->pv_flags &= ~PVF_NC; 1610 } 1611 } else 1612 if (opte & L2_S_PROT_W) { 1613 /* 1614 * Entry is writable/cacheable: check if pmap 1615 * is current if it is flush it, otherwise it 1616 * won't be in the cache 1617 */ 1618 if (PV_BEEN_EXECD(oflags)) 1619 pmap_idcache_wbinv_range(pm, pv->pv_va, 1620 PAGE_SIZE); 1621 else 1622 if (PV_BEEN_REFD(oflags)) 1623 pmap_dcache_wb_range(pm, pv->pv_va, 1624 PAGE_SIZE, 1625 (maskbits & PVF_REF) ? TRUE : FALSE, 1626 FALSE); 1627 } 1628 1629 /* make the pte read only */ 1630 npte &= ~L2_S_PROT_W; 1631 1632 if (maskbits & PVF_WRITE) { 1633 /* 1634 * Keep alias accounting up to date 1635 */ 1636 if (pv->pv_pmap == pmap_kernel()) { 1637 if (oflags & PVF_WRITE) { 1638 pg->md.krw_mappings--; 1639 pg->md.kro_mappings++; 1640 } 1641 } else 1642 if (oflags & PVF_WRITE) { 1643 pg->md.urw_mappings--; 1644 pg->md.uro_mappings++; 1645 } 1646 } 1647 } 1648 1649 if (maskbits & PVF_REF) { 1650 if ((pv->pv_flags & PVF_NC) == 0 && 1651 (maskbits & (PVF_WRITE|PVF_MOD)) == 0) { 1652 /* 1653 * Check npte here; we may have already 1654 * done the wbinv above, and the validity 1655 * of the PTE is the same for opte and 1656 * npte. 1657 */ 1658 if (npte & L2_S_PROT_W) { 1659 if (PV_BEEN_EXECD(oflags)) 1660 pmap_idcache_wbinv_range(pm, 1661 pv->pv_va, PAGE_SIZE); 1662 else 1663 if (PV_BEEN_REFD(oflags)) 1664 pmap_dcache_wb_range(pm, 1665 pv->pv_va, PAGE_SIZE, 1666 TRUE, FALSE); 1667 } else 1668 if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) { 1669 /* XXXJRT need idcache_inv_range */ 1670 if (PV_BEEN_EXECD(oflags)) 1671 pmap_idcache_wbinv_range(pm, 1672 pv->pv_va, PAGE_SIZE); 1673 else 1674 if (PV_BEEN_REFD(oflags)) 1675 pmap_dcache_wb_range(pm, 1676 pv->pv_va, PAGE_SIZE, 1677 TRUE, TRUE); 1678 } 1679 } 1680 1681 /* 1682 * Make the PTE invalid so that we will take a 1683 * page fault the next time the mapping is 1684 * referenced. 1685 */ 1686 npte &= ~L2_TYPE_MASK; 1687 npte |= L2_TYPE_INV; 1688 } 1689 1690 if (npte != opte) { 1691 *ptep = npte; 1692 PTE_SYNC(ptep); 1693 /* Flush the TLB entry if a current pmap. */ 1694 if (PV_BEEN_EXECD(oflags)) 1695 pmap_tlb_flushID_SE(pm, pv->pv_va); 1696 else 1697 if (PV_BEEN_REFD(oflags)) 1698 pmap_tlb_flushD_SE(pm, pv->pv_va); 1699 } 1700 1701#if 0 1702 pmap_release_pmap_lock(pm); 1703#endif 1704 1705 } 1706 1707#if 0 1708 simple_unlock(&pg->mdpage.pvh_slock); 1709 PMAP_HEAD_TO_MAP_UNLOCK(); 1710#endif 1711} 1712 1713/* 1714 * main pv_entry manipulation functions: 1715 * pmap_enter_pv: enter a mapping onto a vm_page list 1716 * pmap_remove_pv: remove a mappiing from a vm_page list 1717 * 1718 * NOTE: pmap_enter_pv expects to lock the pvh itself 1719 * pmap_remove_pv expects te caller to lock the pvh before calling 1720 */ 1721 1722/* 1723 * pmap_enter_pv: enter a mapping onto a vm_page lst 1724 * 1725 * => caller should hold the proper lock on pmap_main_lock 1726 * => caller should have pmap locked 1727 * => we will gain the lock on the vm_page and allocate the new pv_entry 1728 * => caller should adjust ptp's wire_count before calling 1729 * => caller should not adjust pmap's wire_count 1730 */ 1731static void 1732pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 1733 vm_offset_t va, u_int flags) 1734{ 1735 1736 1737 pve->pv_pmap = pm; 1738 pve->pv_va = va; 1739 pve->pv_flags = flags; 1740 1741#if 0 1742 mtx_lock(&pg->md.pvh_mtx); 1743 TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); 1744#endif 1745 1746 TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); 1747 pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 1748 if (pm == pmap_kernel()) { 1749 if (flags & PVF_WRITE) 1750 pg->md.krw_mappings++; 1751 else 1752 pg->md.kro_mappings++; 1753 } 1754 if (flags & PVF_WRITE) 1755 pg->md.urw_mappings++; 1756 else 1757 pg->md.uro_mappings++; 1758#if 0 1759 mtx_unlock(&pg->md.pvh_mtx); 1760#endif 1761 if (pve->pv_flags & PVF_WIRED) 1762 ++pm->pm_stats.wired_count; 1763} 1764 1765/* 1766 * 1767 * pmap_find_pv: Find a pv entry 1768 * 1769 * => caller should hold lock on vm_page 1770 */ 1771static PMAP_INLINE struct pv_entry * 1772pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1773{ 1774 struct pv_entry *pv; 1775 1776 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) 1777 if (pm == pv->pv_pmap && va == pv->pv_va) 1778 break; 1779 return (pv); 1780} 1781 1782/* 1783 * vector_page_setprot: 1784 * 1785 * Manipulate the protection of the vector page. 1786 */ 1787void 1788vector_page_setprot(int prot) 1789{ 1790 struct l2_bucket *l2b; 1791 pt_entry_t *ptep; 1792 1793 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 1794 1795 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 1796 1797 *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 1798 PTE_SYNC(ptep); 1799 cpu_tlb_flushD_SE(vector_page); 1800 cpu_cpwait(); 1801} 1802 1803/* 1804 * pmap_remove_pv: try to remove a mapping from a pv_list 1805 * 1806 * => caller should hold proper lock on pmap_main_lock 1807 * => pmap should be locked 1808 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1809 * => caller should adjust ptp's wire_count and free PTP if needed 1810 * => caller should NOT adjust pmap's wire_count 1811 * => we return the removed pve 1812 */ 1813static struct pv_entry * 1814pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1815{ 1816 struct pv_entry *pve, **prevptr; 1817 1818 1819 prevptr = &TAILQ_FIRST(&pg->md.pv_list);/* previous pv_entry pointer */ 1820 pve = *prevptr; 1821 1822 while (pve) { 1823 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 1824 *prevptr = TAILQ_NEXT(pve, pv_list); /* remove it! */ 1825 if (pve->pv_flags & PVF_WIRED) 1826 --pm->pm_stats.wired_count; 1827 if (pm == pmap_kernel()) { 1828 if (pve->pv_flags & PVF_WRITE) 1829 pg->md.krw_mappings--; 1830 else 1831 pg->md.kro_mappings--; 1832 } else 1833 if (pve->pv_flags & PVF_WRITE) 1834 pg->md.urw_mappings--; 1835 else 1836 pg->md.uro_mappings--; 1837 break; 1838 } 1839 prevptr = &TAILQ_NEXT(pve, pv_list); 1840 pve = TAILQ_NEXT(pve, pv_list); 1841 } 1842 1843 return(pve); /* return removed pve */ 1844} 1845/* 1846 * 1847 * pmap_modify_pv: Update pv flags 1848 * 1849 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1850 * => caller should NOT adjust pmap's wire_count 1851 * => caller must call pmap_vac_me_harder() if writable status of a page 1852 * may have changed. 1853 * => we return the old flags 1854 * 1855 * Modify a physical-virtual mapping in the pv table 1856 */ 1857static u_int 1858pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, 1859 u_int clr_mask, u_int set_mask) 1860{ 1861 struct pv_entry *npv; 1862 u_int flags, oflags; 1863 1864 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1865 return (0); 1866 1867 /* 1868 * There is at least one VA mapping this page. 1869 */ 1870 1871 if (clr_mask & (PVF_REF | PVF_MOD)) 1872 pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1873 1874 oflags = npv->pv_flags; 1875 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1876 1877 if ((flags ^ oflags) & PVF_WIRED) { 1878 if (flags & PVF_WIRED) 1879 ++pm->pm_stats.wired_count; 1880 else 1881 --pm->pm_stats.wired_count; 1882 } 1883 1884 if ((flags ^ oflags) & PVF_WRITE) { 1885 if (pm == pmap_kernel()) { 1886 if (flags & PVF_WRITE) { 1887 pg->md.krw_mappings++; 1888 pg->md.kro_mappings--; 1889 } else { 1890 pg->md.kro_mappings++; 1891 pg->md.krw_mappings--; 1892 } 1893 } else 1894 if (flags & PVF_WRITE) { 1895 pg->md.urw_mappings++; 1896 pg->md.uro_mappings--; 1897 } else { 1898 pg->md.uro_mappings++; 1899 pg->md.urw_mappings--; 1900 } 1901 } 1902 1903 return (oflags); 1904} 1905 1906/* Function to set the debug level of the pmap code */ 1907#ifdef PMAP_DEBUG 1908void 1909pmap_debug(int level) 1910{ 1911 pmap_debug_level = level; 1912 dprintf("pmap_debug: level=%d\n", pmap_debug_level); 1913} 1914#endif /* PMAP_DEBUG */ 1915 1916 1917void 1918pmap_pinit0(struct pmap *pmap) 1919{ 1920 PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); 1921 1922 dprintf("pmap_pinit0: pmap = %08x, pm_pdir = %08x\n", 1923 (u_int32_t) pmap, (u_int32_t) pmap->pm_pdir); 1924 pmap_pinit(pmap); 1925} 1926 1927 1928/* 1929 * Initialize the pmap module. 1930 * Called by vm_init, to initialize any structures that the pmap 1931 * system needs to map virtual memory. 1932 * pmap_init has been enhanced to support in a fairly consistant 1933 * way, discontiguous physical memory. 1934 */ 1935void 1936pmap_init(void) 1937{ 1938 int i; 1939 1940 PDEBUG(1, printf("pmap_init: phys_start = %08x\n")); 1941 /* 1942 * Allocate memory for random pmap data structures. Includes the 1943 * pv_head_table. 1944 */ 1945 for(i = 0; i < vm_page_array_size; i++) { 1946 vm_page_t m; 1947 1948 m = &vm_page_array[i]; 1949 TAILQ_INIT(&m->md.pv_list); 1950 m->md.pv_list_count = 0; 1951 } 1952 1953 /* 1954 * init the pv free list 1955 */ 1956 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 1957 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1958 uma_prealloc(pvzone, MINPV); 1959 l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), 1960 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1961 UMA_ZONE_VM | UMA_ZONE_NOFREE); 1962 /* 1963 * Now it is safe to enable pv_table recording. 1964 */ 1965 pmap_initialized = TRUE; 1966 PDEBUG(1, printf("pmap_init: done!\n")); 1967} 1968 1969int 1970pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) 1971{ 1972 struct l2_dtable *l2; 1973 struct l2_bucket *l2b; 1974 pd_entry_t *pl1pd, l1pd; 1975 pt_entry_t *ptep, pte; 1976 vm_paddr_t pa; 1977 u_int l1idx; 1978 int rv = 0; 1979 1980#if 0 1981 PMAP_MAP_TO_HEAD_LOCK(); 1982 pmap_acquire_pmap_lock(pm); 1983#endif 1984 l1idx = L1_IDX(va); 1985 1986 /* 1987 * If there is no l2_dtable for this address, then the process 1988 * has no business accessing it. 1989 * 1990 * Note: This will catch userland processes trying to access 1991 * kernel addresses. 1992 */ 1993 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1994 if (l2 == NULL) 1995 goto out; 1996 1997 /* 1998 * Likewise if there is no L2 descriptor table 1999 */ 2000 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2001 if (l2b->l2b_kva == NULL) 2002 goto out; 2003 2004 /* 2005 * Check the PTE itself. 2006 */ 2007 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2008 pte = *ptep; 2009 if (pte == 0) 2010 goto out; 2011 2012 /* 2013 * Catch a userland access to the vector page mapped at 0x0 2014 */ 2015 if (user && (pte & L2_S_PROT_U) == 0) 2016 goto out; 2017 2018 pa = l2pte_pa(pte); 2019 2020 if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { 2021 /* 2022 * This looks like a good candidate for "page modified" 2023 * emulation... 2024 */ 2025 struct pv_entry *pv; 2026 struct vm_page *pg; 2027 2028 /* Extract the physical address of the page */ 2029 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 2030 goto out; 2031 } 2032 /* Get the current flags for this page. */ 2033 2034 pv = pmap_find_pv(pg, pm, va); 2035 if (pv == NULL) { 2036 goto out; 2037 } 2038 2039 /* 2040 * Do the flags say this page is writable? If not then it 2041 * is a genuine write fault. If yes then the write fault is 2042 * our fault as we did not reflect the write access in the 2043 * PTE. Now we know a write has occurred we can correct this 2044 * and also set the modified bit 2045 */ 2046 if ((pv->pv_flags & PVF_WRITE) == 0) { 2047 goto out; 2048 } 2049 2050 pg->md.pvh_attrs |= PVF_REF | PVF_MOD; 2051 pv->pv_flags |= PVF_REF | PVF_MOD; 2052 2053 /* 2054 * Re-enable write permissions for the page. No need to call 2055 * pmap_vac_me_harder(), since this is just a 2056 * modified-emulation fault, and the PVF_WRITE bit isn't 2057 * changing. We've already set the cacheable bits based on 2058 * the assumption that we can write to this page. 2059 */ 2060 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 2061 PTE_SYNC(ptep); 2062 rv = 1; 2063 } else 2064 if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 2065 /* 2066 * This looks like a good candidate for "page referenced" 2067 * emulation. 2068 */ 2069 struct pv_entry *pv; 2070 struct vm_page *pg; 2071 2072 /* Extract the physical address of the page */ 2073 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2074 goto out; 2075 2076 /* Get the current flags for this page. */ 2077 2078 pv = pmap_find_pv(pg, pm, va); 2079 if (pv == NULL) { 2080 goto out; 2081 } 2082 2083 pg->md.pvh_attrs |= PVF_REF; 2084 pv->pv_flags |= PVF_REF; 2085 2086 2087 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 2088 PTE_SYNC(ptep); 2089 rv = 1; 2090 } 2091 2092 /* 2093 * We know there is a valid mapping here, so simply 2094 * fix up the L1 if necessary. 2095 */ 2096 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2097 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 2098 if (*pl1pd != l1pd) { 2099 *pl1pd = l1pd; 2100 PTE_SYNC(pl1pd); 2101 rv = 1; 2102 } 2103 2104#ifdef CPU_SA110 2105 /* 2106 * There are bugs in the rev K SA110. This is a check for one 2107 * of them. 2108 */ 2109 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 2110 curcpu()->ci_arm_cpurev < 3) { 2111 /* Always current pmap */ 2112 if (l2pte_valid(pte)) { 2113 extern int kernel_debug; 2114 if (kernel_debug & 1) { 2115 struct proc *p = curlwp->l_proc; 2116 printf("prefetch_abort: page is already " 2117 "mapped - pte=%p *pte=%08x\n", ptep, pte); 2118 printf("prefetch_abort: pc=%08lx proc=%p " 2119 "process=%s\n", va, p, p->p_comm); 2120 printf("prefetch_abort: far=%08x fs=%x\n", 2121 cpu_faultaddress(), cpu_faultstatus()); 2122 } 2123#ifdef DDB 2124 if (kernel_debug & 2) 2125 Debugger(); 2126#endif 2127 rv = 1; 2128 } 2129 } 2130#endif /* CPU_SA110 */ 2131 2132#ifdef DEBUG 2133 /* 2134 * If 'rv == 0' at this point, it generally indicates that there is a 2135 * stale TLB entry for the faulting address. This happens when two or 2136 * more processes are sharing an L1. Since we don't flush the TLB on 2137 * a context switch between such processes, we can take domain faults 2138 * for mappings which exist at the same VA in both processes. EVEN IF 2139 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 2140 * example. 2141 * 2142 * This is extremely likely to happen if pmap_enter() updated the L1 2143 * entry for a recently entered mapping. In this case, the TLB is 2144 * flushed for the new mapping, but there may still be TLB entries for 2145 * other mappings belonging to other processes in the 1MB range 2146 * covered by the L1 entry. 2147 * 2148 * Since 'rv == 0', we know that the L1 already contains the correct 2149 * value, so the fault must be due to a stale TLB entry. 2150 * 2151 * Since we always need to flush the TLB anyway in the case where we 2152 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 2153 * stale TLB entries dynamically. 2154 * 2155 * However, the above condition can ONLY happen if the current L1 is 2156 * being shared. If it happens when the L1 is unshared, it indicates 2157 * that other parts of the pmap are not doing their job WRT managing 2158 * the TLB. 2159 */ 2160 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 2161 extern int last_fault_code; 2162 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 2163 pm, va, ftype); 2164 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 2165 l2, l2b, ptep, pl1pd); 2166 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 2167 pte, l1pd, last_fault_code); 2168#ifdef DDB 2169 Debugger(); 2170#endif 2171 } 2172#endif 2173 2174 cpu_tlb_flushID_SE(va); 2175 cpu_cpwait(); 2176 2177 rv = 1; 2178 2179out: 2180#if 0 2181 pmap_release_pmap_lock(pm); 2182 PMAP_MAP_TO_HEAD_UNLOCK(); 2183#endif 2184 return (rv); 2185} 2186 2187/* 2188 * Initialize the address space (zone) for the pv_entries. Set a 2189 * high water mark so that the system can recover from excessive 2190 * numbers of pv entries. 2191 */ 2192void 2193pmap_init2() 2194{ 2195 int shpgperproc = PMAP_SHPGPERPROC; 2196 struct l2_bucket *l2b; 2197 struct l1_ttable *l1; 2198 pd_entry_t *pl1pt; 2199 pt_entry_t *ptep, pte; 2200 vm_offset_t va, eva; 2201 u_int loop, needed; 2202 int i; 2203 2204 2205 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 2206 2207 pv_entry_max = shpgperproc * maxproc + vm_page_array_size; 2208 pv_entry_high_water = 9 * (pv_entry_max / 10); 2209 l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, 2210 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 2211 uma_prealloc(l2zone, 512); 2212 2213 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 2214 uma_zone_set_obj(l2zone, &l2zone_obj, pv_entry_max); 2215 2216 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 2217 needed -= 1; 2218 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 2219 2220 for (loop = 0; loop < needed; loop++, l1++) { 2221 /* Allocate a L1 page table */ 2222 va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 2223 0xffffffff, L1_TABLE_SIZE, 0); 2224 2225 if (va == 0) 2226 panic("Cannot allocate L1 KVM"); 2227 2228 2229 eva = va + L1_TABLE_SIZE; 2230 pl1pt = (pd_entry_t *)va; 2231 2232 for (i = 0; i < (L1_TABLE_SIZE / PAGE_SIZE) && va < eva; i++) { 2233 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2234 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2235 pte = *ptep; 2236 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 2237 *ptep = pte; 2238 PTE_SYNC(ptep); 2239 cpu_tlb_flushD_SE(va); 2240 2241 va += PAGE_SIZE; 2242 } 2243 pmap_init_l1(l1, pl1pt); 2244 } 2245 2246 2247#ifdef DEBUG 2248 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 2249 needed); 2250#endif 2251} 2252 2253/* 2254 * This is used to stuff certain critical values into the PCB where they 2255 * can be accessed quickly from cpu_switch() et al. 2256 */ 2257void 2258pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 2259{ 2260 struct l2_bucket *l2b; 2261 2262 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 2263 pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 2264 (DOMAIN_CLIENT << (pm->pm_domain * 2)); 2265 pcb->pcb_cstate = (void *)&pm->pm_cstate; 2266 2267 if (vector_page < KERNBASE) { 2268 pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 2269 l2b = pmap_get_l2_bucket(pm, vector_page); 2270 pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | 2271 L1_C_DOM(pm->pm_domain); 2272 } else 2273 pcb->pcb_pl1vec = NULL; 2274} 2275 2276void 2277pmap_activate(struct thread *td) 2278{ 2279 pmap_t pm; 2280 struct pcb *pcb; 2281 int s; 2282 2283 pm = td->td_proc->p_vmspace->vm_map.pmap; 2284 pcb = td->td_pcb; 2285 2286 critical_enter(); 2287 pmap_set_pcb_pagedir(pm, pcb); 2288 2289 if (td == curthread) { 2290 u_int cur_dacr, cur_ttb; 2291 2292 __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 2293 __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); 2294 2295 cur_ttb &= ~(L1_TABLE_SIZE - 1); 2296 2297 if (cur_ttb == (u_int)pcb->pcb_pagedir && 2298 cur_dacr == pcb->pcb_dacr) { 2299 /* 2300 * No need to switch address spaces. 2301 */ 2302 critical_exit(); 2303 return; 2304 } 2305 2306 disable_interrupts(I32_bit | F32_bit); 2307 2308 /* 2309 * We MUST, I repeat, MUST fix up the L1 entry corresponding 2310 * to 'vector_page' in the incoming L1 table before switching 2311 * to it otherwise subsequent interrupts/exceptions (including 2312 * domain faults!) will jump into hyperspace. 2313 */ 2314 if (pcb->pcb_pl1vec) { 2315 2316 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 2317 /* 2318 * Don't need to PTE_SYNC() at this point since 2319 * cpu_setttb() is about to flush both the cache 2320 * and the TLB. 2321 */ 2322 } 2323 2324 cpu_domains(pcb->pcb_dacr); 2325 cpu_setttb(pcb->pcb_pagedir); 2326 2327 enable_interrupts(I32_bit | F32_bit); 2328 2329 /* 2330 * Flag any previous userland pmap as being NOT 2331 * resident in the cache/tlb. 2332 */ 2333 if (pmap_cache_state && pmap_cache_state != &pm->pm_cstate) 2334 pmap_cache_state->cs_all = 0; 2335 2336 /* 2337 * The new pmap, however, IS resident. 2338 */ 2339 pmap_cache_state = &pm->pm_cstate; 2340 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 2341 splx(s); 2342 } 2343 critical_exit(); 2344} 2345 2346static int 2347pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) 2348{ 2349 pd_entry_t *pdep, pde; 2350 pt_entry_t *ptep, pte; 2351 vm_offset_t pa; 2352 int rv = 0; 2353 2354 /* 2355 * Make sure the descriptor itself has the correct cache mode 2356 */ 2357 pdep = &kl1[L1_IDX(va)]; 2358 pde = *pdep; 2359 2360 if (l1pte_section_p(pde)) { 2361 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 2362 *pdep = (pde & ~L1_S_CACHE_MASK) | 2363 pte_l1_s_cache_mode_pt; 2364 PTE_SYNC(pdep); 2365 cpu_dcache_wbinv_range((vm_offset_t)pdep, 2366 sizeof(*pdep)); 2367 rv = 1; 2368 } 2369 } else { 2370 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2371 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2372 if (ptep == NULL) 2373 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 2374 2375 ptep = &ptep[l2pte_index(va)]; 2376 pte = *ptep; 2377 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 2378 *ptep = (pte & ~L2_S_CACHE_MASK) | 2379 pte_l2_s_cache_mode_pt; 2380 PTE_SYNC(ptep); 2381 cpu_dcache_wbinv_range((vm_offset_t)ptep, 2382 sizeof(*ptep)); 2383 rv = 1; 2384 } 2385 } 2386 2387 return (rv); 2388} 2389 2390static void 2391pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, 2392 pt_entry_t **ptep) 2393{ 2394 vm_offset_t va = *availp; 2395 struct l2_bucket *l2b; 2396 2397 if (ptep) { 2398 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2399 if (l2b == NULL) 2400 panic("pmap_alloc_specials: no l2b for 0x%x", va); 2401 2402 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 2403 } 2404 2405 *vap = va; 2406 *availp = va + (PAGE_SIZE * pages); 2407} 2408 2409/* 2410 * Bootstrap the system enough to run with virtual memory. 2411 * 2412 * On the arm this is called after mapping has already been enabled 2413 * and just syncs the pmap module with what has already been done. 2414 * [We can't call it easily with mapping off since the kernel is not 2415 * mapped with PA == VA, hence we would have to relocate every address 2416 * from the linked base (virtual) address "KERNBASE" to the actual 2417 * (physical) address starting relative to 0] 2418 */ 2419#define PMAP_STATIC_L2_SIZE 16 2420void 2421pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt) 2422{ 2423 static struct l1_ttable static_l1; 2424 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 2425 struct l1_ttable *l1 = &static_l1; 2426 struct l2_dtable *l2; 2427 struct l2_bucket *l2b; 2428 pd_entry_t pde; 2429 pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; 2430 pt_entry_t *ptep; 2431 vm_paddr_t pa; 2432 vm_offset_t va; 2433 int l1idx, l2idx, l2next = 0; 2434 2435 PDEBUG(1, printf("firstaddr = %08x, loadaddr = %08x\n", 2436 firstaddr, loadaddr)); 2437 2438 virtual_avail = firstaddr; 2439 kernel_pmap = &kernel_pmap_store; 2440 kernel_pmap->pm_l1 = l1; 2441/* 2442 * Scan the L1 translation table created by initarm() and create 2443 * the required metadata for all valid mappings found in it. 2444 */ 2445 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 2446 pde = kernel_l1pt[l1idx]; 2447 2448 /* 2449 * We're only interested in Coarse mappings. 2450 * pmap_extract() can deal with section mappings without 2451 * recourse to checking L2 metadata. 2452 */ 2453 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 2454 continue; 2455 2456 /* 2457 * Lookup the KVA of this L2 descriptor table 2458 */ 2459 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2460 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2461 2462 if (ptep == NULL) { 2463 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 2464 (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); 2465 } 2466 2467 /* 2468 * Fetch the associated L2 metadata structure. 2469 * Allocate a new one if necessary. 2470 */ 2471 if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { 2472 if (l2next == PMAP_STATIC_L2_SIZE) 2473 panic("pmap_bootstrap: out of static L2s"); 2474 kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = 2475 &static_l2[l2next++]; 2476 } 2477 2478 /* 2479 * One more L1 slot tracked... 2480 */ 2481 l2->l2_occupancy++; 2482 2483 /* 2484 * Fill in the details of the L2 descriptor in the 2485 * appropriate bucket. 2486 */ 2487 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2488 l2b->l2b_kva = ptep; 2489 l2b->l2b_phys = pa; 2490 l2b->l2b_l1idx = l1idx; 2491 2492 /* 2493 * Establish an initial occupancy count for this descriptor 2494 */ 2495 for (l2idx = 0; 2496 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 2497 l2idx++) { 2498 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 2499 l2b->l2b_occupancy++; 2500 } 2501 } 2502 2503 /* 2504 * Make sure the descriptor itself has the correct cache mode. 2505 * If not, fix it, but whine about the problem. Port-meisters 2506 * should consider this a clue to fix up their initarm() 2507 * function. :) 2508 */ 2509 if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { 2510 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2511 "L2 pte @ %p\n", ptep); 2512 } 2513 } 2514 2515 2516 /* 2517 * Initialize protection array. 2518 */ 2519 arm_protection_init(); 2520 2521 /* 2522 * Ensure the primary (kernel) L1 has the correct cache mode for 2523 * a page table. Bitch if it is not correctly set. 2524 */ 2525 for (va = (vm_offset_t)kernel_l1pt; 2526 va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 2527 if (pmap_set_pt_cache_mode(kernel_l1pt, va)) 2528 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2529 "primary L1 @ 0x%x\n", va); 2530 } 2531 2532 cpu_dcache_wbinv_all(); 2533 cpu_tlb_flushID(); 2534 cpu_cpwait(); 2535 2536 kernel_pmap->pm_active = -1; 2537 kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; 2538 TAILQ_INIT(&kernel_pmap->pm_pvlist); 2539 LIST_INIT(&allpmaps); 2540 LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); 2541 2542 /* 2543 * Reserve some special page table entries/VA space for temporary 2544 * mapping of pages. 2545 */ 2546#define SYSMAP(c, p, v, n) \ 2547 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 2548 2549 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 2550 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); 2551 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 2552 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); 2553 TAILQ_INIT(&l1_lru_list); 2554 mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); 2555 pmap_init_l1(l1, kernel_l1pt); 2556 cpu_dcache_wbinv_all(); 2557 2558 virtual_avail = round_page(virtual_avail); 2559 virtual_end = lastaddr; 2560 kernel_vm_end = virtual_end; 2561} 2562 2563/*************************************************** 2564 * Pmap allocation/deallocation routines. 2565 ***************************************************/ 2566 2567/* 2568 * Release any resources held by the given physical map. 2569 * Called when a pmap initialized by pmap_pinit is being released. 2570 * Should only be called if the map contains no valid mappings. 2571 */ 2572void 2573pmap_release(pmap_t pmap) 2574{ 2575 pmap_free_l1(pmap); 2576 dprintf("pmap_release()\n"); 2577} 2578 2579 2580/* 2581 * grow the number of kernel page table entries, if needed 2582 */ 2583void 2584pmap_growkernel(vm_offset_t addr) 2585{ 2586 2587} 2588 2589 2590/* 2591 * pmap_page_protect: 2592 * 2593 * Lower the permission for all mappings to a given page. 2594 */ 2595void 2596pmap_page_protect(vm_page_t m, vm_prot_t prot) 2597{ 2598 2599 if ((prot & VM_PROT_WRITE) == 0) { 2600 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 2601 pmap_changebit(m, AP_KRWURW, FALSE); 2602 } else { 2603 pmap_remove_all(m); 2604 } 2605 } 2606} 2607 2608 2609#define PMAP_REMOVE_PAGES_CURPROC_ONLY 2610/* 2611 * Remove all pages from specified address space 2612 * this aids process exit speeds. Also, this code 2613 * is special cased for current process only, but 2614 * can have the more generic (and slightly slower) 2615 * mode enabled. This is much faster than pmap_remove 2616 * in the case of running down an entire address space. 2617 */ 2618void 2619pmap_remove_pages(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) 2620{ 2621 struct l2_bucket *l2b; 2622 pt_entry_t *pte, tpte; 2623 pv_entry_t pv, npv; 2624 vm_page_t m; 2625 2626#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY 2627 if (!curproc || (pmap != vmspace_pmap(curproc->p_vmspace))) { 2628 printf("warning: pmap_remove_pages called with non-current pmap\n"); 2629 return; 2630 } 2631#endif 2632 2633 vm_page_lock_queues(); 2634 for(pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2635 if (pv->pv_va >= eva || pv->pv_va < sva) { 2636 npv = TAILQ_NEXT(pv, pv_plist); 2637 continue; 2638 } 2639 2640 /* 2641 * We cannot remove a wired pages from a process' mapping 2642 * at this time 2643 */ 2644 if (pv->pv_flags & PT_W) { 2645 npv = TAILQ_NEXT(pv, pv_plist); 2646 continue; 2647 } 2648 l2b = pmap_get_l2_bucket(pmap_kernel(), pv->pv_va); 2649 pte = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2650 tpte = *pte; 2651 *pte = 0; 2652 2653 m = PHYS_TO_VM_PAGE(tpte); 2654 2655 KASSERT(m < &vm_page_array[vm_page_array_size], 2656 ("pmap_remove_pages: bad tpte %x", tpte)); 2657 2658 pv->pv_pmap->pm_stats.resident_count--; 2659 2660 /* 2661 * Update the vm_page_t clean and reference bits. 2662 */ 2663 vm_page_dirty(m); 2664 2665 npv = TAILQ_NEXT(pv, pv_plist); 2666 TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); 2667 2668 m->md.pv_list_count--; 2669 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2670 if (TAILQ_FIRST(&m->md.pv_list) == NULL) 2671 vm_page_flag_clear(m, PG_WRITEABLE); 2672 2673 pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); 2674 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2675 pmap_free_pv_entry(pv); 2676 } 2677 pmap_invalidate_tlb_all(pmap); 2678 vm_page_unlock_queues(); 2679} 2680 2681 2682/*************************************************** 2683 * Low level mapping routines..... 2684 ***************************************************/ 2685 2686/* 2687 * add a wired page to the kva 2688 * note that in order for the mapping to take effect -- you 2689 * should do a invltlb after doing the pmap_kenter... 2690 */ 2691PMAP_INLINE void 2692pmap_kenter(vm_offset_t va, vm_offset_t pa) 2693{ 2694 struct l2_bucket *l2b; 2695 pt_entry_t *pte; 2696 pt_entry_t opte; 2697 2698 PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", 2699 (uint32_t) va, (uint32_t) pa)); 2700 2701 2702 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2703 KASSERT(l2b != NULL, ("No L2 Bucket")); 2704 pte = &l2b->l2b_kva[l2pte_index(va)]; 2705 opte = *pte; 2706 PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", 2707 (uint32_t) pte, opte, *pte)); 2708 if (l2pte_valid(opte)) { 2709 cpu_dcache_wbinv_range(va, PAGE_SIZE); 2710 cpu_tlb_flushD_SE(va); 2711 cpu_cpwait(); 2712 } else 2713 if (opte == 0) 2714 l2b->l2b_occupancy++; 2715 *pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, 2716 VM_PROT_READ | VM_PROT_WRITE) | pte_l2_s_cache_mode; 2717 PTE_SYNC(pte); 2718 2719} 2720 2721 2722/* 2723 * remove a page from the kernel pagetables 2724 */ 2725PMAP_INLINE void 2726pmap_kremove(vm_offset_t va) 2727{ 2728 pt_entry_t *pte; 2729 2730 pte = (pt_entry_t *)vtopte(va); 2731 *pte = 0; 2732 pmap_invalidate_page(kernel_pmap, va); 2733} 2734 2735 2736/* 2737 * Used to map a range of physical addresses into kernel 2738 * virtual address space. 2739 * 2740 * The value passed in '*virt' is a suggested virtual address for 2741 * the mapping. Architectures which can support a direct-mapped 2742 * physical to virtual region can return the appropriate address 2743 * within that region, leaving '*virt' unchanged. Other 2744 * architectures should map the pages starting at '*virt' and 2745 * update '*virt' with the first usable address after the mapped 2746 * region. 2747 */ 2748vm_offset_t 2749pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 2750{ 2751 vm_offset_t sva = *virt; 2752 vm_offset_t va = sva; 2753 2754 PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " 2755 "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, 2756 prot)); 2757 2758 while (start < end) { 2759 pmap_kenter(va, start); 2760 va += PAGE_SIZE; 2761 start += PAGE_SIZE; 2762 } 2763 *virt = va; 2764 return (sva); 2765} 2766 2767 2768/* 2769 * Add a list of wired pages to the kva 2770 * this routine is only used for temporary 2771 * kernel mappings that do not need to have 2772 * page modification or references recorded. 2773 * Note that old mappings are simply written 2774 * over. The page *must* be wired. 2775 */ 2776void 2777pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 2778{ 2779 int i; 2780 2781 for (i = 0; i < count; i++) { 2782 pmap_kenter(va, VM_PAGE_TO_PHYS(m[i])); 2783 va += PAGE_SIZE; 2784 } 2785} 2786 2787 2788/* 2789 * this routine jerks page mappings from the 2790 * kernel -- it is meant only for temporary mappings. 2791 */ 2792void 2793pmap_qremove(vm_offset_t va, int count) 2794{ 2795 int i; 2796 2797 for (i = 0; i < count; i++) { 2798 pmap_kremove(va); 2799 va += PAGE_SIZE; 2800 } 2801} 2802 2803 2804/* 2805 * pmap_object_init_pt preloads the ptes for a given object 2806 * into the specified pmap. This eliminates the blast of soft 2807 * faults on process startup and immediately after an mmap. 2808 */ 2809void 2810pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 2811 vm_pindex_t pindex, vm_size_t size) 2812{ 2813 printf("pmap_object_init_pt()\n"); 2814} 2815 2816 2817/* 2818 * pmap_is_prefaultable: 2819 * 2820 * Return whether or not the specified virtual address is elgible 2821 * for prefault. 2822 */ 2823boolean_t 2824pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 2825{ 2826 return (FALSE); 2827} 2828 2829static PMAP_INLINE void 2830pmap_invalidate_page(pmap_t pmap, vm_offset_t va) 2831{ 2832 /* TODO: Invalidate I+D_SE */ 2833 __asm("mcr p15, 0, r0, c7, c7, 0"); 2834} 2835 2836#if 0 2837static PMAP_INLINE void 2838pmap_invalidate_tlb(pmap_t pmap, vm_offset_t va) 2839{ 2840 __asm("mcr p15, 0, r0, c8, c7, 0"); 2841 /* TODO: Invalidate TLB */ 2842} 2843#endif 2844 2845static PMAP_INLINE void 2846pmap_invalidate_tlb_all(pmap_t pmap) 2847{ 2848 __asm("mcr p15, 0, r0, c8, c7, 0"); 2849 /* TODO: Invalidate all TLB */ 2850} 2851 2852static PMAP_INLINE void 2853pmap_changebit(vm_page_t m, int bit, boolean_t setem) 2854{ 2855 pv_entry_t pv; 2856 pt_entry_t *pte; 2857 int s; 2858 2859 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 2860 return; 2861 2862 s = splvm(); 2863 2864 /* 2865 * Loop over all current mappings setting/clearing as appropos 2866 */ 2867 for (pv = TAILQ_FIRST(&m->md.pv_list); 2868 pv; 2869 pv = TAILQ_NEXT(pv, pv_list)) { 2870 2871 /* 2872 * don't write protect pager mappings 2873 */ 2874 if (!setem && bit == AP_KRWURW) { 2875 if (!pmap_track_modified(pv->pv_va)) 2876 continue; 2877 } 2878 2879#if defined(PMAP_DEBUG) 2880 if (!pv->pv_pmap) { 2881 printf("Null pmap (cb) at va: 0x%x\n", (uint32_t) pv->pv_va); 2882 continue; 2883 } 2884#endif 2885 2886 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2887 2888 if (setem) { 2889 *pte |= bit; 2890 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2891 } else { 2892 pt_entry_t pbits = *pte; 2893 if (pbits & bit) { 2894 *pte = pbits & ~bit; 2895 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2896 } 2897 } 2898 } 2899 splx(s); 2900} 2901 2902/* 2903 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 2904 * Returns TRUE if the mapping exists, else FALSE. 2905 * 2906 * NOTE: This function is only used by a couple of arm-specific modules. 2907 * It is not safe to take any pmap locks here, since we could be right 2908 * in the middle of debugging the pmap anyway... 2909 * 2910 * It is possible for this routine to return FALSE even though a valid 2911 * mapping does exist. This is because we don't lock, so the metadata 2912 * state may be inconsistent. 2913 * 2914 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 2915 * a "section" mapping. 2916 */ 2917boolean_t 2918pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) 2919{ 2920 struct l2_dtable *l2; 2921 pd_entry_t *pl1pd, l1pd; 2922 pt_entry_t *ptep; 2923 u_short l1idx; 2924 2925 if (pm->pm_l1 == NULL) 2926 return (FALSE); 2927 2928 l1idx = L1_IDX(va); 2929 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2930 l1pd = *pl1pd; 2931 2932 if (l1pte_section_p(l1pd)) { 2933 *ptp = NULL; 2934 return (TRUE); 2935 } 2936 2937 if (pm->pm_l2 == NULL) 2938 return (FALSE); 2939 2940 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2941 2942 if (l2 == NULL || 2943 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 2944 return (FALSE); 2945 } 2946 2947 *ptp = &ptep[l2pte_index(va)]; 2948 return (TRUE); 2949} 2950 2951/* 2952 * Routine: pmap_remove_all 2953 * Function: 2954 * Removes this physical page from 2955 * all physical maps in which it resides. 2956 * Reflects back modify bits to the pager. 2957 * 2958 * Notes: 2959 * Original versions of this routine were very 2960 * inefficient because they iteratively called 2961 * pmap_remove (slow...) 2962 */ 2963void 2964pmap_remove_all(vm_page_t m) 2965{ 2966 pv_entry_t pv; 2967 pt_entry_t *pte, tpte; 2968 int s; 2969 2970#if defined(PMAP_DEBUG) 2971 /* 2972 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 2973 * pages! 2974 */ 2975 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) { 2976 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%x", VM_PAGE_TO_PHYS(m)); 2977 } 2978#endif 2979 2980 s = splvm(); 2981 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 2982 pv->pv_pmap->pm_stats.resident_count--; 2983 2984 pte = pmap_pte(pv->pv_pmap, pv->pv_va); 2985 2986 tpte = atomic_readandclear_int(pte); 2987 if (pv->pv_flags & PT_W) 2988 pv->pv_pmap->pm_stats.wired_count--; 2989 2990 pmap_invalidate_page(pv->pv_pmap, pv->pv_va); 2991 2992 TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); 2993 m->md.pv_list_count--; 2994 pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); 2995 pmap_idcache_wbinv_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE); 2996 pmap_free_pv_entry(pv); 2997 } 2998 2999 splx(s); 3000} 3001 3002 3003/* 3004 * Set the physical protection on the 3005 * specified range of this map as requested. 3006 */ 3007void 3008pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 3009{ 3010 struct l2_bucket *l2b; 3011 pt_entry_t *ptep, pte; 3012 vm_offset_t next_bucket; 3013 u_int flags; 3014 int flush; 3015 3016 if ((prot & VM_PROT_READ) == 0) { 3017 pmap_remove(pm, sva, eva); 3018 return; 3019 } 3020 3021 if (prot & VM_PROT_WRITE) { 3022 /* 3023 * If this is a read->write transition, just ignore it and let 3024 * uvm_fault() take care of it later. 3025 */ 3026 return; 3027 } 3028 3029 3030 /* 3031 * OK, at this point, we know we're doing write-protect operation. 3032 * If the pmap is active, write-back the range. 3033 */ 3034 pmap_dcache_wb_range(pm, sva, eva - sva, FALSE, FALSE); 3035 3036 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 3037 flags = 0; 3038 3039 while (sva < eva) { 3040 next_bucket = L2_NEXT_BUCKET(sva); 3041 if (next_bucket > eva) 3042 next_bucket = eva; 3043 3044 l2b = pmap_get_l2_bucket(pm, sva); 3045 if (l2b == NULL) { 3046 sva = next_bucket; 3047 continue; 3048 } 3049 3050 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3051 3052 while (sva < next_bucket) { 3053 if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { 3054 struct vm_page *pg; 3055 u_int f; 3056 3057 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3058 pte &= ~L2_S_PROT_W; 3059 *ptep = pte; 3060 PTE_SYNC(ptep); 3061 3062 if (pg != NULL) { 3063 f = pmap_modify_pv(pg, pm, sva, 3064 PVF_WRITE, 0); 3065 pmap_vac_me_harder(pg, pm, sva); 3066 } else 3067 f = PVF_REF | PVF_EXEC; 3068 3069 if (flush >= 0) { 3070 flush++; 3071 flags |= f; 3072 } else 3073 if (PV_BEEN_EXECD(f)) 3074 pmap_tlb_flushID_SE(pm, sva); 3075 else 3076 if (PV_BEEN_REFD(f)) 3077 pmap_tlb_flushD_SE(pm, sva); 3078 } 3079 3080 sva += PAGE_SIZE; 3081 ptep++; 3082 } 3083 } 3084 3085 3086 if (flush) { 3087 if (PV_BEEN_EXECD(flags)) 3088 pmap_tlb_flushID(pm); 3089 else 3090 if (PV_BEEN_REFD(flags)) 3091 pmap_tlb_flushD(pm); 3092 } 3093 3094} 3095 3096 3097/* 3098 * Insert the given physical page (p) at 3099 * the specified virtual address (v) in the 3100 * target physical map with the protection requested. 3101 * 3102 * If specified, the page will be wired down, meaning 3103 * that the related pte can not be reclaimed. 3104 * 3105 * NB: This is the only routine which MAY NOT lazy-evaluate 3106 * or lose information. That is, this routine must actually 3107 * insert this page into the given map NOW. 3108 */ 3109vm_offset_t getttb(void); 3110void 3111pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3112 boolean_t wired) 3113{ 3114 struct l2_bucket *l2b; 3115 struct vm_page *opg; 3116 struct pv_entry *pve; 3117 pt_entry_t *ptep, npte, opte; 3118 u_int nflags; 3119 u_int oflags; 3120 vm_paddr_t pa; 3121 3122 3123 if (va == vector_page) { 3124 pa = systempage.pv_pa; 3125 m = NULL; 3126 } else 3127 pa = VM_PAGE_TO_PHYS(m); 3128 nflags = 0; 3129 if (prot & VM_PROT_WRITE) 3130 nflags |= PVF_WRITE; 3131 if (prot & VM_PROT_EXECUTE) 3132 nflags |= PVF_EXEC; 3133 if (wired) 3134 nflags |= PVF_WIRED; 3135 PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " 3136 "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); 3137 3138 if (pmap == pmap_kernel()) 3139 l2b = pmap_get_l2_bucket(pmap, va); 3140 else { 3141 l2b = pmap_alloc_l2_bucket(pmap, va); 3142 } 3143 if (l2b == NULL) 3144 panic("pmap_enter: failed to allocate l2 bucket"); 3145 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3146 /* 3147 * Page table entry not valid, we need a new PT page 3148 */ 3149 if (ptep == NULL) 3150 panic("pmap_enter: invalid page table pte=%p, va=0x%x 0x%x\n", 3151 ptep, va, UPT_MIN_ADDRESS); 3152 3153 opte = *(vm_offset_t *)ptep; 3154 npte = pa; 3155 oflags = 0; 3156 3157 if (opte) { 3158 /* 3159 * There is already a mapping at this address. 3160 * If the physical address is different, lookup the 3161 * vm_page. 3162 */ 3163 if (l2pte_pa(opte) != pa) 3164 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3165 else 3166 opg = m; 3167 } else 3168 opg = NULL; 3169 3170 if (m && !(m->flags & PG_UNMANAGED)) { 3171 /* 3172 * This is to be a managed mapping. 3173 */ 3174 if ((prot & (VM_PROT_ALL)) || 3175 (m->md.pvh_attrs & PVF_REF)) { 3176 /* 3177 * - The access type indicates that we don't need 3178 * to do referenced emulation. 3179 * OR 3180 * - The physical page has already been referenced 3181 * so no need to re-do referenced emulation here. 3182 */ 3183 npte |= L2_S_PROTO; 3184 3185 nflags |= PVF_REF; 3186 3187 if (((prot & VM_PROT_WRITE) != 0 && 3188 ((m->flags & PG_WRITEABLE) || 3189 (m->md.pvh_attrs & PVF_MOD) != 0))) { 3190 /* 3191 * This is a writable mapping, and the 3192 * page's mod state indicates it has 3193 * already been modified. Make it 3194 * writable from the outset. 3195 */ 3196 npte |= L2_S_PROT_W; 3197 nflags |= PVF_MOD; 3198 } 3199 } else { 3200 /* 3201 * Need to do page referenced emulation. 3202 */ 3203 npte |= L2_TYPE_INV; 3204 } 3205 3206 npte |= pte_l2_s_cache_mode; 3207 3208 if (m == opg) { 3209 /* 3210 * We're changing the attrs of an existing mapping. 3211 */ 3212#if 0 3213 simple_lock(&pg->mdpage.pvh_slock); 3214#endif 3215 oflags = pmap_modify_pv(m, pmap, va, 3216 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3217 PVF_MOD | PVF_REF, nflags); 3218#if 0 3219 simple_unlock(&pg->mdpage.pvh_slock); 3220#endif 3221 3222 /* 3223 * We may need to flush the cache if we're 3224 * doing rw-ro... 3225 */ 3226 if (pmap->pm_cstate.cs_cache_d && 3227 (oflags & PVF_NC) == 0 && 3228 (opte & L2_S_PROT_W) != 0 && 3229 (prot & VM_PROT_WRITE) == 0) 3230 cpu_dcache_wb_range(va, PAGE_SIZE); 3231 } else { 3232 /* 3233 * New mapping, or changing the backing page 3234 * of an existing mapping. 3235 */ 3236 if (opg) { 3237 /* 3238 * Replacing an existing mapping with a new one. 3239 * It is part of our managed memory so we 3240 * must remove it from the PV list 3241 */ 3242#if 0 3243 simple_lock(&opg->mdpage.pvh_slock); 3244#endif 3245 pve = pmap_remove_pv(opg, pmap, va); 3246 pmap_vac_me_harder(opg, pmap, 0); 3247#if 0 3248 simple_unlock(&opg->mdpage.pvh_slock); 3249#endif 3250 oflags = pve->pv_flags; 3251 3252 /* 3253 * If the old mapping was valid (ref/mod 3254 * emulation creates 'invalid' mappings 3255 * initially) then make sure to frob 3256 * the cache. 3257 */ 3258 if ((oflags & PVF_NC) == 0 && 3259 l2pte_valid(opte)) { 3260 if (PV_BEEN_EXECD(oflags)) { 3261 pmap_idcache_wbinv_range(pmap, va, 3262 PAGE_SIZE); 3263 } else 3264 if (PV_BEEN_REFD(oflags)) { 3265 pmap_dcache_wb_range(pmap, va, 3266 PAGE_SIZE, TRUE, 3267 (oflags & PVF_WRITE) == 0); 3268 } 3269 } 3270 } else 3271 if ((pve = pmap_get_pv_entry()) == NULL) { 3272 panic("pmap_enter: no pv entries"); 3273 3274 } 3275 3276 pmap_enter_pv(m, pve, pmap, va, nflags); 3277 } 3278 } else { 3279 /* 3280 * We're mapping an unmanaged page. 3281 * These are always readable, and possibly writable, from 3282 * the get go as we don't need to track ref/mod status. 3283 */ 3284 npte |= L2_S_PROTO; 3285 if (prot & VM_PROT_WRITE) { 3286 npte |= L2_S_PROT_W; 3287 } 3288 3289 /* 3290 * Make sure the vector table is mapped cacheable 3291 */ 3292 if (pmap != pmap_kernel() && va == vector_page) 3293 npte |= pte_l2_s_cache_mode; 3294 if (opg) { 3295 /* 3296 * Looks like there's an existing 'managed' mapping 3297 * at this address. 3298 */ 3299#if 0 3300 simple_lock(&opg->mdpage.pvh_slock); 3301#endif 3302 pve = pmap_remove_pv(opg, pmap, va); 3303 pmap_vac_me_harder(opg, pmap, 0); 3304#if 0 3305 simple_unlock(&opg->mdpage.pvh_slock); 3306#endif 3307 oflags = pve->pv_flags; 3308 3309 if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { 3310 if (PV_BEEN_EXECD(oflags)) 3311 pmap_idcache_wbinv_range(pmap, va, 3312 PAGE_SIZE); 3313 else 3314 if (PV_BEEN_REFD(oflags)) 3315 pmap_dcache_wb_range(pmap, va, PAGE_SIZE, 3316 TRUE, (oflags & PVF_WRITE) == 0); 3317 } 3318 } 3319 } 3320 3321 /* 3322 * Make sure userland mappings get the right permissions 3323 */ 3324 if (pmap != pmap_kernel() && va != vector_page) { 3325 npte |= L2_S_PROT_U; 3326 } 3327 3328 /* 3329 * Keep the stats up to date 3330 */ 3331 if (opte == 0) { 3332 l2b->l2b_occupancy++; 3333 pmap->pm_stats.resident_count++; 3334 } 3335 3336 3337 /* 3338 * If this is just a wiring change, the two PTEs will be 3339 * identical, so there's no need to update the page table. 3340 */ 3341 if (npte != opte) { 3342 boolean_t is_cached = pmap_is_cached(pmap); 3343 3344 *ptep = npte; 3345 if (is_cached) { 3346 /* 3347 * We only need to frob the cache/tlb if this pmap 3348 * is current 3349 */ 3350 PTE_SYNC(ptep); 3351 if (L1_IDX(va) != L1_IDX(vector_page) && 3352 l2pte_valid(npte)) { 3353 /* 3354 * This mapping is likely to be accessed as 3355 * soon as we return to userland. Fix up the 3356 * L1 entry to avoid taking another 3357 * page/domain fault. 3358 */ 3359 pd_entry_t *pl1pd, l1pd; 3360 3361 3362 pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; 3363 l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | 3364 L1_C_PROTO; 3365 if (*pl1pd != l1pd) { 3366 *pl1pd = l1pd; 3367 PTE_SYNC(pl1pd); 3368 } 3369 } 3370 } 3371 3372 if (PV_BEEN_EXECD(oflags)) 3373 pmap_tlb_flushID_SE(pmap, va); 3374 else 3375 if (PV_BEEN_REFD(oflags)) 3376 pmap_tlb_flushD_SE(pmap, va); 3377 3378 3379 if (m && !(m->flags & PG_UNMANAGED)) { 3380#if 0 3381 simple_lock(&pg->mdpage.pvh_slock); 3382#endif 3383 pmap_vac_me_harder(m, pmap, va); 3384#if 0 3385 simple_unlock(&pg->mdpage.pvh_slock); 3386#endif 3387 } 3388 } 3389} 3390 3391/* 3392 * this code makes some *MAJOR* assumptions: 3393 * 1. Current pmap & pmap exists. 3394 * 2. Not wired. 3395 * 3. Read access. 3396 * 4. No page table pages. 3397 * 5. Tlbflush is deferred to calling procedure. 3398 * 6. Page IS managed. 3399 * but is *MUCH* faster than pmap_enter... 3400 */ 3401 3402vm_page_t 3403pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) 3404{ 3405 pmap_enter(pmap, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); 3406 return (NULL); 3407} 3408 3409/* 3410 * Routine: pmap_change_wiring 3411 * Function: Change the wiring attribute for a map/virtual-address 3412 * pair. 3413 * In/out conditions: 3414 * The mapping must already exist in the pmap. 3415 */ 3416void 3417pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 3418{ 3419 struct l2_bucket *l2b; 3420 pt_entry_t *ptep, pte; 3421 vm_page_t pg; 3422 3423 l2b = pmap_get_l2_bucket(pmap, va); 3424 KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); 3425 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3426 pte = *ptep; 3427 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3428 if (pg) 3429 pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); 3430} 3431 3432 3433/* 3434 * Copy the range specified by src_addr/len 3435 * from the source map to the range dst_addr/len 3436 * in the destination map. 3437 * 3438 * This routine is only advisory and need not do anything. 3439 */ 3440void 3441pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 3442 vm_size_t len, vm_offset_t src_addr) 3443{ 3444} 3445 3446 3447/* 3448 * this routine defines the region(s) of memory that should 3449 * not be tested for the modified bit. 3450 */ 3451static PMAP_INLINE int 3452pmap_track_modified(vm_offset_t va) 3453{ 3454 if ((va < clean_sva) || (va >= clean_eva)) 3455 return 1; 3456 else 3457 return 0; 3458} 3459 3460 3461/* 3462 * Routine: pmap_pte 3463 * Function: 3464 * Extract the page table entry associated 3465 * with the given map/virtual_address pair. 3466 */ 3467static pt_entry_t * 3468pmap_pte(pmap_t pmap, vm_offset_t va) 3469{ 3470 struct l2_bucket *l2b; 3471 3472 l2b = pmap_get_l2_bucket(pmap, va); 3473 if (l2b == NULL) 3474 return (NULL); 3475 return (&l2b->l2b_kva[l2pte_index(va)]); 3476} 3477 3478vm_paddr_t 3479pmap_kextract(vm_offset_t va) 3480{ 3481 return (pmap_extract(pmap_kernel(), va)); 3482} 3483 3484/* 3485 * Routine: pmap_extract 3486 * Function: 3487 * Extract the physical page address associated 3488 * with the given map/virtual_address pair. 3489 */ 3490vm_paddr_t 3491pmap_extract(pmap_t pm, vm_offset_t va) 3492{ 3493 struct l2_dtable *l2; 3494 pd_entry_t *pl1pd, l1pd; 3495 pt_entry_t *ptep, pte; 3496 vm_paddr_t pa; 3497 u_int l1idx; 3498 l1idx = L1_IDX(va); 3499 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3500 l1pd = *pl1pd; 3501 3502 if (l1pte_section_p(l1pd)) { 3503 /* 3504 * These should only happen for pmap_kernel() 3505 */ 3506 KASSERT(pm == pmap_kernel(), ("huh")); 3507 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3508 } else { 3509 /* 3510 * Note that we can't rely on the validity of the L1 3511 * descriptor as an indication that a mapping exists. 3512 * We have to look it up in the L2 dtable. 3513 */ 3514 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3515 3516 if (l2 == NULL || 3517 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3518 return (0); 3519 } 3520 3521 ptep = &ptep[l2pte_index(va)]; 3522 pte = *ptep; 3523 3524 if (pte == 0) 3525 return (0); 3526 3527 switch (pte & L2_TYPE_MASK) { 3528 case L2_TYPE_L: 3529 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3530 break; 3531 3532 default: 3533 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3534 break; 3535 } 3536 } 3537 3538 return (pa); 3539} 3540 3541vm_page_t 3542pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3543{ 3544 vm_paddr_t pa; 3545 vm_page_t m; 3546 3547 m = NULL; 3548 mtx_lock(&Giant); 3549 if ((pa = pmap_extract(pmap, va)) != 0) { 3550 m = PHYS_TO_VM_PAGE(pa); 3551 vm_page_lock_queues(); 3552 vm_page_hold(m); 3553 vm_page_unlock_queues(); 3554 } 3555 mtx_unlock(&Giant); 3556 return (m); 3557} 3558/* 3559 * After removing a page table entry, this routine is used to 3560 * conditionally free the page, and manage the hold/wire counts. 3561 */ 3562static int 3563pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t mpte) 3564{ 3565 if (va >= UPT_MIN_ADDRESS) 3566 return 0; 3567 3568 return pmap_unwire_pte_hold(pmap, mpte); 3569} 3570 3571void 3572pmap_update(pmap_t pm) 3573{ 3574 3575 if (pmap_is_current(pm)) { 3576 /* 3577 * If we're dealing with a current userland pmap, move its L1 3578 * to the end of the LRU. 3579 */ 3580 if (pm != pmap_kernel()) 3581 pmap_use_l1(pm); 3582 3583 /* 3584 * We can assume we're done with frobbing the cache/tlb for 3585 * now. Make sure any future pmap ops don't skip cache/tlb 3586 * flushes. 3587 */ 3588 pm->pm_cstate.cs_all = PMAP_CACHE_STATE_ALL; 3589 } 3590 3591 /* 3592 * make sure TLB/cache operations have completed. 3593 */ 3594 cpu_cpwait(); 3595} 3596 3597 3598/* 3599 * Initialize a preallocated and zeroed pmap structure, 3600 * such as one in a vmspace structure. 3601 */ 3602 3603void 3604pmap_pinit(pmap_t pmap) 3605{ 3606 PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); 3607 3608 pmap_alloc_l1(pmap); 3609 bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); 3610 3611 LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); 3612 pmap->pm_count = 1; 3613 pmap->pm_active = 0; 3614 pmap->pm_ptphint = NULL; 3615 3616 TAILQ_INIT(&pmap->pm_pvlist); 3617 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 3618 pmap->pm_stats.resident_count = 1; 3619 if (vector_page < KERNBASE) { 3620 pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa), 3621 VM_PROT_READ, 1); 3622 pmap_update(pmap); 3623 } 3624} 3625 3626 3627/*************************************************** 3628 * page management routines. 3629 ***************************************************/ 3630 3631 3632static PMAP_INLINE void 3633pmap_free_pv_entry(pv_entry_t pv) 3634{ 3635 pv_entry_count--; 3636 uma_zfree(pvzone, pv); 3637} 3638 3639 3640/* 3641 * get a new pv_entry, allocating a block from the system 3642 * when needed. 3643 * the memory allocation is performed bypassing the malloc code 3644 * because of the possibility of allocations at interrupt time. 3645 */ 3646static pv_entry_t 3647pmap_get_pv_entry(void) 3648{ 3649 pv_entry_t ret_value; 3650 3651 pv_entry_count++; 3652 if (pv_entry_high_water && 3653 (pv_entry_count > pv_entry_high_water) && 3654 (pmap_pagedaemon_waken == 0)) { 3655 pmap_pagedaemon_waken = 1; 3656 wakeup (&vm_pages_needed); 3657 } 3658 ret_value = uma_zalloc(pvzone, M_NOWAIT); 3659 return ret_value; 3660} 3661 3662 3663/* 3664 * Remove the given range of addresses from the specified map. 3665 * 3666 * It is assumed that the start and end are properly 3667 * rounded to the page size. 3668 */ 3669#define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3670void 3671pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 3672{ 3673 struct l2_bucket *l2b; 3674 vm_offset_t next_bucket; 3675 pt_entry_t *ptep; 3676 u_int cleanlist_idx, total, cnt; 3677 struct { 3678 vm_offset_t va; 3679 pt_entry_t *pte; 3680 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3681 u_int mappings, is_exec, is_refd; 3682 3683 3684 /* 3685 * we lock in the pmap => pv_head direction 3686 */ 3687#if 0 3688 PMAP_MAP_TO_HEAD_LOCK(); 3689 pmap_acquire_pmap_lock(pm); 3690#endif 3691 3692 if (!pmap_is_cached(pm)) { 3693 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3694 } else 3695 cleanlist_idx = 0; 3696 3697 total = 0; 3698 3699 while (sva < eva) { 3700 /* 3701 * Do one L2 bucket's worth at a time. 3702 */ 3703 next_bucket = L2_NEXT_BUCKET(sva); 3704 if (next_bucket > eva) 3705 next_bucket = eva; 3706 3707 l2b = pmap_get_l2_bucket(pm, sva); 3708 if (l2b == NULL) { 3709 sva = next_bucket; 3710 continue; 3711 } 3712 3713 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3714 mappings = 0; 3715 3716 while (sva < next_bucket) { 3717 struct vm_page *pg; 3718 pt_entry_t pte; 3719 vm_paddr_t pa; 3720 3721 pte = *ptep; 3722 3723 if (pte == 0) { 3724 /* 3725 * Nothing here, move along 3726 */ 3727 sva += PAGE_SIZE; 3728 ptep++; 3729 continue; 3730 } 3731 3732 pm->pm_stats.resident_count--; 3733 pa = l2pte_pa(pte); 3734 is_exec = 0; 3735 is_refd = 1; 3736 3737 /* 3738 * Update flags. In a number of circumstances, 3739 * we could cluster a lot of these and do a 3740 * number of sequential pages in one go. 3741 */ 3742 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3743 struct pv_entry *pve; 3744#if 0 3745 simple_lock(&pg->mdpage.pvh_slock); 3746#endif 3747 pve = pmap_remove_pv(pg, pm, sva); 3748 pmap_vac_me_harder(pg, pm, 0); 3749#if 0 3750 simple_unlock(&pg->mdpage.pvh_slock); 3751#endif 3752 if (pve != NULL) { 3753 is_exec = 3754 PV_BEEN_EXECD(pve->pv_flags); 3755 is_refd = 3756 PV_BEEN_REFD(pve->pv_flags); 3757 pmap_free_pv_entry(pve); 3758 } 3759 } 3760 3761 if (!l2pte_valid(pte)) { 3762 *ptep = 0; 3763 PTE_SYNC_CURRENT(pm, ptep); 3764 sva += PAGE_SIZE; 3765 ptep++; 3766 mappings++; 3767 continue; 3768 } 3769 3770 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3771 /* Add to the clean list. */ 3772 cleanlist[cleanlist_idx].pte = ptep; 3773 cleanlist[cleanlist_idx].va = 3774 sva | (is_exec & 1); 3775 cleanlist_idx++; 3776 } else 3777 if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3778 /* Nuke everything if needed. */ 3779 pmap_idcache_wbinv_all(pm); 3780 pmap_tlb_flushID(pm); 3781 3782 /* 3783 * Roll back the previous PTE list, 3784 * and zero out the current PTE. 3785 */ 3786 for (cnt = 0; 3787 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3788 *cleanlist[cnt].pte = 0; 3789 } 3790 *ptep = 0; 3791 PTE_SYNC(ptep); 3792 cleanlist_idx++; 3793 } else { 3794 *ptep = 0; 3795 PTE_SYNC(ptep); 3796 if (is_exec) 3797 pmap_tlb_flushID_SE(pm, sva); 3798 else 3799 if (is_refd) 3800 pmap_tlb_flushD_SE(pm, sva); 3801 } 3802 3803 sva += PAGE_SIZE; 3804 ptep++; 3805 mappings++; 3806 } 3807 3808 /* 3809 * Deal with any left overs 3810 */ 3811 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3812 total += cleanlist_idx; 3813 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3814 if (pm->pm_cstate.cs_all != 0) { 3815 vm_offset_t clva = 3816 cleanlist[cnt].va & ~1; 3817 if (cleanlist[cnt].va & 1) { 3818 pmap_idcache_wbinv_range(pm, 3819 clva, PAGE_SIZE); 3820 pmap_tlb_flushID_SE(pm, clva); 3821 } else { 3822 pmap_dcache_wb_range(pm, 3823 clva, PAGE_SIZE, TRUE, 3824 FALSE); 3825 pmap_tlb_flushD_SE(pm, clva); 3826 } 3827 } 3828 *cleanlist[cnt].pte = 0; 3829 PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte); 3830 } 3831 3832 /* 3833 * If it looks like we're removing a whole bunch 3834 * of mappings, it's faster to just write-back 3835 * the whole cache now and defer TLB flushes until 3836 * pmap_update() is called. 3837 */ 3838 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3839 cleanlist_idx = 0; 3840 else { 3841 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3842 pmap_idcache_wbinv_all(pm); 3843 } 3844 } 3845 3846 pmap_free_l2_bucket(pm, l2b, mappings); 3847 } 3848 3849#if 0 3850 pmap_release_pmap_lock(pm); 3851 PMAP_MAP_TO_HEAD_UNLOCK(); 3852#endif 3853} 3854 3855 3856 3857static PMAP_INLINE int 3858pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) 3859{ 3860 return 0; 3861} 3862 3863 3864/* 3865 * pmap_zero_page() 3866 * 3867 * Zero a given physical page by mapping it at a page hook point. 3868 * In doing the zero page op, the page we zero is mapped cachable, as with 3869 * StrongARM accesses to non-cached pages are non-burst making writing 3870 * _any_ bulk data very slow. 3871 */ 3872#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 3873void 3874pmap_zero_page_generic(vm_paddr_t phys, int off, int size) 3875{ 3876#ifdef DEBUG 3877 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 3878 3879 if (pg->md.pvh_list != NULL) 3880 panic("pmap_zero_page: page has mappings"); 3881#endif 3882 3883 3884 /* 3885 * Hook in the page, zero it, and purge the cache for that 3886 * zeroed page. Invalidate the TLB as needed. 3887 */ 3888 *cdst_pte = L2_S_PROTO | phys | 3889 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 3890 PTE_SYNC(cdst_pte); 3891 cpu_tlb_flushD_SE(cdstp); 3892 cpu_cpwait(); 3893 if (off || size) 3894 bzero((void *)(cdstp + off), size); 3895 else 3896 bzero_page(cdstp); 3897 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 3898} 3899#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 3900 3901#if ARM_MMU_XSCALE == 1 3902void 3903pmap_zero_page_xscale(vm_paddr_t phys, int off, int size) 3904{ 3905#ifdef DEBUG 3906 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 3907 3908 if (pg->md.pvh_list != NULL) 3909 panic("pmap_zero_page: page has mappings"); 3910#endif 3911 3912 3913 /* 3914 * Hook in the page, zero it, and purge the cache for that 3915 * zeroed page. Invalidate the TLB as needed. 3916 */ 3917 *cdst_pte = L2_S_PROTO | phys | 3918 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 3919 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 3920 PTE_SYNC(cdst_pte); 3921 cpu_tlb_flushD_SE(cdstp); 3922 cpu_cpwait(); 3923 if (off || size) 3924 bzero((void *)(cdstp + off), size); 3925 else 3926 bzero_page(cdstp); 3927 xscale_cache_clean_minidata(); 3928} 3929 3930/* 3931 * Change the PTEs for the specified kernel mappings such that they 3932 * will use the mini data cache instead of the main data cache. 3933 */ 3934void 3935pmap_uarea(vm_offset_t va) 3936{ 3937 struct l2_bucket *l2b; 3938 pt_entry_t *ptep, *sptep, pte; 3939 vm_offset_t next_bucket, eva; 3940 3941#if (ARM_NMMUS > 1) 3942 if (xscale_use_minidata == 0) 3943 return; 3944#endif 3945 3946 eva = va + USPACE; 3947 3948 while (va < eva) { 3949 next_bucket = L2_NEXT_BUCKET(va); 3950 if (next_bucket > eva) 3951 next_bucket = eva; 3952 3953 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 3954 3955 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 3956 3957 while (va < next_bucket) { 3958 pte = *ptep; 3959 if (!l2pte_minidata(pte)) { 3960 cpu_dcache_wbinv_range(va, PAGE_SIZE); 3961 cpu_tlb_flushD_SE(va); 3962 *ptep = pte & ~L2_B; 3963 } 3964 ptep++; 3965 va += PAGE_SIZE; 3966 } 3967 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 3968 } 3969 cpu_cpwait(); 3970} 3971#endif /* ARM_MMU_XSCALE == 1 */ 3972 3973/* 3974 * pmap_zero_page zeros the specified hardware page by mapping 3975 * the page into KVM and using bzero to clear its contents. 3976 */ 3977void 3978pmap_zero_page(vm_page_t m) 3979{ 3980 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), 0, 0); 3981} 3982 3983 3984/* 3985 * pmap_zero_page_area zeros the specified hardware page by mapping 3986 * the page into KVM and using bzero to clear its contents. 3987 * 3988 * off and size may not cover an area beyond a single hardware page. 3989 */ 3990void 3991pmap_zero_page_area(vm_page_t m, int off, int size) 3992{ 3993 3994 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), off, size); 3995} 3996 3997 3998/* 3999 * pmap_zero_page_idle zeros the specified hardware page by mapping 4000 * the page into KVM and using bzero to clear its contents. This 4001 * is intended to be called from the vm_pagezero process only and 4002 * outside of Giant. 4003 */ 4004void 4005pmap_zero_page_idle(vm_page_t m) 4006{ 4007 unsigned int i; 4008 int *ptr; 4009 vm_paddr_t phys = VM_PAGE_TO_PHYS(m); 4010 4011 pmap_zero_page(m); 4012 return; 4013 *cdst_pte = L2_S_PROTO | phys | 4014 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4015 PTE_SYNC(cdst_pte); 4016 cpu_tlb_flushD_SE(cdstp); 4017 cpu_cpwait(); 4018 4019 for (i = 0, ptr = (int *)cdstp; 4020 i < (PAGE_SIZE / sizeof(int)); i++) { 4021 *ptr = 0; 4022 } 4023 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4024} 4025 4026/* 4027 * pmap_clean_page() 4028 * 4029 * This is a local function used to work out the best strategy to clean 4030 * a single page referenced by its entry in the PV table. It's used by 4031 * pmap_copy_page, pmap_zero page and maybe some others later on. 4032 * 4033 * Its policy is effectively: 4034 * o If there are no mappings, we don't bother doing anything with the cache. 4035 * o If there is one mapping, we clean just that page. 4036 * o If there are multiple mappings, we clean the entire cache. 4037 * 4038 * So that some functions can be further optimised, it returns 0 if it didn't 4039 * clean the entire cache, or 1 if it did. 4040 * 4041 * XXX One bug in this routine is that if the pv_entry has a single page 4042 * mapped at 0x00000000 a whole cache clean will be performed rather than 4043 * just the 1 page. Since this should not occur in everyday use and if it does 4044 * it will just result in not the most efficient clean for the page. 4045 */ 4046static int 4047pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 4048{ 4049 pmap_t pm, pm_to_clean = NULL; 4050 struct pv_entry *npv; 4051 u_int cache_needs_cleaning = 0; 4052 u_int flags = 0; 4053 vm_offset_t page_to_clean = 0; 4054 4055 if (pv == NULL) { 4056 /* nothing mapped in so nothing to flush */ 4057 return (0); 4058 } 4059 4060 /* 4061 * Since we flush the cache each time we change to a different 4062 * user vmspace, we only need to flush the page if it is in the 4063 * current pmap. 4064 */ 4065 if (curproc) 4066 pm = curproc->p_vmspace->vm_map.pmap; 4067 else 4068 pm = pmap_kernel(); 4069 4070 for (npv = pv; npv; npv = TAILQ_NEXT(npv, pv_list)) { 4071 if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) { 4072 flags |= npv->pv_flags; 4073 /* 4074 * The page is mapped non-cacheable in 4075 * this map. No need to flush the cache. 4076 */ 4077 if (npv->pv_flags & PVF_NC) { 4078#ifdef DIAGNOSTIC 4079 if (cache_needs_cleaning) 4080 panic("pmap_clean_page: " 4081 "cache inconsistency"); 4082#endif 4083 break; 4084 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 4085 continue; 4086 if (cache_needs_cleaning) { 4087 page_to_clean = 0; 4088 break; 4089 } else { 4090 page_to_clean = npv->pv_va; 4091 pm_to_clean = npv->pv_pmap; 4092 } 4093 cache_needs_cleaning = 1; 4094 } 4095 } 4096 if (page_to_clean) { 4097 if (PV_BEEN_EXECD(flags)) 4098 pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, 4099 PAGE_SIZE); 4100 else 4101 pmap_dcache_wb_range(pm_to_clean, page_to_clean, 4102 PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); 4103 } else if (cache_needs_cleaning) { 4104 if (PV_BEEN_EXECD(flags)) 4105 pmap_idcache_wbinv_all(pm); 4106 else 4107 pmap_dcache_wbinv_all(pm); 4108 return (1); 4109 } 4110 return (0); 4111} 4112 4113/* 4114 * pmap_copy_page copies the specified (machine independent) 4115 * page by mapping the page into virtual memory and using 4116 * bcopy to copy the page, one machine dependent page at a 4117 * time. 4118 */ 4119 4120/* 4121 * pmap_copy_page() 4122 * 4123 * Copy one physical page into another, by mapping the pages into 4124 * hook points. The same comment regarding cachability as in 4125 * pmap_zero_page also applies here. 4126 */ 4127#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 4128void 4129pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) 4130{ 4131 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4132#ifdef DEBUG 4133 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4134 4135 if (dst_pg->md.pvh_list != NULL) 4136 panic("pmap_copy_page: dst page has mappings"); 4137#endif 4138 4139 4140 /* 4141 * Clean the source page. Hold the source page's lock for 4142 * the duration of the copy so that no other mappings can 4143 * be created while we have a potentially aliased mapping. 4144 */ 4145#if 0 4146 mtx_lock(&src_pg->md.pvh_mtx); 4147#endif 4148 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4149 4150 /* 4151 * Map the pages into the page hook points, copy them, and purge 4152 * the cache for the appropriate page. Invalidate the TLB 4153 * as required. 4154 */ 4155 *csrc_pte = L2_S_PROTO | src | 4156 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 4157 PTE_SYNC(csrc_pte); 4158 *cdst_pte = L2_S_PROTO | dst | 4159 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4160 PTE_SYNC(cdst_pte); 4161 cpu_tlb_flushD_SE(csrcp); 4162 cpu_tlb_flushD_SE(cdstp); 4163 cpu_cpwait(); 4164 bcopy_page(csrcp, cdstp); 4165 cpu_dcache_inv_range(csrcp, PAGE_SIZE); 4166#if 0 4167 mtx_lock(&src_pg->md.pvh_mtx); 4168#endif 4169 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4170} 4171#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 4172 4173#if ARM_MMU_XSCALE == 1 4174void 4175pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) 4176{ 4177 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4178#ifdef DEBUG 4179 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4180 4181 if (dst_pg->md.pvh_list != NULL) 4182 panic("pmap_copy_page: dst page has mappings"); 4183#endif 4184 4185 4186 /* 4187 * Clean the source page. Hold the source page's lock for 4188 * the duration of the copy so that no other mappings can 4189 * be created while we have a potentially aliased mapping. 4190 */ 4191 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4192 4193 /* 4194 * Map the pages into the page hook points, copy them, and purge 4195 * the cache for the appropriate page. Invalidate the TLB 4196 * as required. 4197 */ 4198 *csrc_pte = L2_S_PROTO | src | 4199 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4200 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4201 PTE_SYNC(csrc_pte); 4202 *cdst_pte = L2_S_PROTO | dst | 4203 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4204 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4205 PTE_SYNC(cdst_pte); 4206 cpu_tlb_flushD_SE(csrcp); 4207 cpu_tlb_flushD_SE(cdstp); 4208 cpu_cpwait(); 4209 bcopy_page(csrcp, cdstp); 4210 xscale_cache_clean_minidata(); 4211} 4212#endif /* ARM_MMU_XSCALE == 1 */ 4213 4214void 4215pmap_copy_page(vm_page_t src, vm_page_t dst) 4216{ 4217 pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); 4218} 4219 4220 4221 4222 4223/* 4224 * this routine returns true if a physical page resides 4225 * in the given pmap. 4226 */ 4227boolean_t 4228pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 4229{ 4230 pv_entry_t pv; 4231 int loops = 0; 4232 int s; 4233 4234 if (!pmap_initialized || (m->flags & PG_FICTITIOUS)) 4235 return (FALSE); 4236 4237 s = splvm(); 4238 4239 /* 4240 * Not found, check current mappings returning immediately 4241 */ 4242 for (pv = TAILQ_FIRST(&m->md.pv_list); 4243 pv; 4244 pv = TAILQ_NEXT(pv, pv_list)) { 4245 if (pv->pv_pmap == pmap) { 4246 splx(s); 4247 return (TRUE); 4248 } 4249 loops++; 4250 if (loops >= 16) 4251 break; 4252 } 4253 splx(s); 4254 return (FALSE); 4255} 4256 4257 4258/* 4259 * pmap_ts_referenced: 4260 * 4261 * Return the count of reference bits for a page, clearing all of them. 4262 */ 4263int 4264pmap_ts_referenced(vm_page_t m) 4265{ 4266 printf("pmap_ts_referenced()\n"); 4267 4268 return (0); 4269} 4270 4271 4272/* 4273 * pmap_is_modified: 4274 * 4275 * Return whether or not the specified physical page was modified 4276 * in any physical maps. 4277 */ 4278boolean_t 4279pmap_is_modified(vm_page_t m) 4280{ 4281 printf("pmap_is_modified()\n"); 4282 4283 return(FALSE); 4284} 4285 4286 4287/* 4288 * Clear the modify bits on the specified physical page. 4289 */ 4290void 4291pmap_clear_modify(vm_page_t m) 4292{ 4293 4294 if (m->md.pvh_attrs & PVF_MOD) 4295 pmap_clearbit(m, PVF_MOD); 4296} 4297 4298 4299/* 4300 * pmap_clear_reference: 4301 * 4302 * Clear the reference bit on the specified physical page. 4303 */ 4304void 4305pmap_clear_reference(vm_page_t m) 4306{ 4307 4308 if (m->md.pvh_attrs & PVF_REF) 4309 pmap_clearbit(m, PVF_REF); 4310} 4311 4312 4313/* 4314 * perform the pmap work for mincore 4315 */ 4316int 4317pmap_mincore(pmap_t pmap, vm_offset_t addr) 4318{ 4319 printf("pmap_mincore()\n"); 4320 4321 return (0); 4322} 4323 4324 4325vm_offset_t 4326pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 4327{ 4328 4329 return(addr); 4330} 4331 4332 4333static void 4334arm_protection_init(void) 4335{ 4336 int *kp, prot; 4337 4338 kp = protection_codes; 4339 for (prot = 0; prot < 8; prot++) { 4340 switch (prot) { 4341 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_NONE: 4342 *kp++ = 0; 4343 break; 4344 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_NONE: 4345 case VM_PROT_READ | VM_PROT_NONE | VM_PROT_EXECUTE: 4346 case VM_PROT_NONE | VM_PROT_NONE | VM_PROT_EXECUTE: 4347 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_NONE: 4348 case VM_PROT_NONE | VM_PROT_WRITE | VM_PROT_EXECUTE: 4349 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_NONE: 4350 case VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE: 4351 *kp++ = PT_AP(AP_KRW); 4352 } 4353 } 4354} 4355 4356 4357/* 4358 * Map a set of physical memory pages into the kernel virtual 4359 * address space. Return a pointer to where it is mapped. This 4360 * routine is intended to be used for mapping device memory, 4361 * NOT real memory. 4362 */ 4363void * 4364pmap_mapdev(vm_offset_t pa, vm_size_t size) 4365{ 4366 vm_offset_t va, tmpva, offset; 4367 pt_entry_t *pte; 4368 4369 /* XXX: pmap_mapdev is wrong. */ 4370 offset = pa & PAGE_MASK; 4371 size = roundup(offset + size, PAGE_SIZE); 4372 4373 GIANT_REQUIRED; 4374 4375 va = kmem_alloc_nofault(kernel_map, size); 4376 if (!va) 4377 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 4378 4379 pa = pa & PG_FRAME; 4380 for (tmpva = va; size > 0;) { 4381 pte = vtopte((vm_offset_t)vtopte(tmpva)); 4382 *pte = L2_PTE(pa, AP_KRW); 4383 size -= PAGE_SIZE; 4384 tmpva += PAGE_SIZE; 4385 pa += PAGE_SIZE; 4386 } 4387 pmap_invalidate_tlb_all(kernel_pmap); 4388 4389 return ((void *)(va + offset)); 4390} 4391 4392#define BOOTSTRAP_DEBUG 4393 4394/* 4395 * pmap_map_section: 4396 * 4397 * Create a single section mapping. 4398 */ 4399void 4400pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4401 int prot, int cache) 4402{ 4403 pd_entry_t *pde = (pd_entry_t *) l1pt; 4404 pd_entry_t fl; 4405 4406 KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); 4407 4408 switch (cache) { 4409 case PTE_NOCACHE: 4410 default: 4411 fl = 0; 4412 break; 4413 4414 case PTE_CACHE: 4415 fl = pte_l1_s_cache_mode; 4416 break; 4417 4418 case PTE_PAGETABLE: 4419 fl = pte_l1_s_cache_mode_pt; 4420 break; 4421 } 4422 4423 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4424 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 4425 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4426 4427} 4428 4429/* 4430 * pmap_link_l2pt: 4431 * 4432 * Link the L2 page table specified by "pa" into the L1 4433 * page table at the slot for "va". 4434 */ 4435void 4436pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) 4437{ 4438 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 4439 u_int slot = va >> L1_S_SHIFT; 4440 4441#ifndef ARM32_NEW_VM_LAYOUT 4442 KASSERT((va & ((L1_S_SIZE * 4) - 1)) == 0, ("blah")); 4443 KASSERT((l2pv->pv_pa & PAGE_MASK) == 0, ("ouin")); 4444#endif 4445 4446 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 4447 4448 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 4449#ifdef ARM32_NEW_VM_LAYOUT 4450 PTE_SYNC(&pde[slot]); 4451#else 4452 pde[slot + 1] = proto | (l2pv->pv_pa + 0x400); 4453 pde[slot + 2] = proto | (l2pv->pv_pa + 0x800); 4454 pde[slot + 3] = proto | (l2pv->pv_pa + 0xc00); 4455 PTE_SYNC_RANGE(&pde[slot + 0], 4); 4456#endif 4457 4458 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 4459 4460 4461} 4462 4463/* 4464 * pmap_map_entry 4465 * 4466 * Create a single page mapping. 4467 */ 4468void 4469pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, 4470 int cache) 4471{ 4472 pd_entry_t *pde = (pd_entry_t *) l1pt; 4473 pt_entry_t fl; 4474 pt_entry_t *pte; 4475 4476 KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); 4477 4478 switch (cache) { 4479 case PTE_NOCACHE: 4480 default: 4481 fl = 0; 4482 break; 4483 4484 case PTE_CACHE: 4485 fl = pte_l2_s_cache_mode; 4486 break; 4487 4488 case PTE_PAGETABLE: 4489 fl = pte_l2_s_cache_mode_pt; 4490 break; 4491 } 4492 4493 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4494 panic("pmap_map_entry: no L2 table for VA 0x%08x", va); 4495 4496#ifndef ARM32_NEW_VM_LAYOUT 4497 pte = (pt_entry_t *) 4498 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4499#else 4500 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4501#endif 4502 4503 if (pte == NULL) 4504 panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); 4505 4506#ifndef ARM32_NEW_VM_LAYOUT 4507 pte[(va >> PAGE_SHIFT) & 0x3ff] = 4508 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 4509 PTE_SYNC(&pte[(va >> PAGE_SHIFT) & 0x3ff]); 4510#else 4511 pte[l2pte_index(va)] = 4512 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 4513 PTE_SYNC(&pte[l2pte_index(va)]); 4514#endif 4515} 4516 4517/* 4518 * pmap_map_chunk: 4519 * 4520 * Map a chunk of memory using the most efficient mappings 4521 * possible (section. large page, small page) into the 4522 * provided L1 and L2 tables at the specified virtual address. 4523 */ 4524vm_size_t 4525pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4526 vm_size_t size, int prot, int cache) 4527{ 4528 pd_entry_t *pde = (pd_entry_t *) l1pt; 4529 pt_entry_t *pte, f1, f2s, f2l; 4530 vm_size_t resid; 4531 int i; 4532 4533 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 4534 4535 if (l1pt == 0) 4536 panic("pmap_map_chunk: no L1 table provided"); 4537 4538#ifdef VERBOSE_INIT_ARM 4539 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 4540 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 4541#endif 4542 4543 switch (cache) { 4544 case PTE_NOCACHE: 4545 default: 4546 f1 = 0; 4547 f2l = 0; 4548 f2s = 0; 4549 break; 4550 4551 case PTE_CACHE: 4552 f1 = pte_l1_s_cache_mode; 4553 f2l = pte_l2_l_cache_mode; 4554 f2s = pte_l2_s_cache_mode; 4555 break; 4556 4557 case PTE_PAGETABLE: 4558 f1 = pte_l1_s_cache_mode_pt; 4559 f2l = pte_l2_l_cache_mode_pt; 4560 f2s = pte_l2_s_cache_mode_pt; 4561 break; 4562 } 4563 4564 size = resid; 4565 4566 while (resid > 0) { 4567 /* See if we can use a section mapping. */ 4568 if (L1_S_MAPPABLE_P(va, pa, resid)) { 4569#ifdef VERBOSE_INIT_ARM 4570 printf("S"); 4571#endif 4572 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4573 L1_S_PROT(PTE_KERNEL, prot) | f1 | 4574 L1_S_DOM(PMAP_DOMAIN_KERNEL); 4575 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4576 va += L1_S_SIZE; 4577 pa += L1_S_SIZE; 4578 resid -= L1_S_SIZE; 4579 continue; 4580 } 4581 4582 /* 4583 * Ok, we're going to use an L2 table. Make sure 4584 * one is actually in the corresponding L1 slot 4585 * for the current VA. 4586 */ 4587 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4588 panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); 4589 4590#ifndef ARM32_NEW_VM_LAYOUT 4591 pte = (pt_entry_t *) 4592 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 4593#else 4594 pte = (pt_entry_t *) kernel_pt_lookup( 4595 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4596#endif 4597 if (pte == NULL) 4598 panic("pmap_map_chunk: can't find L2 table for VA" 4599 "0x%08x", va); 4600 4601 /* See if we can use a L2 large page mapping. */ 4602 if (L2_L_MAPPABLE_P(va, pa, resid)) { 4603#ifdef VERBOSE_INIT_ARM 4604 printf("L"); 4605#endif 4606 for (i = 0; i < 16; i++) { 4607#ifndef ARM32_NEW_VM_LAYOUT 4608 pte[((va >> PAGE_SHIFT) & 0x3f0) + i] = 4609 L2_L_PROTO | pa | 4610 L2_L_PROT(PTE_KERNEL, prot) | f2l; 4611 PTE_SYNC(&pte[((va >> PAGE_SHIFT) & 0x3f0) + i]); 4612#else 4613 pte[l2pte_index(va) + i] = 4614 L2_L_PROTO | pa | 4615 L2_L_PROT(PTE_KERNEL, prot) | f2l; 4616 PTE_SYNC(&pte[l2pte_index(va) + i]); 4617#endif 4618 } 4619 va += L2_L_SIZE; 4620 pa += L2_L_SIZE; 4621 resid -= L2_L_SIZE; 4622 continue; 4623 } 4624 4625 /* Use a small page mapping. */ 4626#ifdef VERBOSE_INIT_ARM 4627 printf("P"); 4628#endif 4629#ifndef ARM32_NEW_VM_LAYOUT 4630 pte[(va >> PAGE_SHIFT) & 0x3ff] = 4631 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 4632 PTE_SYNC(&pte[(va >> PAGE_SHIFT) & 0x3ff]); 4633#else 4634 pte[l2pte_index(va)] = 4635 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 4636 PTE_SYNC(&pte[l2pte_index(va)]); 4637#endif 4638 va += PAGE_SIZE; 4639 pa += PAGE_SIZE; 4640 resid -= PAGE_SIZE; 4641 } 4642#ifdef VERBOSE_INIT_ARM 4643 printf("\n"); 4644#endif 4645 return (size); 4646 4647} 4648 4649