pmap-v4.c revision 159474
1/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ 2/*- 3 * Copyright 2004 Olivier Houchard. 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38/*- 39 * Copyright (c) 2002-2003 Wasabi Systems, Inc. 40 * Copyright (c) 2001 Richard Earnshaw 41 * Copyright (c) 2001-2002 Christopher Gilbert 42 * All rights reserved. 43 * 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. The name of the company nor the name of the author may be used to 50 * endorse or promote products derived from this software without specific 51 * prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 54 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 55 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 56 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 57 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 58 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 59 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 */ 65/*- 66 * Copyright (c) 1999 The NetBSD Foundation, Inc. 67 * All rights reserved. 68 * 69 * This code is derived from software contributed to The NetBSD Foundation 70 * by Charles M. Hannum. 71 * 72 * Redistribution and use in source and binary forms, with or without 73 * modification, are permitted provided that the following conditions 74 * are met: 75 * 1. Redistributions of source code must retain the above copyright 76 * notice, this list of conditions and the following disclaimer. 77 * 2. Redistributions in binary form must reproduce the above copyright 78 * notice, this list of conditions and the following disclaimer in the 79 * documentation and/or other materials provided with the distribution. 80 * 3. All advertising materials mentioning features or use of this software 81 * must display the following acknowledgement: 82 * This product includes software developed by the NetBSD 83 * Foundation, Inc. and its contributors. 84 * 4. Neither the name of The NetBSD Foundation nor the names of its 85 * contributors may be used to endorse or promote products derived 86 * from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 89 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 90 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 91 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 92 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 93 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 94 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 95 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 96 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 97 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 98 * POSSIBILITY OF SUCH DAMAGE. 99 */ 100 101/*- 102 * Copyright (c) 1994-1998 Mark Brinicombe. 103 * Copyright (c) 1994 Brini. 104 * All rights reserved. 105 * 106 * This code is derived from software written for Brini by Mark Brinicombe 107 * 108 * Redistribution and use in source and binary forms, with or without 109 * modification, are permitted provided that the following conditions 110 * are met: 111 * 1. Redistributions of source code must retain the above copyright 112 * notice, this list of conditions and the following disclaimer. 113 * 2. Redistributions in binary form must reproduce the above copyright 114 * notice, this list of conditions and the following disclaimer in the 115 * documentation and/or other materials provided with the distribution. 116 * 3. All advertising materials mentioning features or use of this software 117 * must display the following acknowledgement: 118 * This product includes software developed by Mark Brinicombe. 119 * 4. The name of the author may not be used to endorse or promote products 120 * derived from this software without specific prior written permission. 121 * 122 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 123 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 124 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 125 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 126 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 127 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 128 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 129 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 130 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 131 * 132 * RiscBSD kernel project 133 * 134 * pmap.c 135 * 136 * Machine dependant vm stuff 137 * 138 * Created : 20/09/94 139 */ 140 141/* 142 * Special compilation symbols 143 * PMAP_DEBUG - Build in pmap_debug_level code 144 */ 145/* Include header files */ 146 147#include "opt_vm.h" 148 149#include <sys/cdefs.h> 150__FBSDID("$FreeBSD: head/sys/arm/arm/pmap.c 159474 2006-06-10 05:20:18Z alc $"); 151#include <sys/param.h> 152#include <sys/systm.h> 153#include <sys/kernel.h> 154#include <sys/proc.h> 155#include <sys/malloc.h> 156#include <sys/msgbuf.h> 157#include <sys/vmmeter.h> 158#include <sys/mman.h> 159#include <sys/smp.h> 160#include <sys/sx.h> 161#include <sys/sched.h> 162 163#include <vm/vm.h> 164#include <vm/uma.h> 165#include <vm/pmap.h> 166#include <vm/vm_kern.h> 167#include <vm/vm_object.h> 168#include <vm/vm_map.h> 169#include <vm/vm_page.h> 170#include <vm/vm_pageout.h> 171#include <vm/vm_extern.h> 172#include <sys/lock.h> 173#include <sys/mutex.h> 174#include <machine/md_var.h> 175#include <machine/vmparam.h> 176#include <machine/cpu.h> 177#include <machine/cpufunc.h> 178#include <machine/pcb.h> 179 180#ifdef PMAP_DEBUG 181#define PDEBUG(_lev_,_stat_) \ 182 if (pmap_debug_level >= (_lev_)) \ 183 ((_stat_)) 184#define dprintf printf 185 186int pmap_debug_level = 0; 187#define PMAP_INLINE 188#else /* PMAP_DEBUG */ 189#define PDEBUG(_lev_,_stat_) /* Nothing */ 190#define dprintf(x, arg...) 191#define PMAP_INLINE __inline 192#endif /* PMAP_DEBUG */ 193 194extern struct pv_addr systempage; 195/* 196 * Internal function prototypes 197 */ 198static void pmap_free_pv_entry (pv_entry_t); 199static pv_entry_t pmap_get_pv_entry(void); 200 201static void pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, 202 vm_prot_t, boolean_t); 203static void pmap_vac_me_harder(struct vm_page *, pmap_t, 204 vm_offset_t); 205static void pmap_vac_me_kpmap(struct vm_page *, pmap_t, 206 vm_offset_t); 207static void pmap_vac_me_user(struct vm_page *, pmap_t, vm_offset_t); 208static void pmap_alloc_l1(pmap_t); 209static void pmap_free_l1(pmap_t); 210static void pmap_use_l1(pmap_t); 211 212static int pmap_clearbit(struct vm_page *, u_int); 213 214static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); 215static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); 216static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); 217static vm_offset_t kernel_pt_lookup(vm_paddr_t); 218 219static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); 220 221vm_offset_t avail_end; /* PA of last available physical page */ 222vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ 223vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ 224vm_offset_t pmap_curmaxkvaddr; 225vm_paddr_t kernel_l1pa; 226 227extern void *end; 228vm_offset_t kernel_vm_end = 0; 229 230struct pmap kernel_pmap_store; 231pmap_t kernel_pmap; 232 233static pt_entry_t *csrc_pte, *cdst_pte; 234static vm_offset_t csrcp, cdstp; 235static struct mtx cmtx; 236 237static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); 238/* 239 * These routines are called when the CPU type is identified to set up 240 * the PTE prototypes, cache modes, etc. 241 * 242 * The variables are always here, just in case LKMs need to reference 243 * them (though, they shouldn't). 244 */ 245 246pt_entry_t pte_l1_s_cache_mode; 247pt_entry_t pte_l1_s_cache_mode_pt; 248pt_entry_t pte_l1_s_cache_mask; 249 250pt_entry_t pte_l2_l_cache_mode; 251pt_entry_t pte_l2_l_cache_mode_pt; 252pt_entry_t pte_l2_l_cache_mask; 253 254pt_entry_t pte_l2_s_cache_mode; 255pt_entry_t pte_l2_s_cache_mode_pt; 256pt_entry_t pte_l2_s_cache_mask; 257 258pt_entry_t pte_l2_s_prot_u; 259pt_entry_t pte_l2_s_prot_w; 260pt_entry_t pte_l2_s_prot_mask; 261 262pt_entry_t pte_l1_s_proto; 263pt_entry_t pte_l1_c_proto; 264pt_entry_t pte_l2_s_proto; 265 266void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); 267void (*pmap_zero_page_func)(vm_paddr_t, int, int); 268/* 269 * Which pmap is currently 'live' in the cache 270 * 271 * XXXSCW: Fix for SMP ... 272 */ 273union pmap_cache_state *pmap_cache_state; 274 275/* static pt_entry_t *msgbufmap;*/ 276struct msgbuf *msgbufp = 0; 277 278extern void bcopy_page(vm_offset_t, vm_offset_t); 279extern void bzero_page(vm_offset_t); 280 281char *_tmppt; 282 283/* 284 * Metadata for L1 translation tables. 285 */ 286struct l1_ttable { 287 /* Entry on the L1 Table list */ 288 SLIST_ENTRY(l1_ttable) l1_link; 289 290 /* Entry on the L1 Least Recently Used list */ 291 TAILQ_ENTRY(l1_ttable) l1_lru; 292 293 /* Track how many domains are allocated from this L1 */ 294 volatile u_int l1_domain_use_count; 295 296 /* 297 * A free-list of domain numbers for this L1. 298 * We avoid using ffs() and a bitmap to track domains since ffs() 299 * is slow on ARM. 300 */ 301 u_int8_t l1_domain_first; 302 u_int8_t l1_domain_free[PMAP_DOMAINS]; 303 304 /* Physical address of this L1 page table */ 305 vm_paddr_t l1_physaddr; 306 307 /* KVA of this L1 page table */ 308 pd_entry_t *l1_kva; 309}; 310 311/* 312 * Convert a virtual address into its L1 table index. That is, the 313 * index used to locate the L2 descriptor table pointer in an L1 table. 314 * This is basically used to index l1->l1_kva[]. 315 * 316 * Each L2 descriptor table represents 1MB of VA space. 317 */ 318#define L1_IDX(va) (((vm_offset_t)(va)) >> L1_S_SHIFT) 319 320/* 321 * L1 Page Tables are tracked using a Least Recently Used list. 322 * - New L1s are allocated from the HEAD. 323 * - Freed L1s are added to the TAIl. 324 * - Recently accessed L1s (where an 'access' is some change to one of 325 * the userland pmaps which owns this L1) are moved to the TAIL. 326 */ 327static TAILQ_HEAD(, l1_ttable) l1_lru_list; 328/* 329 * A list of all L1 tables 330 */ 331static SLIST_HEAD(, l1_ttable) l1_list; 332static struct mtx l1_lru_lock; 333 334/* 335 * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. 336 * 337 * This is normally 16MB worth L2 page descriptors for any given pmap. 338 * Reference counts are maintained for L2 descriptors so they can be 339 * freed when empty. 340 */ 341struct l2_dtable { 342 /* The number of L2 page descriptors allocated to this l2_dtable */ 343 u_int l2_occupancy; 344 345 /* List of L2 page descriptors */ 346 struct l2_bucket { 347 pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ 348 vm_paddr_t l2b_phys; /* Physical address of same */ 349 u_short l2b_l1idx; /* This L2 table's L1 index */ 350 u_short l2b_occupancy; /* How many active descriptors */ 351 } l2_bucket[L2_BUCKET_SIZE]; 352}; 353 354/* pmap_kenter_internal flags */ 355#define KENTER_CACHE 0x1 356#define KENTER_USER 0x2 357 358/* 359 * Given an L1 table index, calculate the corresponding l2_dtable index 360 * and bucket index within the l2_dtable. 361 */ 362#define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ 363 (L2_SIZE - 1)) 364#define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) 365 366/* 367 * Given a virtual address, this macro returns the 368 * virtual address required to drop into the next L2 bucket. 369 */ 370#define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) 371 372/* 373 * L2 allocation. 374 */ 375#define pmap_alloc_l2_dtable() \ 376 (void*)uma_zalloc(l2table_zone, M_NOWAIT) 377#define pmap_free_l2_dtable(l2) \ 378 uma_zfree(l2table_zone, l2) 379 380/* 381 * We try to map the page tables write-through, if possible. However, not 382 * all CPUs have a write-through cache mode, so on those we have to sync 383 * the cache when we frob page tables. 384 * 385 * We try to evaluate this at compile time, if possible. However, it's 386 * not always possible to do that, hence this run-time var. 387 */ 388int pmap_needs_pte_sync; 389 390/* 391 * Macro to determine if a mapping might be resident in the 392 * instruction cache and/or TLB 393 */ 394#define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) 395 396/* 397 * Macro to determine if a mapping might be resident in the 398 * data cache and/or TLB 399 */ 400#define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) 401 402#ifndef PMAP_SHPGPERPROC 403#define PMAP_SHPGPERPROC 200 404#endif 405 406#define pmap_is_current(pm) ((pm) == pmap_kernel() || \ 407 curproc->p_vmspace->vm_map.pmap == (pm)) 408static uma_zone_t pvzone; 409uma_zone_t l2zone; 410static uma_zone_t l2table_zone; 411static vm_offset_t pmap_kernel_l2dtable_kva; 412static vm_offset_t pmap_kernel_l2ptp_kva; 413static vm_paddr_t pmap_kernel_l2ptp_phys; 414static struct vm_object pvzone_obj; 415static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; 416int pmap_pagedaemon_waken = 0; 417 418/* 419 * This list exists for the benefit of pmap_map_chunk(). It keeps track 420 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 421 * find them as necessary. 422 * 423 * Note that the data on this list MUST remain valid after initarm() returns, 424 * as pmap_bootstrap() uses it to contruct L2 table metadata. 425 */ 426SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 427 428static void 429pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) 430{ 431 int i; 432 433 l1->l1_kva = l1pt; 434 l1->l1_domain_use_count = 0; 435 l1->l1_domain_first = 0; 436 437 for (i = 0; i < PMAP_DOMAINS; i++) 438 l1->l1_domain_free[i] = i + 1; 439 440 /* 441 * Copy the kernel's L1 entries to each new L1. 442 */ 443 if (l1pt != pmap_kernel()->pm_l1->l1_kva) 444 memcpy(l1pt, pmap_kernel()->pm_l1->l1_kva, L1_TABLE_SIZE); 445 446 if ((l1->l1_physaddr = pmap_extract(pmap_kernel(), (vm_offset_t)l1pt)) == 0) 447 panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); 448 SLIST_INSERT_HEAD(&l1_list, l1, l1_link); 449 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 450} 451 452static vm_offset_t 453kernel_pt_lookup(vm_paddr_t pa) 454{ 455 struct pv_addr *pv; 456 457 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 458 if (pv->pv_pa == pa) 459 return (pv->pv_va); 460 } 461 return (0); 462} 463 464#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 465void 466pmap_pte_init_generic(void) 467{ 468 469 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 470 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 471 472 pte_l2_l_cache_mode = L2_B|L2_C; 473 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 474 475 pte_l2_s_cache_mode = L2_B|L2_C; 476 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 477 478 /* 479 * If we have a write-through cache, set B and C. If 480 * we have a write-back cache, then we assume setting 481 * only C will make those pages write-through. 482 */ 483 if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { 484 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 485 pte_l2_l_cache_mode_pt = L2_B|L2_C; 486 pte_l2_s_cache_mode_pt = L2_B|L2_C; 487 } else { 488 pte_l1_s_cache_mode_pt = L1_S_C; 489 pte_l2_l_cache_mode_pt = L2_C; 490 pte_l2_s_cache_mode_pt = L2_C; 491 } 492 493 pte_l2_s_prot_u = L2_S_PROT_U_generic; 494 pte_l2_s_prot_w = L2_S_PROT_W_generic; 495 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 496 497 pte_l1_s_proto = L1_S_PROTO_generic; 498 pte_l1_c_proto = L1_C_PROTO_generic; 499 pte_l2_s_proto = L2_S_PROTO_generic; 500 501 pmap_copy_page_func = pmap_copy_page_generic; 502 pmap_zero_page_func = pmap_zero_page_generic; 503} 504 505#if defined(CPU_ARM8) 506void 507pmap_pte_init_arm8(void) 508{ 509 510 /* 511 * ARM8 is compatible with generic, but we need to use 512 * the page tables uncached. 513 */ 514 pmap_pte_init_generic(); 515 516 pte_l1_s_cache_mode_pt = 0; 517 pte_l2_l_cache_mode_pt = 0; 518 pte_l2_s_cache_mode_pt = 0; 519} 520#endif /* CPU_ARM8 */ 521 522#if defined(CPU_ARM9) && defined(ARM9_CACHE_WRITE_THROUGH) 523void 524pmap_pte_init_arm9(void) 525{ 526 527 /* 528 * ARM9 is compatible with generic, but we want to use 529 * write-through caching for now. 530 */ 531 pmap_pte_init_generic(); 532 533 pte_l1_s_cache_mode = L1_S_C; 534 pte_l2_l_cache_mode = L2_C; 535 pte_l2_s_cache_mode = L2_C; 536 537 pte_l1_s_cache_mode_pt = L1_S_C; 538 pte_l2_l_cache_mode_pt = L2_C; 539 pte_l2_s_cache_mode_pt = L2_C; 540} 541#endif /* CPU_ARM9 */ 542#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 543 544#if defined(CPU_ARM10) 545void 546pmap_pte_init_arm10(void) 547{ 548 549 /* 550 * ARM10 is compatible with generic, but we want to use 551 * write-through caching for now. 552 */ 553 pmap_pte_init_generic(); 554 555 pte_l1_s_cache_mode = L1_S_B | L1_S_C; 556 pte_l2_l_cache_mode = L2_B | L2_C; 557 pte_l2_s_cache_mode = L2_B | L2_C; 558 559 pte_l1_s_cache_mode_pt = L1_S_C; 560 pte_l2_l_cache_mode_pt = L2_C; 561 pte_l2_s_cache_mode_pt = L2_C; 562 563} 564#endif /* CPU_ARM10 */ 565 566#if ARM_MMU_SA1 == 1 567void 568pmap_pte_init_sa1(void) 569{ 570 571 /* 572 * The StrongARM SA-1 cache does not have a write-through 573 * mode. So, do the generic initialization, then reset 574 * the page table cache mode to B=1,C=1, and note that 575 * the PTEs need to be sync'd. 576 */ 577 pmap_pte_init_generic(); 578 579 pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; 580 pte_l2_l_cache_mode_pt = L2_B|L2_C; 581 pte_l2_s_cache_mode_pt = L2_B|L2_C; 582 583 pmap_needs_pte_sync = 1; 584} 585#endif /* ARM_MMU_SA1 == 1*/ 586 587#if ARM_MMU_XSCALE == 1 588#if (ARM_NMMUS > 1) 589static u_int xscale_use_minidata; 590#endif 591 592void 593pmap_pte_init_xscale(void) 594{ 595 uint32_t auxctl; 596 int write_through = 0; 597 598 pte_l1_s_cache_mode = L1_S_B|L1_S_C|L1_S_XSCALE_P; 599 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 600 601 pte_l2_l_cache_mode = L2_B|L2_C; 602 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 603 604 pte_l2_s_cache_mode = L2_B|L2_C; 605 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 606 607 pte_l1_s_cache_mode_pt = L1_S_C; 608 pte_l2_l_cache_mode_pt = L2_C; 609 pte_l2_s_cache_mode_pt = L2_C; 610#ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE 611 /* 612 * The XScale core has an enhanced mode where writes that 613 * miss the cache cause a cache line to be allocated. This 614 * is significantly faster than the traditional, write-through 615 * behavior of this case. 616 */ 617 pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); 618 pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); 619 pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); 620#endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ 621#ifdef XSCALE_CACHE_WRITE_THROUGH 622 /* 623 * Some versions of the XScale core have various bugs in 624 * their cache units, the work-around for which is to run 625 * the cache in write-through mode. Unfortunately, this 626 * has a major (negative) impact on performance. So, we 627 * go ahead and run fast-and-loose, in the hopes that we 628 * don't line up the planets in a way that will trip the 629 * bugs. 630 * 631 * However, we give you the option to be slow-but-correct. 632 */ 633 write_through = 1; 634#elif defined(XSCALE_CACHE_WRITE_BACK) 635 /* force write back cache mode */ 636 write_through = 0; 637#elif defined(CPU_XSCALE_PXA2X0) 638 /* 639 * Intel PXA2[15]0 processors are known to have a bug in 640 * write-back cache on revision 4 and earlier (stepping 641 * A[01] and B[012]). Fixed for C0 and later. 642 */ 643 { 644 uint32_t id, type; 645 646 id = cpufunc_id(); 647 type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); 648 649 if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { 650 if ((id & CPU_ID_REVISION_MASK) < 5) { 651 /* write through for stepping A0-1 and B0-2 */ 652 write_through = 1; 653 } 654 } 655 } 656#endif /* XSCALE_CACHE_WRITE_THROUGH */ 657 658 if (write_through) { 659 pte_l1_s_cache_mode = L1_S_C; 660 pte_l2_l_cache_mode = L2_C; 661 pte_l2_s_cache_mode = L2_C; 662 } 663 664#if (ARM_NMMUS > 1) 665 xscale_use_minidata = 1; 666#endif 667 668 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 669 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 670 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 671 672 pte_l1_s_proto = L1_S_PROTO_xscale; 673 pte_l1_c_proto = L1_C_PROTO_xscale; 674 pte_l2_s_proto = L2_S_PROTO_xscale; 675 676 pmap_copy_page_func = pmap_copy_page_xscale; 677 pmap_zero_page_func = pmap_zero_page_xscale; 678 679 /* 680 * Disable ECC protection of page table access, for now. 681 */ 682 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 683 auxctl &= ~XSCALE_AUXCTL_P; 684 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 685} 686 687/* 688 * xscale_setup_minidata: 689 * 690 * Set up the mini-data cache clean area. We require the 691 * caller to allocate the right amount of physically and 692 * virtually contiguous space. 693 */ 694extern vm_offset_t xscale_minidata_clean_addr; 695extern vm_size_t xscale_minidata_clean_size; /* already initialized */ 696void 697xscale_setup_minidata(vm_offset_t l1pt, vm_offset_t va, vm_paddr_t pa) 698{ 699 pd_entry_t *pde = (pd_entry_t *) l1pt; 700 pt_entry_t *pte; 701 vm_size_t size; 702 uint32_t auxctl; 703 704 xscale_minidata_clean_addr = va; 705 706 /* Round it to page size. */ 707 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 708 709 for (; size != 0; 710 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 711 pte = (pt_entry_t *) kernel_pt_lookup( 712 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 713 if (pte == NULL) 714 panic("xscale_setup_minidata: can't find L2 table for " 715 "VA 0x%08x", (u_int32_t) va); 716 pte[l2pte_index(va)] = 717 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 718 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 719 } 720 721 /* 722 * Configure the mini-data cache for write-back with 723 * read/write-allocate. 724 * 725 * NOTE: In order to reconfigure the mini-data cache, we must 726 * make sure it contains no valid data! In order to do that, 727 * we must issue a global data cache invalidate command! 728 * 729 * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! 730 * THIS IS VERY IMPORTANT! 731 */ 732 733 /* Invalidate data and mini-data. */ 734 __asm __volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); 735 __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); 736 auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; 737 __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); 738} 739#endif 740 741/* 742 * Allocate an L1 translation table for the specified pmap. 743 * This is called at pmap creation time. 744 */ 745static void 746pmap_alloc_l1(pmap_t pm) 747{ 748 struct l1_ttable *l1; 749 u_int8_t domain; 750 751 /* 752 * Remove the L1 at the head of the LRU list 753 */ 754 mtx_lock(&l1_lru_lock); 755 l1 = TAILQ_FIRST(&l1_lru_list); 756 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 757 758 /* 759 * Pick the first available domain number, and update 760 * the link to the next number. 761 */ 762 domain = l1->l1_domain_first; 763 l1->l1_domain_first = l1->l1_domain_free[domain]; 764 765 /* 766 * If there are still free domain numbers in this L1, 767 * put it back on the TAIL of the LRU list. 768 */ 769 if (++l1->l1_domain_use_count < PMAP_DOMAINS) 770 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 771 772 mtx_unlock(&l1_lru_lock); 773 774 /* 775 * Fix up the relevant bits in the pmap structure 776 */ 777 pm->pm_l1 = l1; 778 pm->pm_domain = domain; 779} 780 781/* 782 * Free an L1 translation table. 783 * This is called at pmap destruction time. 784 */ 785static void 786pmap_free_l1(pmap_t pm) 787{ 788 struct l1_ttable *l1 = pm->pm_l1; 789 790 mtx_lock(&l1_lru_lock); 791 792 /* 793 * If this L1 is currently on the LRU list, remove it. 794 */ 795 if (l1->l1_domain_use_count < PMAP_DOMAINS) 796 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 797 798 /* 799 * Free up the domain number which was allocated to the pmap 800 */ 801 l1->l1_domain_free[pm->pm_domain] = l1->l1_domain_first; 802 l1->l1_domain_first = pm->pm_domain; 803 l1->l1_domain_use_count--; 804 805 /* 806 * The L1 now must have at least 1 free domain, so add 807 * it back to the LRU list. If the use count is zero, 808 * put it at the head of the list, otherwise it goes 809 * to the tail. 810 */ 811 if (l1->l1_domain_use_count == 0) { 812 TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); 813 } else 814 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 815 816 mtx_unlock(&l1_lru_lock); 817} 818 819static PMAP_INLINE void 820pmap_use_l1(pmap_t pm) 821{ 822 struct l1_ttable *l1; 823 824 /* 825 * Do nothing if we're in interrupt context. 826 * Access to an L1 by the kernel pmap must not affect 827 * the LRU list. 828 */ 829 if (pm == pmap_kernel()) 830 return; 831 832 l1 = pm->pm_l1; 833 834 /* 835 * If the L1 is not currently on the LRU list, just return 836 */ 837 if (l1->l1_domain_use_count == PMAP_DOMAINS) 838 return; 839 840 mtx_lock(&l1_lru_lock); 841 842 /* 843 * Check the use count again, now that we've acquired the lock 844 */ 845 if (l1->l1_domain_use_count == PMAP_DOMAINS) { 846 mtx_unlock(&l1_lru_lock); 847 return; 848 } 849 850 /* 851 * Move the L1 to the back of the LRU list 852 */ 853 TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); 854 TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); 855 856 mtx_unlock(&l1_lru_lock); 857} 858 859 860/* 861 * Returns a pointer to the L2 bucket associated with the specified pmap 862 * and VA, or NULL if no L2 bucket exists for the address. 863 */ 864static PMAP_INLINE struct l2_bucket * 865pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) 866{ 867 struct l2_dtable *l2; 868 struct l2_bucket *l2b; 869 u_short l1idx; 870 871 l1idx = L1_IDX(va); 872 873 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || 874 (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) 875 return (NULL); 876 877 return (l2b); 878} 879 880/* 881 * Returns a pointer to the L2 bucket associated with the specified pmap 882 * and VA. 883 * 884 * If no L2 bucket exists, perform the necessary allocations to put an L2 885 * bucket/page table in place. 886 * 887 * Note that if a new L2 bucket/page was allocated, the caller *must* 888 * increment the bucket occupancy counter appropriately *before* 889 * releasing the pmap's lock to ensure no other thread or cpu deallocates 890 * the bucket/page in the meantime. 891 */ 892static struct l2_bucket * 893pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) 894{ 895 struct l2_dtable *l2; 896 struct l2_bucket *l2b; 897 u_short l1idx; 898 899 l1idx = L1_IDX(va); 900 901 PMAP_ASSERT_LOCKED(pm); 902 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 903 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 904 /* 905 * No mapping at this address, as there is 906 * no entry in the L1 table. 907 * Need to allocate a new l2_dtable. 908 */ 909again_l2table: 910 PMAP_UNLOCK(pm); 911 vm_page_unlock_queues(); 912 if ((l2 = pmap_alloc_l2_dtable()) == NULL) { 913 vm_page_lock_queues(); 914 PMAP_LOCK(pm); 915 return (NULL); 916 } 917 vm_page_lock_queues(); 918 PMAP_LOCK(pm); 919 if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { 920 PMAP_UNLOCK(pm); 921 vm_page_unlock_queues(); 922 uma_zfree(l2table_zone, l2); 923 vm_page_lock_queues(); 924 PMAP_LOCK(pm); 925 l2 = pm->pm_l2[L2_IDX(l1idx)]; 926 if (l2 == NULL) 927 goto again_l2table; 928 /* 929 * Someone already allocated the l2_dtable while 930 * we were doing the same. 931 */ 932 } else { 933 bzero(l2, sizeof(*l2)); 934 /* 935 * Link it into the parent pmap 936 */ 937 pm->pm_l2[L2_IDX(l1idx)] = l2; 938 } 939 } 940 941 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 942 943 /* 944 * Fetch pointer to the L2 page table associated with the address. 945 */ 946 if (l2b->l2b_kva == NULL) { 947 pt_entry_t *ptep; 948 949 /* 950 * No L2 page table has been allocated. Chances are, this 951 * is because we just allocated the l2_dtable, above. 952 */ 953again_ptep: 954 PMAP_UNLOCK(pm); 955 vm_page_unlock_queues(); 956 ptep = (void*)uma_zalloc(l2zone, M_NOWAIT); 957 vm_page_lock_queues(); 958 PMAP_LOCK(pm); 959 if (l2b->l2b_kva != 0) { 960 /* We lost the race. */ 961 PMAP_UNLOCK(pm); 962 vm_page_unlock_queues(); 963 uma_zfree(l2zone, ptep); 964 vm_page_lock_queues(); 965 PMAP_LOCK(pm); 966 if (l2b->l2b_kva == 0) 967 goto again_ptep; 968 return (l2b); 969 } 970 l2b->l2b_phys = vtophys(ptep); 971 if (ptep == NULL) { 972 /* 973 * Oops, no more L2 page tables available at this 974 * time. We may need to deallocate the l2_dtable 975 * if we allocated a new one above. 976 */ 977 if (l2->l2_occupancy == 0) { 978 pm->pm_l2[L2_IDX(l1idx)] = NULL; 979 pmap_free_l2_dtable(l2); 980 } 981 return (NULL); 982 } 983 984 l2->l2_occupancy++; 985 l2b->l2b_kva = ptep; 986 l2b->l2b_l1idx = l1idx; 987 } 988 989 return (l2b); 990} 991 992static PMAP_INLINE void 993#ifndef PMAP_INCLUDE_PTE_SYNC 994pmap_free_l2_ptp(pt_entry_t *l2) 995#else 996pmap_free_l2_ptp(boolean_t need_sync, pt_entry_t *l2) 997#endif 998{ 999#ifdef PMAP_INCLUDE_PTE_SYNC 1000 /* 1001 * Note: With a write-back cache, we may need to sync this 1002 * L2 table before re-using it. 1003 * This is because it may have belonged to a non-current 1004 * pmap, in which case the cache syncs would have been 1005 * skipped when the pages were being unmapped. If the 1006 * L2 table were then to be immediately re-allocated to 1007 * the *current* pmap, it may well contain stale mappings 1008 * which have not yet been cleared by a cache write-back 1009 * and so would still be visible to the mmu. 1010 */ 1011 if (need_sync) 1012 PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1013#endif 1014 uma_zfree(l2zone, l2); 1015} 1016/* 1017 * One or more mappings in the specified L2 descriptor table have just been 1018 * invalidated. 1019 * 1020 * Garbage collect the metadata and descriptor table itself if necessary. 1021 * 1022 * The pmap lock must be acquired when this is called (not necessary 1023 * for the kernel pmap). 1024 */ 1025static void 1026pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) 1027{ 1028 struct l2_dtable *l2; 1029 pd_entry_t *pl1pd, l1pd; 1030 pt_entry_t *ptep; 1031 u_short l1idx; 1032 1033 1034 /* 1035 * Update the bucket's reference count according to how many 1036 * PTEs the caller has just invalidated. 1037 */ 1038 l2b->l2b_occupancy -= count; 1039 1040 /* 1041 * Note: 1042 * 1043 * Level 2 page tables allocated to the kernel pmap are never freed 1044 * as that would require checking all Level 1 page tables and 1045 * removing any references to the Level 2 page table. See also the 1046 * comment elsewhere about never freeing bootstrap L2 descriptors. 1047 * 1048 * We make do with just invalidating the mapping in the L2 table. 1049 * 1050 * This isn't really a big deal in practice and, in fact, leads 1051 * to a performance win over time as we don't need to continually 1052 * alloc/free. 1053 */ 1054 if (l2b->l2b_occupancy > 0 || pm == pmap_kernel()) 1055 return; 1056 1057 /* 1058 * There are no more valid mappings in this level 2 page table. 1059 * Go ahead and NULL-out the pointer in the bucket, then 1060 * free the page table. 1061 */ 1062 l1idx = l2b->l2b_l1idx; 1063 ptep = l2b->l2b_kva; 1064 l2b->l2b_kva = NULL; 1065 1066 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 1067 1068 /* 1069 * If the L1 slot matches the pmap's domain 1070 * number, then invalidate it. 1071 */ 1072 l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); 1073 if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { 1074 *pl1pd = 0; 1075 PTE_SYNC(pl1pd); 1076 } 1077 1078 /* 1079 * Release the L2 descriptor table back to the pool cache. 1080 */ 1081#ifndef PMAP_INCLUDE_PTE_SYNC 1082 pmap_free_l2_ptp(ptep); 1083#else 1084 pmap_free_l2_ptp(!pmap_is_current(pm), ptep); 1085#endif 1086 1087 /* 1088 * Update the reference count in the associated l2_dtable 1089 */ 1090 l2 = pm->pm_l2[L2_IDX(l1idx)]; 1091 if (--l2->l2_occupancy > 0) 1092 return; 1093 1094 /* 1095 * There are no more valid mappings in any of the Level 1 1096 * slots managed by this l2_dtable. Go ahead and NULL-out 1097 * the pointer in the parent pmap and free the l2_dtable. 1098 */ 1099 pm->pm_l2[L2_IDX(l1idx)] = NULL; 1100 pmap_free_l2_dtable(l2); 1101} 1102 1103/* 1104 * Pool cache constructors for L2 descriptor tables, metadata and pmap 1105 * structures. 1106 */ 1107static int 1108pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) 1109{ 1110#ifndef PMAP_INCLUDE_PTE_SYNC 1111 struct l2_bucket *l2b; 1112 pt_entry_t *ptep, pte; 1113#ifdef ARM_USE_SMALL_ALLOC 1114 pd_entry_t *pde; 1115#endif 1116 vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; 1117 1118 /* 1119 * The mappings for these page tables were initially made using 1120 * pmap_kenter() by the pool subsystem. Therefore, the cache- 1121 * mode will not be right for page table mappings. To avoid 1122 * polluting the pmap_kenter() code with a special case for 1123 * page tables, we simply fix up the cache-mode here if it's not 1124 * correct. 1125 */ 1126#ifdef ARM_USE_SMALL_ALLOC 1127 pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)]; 1128 if (!l1pte_section_p(*pde)) { 1129#endif 1130 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 1131 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1132 pte = *ptep; 1133 1134 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 1135 /* 1136 * Page tables must have the cache-mode set to 1137 * Write-Thru. 1138 */ 1139 *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 1140 PTE_SYNC(ptep); 1141 cpu_tlb_flushD_SE(va); 1142 cpu_cpwait(); 1143 } 1144 1145#ifdef ARM_USE_SMALL_ALLOC 1146 } 1147#endif 1148#endif 1149 memset(mem, 0, L2_TABLE_SIZE_REAL); 1150 PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 1151 return (0); 1152} 1153 1154/* 1155 * A bunch of routines to conditionally flush the caches/TLB depending 1156 * on whether the specified pmap actually needs to be flushed at any 1157 * given time. 1158 */ 1159static PMAP_INLINE void 1160pmap_tlb_flushID_SE(pmap_t pm, vm_offset_t va) 1161{ 1162 1163 if (pmap_is_current(pm)) 1164 cpu_tlb_flushID_SE(va); 1165} 1166 1167static PMAP_INLINE void 1168pmap_tlb_flushD_SE(pmap_t pm, vm_offset_t va) 1169{ 1170 1171 if (pmap_is_current(pm)) 1172 cpu_tlb_flushD_SE(va); 1173} 1174 1175static PMAP_INLINE void 1176pmap_tlb_flushID(pmap_t pm) 1177{ 1178 1179 if (pmap_is_current(pm)) 1180 cpu_tlb_flushID(); 1181} 1182static PMAP_INLINE void 1183pmap_tlb_flushD(pmap_t pm) 1184{ 1185 1186 if (pmap_is_current(pm)) 1187 cpu_tlb_flushD(); 1188} 1189 1190static PMAP_INLINE void 1191pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) 1192{ 1193 1194 if (pmap_is_current(pm)) 1195 cpu_idcache_wbinv_range(va, len); 1196} 1197 1198static PMAP_INLINE void 1199pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, 1200 boolean_t do_inv, boolean_t rd_only) 1201{ 1202 1203 if (pmap_is_current(pm)) { 1204 if (do_inv) { 1205 if (rd_only) 1206 cpu_dcache_inv_range(va, len); 1207 else 1208 cpu_dcache_wbinv_range(va, len); 1209 } else 1210 if (!rd_only) 1211 cpu_dcache_wb_range(va, len); 1212 } 1213} 1214 1215static PMAP_INLINE void 1216pmap_idcache_wbinv_all(pmap_t pm) 1217{ 1218 1219 if (pmap_is_current(pm)) 1220 cpu_idcache_wbinv_all(); 1221} 1222 1223static PMAP_INLINE void 1224pmap_dcache_wbinv_all(pmap_t pm) 1225{ 1226 1227 if (pmap_is_current(pm)) 1228 cpu_dcache_wbinv_all(); 1229} 1230 1231/* 1232 * PTE_SYNC_CURRENT: 1233 * 1234 * Make sure the pte is written out to RAM. 1235 * We need to do this for one of two cases: 1236 * - We're dealing with the kernel pmap 1237 * - There is no pmap active in the cache/tlb. 1238 * - The specified pmap is 'active' in the cache/tlb. 1239 */ 1240#ifdef PMAP_INCLUDE_PTE_SYNC 1241#define PTE_SYNC_CURRENT(pm, ptep) \ 1242do { \ 1243 if (PMAP_NEEDS_PTE_SYNC && \ 1244 pmap_is_current(pm)) \ 1245 PTE_SYNC(ptep); \ 1246} while (/*CONSTCOND*/0) 1247#else 1248#define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ 1249#endif 1250 1251/* 1252 * Since we have a virtually indexed cache, we may need to inhibit caching if 1253 * there is more than one mapping and at least one of them is writable. 1254 * Since we purge the cache on every context switch, we only need to check for 1255 * other mappings within the same pmap, or kernel_pmap. 1256 * This function is also called when a page is unmapped, to possibly reenable 1257 * caching on any remaining mappings. 1258 * 1259 * The code implements the following logic, where: 1260 * 1261 * KW = # of kernel read/write pages 1262 * KR = # of kernel read only pages 1263 * UW = # of user read/write pages 1264 * UR = # of user read only pages 1265 * 1266 * KC = kernel mapping is cacheable 1267 * UC = user mapping is cacheable 1268 * 1269 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 1270 * +--------------------------------------------- 1271 * UW=0,UR=0 | --- KC=1 KC=1 KC=0 1272 * UW=0,UR>0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 1273 * UW=1,UR=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1274 * UW>1,UR>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 1275 */ 1276 1277static const int pmap_vac_flags[4][4] = { 1278 {-1, 0, 0, PVF_KNC}, 1279 {0, 0, PVF_NC, PVF_NC}, 1280 {0, PVF_NC, PVF_NC, PVF_NC}, 1281 {PVF_UNC, PVF_NC, PVF_NC, PVF_NC} 1282}; 1283 1284static PMAP_INLINE int 1285pmap_get_vac_flags(const struct vm_page *pg) 1286{ 1287 int kidx, uidx; 1288 1289 kidx = 0; 1290 if (pg->md.kro_mappings || pg->md.krw_mappings > 1) 1291 kidx |= 1; 1292 if (pg->md.krw_mappings) 1293 kidx |= 2; 1294 1295 uidx = 0; 1296 if (pg->md.uro_mappings || pg->md.urw_mappings > 1) 1297 uidx |= 1; 1298 if (pg->md.urw_mappings) 1299 uidx |= 2; 1300 1301 return (pmap_vac_flags[uidx][kidx]); 1302} 1303 1304static __inline void 1305pmap_vac_me_harder(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1306{ 1307 int nattr; 1308 1309 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1310 nattr = pmap_get_vac_flags(pg); 1311 1312 if (nattr < 0) { 1313 pg->md.pvh_attrs &= ~PVF_NC; 1314 return; 1315 } 1316 1317 if (nattr == 0 && (pg->md.pvh_attrs & PVF_NC) == 0) { 1318 return; 1319 } 1320 1321 if (pm == pmap_kernel()) 1322 pmap_vac_me_kpmap(pg, pm, va); 1323 else 1324 pmap_vac_me_user(pg, pm, va); 1325 1326 pg->md.pvh_attrs = (pg->md.pvh_attrs & ~PVF_NC) | nattr; 1327} 1328 1329static void 1330pmap_vac_me_kpmap(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1331{ 1332 u_int u_cacheable, u_entries; 1333 struct pv_entry *pv; 1334 pmap_t last_pmap = pm; 1335 1336 /* 1337 * Pass one, see if there are both kernel and user pmaps for 1338 * this page. Calculate whether there are user-writable or 1339 * kernel-writable pages. 1340 */ 1341 u_cacheable = 0; 1342 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1343 if (pv->pv_pmap != pm && (pv->pv_flags & PVF_NC) == 0) 1344 u_cacheable++; 1345 } 1346 1347 u_entries = pg->md.urw_mappings + pg->md.uro_mappings; 1348 1349 /* 1350 * We know we have just been updating a kernel entry, so if 1351 * all user pages are already cacheable, then there is nothing 1352 * further to do. 1353 */ 1354 if (pg->md.k_mappings == 0 && u_cacheable == u_entries) 1355 return; 1356 1357 if (u_entries) { 1358 /* 1359 * Scan over the list again, for each entry, if it 1360 * might not be set correctly, call pmap_vac_me_user 1361 * to recalculate the settings. 1362 */ 1363 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1364 /* 1365 * We know kernel mappings will get set 1366 * correctly in other calls. We also know 1367 * that if the pmap is the same as last_pmap 1368 * then we've just handled this entry. 1369 */ 1370 if (pv->pv_pmap == pm || pv->pv_pmap == last_pmap) 1371 continue; 1372 1373 /* 1374 * If there are kernel entries and this page 1375 * is writable but non-cacheable, then we can 1376 * skip this entry also. 1377 */ 1378 if (pg->md.k_mappings && 1379 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 1380 (PVF_NC | PVF_WRITE)) 1381 continue; 1382 1383 /* 1384 * Similarly if there are no kernel-writable 1385 * entries and the page is already 1386 * read-only/cacheable. 1387 */ 1388 if (pg->md.krw_mappings == 0 && 1389 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 1390 continue; 1391 1392 /* 1393 * For some of the remaining cases, we know 1394 * that we must recalculate, but for others we 1395 * can't tell if they are correct or not, so 1396 * we recalculate anyway. 1397 */ 1398 pmap_vac_me_user(pg, (last_pmap = pv->pv_pmap), 0); 1399 } 1400 1401 if (pg->md.k_mappings == 0) 1402 return; 1403 } 1404 1405 pmap_vac_me_user(pg, pm, va); 1406} 1407 1408static void 1409pmap_vac_me_user(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1410{ 1411 pmap_t kpmap = pmap_kernel(); 1412 struct pv_entry *pv, *npv; 1413 struct l2_bucket *l2b; 1414 pt_entry_t *ptep, pte; 1415 u_int entries = 0; 1416 u_int writable = 0; 1417 u_int cacheable_entries = 0; 1418 u_int kern_cacheable = 0; 1419 u_int other_writable = 0; 1420 1421 /* 1422 * Count mappings and writable mappings in this pmap. 1423 * Include kernel mappings as part of our own. 1424 * Keep a pointer to the first one. 1425 */ 1426 npv = TAILQ_FIRST(&pg->md.pv_list); 1427 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1428 /* Count mappings in the same pmap */ 1429 if (pm == pv->pv_pmap || kpmap == pv->pv_pmap) { 1430 if (entries++ == 0) 1431 npv = pv; 1432 1433 /* Cacheable mappings */ 1434 if ((pv->pv_flags & PVF_NC) == 0) { 1435 cacheable_entries++; 1436 if (kpmap == pv->pv_pmap) 1437 kern_cacheable++; 1438 } 1439 1440 /* Writable mappings */ 1441 if (pv->pv_flags & PVF_WRITE) 1442 ++writable; 1443 } else 1444 if (pv->pv_flags & PVF_WRITE) 1445 other_writable = 1; 1446 } 1447 1448 /* 1449 * Enable or disable caching as necessary. 1450 * Note: the first entry might be part of the kernel pmap, 1451 * so we can't assume this is indicative of the state of the 1452 * other (maybe non-kpmap) entries. 1453 */ 1454 if ((entries > 1 && writable) || 1455 (entries > 0 && pm == kpmap && other_writable)) { 1456 if (cacheable_entries == 0) 1457 return; 1458 1459 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1460 if ((pm != pv->pv_pmap && kpmap != pv->pv_pmap) || 1461 (pv->pv_flags & PVF_NC)) 1462 continue; 1463 1464 pv->pv_flags |= PVF_NC; 1465 1466 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1467 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1468 pte = *ptep & ~L2_S_CACHE_MASK; 1469 1470 if ((va != pv->pv_va || pm != pv->pv_pmap) && 1471 l2pte_valid(pte)) { 1472 if (PV_BEEN_EXECD(pv->pv_flags)) { 1473 pmap_idcache_wbinv_range(pv->pv_pmap, 1474 pv->pv_va, PAGE_SIZE); 1475 pmap_tlb_flushID_SE(pv->pv_pmap, 1476 pv->pv_va); 1477 } else 1478 if (PV_BEEN_REFD(pv->pv_flags)) { 1479 pmap_dcache_wb_range(pv->pv_pmap, 1480 pv->pv_va, PAGE_SIZE, TRUE, 1481 (pv->pv_flags & PVF_WRITE) == 0); 1482 pmap_tlb_flushD_SE(pv->pv_pmap, 1483 pv->pv_va); 1484 } 1485 } 1486 1487 *ptep = pte; 1488 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1489 } 1490 cpu_cpwait(); 1491 } else 1492 if (entries > cacheable_entries) { 1493 /* 1494 * Turn cacheing back on for some pages. If it is a kernel 1495 * page, only do so if there are no other writable pages. 1496 */ 1497 for (pv = npv; pv; pv = TAILQ_NEXT(pv, pv_list)) { 1498 if (!(pv->pv_flags & PVF_NC) || (pm != pv->pv_pmap && 1499 (kpmap != pv->pv_pmap || other_writable))) 1500 continue; 1501 1502 pv->pv_flags &= ~PVF_NC; 1503 1504 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 1505 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 1506 pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; 1507 1508 if (l2pte_valid(pte)) { 1509 if (PV_BEEN_EXECD(pv->pv_flags)) { 1510 pmap_tlb_flushID_SE(pv->pv_pmap, 1511 pv->pv_va); 1512 } else 1513 if (PV_BEEN_REFD(pv->pv_flags)) { 1514 pmap_tlb_flushD_SE(pv->pv_pmap, 1515 pv->pv_va); 1516 } 1517 } 1518 1519 *ptep = pte; 1520 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 1521 } 1522 } 1523} 1524 1525/* 1526 * Modify pte bits for all ptes corresponding to the given physical address. 1527 * We use `maskbits' rather than `clearbits' because we're always passing 1528 * constants and the latter would require an extra inversion at run-time. 1529 */ 1530static int 1531pmap_clearbit(struct vm_page *pg, u_int maskbits) 1532{ 1533 struct l2_bucket *l2b; 1534 struct pv_entry *pv; 1535 pt_entry_t *ptep, npte, opte; 1536 pmap_t pm; 1537 vm_offset_t va; 1538 u_int oflags; 1539 int count = 0; 1540 1541 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1542 1543 /* 1544 * Clear saved attributes (modify, reference) 1545 */ 1546 pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); 1547 1548 if (TAILQ_EMPTY(&pg->md.pv_list)) { 1549 return (0); 1550 } 1551 1552 /* 1553 * Loop over all current mappings setting/clearing as appropos 1554 */ 1555 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { 1556 va = pv->pv_va; 1557 pm = pv->pv_pmap; 1558 oflags = pv->pv_flags; 1559 pv->pv_flags &= ~maskbits; 1560 1561 PMAP_LOCK(pm); 1562 1563 l2b = pmap_get_l2_bucket(pm, va); 1564 1565 ptep = &l2b->l2b_kva[l2pte_index(va)]; 1566 npte = opte = *ptep; 1567 1568 if (maskbits & (PVF_WRITE|PVF_MOD)) { 1569 if ((pv->pv_flags & PVF_NC)) { 1570 /* 1571 * Entry is not cacheable: 1572 * 1573 * Don't turn caching on again if this is a 1574 * modified emulation. This would be 1575 * inconsitent with the settings created by 1576 * pmap_vac_me_harder(). Otherwise, it's safe 1577 * to re-enable cacheing. 1578 * 1579 * There's no need to call pmap_vac_me_harder() 1580 * here: all pages are losing their write 1581 * permission. 1582 */ 1583 if (maskbits & PVF_WRITE) { 1584 npte |= pte_l2_s_cache_mode; 1585 pv->pv_flags &= ~PVF_NC; 1586 } 1587 } else 1588 if (opte & L2_S_PROT_W) { 1589 vm_page_dirty(pg); 1590 /* 1591 * Entry is writable/cacheable: check if pmap 1592 * is current if it is flush it, otherwise it 1593 * won't be in the cache 1594 */ 1595 if (PV_BEEN_EXECD(oflags)) 1596 pmap_idcache_wbinv_range(pm, pv->pv_va, 1597 PAGE_SIZE); 1598 else 1599 if (PV_BEEN_REFD(oflags)) 1600 pmap_dcache_wb_range(pm, pv->pv_va, 1601 PAGE_SIZE, 1602 (maskbits & PVF_REF) ? TRUE : FALSE, 1603 FALSE); 1604 } 1605 1606 /* make the pte read only */ 1607 npte &= ~L2_S_PROT_W; 1608 1609 if (maskbits & PVF_WRITE) { 1610 /* 1611 * Keep alias accounting up to date 1612 */ 1613 if (pv->pv_pmap == pmap_kernel()) { 1614 if (oflags & PVF_WRITE) { 1615 pg->md.krw_mappings--; 1616 pg->md.kro_mappings++; 1617 } 1618 } else 1619 if (oflags & PVF_WRITE) { 1620 pg->md.urw_mappings--; 1621 pg->md.uro_mappings++; 1622 } 1623 } 1624 } 1625 1626 if (maskbits & PVF_REF) { 1627 if ((pv->pv_flags & PVF_NC) == 0 && 1628 (maskbits & (PVF_WRITE|PVF_MOD)) == 0) { 1629 /* 1630 * Check npte here; we may have already 1631 * done the wbinv above, and the validity 1632 * of the PTE is the same for opte and 1633 * npte. 1634 */ 1635 if (npte & L2_S_PROT_W) { 1636 if (PV_BEEN_EXECD(oflags)) 1637 pmap_idcache_wbinv_range(pm, 1638 pv->pv_va, PAGE_SIZE); 1639 else 1640 if (PV_BEEN_REFD(oflags)) 1641 pmap_dcache_wb_range(pm, 1642 pv->pv_va, PAGE_SIZE, 1643 TRUE, FALSE); 1644 } else 1645 if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) { 1646 /* XXXJRT need idcache_inv_range */ 1647 if (PV_BEEN_EXECD(oflags)) 1648 pmap_idcache_wbinv_range(pm, 1649 pv->pv_va, PAGE_SIZE); 1650 else 1651 if (PV_BEEN_REFD(oflags)) 1652 pmap_dcache_wb_range(pm, 1653 pv->pv_va, PAGE_SIZE, 1654 TRUE, TRUE); 1655 } 1656 } 1657 1658 /* 1659 * Make the PTE invalid so that we will take a 1660 * page fault the next time the mapping is 1661 * referenced. 1662 */ 1663 npte &= ~L2_TYPE_MASK; 1664 npte |= L2_TYPE_INV; 1665 } 1666 1667 if (npte != opte) { 1668 count++; 1669 *ptep = npte; 1670 PTE_SYNC(ptep); 1671 /* Flush the TLB entry if a current pmap. */ 1672 if (PV_BEEN_EXECD(oflags)) 1673 pmap_tlb_flushID_SE(pm, pv->pv_va); 1674 else 1675 if (PV_BEEN_REFD(oflags)) 1676 pmap_tlb_flushD_SE(pm, pv->pv_va); 1677 } 1678 1679 PMAP_UNLOCK(pm); 1680 1681 } 1682 1683 if (maskbits & PVF_WRITE) 1684 vm_page_flag_clear(pg, PG_WRITEABLE); 1685 return (count); 1686} 1687 1688/* 1689 * main pv_entry manipulation functions: 1690 * pmap_enter_pv: enter a mapping onto a vm_page list 1691 * pmap_remove_pv: remove a mappiing from a vm_page list 1692 * 1693 * NOTE: pmap_enter_pv expects to lock the pvh itself 1694 * pmap_remove_pv expects te caller to lock the pvh before calling 1695 */ 1696 1697/* 1698 * pmap_enter_pv: enter a mapping onto a vm_page lst 1699 * 1700 * => caller should hold the proper lock on pmap_main_lock 1701 * => caller should have pmap locked 1702 * => we will gain the lock on the vm_page and allocate the new pv_entry 1703 * => caller should adjust ptp's wire_count before calling 1704 * => caller should not adjust pmap's wire_count 1705 */ 1706static void 1707pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 1708 vm_offset_t va, u_int flags) 1709{ 1710 1711 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1712 PMAP_ASSERT_LOCKED(pm); 1713 pve->pv_pmap = pm; 1714 pve->pv_va = va; 1715 pve->pv_flags = flags; 1716 1717 TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); 1718 TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); 1719 pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); 1720 if (pm == pmap_kernel()) { 1721 if (flags & PVF_WRITE) 1722 pg->md.krw_mappings++; 1723 else 1724 pg->md.kro_mappings++; 1725 } 1726 if (flags & PVF_WRITE) 1727 pg->md.urw_mappings++; 1728 else 1729 pg->md.uro_mappings++; 1730 pg->md.pv_list_count++; 1731 if (pve->pv_flags & PVF_WIRED) 1732 ++pm->pm_stats.wired_count; 1733 vm_page_flag_set(pg, PG_REFERENCED); 1734} 1735 1736/* 1737 * 1738 * pmap_find_pv: Find a pv entry 1739 * 1740 * => caller should hold lock on vm_page 1741 */ 1742static PMAP_INLINE struct pv_entry * 1743pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1744{ 1745 struct pv_entry *pv; 1746 1747 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1748 TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) 1749 if (pm == pv->pv_pmap && va == pv->pv_va) 1750 break; 1751 return (pv); 1752} 1753 1754/* 1755 * vector_page_setprot: 1756 * 1757 * Manipulate the protection of the vector page. 1758 */ 1759void 1760vector_page_setprot(int prot) 1761{ 1762 struct l2_bucket *l2b; 1763 pt_entry_t *ptep; 1764 1765 l2b = pmap_get_l2_bucket(pmap_kernel(), vector_page); 1766 1767 ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; 1768 1769 *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 1770 PTE_SYNC(ptep); 1771 cpu_tlb_flushD_SE(vector_page); 1772 cpu_cpwait(); 1773} 1774 1775/* 1776 * pmap_remove_pv: try to remove a mapping from a pv_list 1777 * 1778 * => caller should hold proper lock on pmap_main_lock 1779 * => pmap should be locked 1780 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1781 * => caller should adjust ptp's wire_count and free PTP if needed 1782 * => caller should NOT adjust pmap's wire_count 1783 * => we return the removed pve 1784 */ 1785 1786static void 1787pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) 1788{ 1789 1790 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1791 PMAP_ASSERT_LOCKED(pm); 1792 TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); 1793 TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); 1794 if (pve->pv_flags & PVF_WIRED) 1795 --pm->pm_stats.wired_count; 1796 pg->md.pv_list_count--; 1797 if (pg->md.pvh_attrs & PVF_MOD) 1798 vm_page_dirty(pg); 1799 if (pm == pmap_kernel()) { 1800 if (pve->pv_flags & PVF_WRITE) 1801 pg->md.krw_mappings--; 1802 else 1803 pg->md.kro_mappings--; 1804 } else 1805 if (pve->pv_flags & PVF_WRITE) 1806 pg->md.urw_mappings--; 1807 else 1808 pg->md.uro_mappings--; 1809 if (TAILQ_FIRST(&pg->md.pv_list) == NULL || 1810 (pg->md.krw_mappings == 0 && pg->md.urw_mappings == 0)) { 1811 pg->md.pvh_attrs &= ~PVF_MOD; 1812 if (TAILQ_FIRST(&pg->md.pv_list) == NULL) 1813 pg->md.pvh_attrs &= ~PVF_REF; 1814 vm_page_flag_clear(pg, PG_WRITEABLE); 1815 } 1816 if (TAILQ_FIRST(&pg->md.pv_list)) 1817 vm_page_flag_set(pg, PG_REFERENCED); 1818 if (pve->pv_flags & PVF_WRITE) 1819 pmap_vac_me_harder(pg, pm, 0); 1820} 1821 1822static struct pv_entry * 1823pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) 1824{ 1825 struct pv_entry *pve; 1826 1827 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1828 pve = TAILQ_FIRST(&pg->md.pv_list); 1829 1830 while (pve) { 1831 if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ 1832 pmap_nuke_pv(pg, pm, pve); 1833 break; 1834 } 1835 pve = TAILQ_NEXT(pve, pv_list); 1836 } 1837 1838 return(pve); /* return removed pve */ 1839} 1840/* 1841 * 1842 * pmap_modify_pv: Update pv flags 1843 * 1844 * => caller should hold lock on vm_page [so that attrs can be adjusted] 1845 * => caller should NOT adjust pmap's wire_count 1846 * => caller must call pmap_vac_me_harder() if writable status of a page 1847 * may have changed. 1848 * => we return the old flags 1849 * 1850 * Modify a physical-virtual mapping in the pv table 1851 */ 1852static u_int 1853pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, 1854 u_int clr_mask, u_int set_mask) 1855{ 1856 struct pv_entry *npv; 1857 u_int flags, oflags; 1858 1859 PMAP_ASSERT_LOCKED(pm); 1860 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1861 if ((npv = pmap_find_pv(pg, pm, va)) == NULL) 1862 return (0); 1863 1864 /* 1865 * There is at least one VA mapping this page. 1866 */ 1867 1868 if (clr_mask & (PVF_REF | PVF_MOD)) 1869 pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); 1870 1871 oflags = npv->pv_flags; 1872 npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; 1873 1874 if ((flags ^ oflags) & PVF_WIRED) { 1875 if (flags & PVF_WIRED) 1876 ++pm->pm_stats.wired_count; 1877 else 1878 --pm->pm_stats.wired_count; 1879 } 1880 1881 if ((flags ^ oflags) & PVF_WRITE) { 1882 if (pm == pmap_kernel()) { 1883 if (flags & PVF_WRITE) { 1884 pg->md.krw_mappings++; 1885 pg->md.kro_mappings--; 1886 } else { 1887 pg->md.kro_mappings++; 1888 pg->md.krw_mappings--; 1889 } 1890 } else 1891 if (flags & PVF_WRITE) { 1892 pg->md.urw_mappings++; 1893 pg->md.uro_mappings--; 1894 } else { 1895 pg->md.uro_mappings++; 1896 pg->md.urw_mappings--; 1897 } 1898 if (pg->md.krw_mappings == 0 && pg->md.urw_mappings == 0) { 1899 pg->md.pvh_attrs &= ~PVF_MOD; 1900 vm_page_flag_clear(pg, PG_WRITEABLE); 1901 } 1902 pmap_vac_me_harder(pg, pm, 0); 1903 } 1904 1905 return (oflags); 1906} 1907 1908/* Function to set the debug level of the pmap code */ 1909#ifdef PMAP_DEBUG 1910void 1911pmap_debug(int level) 1912{ 1913 pmap_debug_level = level; 1914 dprintf("pmap_debug: level=%d\n", pmap_debug_level); 1915} 1916#endif /* PMAP_DEBUG */ 1917 1918void 1919pmap_pinit0(struct pmap *pmap) 1920{ 1921 PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); 1922 1923 dprintf("pmap_pinit0: pmap = %08x, pm_pdir = %08x\n", 1924 (u_int32_t) pmap, (u_int32_t) pmap->pm_pdir); 1925 bcopy(kernel_pmap, pmap, sizeof(*pmap)); 1926 bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); 1927 PMAP_LOCK_INIT(pmap); 1928} 1929 1930/* 1931 * Initialize a vm_page's machine-dependent fields. 1932 */ 1933void 1934pmap_page_init(vm_page_t m) 1935{ 1936 1937 TAILQ_INIT(&m->md.pv_list); 1938 m->md.pv_list_count = 0; 1939} 1940 1941/* 1942 * Initialize the pmap module. 1943 * Called by vm_init, to initialize any structures that the pmap 1944 * system needs to map virtual memory. 1945 */ 1946void 1947pmap_init(void) 1948{ 1949 int shpgperproc = PMAP_SHPGPERPROC; 1950 1951 PDEBUG(1, printf("pmap_init: phys_start = %08x\n")); 1952 1953 /* 1954 * init the pv free list 1955 */ 1956 pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, 1957 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1958 /* 1959 * Now it is safe to enable pv_table recording. 1960 */ 1961 PDEBUG(1, printf("pmap_init: done!\n")); 1962 1963 TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); 1964 1965 pv_entry_max = shpgperproc * maxproc + vm_page_array_size; 1966 pv_entry_high_water = 9 * (pv_entry_max / 10); 1967 l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, 1968 NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); 1969 l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), 1970 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 1971 UMA_ZONE_VM | UMA_ZONE_NOFREE); 1972 1973 uma_zone_set_obj(pvzone, &pvzone_obj, pv_entry_max); 1974 1975} 1976 1977int 1978pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) 1979{ 1980 struct l2_dtable *l2; 1981 struct l2_bucket *l2b; 1982 pd_entry_t *pl1pd, l1pd; 1983 pt_entry_t *ptep, pte; 1984 vm_paddr_t pa; 1985 u_int l1idx; 1986 int rv = 0; 1987 1988 l1idx = L1_IDX(va); 1989 vm_page_lock_queues(); 1990 PMAP_LOCK(pm); 1991 1992 /* 1993 * If there is no l2_dtable for this address, then the process 1994 * has no business accessing it. 1995 * 1996 * Note: This will catch userland processes trying to access 1997 * kernel addresses. 1998 */ 1999 l2 = pm->pm_l2[L2_IDX(l1idx)]; 2000 if (l2 == NULL) 2001 goto out; 2002 2003 /* 2004 * Likewise if there is no L2 descriptor table 2005 */ 2006 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2007 if (l2b->l2b_kva == NULL) 2008 goto out; 2009 2010 /* 2011 * Check the PTE itself. 2012 */ 2013 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2014 pte = *ptep; 2015 if (pte == 0) 2016 goto out; 2017 2018 /* 2019 * Catch a userland access to the vector page mapped at 0x0 2020 */ 2021 if (user && (pte & L2_S_PROT_U) == 0) 2022 goto out; 2023 if (va == vector_page) 2024 goto out; 2025 2026 pa = l2pte_pa(pte); 2027 2028 if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { 2029 /* 2030 * This looks like a good candidate for "page modified" 2031 * emulation... 2032 */ 2033 struct pv_entry *pv; 2034 struct vm_page *pg; 2035 2036 /* Extract the physical address of the page */ 2037 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { 2038 goto out; 2039 } 2040 /* Get the current flags for this page. */ 2041 2042 pv = pmap_find_pv(pg, pm, va); 2043 if (pv == NULL) { 2044 goto out; 2045 } 2046 2047 /* 2048 * Do the flags say this page is writable? If not then it 2049 * is a genuine write fault. If yes then the write fault is 2050 * our fault as we did not reflect the write access in the 2051 * PTE. Now we know a write has occurred we can correct this 2052 * and also set the modified bit 2053 */ 2054 if ((pv->pv_flags & PVF_WRITE) == 0) { 2055 goto out; 2056 } 2057 2058 pg->md.pvh_attrs |= PVF_REF | PVF_MOD; 2059 vm_page_dirty(pg); 2060 pv->pv_flags |= PVF_REF | PVF_MOD; 2061 2062 /* 2063 * Re-enable write permissions for the page. No need to call 2064 * pmap_vac_me_harder(), since this is just a 2065 * modified-emulation fault, and the PVF_WRITE bit isn't 2066 * changing. We've already set the cacheable bits based on 2067 * the assumption that we can write to this page. 2068 */ 2069 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 2070 PTE_SYNC(ptep); 2071 rv = 1; 2072 } else 2073 if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { 2074 /* 2075 * This looks like a good candidate for "page referenced" 2076 * emulation. 2077 */ 2078 struct pv_entry *pv; 2079 struct vm_page *pg; 2080 2081 /* Extract the physical address of the page */ 2082 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2083 goto out; 2084 /* Get the current flags for this page. */ 2085 2086 pv = pmap_find_pv(pg, pm, va); 2087 if (pv == NULL) 2088 goto out; 2089 2090 pg->md.pvh_attrs |= PVF_REF; 2091 pv->pv_flags |= PVF_REF; 2092 2093 2094 *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; 2095 PTE_SYNC(ptep); 2096 rv = 1; 2097 } 2098 2099 /* 2100 * We know there is a valid mapping here, so simply 2101 * fix up the L1 if necessary. 2102 */ 2103 pl1pd = &pm->pm_l1->l1_kva[l1idx]; 2104 l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; 2105 if (*pl1pd != l1pd) { 2106 *pl1pd = l1pd; 2107 PTE_SYNC(pl1pd); 2108 rv = 1; 2109 } 2110 2111#ifdef CPU_SA110 2112 /* 2113 * There are bugs in the rev K SA110. This is a check for one 2114 * of them. 2115 */ 2116 if (rv == 0 && curcpu()->ci_arm_cputype == CPU_ID_SA110 && 2117 curcpu()->ci_arm_cpurev < 3) { 2118 /* Always current pmap */ 2119 if (l2pte_valid(pte)) { 2120 extern int kernel_debug; 2121 if (kernel_debug & 1) { 2122 struct proc *p = curlwp->l_proc; 2123 printf("prefetch_abort: page is already " 2124 "mapped - pte=%p *pte=%08x\n", ptep, pte); 2125 printf("prefetch_abort: pc=%08lx proc=%p " 2126 "process=%s\n", va, p, p->p_comm); 2127 printf("prefetch_abort: far=%08x fs=%x\n", 2128 cpu_faultaddress(), cpu_faultstatus()); 2129 } 2130#ifdef DDB 2131 if (kernel_debug & 2) 2132 Debugger(); 2133#endif 2134 rv = 1; 2135 } 2136 } 2137#endif /* CPU_SA110 */ 2138 2139#ifdef DEBUG 2140 /* 2141 * If 'rv == 0' at this point, it generally indicates that there is a 2142 * stale TLB entry for the faulting address. This happens when two or 2143 * more processes are sharing an L1. Since we don't flush the TLB on 2144 * a context switch between such processes, we can take domain faults 2145 * for mappings which exist at the same VA in both processes. EVEN IF 2146 * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for 2147 * example. 2148 * 2149 * This is extremely likely to happen if pmap_enter() updated the L1 2150 * entry for a recently entered mapping. In this case, the TLB is 2151 * flushed for the new mapping, but there may still be TLB entries for 2152 * other mappings belonging to other processes in the 1MB range 2153 * covered by the L1 entry. 2154 * 2155 * Since 'rv == 0', we know that the L1 already contains the correct 2156 * value, so the fault must be due to a stale TLB entry. 2157 * 2158 * Since we always need to flush the TLB anyway in the case where we 2159 * fixed up the L1, or frobbed the L2 PTE, we effectively deal with 2160 * stale TLB entries dynamically. 2161 * 2162 * However, the above condition can ONLY happen if the current L1 is 2163 * being shared. If it happens when the L1 is unshared, it indicates 2164 * that other parts of the pmap are not doing their job WRT managing 2165 * the TLB. 2166 */ 2167 if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { 2168 extern int last_fault_code; 2169 printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", 2170 pm, va, ftype); 2171 printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", 2172 l2, l2b, ptep, pl1pd); 2173 printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", 2174 pte, l1pd, last_fault_code); 2175#ifdef DDB 2176 Debugger(); 2177#endif 2178 } 2179#endif 2180 2181 cpu_tlb_flushID_SE(va); 2182 cpu_cpwait(); 2183 2184 rv = 1; 2185 2186out: 2187 vm_page_unlock_queues(); 2188 PMAP_UNLOCK(pm); 2189 return (rv); 2190} 2191 2192void 2193pmap_postinit(void) 2194{ 2195 struct l2_bucket *l2b; 2196 struct l1_ttable *l1; 2197 pd_entry_t *pl1pt; 2198 pt_entry_t *ptep, pte; 2199 vm_offset_t va, eva; 2200 u_int loop, needed; 2201 2202 needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); 2203 needed -= 1; 2204 l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); 2205 2206 for (loop = 0; loop < needed; loop++, l1++) { 2207 /* Allocate a L1 page table */ 2208 va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 2209 0xffffffff, L1_TABLE_SIZE, 0); 2210 2211 if (va == 0) 2212 panic("Cannot allocate L1 KVM"); 2213 2214 eva = va + L1_TABLE_SIZE; 2215 pl1pt = (pd_entry_t *)va; 2216 2217 while (va < eva) { 2218 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2219 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2220 pte = *ptep; 2221 pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; 2222 *ptep = pte; 2223 PTE_SYNC(ptep); 2224 cpu_tlb_flushD_SE(va); 2225 2226 va += PAGE_SIZE; 2227 } 2228 pmap_init_l1(l1, pl1pt); 2229 } 2230 2231 2232#ifdef DEBUG 2233 printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", 2234 needed); 2235#endif 2236} 2237 2238/* 2239 * This is used to stuff certain critical values into the PCB where they 2240 * can be accessed quickly from cpu_switch() et al. 2241 */ 2242void 2243pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) 2244{ 2245 struct l2_bucket *l2b; 2246 2247 pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; 2248 pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | 2249 (DOMAIN_CLIENT << (pm->pm_domain * 2)); 2250 2251 if (vector_page < KERNBASE) { 2252 pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; 2253 l2b = pmap_get_l2_bucket(pm, vector_page); 2254 pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | 2255 L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); 2256 } else 2257 pcb->pcb_pl1vec = NULL; 2258} 2259 2260void 2261pmap_activate(struct thread *td) 2262{ 2263 pmap_t pm; 2264 struct pcb *pcb; 2265 int s; 2266 2267 pm = vmspace_pmap(td->td_proc->p_vmspace); 2268 pcb = td->td_pcb; 2269 2270 critical_enter(); 2271 pmap_set_pcb_pagedir(pm, pcb); 2272 2273 if (td == curthread) { 2274 u_int cur_dacr, cur_ttb; 2275 2276 __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); 2277 __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); 2278 2279 cur_ttb &= ~(L1_TABLE_SIZE - 1); 2280 2281 if (cur_ttb == (u_int)pcb->pcb_pagedir && 2282 cur_dacr == pcb->pcb_dacr) { 2283 /* 2284 * No need to switch address spaces. 2285 */ 2286 critical_exit(); 2287 return; 2288 } 2289 2290 2291 /* 2292 * We MUST, I repeat, MUST fix up the L1 entry corresponding 2293 * to 'vector_page' in the incoming L1 table before switching 2294 * to it otherwise subsequent interrupts/exceptions (including 2295 * domain faults!) will jump into hyperspace. 2296 */ 2297 if (pcb->pcb_pl1vec) { 2298 2299 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 2300 /* 2301 * Don't need to PTE_SYNC() at this point since 2302 * cpu_setttb() is about to flush both the cache 2303 * and the TLB. 2304 */ 2305 } 2306 2307 cpu_domains(pcb->pcb_dacr); 2308 cpu_setttb(pcb->pcb_pagedir); 2309 2310 splx(s); 2311 } 2312 critical_exit(); 2313} 2314 2315static int 2316pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) 2317{ 2318 pd_entry_t *pdep, pde; 2319 pt_entry_t *ptep, pte; 2320 vm_offset_t pa; 2321 int rv = 0; 2322 2323 /* 2324 * Make sure the descriptor itself has the correct cache mode 2325 */ 2326 pdep = &kl1[L1_IDX(va)]; 2327 pde = *pdep; 2328 2329 if (l1pte_section_p(pde)) { 2330 if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { 2331 *pdep = (pde & ~L1_S_CACHE_MASK) | 2332 pte_l1_s_cache_mode_pt; 2333 PTE_SYNC(pdep); 2334 cpu_dcache_wbinv_range((vm_offset_t)pdep, 2335 sizeof(*pdep)); 2336 rv = 1; 2337 } 2338 } else { 2339 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2340 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2341 if (ptep == NULL) 2342 panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); 2343 2344 ptep = &ptep[l2pte_index(va)]; 2345 pte = *ptep; 2346 if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { 2347 *ptep = (pte & ~L2_S_CACHE_MASK) | 2348 pte_l2_s_cache_mode_pt; 2349 PTE_SYNC(ptep); 2350 cpu_dcache_wbinv_range((vm_offset_t)ptep, 2351 sizeof(*ptep)); 2352 rv = 1; 2353 } 2354 } 2355 2356 return (rv); 2357} 2358 2359static void 2360pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, 2361 pt_entry_t **ptep) 2362{ 2363 vm_offset_t va = *availp; 2364 struct l2_bucket *l2b; 2365 2366 if (ptep) { 2367 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2368 if (l2b == NULL) 2369 panic("pmap_alloc_specials: no l2b for 0x%x", va); 2370 2371 *ptep = &l2b->l2b_kva[l2pte_index(va)]; 2372 } 2373 2374 *vap = va; 2375 *availp = va + (PAGE_SIZE * pages); 2376} 2377 2378/* 2379 * Bootstrap the system enough to run with virtual memory. 2380 * 2381 * On the arm this is called after mapping has already been enabled 2382 * and just syncs the pmap module with what has already been done. 2383 * [We can't call it easily with mapping off since the kernel is not 2384 * mapped with PA == VA, hence we would have to relocate every address 2385 * from the linked base (virtual) address "KERNBASE" to the actual 2386 * (physical) address starting relative to 0] 2387 */ 2388#define PMAP_STATIC_L2_SIZE 16 2389#ifdef ARM_USE_SMALL_ALLOC 2390extern struct mtx smallalloc_mtx; 2391extern vm_offset_t alloc_curaddr; 2392extern vm_offset_t alloc_firstaddr; 2393#endif 2394 2395void 2396pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt) 2397{ 2398 static struct l1_ttable static_l1; 2399 static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; 2400 struct l1_ttable *l1 = &static_l1; 2401 struct l2_dtable *l2; 2402 struct l2_bucket *l2b; 2403 pd_entry_t pde; 2404 pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; 2405 pt_entry_t *ptep; 2406 vm_paddr_t pa; 2407 vm_offset_t va; 2408 vm_size_t size; 2409 int l1idx, l2idx, l2next = 0; 2410 2411 PDEBUG(1, printf("firstaddr = %08x, loadaddr = %08x\n", 2412 firstaddr, loadaddr)); 2413 2414 virtual_avail = firstaddr; 2415 kernel_pmap = &kernel_pmap_store; 2416 kernel_pmap->pm_l1 = l1; 2417 kernel_l1pa = l1pt->pv_pa; 2418 2419 /* 2420 * Scan the L1 translation table created by initarm() and create 2421 * the required metadata for all valid mappings found in it. 2422 */ 2423 for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { 2424 pde = kernel_l1pt[l1idx]; 2425 2426 /* 2427 * We're only interested in Coarse mappings. 2428 * pmap_extract() can deal with section mappings without 2429 * recourse to checking L2 metadata. 2430 */ 2431 if ((pde & L1_TYPE_MASK) != L1_TYPE_C) 2432 continue; 2433 2434 /* 2435 * Lookup the KVA of this L2 descriptor table 2436 */ 2437 pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); 2438 ptep = (pt_entry_t *)kernel_pt_lookup(pa); 2439 2440 if (ptep == NULL) { 2441 panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", 2442 (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); 2443 } 2444 2445 /* 2446 * Fetch the associated L2 metadata structure. 2447 * Allocate a new one if necessary. 2448 */ 2449 if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { 2450 if (l2next == PMAP_STATIC_L2_SIZE) 2451 panic("pmap_bootstrap: out of static L2s"); 2452 kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = 2453 &static_l2[l2next++]; 2454 } 2455 2456 /* 2457 * One more L1 slot tracked... 2458 */ 2459 l2->l2_occupancy++; 2460 2461 /* 2462 * Fill in the details of the L2 descriptor in the 2463 * appropriate bucket. 2464 */ 2465 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2466 l2b->l2b_kva = ptep; 2467 l2b->l2b_phys = pa; 2468 l2b->l2b_l1idx = l1idx; 2469 2470 /* 2471 * Establish an initial occupancy count for this descriptor 2472 */ 2473 for (l2idx = 0; 2474 l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); 2475 l2idx++) { 2476 if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { 2477 l2b->l2b_occupancy++; 2478 } 2479 } 2480 2481 /* 2482 * Make sure the descriptor itself has the correct cache mode. 2483 * If not, fix it, but whine about the problem. Port-meisters 2484 * should consider this a clue to fix up their initarm() 2485 * function. :) 2486 */ 2487 if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { 2488 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2489 "L2 pte @ %p\n", ptep); 2490 } 2491 } 2492 2493 2494 /* 2495 * Ensure the primary (kernel) L1 has the correct cache mode for 2496 * a page table. Bitch if it is not correctly set. 2497 */ 2498 for (va = (vm_offset_t)kernel_l1pt; 2499 va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { 2500 if (pmap_set_pt_cache_mode(kernel_l1pt, va)) 2501 printf("pmap_bootstrap: WARNING! wrong cache mode for " 2502 "primary L1 @ 0x%x\n", va); 2503 } 2504 2505 cpu_dcache_wbinv_all(); 2506 cpu_tlb_flushID(); 2507 cpu_cpwait(); 2508 2509 PMAP_LOCK_INIT(kernel_pmap); 2510 kernel_pmap->pm_active = -1; 2511 kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; 2512 TAILQ_INIT(&kernel_pmap->pm_pvlist); 2513 2514 /* 2515 * Reserve some special page table entries/VA space for temporary 2516 * mapping of pages. 2517 */ 2518#define SYSMAP(c, p, v, n) \ 2519 v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); 2520 2521 pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); 2522 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); 2523 pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); 2524 pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); 2525 size = ((lastaddr - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; 2526 pmap_alloc_specials(&virtual_avail, 2527 round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, 2528 &pmap_kernel_l2ptp_kva, NULL); 2529 2530 size = (size + (L2_BUCKET_SIZE - 1)) / L2_BUCKET_SIZE; 2531 pmap_alloc_specials(&virtual_avail, 2532 round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, 2533 &pmap_kernel_l2dtable_kva, NULL); 2534 2535 pmap_alloc_specials(&virtual_avail, 2536 1, (vm_offset_t*)&_tmppt, NULL); 2537 SLIST_INIT(&l1_list); 2538 TAILQ_INIT(&l1_lru_list); 2539 mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); 2540 pmap_init_l1(l1, kernel_l1pt); 2541 cpu_dcache_wbinv_all(); 2542 2543 virtual_avail = round_page(virtual_avail); 2544 virtual_end = lastaddr; 2545 kernel_vm_end = pmap_curmaxkvaddr; 2546 arm_nocache_startaddr = lastaddr; 2547 mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); 2548 2549#ifdef ARM_USE_SMALL_ALLOC 2550 mtx_init(&smallalloc_mtx, "Small alloc page list", NULL, MTX_DEF); 2551 alloc_firstaddr = alloc_curaddr = arm_nocache_startaddr + 2552 ARM_NOCACHE_KVA_SIZE; 2553#endif 2554} 2555 2556/*************************************************** 2557 * Pmap allocation/deallocation routines. 2558 ***************************************************/ 2559 2560/* 2561 * Release any resources held by the given physical map. 2562 * Called when a pmap initialized by pmap_pinit is being released. 2563 * Should only be called if the map contains no valid mappings. 2564 */ 2565void 2566pmap_release(pmap_t pmap) 2567{ 2568 struct pcb *pcb; 2569 2570 pmap_idcache_wbinv_all(pmap); 2571 pmap_tlb_flushID(pmap); 2572 cpu_cpwait(); 2573 if (vector_page < KERNBASE) { 2574 struct pcb *curpcb = PCPU_GET(curpcb); 2575 pcb = thread0.td_pcb; 2576 if (pmap_is_current(pmap)) { 2577 /* 2578 * Frob the L1 entry corresponding to the vector 2579 * page so that it contains the kernel pmap's domain 2580 * number. This will ensure pmap_remove() does not 2581 * pull the current vector page out from under us. 2582 */ 2583 critical_enter(); 2584 *pcb->pcb_pl1vec = pcb->pcb_l1vec; 2585 cpu_domains(pcb->pcb_dacr); 2586 cpu_setttb(pcb->pcb_pagedir); 2587 critical_exit(); 2588 } 2589 pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); 2590 /* 2591 * Make sure cpu_switch(), et al, DTRT. This is safe to do 2592 * since this process has no remaining mappings of its own. 2593 */ 2594 curpcb->pcb_pl1vec = pcb->pcb_pl1vec; 2595 curpcb->pcb_l1vec = pcb->pcb_l1vec; 2596 curpcb->pcb_dacr = pcb->pcb_dacr; 2597 curpcb->pcb_pagedir = pcb->pcb_pagedir; 2598 2599 } 2600 pmap_free_l1(pmap); 2601 PMAP_LOCK_DESTROY(pmap); 2602 2603 dprintf("pmap_release()\n"); 2604} 2605 2606 2607 2608/* 2609 * Helper function for pmap_grow_l2_bucket() 2610 */ 2611static __inline int 2612pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) 2613{ 2614 struct l2_bucket *l2b; 2615 pt_entry_t *ptep; 2616 vm_paddr_t pa; 2617 struct vm_page *pg; 2618 2619 pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); 2620 if (pg == NULL) 2621 return (1); 2622 pa = VM_PAGE_TO_PHYS(pg); 2623 2624 if (pap) 2625 *pap = pa; 2626 2627 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2628 2629 ptep = &l2b->l2b_kva[l2pte_index(va)]; 2630 *ptep = L2_S_PROTO | pa | cache_mode | 2631 L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); 2632 PTE_SYNC(ptep); 2633 return (0); 2634} 2635 2636/* 2637 * This is the same as pmap_alloc_l2_bucket(), except that it is only 2638 * used by pmap_growkernel(). 2639 */ 2640static __inline struct l2_bucket * 2641pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) 2642{ 2643 struct l2_dtable *l2; 2644 struct l2_bucket *l2b; 2645 struct l1_ttable *l1; 2646 pd_entry_t *pl1pd; 2647 u_short l1idx; 2648 vm_offset_t nva; 2649 2650 l1idx = L1_IDX(va); 2651 2652 if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { 2653 /* 2654 * No mapping at this address, as there is 2655 * no entry in the L1 table. 2656 * Need to allocate a new l2_dtable. 2657 */ 2658 nva = pmap_kernel_l2dtable_kva; 2659 if ((nva & PAGE_MASK) == 0) { 2660 /* 2661 * Need to allocate a backing page 2662 */ 2663 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 2664 return (NULL); 2665 } 2666 2667 l2 = (struct l2_dtable *)nva; 2668 nva += sizeof(struct l2_dtable); 2669 2670 if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & 2671 PAGE_MASK)) { 2672 /* 2673 * The new l2_dtable straddles a page boundary. 2674 * Map in another page to cover it. 2675 */ 2676 if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) 2677 return (NULL); 2678 } 2679 2680 pmap_kernel_l2dtable_kva = nva; 2681 2682 /* 2683 * Link it into the parent pmap 2684 */ 2685 pm->pm_l2[L2_IDX(l1idx)] = l2; 2686 memset(l2, 0, sizeof(*l2)); 2687 } 2688 2689 l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; 2690 2691 /* 2692 * Fetch pointer to the L2 page table associated with the address. 2693 */ 2694 if (l2b->l2b_kva == NULL) { 2695 pt_entry_t *ptep; 2696 2697 /* 2698 * No L2 page table has been allocated. Chances are, this 2699 * is because we just allocated the l2_dtable, above. 2700 */ 2701 nva = pmap_kernel_l2ptp_kva; 2702 ptep = (pt_entry_t *)nva; 2703 if ((nva & PAGE_MASK) == 0) { 2704 /* 2705 * Need to allocate a backing page 2706 */ 2707 if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, 2708 &pmap_kernel_l2ptp_phys)) 2709 return (NULL); 2710 PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); 2711 } 2712 memset(ptep, 0, L2_TABLE_SIZE_REAL); 2713 l2->l2_occupancy++; 2714 l2b->l2b_kva = ptep; 2715 l2b->l2b_l1idx = l1idx; 2716 l2b->l2b_phys = pmap_kernel_l2ptp_phys; 2717 2718 pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; 2719 pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; 2720 } 2721 2722 /* Distribute new L1 entry to all other L1s */ 2723 SLIST_FOREACH(l1, &l1_list, l1_link) { 2724 pl1pd = &l1->l1_kva[L1_IDX(va)]; 2725 *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | 2726 L1_C_PROTO; 2727 PTE_SYNC(pl1pd); 2728 } 2729 2730 return (l2b); 2731} 2732 2733 2734/* 2735 * grow the number of kernel page table entries, if needed 2736 */ 2737void 2738pmap_growkernel(vm_offset_t addr) 2739{ 2740 pmap_t kpm = pmap_kernel(); 2741 int s; 2742 2743 if (addr <= pmap_curmaxkvaddr) 2744 return; /* we are OK */ 2745 2746 /* 2747 * whoops! we need to add kernel PTPs 2748 */ 2749 2750 s = splhigh(); /* to be safe */ 2751 2752 /* Map 1MB at a time */ 2753 for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) 2754 pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); 2755 2756 /* 2757 * flush out the cache, expensive but growkernel will happen so 2758 * rarely 2759 */ 2760 cpu_dcache_wbinv_all(); 2761 cpu_tlb_flushD(); 2762 cpu_cpwait(); 2763 kernel_vm_end = pmap_curmaxkvaddr; 2764 2765} 2766 2767 2768/* 2769 * pmap_page_protect: 2770 * 2771 * Lower the permission for all mappings to a given page. 2772 */ 2773void 2774pmap_page_protect(vm_page_t m, vm_prot_t prot) 2775{ 2776 switch(prot) { 2777 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2778 case VM_PROT_READ|VM_PROT_WRITE: 2779 return; 2780 2781 case VM_PROT_READ: 2782 case VM_PROT_READ|VM_PROT_EXECUTE: 2783 pmap_clearbit(m, PVF_WRITE); 2784 break; 2785 2786 default: 2787 pmap_remove_all(m); 2788 break; 2789 } 2790 2791} 2792 2793 2794/* 2795 * Remove all pages from specified address space 2796 * this aids process exit speeds. Also, this code 2797 * is special cased for current process only, but 2798 * can have the more generic (and slightly slower) 2799 * mode enabled. This is much faster than pmap_remove 2800 * in the case of running down an entire address space. 2801 */ 2802void 2803pmap_remove_pages(pmap_t pmap) 2804{ 2805 struct pv_entry *pv, *npv; 2806 struct l2_bucket *l2b = NULL; 2807 vm_page_t m; 2808 pt_entry_t *pt; 2809 2810 vm_page_lock_queues(); 2811 PMAP_LOCK(pmap); 2812 for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { 2813 if (pv->pv_flags & PVF_WIRED) { 2814 /* The page is wired, cannot remove it now. */ 2815 npv = TAILQ_NEXT(pv, pv_plist); 2816 continue; 2817 } 2818 pmap->pm_stats.resident_count--; 2819 l2b = pmap_get_l2_bucket(pmap, pv->pv_va); 2820 KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); 2821 pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 2822 m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); 2823 *pt = 0; 2824 PTE_SYNC(pt); 2825 npv = TAILQ_NEXT(pv, pv_plist); 2826 pmap_nuke_pv(m, pmap, pv); 2827 if (TAILQ_EMPTY(&m->md.pv_list)) 2828 vm_page_flag_clear(m, PG_WRITEABLE); 2829 pmap_free_pv_entry(pv); 2830 } 2831 vm_page_unlock_queues(); 2832 cpu_idcache_wbinv_all(); 2833 cpu_tlb_flushID(); 2834 cpu_cpwait(); 2835 PMAP_UNLOCK(pmap); 2836} 2837 2838 2839/*************************************************** 2840 * Low level mapping routines..... 2841 ***************************************************/ 2842 2843/* Map a section into the KVA. */ 2844 2845void 2846pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) 2847{ 2848 pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, 2849 VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); 2850 struct l1_ttable *l1; 2851 2852 KASSERT(((va | pa) & L1_S_OFFSET) == 0, 2853 ("Not a valid section mapping")); 2854 if (flags & SECTION_CACHE) 2855 pd |= pte_l1_s_cache_mode; 2856 else if (flags & SECTION_PT) 2857 pd |= pte_l1_s_cache_mode_pt; 2858 SLIST_FOREACH(l1, &l1_list, l1_link) { 2859 l1->l1_kva[L1_IDX(va)] = pd; 2860 PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); 2861 } 2862} 2863 2864/* 2865 * add a wired page to the kva 2866 * note that in order for the mapping to take effect -- you 2867 * should do a invltlb after doing the pmap_kenter... 2868 */ 2869static PMAP_INLINE void 2870pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) 2871{ 2872 struct l2_bucket *l2b; 2873 pt_entry_t *pte; 2874 pt_entry_t opte; 2875 PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", 2876 (uint32_t) va, (uint32_t) pa)); 2877 2878 2879 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2880 if (l2b == NULL) 2881 l2b = pmap_grow_l2_bucket(pmap_kernel(), va); 2882 KASSERT(l2b != NULL, ("No L2 Bucket")); 2883 pte = &l2b->l2b_kva[l2pte_index(va)]; 2884 opte = *pte; 2885 PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", 2886 (uint32_t) pte, opte, *pte)); 2887 if (l2pte_valid(opte)) { 2888 cpu_dcache_wbinv_range(va, PAGE_SIZE); 2889 cpu_tlb_flushD_SE(va); 2890 cpu_cpwait(); 2891 } else { 2892 if (opte == 0) 2893 l2b->l2b_occupancy++; 2894 } 2895 *pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, 2896 VM_PROT_READ | VM_PROT_WRITE); 2897 if (flags & KENTER_CACHE) 2898 *pte |= pte_l2_s_cache_mode; 2899 if (flags & KENTER_USER) 2900 *pte |= L2_S_PROT_U; 2901 PTE_SYNC(pte); 2902} 2903 2904void 2905pmap_kenter(vm_offset_t va, vm_paddr_t pa) 2906{ 2907 pmap_kenter_internal(va, pa, KENTER_CACHE); 2908} 2909 2910void 2911pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) 2912{ 2913 2914 pmap_kenter_internal(va, pa, 0); 2915} 2916 2917void 2918pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) 2919{ 2920 2921 pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); 2922 /* 2923 * Call pmap_fault_fixup now, to make sure we'll have no exception 2924 * at the first use of the new address, or bad things will happen, 2925 * as we use one of these addresses in the exception handlers. 2926 */ 2927 pmap_fault_fixup(pmap_kernel(), va, VM_PROT_READ|VM_PROT_WRITE, 1); 2928} 2929 2930/* 2931 * remove a page rom the kernel pagetables 2932 */ 2933PMAP_INLINE void 2934pmap_kremove(vm_offset_t va) 2935{ 2936 struct l2_bucket *l2b; 2937 pt_entry_t *pte, opte; 2938 2939 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 2940 if (!l2b) 2941 return; 2942 KASSERT(l2b != NULL, ("No L2 Bucket")); 2943 pte = &l2b->l2b_kva[l2pte_index(va)]; 2944 opte = *pte; 2945 if (l2pte_valid(opte)) { 2946 cpu_dcache_wbinv_range(va, PAGE_SIZE); 2947 cpu_tlb_flushD_SE(va); 2948 cpu_cpwait(); 2949 *pte = 0; 2950 } 2951} 2952 2953 2954/* 2955 * Used to map a range of physical addresses into kernel 2956 * virtual address space. 2957 * 2958 * The value passed in '*virt' is a suggested virtual address for 2959 * the mapping. Architectures which can support a direct-mapped 2960 * physical to virtual region can return the appropriate address 2961 * within that region, leaving '*virt' unchanged. Other 2962 * architectures should map the pages starting at '*virt' and 2963 * update '*virt' with the first usable address after the mapped 2964 * region. 2965 */ 2966vm_offset_t 2967pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) 2968{ 2969 vm_offset_t sva = *virt; 2970 vm_offset_t va = sva; 2971 2972 PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " 2973 "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, 2974 prot)); 2975 2976 while (start < end) { 2977 pmap_kenter(va, start); 2978 va += PAGE_SIZE; 2979 start += PAGE_SIZE; 2980 } 2981 *virt = va; 2982 return (sva); 2983} 2984 2985static void 2986pmap_wb_page(vm_page_t m) 2987{ 2988 struct pv_entry *pv; 2989 2990 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 2991 pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, FALSE, 2992 (pv->pv_flags & PVF_WRITE) == 0); 2993} 2994 2995static void 2996pmap_inv_page(vm_page_t m) 2997{ 2998 struct pv_entry *pv; 2999 3000 TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) 3001 pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE); 3002} 3003/* 3004 * Add a list of wired pages to the kva 3005 * this routine is only used for temporary 3006 * kernel mappings that do not need to have 3007 * page modification or references recorded. 3008 * Note that old mappings are simply written 3009 * over. The page *must* be wired. 3010 */ 3011void 3012pmap_qenter(vm_offset_t va, vm_page_t *m, int count) 3013{ 3014 int i; 3015 3016 for (i = 0; i < count; i++) { 3017 pmap_wb_page(m[i]); 3018 pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), 3019 KENTER_CACHE); 3020 va += PAGE_SIZE; 3021 } 3022} 3023 3024 3025/* 3026 * this routine jerks page mappings from the 3027 * kernel -- it is meant only for temporary mappings. 3028 */ 3029void 3030pmap_qremove(vm_offset_t va, int count) 3031{ 3032 vm_paddr_t pa; 3033 int i; 3034 3035 for (i = 0; i < count; i++) { 3036 pa = vtophys(va); 3037 if (pa) { 3038 pmap_inv_page(PHYS_TO_VM_PAGE(pa)); 3039 pmap_kremove(va); 3040 } 3041 va += PAGE_SIZE; 3042 } 3043} 3044 3045 3046/* 3047 * pmap_object_init_pt preloads the ptes for a given object 3048 * into the specified pmap. This eliminates the blast of soft 3049 * faults on process startup and immediately after an mmap. 3050 */ 3051void 3052pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, 3053 vm_pindex_t pindex, vm_size_t size) 3054{ 3055 3056 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 3057 KASSERT(object->type == OBJT_DEVICE, 3058 ("pmap_object_init_pt: non-device object")); 3059} 3060 3061 3062/* 3063 * pmap_is_prefaultable: 3064 * 3065 * Return whether or not the specified virtual address is elgible 3066 * for prefault. 3067 */ 3068boolean_t 3069pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) 3070{ 3071 pd_entry_t *pde; 3072 pt_entry_t *pte; 3073 3074 if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) 3075 return (FALSE); 3076 KASSERT(pte != NULL, ("Valid mapping but no pte ?")); 3077 if (*pte == 0) 3078 return (TRUE); 3079 return (FALSE); 3080} 3081 3082/* 3083 * Fetch pointers to the PDE/PTE for the given pmap/VA pair. 3084 * Returns TRUE if the mapping exists, else FALSE. 3085 * 3086 * NOTE: This function is only used by a couple of arm-specific modules. 3087 * It is not safe to take any pmap locks here, since we could be right 3088 * in the middle of debugging the pmap anyway... 3089 * 3090 * It is possible for this routine to return FALSE even though a valid 3091 * mapping does exist. This is because we don't lock, so the metadata 3092 * state may be inconsistent. 3093 * 3094 * NOTE: We can return a NULL *ptp in the case where the L1 pde is 3095 * a "section" mapping. 3096 */ 3097boolean_t 3098pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) 3099{ 3100 struct l2_dtable *l2; 3101 pd_entry_t *pl1pd, l1pd; 3102 pt_entry_t *ptep; 3103 u_short l1idx; 3104 3105 if (pm->pm_l1 == NULL) 3106 return (FALSE); 3107 3108 l1idx = L1_IDX(va); 3109 *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; 3110 l1pd = *pl1pd; 3111 3112 if (l1pte_section_p(l1pd)) { 3113 *ptp = NULL; 3114 return (TRUE); 3115 } 3116 3117 if (pm->pm_l2 == NULL) 3118 return (FALSE); 3119 3120 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3121 3122 if (l2 == NULL || 3123 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3124 return (FALSE); 3125 } 3126 3127 *ptp = &ptep[l2pte_index(va)]; 3128 return (TRUE); 3129} 3130 3131/* 3132 * Routine: pmap_remove_all 3133 * Function: 3134 * Removes this physical page from 3135 * all physical maps in which it resides. 3136 * Reflects back modify bits to the pager. 3137 * 3138 * Notes: 3139 * Original versions of this routine were very 3140 * inefficient because they iteratively called 3141 * pmap_remove (slow...) 3142 */ 3143void 3144pmap_remove_all(vm_page_t m) 3145{ 3146 pv_entry_t pv; 3147 pt_entry_t *ptep, pte; 3148 struct l2_bucket *l2b; 3149 boolean_t flush = FALSE; 3150 pmap_t curpm; 3151 int flags = 0; 3152 3153#if defined(PMAP_DEBUG) 3154 /* 3155 * XXX this makes pmap_page_protect(NONE) illegal for non-managed 3156 * pages! 3157 */ 3158 if (m->flags & PG_FICTITIOUS) { 3159 panic("pmap_page_protect: illegal for unmanaged page, va: 0x%x", VM_PAGE_TO_PHYS(m)); 3160 } 3161#endif 3162 3163 if (TAILQ_EMPTY(&m->md.pv_list)) 3164 return; 3165 curpm = vmspace_pmap(curproc->p_vmspace); 3166 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { 3167 if (flush == FALSE && (pv->pv_pmap == curpm || 3168 pv->pv_pmap == pmap_kernel())) 3169 flush = TRUE; 3170 PMAP_LOCK(pv->pv_pmap); 3171 l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); 3172 KASSERT(l2b != NULL, ("No l2 bucket")); 3173 ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; 3174 pte = *ptep; 3175 *ptep = 0; 3176 PTE_SYNC_CURRENT(pv->pv_pmap, ptep); 3177 pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); 3178 if (pv->pv_flags & PVF_WIRED) 3179 pv->pv_pmap->pm_stats.wired_count--; 3180 pv->pv_pmap->pm_stats.resident_count--; 3181 flags |= pv->pv_flags; 3182 pmap_nuke_pv(m, pv->pv_pmap, pv); 3183 PMAP_UNLOCK(pv->pv_pmap); 3184 pmap_free_pv_entry(pv); 3185 } 3186 3187 if (flush) { 3188 if (PV_BEEN_EXECD(flags)) 3189 pmap_tlb_flushID(curpm); 3190 else 3191 pmap_tlb_flushD(curpm); 3192 } 3193 vm_page_flag_clear(m, PG_WRITEABLE); 3194} 3195 3196 3197/* 3198 * Set the physical protection on the 3199 * specified range of this map as requested. 3200 */ 3201void 3202pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) 3203{ 3204 struct l2_bucket *l2b; 3205 pt_entry_t *ptep, pte; 3206 vm_offset_t next_bucket; 3207 u_int flags; 3208 int flush; 3209 3210 if ((prot & VM_PROT_READ) == 0) { 3211 pmap_remove(pm, sva, eva); 3212 return; 3213 } 3214 3215 if (prot & VM_PROT_WRITE) { 3216 /* 3217 * If this is a read->write transition, just ignore it and let 3218 * vm_fault() take care of it later. 3219 */ 3220 return; 3221 } 3222 3223 vm_page_lock_queues(); 3224 PMAP_LOCK(pm); 3225 3226 /* 3227 * OK, at this point, we know we're doing write-protect operation. 3228 * If the pmap is active, write-back the range. 3229 */ 3230 pmap_dcache_wb_range(pm, sva, eva - sva, FALSE, FALSE); 3231 3232 flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; 3233 flags = 0; 3234 3235 while (sva < eva) { 3236 next_bucket = L2_NEXT_BUCKET(sva); 3237 if (next_bucket > eva) 3238 next_bucket = eva; 3239 3240 l2b = pmap_get_l2_bucket(pm, sva); 3241 if (l2b == NULL) { 3242 sva = next_bucket; 3243 continue; 3244 } 3245 3246 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3247 3248 while (sva < next_bucket) { 3249 if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { 3250 struct vm_page *pg; 3251 u_int f; 3252 3253 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3254 pte &= ~L2_S_PROT_W; 3255 *ptep = pte; 3256 PTE_SYNC(ptep); 3257 3258 if (pg != NULL) { 3259 f = pmap_modify_pv(pg, pm, sva, 3260 PVF_WRITE, 0); 3261 pmap_vac_me_harder(pg, pm, sva); 3262 vm_page_dirty(pg); 3263 } else 3264 f = PVF_REF | PVF_EXEC; 3265 3266 if (flush >= 0) { 3267 flush++; 3268 flags |= f; 3269 } else 3270 if (PV_BEEN_EXECD(f)) 3271 pmap_tlb_flushID_SE(pm, sva); 3272 else 3273 if (PV_BEEN_REFD(f)) 3274 pmap_tlb_flushD_SE(pm, sva); 3275 } 3276 3277 sva += PAGE_SIZE; 3278 ptep++; 3279 } 3280 } 3281 3282 3283 if (flush) { 3284 if (PV_BEEN_EXECD(flags)) 3285 pmap_tlb_flushID(pm); 3286 else 3287 if (PV_BEEN_REFD(flags)) 3288 pmap_tlb_flushD(pm); 3289 } 3290 vm_page_unlock_queues(); 3291 3292 PMAP_UNLOCK(pm); 3293} 3294 3295 3296/* 3297 * Insert the given physical page (p) at 3298 * the specified virtual address (v) in the 3299 * target physical map with the protection requested. 3300 * 3301 * If specified, the page will be wired down, meaning 3302 * that the related pte can not be reclaimed. 3303 * 3304 * NB: This is the only routine which MAY NOT lazy-evaluate 3305 * or lose information. That is, this routine must actually 3306 * insert this page into the given map NOW. 3307 */ 3308 3309void 3310pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3311 boolean_t wired) 3312{ 3313 3314 vm_page_lock_queues(); 3315 PMAP_LOCK(pmap); 3316 pmap_enter_locked(pmap, va, m, prot, wired); 3317 vm_page_unlock_queues(); 3318 PMAP_UNLOCK(pmap); 3319} 3320 3321/* 3322 * The page queues and pmap must be locked. 3323 */ 3324static void 3325pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3326 boolean_t wired) 3327{ 3328 struct l2_bucket *l2b = NULL; 3329 struct vm_page *opg; 3330 struct pv_entry *pve = NULL; 3331 pt_entry_t *ptep, npte, opte; 3332 u_int nflags; 3333 u_int oflags; 3334 vm_paddr_t pa; 3335 3336 PMAP_ASSERT_LOCKED(pmap); 3337 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 3338 if (va == vector_page) { 3339 pa = systempage.pv_pa; 3340 m = NULL; 3341 } else 3342 pa = VM_PAGE_TO_PHYS(m); 3343 nflags = 0; 3344 if (prot & VM_PROT_WRITE) 3345 nflags |= PVF_WRITE; 3346 if (prot & VM_PROT_EXECUTE) 3347 nflags |= PVF_EXEC; 3348 if (wired) 3349 nflags |= PVF_WIRED; 3350 PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " 3351 "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired)); 3352 3353 if (pmap == pmap_kernel()) { 3354 l2b = pmap_get_l2_bucket(pmap, va); 3355 if (l2b == NULL) 3356 l2b = pmap_grow_l2_bucket(pmap, va); 3357 } else 3358 l2b = pmap_alloc_l2_bucket(pmap, va); 3359 KASSERT(l2b != NULL, 3360 ("pmap_enter: failed to allocate l2 bucket")); 3361 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3362 3363 opte = *ptep; 3364 npte = pa; 3365 oflags = 0; 3366 if (opte) { 3367 /* 3368 * There is already a mapping at this address. 3369 * If the physical address is different, lookup the 3370 * vm_page. 3371 */ 3372 if (l2pte_pa(opte) != pa) 3373 opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); 3374 else 3375 opg = m; 3376 } else 3377 opg = NULL; 3378 3379 if ((prot & (VM_PROT_ALL)) || 3380 (!m || m->md.pvh_attrs & PVF_REF)) { 3381 /* 3382 * - The access type indicates that we don't need 3383 * to do referenced emulation. 3384 * OR 3385 * - The physical page has already been referenced 3386 * so no need to re-do referenced emulation here. 3387 */ 3388 npte |= L2_S_PROTO; 3389 3390 nflags |= PVF_REF; 3391 3392 if (m && ((prot & VM_PROT_WRITE) != 0 || 3393 (m->md.pvh_attrs & PVF_MOD))) { 3394 /* 3395 * This is a writable mapping, and the 3396 * page's mod state indicates it has 3397 * already been modified. Make it 3398 * writable from the outset. 3399 */ 3400 nflags |= PVF_MOD; 3401 if (!(m->md.pvh_attrs & PVF_MOD)) 3402 vm_page_dirty(m); 3403 } 3404 if (m && opte) 3405 vm_page_flag_set(m, PG_REFERENCED); 3406 } else { 3407 /* 3408 * Need to do page referenced emulation. 3409 */ 3410 npte |= L2_TYPE_INV; 3411 } 3412 3413 if (prot & VM_PROT_WRITE) 3414 npte |= L2_S_PROT_W; 3415 npte |= pte_l2_s_cache_mode; 3416 if (m && m == opg) { 3417 /* 3418 * We're changing the attrs of an existing mapping. 3419 */ 3420 oflags = pmap_modify_pv(m, pmap, va, 3421 PVF_WRITE | PVF_EXEC | PVF_WIRED | 3422 PVF_MOD | PVF_REF, nflags); 3423 3424 /* 3425 * We may need to flush the cache if we're 3426 * doing rw-ro... 3427 */ 3428 if (pmap_is_current(pmap) && 3429 (oflags & PVF_NC) == 0 && 3430 (opte & L2_S_PROT_W) != 0 && 3431 (prot & VM_PROT_WRITE) == 0) 3432 cpu_dcache_wb_range(va, PAGE_SIZE); 3433 } else { 3434 /* 3435 * New mapping, or changing the backing page 3436 * of an existing mapping. 3437 */ 3438 if (opg) { 3439 /* 3440 * Replacing an existing mapping with a new one. 3441 * It is part of our managed memory so we 3442 * must remove it from the PV list 3443 */ 3444 pve = pmap_remove_pv(opg, pmap, va); 3445 if (m && (m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) && 3446 pve) 3447 pmap_free_pv_entry(pve); 3448 else if (!pve && 3449 !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) 3450 pve = pmap_get_pv_entry(); 3451 KASSERT(pve != NULL, ("No pv")); 3452 oflags = pve->pv_flags; 3453 3454 /* 3455 * If the old mapping was valid (ref/mod 3456 * emulation creates 'invalid' mappings 3457 * initially) then make sure to frob 3458 * the cache. 3459 */ 3460 if ((oflags & PVF_NC) == 0 && 3461 l2pte_valid(opte)) { 3462 if (PV_BEEN_EXECD(oflags)) { 3463 pmap_idcache_wbinv_range(pmap, va, 3464 PAGE_SIZE); 3465 } else 3466 if (PV_BEEN_REFD(oflags)) { 3467 pmap_dcache_wb_range(pmap, va, 3468 PAGE_SIZE, TRUE, 3469 (oflags & PVF_WRITE) == 0); 3470 } 3471 } 3472 } else if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) 3473 if ((pve = pmap_get_pv_entry()) == NULL) { 3474 panic("pmap_enter: no pv entries"); 3475 } 3476 if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) { 3477 KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, 3478 ("pmap_enter: managed mapping within the clean submap")); 3479 pmap_enter_pv(m, pve, pmap, va, nflags); 3480 } 3481 } 3482 /* 3483 * Make sure userland mappings get the right permissions 3484 */ 3485 if (pmap != pmap_kernel() && va != vector_page) { 3486 npte |= L2_S_PROT_U; 3487 } 3488 3489 /* 3490 * Keep the stats up to date 3491 */ 3492 if (opte == 0) { 3493 l2b->l2b_occupancy++; 3494 pmap->pm_stats.resident_count++; 3495 } 3496 3497 3498 /* 3499 * If this is just a wiring change, the two PTEs will be 3500 * identical, so there's no need to update the page table. 3501 */ 3502 if (npte != opte) { 3503 boolean_t is_cached = pmap_is_current(pmap); 3504 3505 *ptep = npte; 3506 if (is_cached) { 3507 /* 3508 * We only need to frob the cache/tlb if this pmap 3509 * is current 3510 */ 3511 PTE_SYNC(ptep); 3512 if (L1_IDX(va) != L1_IDX(vector_page) && 3513 l2pte_valid(npte)) { 3514 /* 3515 * This mapping is likely to be accessed as 3516 * soon as we return to userland. Fix up the 3517 * L1 entry to avoid taking another 3518 * page/domain fault. 3519 */ 3520 pd_entry_t *pl1pd, l1pd; 3521 3522 pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; 3523 l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | 3524 L1_C_PROTO; 3525 if (*pl1pd != l1pd) { 3526 *pl1pd = l1pd; 3527 PTE_SYNC(pl1pd); 3528 } 3529 } 3530 } 3531 3532 if (PV_BEEN_EXECD(oflags)) 3533 pmap_tlb_flushID_SE(pmap, va); 3534 else if (PV_BEEN_REFD(oflags)) 3535 pmap_tlb_flushD_SE(pmap, va); 3536 3537 3538 if (m) 3539 pmap_vac_me_harder(m, pmap, va); 3540 } 3541} 3542 3543/* 3544 * Maps a sequence of resident pages belonging to the same object. 3545 * The sequence begins with the given page m_start. This page is 3546 * mapped at the given virtual address start. Each subsequent page is 3547 * mapped at a virtual address that is offset from start by the same 3548 * amount as the page is offset from m_start within the object. The 3549 * last page in the sequence is the page with the largest offset from 3550 * m_start that can be mapped at a virtual address less than the given 3551 * virtual address end. Not every virtual page between start and end 3552 * is mapped; only those for which a resident page exists with the 3553 * corresponding offset from m_start are mapped. 3554 */ 3555void 3556pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, 3557 vm_page_t m_start, vm_prot_t prot) 3558{ 3559 vm_page_t m; 3560 vm_pindex_t diff, psize; 3561 3562 psize = atop(end - start); 3563 m = m_start; 3564 PMAP_LOCK(pmap); 3565 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { 3566 pmap_enter_locked(pmap, start + ptoa(diff), m, prot & 3567 (VM_PROT_READ | VM_PROT_EXECUTE), FALSE); 3568 m = TAILQ_NEXT(m, listq); 3569 } 3570 PMAP_UNLOCK(pmap); 3571} 3572 3573/* 3574 * this code makes some *MAJOR* assumptions: 3575 * 1. Current pmap & pmap exists. 3576 * 2. Not wired. 3577 * 3. Read access. 3578 * 4. No page table pages. 3579 * but is *MUCH* faster than pmap_enter... 3580 */ 3581 3582vm_page_t 3583pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, 3584 vm_page_t mpte) 3585{ 3586 3587 PMAP_LOCK(pmap); 3588 pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), 3589 FALSE); 3590 PMAP_UNLOCK(pmap); 3591 return (NULL); 3592} 3593 3594/* 3595 * Routine: pmap_change_wiring 3596 * Function: Change the wiring attribute for a map/virtual-address 3597 * pair. 3598 * In/out conditions: 3599 * The mapping must already exist in the pmap. 3600 */ 3601void 3602pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired) 3603{ 3604 struct l2_bucket *l2b; 3605 pt_entry_t *ptep, pte; 3606 vm_page_t pg; 3607 3608 vm_page_lock_queues(); 3609 PMAP_LOCK(pmap); 3610 l2b = pmap_get_l2_bucket(pmap, va); 3611 KASSERT(l2b, ("No l2b bucket in pmap_change_wiring")); 3612 ptep = &l2b->l2b_kva[l2pte_index(va)]; 3613 pte = *ptep; 3614 pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); 3615 if (pg) 3616 pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired); 3617 vm_page_unlock_queues(); 3618 PMAP_UNLOCK(pmap); 3619} 3620 3621 3622/* 3623 * Copy the range specified by src_addr/len 3624 * from the source map to the range dst_addr/len 3625 * in the destination map. 3626 * 3627 * This routine is only advisory and need not do anything. 3628 */ 3629void 3630pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, 3631 vm_size_t len, vm_offset_t src_addr) 3632{ 3633} 3634 3635 3636/* 3637 * Routine: pmap_extract 3638 * Function: 3639 * Extract the physical page address associated 3640 * with the given map/virtual_address pair. 3641 */ 3642vm_paddr_t 3643pmap_extract(pmap_t pm, vm_offset_t va) 3644{ 3645 struct l2_dtable *l2; 3646 pd_entry_t l1pd; 3647 pt_entry_t *ptep, pte; 3648 vm_paddr_t pa; 3649 u_int l1idx; 3650 l1idx = L1_IDX(va); 3651 3652 PMAP_LOCK(pm); 3653 l1pd = pm->pm_l1->l1_kva[l1idx]; 3654 if (l1pte_section_p(l1pd)) { 3655 /* 3656 * These should only happen for pmap_kernel() 3657 */ 3658 KASSERT(pm == pmap_kernel(), ("huh")); 3659 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3660 } else { 3661 /* 3662 * Note that we can't rely on the validity of the L1 3663 * descriptor as an indication that a mapping exists. 3664 * We have to look it up in the L2 dtable. 3665 */ 3666 l2 = pm->pm_l2[L2_IDX(l1idx)]; 3667 3668 if (l2 == NULL || 3669 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3670 PMAP_UNLOCK(pm); 3671 return (0); 3672 } 3673 3674 ptep = &ptep[l2pte_index(va)]; 3675 pte = *ptep; 3676 3677 if (pte == 0) { 3678 PMAP_UNLOCK(pm); 3679 return (0); 3680 } 3681 3682 switch (pte & L2_TYPE_MASK) { 3683 case L2_TYPE_L: 3684 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3685 break; 3686 3687 default: 3688 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3689 break; 3690 } 3691 } 3692 3693 PMAP_UNLOCK(pm); 3694 return (pa); 3695} 3696 3697/* 3698 * Atomically extract and hold the physical page with the given 3699 * pmap and virtual address pair if that mapping permits the given 3700 * protection. 3701 * 3702 */ 3703vm_page_t 3704pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) 3705{ 3706 struct l2_dtable *l2; 3707 pd_entry_t l1pd; 3708 pt_entry_t *ptep, pte; 3709 vm_paddr_t pa; 3710 vm_page_t m = NULL; 3711 u_int l1idx; 3712 l1idx = L1_IDX(va); 3713 3714 vm_page_lock_queues(); 3715 PMAP_LOCK(pmap); 3716 l1pd = pmap->pm_l1->l1_kva[l1idx]; 3717 if (l1pte_section_p(l1pd)) { 3718 /* 3719 * These should only happen for pmap_kernel() 3720 */ 3721 KASSERT(pmap == pmap_kernel(), ("huh")); 3722 pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); 3723 if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { 3724 m = PHYS_TO_VM_PAGE(pa); 3725 vm_page_hold(m); 3726 } 3727 3728 } else { 3729 /* 3730 * Note that we can't rely on the validity of the L1 3731 * descriptor as an indication that a mapping exists. 3732 * We have to look it up in the L2 dtable. 3733 */ 3734 l2 = pmap->pm_l2[L2_IDX(l1idx)]; 3735 3736 if (l2 == NULL || 3737 (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { 3738 PMAP_UNLOCK(pmap); 3739 vm_page_unlock_queues(); 3740 return (NULL); 3741 } 3742 3743 ptep = &ptep[l2pte_index(va)]; 3744 pte = *ptep; 3745 3746 if (pte == 0) { 3747 PMAP_UNLOCK(pmap); 3748 vm_page_unlock_queues(); 3749 return (NULL); 3750 } 3751 if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { 3752 switch (pte & L2_TYPE_MASK) { 3753 case L2_TYPE_L: 3754 pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); 3755 break; 3756 3757 default: 3758 pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); 3759 break; 3760 } 3761 m = PHYS_TO_VM_PAGE(pa); 3762 vm_page_hold(m); 3763 } 3764 } 3765 3766 PMAP_UNLOCK(pmap); 3767 vm_page_unlock_queues(); 3768 return (m); 3769} 3770 3771/* 3772 * Initialize a preallocated and zeroed pmap structure, 3773 * such as one in a vmspace structure. 3774 */ 3775 3776void 3777pmap_pinit(pmap_t pmap) 3778{ 3779 PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); 3780 3781 PMAP_LOCK_INIT(pmap); 3782 pmap_alloc_l1(pmap); 3783 bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); 3784 3785 pmap->pm_count = 1; 3786 pmap->pm_active = 0; 3787 3788 TAILQ_INIT(&pmap->pm_pvlist); 3789 bzero(&pmap->pm_stats, sizeof pmap->pm_stats); 3790 pmap->pm_stats.resident_count = 1; 3791 if (vector_page < KERNBASE) { 3792 pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa), 3793 VM_PROT_READ, 1); 3794 } 3795} 3796 3797 3798/*************************************************** 3799 * page management routines. 3800 ***************************************************/ 3801 3802 3803static void 3804pmap_free_pv_entry(pv_entry_t pv) 3805{ 3806 pv_entry_count--; 3807 uma_zfree(pvzone, pv); 3808} 3809 3810 3811/* 3812 * get a new pv_entry, allocating a block from the system 3813 * when needed. 3814 * the memory allocation is performed bypassing the malloc code 3815 * because of the possibility of allocations at interrupt time. 3816 */ 3817static pv_entry_t 3818pmap_get_pv_entry(void) 3819{ 3820 pv_entry_t ret_value; 3821 3822 pv_entry_count++; 3823 if ((pv_entry_count > pv_entry_high_water) && 3824 (pmap_pagedaemon_waken == 0)) { 3825 pmap_pagedaemon_waken = 1; 3826 wakeup (&vm_pages_needed); 3827 } 3828 ret_value = uma_zalloc(pvzone, M_NOWAIT); 3829 return ret_value; 3830} 3831 3832 3833/* 3834 * Remove the given range of addresses from the specified map. 3835 * 3836 * It is assumed that the start and end are properly 3837 * rounded to the page size. 3838 */ 3839#define PMAP_REMOVE_CLEAN_LIST_SIZE 3 3840void 3841pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) 3842{ 3843 struct l2_bucket *l2b; 3844 vm_offset_t next_bucket; 3845 pt_entry_t *ptep; 3846 u_int cleanlist_idx, total, cnt; 3847 struct { 3848 vm_offset_t va; 3849 pt_entry_t *pte; 3850 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 3851 u_int mappings, is_exec, is_refd; 3852 int flushall = 0; 3853 3854 3855 /* 3856 * we lock in the pmap => pv_head direction 3857 */ 3858 3859 vm_page_lock_queues(); 3860 PMAP_LOCK(pm); 3861 if (!pmap_is_current(pm)) { 3862 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3863 } else 3864 cleanlist_idx = 0; 3865 3866 total = 0; 3867 while (sva < eva) { 3868 /* 3869 * Do one L2 bucket's worth at a time. 3870 */ 3871 next_bucket = L2_NEXT_BUCKET(sva); 3872 if (next_bucket > eva) 3873 next_bucket = eva; 3874 3875 l2b = pmap_get_l2_bucket(pm, sva); 3876 if (l2b == NULL) { 3877 sva = next_bucket; 3878 continue; 3879 } 3880 3881 ptep = &l2b->l2b_kva[l2pte_index(sva)]; 3882 mappings = 0; 3883 3884 while (sva < next_bucket) { 3885 struct vm_page *pg; 3886 pt_entry_t pte; 3887 vm_paddr_t pa; 3888 3889 pte = *ptep; 3890 3891 if (pte == 0) { 3892 /* 3893 * Nothing here, move along 3894 */ 3895 sva += PAGE_SIZE; 3896 ptep++; 3897 continue; 3898 } 3899 3900 pm->pm_stats.resident_count--; 3901 pa = l2pte_pa(pte); 3902 is_exec = 0; 3903 is_refd = 1; 3904 3905 /* 3906 * Update flags. In a number of circumstances, 3907 * we could cluster a lot of these and do a 3908 * number of sequential pages in one go. 3909 */ 3910 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 3911 struct pv_entry *pve; 3912 3913 pve = pmap_remove_pv(pg, pm, sva); 3914 if (pve) { 3915 is_exec = PV_BEEN_EXECD(pve->pv_flags); 3916 is_refd = PV_BEEN_REFD(pve->pv_flags); 3917 pmap_free_pv_entry(pve); 3918 } 3919 } 3920 3921 if (!l2pte_valid(pte)) { 3922 *ptep = 0; 3923 PTE_SYNC_CURRENT(pm, ptep); 3924 sva += PAGE_SIZE; 3925 ptep++; 3926 mappings++; 3927 continue; 3928 } 3929 3930 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 3931 /* Add to the clean list. */ 3932 cleanlist[cleanlist_idx].pte = ptep; 3933 cleanlist[cleanlist_idx].va = 3934 sva | (is_exec & 1); 3935 cleanlist_idx++; 3936 } else 3937 if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 3938 /* Nuke everything if needed. */ 3939 pmap_idcache_wbinv_all(pm); 3940 pmap_tlb_flushID(pm); 3941 3942 /* 3943 * Roll back the previous PTE list, 3944 * and zero out the current PTE. 3945 */ 3946 for (cnt = 0; 3947 cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 3948 *cleanlist[cnt].pte = 0; 3949 } 3950 *ptep = 0; 3951 PTE_SYNC(ptep); 3952 cleanlist_idx++; 3953 flushall = 1; 3954 } else { 3955 *ptep = 0; 3956 PTE_SYNC(ptep); 3957 if (is_exec) 3958 pmap_tlb_flushID_SE(pm, sva); 3959 else 3960 if (is_refd) 3961 pmap_tlb_flushD_SE(pm, sva); 3962 } 3963 3964 sva += PAGE_SIZE; 3965 ptep++; 3966 mappings++; 3967 } 3968 3969 /* 3970 * Deal with any left overs 3971 */ 3972 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 3973 total += cleanlist_idx; 3974 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 3975 vm_offset_t clva = 3976 cleanlist[cnt].va & ~1; 3977 if (cleanlist[cnt].va & 1) { 3978 pmap_idcache_wbinv_range(pm, 3979 clva, PAGE_SIZE); 3980 pmap_tlb_flushID_SE(pm, clva); 3981 } else { 3982 pmap_dcache_wb_range(pm, 3983 clva, PAGE_SIZE, TRUE, 3984 FALSE); 3985 pmap_tlb_flushD_SE(pm, clva); 3986 } 3987 *cleanlist[cnt].pte = 0; 3988 PTE_SYNC_CURRENT(pm, cleanlist[cnt].pte); 3989 } 3990 3991 if (total <= PMAP_REMOVE_CLEAN_LIST_SIZE) 3992 cleanlist_idx = 0; 3993 else { 3994 /* 3995 * We are removing so much entries it's just 3996 * easier to flush the whole cache. 3997 */ 3998 cleanlist_idx = PMAP_REMOVE_CLEAN_LIST_SIZE + 1; 3999 pmap_idcache_wbinv_all(pm); 4000 flushall = 1; 4001 } 4002 } 4003 4004 pmap_free_l2_bucket(pm, l2b, mappings); 4005 } 4006 4007 vm_page_unlock_queues(); 4008 if (flushall) 4009 cpu_tlb_flushID(); 4010 PMAP_UNLOCK(pm); 4011} 4012 4013 4014 4015 4016/* 4017 * pmap_zero_page() 4018 * 4019 * Zero a given physical page by mapping it at a page hook point. 4020 * In doing the zero page op, the page we zero is mapped cachable, as with 4021 * StrongARM accesses to non-cached pages are non-burst making writing 4022 * _any_ bulk data very slow. 4023 */ 4024#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 4025void 4026pmap_zero_page_generic(vm_paddr_t phys, int off, int size) 4027{ 4028#ifdef DEBUG 4029 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 4030 4031 if (pg->md.pvh_list != NULL) 4032 panic("pmap_zero_page: page has mappings"); 4033#endif 4034 4035 if (_arm_bzero && 4036 _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) 4037 return; 4038 4039 4040 mtx_lock(&cmtx); 4041 /* 4042 * Hook in the page, zero it, and purge the cache for that 4043 * zeroed page. Invalidate the TLB as needed. 4044 */ 4045 *cdst_pte = L2_S_PROTO | phys | 4046 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4047 PTE_SYNC(cdst_pte); 4048 cpu_tlb_flushD_SE(cdstp); 4049 cpu_cpwait(); 4050 if (off || size != PAGE_SIZE) 4051 bzero((void *)(cdstp + off), size); 4052 else 4053 bzero_page(cdstp); 4054 mtx_unlock(&cmtx); 4055 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4056} 4057#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 4058 4059#if ARM_MMU_XSCALE == 1 4060void 4061pmap_zero_page_xscale(vm_paddr_t phys, int off, int size) 4062{ 4063 4064 if (_arm_bzero && 4065 _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) 4066 return; 4067 mtx_lock(&cmtx); 4068 /* 4069 * Hook in the page, zero it, and purge the cache for that 4070 * zeroed page. Invalidate the TLB as needed. 4071 */ 4072 *cdst_pte = L2_S_PROTO | phys | 4073 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4074 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4075 PTE_SYNC(cdst_pte); 4076 cpu_tlb_flushD_SE(cdstp); 4077 cpu_cpwait(); 4078 if (off || size != PAGE_SIZE) 4079 bzero((void *)(cdstp + off), size); 4080 else 4081 bzero_page(cdstp); 4082 mtx_unlock(&cmtx); 4083 xscale_cache_clean_minidata(); 4084} 4085 4086/* 4087 * Change the PTEs for the specified kernel mappings such that they 4088 * will use the mini data cache instead of the main data cache. 4089 */ 4090void 4091pmap_use_minicache(vm_offset_t va, vm_size_t size) 4092{ 4093 struct l2_bucket *l2b; 4094 pt_entry_t *ptep, *sptep, pte; 4095 vm_offset_t next_bucket, eva; 4096 4097#if (ARM_NMMUS > 1) 4098 if (xscale_use_minidata == 0) 4099 return; 4100#endif 4101 4102 eva = va + size; 4103 4104 while (va < eva) { 4105 next_bucket = L2_NEXT_BUCKET(va); 4106 if (next_bucket > eva) 4107 next_bucket = eva; 4108 4109 l2b = pmap_get_l2_bucket(pmap_kernel(), va); 4110 4111 sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; 4112 4113 while (va < next_bucket) { 4114 pte = *ptep; 4115 if (!l2pte_minidata(pte)) { 4116 cpu_dcache_wbinv_range(va, PAGE_SIZE); 4117 cpu_tlb_flushD_SE(va); 4118 *ptep = pte & ~L2_B; 4119 } 4120 ptep++; 4121 va += PAGE_SIZE; 4122 } 4123 PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); 4124 } 4125 cpu_cpwait(); 4126} 4127#endif /* ARM_MMU_XSCALE == 1 */ 4128 4129/* 4130 * pmap_zero_page zeros the specified hardware page by mapping 4131 * the page into KVM and using bzero to clear its contents. 4132 */ 4133void 4134pmap_zero_page(vm_page_t m) 4135{ 4136 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), 0, PAGE_SIZE); 4137} 4138 4139 4140/* 4141 * pmap_zero_page_area zeros the specified hardware page by mapping 4142 * the page into KVM and using bzero to clear its contents. 4143 * 4144 * off and size may not cover an area beyond a single hardware page. 4145 */ 4146void 4147pmap_zero_page_area(vm_page_t m, int off, int size) 4148{ 4149 4150 pmap_zero_page_func(VM_PAGE_TO_PHYS(m), off, size); 4151} 4152 4153 4154/* 4155 * pmap_zero_page_idle zeros the specified hardware page by mapping 4156 * the page into KVM and using bzero to clear its contents. This 4157 * is intended to be called from the vm_pagezero process only and 4158 * outside of Giant. 4159 */ 4160void 4161pmap_zero_page_idle(vm_page_t m) 4162{ 4163 4164 pmap_zero_page(m); 4165} 4166 4167#if 0 4168/* 4169 * pmap_clean_page() 4170 * 4171 * This is a local function used to work out the best strategy to clean 4172 * a single page referenced by its entry in the PV table. It's used by 4173 * pmap_copy_page, pmap_zero page and maybe some others later on. 4174 * 4175 * Its policy is effectively: 4176 * o If there are no mappings, we don't bother doing anything with the cache. 4177 * o If there is one mapping, we clean just that page. 4178 * o If there are multiple mappings, we clean the entire cache. 4179 * 4180 * So that some functions can be further optimised, it returns 0 if it didn't 4181 * clean the entire cache, or 1 if it did. 4182 * 4183 * XXX One bug in this routine is that if the pv_entry has a single page 4184 * mapped at 0x00000000 a whole cache clean will be performed rather than 4185 * just the 1 page. Since this should not occur in everyday use and if it does 4186 * it will just result in not the most efficient clean for the page. 4187 */ 4188static int 4189pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 4190{ 4191 pmap_t pm, pm_to_clean = NULL; 4192 struct pv_entry *npv; 4193 u_int cache_needs_cleaning = 0; 4194 u_int flags = 0; 4195 vm_offset_t page_to_clean = 0; 4196 4197 if (pv == NULL) { 4198 /* nothing mapped in so nothing to flush */ 4199 return (0); 4200 } 4201 4202 /* 4203 * Since we flush the cache each time we change to a different 4204 * user vmspace, we only need to flush the page if it is in the 4205 * current pmap. 4206 */ 4207 if (curthread) 4208 pm = vmspace_pmap(curproc->p_vmspace); 4209 else 4210 pm = pmap_kernel(); 4211 4212 for (npv = pv; npv; npv = TAILQ_NEXT(npv, pv_list)) { 4213 if (npv->pv_pmap == pmap_kernel() || npv->pv_pmap == pm) { 4214 flags |= npv->pv_flags; 4215 /* 4216 * The page is mapped non-cacheable in 4217 * this map. No need to flush the cache. 4218 */ 4219 if (npv->pv_flags & PVF_NC) { 4220#ifdef DIAGNOSTIC 4221 if (cache_needs_cleaning) 4222 panic("pmap_clean_page: " 4223 "cache inconsistency"); 4224#endif 4225 break; 4226 } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) 4227 continue; 4228 if (cache_needs_cleaning) { 4229 page_to_clean = 0; 4230 break; 4231 } else { 4232 page_to_clean = npv->pv_va; 4233 pm_to_clean = npv->pv_pmap; 4234 } 4235 cache_needs_cleaning = 1; 4236 } 4237 } 4238 if (page_to_clean) { 4239 if (PV_BEEN_EXECD(flags)) 4240 pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, 4241 PAGE_SIZE); 4242 else 4243 pmap_dcache_wb_range(pm_to_clean, page_to_clean, 4244 PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); 4245 } else if (cache_needs_cleaning) { 4246 if (PV_BEEN_EXECD(flags)) 4247 pmap_idcache_wbinv_all(pm); 4248 else 4249 pmap_dcache_wbinv_all(pm); 4250 return (1); 4251 } 4252 return (0); 4253} 4254#endif 4255 4256/* 4257 * pmap_copy_page copies the specified (machine independent) 4258 * page by mapping the page into virtual memory and using 4259 * bcopy to copy the page, one machine dependent page at a 4260 * time. 4261 */ 4262 4263/* 4264 * pmap_copy_page() 4265 * 4266 * Copy one physical page into another, by mapping the pages into 4267 * hook points. The same comment regarding cachability as in 4268 * pmap_zero_page also applies here. 4269 */ 4270#if (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 4271void 4272pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) 4273{ 4274#if 0 4275 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4276#endif 4277#ifdef DEBUG 4278 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4279 4280 if (dst_pg->md.pvh_list != NULL) 4281 panic("pmap_copy_page: dst page has mappings"); 4282#endif 4283 4284 4285 /* 4286 * Clean the source page. Hold the source page's lock for 4287 * the duration of the copy so that no other mappings can 4288 * be created while we have a potentially aliased mapping. 4289 */ 4290#if 0 4291 /* 4292 * XXX: Not needed while we call cpu_dcache_wbinv_all() in 4293 * pmap_copy_page(). 4294 */ 4295 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4296#endif 4297 /* 4298 * Map the pages into the page hook points, copy them, and purge 4299 * the cache for the appropriate page. Invalidate the TLB 4300 * as required. 4301 */ 4302 mtx_lock(&cmtx); 4303 *csrc_pte = L2_S_PROTO | src | 4304 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 4305 PTE_SYNC(csrc_pte); 4306 *cdst_pte = L2_S_PROTO | dst | 4307 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 4308 PTE_SYNC(cdst_pte); 4309 cpu_tlb_flushD_SE(csrcp); 4310 cpu_tlb_flushD_SE(cdstp); 4311 cpu_cpwait(); 4312 bcopy_page(csrcp, cdstp); 4313 mtx_unlock(&cmtx); 4314 cpu_dcache_inv_range(csrcp, PAGE_SIZE); 4315 cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); 4316} 4317#endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 4318 4319#if ARM_MMU_XSCALE == 1 4320void 4321pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) 4322{ 4323#if 0 4324 /* XXX: Only needed for pmap_clean_page(), which is commented out. */ 4325 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 4326#endif 4327#ifdef DEBUG 4328 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 4329 4330 if (dst_pg->md.pvh_list != NULL) 4331 panic("pmap_copy_page: dst page has mappings"); 4332#endif 4333 4334 4335 /* 4336 * Clean the source page. Hold the source page's lock for 4337 * the duration of the copy so that no other mappings can 4338 * be created while we have a potentially aliased mapping. 4339 */ 4340#if 0 4341 /* 4342 * XXX: Not needed while we call cpu_dcache_wbinv_all() in 4343 * pmap_copy_page(). 4344 */ 4345 (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); 4346#endif 4347 /* 4348 * Map the pages into the page hook points, copy them, and purge 4349 * the cache for the appropriate page. Invalidate the TLB 4350 * as required. 4351 */ 4352 mtx_lock(&cmtx); 4353 *csrc_pte = L2_S_PROTO | src | 4354 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 4355 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4356 PTE_SYNC(csrc_pte); 4357 *cdst_pte = L2_S_PROTO | dst | 4358 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 4359 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 4360 PTE_SYNC(cdst_pte); 4361 cpu_tlb_flushD_SE(csrcp); 4362 cpu_tlb_flushD_SE(cdstp); 4363 cpu_cpwait(); 4364 bcopy_page(csrcp, cdstp); 4365 mtx_unlock(&cmtx); 4366 xscale_cache_clean_minidata(); 4367} 4368#endif /* ARM_MMU_XSCALE == 1 */ 4369 4370void 4371pmap_copy_page(vm_page_t src, vm_page_t dst) 4372{ 4373 cpu_dcache_wbinv_all(); 4374 if (_arm_memcpy && 4375 _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), 4376 (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) 4377 return; 4378 pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); 4379} 4380 4381 4382 4383 4384/* 4385 * this routine returns true if a physical page resides 4386 * in the given pmap. 4387 */ 4388boolean_t 4389pmap_page_exists_quick(pmap_t pmap, vm_page_t m) 4390{ 4391 pv_entry_t pv; 4392 int loops = 0; 4393 int s; 4394 4395 if (m->flags & PG_FICTITIOUS) 4396 return (FALSE); 4397 4398 s = splvm(); 4399 4400 /* 4401 * Not found, check current mappings returning immediately 4402 */ 4403 for (pv = TAILQ_FIRST(&m->md.pv_list); 4404 pv; 4405 pv = TAILQ_NEXT(pv, pv_list)) { 4406 if (pv->pv_pmap == pmap) { 4407 splx(s); 4408 return (TRUE); 4409 } 4410 loops++; 4411 if (loops >= 16) 4412 break; 4413 } 4414 splx(s); 4415 return (FALSE); 4416} 4417 4418 4419/* 4420 * pmap_ts_referenced: 4421 * 4422 * Return the count of reference bits for a page, clearing all of them. 4423 */ 4424int 4425pmap_ts_referenced(vm_page_t m) 4426{ 4427 return (pmap_clearbit(m, PVF_REF)); 4428} 4429 4430 4431boolean_t 4432pmap_is_modified(vm_page_t m) 4433{ 4434 4435 if (m->md.pvh_attrs & PVF_MOD) 4436 return (TRUE); 4437 4438 return(FALSE); 4439} 4440 4441 4442/* 4443 * Clear the modify bits on the specified physical page. 4444 */ 4445void 4446pmap_clear_modify(vm_page_t m) 4447{ 4448 4449 if (m->md.pvh_attrs & PVF_MOD) 4450 pmap_clearbit(m, PVF_MOD); 4451} 4452 4453 4454/* 4455 * pmap_clear_reference: 4456 * 4457 * Clear the reference bit on the specified physical page. 4458 */ 4459void 4460pmap_clear_reference(vm_page_t m) 4461{ 4462 4463 if (m->md.pvh_attrs & PVF_REF) 4464 pmap_clearbit(m, PVF_REF); 4465} 4466 4467 4468/* 4469 * perform the pmap work for mincore 4470 */ 4471int 4472pmap_mincore(pmap_t pmap, vm_offset_t addr) 4473{ 4474 printf("pmap_mincore()\n"); 4475 4476 return (0); 4477} 4478 4479 4480vm_offset_t 4481pmap_addr_hint(vm_object_t obj, vm_offset_t addr, vm_size_t size) 4482{ 4483 4484 return(addr); 4485} 4486 4487 4488/* 4489 * Map a set of physical memory pages into the kernel virtual 4490 * address space. Return a pointer to where it is mapped. This 4491 * routine is intended to be used for mapping device memory, 4492 * NOT real memory. 4493 */ 4494void * 4495pmap_mapdev(vm_offset_t pa, vm_size_t size) 4496{ 4497 vm_offset_t va, tmpva, offset; 4498 4499 offset = pa & PAGE_MASK; 4500 size = roundup(size, PAGE_SIZE); 4501 4502 GIANT_REQUIRED; 4503 4504 va = kmem_alloc_nofault(kernel_map, size); 4505 if (!va) 4506 panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); 4507 for (tmpva = va; size > 0;) { 4508 pmap_kenter_internal(tmpva, pa, 0); 4509 size -= PAGE_SIZE; 4510 tmpva += PAGE_SIZE; 4511 pa += PAGE_SIZE; 4512 } 4513 4514 return ((void *)(va + offset)); 4515} 4516 4517#define BOOTSTRAP_DEBUG 4518 4519/* 4520 * pmap_map_section: 4521 * 4522 * Create a single section mapping. 4523 */ 4524void 4525pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4526 int prot, int cache) 4527{ 4528 pd_entry_t *pde = (pd_entry_t *) l1pt; 4529 pd_entry_t fl; 4530 4531 KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); 4532 4533 switch (cache) { 4534 case PTE_NOCACHE: 4535 default: 4536 fl = 0; 4537 break; 4538 4539 case PTE_CACHE: 4540 fl = pte_l1_s_cache_mode; 4541 break; 4542 4543 case PTE_PAGETABLE: 4544 fl = pte_l1_s_cache_mode_pt; 4545 break; 4546 } 4547 4548 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4549 L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); 4550 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4551 4552} 4553 4554/* 4555 * pmap_link_l2pt: 4556 * 4557 * Link the L2 page table specified by "pa" into the L1 4558 * page table at the slot for "va". 4559 */ 4560void 4561pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) 4562{ 4563 pd_entry_t *pde = (pd_entry_t *) l1pt, proto; 4564 u_int slot = va >> L1_S_SHIFT; 4565 4566 proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; 4567 4568 pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); 4569 PTE_SYNC(&pde[slot]); 4570 4571 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 4572 4573 4574} 4575 4576/* 4577 * pmap_map_entry 4578 * 4579 * Create a single page mapping. 4580 */ 4581void 4582pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, 4583 int cache) 4584{ 4585 pd_entry_t *pde = (pd_entry_t *) l1pt; 4586 pt_entry_t fl; 4587 pt_entry_t *pte; 4588 4589 KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); 4590 4591 switch (cache) { 4592 case PTE_NOCACHE: 4593 default: 4594 fl = 0; 4595 break; 4596 4597 case PTE_CACHE: 4598 fl = pte_l2_s_cache_mode; 4599 break; 4600 4601 case PTE_PAGETABLE: 4602 fl = pte_l2_s_cache_mode_pt; 4603 break; 4604 } 4605 4606 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4607 panic("pmap_map_entry: no L2 table for VA 0x%08x", va); 4608 4609 pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4610 4611 if (pte == NULL) 4612 panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); 4613 4614 pte[l2pte_index(va)] = 4615 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; 4616 PTE_SYNC(&pte[l2pte_index(va)]); 4617} 4618 4619/* 4620 * pmap_map_chunk: 4621 * 4622 * Map a chunk of memory using the most efficient mappings 4623 * possible (section. large page, small page) into the 4624 * provided L1 and L2 tables at the specified virtual address. 4625 */ 4626vm_size_t 4627pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, 4628 vm_size_t size, int prot, int cache) 4629{ 4630 pd_entry_t *pde = (pd_entry_t *) l1pt; 4631 pt_entry_t *pte, f1, f2s, f2l; 4632 vm_size_t resid; 4633 int i; 4634 4635 resid = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); 4636 4637 if (l1pt == 0) 4638 panic("pmap_map_chunk: no L1 table provided"); 4639 4640#ifdef VERBOSE_INIT_ARM 4641 printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " 4642 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 4643#endif 4644 4645 switch (cache) { 4646 case PTE_NOCACHE: 4647 default: 4648 f1 = 0; 4649 f2l = 0; 4650 f2s = 0; 4651 break; 4652 4653 case PTE_CACHE: 4654 f1 = pte_l1_s_cache_mode; 4655 f2l = pte_l2_l_cache_mode; 4656 f2s = pte_l2_s_cache_mode; 4657 break; 4658 4659 case PTE_PAGETABLE: 4660 f1 = pte_l1_s_cache_mode_pt; 4661 f2l = pte_l2_l_cache_mode_pt; 4662 f2s = pte_l2_s_cache_mode_pt; 4663 break; 4664 } 4665 4666 size = resid; 4667 4668 while (resid > 0) { 4669 /* See if we can use a section mapping. */ 4670 if (L1_S_MAPPABLE_P(va, pa, resid)) { 4671#ifdef VERBOSE_INIT_ARM 4672 printf("S"); 4673#endif 4674 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 4675 L1_S_PROT(PTE_KERNEL, prot) | f1 | 4676 L1_S_DOM(PMAP_DOMAIN_KERNEL); 4677 PTE_SYNC(&pde[va >> L1_S_SHIFT]); 4678 va += L1_S_SIZE; 4679 pa += L1_S_SIZE; 4680 resid -= L1_S_SIZE; 4681 continue; 4682 } 4683 4684 /* 4685 * Ok, we're going to use an L2 table. Make sure 4686 * one is actually in the corresponding L1 slot 4687 * for the current VA. 4688 */ 4689 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 4690 panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); 4691 4692 pte = (pt_entry_t *) kernel_pt_lookup( 4693 pde[L1_IDX(va)] & L1_C_ADDR_MASK); 4694 if (pte == NULL) 4695 panic("pmap_map_chunk: can't find L2 table for VA" 4696 "0x%08x", va); 4697 /* See if we can use a L2 large page mapping. */ 4698 if (L2_L_MAPPABLE_P(va, pa, resid)) { 4699#ifdef VERBOSE_INIT_ARM 4700 printf("L"); 4701#endif 4702 for (i = 0; i < 16; i++) { 4703 pte[l2pte_index(va) + i] = 4704 L2_L_PROTO | pa | 4705 L2_L_PROT(PTE_KERNEL, prot) | f2l; 4706 PTE_SYNC(&pte[l2pte_index(va) + i]); 4707 } 4708 va += L2_L_SIZE; 4709 pa += L2_L_SIZE; 4710 resid -= L2_L_SIZE; 4711 continue; 4712 } 4713 4714 /* Use a small page mapping. */ 4715#ifdef VERBOSE_INIT_ARM 4716 printf("P"); 4717#endif 4718 pte[l2pte_index(va)] = 4719 L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; 4720 PTE_SYNC(&pte[l2pte_index(va)]); 4721 va += PAGE_SIZE; 4722 pa += PAGE_SIZE; 4723 resid -= PAGE_SIZE; 4724 } 4725#ifdef VERBOSE_INIT_ARM 4726 printf("\n"); 4727#endif 4728 return (size); 4729 4730} 4731 4732/********************** Static device map routines ***************************/ 4733 4734static const struct pmap_devmap *pmap_devmap_table; 4735 4736/* 4737 * Register the devmap table. This is provided in case early console 4738 * initialization needs to register mappings created by bootstrap code 4739 * before pmap_devmap_bootstrap() is called. 4740 */ 4741void 4742pmap_devmap_register(const struct pmap_devmap *table) 4743{ 4744 4745 pmap_devmap_table = table; 4746} 4747 4748/* 4749 * Map all of the static regions in the devmap table, and remember 4750 * the devmap table so other parts of the kernel can look up entries 4751 * later. 4752 */ 4753void 4754pmap_devmap_bootstrap(vm_offset_t l1pt, const struct pmap_devmap *table) 4755{ 4756 int i; 4757 4758 pmap_devmap_table = table; 4759 4760 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4761#ifdef VERBOSE_INIT_ARM 4762 printf("devmap: %08x -> %08x @ %08x\n", 4763 pmap_devmap_table[i].pd_pa, 4764 pmap_devmap_table[i].pd_pa + 4765 pmap_devmap_table[i].pd_size - 1, 4766 pmap_devmap_table[i].pd_va); 4767#endif 4768 pmap_map_chunk(l1pt, pmap_devmap_table[i].pd_va, 4769 pmap_devmap_table[i].pd_pa, 4770 pmap_devmap_table[i].pd_size, 4771 pmap_devmap_table[i].pd_prot, 4772 pmap_devmap_table[i].pd_cache); 4773 } 4774} 4775 4776const struct pmap_devmap * 4777pmap_devmap_find_pa(vm_paddr_t pa, vm_size_t size) 4778{ 4779 int i; 4780 4781 if (pmap_devmap_table == NULL) 4782 return (NULL); 4783 4784 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4785 if (pa >= pmap_devmap_table[i].pd_pa && 4786 pa + size <= pmap_devmap_table[i].pd_pa + 4787 pmap_devmap_table[i].pd_size) 4788 return (&pmap_devmap_table[i]); 4789 } 4790 4791 return (NULL); 4792} 4793 4794const struct pmap_devmap * 4795pmap_devmap_find_va(vm_offset_t va, vm_size_t size) 4796{ 4797 int i; 4798 4799 if (pmap_devmap_table == NULL) 4800 return (NULL); 4801 4802 for (i = 0; pmap_devmap_table[i].pd_size != 0; i++) { 4803 if (va >= pmap_devmap_table[i].pd_va && 4804 va + size <= pmap_devmap_table[i].pd_va + 4805 pmap_devmap_table[i].pd_size) 4806 return (&pmap_devmap_table[i]); 4807 } 4808 4809 return (NULL); 4810} 4811 4812