1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58 59/* 60 * File: pmap.h 61 * 62 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 63 * Date: 1985 64 * 65 * Machine-dependent structures for the physical map module. 66 */ 67#ifdef KERNEL_PRIVATE 68#ifndef _PMAP_MACHINE_ 69#define _PMAP_MACHINE_ 1 70 71#ifndef ASSEMBLER 72 73#include <platforms.h> 74 75#include <mach/kern_return.h> 76#include <mach/machine/vm_types.h> 77#include <mach/vm_prot.h> 78#include <mach/vm_statistics.h> 79#include <mach/machine/vm_param.h> 80#include <kern/kern_types.h> 81#include <kern/thread.h> 82#include <kern/lock.h> 83 84#include <i386/mp.h> 85#include <i386/proc_reg.h> 86 87/* 88 * Define the generic in terms of the specific 89 */ 90 91#define INTEL_PGBYTES I386_PGBYTES 92#define INTEL_PGSHIFT I386_PGSHIFT 93#define intel_btop(x) i386_btop(x) 94#define intel_ptob(x) i386_ptob(x) 95#define intel_round_page(x) i386_round_page(x) 96#define intel_trunc_page(x) i386_trunc_page(x) 97#define trunc_intel_to_vm(x) trunc_i386_to_vm(x) 98#define round_intel_to_vm(x) round_i386_to_vm(x) 99#define vm_to_intel(x) vm_to_i386(x) 100 101/* 102 * i386/i486/i860 Page Table Entry 103 */ 104 105#endif /* ASSEMBLER */ 106 107#define NPGPTD 4 108#define PDESHIFT 21 109#define PTEMASK 0x1ff 110#define PTEINDX 3 111 112#define PTESHIFT 12 113 114#define PDESIZE sizeof(pd_entry_t) /* for assembly files */ 115#define PTESIZE sizeof(pt_entry_t) /* for assembly files */ 116 117#define INTEL_OFFMASK (I386_PGBYTES - 1) 118#define PG_FRAME 0x000FFFFFFFFFF000ULL 119#define NPTEPG (PAGE_SIZE/(sizeof (pt_entry_t))) 120#define NPTDPG (PAGE_SIZE/(sizeof (pd_entry_t))) 121 122#define NBPTD (NPGPTD << PAGE_SHIFT) 123#define NPDEPTD (NBPTD / (sizeof (pd_entry_t))) 124#define NPDEPG (PAGE_SIZE/(sizeof (pd_entry_t))) 125#define NBPDE (1 << PDESHIFT) 126#define PDEMASK (NBPDE - 1) 127 128 /* cleanly define parameters for all the page table levels */ 129typedef uint64_t pml4_entry_t; 130#define NPML4PG (PAGE_SIZE/(sizeof (pml4_entry_t))) 131#define PML4SHIFT 39 132#define PML4PGSHIFT 9 133#define NBPML4 (1ULL << PML4SHIFT) 134#define PML4MASK (NBPML4-1) 135#define PML4_ENTRY_NULL ((pml4_entry_t *) 0) 136 137typedef uint64_t pdpt_entry_t; 138#define NPDPTPG (PAGE_SIZE/(sizeof (pdpt_entry_t))) 139#define PDPTSHIFT 30 140#define PDPTPGSHIFT 9 141#define NBPDPT (1 << PDPTSHIFT) 142#define PDPTMASK (NBPDPT-1) 143#define PDPT_ENTRY_NULL ((pdpt_entry_t *) 0) 144 145typedef uint64_t pd_entry_t; 146#define NPDPG (PAGE_SIZE/(sizeof (pd_entry_t))) 147#define PDSHIFT 21 148#define PDPGSHIFT 9 149#define NBPD (1 << PDSHIFT) 150#define PDMASK (NBPD-1) 151#define PD_ENTRY_NULL ((pd_entry_t *) 0) 152 153typedef uint64_t pt_entry_t; 154#define NPTPG (PAGE_SIZE/(sizeof (pt_entry_t))) 155#define PTSHIFT 12 156#define PTPGSHIFT 9 157#define NBPT (1 << PTSHIFT) 158#define PTMASK (NBPT-1) 159#define PT_ENTRY_NULL ((pt_entry_t *) 0) 160 161typedef uint64_t pmap_paddr_t; 162 163/* 164 * Atomic 64-bit store of a page table entry. 165 */ 166static inline void 167pmap_store_pte(pt_entry_t *entryp, pt_entry_t value) 168{ 169 /* 170 * Load the new value into %ecx:%ebx 171 * Load the old value into %edx:%eax 172 * Compare-exchange-8bytes at address entryp (loaded in %edi) 173 * If the compare succeeds, the new value will have been stored. 174 * Otherwise, the old value changed and reloaded, so try again. 175 */ 176 __asm__ volatile( 177 " movl (%0), %%eax \n\t" 178 " movl 4(%0), %%edx \n\t" 179 "1: \n\t" 180 " cmpxchg8b (%0) \n\t" 181 " jnz 1b" 182 : 183 : "D" (entryp), 184 "b" ((uint32_t)value), 185 "c" ((uint32_t)(value >> 32)) 186 : "eax", "edx", "memory"); 187} 188 189/* 190 * Atomic 64-bit compare and exchange of a page table entry. 191 */ 192static inline boolean_t 193pmap_cmpx_pte(pt_entry_t *entryp, pt_entry_t old, pt_entry_t new) 194{ 195 boolean_t ret; 196 197 /* 198 * Load the old value into %edx:%eax 199 * Load the new value into %ecx:%ebx 200 * Compare-exchange-8bytes at address entryp (loaded in %edi) 201 * If the compare succeeds, the new value is stored, return TRUE. 202 * Otherwise, no swap is made, return FALSE. 203 */ 204 asm volatile( 205 " lock; cmpxchg8b (%1) \n\t" 206 " setz %%al \n\t" 207 " movzbl %%al,%0" 208 : "=a" (ret) 209 : "D" (entryp), 210 "a" ((uint32_t)old), 211 "d" ((uint32_t)(old >> 32)), 212 "b" ((uint32_t)new), 213 "c" ((uint32_t)(new >> 32)) 214 : "memory"); 215 return ret; 216} 217 218#define pmap_update_pte(entryp, old, new) \ 219 while (!pmap_cmpx_pte((entryp), (old), (new))) 220 221 222/* in 64 bit spaces, the number of each type of page in the page tables */ 223#define NPML4PGS (1ULL * (PAGE_SIZE/(sizeof (pml4_entry_t)))) 224#define NPDPTPGS (NPML4PGS * (PAGE_SIZE/(sizeof (pdpt_entry_t)))) 225#define NPDEPGS (NPDPTPGS * (PAGE_SIZE/(sizeof (pd_entry_t)))) 226#define NPTEPGS (NPDEPGS * (PAGE_SIZE/(sizeof (pt_entry_t)))) 227 228/* 229 * The 64-bit kernel is remapped in uber-space which is at the base 230 * the highest 4th-level directory (KERNEL_UBER_PML4_INDEX). That is, 231 * 512GB from the top of virtual space (or zero). 232 */ 233#define KERNEL_UBER_PML4_INDEX 511 234#define KERNEL_UBER_BASE (0ULL - NBPML4) 235#define KERNEL_UBER_BASE_HI32 ((uint32_t)(KERNEL_UBER_BASE >> 32)) 236 237#define VM_WIMG_COPYBACK VM_MEM_COHERENT 238#define VM_WIMG_DEFAULT VM_MEM_COHERENT 239/* ?? intel ?? */ 240#define VM_WIMG_IO (VM_MEM_COHERENT | \ 241 VM_MEM_NOT_CACHEABLE | VM_MEM_GUARDED) 242#define VM_WIMG_WTHRU (VM_MEM_WRITE_THROUGH | VM_MEM_COHERENT | VM_MEM_GUARDED) 243/* write combining mode, aka store gather */ 244#define VM_WIMG_WCOMB (VM_MEM_NOT_CACHEABLE | VM_MEM_COHERENT) 245 246/* 247 * Pte related macros 248 */ 249#define VADDR(pdi, pti) ((vm_offset_t)(((pdi)<<PDESHIFT)|((pti)<<PTESHIFT))) 250#define VADDR64(pmi, pdi, pti) ((vm_offset_t)(((pmi)<<PLM4SHIFT))((pdi)<<PDESHIFT)|((pti)<<PTESHIFT)) 251 252/* 253 * Size of Kernel address space. This is the number of page table pages 254 * (4MB each) to use for the kernel. 256 pages == 1 Gigabyte. 255 * This **MUST** be a multiple of 4 (eg: 252, 256, 260, etc). 256 */ 257#ifndef KVA_PAGES 258#define KVA_PAGES 1024 259#endif 260 261#ifndef NKPT 262#define NKPT 500 /* actual number of kernel page tables */ 263#endif 264#ifndef NKPDE 265#define NKPDE (KVA_PAGES - 1) /* addressable number of page tables/pde's */ 266#endif 267 268 269enum high_cpu_types { 270 HIGH_CPU_ISS0, 271 HIGH_CPU_ISS1, 272 HIGH_CPU_DESC, 273 HIGH_CPU_LDT_BEGIN, 274 HIGH_CPU_LDT_END = HIGH_CPU_LDT_BEGIN + (LDTSZ / 512) - 1, 275 HIGH_CPU_END 276}; 277 278enum high_fixed_addresses { 279 HIGH_FIXED_TRAMPS, /* must be first */ 280 HIGH_FIXED_TRAMPS_END, 281 HIGH_FIXED_GDT, 282 HIGH_FIXED_IDT, 283 HIGH_FIXED_LDT_BEGIN, 284 HIGH_FIXED_LDT_END = HIGH_FIXED_LDT_BEGIN + (LDTSZ / 512) - 1, 285 HIGH_FIXED_KTSS, 286 HIGH_FIXED_DFTSS, 287 HIGH_FIXED_DBTSS, 288 HIGH_FIXED_CPUS_BEGIN, 289 HIGH_FIXED_CPUS_END = HIGH_FIXED_CPUS_BEGIN + (HIGH_CPU_END * MAX_CPUS) - 1, 290}; 291 292 293/* XXX64 below PTDI values need cleanup */ 294/* 295 * The *PTDI values control the layout of virtual memory 296 * 297 */ 298#define KPTDI (0x000)/* start of kernel virtual pde's */ 299#define PTDPTDI (0x7F4) /* ptd entry that points to ptd! */ 300#define APTDPTDI (0x7F8) /* alt ptd entry that points to APTD */ 301#define UMAXPTDI (0x7F8) /* ptd entry for user space end */ 302#define UMAXPTEOFF (NPTEPG) /* pte entry for user space end */ 303 304#define KERNBASE VADDR(KPTDI,0) 305 306/* 307 * Convert address offset to directory address 308 * containing the page table pointer - legacy 309 */ 310/*#define pmap_pde(m,v) (&((m)->dirbase[(vm_offset_t)(v) >> PDESHIFT]))*/ 311 312#define HIGH_MEM_BASE ((uint32_t)( -NBPDE) ) /* shared gdt etc seg addr */ /* XXX64 ?? */ 313#define pmap_index_to_virt(x) (HIGH_MEM_BASE | ((unsigned)(x) << PAGE_SHIFT)) 314 315/* 316 * Convert address offset to page descriptor index 317 */ 318#define pdenum(pmap, a) (((vm_offset_t)(a) >> PDESHIFT) & PDEMASK) 319 320#define pdeidx(pmap, a) (((a) >> PDSHIFT) & ((1ULL<<(48 - PDSHIFT)) -1)) 321#define pdptidx(pmap, a) (((a) >> PDPTSHIFT) & ((1ULL<<(48 - PDPTSHIFT)) -1)) 322#define pml4idx(pmap, a) (((a) >> PML4SHIFT) & ((1ULL<<(48 - PML4SHIFT)) -1)) 323 324/* 325 * Convert page descriptor index to user virtual address 326 */ 327#define pdetova(a) ((vm_offset_t)(a) << PDESHIFT) 328 329/* 330 * Convert address offset to page table index 331 */ 332#define ptenum(a) (((vm_offset_t)(a) >> PTESHIFT) & PTEMASK) 333 334/* 335 * Hardware pte bit definitions (to be used directly on the ptes 336 * without using the bit fields). 337 */ 338 339#define INTEL_PTE_VALID 0x00000001 340#define INTEL_PTE_WRITE 0x00000002 341#define INTEL_PTE_RW 0x00000002 342#define INTEL_PTE_USER 0x00000004 343#define INTEL_PTE_WTHRU 0x00000008 344#define INTEL_PTE_NCACHE 0x00000010 345#define INTEL_PTE_REF 0x00000020 346#define INTEL_PTE_MOD 0x00000040 347#define INTEL_PTE_PS 0x00000080 348#define INTEL_PTE_GLOBAL 0x00000100 349#define INTEL_PTE_WIRED 0x00000200 350#define INTEL_PTE_PFN PG_FRAME 351#define INTEL_PTE_PTA 0x00000080 352 353#define INTEL_PTE_NX (1ULL << 63) 354 355#define INTEL_PTE_INVALID 0 356 357#define pa_to_pte(a) ((a) & INTEL_PTE_PFN) /* XXX */ 358#define pte_to_pa(p) ((p) & INTEL_PTE_PFN) /* XXX */ 359#define pte_increment_pa(p) ((p) += INTEL_OFFMASK+1) 360 361#define pte_kernel_rw(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_RW)) 362#define pte_kernel_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID)) 363#define pte_user_rw(p) ((pt_entry)t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER|INTEL_PTE_RW)) 364#define pte_user_ro(p) ((pt_entry_t)(pa_to_pte(p) | INTEL_PTE_VALID|INTEL_PTE_USER)) 365 366#define PMAP_DEFAULT_CACHE 0 367#define PMAP_INHIBIT_CACHE 1 368#define PMAP_GUARDED_CACHE 2 369#define PMAP_ACTIVATE_CACHE 4 370#define PMAP_NO_GUARD_CACHE 8 371 372 373#ifndef ASSEMBLER 374 375#include <sys/queue.h> 376 377/* 378 * Address of current and alternate address space page table maps 379 * and directories. 380 */ 381 382extern pt_entry_t PTmap[], APTmap[], Upte; 383extern pd_entry_t PTD[], APTD[], PTDpde[], APTDpde[], Upde; 384 385extern pd_entry_t *IdlePTD; /* physical address of "Idle" state directory */ 386extern pdpt_entry_t *IdlePDPT; 387 388extern pmap_paddr_t lo_kernel_cr3; 389 390extern pml4_entry_t *IdlePML4; 391extern pdpt_entry_t *IdlePDPT64; 392extern addr64_t kernel64_cr3; 393extern boolean_t no_shared_cr3; 394 395extern uint64_t pmap_pv_hashlist_walks; 396extern uint64_t pmap_pv_hashlist_cnts; 397extern uint32_t pmap_pv_hashlist_max; 398 399/* 400 * virtual address to page table entry and 401 * to physical address. Likewise for alternate address space. 402 * Note: these work recursively, thus vtopte of a pte will give 403 * the corresponding pde that in turn maps it. 404 */ 405#define vtopte(va) (PTmap + i386_btop((vm_offset_t)va)) 406 407 408typedef volatile long cpu_set; /* set of CPUs - must be <= 32 */ 409 /* changed by other processors */ 410struct md_page { 411 int pv_list_count; 412 TAILQ_HEAD(,pv_entry) pv_list; 413}; 414 415#include <vm/vm_page.h> 416 417/* 418 * For each vm_page_t, there is a list of all currently 419 * valid virtual mappings of that page. An entry is 420 * a pv_entry_t; the list is the pv_table. 421 */ 422 423struct pmap { 424 pd_entry_t *dirbase; /* page directory pointer */ 425 pmap_paddr_t pdirbase; /* phys. address of dirbase */ 426 vm_object_t pm_obj; /* object to hold pde's */ 427 int ref_count; /* reference count */ 428 int nx_enabled; 429 task_map_t pm_task_map; 430 decl_simple_lock_data(,lock) /* lock on map */ 431 struct pmap_statistics stats; /* map statistics */ 432 vm_offset_t pm_hold; /* true pdpt zalloc addr */ 433 pmap_paddr_t pm_cr3; /* physical addr */ 434 pdpt_entry_t *pm_pdpt; /* KVA of 3rd level page */ 435 pml4_entry_t *pm_pml4; /* VKA of top level */ 436 vm_object_t pm_obj_pdpt; /* holds pdpt pages */ 437 vm_object_t pm_obj_pml4; /* holds pml4 pages */ 438 vm_object_t pm_obj_top; /* holds single top level page */ 439 boolean_t pm_shared; 440}; 441 442 443#define PMAP_PDPT_FIRST_WINDOW 0 444#define PMAP_PDPT_NWINDOWS 4 445#define PMAP_PDE_FIRST_WINDOW (PMAP_PDPT_NWINDOWS) 446#define PMAP_PDE_NWINDOWS 4 447#define PMAP_PTE_FIRST_WINDOW (PMAP_PDE_FIRST_WINDOW + PMAP_PDE_NWINDOWS) 448#define PMAP_PTE_NWINDOWS 4 449 450#define PMAP_NWINDOWS_FIRSTFREE (PMAP_PTE_FIRST_WINDOW + PMAP_PTE_NWINDOWS) 451#define PMAP_WINDOW_SIZE 8 452#define PMAP_NWINDOWS (PMAP_NWINDOWS_FIRSTFREE + PMAP_WINDOW_SIZE) 453 454typedef struct { 455 pt_entry_t *prv_CMAP; 456 caddr_t prv_CADDR; 457} mapwindow_t; 458 459typedef struct cpu_pmap { 460 int pdpt_window_index; 461 int pde_window_index; 462 int pte_window_index; 463 mapwindow_t mapwindow[PMAP_NWINDOWS]; 464} cpu_pmap_t; 465 466 467extern mapwindow_t *pmap_get_mapwindow(pt_entry_t pentry); 468extern void pmap_put_mapwindow(mapwindow_t *map); 469 470 471typedef struct pmap_memory_regions { 472 ppnum_t base; 473 ppnum_t end; 474 ppnum_t alloc; 475 uint32_t type; 476} pmap_memory_region_t; 477 478unsigned pmap_memory_region_count; 479unsigned pmap_memory_region_current; 480 481#define PMAP_MEMORY_REGIONS_SIZE 128 482 483extern pmap_memory_region_t pmap_memory_regions[]; 484 485static inline void set_dirbase(pmap_t tpmap, __unused int tcpu) { 486 current_cpu_datap()->cpu_task_cr3 = (pmap_paddr_t)((tpmap)->pm_cr3); 487 current_cpu_datap()->cpu_task_map = tpmap->pm_task_map; 488} 489 490/* 491 * External declarations for PMAP_ACTIVATE. 492 */ 493 494extern void process_pmap_updates(void); 495extern void pmap_update_interrupt(void); 496 497/* 498 * Machine dependent routines that are used only for i386/i486/i860. 499 */ 500 501extern addr64_t (kvtophys)( 502 vm_offset_t addr); 503 504extern void pmap_expand( 505 pmap_t pmap, 506 vm_map_offset_t addr); 507 508extern pt_entry_t *pmap_pte( 509 struct pmap *pmap, 510 vm_map_offset_t addr); 511 512extern pd_entry_t *pmap_pde( 513 struct pmap *pmap, 514 vm_map_offset_t addr); 515 516extern pd_entry_t *pmap64_pde( 517 struct pmap *pmap, 518 vm_map_offset_t addr); 519 520extern pdpt_entry_t *pmap64_pdpt( 521 struct pmap *pmap, 522 vm_map_offset_t addr); 523 524extern vm_offset_t pmap_map( 525 vm_offset_t virt, 526 vm_map_offset_t start, 527 vm_map_offset_t end, 528 vm_prot_t prot, 529 unsigned int flags); 530 531extern vm_offset_t pmap_map_bd( 532 vm_offset_t virt, 533 vm_map_offset_t start, 534 vm_map_offset_t end, 535 vm_prot_t prot, 536 unsigned int flags); 537 538extern void pmap_bootstrap( 539 vm_offset_t load_start, 540 boolean_t IA32e); 541 542extern boolean_t pmap_valid_page( 543 ppnum_t pn); 544 545extern int pmap_list_resident_pages( 546 struct pmap *pmap, 547 vm_offset_t *listp, 548 int space); 549 550extern void pmap_commpage32_init( 551 vm_offset_t kernel, 552 vm_offset_t user, 553 int count); 554extern void pmap_commpage64_init( 555 vm_offset_t kernel, 556 vm_map_offset_t user, 557 int count); 558 559extern struct cpu_pmap *pmap_cpu_alloc( 560 boolean_t is_boot_cpu); 561extern void pmap_cpu_free( 562 struct cpu_pmap *cp); 563 564extern void pmap_map_block( 565 pmap_t pmap, 566 addr64_t va, 567 ppnum_t pa, 568 uint32_t size, 569 vm_prot_t prot, 570 int attr, 571 unsigned int flags); 572 573extern void invalidate_icache(vm_offset_t addr, unsigned cnt, int phys); 574extern void flush_dcache(vm_offset_t addr, unsigned count, int phys); 575extern ppnum_t pmap_find_phys(pmap_t map, addr64_t va); 576 577extern void pmap_cpu_init(void); 578extern void pmap_disable_NX(pmap_t pmap); 579extern void pmap_set_4GB_pagezero(pmap_t pmap); 580extern void pmap_clear_4GB_pagezero(pmap_t pmap); 581extern void pmap_load_kernel_cr3(void); 582extern vm_offset_t pmap_cpu_high_map_vaddr(int, enum high_cpu_types); 583extern vm_offset_t pmap_high_map_vaddr(enum high_cpu_types); 584extern vm_offset_t pmap_high_map(pt_entry_t, enum high_cpu_types); 585extern vm_offset_t pmap_cpu_high_shared_remap(int, enum high_cpu_types, vm_offset_t, int); 586extern vm_offset_t pmap_high_shared_remap(enum high_fixed_addresses, vm_offset_t, int); 587 588extern void pt_fake_zone_info(int *, vm_size_t *, vm_size_t *, vm_size_t *, vm_size_t *, int *, int *); 589 590 591 592/* 593 * Macros for speed. 594 */ 595 596 597#include <kern/spl.h> 598 599#if defined(PMAP_ACTIVATE_KERNEL) 600#undef PMAP_ACTIVATE_KERNEL 601#undef PMAP_DEACTIVATE_KERNEL 602#undef PMAP_ACTIVATE_USER 603#undef PMAP_DEACTIVATE_USER 604#endif 605 606 607#define PMAP_ACTIVATE_KERNEL(my_cpu) { \ 608 spl_t spl; \ 609 \ 610 spl = splhigh(); \ 611 if (current_cpu_datap()->cpu_tlb_invalid) \ 612 process_pmap_updates(); \ 613 splx(spl); \ 614} 615 616#define PMAP_DEACTIVATE_KERNEL(my_cpu) { \ 617 spl_t spl; \ 618 \ 619 spl = splhigh(); \ 620 process_pmap_updates(); \ 621 splx(spl); \ 622} 623 624 625#define PMAP_ACTIVATE_MAP(map, my_cpu) { \ 626 register pmap_t tpmap; \ 627 \ 628 tpmap = vm_map_pmap(map); \ 629 set_dirbase(tpmap, my_cpu); \ 630} 631 632#define PMAP_DEACTIVATE_MAP(map, my_cpu) \ 633 if (vm_map_pmap(map)->pm_task_map == TASK_MAP_64BIT_SHARED) \ 634 pmap_load_kernel_cr3(); 635 636 637#define PMAP_ACTIVATE_USER(th, my_cpu) { \ 638 spl_t spl; \ 639 \ 640 spl = splhigh(); \ 641 PMAP_ACTIVATE_MAP(th->map, my_cpu) \ 642 splx(spl); \ 643} 644 645#define PMAP_DEACTIVATE_USER(th, my_cpu) 646 647 648#define PMAP_SWITCH_CONTEXT(old_th, new_th, my_cpu) { \ 649 spl_t spl; \ 650 pt_entry_t *kpdp; \ 651 pt_entry_t *updp; \ 652 int i; \ 653 int need_flush; \ 654 \ 655 need_flush = 0; \ 656 spl = splhigh(); \ 657 if (old_th->map != new_th->map) { \ 658 PMAP_DEACTIVATE_MAP(old_th->map, my_cpu); \ 659 PMAP_ACTIVATE_MAP(new_th->map, my_cpu); \ 660 } \ 661 kpdp = current_cpu_datap()->cpu_copywindow_pdp; \ 662 for (i = 0; i < NCOPY_WINDOWS; i++) { \ 663 if (new_th->machine.copy_window[i].user_base != (user_addr_t)-1) { \ 664 updp = pmap_pde(new_th->map->pmap, \ 665 new_th->machine.copy_window[i].user_base);\ 666 pmap_store_pte(kpdp, updp ? *updp : 0); \ 667 } \ 668 kpdp++; \ 669 } \ 670 splx(spl); \ 671 if (new_th->machine.copyio_state == WINDOWS_OPENED) \ 672 need_flush = 1; \ 673 else \ 674 new_th->machine.copyio_state = WINDOWS_DIRTY; \ 675 if (new_th->machine.physwindow_pte) { \ 676 pmap_store_pte((current_cpu_datap()->cpu_physwindow_ptep), \ 677 new_th->machine.physwindow_pte); \ 678 if (need_flush == 0) \ 679 invlpg((uintptr_t)current_cpu_datap()->cpu_physwindow_base);\ 680 } \ 681 if (need_flush) \ 682 flush_tlb(); \ 683} 684 685#define PMAP_SWITCH_USER(th, new_map, my_cpu) { \ 686 spl_t spl; \ 687 \ 688 spl = splhigh(); \ 689 PMAP_DEACTIVATE_MAP(th->map, my_cpu); \ 690 th->map = new_map; \ 691 PMAP_ACTIVATE_MAP(th->map, my_cpu); \ 692 splx(spl); \ 693 inval_copy_windows(th); \ 694} 695 696/* 697 * Marking the current cpu's cr3 inactive is achieved by setting its lsb. 698 * Marking the current cpu's cr3 active once more involves clearng this bit. 699 * Note that valid page tables are page-aligned and so the bottom 12 bits 700 * are noramlly zero. 701 * We can only mark the current cpu active/inactive but we can test any cpu. 702 */ 703#define CPU_CR3_MARK_INACTIVE() \ 704 current_cpu_datap()->cpu_active_cr3 |= 1 705 706#define CPU_CR3_MARK_ACTIVE() \ 707 current_cpu_datap()->cpu_active_cr3 &= ~1 708 709#define CPU_CR3_IS_ACTIVE(cpu) \ 710 ((cpu_datap(cpu)->cpu_active_cr3 & 1) == 0) 711 712#define CPU_GET_ACTIVE_CR3(cpu) \ 713 (cpu_datap(cpu)->cpu_active_cr3 & ~1) 714 715#define MARK_CPU_IDLE(my_cpu) { \ 716 /* \ 717 * Mark this cpu idle, and remove it from the active set, \ 718 * since it is not actively using any pmap. Signal_cpus \ 719 * will notice that it is idle, and avoid signaling it, \ 720 * but will queue the update request for when the cpu \ 721 * becomes active. \ 722 */ \ 723 int s = splhigh(); \ 724 if (!cpu_mode_is64bit() || no_shared_cr3) \ 725 process_pmap_updates(); \ 726 else \ 727 pmap_load_kernel_cr3(); \ 728 CPU_CR3_MARK_INACTIVE(); \ 729 __asm__ volatile("mfence"); \ 730 splx(s); \ 731} 732 733#define MARK_CPU_ACTIVE(my_cpu) { \ 734 \ 735 int s = splhigh(); \ 736 /* \ 737 * If a kernel_pmap update was requested while this cpu \ 738 * was idle, process it as if we got the interrupt. \ 739 * Before doing so, remove this cpu from the idle set. \ 740 * Since we do not grab any pmap locks while we flush \ 741 * our TLB, another cpu may start an update operation \ 742 * before we finish. Removing this cpu from the idle \ 743 * set assures that we will receive another update \ 744 * interrupt if this happens. \ 745 */ \ 746 CPU_CR3_MARK_ACTIVE(); \ 747 __asm__ volatile("mfence"); \ 748 \ 749 if (current_cpu_datap()->cpu_tlb_invalid) \ 750 process_pmap_updates(); \ 751 splx(s); \ 752} 753 754#define PMAP_CONTEXT(pmap, thread) 755 756#define pmap_kernel_va(VA) \ 757 ((((vm_offset_t) (VA)) >= vm_min_kernel_address) && \ 758 (((vm_offset_t) (VA)) <= vm_max_kernel_address)) 759 760 761#define pmap_resident_count(pmap) ((pmap)->stats.resident_count) 762#define pmap_resident_max(pmap) ((pmap)->stats.resident_max) 763#define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr) 764#define pmap_attribute(pmap,addr,size,attr,value) \ 765 (KERN_INVALID_ADDRESS) 766#define pmap_attribute_cache_sync(addr,size,attr,value) \ 767 (KERN_INVALID_ADDRESS) 768 769#define MACHINE_PMAP_IS_EMPTY 1 770extern boolean_t pmap_is_empty(pmap_t pmap, 771 vm_map_offset_t start, 772 vm_map_offset_t end); 773 774#endif /* ASSEMBLER */ 775 776 777#endif /* _PMAP_MACHINE_ */ 778 779 780#endif /* KERNEL_PRIVATE */ 781