1/* 2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1990,1991,1992 The University of Utah and 34 * the Center for Software Science (CSS). 35 * Copyright (c) 1991,1987 Carnegie Mellon University. 36 * All rights reserved. 37 * 38 * Permission to use, copy, modify and distribute this software and its 39 * documentation is hereby granted, provided that both the copyright 40 * notice and this permission notice appear in all copies of the 41 * software, derivative works or modified versions, and any portions 42 * thereof, and that both notices appear in supporting documentation, 43 * and that all advertising materials mentioning features or use of 44 * this software display the following acknowledgement: ``This product 45 * includes software developed by the Center for Software Science at 46 * the University of Utah.'' 47 * 48 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSS ALLOW FREE USE OF 49 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY 50 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF 51 * THIS SOFTWARE. 52 * 53 * CSS requests users of this software to return to css-dist@cs.utah.edu any 54 * improvements that they make and grant CSS redistribution rights. 55 * 56 * Carnegie Mellon requests users of this software to return to 57 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 58 * School of Computer Science 59 * Carnegie Mellon University 60 * Pittsburgh PA 15213-3890 61 * any improvements or extensions that they make and grant Carnegie Mellon 62 * the rights to redistribute these changes. 63 * 64 * Utah $Hdr: pmap.c 1.28 92/06/23$ 65 * Author: Mike Hibler, Bob Wheeler, University of Utah CSS, 10/90 66 */ 67 68/* 69 * Manages physical address maps for powerpc. 70 * 71 * In addition to hardware address maps, this 72 * module is called upon to provide software-use-only 73 * maps which may or may not be stored in the same 74 * form as hardware maps. These pseudo-maps are 75 * used to store intermediate results from copy 76 * operations to and from address spaces. 77 * 78 * Since the information managed by this module is 79 * also stored by the logical address mapping module, 80 * this module may throw away valid virtual-to-physical 81 * mappings at almost any time. However, invalidations 82 * of virtual-to-physical mappings must be done as 83 * requested. 84 * 85 * In order to cope with hardware architectures which 86 * make virtual-to-physical map invalidates expensive, 87 * this module may delay invalidate or reduced protection 88 * operations until such time as they are actually 89 * necessary. This module is given full information to 90 * when physical maps must be made correct. 91 * 92 */ 93 94#include <zone_debug.h> 95#include <debug.h> 96#include <mach_kgdb.h> 97#include <mach_vm_debug.h> 98#include <db_machine_commands.h> 99 100#include <kern/thread.h> 101#include <kern/simple_lock.h> 102#include <mach/vm_attributes.h> 103#include <mach/vm_param.h> 104#include <vm/vm_kern.h> 105#include <kern/spl.h> 106 107#include <kern/misc_protos.h> 108#include <ppc/misc_protos.h> 109#include <ppc/proc_reg.h> 110 111#include <vm/pmap.h> 112#include <vm/vm_map.h> 113#include <vm/vm_page.h> 114 115#include <ppc/pmap.h> 116#include <ppc/mem.h> 117#include <ppc/mappings.h> 118 119#include <ppc/new_screen.h> 120#include <ppc/Firmware.h> 121#include <ppc/savearea.h> 122#include <ppc/cpu_internal.h> 123#include <ppc/exception.h> 124#include <ppc/low_trace.h> 125#include <ppc/lowglobals.h> 126#include <ppc/limits.h> 127#include <ddb/db_output.h> 128#include <machine/cpu_capabilities.h> 129 130#include <vm/vm_protos.h> /* must be last */ 131 132 133extern unsigned int avail_remaining; 134unsigned int debugbackpocket; /* (TEST/DEBUG) */ 135 136vm_offset_t first_free_virt; 137unsigned int current_free_region; /* Used in pmap_next_page */ 138 139pmapTransTab *pmapTrans; /* Point to the hash to pmap translations */ 140struct phys_entry *phys_table; 141 142/* forward */ 143static void pmap_map_physical(void); 144static void pmap_map_iohole(addr64_t paddr, addr64_t size); 145void pmap_activate(pmap_t pmap, thread_t th, int which_cpu); 146void pmap_deactivate(pmap_t pmap, thread_t th, int which_cpu); 147 148extern void hw_hash_init(void); 149 150/* NOTE: kernel_pmap_store must be in V=R storage and aligned!!!!!!!!!!!!!! */ 151 152extern struct pmap kernel_pmap_store; 153pmap_t kernel_pmap; /* Pointer to kernel pmap and anchor for in-use pmaps */ 154addr64_t kernel_pmap_phys; /* Pointer to kernel pmap and anchor for in-use pmaps, physical address */ 155pmap_t cursor_pmap; /* Pointer to last pmap allocated or previous if removed from in-use list */ 156pmap_t sharedPmap; /* Pointer to common pmap for 64-bit address spaces */ 157struct zone *pmap_zone; /* zone of pmap structures */ 158boolean_t pmap_initialized = FALSE; 159 160int ppc_max_pmaps; /* Maximum number of concurrent address spaces allowed. This is machine dependent */ 161addr64_t vm_max_address; /* Maximum effective address supported */ 162addr64_t vm_max_physical; /* Maximum physical address supported */ 163 164/* 165 * Physical-to-virtual translations are handled by inverted page table 166 * structures, phys_tables. Multiple mappings of a single page are handled 167 * by linking the affected mapping structures. We initialise one region 168 * for phys_tables of the physical memory we know about, but more may be 169 * added as it is discovered (eg. by drivers). 170 */ 171 172/* 173 * free pmap list. caches the first free_pmap_max pmaps that are freed up 174 */ 175int free_pmap_max = 32; 176int free_pmap_count; 177pmap_t free_pmap_list; 178decl_simple_lock_data(,free_pmap_lock) 179 180/* 181 * Function to get index into phys_table for a given physical address 182 */ 183 184struct phys_entry *pmap_find_physentry(ppnum_t pa) 185{ 186 int i; 187 unsigned int entry; 188 189 for (i = pmap_mem_regions_count - 1; i >= 0; i--) { 190 if (pa < pmap_mem_regions[i].mrStart) continue; /* See if we fit in this region */ 191 if (pa > pmap_mem_regions[i].mrEnd) continue; /* Check the end too */ 192 193 entry = (unsigned int)pmap_mem_regions[i].mrPhysTab + ((pa - pmap_mem_regions[i].mrStart) * sizeof(phys_entry_t)); 194 return (struct phys_entry *)entry; 195 } 196// kprintf("DEBUG - pmap_find_physentry: page 0x%08X not found\n", pa); 197 return NULL; 198} 199 200/* 201 * kern_return_t 202 * pmap_add_physical_memory(vm_offset_t spa, vm_offset_t epa, 203 * boolean_t available, unsigned int attr) 204 * 205 * THIS IS NOT SUPPORTED 206 */ 207kern_return_t 208pmap_add_physical_memory( 209 __unused vm_offset_t spa, 210 __unused vm_offset_t epa, 211 __unused boolean_t available, 212 __unused unsigned int attr) 213{ 214 215 panic("Forget it! You can't map no more memory, you greedy puke!\n"); 216 return KERN_SUCCESS; 217} 218 219/* 220 * pmap_map(va, spa, epa, prot) 221 * is called during boot to map memory in the kernel's address map. 222 * A virtual address range starting at "va" is mapped to the physical 223 * address range "spa" to "epa" with machine independent protection 224 * "prot". 225 * 226 * "va", "spa", and "epa" are byte addresses and must be on machine 227 * independent page boundaries. 228 * 229 * Pages with a contiguous virtual address range, the same protection, and attributes. 230 * therefore, we map it with a single block. 231 * 232 * Note that this call will only map into 32-bit space 233 * 234 */ 235 236vm_offset_t 237pmap_map( 238 vm_offset_t va, 239 vm_offset_t spa, 240 vm_offset_t epa, 241 vm_prot_t prot, 242 unsigned int flags) 243{ 244 unsigned int mflags; 245 addr64_t colladr; 246 mflags = 0; /* Make sure this is initialized to nothing special */ 247 if(!(flags & VM_WIMG_USE_DEFAULT)) { /* Are they supplying the attributes? */ 248 mflags = mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1); /* Convert to our mapping_make flags */ 249 } 250 251 if (spa == epa) return(va); 252 253 assert(epa > spa); 254 255 colladr = mapping_make(kernel_pmap, (addr64_t)va, (ppnum_t)(spa >> 12), 256 (mmFlgBlock | mmFlgPerm), (epa - spa) >> 12, (prot & VM_PROT_ALL) ); 257 258 if(colladr) { /* Was something already mapped in the range? */ 259 panic("pmap_map: attempt to map previously mapped range - va = %08X, pa = %08X, epa = %08X, collision = %016llX\n", 260 va, spa, epa, colladr); 261 } 262 return(va); 263} 264 265/* 266 * pmap_map_physical() 267 * Maps physical memory into the kernel's address map beginning at lgPMWvaddr, the 268 * physical memory window. 269 * 270 */ 271void 272pmap_map_physical(void) 273{ 274 unsigned region; 275 uint64_t msize, size; 276 addr64_t paddr, vaddr, colladdr; 277 278 /* Iterate over physical memory regions, block mapping each into the kernel's address map */ 279 for (region = 0; region < (unsigned)pmap_mem_regions_count; region++) { 280 paddr = ((addr64_t)pmap_mem_regions[region].mrStart << 12); /* Get starting physical address */ 281 size = (((addr64_t)pmap_mem_regions[region].mrEnd + 1) << 12) - paddr; 282 283 vaddr = paddr + lowGlo.lgPMWvaddr; /* Get starting virtual address */ 284 285 while (size > 0) { 286 287 msize = ((size > 0x0000020000000000ULL) ? 0x0000020000000000ULL : size); /* Get size, but no more than 2TBs */ 288 289 colladdr = mapping_make(kernel_pmap, vaddr, (paddr >> 12), 290 (mmFlgBlock | mmFlgPerm), (msize >> 12), 291 (VM_PROT_READ | VM_PROT_WRITE)); 292 if (colladdr) { 293 panic ("pmap_map_physical: mapping failure - va = %016llX, pa = %016llX, size = %016llX, collision = %016llX\n", 294 vaddr, (paddr >> 12), (msize >> 12), colladdr); 295 } 296 297 vaddr = vaddr + (uint64_t)msize; /* Point to the next virtual addr */ 298 paddr = paddr + (uint64_t)msize; /* Point to the next physical addr */ 299 size -= msize; 300 } 301 } 302} 303 304/* 305 * pmap_map_iohole(addr64_t paddr, addr64_t size) 306 * Maps an I/O hole into the kernel's address map at its proper offset in 307 * the physical memory window. 308 * 309 */ 310void 311pmap_map_iohole(addr64_t paddr, addr64_t size) 312{ 313 314 addr64_t vaddr, colladdr, msize; 315 316 vaddr = paddr + lowGlo.lgPMWvaddr; /* Get starting virtual address */ 317 318 while (size > 0) { 319 320 msize = ((size > 0x0000020000000000ULL) ? 0x0000020000000000ULL : size); /* Get size, but no more than 2TBs */ 321 322 colladdr = mapping_make(kernel_pmap, vaddr, (paddr >> 12), 323 (mmFlgBlock | mmFlgPerm | mmFlgGuarded | mmFlgCInhib), (msize >> 12), 324 (VM_PROT_READ | VM_PROT_WRITE)); 325 if (colladdr) { 326 panic ("pmap_map_iohole: mapping failed - va = %016llX, pa = %016llX, size = %016llX, collision = %016llX\n", 327 vaddr, (paddr >> 12), (msize >> 12), colladdr); 328 } 329 330 vaddr = vaddr + (uint64_t)msize; /* Point to the next virtual addr */ 331 paddr = paddr + (uint64_t)msize; /* Point to the next physical addr */ 332 size -= msize; 333 } 334} 335 336/* 337 * Bootstrap the system enough to run with virtual memory. 338 * Map the kernel's code and data, and allocate the system page table. 339 * Called with mapping done by BATs. Page_size must already be set. 340 * 341 * Parameters: 342 * msize: Total memory present 343 * first_avail: First virtual address available 344 * kmapsize: Size of kernel text and data 345 */ 346void 347pmap_bootstrap(uint64_t msize, vm_offset_t *first_avail, unsigned int kmapsize) 348{ 349 vm_offset_t addr; 350 vm_size_t size; 351 unsigned int i, num, mapsize, vmpagesz, vmmapsz, nbits; 352 signed bank; 353 uint64_t tmemsize; 354 uint_t htslop; 355 vm_offset_t first_used_addr, PCAsize; 356 struct phys_entry *phys_entry; 357 358 *first_avail = round_page(*first_avail); /* Make sure we start out on a page boundary */ 359 vm_last_addr = VM_MAX_KERNEL_ADDRESS; /* Set the highest address know to VM */ 360 361 /* 362 * Initialize kernel pmap 363 */ 364 kernel_pmap = &kernel_pmap_store; 365 kernel_pmap_phys = (addr64_t)(uintptr_t)&kernel_pmap_store; 366 cursor_pmap = &kernel_pmap_store; 367 368 kernel_pmap->pmap_link.next = (queue_t)kernel_pmap; /* Set up anchor forward */ 369 kernel_pmap->pmap_link.prev = (queue_t)kernel_pmap; /* Set up anchor reverse */ 370 kernel_pmap->ref_count = 1; 371 kernel_pmap->pmapFlags = pmapKeyDef; /* Set the default keys */ 372 kernel_pmap->pmapFlags |= pmapNXdisabled; 373 kernel_pmap->pmapCCtl = pmapCCtlVal; /* Initialize cache control */ 374 kernel_pmap->space = PPC_SID_KERNEL; 375 kernel_pmap->pmapvr = 0; /* Virtual = Real */ 376 377/* 378 * IBM's recommended hash table size is one PTEG for every 2 physical pages. 379 * However, we have found that OSX rarely uses more than 4 PTEs in a PTEG 380 * with this size table. Therefore, by default we allocate a hash table 381 * one half IBM's recommended size, ie one PTEG per 4 pages. The "ht_shift" boot-arg 382 * can be used to override the default hash table size. 383 * We will allocate the hash table in physical RAM, outside of kernel virtual memory, 384 * at the top of the highest bank that will contain it. 385 * Note that "bank" doesn't refer to a physical memory slot here, it is a range of 386 * physically contiguous memory. 387 * 388 * The PCA will go there as well, immediately before the hash table. 389 */ 390 391 nbits = cntlzw(((msize << 1) - 1) >> 32); /* Get first bit in upper half */ 392 if (nbits == 32) /* If upper half was empty, find bit in bottom half */ 393 nbits = nbits + cntlzw((uint_t)((msize << 1) - 1)); 394 tmemsize = 0x8000000000000000ULL >> nbits; /* Get memory size rounded up to power of 2 */ 395 396 /* Calculate hash table size: First, make sure we don't overflow 32-bit arithmetic. */ 397 if (tmemsize > 0x0000002000000000ULL) 398 tmemsize = 0x0000002000000000ULL; 399 400 /* Second, calculate IBM recommended hash table size, ie one PTEG per 2 physical pages */ 401 hash_table_size = (uint_t)(tmemsize >> 13) * PerProcTable[0].ppe_vaddr->pf.pfPTEG; 402 403 /* Third, cut this in half to produce the OSX default, ie one PTEG per 4 physical pages */ 404 hash_table_size >>= 1; 405 406 /* Fourth, adjust default size per "ht_shift" boot arg */ 407 if (hash_table_shift >= 0) /* if positive, make size bigger */ 408 hash_table_size <<= hash_table_shift; 409 else /* if "ht_shift" is negative, make smaller */ 410 hash_table_size >>= (-hash_table_shift); 411 412 /* Fifth, make sure we are at least minimum size */ 413 if (hash_table_size < (256 * 1024)) 414 hash_table_size = (256 * 1024); 415 416 while(1) { /* Try to fit hash table in PCA into contiguous memory */ 417 418 if(hash_table_size < (256 * 1024)) { /* Have we dropped too short? This should never, ever happen */ 419 panic("pmap_bootstrap: Can't find space for hash table\n"); /* This will never print, system isn't up far enough... */ 420 } 421 422 PCAsize = (hash_table_size / PerProcTable[0].ppe_vaddr->pf.pfPTEG) * sizeof(PCA_t); /* Get total size of PCA table */ 423 PCAsize = round_page(PCAsize); /* Make sure it is at least a page long */ 424 425 for(bank = pmap_mem_regions_count - 1; bank >= 0; bank--) { /* Search backwards through banks */ 426 427 hash_table_base = ((addr64_t)pmap_mem_regions[bank].mrEnd << 12) - hash_table_size + PAGE_SIZE; /* Get tenative address */ 428 429 htslop = hash_table_base & (hash_table_size - 1); /* Get the extra that we will round down when we align */ 430 hash_table_base = hash_table_base & -(addr64_t)hash_table_size; /* Round down to correct boundary */ 431 432 if((hash_table_base - round_page(PCAsize)) >= ((addr64_t)pmap_mem_regions[bank].mrStart << 12)) break; /* Leave if we fit */ 433 } 434 435 if(bank >= 0) break; /* We are done if we found a suitable bank */ 436 437 hash_table_size = hash_table_size >> 1; /* Try the next size down */ 438 } 439 440 if(htslop) { /* If there was slop (i.e., wasted pages for alignment) add a new region */ 441 for(i = pmap_mem_regions_count - 1; i >= (unsigned)bank; i--) { /* Copy from end to our bank, including our bank */ 442 pmap_mem_regions[i + 1].mrStart = pmap_mem_regions[i].mrStart; /* Set the start of the bank */ 443 pmap_mem_regions[i + 1].mrAStart = pmap_mem_regions[i].mrAStart; /* Set the start of allocatable area */ 444 pmap_mem_regions[i + 1].mrEnd = pmap_mem_regions[i].mrEnd; /* Set the end address of bank */ 445 pmap_mem_regions[i + 1].mrAEnd = pmap_mem_regions[i].mrAEnd; /* Set the end address of allocatable area */ 446 } 447 448 pmap_mem_regions[i + 1].mrStart = (hash_table_base + hash_table_size) >> 12; /* Set the start of the next bank to the start of the slop area */ 449 pmap_mem_regions[i + 1].mrAStart = (hash_table_base + hash_table_size) >> 12; /* Set the start of allocatable area to the start of the slop area */ 450 pmap_mem_regions[i].mrEnd = (hash_table_base + hash_table_size - 4096) >> 12; /* Set the end of our bank to the end of the hash table */ 451 452 } 453 454 pmap_mem_regions[bank].mrAEnd = (hash_table_base - PCAsize - 4096) >> 12; /* Set the maximum allocatable in this bank */ 455 456 hw_hash_init(); /* Initiaize the hash table and PCA */ 457 hw_setup_trans(); /* Set up hardware registers needed for translation */ 458 459/* 460 * The hash table is now all initialized and so is the PCA. Go on to do the rest of it. 461 * This allocation is from the bottom up. 462 */ 463 464 num = atop_64(msize); /* Get number of pages in all of memory */ 465 466/* Figure out how much we need to allocate */ 467 468 size = (vm_size_t) ( 469 (InitialSaveBloks * PAGE_SIZE) + /* Allow space for the initial context saveareas */ 470 (BackPocketSaveBloks * PAGE_SIZE) + /* For backpocket saveareas */ 471 trcWork.traceSize + /* Size of trace table */ 472 ((((1 << maxAdrSpb) * sizeof(pmapTransTab)) + 4095) & -4096) + /* Size of pmap translate table */ 473 (((num * sizeof(struct phys_entry)) + 4095) & -4096) /* For the physical entries */ 474 ); 475 476 mapsize = size = round_page(size); /* Get size of area to map that we just calculated */ 477 mapsize = mapsize + kmapsize; /* Account for the kernel text size */ 478 479 vmpagesz = round_page(num * sizeof(struct vm_page)); /* Allow for all vm_pages needed to map physical mem */ 480 vmmapsz = round_page((num / 8) * sizeof(struct vm_map_entry)); /* Allow for vm_maps */ 481 482 mapsize = mapsize + vmpagesz + vmmapsz; /* Add the VM system estimates into the grand total */ 483 484 mapsize = mapsize + (4 * 1024 * 1024); /* Allow for 4 meg of extra mappings */ 485 mapsize = ((mapsize / PAGE_SIZE) + MAPPERBLOK - 1) / MAPPERBLOK; /* Get number of blocks of mappings we need */ 486 mapsize = mapsize + ((mapsize + MAPPERBLOK - 1) / MAPPERBLOK); /* Account for the mappings themselves */ 487 488 size = size + (mapsize * PAGE_SIZE); /* Get the true size we need */ 489 490 /* hash table must be aligned to its size */ 491 492 addr = *first_avail; /* Set the address to start allocations */ 493 first_used_addr = addr; /* Remember where we started */ 494 495 bzero((char *)addr, size); /* Clear everything that we are allocating */ 496 497 savearea_init(addr); /* Initialize the savearea chains and data */ 498 499 addr = (vm_offset_t)((unsigned int)addr + ((InitialSaveBloks + BackPocketSaveBloks) * PAGE_SIZE)); /* Point past saveareas */ 500 501 trcWork.traceCurr = (unsigned int)addr; /* Set first trace slot to use */ 502 trcWork.traceStart = (unsigned int)addr; /* Set start of trace table */ 503 trcWork.traceEnd = (unsigned int)addr + trcWork.traceSize; /* Set end of trace table */ 504 505 addr = (vm_offset_t)trcWork.traceEnd; /* Set next allocatable location */ 506 507 pmapTrans = (pmapTransTab *)addr; /* Point to the pmap to hash translation table */ 508 509 pmapTrans[PPC_SID_KERNEL].pmapPAddr = (addr64_t)((uintptr_t)kernel_pmap); /* Initialize the kernel pmap in the translate table */ 510 pmapTrans[PPC_SID_KERNEL].pmapVAddr = CAST_DOWN(unsigned int, kernel_pmap); /* Initialize the kernel pmap in the translate table */ 511 512 addr += ((((1 << maxAdrSpb) * sizeof(pmapTransTab)) + 4095) & -4096); /* Point past pmap translate table */ 513 514/* NOTE: the phys_table must be within the first 2GB of physical RAM. This makes sure we only need to do 32-bit arithmetic */ 515 516 phys_entry = (struct phys_entry *) addr; /* Get pointer to physical table */ 517 518 for (bank = 0; (unsigned)bank < pmap_mem_regions_count; bank++) { /* Set pointer and initialize all banks of ram */ 519 520 pmap_mem_regions[bank].mrPhysTab = phys_entry; /* Set pointer to the physical table for this bank */ 521 522 phys_entry = phys_entry + (pmap_mem_regions[bank].mrEnd - pmap_mem_regions[bank].mrStart + 1); /* Point to the next */ 523 } 524 525 addr += (((num * sizeof(struct phys_entry)) + 4095) & -4096); /* Step on past the physical entries */ 526 527/* 528 * Remaining space is for mapping entries. Tell the initializer routine that 529 * the mapping system can't release this block because it's permanently assigned 530 */ 531 532 mapping_init(); /* Initialize the mapping tables */ 533 534 for(i = addr; i < first_used_addr + size; i += PAGE_SIZE) { /* Add initial mapping blocks */ 535 mapping_free_init(i, 1, 0); /* Pass block address and say that this one is not releasable */ 536 } 537 mapCtl.mapcmin = MAPPERBLOK; /* Make sure we only adjust one at a time */ 538 539 /* Map V=R the page tables */ 540 pmap_map(first_used_addr, first_used_addr, 541 round_page(first_used_addr + size), VM_PROT_READ | VM_PROT_WRITE, VM_WIMG_USE_DEFAULT); 542 543 *first_avail = round_page(first_used_addr + size); /* Set next available page */ 544 first_free_virt = *first_avail; /* Ditto */ 545 546 /* For 64-bit machines, block map physical memory and the I/O hole into kernel space */ 547 if(BootProcInfo.pf.Available & pf64Bit) { /* Are we on a 64-bit machine? */ 548 lowGlo.lgPMWvaddr = PHYS_MEM_WINDOW_VADDR; /* Initialize the physical memory window's virtual address */ 549 550 pmap_map_physical(); /* Block map physical memory into the window */ 551 552 pmap_map_iohole(IO_MEM_WINDOW_VADDR, IO_MEM_WINDOW_SIZE); 553 /* Block map the I/O hole */ 554 } 555 556 /* All the rest of memory is free - add it to the free 557 * regions so that it can be allocated by pmap_steal 558 */ 559 560 pmap_mem_regions[0].mrAStart = (*first_avail >> 12); /* Set up the free area to start allocations (always in the first bank) */ 561 562 current_free_region = 0; /* Set that we will start allocating in bank 0 */ 563 avail_remaining = 0; /* Clear free page count */ 564 for(bank = 0; (unsigned)bank < pmap_mem_regions_count; bank++) { /* Total up all of the pages in the system that are available */ 565 avail_remaining += (pmap_mem_regions[bank].mrAEnd - pmap_mem_regions[bank].mrAStart) + 1; /* Add in allocatable pages in this bank */ 566 } 567 568 569} 570 571/* 572 * pmap_init(spa, epa) 573 * finishes the initialization of the pmap module. 574 * This procedure is called from vm_mem_init() in vm/vm_init.c 575 * to initialize any remaining data structures that the pmap module 576 * needs to map virtual memory (VM is already ON). 577 * 578 * Note that the pmap needs to be sized and aligned to 579 * a power of two. This is because it is used both in virtual and 580 * real so it can't span a page boundary. 581 */ 582 583void 584pmap_init(void) 585{ 586 587 pmap_zone = zinit(pmapSize, 400 * pmapSize, 4096, "pmap"); 588#if ZONE_DEBUG 589 zone_debug_disable(pmap_zone); /* Can't debug this one 'cause it messes with size and alignment */ 590#endif /* ZONE_DEBUG */ 591 592 pmap_initialized = TRUE; 593 594 /* 595 * Initialize list of freed up pmaps 596 */ 597 free_pmap_list = NULL; /* Set that there are no free pmaps */ 598 free_pmap_count = 0; 599 simple_lock_init(&free_pmap_lock, 0); 600 601} 602 603unsigned int pmap_free_pages(void) 604{ 605 return avail_remaining; 606} 607 608/* 609 * This function allocates physical pages. 610 */ 611 612/* Non-optimal, but only used for virtual memory startup. 613 * Allocate memory from a table of free physical addresses 614 * If there are no more free entries, too bad. 615 */ 616 617boolean_t 618pmap_next_page(ppnum_t *addrp) 619{ 620 unsigned int i; 621 622 if(current_free_region >= pmap_mem_regions_count) return FALSE; /* Return failure if we have used everything... */ 623 624 for(i = current_free_region; i < pmap_mem_regions_count; i++) { /* Find the next bank with free pages */ 625 if(pmap_mem_regions[i].mrAStart <= pmap_mem_regions[i].mrAEnd) break; /* Found one */ 626 } 627 628 current_free_region = i; /* Set our current bank */ 629 if(i >= pmap_mem_regions_count) return FALSE; /* Couldn't find a free page */ 630 631 *addrp = pmap_mem_regions[i].mrAStart; /* Allocate the page */ 632 pmap_mem_regions[i].mrAStart = pmap_mem_regions[i].mrAStart + 1; /* Set the next one to go */ 633 avail_remaining--; /* Drop free count */ 634 635 return TRUE; 636} 637 638void pmap_virtual_space( 639 vm_offset_t *startp, 640 vm_offset_t *endp) 641{ 642 *startp = round_page(first_free_virt); 643 *endp = vm_last_addr; 644} 645 646/* 647 * pmap_create 648 * 649 * Create and return a physical map. 650 * 651 * If the size specified for the map is zero, the map is an actual physical 652 * map, and may be referenced by the hardware. 653 * 654 * A pmap is either in the free list or in the in-use list. The only use 655 * of the in-use list (aside from debugging) is to handle the VSID wrap situation. 656 * Whenever a new pmap is allocated (i.e., not recovered from the free list). The 657 * in-use list is matched until a hole in the VSID sequence is found. (Note 658 * that the in-use pmaps are queued in VSID sequence order.) This is all done 659 * while free_pmap_lock is held. 660 * 661 * If the size specified is non-zero, the map will be used in software 662 * only, and is bounded by that size. 663 */ 664pmap_t 665pmap_create(vm_map_size_t size, __unused boolean_t is_64bit) 666{ 667 pmap_t pmap, ckpmap, fore; 668 int s; 669 unsigned int currSID; 670 addr64_t physpmap; 671 672 /* 673 * A software use-only map doesn't even need a pmap structure. 674 */ 675 if (size) 676 return(PMAP_NULL); 677 678 /* 679 * If there is a pmap in the pmap free list, reuse it. 680 * Note that we use free_pmap_list for all chaining of pmaps, both to 681 * the free list and the in use chain (anchored from kernel_pmap). 682 */ 683 s = splhigh(); 684 simple_lock(&free_pmap_lock); 685 686 if(free_pmap_list) { /* Any free? */ 687 pmap = free_pmap_list; /* Yes, allocate it */ 688 free_pmap_list = (pmap_t)pmap->freepmap; /* Dequeue this one (we chain free ones through freepmap) */ 689 free_pmap_count--; 690 } 691 else { 692 simple_unlock(&free_pmap_lock); /* Unlock just in case */ 693 splx(s); 694 695 pmap = (pmap_t) zalloc(pmap_zone); /* Get one */ 696 if (pmap == PMAP_NULL) return(PMAP_NULL); /* Handle out-of-memory condition */ 697 698 bzero((char *)pmap, pmapSize); /* Clean up the pmap */ 699 700 s = splhigh(); 701 simple_lock(&free_pmap_lock); /* Lock it back up */ 702 703 ckpmap = cursor_pmap; /* Get starting point for free ID search */ 704 currSID = ckpmap->spaceNum; /* Get the actual space ID number */ 705 706 while(1) { /* Keep trying until something happens */ 707 708 currSID = (currSID + 1) & (maxAdrSp - 1); /* Get the next in the sequence */ 709 if(((currSID * incrVSID) & (maxAdrSp - 1)) == invalSpace) continue; /* Skip the space we have reserved */ 710 ckpmap = (pmap_t)ckpmap->pmap_link.next; /* On to the next in-use pmap */ 711 712 if(ckpmap->spaceNum != currSID) break; /* If we are out of sequence, this is free */ 713 714 if(ckpmap == cursor_pmap) { /* See if we have 2^20 already allocated */ 715 panic("pmap_create: Maximum number (%d) active address spaces reached\n", maxAdrSp); /* Die pig dog */ 716 } 717 } 718 719 pmap->space = (currSID * incrVSID) & (maxAdrSp - 1); /* Calculate the actual VSID */ 720 pmap->spaceNum = currSID; /* Set the space ID number */ 721/* 722 * Now we link into the chain just before the out of sequence guy. 723 */ 724 725 fore = (pmap_t)ckpmap->pmap_link.prev; /* Get the current's previous */ 726 pmap->pmap_link.next = (queue_t)ckpmap; /* My next points to the current */ 727 fore->pmap_link.next = (queue_t)pmap; /* Current's previous's next points to me */ 728 pmap->pmap_link.prev = (queue_t)fore; /* My prev points to what the current pointed to */ 729 ckpmap->pmap_link.prev = (queue_t)pmap; /* Current's prev points to me */ 730 731 physpmap = ((addr64_t)pmap_find_phys(kernel_pmap, (addr64_t)((uintptr_t)pmap)) << 12) | (addr64_t)((unsigned int)pmap & 0xFFF); /* Get the physical address of the pmap */ 732 733 pmap->pmapvr = (addr64_t)((uintptr_t)pmap) ^ physpmap; /* Make V to R translation mask */ 734 735 pmapTrans[pmap->space].pmapPAddr = physpmap; /* Set translate table physical to point to us */ 736 pmapTrans[pmap->space].pmapVAddr = CAST_DOWN(unsigned int, pmap); /* Set translate table virtual to point to us */ 737 } 738 739 pmap->pmapVmmExt = NULL; /* Clear VMM extension block vaddr */ 740 pmap->pmapVmmExtPhys = 0; /* and the paddr, too */ 741 pmap->pmapFlags = pmapKeyDef; /* Set default key */ 742 pmap->pmapCCtl = pmapCCtlVal; /* Initialize cache control */ 743 pmap->ref_count = 1; 744 pmap->stats.resident_count = 0; 745 pmap->stats.wired_count = 0; 746 pmap->pmapSCSubTag = 0x0000000000000000ULL; /* Make sure this is clean an tidy */ 747 simple_unlock(&free_pmap_lock); 748 749 splx(s); 750 return(pmap); 751} 752 753/* 754 * pmap_destroy 755 * 756 * Gives up a reference to the specified pmap. When the reference count 757 * reaches zero the pmap structure is added to the pmap free list. 758 * 759 * Should only be called if the map contains no valid mappings. 760 */ 761void 762pmap_destroy(pmap_t pmap) 763{ 764 uint32_t ref_count; 765 spl_t s; 766 pmap_t fore, aft; 767 768 if (pmap == PMAP_NULL) 769 return; 770 771 if ((ref_count = hw_atomic_sub(&pmap->ref_count, 1)) == UINT_MAX) /* underflow */ 772 panic("pmap_destroy(): ref_count < 0"); 773 774 if (ref_count > 0) 775 return; /* Still more users, leave now... */ 776 777 if (!(pmap->pmapFlags & pmapVMgsaa)) { /* Don't try this for a shadow assist guest */ 778 pmap_unmap_sharedpage(pmap); /* Remove any mapping of page -1 */ 779 } 780 781#ifdef notdef 782 if(pmap->stats.resident_count != 0) 783 panic("PMAP_DESTROY: pmap not empty"); 784#else 785 if(pmap->stats.resident_count != 0) { 786 pmap_remove(pmap, 0, 0xFFFFFFFFFFFFF000ULL); 787 } 788#endif 789 790 /* 791 * Add the pmap to the pmap free list. 792 */ 793 794 s = splhigh(); 795 /* 796 * Add the pmap to the pmap free list. 797 */ 798 simple_lock(&free_pmap_lock); 799 800 if (free_pmap_count <= free_pmap_max) { /* Do we have enough spares? */ 801 802 pmap->freepmap = free_pmap_list; /* Queue in front */ 803 free_pmap_list = pmap; 804 free_pmap_count++; 805 simple_unlock(&free_pmap_lock); 806 807 } else { 808 if(cursor_pmap == pmap) cursor_pmap = (pmap_t)pmap->pmap_link.prev; /* If we are releasing the cursor, back up */ 809 fore = (pmap_t)pmap->pmap_link.prev; 810 aft = (pmap_t)pmap->pmap_link.next; 811 fore->pmap_link.next = pmap->pmap_link.next; /* My previous's next is my next */ 812 aft->pmap_link.prev = pmap->pmap_link.prev; /* My next's previous is my previous */ 813 simple_unlock(&free_pmap_lock); 814 pmapTrans[pmap->space].pmapPAddr = -1; /* Invalidate the translate table physical */ 815 pmapTrans[pmap->space].pmapVAddr = -1; /* Invalidate the translate table virtual */ 816 zfree(pmap_zone, pmap); 817 } 818 splx(s); 819} 820 821/* 822 * pmap_reference(pmap) 823 * gains a reference to the specified pmap. 824 */ 825void 826pmap_reference(pmap_t pmap) 827{ 828 if (pmap != PMAP_NULL) 829 (void)hw_atomic_add(&pmap->ref_count, 1); /* Bump the count */ 830} 831 832/* 833 * pmap_remove_some_phys 834 * 835 * Removes mappings of the associated page from the specified pmap 836 * 837 */ 838void pmap_remove_some_phys( 839 pmap_t pmap, 840 vm_offset_t pa) 841{ 842 register struct phys_entry *pp; 843 register struct mapping *mp; 844 unsigned int pindex; 845 846 if (pmap == PMAP_NULL) { /* This should never be called with a null pmap */ 847 panic("pmap_remove_some_phys: null pmap\n"); 848 } 849 850 pp = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ 851 if (pp == 0) return; /* Leave if not in physical RAM */ 852 853 do { /* Keep going until we toss all pages from this pmap */ 854 if (pmap->pmapFlags & pmapVMhost) { 855 mp = hw_purge_phys(pp); /* Toss a map */ 856 switch ((unsigned int)mp & mapRetCode) { 857 case mapRtOK: 858 mapping_free(mp); /* Return mapping to free inventory */ 859 break; 860 case mapRtGuest: 861 break; /* Don't try to return a guest mapping */ 862 case mapRtEmpty: 863 break; /* Physent chain empty, we're done */ 864 case mapRtNotFnd: 865 break; /* Mapping disappeared on us, retry */ 866 default: 867 panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %p, pmap = %p, code = %p\n", 868 pp, pmap, mp); /* Handle failure with our usual lack of tact */ 869 } 870 } else { 871 mp = hw_purge_space(pp, pmap); /* Toss a map */ 872 switch ((unsigned int)mp & mapRetCode) { 873 case mapRtOK: 874 mapping_free(mp); /* Return mapping to free inventory */ 875 break; 876 case mapRtEmpty: 877 break; /* Physent chain empty, we're done */ 878 case mapRtNotFnd: 879 break; /* Mapping disappeared on us, retry */ 880 default: 881 panic("pmap_remove_some_phys: hw_purge_phys failed - pp = %p, pmap = %p, code = %p\n", 882 pp, pmap, mp); /* Handle failure with our usual lack of tact */ 883 } 884 } 885 } while (mapRtEmpty != ((unsigned int)mp & mapRetCode)); 886 887#if DEBUG 888 if ((pmap->pmapFlags & pmapVMhost) && !pmap_verify_free(pa)) 889 panic("pmap_remove_some_phys: cruft left behind - pa = %08X, pmap = %p\n", pa, pmap); 890#endif 891 892 return; /* Leave... */ 893} 894 895/* 896 * pmap_remove(pmap, s, e) 897 * unmaps all virtual addresses v in the virtual address 898 * range determined by [s, e) and pmap. 899 * s and e must be on machine independent page boundaries and 900 * s must be less than or equal to e. 901 * 902 * Note that pmap_remove does not remove any mappings in nested pmaps. We just 903 * skip those segments. 904 */ 905void 906pmap_remove( 907 pmap_t pmap, 908 addr64_t sva, 909 addr64_t eva) 910{ 911 addr64_t va, endva; 912 913 if (pmap == PMAP_NULL) return; /* Leave if software pmap */ 914 915 916 /* It is just possible that eva might have wrapped around to zero, 917 * and sometimes we get asked to liberate something of size zero 918 * even though it's dumb (eg. after zero length read_overwrites) 919 */ 920 assert(eva >= sva); 921 922 /* If these are not page aligned the loop might not terminate */ 923 assert((sva == trunc_page_64(sva)) && (eva == trunc_page_64(eva))); 924 925 va = sva & -4096LL; /* Round start down to a page */ 926 endva = eva & -4096LL; /* Round end down to a page */ 927 928 while(1) { /* Go until we finish the range */ 929 va = mapping_remove(pmap, va); /* Remove the mapping and see what's next */ 930 va = va & -4096LL; /* Make sure the "not found" indication is clear */ 931 if((va == 0) || (va >= endva)) break; /* End loop if we finish range or run off the end */ 932 } 933 934} 935 936/* 937 * Routine: 938 * pmap_page_protect 939 * 940 * Function: 941 * Lower the permission for all mappings to a given page. 942 */ 943void 944pmap_page_protect( 945 ppnum_t pa, 946 vm_prot_t prot) 947{ 948 register struct phys_entry *pp; 949 boolean_t remove; 950 unsigned int pindex; 951 mapping_t *mp; 952 953 954 switch (prot & VM_PROT_ALL) { 955 case VM_PROT_READ: 956 case VM_PROT_READ|VM_PROT_EXECUTE: 957 remove = FALSE; 958 break; 959 case VM_PROT_ALL: 960 return; 961 default: 962 remove = TRUE; 963 break; 964 } 965 966 967 pp = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ 968 if (pp == 0) return; /* Leave if not in physical RAM */ 969 970 if (remove) { /* If the protection was set to none, we'll remove all mappings */ 971 972 do { /* Keep going until we toss all pages from this physical page */ 973 mp = hw_purge_phys(pp); /* Toss a map */ 974 switch ((unsigned int)mp & mapRetCode) { 975 case mapRtOK: 976 mapping_free(mp); /* Return mapping to free inventory */ 977 break; 978 case mapRtGuest: 979 break; /* Don't try to return a guest mapping */ 980 case mapRtNotFnd: 981 break; /* Mapping disappeared on us, retry */ 982 case mapRtEmpty: 983 break; /* Physent chain empty, we're done */ 984 default: panic("pmap_page_protect: hw_purge_phys failed - pp = %p, code = %p\n", 985 pp, mp); /* Handle failure with our usual lack of tact */ 986 } 987 } while (mapRtEmpty != ((unsigned int)mp & mapRetCode)); 988 989#if DEBUG 990 if (!pmap_verify_free(pa)) 991 panic("pmap_page_protect: cruft left behind - pa = %08X\n", pa); 992#endif 993 994 return; /* Leave... */ 995 } 996 997/* When we get here, it means that we are to change the protection for a 998 * physical page. 999 */ 1000 1001 mapping_protect_phys(pa, (prot & VM_PROT_ALL) ); /* Change protection of all mappings to page. */ 1002 1003} 1004 1005/* 1006 * Routine: 1007 * pmap_disconnect 1008 * 1009 * Function: 1010 * Disconnect all mappings for this page and return reference and change status 1011 * in generic format. 1012 * 1013 */ 1014unsigned int pmap_disconnect( 1015 ppnum_t pa) 1016{ 1017 register struct phys_entry *pp; 1018 unsigned int pindex; 1019 mapping_t *mp; 1020 1021 pp = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ 1022 if (pp == 0) return (0); /* Return null ref and chg if not in physical RAM */ 1023 do { /* Iterate until all mappings are dead and gone */ 1024 mp = hw_purge_phys(pp); /* Disconnect a mapping */ 1025 if (!mp) break; /* All mappings are gone, leave the loop */ 1026 switch ((unsigned int)mp & mapRetCode) { 1027 case mapRtOK: 1028 mapping_free(mp); /* Return mapping to free inventory */ 1029 break; 1030 case mapRtGuest: 1031 break; /* Don't try to return a guest mapping */ 1032 case mapRtNotFnd: 1033 break; /* Mapping disappeared on us, retry */ 1034 case mapRtEmpty: 1035 break; /* Physent chain empty, we're done */ 1036 default: panic("hw_purge_phys: hw_purge_phys failed - pp = %p, code = %p\n", 1037 pp, mp); /* Handle failure with our usual lack of tact */ 1038 } 1039 } while (mapRtEmpty != ((unsigned int)mp & mapRetCode)); 1040 1041#if DEBUG 1042 if (!pmap_verify_free(pa)) 1043 panic("pmap_disconnect: cruft left behind - pa = %08X\n", pa); 1044#endif 1045 1046 return (mapping_tst_refmod(pa)); /* Return page ref and chg in generic format */ 1047} 1048 1049/* 1050 * pmap_protect(pmap, s, e, prot) 1051 * changes the protection on all virtual addresses v in the 1052 * virtual address range determined by [s, e] and pmap to prot. 1053 * s and e must be on machine independent page boundaries and 1054 * s must be less than or equal to e. 1055 * 1056 * Note that any requests to change the protection of a nested pmap are 1057 * ignored. Those changes MUST be done by calling this with the correct pmap. 1058 */ 1059void pmap_protect( 1060 pmap_t pmap, 1061 vm_map_offset_t sva, 1062 vm_map_offset_t eva, 1063 vm_prot_t prot) 1064{ 1065 1066 addr64_t va, endva; 1067 1068 if (pmap == PMAP_NULL) return; /* Do nothing if no pmap */ 1069 1070 if (prot == VM_PROT_NONE) { /* Should we kill the address range?? */ 1071 pmap_remove(pmap, (addr64_t)sva, (addr64_t)eva); /* Yeah, dump 'em */ 1072 return; /* Leave... */ 1073 } 1074 1075 va = sva & -4096LL; /* Round start down to a page */ 1076 endva = eva & -4096LL; /* Round end down to a page */ 1077 1078 while(1) { /* Go until we finish the range */ 1079 mapping_protect(pmap, va, (prot & VM_PROT_ALL), &va); /* Change the protection and see what's next */ 1080 if((va == 0) || (va >= endva)) break; /* End loop if we finish range or run off the end */ 1081 } 1082 1083} 1084 1085 1086 1087/* 1088 * pmap_enter 1089 * 1090 * Create a translation for the virtual address (virt) to the physical 1091 * address (phys) in the pmap with the protection requested. If the 1092 * translation is wired then we can not allow a full page fault, i.e., 1093 * the mapping control block is not eligible to be stolen in a low memory 1094 * condition. 1095 * 1096 * NB: This is the only routine which MAY NOT lazy-evaluate 1097 * or lose information. That is, this routine must actually 1098 * insert this page into the given map NOW. 1099 */ 1100void 1101pmap_enter(pmap_t pmap, vm_map_offset_t va, ppnum_t pa, vm_prot_t prot, 1102 unsigned int flags, __unused boolean_t wired) 1103{ 1104 unsigned int mflags; 1105 addr64_t colva; 1106 1107 if (pmap == PMAP_NULL) return; /* Leave if software pmap */ 1108 1109 mflags = 0; /* Make sure this is initialized to nothing special */ 1110 if(!(flags & VM_WIMG_USE_DEFAULT)) { /* Are they supplying the attributes? */ 1111 mflags = mmFlgUseAttr | (flags & VM_MEM_GUARDED) | ((flags & VM_MEM_NOT_CACHEABLE) >> 1); /* Convert to our mapping_make flags */ 1112 } 1113 1114/* 1115 * It is possible to hang here if another processor is remapping any pages we collide with and are removing 1116 */ 1117 1118 while(1) { /* Keep trying the enter until it goes in */ 1119 1120 colva = mapping_make(pmap, va, pa, mflags, 1, (prot & VM_PROT_ALL) ); /* Enter the mapping into the pmap */ 1121 1122 if(!colva) break; /* If there were no collisions, we are done... */ 1123 1124 mapping_remove(pmap, colva); /* Remove the mapping that collided */ 1125 } 1126} 1127 1128/* 1129 * Enters translations for odd-sized V=F blocks. 1130 * 1131 * The higher level VM map should be locked to insure that we don't have a 1132 * double diddle here. 1133 * 1134 * We panic if we get a block that overlaps with another. We do not merge adjacent 1135 * blocks because removing any address within a block removes the entire block and if 1136 * would really mess things up if we trashed too much. 1137 * 1138 * Once a block is mapped, it is unmutable, that is, protection, catch mode, etc. can 1139 * not be changed. The block must be unmapped and then remapped with the new stuff. 1140 * We also do not keep track of reference or change flags. 1141 * 1142 * Any block that is larger than 256MB must be a multiple of 32MB. We panic if it is not. 1143 * 1144 * Note that pmap_map_block_rc is the same but doesn't panic if collision. 1145 * 1146 */ 1147 1148void pmap_map_block(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags) { /* Map an autogenned block */ 1149 1150 unsigned int mflags; 1151 addr64_t colva; 1152 1153 1154 if (pmap == PMAP_NULL) { /* Did they give us a pmap? */ 1155 panic("pmap_map_block: null pmap\n"); /* No, like that's dumb... */ 1156 } 1157 1158// kprintf("pmap_map_block: (%08X) va = %016llX, pa = %08X, size = %08X, prot = %08X, attr = %08X, flags = %08X\n", /* (BRINGUP) */ 1159// current_thread(), va, pa, size, prot, attr, flags); /* (BRINGUP) */ 1160 1161 mflags = mmFlgBlock | mmFlgUseAttr | (attr & VM_MEM_GUARDED) | ((attr & VM_MEM_NOT_CACHEABLE) >> 1); /* Convert to our mapping_make flags */ 1162 if(flags) mflags |= mmFlgPerm; /* Mark permanent if requested */ 1163 1164 colva = mapping_make(pmap, va, pa, mflags, size, prot); /* Enter the mapping into the pmap */ 1165 1166 if(colva) { /* If there was a collision, panic */ 1167 panic("pmap_map_block: mapping error %d, pmap = %p, va = %016llX\n", (uint32_t)(colva & mapRetCode), pmap, va); 1168 } 1169 1170 return; /* Return */ 1171} 1172 1173int pmap_map_block_rc(pmap_t pmap, addr64_t va, ppnum_t pa, uint32_t size, vm_prot_t prot, int attr, unsigned int flags) { /* Map an autogenned block */ 1174 1175 unsigned int mflags; 1176 addr64_t colva; 1177 1178 1179 if (pmap == PMAP_NULL) { /* Did they give us a pmap? */ 1180 panic("pmap_map_block_rc: null pmap\n"); /* No, like that's dumb... */ 1181 } 1182 1183 mflags = mmFlgBlock | mmFlgUseAttr | (attr & VM_MEM_GUARDED) | ((attr & VM_MEM_NOT_CACHEABLE) >> 1); /* Convert to our mapping_make flags */ 1184 if(flags) mflags |= mmFlgPerm; /* Mark permanent if requested */ 1185 1186 colva = mapping_make(pmap, va, pa, mflags, size, prot); /* Enter the mapping into the pmap */ 1187 1188 if(colva) return 0; /* If there was a collision, fail */ 1189 1190 return 1; /* Return true of we worked */ 1191} 1192 1193/* 1194 * pmap_extract(pmap, va) 1195 * returns the physical address corrsponding to the 1196 * virtual address specified by pmap and va if the 1197 * virtual address is mapped and 0 if it is not. 1198 * Note: we assume nothing is ever mapped to phys 0. 1199 * 1200 * NOTE: This call always will fail for physical addresses greater than 0xFFFFF000. 1201 */ 1202vm_offset_t pmap_extract(pmap_t pmap, vm_map_offset_t va) { 1203 1204 spl_t spl; 1205 register struct mapping *mp; 1206 register vm_offset_t pa; 1207 addr64_t nextva; 1208 ppnum_t ppoffset; 1209 unsigned int gva; 1210 1211#ifdef BOGUSCOMPAT 1212 panic("pmap_extract: THIS CALL IS BOGUS. NEVER USE IT EVER. So there...\n"); /* Don't use this */ 1213#else 1214 1215 gva = (unsigned int)va; /* Make sure we don't have a sign */ 1216 1217 spl = splhigh(); /* We can't allow any loss of control here */ 1218 1219 mp = mapping_find(pmap, (addr64_t)gva, &nextva,1); /* Find the mapping for this address */ 1220 1221 if(!mp) { /* Is the page mapped? */ 1222 splx(spl); /* Enable interrupts */ 1223 return 0; /* Pass back 0 if not found */ 1224 } 1225 1226 ppoffset = (ppnum_t)(((gva & -4096LL) - (mp->mpVAddr & -4096LL)) >> 12); /* Get offset from va to base va */ 1227 1228 1229 pa = mp->mpPAddr + ppoffset; /* Remember ppage because mapping may vanish after drop call */ 1230 1231 mapping_drop_busy(mp); /* We have everything we need from the mapping */ 1232 splx(spl); /* Restore 'rupts */ 1233 1234 if(pa > maxPPage32) return 0; /* Force large addresses to fail */ 1235 1236 pa = (pa << 12) | (va & 0xFFF); /* Convert physical page number to address */ 1237 1238#endif 1239 return pa; /* Return physical address or 0 */ 1240} 1241 1242/* 1243 * ppnum_t pmap_find_phys(pmap, addr64_t va) 1244 * returns the physical page corrsponding to the 1245 * virtual address specified by pmap and va if the 1246 * virtual address is mapped and 0 if it is not. 1247 * Note: we assume nothing is ever mapped to phys 0. 1248 * 1249 */ 1250ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va) { 1251 1252 spl_t spl; 1253 register struct mapping *mp; 1254 ppnum_t pa, ppoffset; 1255 addr64_t nextva; 1256 1257 spl = splhigh(); /* We can't allow any loss of control here */ 1258 1259 mp = mapping_find(pmap, va, &nextva, 1); /* Find the mapping for this address */ 1260 1261 if(!mp) { /* Is the page mapped? */ 1262 splx(spl); /* Enable interrupts */ 1263 return 0; /* Pass back 0 if not found */ 1264 } 1265 1266 1267 ppoffset = (ppnum_t)(((va & -4096LL) - (mp->mpVAddr & -4096LL)) >> 12); /* Get offset from va to base va */ 1268 1269 pa = mp->mpPAddr + ppoffset; /* Get the actual physical address */ 1270 1271 mapping_drop_busy(mp); /* We have everything we need from the mapping */ 1272 1273 splx(spl); /* Restore 'rupts */ 1274 return pa; /* Return physical address or 0 */ 1275} 1276 1277 1278/* 1279 * pmap_attributes: 1280 * 1281 * Set/Get special memory attributes; not implemented. 1282 * 1283 * Note: 'VAL_GET_INFO' is used to return info about a page. 1284 * If less than 1 page is specified, return the physical page 1285 * mapping and a count of the number of mappings to that page. 1286 * If more than one page is specified, return the number 1287 * of resident pages and the number of shared (more than 1288 * one mapping) pages in the range; 1289 * 1290 * 1291 */ 1292kern_return_t 1293pmap_attribute( 1294 __unused pmap_t pmap, 1295 __unused vm_map_offset_t address, 1296 __unused vm_map_size_t size, 1297 __unused vm_machine_attribute_t attribute, 1298 __unused vm_machine_attribute_val_t* value) 1299{ 1300 1301 return KERN_INVALID_ARGUMENT; 1302 1303} 1304 1305 1306 1307unsigned int pmap_cache_attributes(ppnum_t pgn) { 1308 1309 unsigned int flags; 1310 struct phys_entry * pp; 1311 1312 // Find physical address 1313 if ((pp = pmap_find_physentry(pgn))) { 1314 // Use physical attributes as default 1315 // NOTE: DEVICE_PAGER_FLAGS are made to line up 1316 flags = VM_MEM_COHERENT; /* We only support coherent memory */ 1317 if (pp->ppLink & ppG) flags |= VM_MEM_GUARDED; /* Add in guarded if it is */ 1318 if (pp->ppLink & ppI) flags |= VM_MEM_NOT_CACHEABLE; /* Add in cache inhibited if so */ 1319 } else 1320 // If no physical, just hard code attributes 1321 flags = VM_WIMG_IO; 1322 1323 return (flags); 1324} 1325 1326 1327 1328/* 1329 * pmap_attribute_cache_sync(vm_offset_t pa) 1330 * 1331 * Invalidates all of the instruction cache on a physical page and 1332 * pushes any dirty data from the data cache for the same physical page 1333 */ 1334 1335kern_return_t pmap_attribute_cache_sync(ppnum_t pp, vm_size_t size, 1336 __unused vm_machine_attribute_t attribute, 1337 __unused vm_machine_attribute_val_t* value) { 1338 1339 spl_t s; 1340 unsigned int i, npages; 1341 1342 npages = round_page(size) >> 12; /* Get the number of pages to do */ 1343 1344 for(i = 0; i < npages; i++) { /* Do all requested pages */ 1345 s = splhigh(); /* No interruptions here */ 1346 sync_ppage(pp + i); /* Go flush data cache and invalidate icache */ 1347 splx(s); /* Allow interruptions */ 1348 } 1349 1350 return KERN_SUCCESS; 1351} 1352 1353/* 1354 * pmap_sync_page_data_phys(ppnum_t pa) 1355 * 1356 * Invalidates all of the instruction cache on a physical page and 1357 * pushes any dirty data from the data cache for the same physical page 1358 */ 1359 1360void pmap_sync_page_data_phys(ppnum_t pa) { 1361 1362 spl_t s; 1363 1364 s = splhigh(); /* No interruptions here */ 1365 sync_ppage(pa); /* Sync up dem caches */ 1366 splx(s); /* Allow interruptions */ 1367 return; 1368} 1369 1370void 1371pmap_sync_page_attributes_phys(ppnum_t pa) 1372{ 1373 pmap_sync_page_data_phys(pa); 1374} 1375 1376#ifdef CURRENTLY_UNUSED_AND_UNTESTED 1377/* 1378 * pmap_collect 1379 * 1380 * Garbage collects the physical map system for pages that are no longer used. 1381 * It isn't implemented or needed or wanted. 1382 */ 1383void 1384pmap_collect(__unused pmap_t pmap) 1385{ 1386 return; 1387} 1388#endif 1389 1390/* 1391 * Routine: pmap_activate 1392 * Function: 1393 * Binds the given physical map to the given 1394 * processor, and returns a hardware map description. 1395 * It isn't implemented or needed or wanted. 1396 */ 1397void 1398pmap_activate( 1399 __unused pmap_t pmap, 1400 __unused thread_t th, 1401 __unused int which_cpu) 1402{ 1403 return; 1404} 1405/* 1406 * pmap_deactivate: 1407 * It isn't implemented or needed or wanted. 1408 */ 1409void 1410pmap_deactivate( 1411 __unused pmap_t pmap, 1412 __unused thread_t th, 1413 __unused int which_cpu) 1414{ 1415 return; 1416} 1417 1418 1419/* 1420 * pmap_pageable(pmap, s, e, pageable) 1421 * Make the specified pages (by pmap, offset) 1422 * pageable (or not) as requested. 1423 * 1424 * A page which is not pageable may not take 1425 * a fault; therefore, its page table entry 1426 * must remain valid for the duration. 1427 * 1428 * This routine is merely advisory; pmap_enter() 1429 * will specify that these pages are to be wired 1430 * down (or not) as appropriate. 1431 * 1432 * (called from vm/vm_fault.c). 1433 */ 1434void 1435pmap_pageable( 1436 __unused pmap_t pmap, 1437 __unused vm_map_offset_t start, 1438 __unused vm_map_offset_t end, 1439 __unused boolean_t pageable) 1440{ 1441 1442 return; /* This is not used... */ 1443 1444} 1445/* 1446 * Routine: pmap_change_wiring 1447 * NOT USED ANYMORE. 1448 */ 1449void 1450pmap_change_wiring( 1451 __unused pmap_t pmap, 1452 __unused vm_map_offset_t va, 1453 __unused boolean_t wired) 1454{ 1455 return; /* This is not used... */ 1456} 1457 1458/* 1459 * pmap_clear_modify(phys) 1460 * clears the hardware modified ("dirty") bit for one 1461 * machine independant page starting at the given 1462 * physical address. phys must be aligned on a machine 1463 * independant page boundary. 1464 */ 1465void 1466pmap_clear_modify(ppnum_t pa) 1467{ 1468 1469 mapping_clr_mod(pa); /* Clear all change bits for physical page */ 1470 1471} 1472 1473/* 1474 * pmap_is_modified(phys) 1475 * returns TRUE if the given physical page has been modified 1476 * since the last call to pmap_clear_modify(). 1477 */ 1478boolean_t 1479pmap_is_modified(register ppnum_t pa) 1480{ 1481 return mapping_tst_mod(pa); /* Check for modified */ 1482 1483} 1484 1485/* 1486 * pmap_clear_reference(phys) 1487 * clears the hardware referenced bit in the given machine 1488 * independant physical page. 1489 * 1490 */ 1491void 1492pmap_clear_reference(ppnum_t pa) 1493{ 1494 mapping_clr_ref(pa); /* Check for modified */ 1495} 1496 1497/* 1498 * pmap_is_referenced(phys) 1499 * returns TRUE if the given physical page has been referenced 1500 * since the last call to pmap_clear_reference(). 1501 */ 1502boolean_t 1503pmap_is_referenced(ppnum_t pa) 1504{ 1505 return mapping_tst_ref(pa); /* Check for referenced */ 1506} 1507 1508/* 1509 * pmap_get_refmod(phys) 1510 * returns the referenced and modified bits of the specified 1511 * physical page. 1512 */ 1513unsigned int 1514pmap_get_refmod(ppnum_t pa) 1515{ 1516 return (mapping_tst_refmod(pa)); 1517} 1518 1519/* 1520 * pmap_clear_refmod(phys, mask) 1521 * clears the referenced and modified bits as specified by the mask 1522 * of the specified physical page. 1523 */ 1524void 1525pmap_clear_refmod(ppnum_t pa, unsigned int mask) 1526{ 1527 mapping_clr_refmod(pa, mask); 1528} 1529 1530/* 1531 * pmap_eligible_for_execute(ppnum_t pa) 1532 * return true if physical address is eligible to contain executable code; 1533 * otherwise, return false 1534 */ 1535boolean_t 1536pmap_eligible_for_execute(ppnum_t pa) 1537{ 1538 phys_entry_t *physent; 1539 unsigned int pindex; 1540 1541 physent = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ 1542 1543 if((!physent) || (physent->ppLink & ppG)) 1544 return 0; /* If there is no physical entry or marked guarded, 1545 the entry is not eligible for execute */ 1546 1547 return 1; /* Otherwise, entry is eligible for execute */ 1548} 1549 1550#if MACH_VM_DEBUG 1551int 1552pmap_list_resident_pages( 1553 __unused pmap_t pmap, 1554 __unused vm_offset_t *listp, 1555 __unused int space) 1556{ 1557 return 0; 1558} 1559#endif /* MACH_VM_DEBUG */ 1560 1561/* 1562 * Locking: 1563 * spl: VM 1564 */ 1565void 1566pmap_copy_part_page( 1567 vm_offset_t src, 1568 vm_offset_t src_offset, 1569 vm_offset_t dst, 1570 vm_offset_t dst_offset, 1571 vm_size_t len) 1572{ 1573 addr64_t fsrc, fdst; 1574 1575 assert((((dst << 12) & PAGE_MASK) + dst_offset + len) <= PAGE_SIZE); 1576 assert((((src << 12) & PAGE_MASK) + src_offset + len) <= PAGE_SIZE); 1577 1578 fsrc = ((addr64_t)src << 12) + src_offset; 1579 fdst = ((addr64_t)dst << 12) + dst_offset; 1580 1581 phys_copy(fsrc, fdst, len); /* Copy the stuff physically */ 1582} 1583 1584void 1585pmap_zero_part_page( 1586 __unused vm_offset_t p, 1587 __unused vm_offset_t offset, 1588 __unused vm_size_t len) 1589{ 1590 panic("pmap_zero_part_page"); 1591} 1592 1593boolean_t pmap_verify_free(ppnum_t pa) { 1594 1595 struct phys_entry *pp; 1596 unsigned int pindex; 1597 1598 pp = mapping_phys_lookup(pa, &pindex); /* Get physical entry */ 1599 if (pp == 0) return FALSE; /* If there isn't one, show no mapping... */ 1600 1601 if(pp->ppLink & ~(ppLock | ppFlags)) return FALSE; /* We have at least one mapping */ 1602 return TRUE; /* No mappings */ 1603} 1604 1605 1606/* Determine if we need to switch space and set up for it if so */ 1607 1608void pmap_switch(pmap_t map) 1609{ 1610 hw_blow_seg(lowGlo.lgUMWvaddr); /* Blow off the first segment */ 1611 hw_blow_seg(lowGlo.lgUMWvaddr + 0x10000000ULL); /* Blow off the second segment */ 1612 1613/* when changing to kernel space, don't bother 1614 * doing anything, the kernel is mapped from here already. 1615 */ 1616 if (map->space == PPC_SID_KERNEL) { /* Are we switching into kernel space? */ 1617 return; /* If so, we don't do anything... */ 1618 } 1619 1620 hw_set_user_space(map); /* Indicate if we need to load the SRs or not */ 1621 return; /* Bye, bye, butterfly... */ 1622} 1623 1624 1625/* 1626 * The PPC pmap can only nest segments of 256MB, aligned on a 256MB boundary. 1627 */ 1628uint64_t pmap_nesting_size_min = 0x10000000ULL; 1629uint64_t pmap_nesting_size_max = 0x10000000ULL; 1630 1631/* 1632 * kern_return_t pmap_nest(grand, subord, vstart, size) 1633 * 1634 * grand = the pmap that we will nest subord into 1635 * subord = the pmap that goes into the grand 1636 * vstart = start of range in pmap to be inserted 1637 * nstart = start of range in pmap nested pmap 1638 * size = Size of nest area (up to 2TB) 1639 * 1640 * Inserts a pmap into another. This is used to implement shared segments. 1641 * On the current PPC processors, this is limited to segment (256MB) aligned 1642 * segment sized ranges. 1643 * 1644 * We actually kinda allow recursive nests. The gating factor is that we do not allow 1645 * nesting on top of something that is already mapped, i.e., the range must be empty. 1646 * 1647 * Note that we depend upon higher level VM locks to insure that things don't change while 1648 * we are doing this. For example, VM should not be doing any pmap enters while it is nesting 1649 * or do 2 nests at once. 1650 */ 1651 1652kern_return_t pmap_nest(pmap_t grand, pmap_t subord, addr64_t vstart, addr64_t nstart, uint64_t size) { 1653 1654 addr64_t vend, colladdr; 1655 unsigned int msize; 1656 int nlists; 1657 mapping_t *mp; 1658 1659 if(size & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this for multiples of 256MB */ 1660 if((size >> 25) > 65536) return KERN_INVALID_VALUE; /* Max size we can nest is 2TB */ 1661 if(vstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this aligned to 256MB */ 1662 if(nstart & 0x0FFFFFFFULL) return KERN_INVALID_VALUE; /* We can only do this aligned to 256MB */ 1663 1664 if(size == 0) { /* Is the size valid? */ 1665 panic("pmap_nest: size is invalid - %016llX\n", size); 1666 } 1667 1668 msize = (size >> 25) - 1; /* Change size to blocks of 32MB */ 1669 1670 nlists = mapSetLists(grand); /* Set number of lists this will be on */ 1671 1672 mp = mapping_alloc(nlists); /* Get a spare mapping block */ 1673 1674 mp->mpFlags = 0x01000000 | mpNest | mpPerm | mpBSu | nlists; /* Make this a permanent nested pmap with a 32MB basic size unit */ 1675 /* Set the flags. Make sure busy count is 1 */ 1676 mp->mpSpace = subord->space; /* Set the address space/pmap lookup ID */ 1677 mp->u.mpBSize = msize; /* Set the size */ 1678 mp->mpPte = 0; /* Set the PTE invalid */ 1679 mp->mpPAddr = 0; /* Set the physical page number */ 1680 mp->mpVAddr = vstart; /* Set the address */ 1681 mp->mpNestReloc = nstart - vstart; /* Set grand to nested vaddr relocation value */ 1682 1683 colladdr = hw_add_map(grand, mp); /* Go add the mapping to the pmap */ 1684 1685 if(colladdr) { /* Did it collide? */ 1686 vend = vstart + size - 4096; /* Point to the last page we would cover in nest */ 1687 panic("pmap_nest: attempt to nest into a non-empty range - pmap = %p, start = %016llX, end = %016llX\n", 1688 grand, vstart, vend); 1689 } 1690 1691 return KERN_SUCCESS; 1692} 1693 1694/* 1695 * kern_return_t pmap_unnest(grand, vaddr, size) 1696 * 1697 * grand = the pmap that we will nest subord into 1698 * vaddr = start of range in pmap to be unnested 1699 * size = size of range in pmap to be unnested 1700 * 1701 * Removes a pmap from another. This is used to implement shared segments. 1702 * On the current PPC processors, this is limited to segment (256MB) aligned 1703 * segment sized ranges. 1704 */ 1705 1706kern_return_t pmap_unnest(pmap_t grand, addr64_t vaddr, uint64_t size) { 1707 1708 unsigned int tstamp, i, mycpu; 1709 addr64_t nextva; 1710 spl_t s; 1711 mapping_t *mp; 1712 1713 if (size != pmap_nesting_size_min || 1714 (vaddr & (pmap_nesting_size_min-1))) { 1715 panic("pmap_unnest(vaddr=0x%016llx, size=0x016%llx): " 1716 "must be 256MB and aligned\n", 1717 vaddr, size); 1718 } 1719 1720 s = splhigh(); /* Make sure interruptions are disabled */ 1721 1722 mp = mapping_find(grand, vaddr, &nextva, 0); /* Find the nested map */ 1723 1724 if(((unsigned int)mp & mapRetCode) != mapRtOK) { /* See if it was even nested */ 1725 panic("pmap_unnest: Attempt to unnest an unnested segment - va = %016llX\n", vaddr); 1726 } 1727 1728 if((mp->mpFlags & mpType) != mpNest) { /* Did we find something other than a nest? */ 1729 panic("pmap_unnest: Attempt to unnest something that is not a nest - va = %016llX\n", vaddr); 1730 } 1731 1732 if(mp->mpVAddr != vaddr) { /* Make sure the address is the same */ 1733 panic("pmap_unnest: Attempt to unnest something that is not at start of nest - va = %016llX\n", vaddr); 1734 } 1735 1736 (void)hw_atomic_and(&mp->mpFlags, ~mpPerm); /* Show that this mapping is now removable */ 1737 1738 mapping_drop_busy(mp); /* Go ahead and release the mapping now */ 1739 1740 splx(s); /* Restore 'rupts */ 1741 1742 (void)mapping_remove(grand, vaddr); /* Toss the nested pmap mapping */ 1743 1744 invalidateSegs(grand); /* Invalidate the pmap segment cache */ 1745 1746/* 1747 * Note that the following will force the segment registers to be reloaded 1748 * on all processors (if they are using the pmap we just changed) before returning. 1749 * 1750 * This is needed. The reason is that until the segment register is 1751 * reloaded, another thread in the same task on a different processor will 1752 * be able to access memory that it isn't allowed to anymore. That can happen 1753 * because access to the subordinate pmap is being removed, but the pmap is still 1754 * valid. 1755 * 1756 * Note that we only kick the other processor if we see that it was using the pmap while we 1757 * were changing it. 1758 */ 1759 1760 1761 for(i=0; i < real_ncpus; i++) { /* Cycle through processors */ 1762 disable_preemption(); 1763 mycpu = cpu_number(); /* Who am I? Am I just a dream? */ 1764 if((unsigned int)grand == PerProcTable[i].ppe_vaddr->ppUserPmapVirt) { /* Is this guy using the changed pmap? */ 1765 1766 PerProcTable[i].ppe_vaddr->ppInvSeg = 1; /* Show that we need to invalidate the segments */ 1767 1768 if(i != mycpu) { 1769 1770 tstamp = PerProcTable[i].ppe_vaddr->ruptStamp[1]; /* Save the processor's last interrupt time stamp */ 1771 if(cpu_signal(i, SIGPcpureq, CPRQsegload, 0) == KERN_SUCCESS) { /* Make sure we see the pmap change */ 1772 if(!hw_cpu_wcng(&PerProcTable[i].ppe_vaddr->ruptStamp[1], tstamp, LockTimeOut)) { /* Wait for the other processors to enter debug */ 1773 panic("pmap_unnest: Other processor (%d) did not see interruption request\n", i); 1774 } 1775 } 1776 } 1777 } 1778 enable_preemption(); 1779 } 1780 1781 return KERN_SUCCESS; /* Bye, bye, butterfly... */ 1782} 1783 1784 1785/* 1786 * void MapUserMemoryWindowInit(void) 1787 * 1788 * Initialize anything we need to in order to map user address space slices into 1789 * the kernel. Primarily used for copy in/out. 1790 * 1791 * Currently we only support one 512MB slot for this purpose. There are two special 1792 * mappings defined for the purpose: the special pmap nest, and linkage mapping. 1793 * 1794 * The special pmap nest (which is allocated in this function) is used as a place holder 1795 * in the kernel's pmap search list. It is 512MB long and covers the address range 1796 * starting at lgUMWvaddr. It points to no actual memory and when the fault handler 1797 * hits in it, it knows to look in the per_proc and start using the linkage 1798 * mapping contained therin. 1799 * 1800 * The linkage mapping is used to glue the user address space slice into the 1801 * kernel. It contains the relocation information used to transform the faulting 1802 * kernel address into the user address space. It also provides the link to the 1803 * user's pmap. This is pointed to by the per_proc and is switched in and out 1804 * whenever there is a context switch. 1805 * 1806 */ 1807 1808void MapUserMemoryWindowInit(void) { 1809 1810 addr64_t colladdr; 1811 int nlists; 1812 mapping_t *mp; 1813 1814 nlists = mapSetLists(kernel_pmap); /* Set number of lists this will be on */ 1815 1816 mp = mapping_alloc(nlists); /* Get a spare mapping block */ 1817 1818 mp->mpFlags = 0x01000000 | mpLinkage | mpPerm | mpBSu | nlists; /* Make this a permanent nested pmap with a 32MB basic size unit */ 1819 /* Set the flags. Make sure busy count is 1 */ 1820 mp->mpSpace = kernel_pmap->space; /* Set the address space/pmap lookup ID */ 1821 mp->u.mpBSize = 15; /* Set the size to 2 segments in 32MB chunks - 1 */ 1822 mp->mpPte = 0; /* Means nothing */ 1823 mp->mpPAddr = 0; /* Means nothing */ 1824 mp->mpVAddr = lowGlo.lgUMWvaddr; /* Set the address range we cover */ 1825 mp->mpNestReloc = 0; /* Means nothing */ 1826 1827 colladdr = hw_add_map(kernel_pmap, mp); /* Go add the mapping to the pmap */ 1828 1829 if(colladdr) { /* Did it collide? */ 1830 panic("MapUserMemoryWindowInit: MapUserMemoryWindow range already mapped\n"); 1831 } 1832 1833 return; 1834} 1835 1836/* 1837 * addr64_t MapUserMemoryWindow(vm_map_t map, vm_offset_t va, size) 1838 * 1839 * map = the vm_map that we are mapping into the kernel 1840 * va = start of the address range we are mapping 1841 * Note that we do not test validty, we chose to trust our fellows... 1842 * 1843 * Maps a 512M slice of a user address space into a predefined kernel range 1844 * on a per-thread basis. We map only the first 256M segment, allowing the 1845 * second 256M segment to fault in as needed. This allows our clients to access 1846 * an arbitrarily aligned operand up to 256M in size. 1847 * 1848 * In the future, the restriction of a predefined range may be loosened. 1849 * 1850 * Builds the proper linkage map to map the user range 1851 * We will round this down to the previous segment boundary and calculate 1852 * the relocation to the kernel slot 1853 * 1854 * We always make a segment table entry here if we need to. This is mainly because of 1855 * copyin/out and if we don't, there will be multiple segment faults for 1856 * each system call. I have seen upwards of 30000 per second. 1857 * 1858 * We do check, however, to see if the slice is already mapped and if so, 1859 * we just exit. This is done for performance reasons. It was found that 1860 * there was a considerable boost in copyin/out performance if we did not 1861 * invalidate the segment at ReleaseUserAddressSpace time, so we dumped the 1862 * restriction that you had to bracket MapUserMemoryWindow. Further, there 1863 * is a yet further boost if you didn't need to map it each time. The theory 1864 * behind this is that many times copies are to or from the same segment and 1865 * done multiple times within the same system call. To take advantage of that, 1866 * we check umwSpace and umwRelo to see if we've already got it. 1867 * 1868 * We also need to half-invalidate the slice when we context switch or go 1869 * back to user state. A half-invalidate does not clear the actual mapping, 1870 * but it does force the MapUserMemoryWindow function to reload the segment 1871 * register/SLBE. If this is not done, we can end up some pretty severe 1872 * performance penalties. If we map a slice, and the cached space/relocation is 1873 * the same, we won't reload the segment registers. Howver, since we ran someone else, 1874 * our SR is cleared and we will take a fault. This is reasonable if we block 1875 * while copying (e.g., we took a page fault), but it is not reasonable when we 1876 * just start. For this reason, we half-invalidate to make sure that the SR is 1877 * explicitly reloaded. 1878 * 1879 * Note that we do not go to the trouble of making a pmap segment cache 1880 * entry for these guys because they are very short term -- 99.99% of the time 1881 * they will be unmapped before the next context switch. 1882 * 1883 */ 1884 1885addr64_t MapUserMemoryWindow( 1886 vm_map_t map, 1887 addr64_t va) { 1888 1889 addr64_t baddrs, reladd; 1890 thread_t thread; 1891 mapping_t *mp; 1892 1893 baddrs = va & 0xFFFFFFFFF0000000ULL; /* Isolate the segment */ 1894 thread = current_thread(); /* Remember our activation */ 1895 1896 reladd = baddrs - lowGlo.lgUMWvaddr; /* Get the relocation from user to kernel */ 1897 1898 if((thread->machine.umwSpace == map->pmap->space) && (thread->machine.umwRelo == reladd)) { /* Already mapped? */ 1899 return ((va & 0x0FFFFFFFULL) | lowGlo.lgUMWvaddr); /* Pass back the kernel address we are to use */ 1900 } 1901 1902 disable_preemption(); /* Don't move... */ 1903 1904 mp = (mapping_t *)&(getPerProc()->ppUMWmp); /* Make up for C */ 1905 thread->machine.umwRelo = reladd; /* Relocation from user to kernel */ 1906 mp->mpNestReloc = reladd; /* Relocation from user to kernel */ 1907 1908 thread->machine.umwSpace = map->pmap->space; /* Set the address space/pmap lookup ID */ 1909 mp->mpSpace = map->pmap->space; /* Set the address space/pmap lookup ID */ 1910 1911/* 1912 * Here we make an assumption that we are going to be using the base pmap's address space. 1913 * If we are wrong, and that would be very, very, very rare, the fault handler will fix us up. 1914 */ 1915 1916 hw_map_seg(map->pmap, lowGlo.lgUMWvaddr, baddrs); /* Make the entry for the first segment */ 1917 1918 enable_preemption(); /* Let's move */ 1919 return ((va & 0x0FFFFFFFULL) | lowGlo.lgUMWvaddr); /* Pass back the kernel address we are to use */ 1920} 1921 1922#if CONFIG_DTRACE 1923/* 1924 * Constrain DTrace copyin/copyout actions 1925 */ 1926extern kern_return_t dtrace_copyio_preflight(addr64_t); 1927extern kern_return_t dtrace_copyio_postflight(addr64_t); 1928 1929kern_return_t dtrace_copyio_preflight(__unused addr64_t va) 1930{ 1931 if (current_map() == kernel_map) 1932 return KERN_FAILURE; 1933 else 1934 return KERN_SUCCESS; 1935} 1936 1937kern_return_t dtrace_copyio_postflight(__unused addr64_t va) 1938{ 1939 thread_t thread = current_thread(); 1940 1941 thread->machine.umwSpace |= umwSwitchAway; 1942 return KERN_SUCCESS; 1943} 1944#endif /* CONFIG_DTRACE */ 1945 1946/* 1947 * kern_return_t pmap_boot_map(size) 1948 * 1949 * size = size of virtual address range to be mapped 1950 * 1951 * This function is used to assign a range of virtual addresses before VM in 1952 * initialized. It starts at VM_MAX_KERNEL_ADDRESS and works downward. 1953 * The variable vm_last_addr contains the current highest possible VM 1954 * assignable address. It is a panic to attempt to call this after VM has 1955 * started up. The only problem is, is that we may not have the serial or 1956 * framebuffer mapped, so we'll never know we died......... 1957 */ 1958 1959vm_offset_t pmap_boot_map(vm_size_t size) { 1960 1961 if(kernel_map != VM_MAP_NULL) { /* Has VM already started? */ 1962 panic("pmap_boot_map: VM started\n"); 1963 } 1964 1965 size = round_page(size); /* Make sure this is in pages */ 1966 vm_last_addr = vm_last_addr - size; /* Allocate the memory */ 1967 return (vm_last_addr + 1); /* Return the vaddr we just allocated */ 1968 1969} 1970 1971 1972/* 1973 * void pmap_init_sharedpage(void); 1974 * 1975 * Hack map for the 64-bit commpage 1976 */ 1977 1978void pmap_init_sharedpage(vm_offset_t cpg){ 1979 1980 addr64_t cva, cpoff; 1981 ppnum_t cpphys; 1982 1983 sharedPmap = pmap_create(0, FALSE); /* Get a pmap to hold the common segment */ 1984 if(!sharedPmap) { /* Check for errors */ 1985 panic("pmap_init_sharedpage: couldn't make sharedPmap\n"); 1986 } 1987 1988 for(cpoff = 0; cpoff < _COMM_PAGE_AREA_USED; cpoff += 4096) { /* Step along now */ 1989 1990 cpphys = pmap_find_phys(kernel_pmap, (addr64_t)cpg + cpoff); 1991 if(!cpphys) { 1992 panic("pmap_init_sharedpage: compage %016llX not mapped in kernel\n", cpg + cpoff); 1993 } 1994 1995 cva = mapping_make(sharedPmap, (addr64_t)((uint32_t)_COMM_PAGE_BASE_ADDRESS) + cpoff, 1996 cpphys, mmFlgPerm, 1, VM_PROT_READ | VM_PROT_EXECUTE); /* Map the page read/execute only */ 1997 if(cva) { /* Check for errors */ 1998 panic("pmap_init_sharedpage: couldn't map commpage page - cva = %016llX\n", cva); 1999 } 2000 2001 } 2002 2003 return; 2004} 2005 2006 2007/* 2008 * void pmap_map_sharedpage(pmap_t pmap); 2009 * 2010 * Maps the last segment in a 64-bit address space 2011 * 2012 * 2013 */ 2014 2015void pmap_map_sharedpage(task_t task, pmap_t pmap){ 2016 2017 kern_return_t ret; 2018 2019 if(task_has_64BitAddr(task) || _cpu_capabilities & k64Bit) { /* Should we map the 64-bit page -1? */ 2020 ret = pmap_nest(pmap, sharedPmap, 0xFFFFFFFFF0000000ULL, 0x00000000F0000000ULL, 2021 0x0000000010000000ULL); /* Nest the highest possible segment to map comm page */ 2022 if(ret != KERN_SUCCESS) { /* Did it work? */ 2023 panic("pmap_map_sharedpage: couldn't nest shared page - ret = %08X\n", ret); 2024 } 2025 } 2026 2027 return; 2028} 2029 2030 2031/* 2032 * void pmap_unmap_sharedpage(pmap_t pmap); 2033 * 2034 * Unmaps the last segment in a 64-bit address space 2035 * 2036 */ 2037 2038void pmap_unmap_sharedpage(pmap_t pmap){ 2039 2040 kern_return_t ret; 2041 mapping_t *mp; 2042 boolean_t inter; 2043 int gotnest; 2044 addr64_t nextva; 2045 2046 if(BootProcInfo.pf.Available & pf64Bit) { /* Are we on a 64-bit machine? */ 2047 2048 inter = ml_set_interrupts_enabled(FALSE); /* Disable interruptions for now */ 2049 mp = hw_find_map(pmap, 0xFFFFFFFFF0000000ULL, &nextva); /* Find the mapping for this address */ 2050 if((unsigned int)mp == mapRtBadLk) { /* Did we lock up ok? */ 2051 panic("pmap_unmap_sharedpage: mapping lock failure - rc = %p, pmap = %p\n", mp, pmap); /* Die... */ 2052 } 2053 2054 gotnest = 0; /* Assume nothing here */ 2055 if(mp) { 2056 gotnest = ((mp->mpFlags & mpType) == mpNest); 2057 /* Remember if we have a nest here */ 2058 mapping_drop_busy(mp); /* We have everything we need from the mapping */ 2059 } 2060 ml_set_interrupts_enabled(inter); /* Put interrupts back to what they were */ 2061 2062 if(!gotnest) return; /* Leave if there isn't any nesting here */ 2063 2064 ret = pmap_unnest(pmap, 0xFFFFFFFFF0000000ULL, 0x0000000010000000ULL); /* Unnest the max 64-bit page */ 2065 2066 if(ret != KERN_SUCCESS) { /* Did it work? */ 2067 panic("pmap_unmap_sharedpage: couldn't unnest shared page - ret = %08X\n", ret); 2068 } 2069 } 2070 2071 return; 2072} 2073 2074 2075/* temporary workaround */ 2076boolean_t 2077coredumpok( 2078 __unused vm_map_t map, 2079 __unused vm_offset_t va) 2080{ 2081 return TRUE; 2082} 2083 2084 2085/* 2086 * disable no-execute capability on 2087 * the specified pmap 2088 */ 2089void pmap_disable_NX(pmap_t pmap) { 2090 2091 pmap->pmapFlags |= pmapNXdisabled; 2092} 2093 2094