1/*
2 * Copyright (c) 2003-2008 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57#include <platforms.h>
58
59#include <mach/i386/vm_param.h>
60
61#include <string.h>
62#include <mach/vm_param.h>
63#include <mach/vm_prot.h>
64#include <mach/machine.h>
65#include <mach/time_value.h>
66#include <kern/spl.h>
67#include <kern/assert.h>
68#include <kern/debug.h>
69#include <kern/misc_protos.h>
70#include <kern/cpu_data.h>
71#include <kern/processor.h>
72#include <vm/vm_page.h>
73#include <vm/pmap.h>
74#include <vm/vm_kern.h>
75#include <i386/pmap.h>
76#include <i386/misc_protos.h>
77#include <i386/cpuid.h>
78#include <mach/thread_status.h>
79#include <pexpert/i386/efi.h>
80#include <i386/i386_lowmem.h>
81#ifdef __x86_64__
82#include <x86_64/lowglobals.h>
83#else
84#include <i386/lowglobals.h>
85#endif
86#include <i386/pal_routines.h>
87
88#include <mach-o/loader.h>
89#include <libkern/kernel_mach_header.h>
90
91
92vm_size_t	mem_size = 0;
93pmap_paddr_t	first_avail = 0;/* first after page tables */
94
95uint64_t	max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
96uint64_t        mem_actual;
97uint64_t	sane_size = 0;  /* Memory size for defaults calculations */
98
99/*
100 * KASLR parameters
101 */
102ppnum_t		vm_kernel_base_page;
103vm_offset_t	vm_kernel_base;
104vm_offset_t	vm_kernel_top;
105vm_offset_t	vm_kernel_stext;
106vm_offset_t	vm_kernel_etext;
107vm_offset_t	vm_kernel_slide;
108
109#define MAXLORESERVE	(32 * 1024 * 1024)
110
111ppnum_t		max_ppnum = 0;
112ppnum_t		lowest_lo = 0;
113ppnum_t		lowest_hi = 0;
114ppnum_t		highest_hi = 0;
115
116enum {PMAP_MAX_RESERVED_RANGES = 32};
117uint32_t pmap_reserved_pages_allocated = 0;
118uint32_t pmap_reserved_range_indices[PMAP_MAX_RESERVED_RANGES];
119uint32_t pmap_last_reserved_range_index = 0;
120uint32_t pmap_reserved_ranges = 0;
121
122extern unsigned int bsd_mbuf_cluster_reserve(boolean_t *);
123
124pmap_paddr_t     avail_start, avail_end;
125vm_offset_t	virtual_avail, virtual_end;
126static pmap_paddr_t	avail_remaining;
127vm_offset_t     static_memory_end = 0;
128
129vm_offset_t	sHIB, eHIB, stext, etext, sdata, edata, sconstdata, econstdata, end;
130
131/*
132 * _mh_execute_header is the mach_header for the currently executing kernel
133 */
134vm_offset_t segTEXTB; unsigned long segSizeTEXT;
135vm_offset_t segDATAB; unsigned long segSizeDATA;
136vm_offset_t segLINKB; unsigned long segSizeLINK;
137vm_offset_t segPRELINKB; unsigned long segSizePRELINK;
138vm_offset_t segHIBB; unsigned long segSizeHIB;
139vm_offset_t sectCONSTB; unsigned long sectSizeConst;
140
141boolean_t doconstro_override = FALSE;
142
143static kernel_segment_command_t *segTEXT, *segDATA;
144static kernel_section_t *cursectTEXT, *lastsectTEXT;
145static kernel_section_t *sectDCONST;
146
147extern uint64_t firmware_Conventional_bytes;
148extern uint64_t firmware_RuntimeServices_bytes;
149extern uint64_t firmware_ACPIReclaim_bytes;
150extern uint64_t firmware_ACPINVS_bytes;
151extern uint64_t firmware_PalCode_bytes;
152extern uint64_t firmware_Reserved_bytes;
153extern uint64_t firmware_Unusable_bytes;
154extern uint64_t firmware_other_bytes;
155uint64_t firmware_MMIO_bytes;
156
157/*
158 * Linker magic to establish the highest address in the kernel.
159 * This is replicated from libsa which marks last_kernel_symbol
160 * but that's not visible from here in osfmk.
161 */
162__asm__(".zerofill __LAST, __last, _kernel_top, 0");
163extern void 	*kernel_top;
164
165#if	DEBUG
166#define	PRINT_PMAP_MEMORY_TABLE
167#define DBG(x...)       kprintf(x)
168#else
169#define DBG(x...)
170#endif /* DEBUG */
171/*
172 * Basic VM initialization.
173 */
174void
175i386_vm_init(uint64_t	maxmem,
176	     boolean_t	IA32e,
177	     boot_args	*args)
178{
179	pmap_memory_region_t *pmptr;
180        pmap_memory_region_t *prev_pmptr;
181	EfiMemoryRange *mptr;
182        unsigned int mcount;
183        unsigned int msize;
184	ppnum_t fap;
185	unsigned int i;
186	unsigned int safeboot;
187	ppnum_t maxpg = 0;
188        uint32_t pmap_type;
189	uint32_t maxloreserve;
190	uint32_t maxdmaaddr;
191	uint32_t  mbuf_reserve = 0;
192	boolean_t mbuf_override = FALSE;
193	boolean_t coalescing_permitted;
194	vm_kernel_base_page = i386_btop(args->kaddr);
195#ifdef __x86_64__
196	vm_offset_t base_address;
197	vm_offset_t static_base_address;
198
199	/*
200	 * Establish the KASLR parameters.
201	 */
202	static_base_address = ml_static_ptovirt(KERNEL_BASE_OFFSET);
203	base_address        = ml_static_ptovirt(args->kaddr);
204	vm_kernel_slide     = base_address - static_base_address;
205	if (args->kslide) {
206		kprintf("KASLR slide: 0x%016lx dynamic\n", vm_kernel_slide);
207		if (vm_kernel_slide != ((vm_offset_t)args->kslide))
208			panic("Kernel base inconsistent with slide - rebased?");
209	} else {
210		/* No slide relative to on-disk symbols */
211		kprintf("KASLR slide: 0x%016lx static and ignored\n",
212			vm_kernel_slide);
213		vm_kernel_slide = 0;
214	}
215
216	/*
217	 * Zero out local relocations to avoid confusing kxld.
218	 * TODO: might be better to move this code to OSKext::initialize
219	 */
220	if (_mh_execute_header.flags & MH_PIE) {
221		struct load_command *loadcmd;
222		uint32_t cmd;
223
224		loadcmd = (struct load_command *)((uintptr_t)&_mh_execute_header +
225						  sizeof (_mh_execute_header));
226
227		for (cmd = 0; cmd < _mh_execute_header.ncmds; cmd++) {
228			if (loadcmd->cmd == LC_DYSYMTAB) {
229				struct dysymtab_command *dysymtab;
230
231				dysymtab = (struct dysymtab_command *)loadcmd;
232				dysymtab->nlocrel = 0;
233				dysymtab->locreloff = 0;
234				kprintf("Hiding local relocations\n");
235				break;
236			}
237			loadcmd = (struct load_command *)((uintptr_t)loadcmd + loadcmd->cmdsize);
238		}
239	}
240
241#endif // __x86_64__
242
243	/*
244	 * Now retrieve addresses for end, edata, and etext
245	 * from MACH-O headers.
246	 */
247	segTEXTB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
248					"__TEXT", &segSizeTEXT);
249	segDATAB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
250					"__DATA", &segSizeDATA);
251	segLINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
252					"__LINKEDIT", &segSizeLINK);
253	segHIBB  = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
254					"__HIB", &segSizeHIB);
255	segPRELINKB = (vm_offset_t) getsegdatafromheader(&_mh_execute_header,
256					"__PRELINK_TEXT", &segSizePRELINK);
257	segTEXT = getsegbynamefromheader(&_mh_execute_header,
258					"__TEXT");
259	segDATA = getsegbynamefromheader(&_mh_execute_header,
260					"__DATA");
261	sectDCONST = getsectbynamefromheader(&_mh_execute_header,
262					"__DATA", "__const");
263	cursectTEXT = lastsectTEXT = firstsect(segTEXT);
264	/* Discover the last TEXT section within the TEXT segment */
265	while ((cursectTEXT = nextsect(segTEXT, cursectTEXT)) != NULL) {
266		lastsectTEXT = cursectTEXT;
267	}
268
269	sHIB  = segHIBB;
270	eHIB  = segHIBB + segSizeHIB;
271	/* Zero-padded from ehib to stext if text is 2M-aligned */
272	stext = segTEXTB;
273#ifdef __x86_64__
274	lowGlo.lgStext = stext;
275#endif
276	etext = (vm_offset_t) round_page_64(lastsectTEXT->addr + lastsectTEXT->size);
277	/* Zero-padded from etext to sdata if text is 2M-aligned */
278	sdata = segDATAB;
279	edata = segDATAB + segSizeDATA;
280
281	sectCONSTB = (vm_offset_t) sectDCONST->addr;
282	sectSizeConst = sectDCONST->size;
283	sconstdata = sectCONSTB;
284	econstdata = sectCONSTB + sectSizeConst;
285
286	if (sectSizeConst & PAGE_MASK) {
287		kernel_section_t *ns = nextsect(segDATA, sectDCONST);
288		if (ns && !(ns->addr & PAGE_MASK))
289			doconstro_override = TRUE;
290	} else
291		doconstro_override = TRUE;
292
293	DBG("segTEXTB    = %p\n", (void *) segTEXTB);
294	DBG("segDATAB    = %p\n", (void *) segDATAB);
295	DBG("segLINKB    = %p\n", (void *) segLINKB);
296	DBG("segHIBB     = %p\n", (void *) segHIBB);
297	DBG("segPRELINKB = %p\n", (void *) segPRELINKB);
298	DBG("sHIB        = %p\n", (void *) sHIB);
299	DBG("eHIB        = %p\n", (void *) eHIB);
300	DBG("stext       = %p\n", (void *) stext);
301	DBG("etext       = %p\n", (void *) etext);
302	DBG("sdata       = %p\n", (void *) sdata);
303	DBG("edata       = %p\n", (void *) edata);
304	DBG("sconstdata  = %p\n", (void *) sconstdata);
305	DBG("econstdata  = %p\n", (void *) econstdata);
306	DBG("kernel_top  = %p\n", (void *) &kernel_top);
307
308	vm_kernel_base  = sHIB;
309	vm_kernel_top   = (vm_offset_t) &kernel_top;
310	vm_kernel_stext = stext;
311	vm_kernel_etext = etext;
312
313	vm_set_page_size();
314
315	/*
316	 * Compute the memory size.
317	 */
318
319	if ((1 == vm_himemory_mode) || PE_parse_boot_argn("-x", &safeboot, sizeof (safeboot))) {
320	        maxpg = 1 << (32 - I386_PGSHIFT);
321	}
322	avail_remaining = 0;
323	avail_end = 0;
324	pmptr = pmap_memory_regions;
325        prev_pmptr = 0;
326	pmap_memory_region_count = pmap_memory_region_current = 0;
327	fap = (ppnum_t) i386_btop(first_avail);
328
329	mptr = (EfiMemoryRange *)ml_static_ptovirt((vm_offset_t)args->MemoryMap);
330        if (args->MemoryMapDescriptorSize == 0)
331	        panic("Invalid memory map descriptor size");
332        msize = args->MemoryMapDescriptorSize;
333        mcount = args->MemoryMapSize / msize;
334
335#define FOURGIG 0x0000000100000000ULL
336#define ONEGIG  0x0000000040000000ULL
337
338	for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
339	        ppnum_t base, top;
340		uint64_t region_bytes = 0;
341
342		if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
343		        kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
344			break;
345		}
346		base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
347		top = (ppnum_t) (((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1);
348
349#if	MR_RSV_TEST
350		static uint32_t nmr = 0;
351		if ((base > 0x20000) && (nmr++ < 4))
352			mptr->Attribute |= EFI_MEMORY_KERN_RESERVED;
353#endif
354		region_bytes = (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
355		pmap_type = mptr->Type;
356
357		switch (mptr->Type) {
358		case kEfiLoaderCode:
359		case kEfiLoaderData:
360		case kEfiBootServicesCode:
361		case kEfiBootServicesData:
362		case kEfiConventionalMemory:
363		        /*
364			 * Consolidate usable memory types into one.
365			 */
366		        pmap_type = kEfiConventionalMemory;
367		        sane_size += region_bytes;
368			firmware_Conventional_bytes += region_bytes;
369			break;
370			/*
371			 * sane_size should reflect the total amount of physical
372			 * RAM in the system, not just the amount that is
373			 * available for the OS to use.
374			 * FIXME:Consider deriving this value from SMBIOS tables
375			 * rather than reverse engineering the memory map.
376			 * Alternatively, see
377			 * <rdar://problem/4642773> Memory map should
378			 * describe all memory
379			 * Firmware on some systems guarantees that the memory
380			 * map is complete via the "RomReservedMemoryTracked"
381			 * feature field--consult that where possible to
382			 * avoid the "round up to 128M" workaround below.
383			 */
384
385		case kEfiRuntimeServicesCode:
386		case kEfiRuntimeServicesData:
387			firmware_RuntimeServices_bytes += region_bytes;
388			sane_size += region_bytes;
389			break;
390		case kEfiACPIReclaimMemory:
391			firmware_ACPIReclaim_bytes += region_bytes;
392			sane_size += region_bytes;
393			break;
394		case kEfiACPIMemoryNVS:
395			firmware_ACPINVS_bytes += region_bytes;
396			sane_size += region_bytes;
397			break;
398		case kEfiPalCode:
399			firmware_PalCode_bytes += region_bytes;
400		        sane_size += region_bytes;
401			break;
402
403		case kEfiReservedMemoryType:
404			firmware_Reserved_bytes += region_bytes;
405			break;
406		case kEfiUnusableMemory:
407			firmware_Unusable_bytes += region_bytes;
408			break;
409		case kEfiMemoryMappedIO:
410		case kEfiMemoryMappedIOPortSpace:
411			firmware_MMIO_bytes += region_bytes;
412			break;
413		default:
414			firmware_other_bytes += region_bytes;
415			break;
416		}
417
418		DBG("EFI region %d: type %u/%d, base 0x%x, top 0x%x %s\n",
419		    i, mptr->Type, pmap_type, base, top,
420		    (mptr->Attribute&EFI_MEMORY_KERN_RESERVED)? "RESERVED" :
421		    (mptr->Attribute&EFI_MEMORY_RUNTIME)? "RUNTIME" : "");
422
423		if (maxpg) {
424		        if (base >= maxpg)
425				break;
426		        top = (top > maxpg) ? maxpg : top;
427		}
428
429		/*
430		 * handle each region
431		 */
432		if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
433		    pmap_type != kEfiConventionalMemory) {
434		        prev_pmptr = 0;
435			continue;
436		} else {
437		        /*
438			 * Usable memory region
439			 */
440		        if (top < I386_LOWMEM_RESERVED ||
441			    !pal_is_usable_memory(base, top)) {
442			        prev_pmptr = 0;
443				continue;
444			}
445			/*
446			 * A range may be marked with with the
447			 * EFI_MEMORY_KERN_RESERVED attribute
448			 * on some systems, to indicate that the range
449			 * must not be made available to devices.
450			 */
451
452			if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED) {
453				if (++pmap_reserved_ranges > PMAP_MAX_RESERVED_RANGES) {
454					panic("Too many reserved ranges %u\n", pmap_reserved_ranges);
455				}
456			}
457
458			if (top < fap) {
459			        /*
460				 * entire range below first_avail
461			         * salvage some low memory pages
462				 * we use some very low memory at startup
463				 * mark as already allocated here
464				 */
465			        if (base >= I386_LOWMEM_RESERVED)
466				        pmptr->base = base;
467				else
468				        pmptr->base = I386_LOWMEM_RESERVED;
469
470				pmptr->end = top;
471
472
473				if ((mptr->Attribute & EFI_MEMORY_KERN_RESERVED) &&
474				    (top < vm_kernel_base_page)) {
475					pmptr->alloc = pmptr->base;
476					pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
477				}
478				else {
479					/*
480					 * mark as already mapped
481					 */
482					pmptr->alloc = top;
483				}
484				pmptr->type = pmap_type;
485				pmptr->attribute = mptr->Attribute;
486			}
487			else if ( (base < fap) && (top > fap) ) {
488			        /*
489				 * spans first_avail
490				 * put mem below first avail in table but
491				 * mark already allocated
492				 */
493			        pmptr->base = base;
494				pmptr->alloc = pmptr->end = (fap - 1);
495				pmptr->type = pmap_type;
496				pmptr->attribute = mptr->Attribute;
497				/*
498				 * we bump these here inline so the accounting
499				 * below works correctly
500				 */
501				pmptr++;
502				pmap_memory_region_count++;
503
504				pmptr->alloc = pmptr->base = fap;
505				pmptr->type = pmap_type;
506				pmptr->attribute = mptr->Attribute;
507				pmptr->end = top;
508
509				if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
510					pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
511			} else {
512			        /*
513				 * entire range useable
514				 */
515			        pmptr->alloc = pmptr->base = base;
516				pmptr->type = pmap_type;
517				pmptr->attribute = mptr->Attribute;
518				pmptr->end = top;
519				if (mptr->Attribute & EFI_MEMORY_KERN_RESERVED)
520					pmap_reserved_range_indices[pmap_last_reserved_range_index++] = pmap_memory_region_count;
521			}
522
523			if (i386_ptob(pmptr->end) > avail_end )
524			        avail_end = i386_ptob(pmptr->end);
525
526			avail_remaining += (pmptr->end - pmptr->base);
527			coalescing_permitted = (prev_pmptr && (pmptr->attribute == prev_pmptr->attribute) && ((pmptr->attribute & EFI_MEMORY_KERN_RESERVED) == 0));
528			/*
529			 * Consolidate contiguous memory regions, if possible
530			 */
531			if (prev_pmptr &&
532			    (pmptr->type == prev_pmptr->type) &&
533			    (coalescing_permitted) &&
534			    (pmptr->base == pmptr->alloc) &&
535			    (pmptr->base == (prev_pmptr->end + 1)))
536			{
537				if (prev_pmptr->end == prev_pmptr->alloc)
538					prev_pmptr->alloc = pmptr->base;
539				prev_pmptr->end = pmptr->end;
540			} else {
541			        pmap_memory_region_count++;
542				prev_pmptr = pmptr;
543				pmptr++;
544			}
545		}
546	}
547
548#ifdef PRINT_PMAP_MEMORY_TABLE
549	{
550        unsigned int j;
551        pmap_memory_region_t *p = pmap_memory_regions;
552        addr64_t region_start, region_end;
553        addr64_t efi_start, efi_end;
554        for (j=0;j<pmap_memory_region_count;j++, p++) {
555            kprintf("pmap region %d type %d base 0x%llx alloc 0x%llx top 0x%llx\n",
556		    j, p->type,
557                    (addr64_t) p->base  << I386_PGSHIFT,
558		    (addr64_t) p->alloc << I386_PGSHIFT,
559		    (addr64_t) p->end   << I386_PGSHIFT);
560            region_start = (addr64_t) p->base << I386_PGSHIFT;
561            region_end = ((addr64_t) p->end << I386_PGSHIFT) - 1;
562	    mptr = (EfiMemoryRange *) ml_static_ptovirt((vm_offset_t)args->MemoryMap);
563            for (i=0; i<mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
564                if (mptr->Type != kEfiLoaderCode &&
565                    mptr->Type != kEfiLoaderData &&
566                    mptr->Type != kEfiBootServicesCode &&
567                    mptr->Type != kEfiBootServicesData &&
568                    mptr->Type != kEfiConventionalMemory) {
569                efi_start = (addr64_t)mptr->PhysicalStart;
570                efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
571                if ((efi_start >= region_start && efi_start <= region_end) ||
572                    (efi_end >= region_start && efi_end <= region_end)) {
573                    kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
574                }
575              }
576            }
577          }
578	}
579#endif
580
581	avail_start = first_avail;
582	mem_actual = sane_size;
583
584	/*
585	 * For user visible memory size, round up to 128 Mb - accounting for the various stolen memory
586	 * not reported by EFI.
587	 */
588
589	sane_size = (sane_size + 128 * MB - 1) & ~((uint64_t)(128 * MB - 1));
590
591	/*
592	 * We cap at KERNEL_MAXMEM bytes (currently 32GB for K32, 96GB for K64).
593	 * Unless overriden by the maxmem= boot-arg
594	 * -- which is a non-zero maxmem argument to this function.
595	 */
596	if (maxmem == 0 && sane_size > KERNEL_MAXMEM) {
597		maxmem = KERNEL_MAXMEM;
598		printf("Physical memory %lld bytes capped at %dGB\n",
599			sane_size, (uint32_t) (KERNEL_MAXMEM/GB));
600	}
601
602	/*
603	 * if user set maxmem, reduce memory sizes
604	 */
605	if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
606		ppnum_t discarded_pages  = (ppnum_t)((sane_size - maxmem) >> I386_PGSHIFT);
607		ppnum_t	highest_pn = 0;
608		ppnum_t	cur_end  = 0;
609		uint64_t	pages_to_use;
610		unsigned	cur_region = 0;
611
612		sane_size = maxmem;
613
614		if (avail_remaining > discarded_pages)
615			avail_remaining -= discarded_pages;
616		else
617			avail_remaining = 0;
618
619		pages_to_use = avail_remaining;
620
621		while (cur_region < pmap_memory_region_count && pages_to_use) {
622		        for (cur_end = pmap_memory_regions[cur_region].base;
623			     cur_end < pmap_memory_regions[cur_region].end && pages_to_use;
624			     cur_end++) {
625			        if (cur_end > highest_pn)
626				        highest_pn = cur_end;
627				pages_to_use--;
628			}
629			if (pages_to_use == 0)
630			        pmap_memory_regions[cur_region].end = cur_end;
631
632			cur_region++;
633		}
634		pmap_memory_region_count = cur_region;
635
636		avail_end = i386_ptob(highest_pn + 1);
637	}
638
639	/*
640	 * mem_size is only a 32 bit container... follow the PPC route
641	 * and pin it to a 2 Gbyte maximum
642	 */
643	if (sane_size > (FOURGIG >> 1))
644	        mem_size = (vm_size_t)(FOURGIG >> 1);
645	else
646	        mem_size = (vm_size_t)sane_size;
647	max_mem = sane_size;
648
649	kprintf("Physical memory %llu MB\n", sane_size/MB);
650
651	max_valid_low_ppnum = (2 * GB) / PAGE_SIZE;
652
653	if (!PE_parse_boot_argn("max_valid_dma_addr", &maxdmaaddr, sizeof (maxdmaaddr))) {
654	        max_valid_dma_address = (uint64_t)4 * (uint64_t)GB;
655	} else {
656	        max_valid_dma_address = ((uint64_t) maxdmaaddr) * MB;
657
658		if ((max_valid_dma_address / PAGE_SIZE) < max_valid_low_ppnum)
659			max_valid_low_ppnum = (ppnum_t)(max_valid_dma_address / PAGE_SIZE);
660	}
661	if (avail_end >= max_valid_dma_address) {
662
663		if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof (maxloreserve))) {
664
665			if (sane_size >= (ONEGIG * 15))
666				maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 4;
667			else if (sane_size >= (ONEGIG * 7))
668				maxloreserve = (MAXLORESERVE / PAGE_SIZE) * 2;
669			else
670				maxloreserve = MAXLORESERVE / PAGE_SIZE;
671
672			mbuf_reserve = bsd_mbuf_cluster_reserve(&mbuf_override) / PAGE_SIZE;
673		} else
674			maxloreserve = (maxloreserve * (1024 * 1024)) / PAGE_SIZE;
675
676		if (maxloreserve) {
677		        vm_lopage_free_limit = maxloreserve;
678
679			if (mbuf_override == TRUE) {
680				vm_lopage_free_limit += mbuf_reserve;
681				vm_lopage_lowater = 0;
682			} else
683				vm_lopage_lowater = vm_lopage_free_limit / 16;
684
685			vm_lopage_refill = TRUE;
686			vm_lopage_needed = TRUE;
687		}
688	}
689
690	/*
691	 *	Initialize kernel physical map.
692	 *	Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
693	 */
694	kprintf("avail_remaining = 0x%lx\n", (unsigned long)avail_remaining);
695	pmap_bootstrap(0, IA32e);
696}
697
698
699unsigned int
700pmap_free_pages(void)
701{
702	return (unsigned int)avail_remaining;
703}
704
705
706boolean_t pmap_next_page_reserved(ppnum_t *);
707
708/*
709 * Pick a page from a "kernel private" reserved range; works around
710 * errata on some hardware.
711 */
712boolean_t
713pmap_next_page_reserved(ppnum_t *pn) {
714	if (pmap_reserved_ranges) {
715		uint32_t n;
716		pmap_memory_region_t *region;
717		for (n = 0; n < pmap_last_reserved_range_index; n++) {
718			uint32_t reserved_index = pmap_reserved_range_indices[n];
719			region = &pmap_memory_regions[reserved_index];
720			if (region->alloc < region->end) {
721				*pn = region->alloc++;
722				avail_remaining--;
723
724				if (*pn > max_ppnum)
725					max_ppnum = *pn;
726
727				if (lowest_lo == 0 || *pn < lowest_lo)
728					lowest_lo = *pn;
729
730				pmap_reserved_pages_allocated++;
731#if DEBUG
732				if (region->alloc == region->end) {
733					kprintf("Exhausted reserved range index: %u, base: 0x%x end: 0x%x, type: 0x%x, attribute: 0x%llx\n", reserved_index, region->base, region->end, region->type, region->attribute);
734				}
735#endif
736				return TRUE;
737			}
738		}
739	}
740	return FALSE;
741}
742
743
744boolean_t
745pmap_next_page_hi(
746	          ppnum_t *pn)
747{
748	pmap_memory_region_t *region;
749	int	n;
750
751	if (pmap_next_page_reserved(pn))
752		return TRUE;
753
754	if (avail_remaining) {
755		for (n = pmap_memory_region_count - 1; n >= 0; n--) {
756			region = &pmap_memory_regions[n];
757
758			if (region->alloc != region->end) {
759				*pn = region->alloc++;
760				avail_remaining--;
761
762				if (*pn > max_ppnum)
763					max_ppnum = *pn;
764
765                                if (lowest_lo == 0 || *pn < lowest_lo)
766                                        lowest_lo = *pn;
767
768                                if (lowest_hi == 0 || *pn < lowest_hi)
769                                        lowest_hi = *pn;
770
771                                if (*pn > highest_hi)
772                                        highest_hi = *pn;
773
774				return TRUE;
775			}
776		}
777	}
778	return FALSE;
779}
780
781
782boolean_t
783pmap_next_page(
784	       ppnum_t *pn)
785{
786	if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
787		if (pmap_memory_regions[pmap_memory_region_current].alloc ==
788				pmap_memory_regions[pmap_memory_region_current].end) {
789			pmap_memory_region_current++;
790			continue;
791		}
792		*pn = pmap_memory_regions[pmap_memory_region_current].alloc++;
793		avail_remaining--;
794
795		if (*pn > max_ppnum)
796			max_ppnum = *pn;
797
798		if (lowest_lo == 0 || *pn < lowest_lo)
799			lowest_lo = *pn;
800
801		return TRUE;
802	}
803	return FALSE;
804}
805
806
807boolean_t
808pmap_valid_page(
809	ppnum_t pn)
810{
811        unsigned int i;
812	pmap_memory_region_t *pmptr = pmap_memory_regions;
813
814	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
815	        if ( (pn >= pmptr->base) && (pn <= pmptr->end) )
816	                return TRUE;
817	}
818	return FALSE;
819}
820
821