1/*
2 * Copyright (c) 2003-2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989, 1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57#include <platforms.h>
58#include <mach_kdb.h>
59
60#include <mach/i386/vm_param.h>
61
62#include <string.h>
63#include <mach/vm_param.h>
64#include <mach/vm_prot.h>
65#include <mach/machine.h>
66#include <mach/time_value.h>
67#include <kern/spl.h>
68#include <kern/assert.h>
69#include <kern/debug.h>
70#include <kern/misc_protos.h>
71#include <kern/cpu_data.h>
72#include <kern/processor.h>
73#include <vm/vm_page.h>
74#include <vm/pmap.h>
75#include <vm/vm_kern.h>
76#include <i386/pmap.h>
77#include <i386/ipl.h>
78#include <i386/misc_protos.h>
79#include <i386/mp_slave_boot.h>
80#include <i386/cpuid.h>
81#include <mach/thread_status.h>
82#include <pexpert/i386/efi.h>
83#include "i386_lowmem.h"
84
85vm_size_t	mem_size = 0;
86vm_offset_t	first_avail = 0;/* first after page tables */
87
88uint64_t	max_mem;        /* Size of physical memory (bytes), adjusted by maxmem */
89uint64_t        mem_actual;
90uint64_t	sane_size = 0;  /* Memory size to use for defaults calculations */
91
92#define MAXBOUNCEPOOL	(128 * 1024 * 1024)
93#define MAXLORESERVE	( 32 * 1024 * 1024)
94
95extern int bsd_mbuf_cluster_reserve(void);
96
97
98uint32_t	bounce_pool_base = 0;
99uint32_t	bounce_pool_size = 0;
100
101static void	reserve_bouncepool(uint32_t);
102
103
104pmap_paddr_t     avail_start, avail_end;
105vm_offset_t	virtual_avail, virtual_end;
106static pmap_paddr_t	avail_remaining;
107vm_offset_t     static_memory_end = 0;
108
109#include	<mach-o/loader.h>
110vm_offset_t	edata, etext, end;
111
112/*
113 * _mh_execute_header is the mach_header for the currently executing
114 * 32 bit kernel
115 */
116extern struct mach_header _mh_execute_header;
117void *sectTEXTB; int sectSizeTEXT;
118void *sectDATAB; int sectSizeDATA;
119void *sectOBJCB; int sectSizeOBJC;
120void *sectLINKB; int sectSizeLINK;
121void *sectPRELINKB; int sectSizePRELINK;
122void *sectHIBB; int sectSizeHIB;
123
124extern void *getsegdatafromheader(struct mach_header *, const char *, int *);
125extern struct segment_command *getsegbyname(const char *);
126extern struct section *firstsect(struct segment_command *);
127extern struct section *nextsect(struct segment_command *, struct section *);
128
129
130void
131i386_macho_zerofill(void)
132{
133	struct segment_command	*sgp;
134	struct section		*sp;
135
136	sgp = getsegbyname("__DATA");
137	if (sgp) {
138		sp = firstsect(sgp);
139		if (sp) {
140			do {
141				if ((sp->flags & S_ZEROFILL))
142					bzero((char *) sp->addr, sp->size);
143			} while ((sp = nextsect(sgp, sp)));
144		}
145	}
146
147	return;
148}
149
150/*
151 * Basic VM initialization.
152 */
153void
154i386_vm_init(uint64_t	maxmem,
155	     boolean_t	IA32e,
156	     boot_args	*args)
157{
158	pmap_memory_region_t *pmptr;
159        pmap_memory_region_t *prev_pmptr;
160	EfiMemoryRange *mptr;
161        unsigned int mcount;
162        unsigned int msize;
163	ppnum_t fap;
164	unsigned int i;
165	unsigned int safeboot;
166	ppnum_t maxpg = 0;
167        uint32_t pmap_type;
168	uint32_t maxbouncepoolsize;
169	uint32_t maxloreserve;
170	uint32_t maxdmaaddr;
171
172	/*
173	 * Now retrieve addresses for end, edata, and etext
174	 * from MACH-O headers.
175	 */
176
177	sectTEXTB = (void *) getsegdatafromheader(
178		&_mh_execute_header, "__TEXT", &sectSizeTEXT);
179	sectDATAB = (void *) getsegdatafromheader(
180		&_mh_execute_header, "__DATA", &sectSizeDATA);
181	sectOBJCB = (void *) getsegdatafromheader(
182		&_mh_execute_header, "__OBJC", &sectSizeOBJC);
183	sectLINKB = (void *) getsegdatafromheader(
184		&_mh_execute_header, "__LINKEDIT", &sectSizeLINK);
185	sectHIBB = (void *)getsegdatafromheader(
186		&_mh_execute_header, "__HIB", &sectSizeHIB);
187	sectPRELINKB = (void *) getsegdatafromheader(
188		&_mh_execute_header, "__PRELINK", &sectSizePRELINK);
189
190	etext = (vm_offset_t) sectTEXTB + sectSizeTEXT;
191	edata = (vm_offset_t) sectDATAB + sectSizeDATA;
192
193	vm_set_page_size();
194
195	/*
196	 * Compute the memory size.
197	 */
198
199	if ((1 == vm_himemory_mode) || PE_parse_boot_argn("-x", &safeboot, sizeof (safeboot))) {
200	        maxpg = 1 << (32 - I386_PGSHIFT);
201	}
202	avail_remaining = 0;
203	avail_end = 0;
204	pmptr = pmap_memory_regions;
205        prev_pmptr = 0;
206	pmap_memory_region_count = pmap_memory_region_current = 0;
207	fap = (ppnum_t) i386_btop(first_avail);
208
209	mptr = (EfiMemoryRange *)args->MemoryMap;
210        if (args->MemoryMapDescriptorSize == 0)
211	        panic("Invalid memory map descriptor size");
212        msize = args->MemoryMapDescriptorSize;
213        mcount = args->MemoryMapSize / msize;
214
215#define FOURGIG 0x0000000100000000ULL
216
217	for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
218	        ppnum_t base, top;
219
220		if (pmap_memory_region_count >= PMAP_MEMORY_REGIONS_SIZE) {
221		        kprintf("WARNING: truncating memory region count at %d\n", pmap_memory_region_count);
222			break;
223		}
224		base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
225		top = (ppnum_t) ((mptr->PhysicalStart) >> I386_PGSHIFT) + mptr->NumberOfPages - 1;
226
227		switch (mptr->Type) {
228		case kEfiLoaderCode:
229		case kEfiLoaderData:
230		case kEfiBootServicesCode:
231		case kEfiBootServicesData:
232		case kEfiConventionalMemory:
233		        /*
234			 * Consolidate usable memory types into one.
235			 */
236		        pmap_type = kEfiConventionalMemory;
237		        sane_size += (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
238			break;
239
240		case kEfiRuntimeServicesCode:
241		case kEfiRuntimeServicesData:
242		case kEfiACPIReclaimMemory:
243		case kEfiACPIMemoryNVS:
244		case kEfiPalCode:
245			/*
246			 * sane_size should reflect the total amount of physical ram
247			 * in the system, not just the amount that is available for
248			 * the OS to use
249			 */
250		        sane_size += (uint64_t)(mptr->NumberOfPages << I386_PGSHIFT);
251			/* fall thru */
252
253		case kEfiUnusableMemory:
254		case kEfiMemoryMappedIO:
255		case kEfiMemoryMappedIOPortSpace:
256		case kEfiReservedMemoryType:
257		default:
258		        pmap_type = mptr->Type;
259		}
260
261		kprintf("EFI region: type = %u/%d,  base = 0x%x,  top = 0x%x\n", mptr->Type, pmap_type, base, top);
262
263		if (maxpg) {
264		        if (base >= maxpg)
265				break;
266		        top = (top > maxpg) ? maxpg : top;
267		}
268
269		/*
270		 * handle each region
271		 */
272		if ((mptr->Attribute & EFI_MEMORY_RUNTIME) == EFI_MEMORY_RUNTIME ||
273		    pmap_type != kEfiConventionalMemory) {
274		        prev_pmptr = 0;
275			continue;
276		} else {
277		        /*
278			 * Usable memory region
279			 */
280		        if (top < I386_LOWMEM_RESERVED) {
281			        prev_pmptr = 0;
282				continue;
283			}
284			if (top < fap) {
285			        /*
286				 * entire range below first_avail
287			         * salvage some low memory pages
288				 * we use some very low memory at startup
289				 * mark as already allocated here
290				 */
291			        if (base >= I386_LOWMEM_RESERVED)
292				        pmptr->base = base;
293				else
294				        pmptr->base = I386_LOWMEM_RESERVED;
295				/*
296				 * mark as already mapped
297				 */
298				pmptr->alloc = pmptr->end = top;
299				pmptr->type = pmap_type;
300			}
301			else if ( (base < fap) && (top > fap) ) {
302			        /*
303				 * spans first_avail
304				 * put mem below first avail in table but
305				 * mark already allocated
306				 */
307			        pmptr->base = base;
308				pmptr->alloc = pmptr->end = (fap - 1);
309				pmptr->type = pmap_type;
310				/*
311				 * we bump these here inline so the accounting
312				 * below works correctly
313				 */
314				pmptr++;
315				pmap_memory_region_count++;
316				pmptr->alloc = pmptr->base = fap;
317				pmptr->type = pmap_type;
318				pmptr->end = top;
319			}
320			else {
321			        /*
322				 * entire range useable
323				 */
324			        pmptr->alloc = pmptr->base = base;
325				pmptr->type = pmap_type;
326				pmptr->end = top;
327			}
328
329			if (i386_ptob(pmptr->end) > avail_end )
330			        avail_end = i386_ptob(pmptr->end);
331
332			avail_remaining += (pmptr->end - pmptr->base);
333
334			/*
335			 * Consolidate contiguous memory regions, if possible
336			 */
337			if (prev_pmptr &&
338			    pmptr->type == prev_pmptr->type &&
339			    pmptr->base == pmptr->alloc &&
340			    pmptr->base == (prev_pmptr->end + 1)) {
341			        prev_pmptr->end = pmptr->end;
342			} else {
343			        pmap_memory_region_count++;
344				prev_pmptr = pmptr;
345				pmptr++;
346			}
347		}
348	}
349
350
351#ifdef PRINT_PMAP_MEMORY_TABLE
352	{
353        unsigned int j;
354        pmap_memory_region_t *p = pmap_memory_regions;
355        vm_offset_t region_start, region_end;
356        vm_offset_t efi_start, efi_end;
357        for (j=0;j<pmap_memory_region_count;j++, p++) {
358            kprintf("type %d base 0x%x alloc 0x%x top 0x%x\n", p->type,
359                    p->base << I386_PGSHIFT, p->alloc << I386_PGSHIFT, p->end << I386_PGSHIFT);
360            region_start = p->base << I386_PGSHIFT;
361            region_end = (p->end << I386_PGSHIFT) - 1;
362            mptr = args->MemoryMap;
363            for (i=0; i<mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
364                if (mptr->Type != kEfiLoaderCode &&
365                    mptr->Type != kEfiLoaderData &&
366                    mptr->Type != kEfiBootServicesCode &&
367                    mptr->Type != kEfiBootServicesData &&
368                    mptr->Type != kEfiConventionalMemory) {
369                efi_start = (vm_offset_t)mptr->PhysicalStart;
370                efi_end = efi_start + ((vm_offset_t)mptr->NumberOfPages << I386_PGSHIFT) - 1;
371                if ((efi_start >= region_start && efi_start <= region_end) ||
372                    (efi_end >= region_start && efi_end <= region_end)) {
373                    kprintf(" *** Overlapping region with EFI runtime region %d\n", i);
374                }
375                }
376
377            }
378        }
379	}
380#endif
381
382	avail_start = first_avail;
383	mem_actual = sane_size;
384
385#define MEG		(1024*1024ULL)
386#define GIG		(1024*MEG)
387
388	/*
389	 * For user visible memory size, round up to 128 Mb - accounting for the various stolen memory
390	 * not reported by EFI.
391	 */
392
393	sane_size = (sane_size + 128 * MEG - 1) & ~((uint64_t)(128 * MEG - 1));
394
395#if defined(__i386__)
396#define K32_MAXMEM	(32*GIG)
397	/*
398	 * For K32 we cap at K32_MAXMEM GB (currently 32GB).
399	 * Unless overriden by the maxmem= boot-arg
400	 * -- which is a non-zero maxmem argument to this function.
401	 */
402	if (maxmem == 0 && sane_size > K32_MAXMEM) {
403		maxmem = K32_MAXMEM;
404		printf("Physical memory %lld bytes capped at %dGB for 32-bit kernel\n",
405			sane_size, (uint32_t) (K32_MAXMEM/GIG));
406	}
407#endif
408	/*
409	 * if user set maxmem, reduce memory sizes
410	 */
411	if ( (maxmem > (uint64_t)first_avail) && (maxmem < sane_size)) {
412		ppnum_t discarded_pages  = (sane_size - maxmem) >> I386_PGSHIFT;
413		ppnum_t	highest_pn = 0;
414		ppnum_t	cur_alloc  = 0;
415		uint64_t	pages_to_use;
416		unsigned	cur_region = 0;
417
418		sane_size = maxmem;
419
420		if (avail_remaining > discarded_pages)
421			avail_remaining -= discarded_pages;
422		else
423			avail_remaining = 0;
424
425		pages_to_use = avail_remaining;
426
427		while (cur_region < pmap_memory_region_count && pages_to_use) {
428		        for (cur_alloc = pmap_memory_regions[cur_region].alloc;
429			     cur_alloc < pmap_memory_regions[cur_region].end && pages_to_use;
430			     cur_alloc++) {
431			        if (cur_alloc > highest_pn)
432				        highest_pn = cur_alloc;
433				pages_to_use--;
434			}
435			if (pages_to_use == 0)
436			        pmap_memory_regions[cur_region].end = cur_alloc;
437
438			cur_region++;
439		}
440		pmap_memory_region_count = cur_region;
441
442		avail_end = i386_ptob(highest_pn + 1);
443	}
444
445	/*
446	 * mem_size is only a 32 bit container... follow the PPC route
447	 * and pin it to a 2 Gbyte maximum
448	 */
449	if (sane_size > (FOURGIG >> 1))
450	        mem_size = (vm_size_t)(FOURGIG >> 1);
451	else
452	        mem_size = (vm_size_t)sane_size;
453	max_mem = sane_size;
454
455	kprintf("Physical memory %llu MB\n", sane_size/MEG);
456
457	if (!PE_parse_boot_argn("max_valid_dma_addr", &maxdmaaddr, sizeof (maxdmaaddr)))
458	        max_valid_dma_address = 1024ULL * 1024ULL * 4096ULL;
459	else
460	        max_valid_dma_address = ((uint64_t) maxdmaaddr) * 1024ULL * 1024ULL;
461
462	if (!PE_parse_boot_argn("maxbouncepool", &maxbouncepoolsize, sizeof (maxbouncepoolsize)))
463	        maxbouncepoolsize = MAXBOUNCEPOOL;
464	else
465	        maxbouncepoolsize = maxbouncepoolsize * (1024 * 1024);
466
467	/*
468	 * bsd_mbuf_cluster_reserve depends on sane_size being set
469	 * in order to correctly determine the size of the mbuf pool
470	 * that will be reserved
471	 */
472	if (!PE_parse_boot_argn("maxloreserve", &maxloreserve, sizeof (maxloreserve)))
473	        maxloreserve = MAXLORESERVE + bsd_mbuf_cluster_reserve();
474	else
475	        maxloreserve = maxloreserve * (1024 * 1024);
476
477
478	if (avail_end >= max_valid_dma_address) {
479	        if (maxbouncepoolsize)
480		        reserve_bouncepool(maxbouncepoolsize);
481
482		if (maxloreserve)
483		        vm_lopage_poolsize = maxloreserve / PAGE_SIZE;
484	}
485
486	/*
487	 *	Initialize kernel physical map.
488	 *	Kernel virtual address starts at VM_KERNEL_MIN_ADDRESS.
489	 */
490	pmap_bootstrap(0, IA32e);
491}
492
493
494unsigned int
495pmap_free_pages(void)
496{
497	return avail_remaining;
498}
499
500
501boolean_t
502pmap_next_page(
503	       ppnum_t *pn)
504{
505
506	if (avail_remaining) while (pmap_memory_region_current < pmap_memory_region_count) {
507	        if (pmap_memory_regions[pmap_memory_region_current].alloc ==
508		    pmap_memory_regions[pmap_memory_region_current].end) {
509		        pmap_memory_region_current++;
510			continue;
511		}
512		*pn = pmap_memory_regions[pmap_memory_region_current].alloc++;
513		avail_remaining--;
514
515		return TRUE;
516	}
517	return FALSE;
518}
519
520
521boolean_t
522pmap_valid_page(
523	ppnum_t pn)
524{
525        unsigned int i;
526	pmap_memory_region_t *pmptr = pmap_memory_regions;
527
528	assert(pn);
529	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
530	        if ( (pn >= pmptr->base) && (pn <= pmptr->end) )
531	                return TRUE;
532	}
533	return FALSE;
534}
535
536
537static void
538reserve_bouncepool(uint32_t bounce_pool_wanted)
539{
540	pmap_memory_region_t *pmptr  = pmap_memory_regions;
541	pmap_memory_region_t *lowest = NULL;
542        unsigned int i;
543	unsigned int pages_needed;
544
545	pages_needed = bounce_pool_wanted / PAGE_SIZE;
546
547	for (i = 0; i < pmap_memory_region_count; i++, pmptr++) {
548	        if ( (pmptr->end - pmptr->alloc) >= pages_needed ) {
549		        if ( (lowest == NULL) || (pmptr->alloc < lowest->alloc) )
550			        lowest = pmptr;
551		}
552	}
553	if ( (lowest != NULL) ) {
554	        bounce_pool_base = lowest->alloc * PAGE_SIZE;
555		bounce_pool_size = bounce_pool_wanted;
556
557		lowest->alloc += pages_needed;
558		avail_remaining -= pages_needed;
559	}
560}
561