1/*
2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
49 *  School of Computer Science
50 *  Carnegie Mellon University
51 *  Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 * NOTICE: This file was modified by McAfee Research in 2004 to introduce
58 * support for mandatory and extensible security protections.  This notice
59 * is included in support of clause 2.2 (b) of the Apple Public License,
60 * Version 2.0.
61 */
62/*
63 */
64
65/*
66 *	Mach kernel startup.
67 */
68
69#include <debug.h>
70#include <xpr_debug.h>
71#include <mach_kdp.h>
72
73#include <mach/boolean.h>
74#include <mach/machine.h>
75#include <mach/thread_act.h>
76#include <mach/task_special_ports.h>
77#include <mach/vm_param.h>
78#include <ipc/ipc_init.h>
79#include <kern/assert.h>
80#include <kern/mach_param.h>
81#include <kern/misc_protos.h>
82#include <kern/clock.h>
83#include <kern/cpu_number.h>
84#include <kern/ledger.h>
85#include <kern/machine.h>
86#include <kern/processor.h>
87#include <kern/sched_prim.h>
88#include <kern/startup.h>
89#include <kern/task.h>
90#include <kern/thread.h>
91#include <kern/timer.h>
92#include <kern/wait_queue.h>
93#include <kern/xpr.h>
94#include <kern/zalloc.h>
95#include <kern/locks.h>
96#include <console/serial_protos.h>
97#include <vm/vm_kern.h>
98#include <vm/vm_init.h>
99#include <vm/vm_map.h>
100#include <vm/vm_object.h>
101#include <vm/vm_page.h>
102#include <vm/vm_pageout.h>
103#include <vm/vm_shared_region.h>
104#include <machine/pmap.h>
105#include <machine/commpage.h>
106#include <libkern/version.h>
107#include <sys/kdebug.h>
108
109#if MACH_KDP
110#include <kdp/kdp.h>
111#endif
112
113#if CONFIG_MACF
114#include <security/mac_mach_internal.h>
115#endif
116
117#if CONFIG_COUNTERS
118#include <pmc/pmc.h>
119#endif
120
121#include <i386/pmCPU.h>
122static void		kernel_bootstrap_thread(void);
123
124static void		load_context(
125					thread_t	thread);
126#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
127extern void cpu_userwindow_init(int);
128extern void cpu_physwindow_init(int);
129#endif
130
131// libkern/OSKextLib.cpp
132extern void	OSKextRemoveKextBootstrap(void);
133
134void scale_setup(void);
135extern void bsd_scale_setup(int);
136extern unsigned int semaphore_max;
137
138/*
139 *	Running in virtual memory, on the interrupt stack.
140 */
141
142extern int serverperfmode;
143
144/* size of kernel trace buffer, disabled by default */
145unsigned int new_nkdbufs = 0;
146unsigned int wake_nkdbufs = 0;
147
148/* mach leak logging */
149int log_leaks = 0;
150int turn_on_log_leaks = 0;
151
152
153void
154kernel_early_bootstrap(void)
155{
156
157	lck_mod_init();
158
159	/*
160	 * Initialize the timer callout world
161	 */
162	timer_call_initialize();
163}
164
165
166void
167kernel_bootstrap(void)
168{
169	kern_return_t	result;
170	thread_t	thread;
171	char		namep[16];
172
173	printf("%s\n", version); /* log kernel version */
174
175#define kernel_bootstrap_kprintf(x...) /* kprintf("kernel_bootstrap: " x) */
176
177	if (PE_parse_boot_argn("-l", namep, sizeof (namep))) /* leaks logging */
178		turn_on_log_leaks = 1;
179
180	PE_parse_boot_argn("trace", &new_nkdbufs, sizeof (new_nkdbufs));
181
182	PE_parse_boot_argn("trace_wake", &wake_nkdbufs, sizeof (wake_nkdbufs));
183
184	/* i386_vm_init already checks for this ; do it aagin anyway */
185        if (PE_parse_boot_argn("serverperfmode", &serverperfmode, sizeof (serverperfmode))) {
186                serverperfmode = 1;
187        }
188	scale_setup();
189
190	kernel_bootstrap_kprintf("calling vm_mem_bootstrap\n");
191	vm_mem_bootstrap();
192
193	kernel_bootstrap_kprintf("calling vm_mem_init\n");
194	vm_mem_init();
195
196	machine_info.memory_size = (uint32_t)mem_size;
197	machine_info.max_mem = max_mem;
198	machine_info.major_version = version_major;
199	machine_info.minor_version = version_minor;
200
201	kernel_bootstrap_kprintf("calling sched_init\n");
202	sched_init();
203
204	kernel_bootstrap_kprintf("calling wait_queue_bootstrap\n");
205	wait_queue_bootstrap();
206
207	kernel_bootstrap_kprintf("calling ipc_bootstrap\n");
208	ipc_bootstrap();
209
210#if CONFIG_MACF
211	mac_policy_init();
212#endif
213	kernel_bootstrap_kprintf("calling ipc_init\n");
214	ipc_init();
215
216	/*
217	 * As soon as the virtual memory system is up, we record
218	 * that this CPU is using the kernel pmap.
219	 */
220	kernel_bootstrap_kprintf("calling PMAP_ACTIVATE_KERNEL\n");
221	PMAP_ACTIVATE_KERNEL(master_cpu);
222
223	kernel_bootstrap_kprintf("calling mapping_free_prime\n");
224	mapping_free_prime();						/* Load up with temporary mapping blocks */
225
226	kernel_bootstrap_kprintf("calling machine_init\n");
227	machine_init();
228
229	kernel_bootstrap_kprintf("calling clock_init\n");
230	clock_init();
231
232	ledger_init();
233
234	/*
235	 *	Initialize the IPC, task, and thread subsystems.
236	 */
237	kernel_bootstrap_kprintf("calling task_init\n");
238	task_init();
239
240	kernel_bootstrap_kprintf("calling thread_init\n");
241	thread_init();
242
243	/*
244	 *	Create a kernel thread to execute the kernel bootstrap.
245	 */
246	kernel_bootstrap_kprintf("calling kernel_thread_create\n");
247	result = kernel_thread_create((thread_continue_t)kernel_bootstrap_thread, NULL, MAXPRI_KERNEL, &thread);
248
249	if (result != KERN_SUCCESS) panic("kernel_bootstrap: result = %08X\n", result);
250
251	thread->state = TH_RUN;
252	thread_deallocate(thread);
253
254	kernel_bootstrap_kprintf("calling load_context - done\n");
255	load_context(thread);
256	/*NOTREACHED*/
257}
258
259int kth_started = 0;
260
261vm_offset_t vm_kernel_addrperm;
262
263/*
264 * Now running in a thread.  Kick off other services,
265 * invoke user bootstrap, enter pageout loop.
266 */
267static void
268kernel_bootstrap_thread(void)
269{
270	processor_t		processor = current_processor();
271
272#define kernel_bootstrap_thread_kprintf(x...) /* kprintf("kernel_bootstrap_thread: " x) */
273	kernel_bootstrap_thread_kprintf("calling idle_thread_create\n");
274	/*
275	 * Create the idle processor thread.
276	 */
277	idle_thread_create(processor);
278
279	/*
280	 * N.B. Do not stick anything else
281	 * before this point.
282	 *
283	 * Start up the scheduler services.
284	 */
285	kernel_bootstrap_thread_kprintf("calling sched_startup\n");
286	sched_startup();
287
288	/*
289	 * Thread lifecycle maintenance (teardown, stack allocation)
290	 */
291	kernel_bootstrap_thread_kprintf("calling thread_daemon_init\n");
292	thread_daemon_init();
293
294	/*
295	 * Thread callout service.
296	 */
297	kernel_bootstrap_thread_kprintf("calling thread_call_initialize\n");
298	thread_call_initialize();
299
300	/*
301	 * Remain on current processor as
302	 * additional processors come online.
303	 */
304	kernel_bootstrap_thread_kprintf("calling thread_bind\n");
305	thread_bind(processor);
306
307	/*
308	 * Kick off memory mapping adjustments.
309	 */
310	kernel_bootstrap_thread_kprintf("calling mapping_adjust\n");
311	mapping_adjust();
312
313	/*
314	 *	Create the clock service.
315	 */
316	kernel_bootstrap_thread_kprintf("calling clock_service_create\n");
317	clock_service_create();
318
319	/*
320	 *	Create the device service.
321	 */
322	device_service_create();
323
324	kth_started = 1;
325
326#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
327	/*
328	 * Create and initialize the physical copy window for processor 0
329	 * This is required before starting kicking off  IOKit.
330	 */
331	cpu_physwindow_init(0);
332#endif
333
334	vm_kernel_reserved_entry_init();
335
336#if MACH_KDP
337	kernel_bootstrap_kprintf("calling kdp_init\n");
338	kdp_init();
339#endif
340
341#if CONFIG_COUNTERS
342	pmc_bootstrap();
343#endif
344
345#if (defined(__i386__) || defined(__x86_64__))
346	if (turn_on_log_leaks && !new_nkdbufs)
347		new_nkdbufs = 200000;
348	start_kern_tracing(new_nkdbufs);
349	if (turn_on_log_leaks)
350		log_leaks = 1;
351#endif
352
353#ifdef	IOKIT
354	PE_init_iokit();
355#endif
356
357	(void) spllo();		/* Allow interruptions */
358
359#if (defined(__i386__) || defined(__x86_64__)) && NCOPY_WINDOWS > 0
360	/*
361	 * Create and initialize the copy window for processor 0
362	 * This also allocates window space for all other processors.
363	 * However, this is dependent on the number of processors - so this call
364	 * must be after IOKit has been started because IOKit performs processor
365	 * discovery.
366	 */
367	cpu_userwindow_init(0);
368#endif
369
370#if (!defined(__i386__) && !defined(__x86_64__))
371	if (turn_on_log_leaks && !new_nkdbufs)
372		new_nkdbufs = 200000;
373	start_kern_tracing(new_nkdbufs);
374	if (turn_on_log_leaks)
375		log_leaks = 1;
376#endif
377
378	/*
379	 *	Initialize the shared region module.
380	 */
381	vm_shared_region_init();
382	vm_commpage_init();
383	vm_commpage_text_init();
384
385#if CONFIG_MACF
386	mac_policy_initmach();
387#endif
388
389	/*
390	 * Initialize the global used for permuting kernel
391	 * addresses that may be exported to userland as tokens
392	 * using VM_KERNEL_ADDRPERM(). Force the random number
393	 * to be odd to avoid mapping a non-zero
394	 * word-aligned address to zero via addition.
395	 */
396	vm_kernel_addrperm = (vm_offset_t)early_random() | 1;
397
398	/*
399	 *	Start the user bootstrap.
400	 */
401#ifdef	MACH_BSD
402	bsd_init();
403#endif
404
405    /*
406     * Get rid of segments used to bootstrap kext loading. This removes
407     * the KLD, PRELINK symtab, LINKEDIT, and symtab segments/load commands.
408     */
409#if 0
410	OSKextRemoveKextBootstrap();
411#endif
412
413	serial_keyboard_init();		/* Start serial keyboard if wanted */
414
415	vm_page_init_local_q();
416
417	thread_bind(PROCESSOR_NULL);
418
419	/*
420	 *	Become the pageout daemon.
421	 */
422	vm_pageout();
423	/*NOTREACHED*/
424}
425
426/*
427 *	slave_main:
428 *
429 *	Load the first thread to start a processor.
430 */
431void
432slave_main(void *machine_param)
433{
434	processor_t		processor = current_processor();
435	thread_t		thread;
436
437	/*
438	 *	Use the idle processor thread if there
439	 *	is no dedicated start up thread.
440	 */
441	if (processor->next_thread == THREAD_NULL) {
442		thread = processor->idle_thread;
443		thread->continuation = (thread_continue_t)processor_start_thread;
444		thread->parameter = machine_param;
445	}
446	else {
447		thread = processor->next_thread;
448		processor->next_thread = THREAD_NULL;
449	}
450
451	load_context(thread);
452	/*NOTREACHED*/
453}
454
455/*
456 *	processor_start_thread:
457 *
458 *	First thread to execute on a started processor.
459 *
460 *	Called at splsched.
461 */
462void
463processor_start_thread(void *machine_param)
464{
465	processor_t		processor = current_processor();
466	thread_t		self = current_thread();
467
468	slave_machine_init(machine_param);
469
470	/*
471	 *	If running the idle processor thread,
472	 *	reenter the idle loop, else terminate.
473	 */
474	if (self == processor->idle_thread)
475		thread_block((thread_continue_t)idle_thread);
476
477	thread_terminate(self);
478	/*NOTREACHED*/
479}
480
481/*
482 *	load_context:
483 *
484 *	Start the first thread on a processor.
485 */
486static void
487load_context(
488	thread_t		thread)
489{
490	processor_t		processor = current_processor();
491
492#define load_context_kprintf(x...) /* kprintf("load_context: " x) */
493
494	load_context_kprintf("calling machine_set_current_thread\n");
495	machine_set_current_thread(thread);
496
497	load_context_kprintf("calling processor_up\n");
498	processor_up(processor);
499
500	PMAP_ACTIVATE_KERNEL(processor->cpu_id);
501
502	/*
503	 * Acquire a stack if none attached.  The panic
504	 * should never occur since the thread is expected
505	 * to have reserved stack.
506	 */
507	load_context_kprintf("thread %p, stack %x, stackptr %x\n", thread,
508			     thread->kernel_stack, thread->machine.iss->sp);
509	if (!thread->kernel_stack) {
510		load_context_kprintf("calling stack_alloc_try\n");
511		if (!stack_alloc_try(thread))
512			panic("load_context");
513	}
514
515	/*
516	 * The idle processor threads are not counted as
517	 * running for load calculations.
518	 */
519	if (!(thread->state & TH_IDLE))
520		sched_run_incr();
521
522	processor->active_thread = thread;
523	processor->current_pri = thread->sched_pri;
524	processor->current_thmode = thread->sched_mode;
525	processor->deadline = UINT64_MAX;
526	thread->last_processor = processor;
527
528	processor->last_dispatch = mach_absolute_time();
529	timer_start(&thread->system_timer, processor->last_dispatch);
530	PROCESSOR_DATA(processor, thread_timer) = PROCESSOR_DATA(processor, kernel_timer) = &thread->system_timer;
531
532	timer_start(&PROCESSOR_DATA(processor, system_state), processor->last_dispatch);
533	PROCESSOR_DATA(processor, current_state) = &PROCESSOR_DATA(processor, system_state);
534
535	PMAP_ACTIVATE_USER(thread, processor->cpu_id);
536
537	load_context_kprintf("calling machine_load_context\n");
538	machine_load_context(thread);
539	/*NOTREACHED*/
540}
541
542void
543scale_setup()
544{
545	int scale = 0;
546#if defined(__LP64__)
547	typeof(task_max) task_max_base = task_max;
548
549	/* Raise limits for servers with >= 16G */
550	if ((serverperfmode != 0) && ((uint64_t)sane_size >= (uint64_t)(16 * 1024 * 1024 *1024ULL))) {
551		scale = (int)((uint64_t)sane_size / (uint64_t)(8 * 1024 * 1024 *1024ULL));
552		/* limit to 128 G */
553		if (scale > 16)
554			scale = 16;
555		task_max_base = 2500;
556	} else if ((uint64_t)sane_size >= (uint64_t)(3 * 1024 * 1024 *1024ULL))
557		scale = 2;
558
559	task_max = MAX(task_max, task_max_base * scale);
560
561	if (scale != 0) {
562		task_threadmax = task_max;
563		thread_max = task_max * 5;
564	}
565
566#endif
567
568	bsd_scale_setup(scale);
569
570	ipc_space_max = SPACE_MAX;
571	ipc_port_max = PORT_MAX;
572	ipc_pset_max = SET_MAX;
573	semaphore_max = SEMAPHORE_MAX;
574}
575
576