1/*
2 * Copyright (c) 2003-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 *	Kernel stack management routines.
30 */
31
32#include <mach/mach_host.h>
33#include <mach/mach_types.h>
34#include <mach/processor_set.h>
35
36#include <kern/kern_types.h>
37#include <kern/mach_param.h>
38#include <kern/processor.h>
39#include <kern/thread.h>
40#include <kern/zalloc.h>
41#include <kern/kalloc.h>
42
43#include <vm/vm_map.h>
44#include <vm/vm_kern.h>
45
46#include <mach_debug.h>
47
48/*
49 *	We allocate stacks from generic kernel VM.
50 *
51 *	The stack_free_list can only be accessed at splsched,
52 *	because stack_alloc_try/thread_invoke operate at splsched.
53 */
54
55decl_simple_lock_data(static,stack_lock_data)
56#define stack_lock()		simple_lock(&stack_lock_data)
57#define stack_unlock()		simple_unlock(&stack_lock_data)
58
59#define STACK_CACHE_SIZE	2
60
61static vm_offset_t		stack_free_list;
62
63static unsigned int		stack_free_count, stack_free_hiwat;		/* free list count */
64static unsigned int		stack_hiwat;
65unsigned int			stack_total;				/* current total count */
66
67static unsigned int		stack_free_target;
68static int				stack_free_delta;
69
70static unsigned int		stack_new_count;						/* total new stack allocations */
71
72static vm_offset_t		stack_addr_mask;
73
74/*
75 *	The next field is at the base of the stack,
76 *	so the low end is left unsullied.
77 */
78#define stack_next(stack)	\
79			(*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
80
81void
82stack_init(void)
83{
84	simple_lock_init(&stack_lock_data, 0);
85
86	if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE))
87		panic("stack_init: stack size %d not a multiple of page size %d\n",	KERNEL_STACK_SIZE, PAGE_SIZE);
88
89	stack_addr_mask = KERNEL_STACK_SIZE - 1;
90}
91
92/*
93 *	stack_alloc:
94 *
95 *	Allocate a stack for a thread, may
96 *	block.
97 */
98void
99stack_alloc(
100	thread_t	thread)
101{
102	vm_offset_t		stack;
103	spl_t			s;
104	int			guard_flags;
105
106	assert(thread->kernel_stack == 0);
107
108	s = splsched();
109	stack_lock();
110	stack = stack_free_list;
111	if (stack != 0) {
112		stack_free_list = stack_next(stack);
113		stack_free_count--;
114	}
115	else {
116		if (++stack_total > stack_hiwat)
117			stack_hiwat = stack_total;
118		stack_new_count++;
119	}
120	stack_free_delta--;
121	stack_unlock();
122	splx(s);
123
124	if (stack == 0) {
125
126		/*
127		 * Request guard pages on either side of the stack.  Ask
128		 * kernel_memory_allocate() for two extra pages to account
129		 * for these.
130		 */
131
132		guard_flags = KMA_GUARD_FIRST | KMA_GUARD_LAST;
133		if (kernel_memory_allocate(kernel_map, &stack,
134					   KERNEL_STACK_SIZE + (2*PAGE_SIZE),
135					   stack_addr_mask,
136					   KMA_KOBJECT | guard_flags)
137		    != KERN_SUCCESS)
138			panic("stack_alloc: kernel_memory_allocate");
139
140		/*
141		 * The stack address that comes back is the address of the lower
142		 * guard page.  Skip past it to get the actual stack base address.
143		 */
144
145		stack += PAGE_SIZE;
146	}
147
148	machine_stack_attach(thread, stack);
149}
150
151/*
152 *	stack_free:
153 *
154 *	Detach and free the stack for a thread.
155 */
156void
157stack_free(
158	thread_t	thread)
159{
160    vm_offset_t		stack = machine_stack_detach(thread);
161
162	assert(stack);
163	if (stack != thread->reserved_stack)
164		stack_free_stack(stack);
165}
166
167void
168stack_free_stack(
169	vm_offset_t		stack)
170{
171	struct stack_cache	*cache;
172	spl_t				s;
173
174	s = splsched();
175	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
176	if (cache->count < STACK_CACHE_SIZE) {
177		stack_next(stack) = cache->free;
178		cache->free = stack;
179		cache->count++;
180	}
181	else {
182		stack_lock();
183		stack_next(stack) = stack_free_list;
184		stack_free_list = stack;
185		if (++stack_free_count > stack_free_hiwat)
186			stack_free_hiwat = stack_free_count;
187		stack_free_delta++;
188		stack_unlock();
189	}
190	splx(s);
191}
192
193/*
194 *	stack_alloc_try:
195 *
196 *	Non-blocking attempt to allocate a
197 *	stack for a thread.
198 *
199 *	Returns TRUE on success.
200 *
201 *	Called at splsched.
202 */
203boolean_t
204stack_alloc_try(
205	thread_t		thread)
206{
207	struct stack_cache	*cache;
208	vm_offset_t			stack;
209
210	cache = &PROCESSOR_DATA(current_processor(), stack_cache);
211	stack = cache->free;
212	if (stack != 0) {
213		cache->free = stack_next(stack);
214		cache->count--;
215	}
216	else {
217		if (stack_free_list != 0) {
218			stack_lock();
219			stack = stack_free_list;
220			if (stack != 0) {
221				stack_free_list = stack_next(stack);
222				stack_free_count--;
223				stack_free_delta--;
224			}
225			stack_unlock();
226		}
227	}
228
229	if (stack != 0 || (stack = thread->reserved_stack) != 0) {
230		machine_stack_attach(thread, stack);
231		return (TRUE);
232	}
233
234	return (FALSE);
235}
236
237static unsigned int		stack_collect_tick, last_stack_tick;
238
239/*
240 *	stack_collect:
241 *
242 *	Free excess kernel stacks, may
243 *	block.
244 */
245void
246stack_collect(void)
247{
248	if (stack_collect_tick != last_stack_tick) {
249		unsigned int	target;
250		vm_offset_t		stack;
251		spl_t			s;
252
253		s = splsched();
254		stack_lock();
255
256		target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
257		target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
258
259		while (stack_free_count > target) {
260			stack = stack_free_list;
261			stack_free_list = stack_next(stack);
262			stack_free_count--; stack_total--;
263			stack_unlock();
264			splx(s);
265
266			/*
267			 * Get the stack base address, then decrement by one page
268			 * to account for the lower guard page.  Add two extra pages
269			 * to the size to account for the guard pages on both ends
270			 * that were originally requested when the stack was allocated
271			 * back in stack_alloc().
272			 */
273
274			stack = vm_map_trunc_page(stack);
275			stack -= PAGE_SIZE;
276			if (vm_map_remove(
277				    kernel_map,
278				    stack,
279				    stack + KERNEL_STACK_SIZE+(2*PAGE_SIZE),
280				    VM_MAP_REMOVE_KUNWIRE)
281			    != KERN_SUCCESS)
282				panic("stack_collect: vm_map_remove");
283			stack = 0;
284
285			s = splsched();
286			stack_lock();
287
288			target = stack_free_target + (STACK_CACHE_SIZE * processor_count);
289			target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
290		}
291
292		last_stack_tick = stack_collect_tick;
293
294		stack_unlock();
295		splx(s);
296	}
297}
298
299/*
300 *	compute_stack_target:
301 *
302 *	Computes a new target free list count
303 *	based on recent alloc / free activity.
304 *
305 *	Limits stack collection to once per
306 *	computation period.
307 */
308void
309compute_stack_target(
310__unused void		*arg)
311{
312	spl_t		s;
313
314	s = splsched();
315	stack_lock();
316
317	if (stack_free_target > 5)
318		stack_free_target = (4 * stack_free_target) / 5;
319	else
320	if (stack_free_target > 0)
321		stack_free_target--;
322
323	stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
324
325	stack_free_delta = 0;
326	stack_collect_tick++;
327
328	stack_unlock();
329	splx(s);
330}
331
332void
333stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
334		     vm_size_t *alloc_size, int *collectable, int *exhaustable)
335{
336	unsigned int	total, hiwat, free;
337	spl_t			s;
338
339	s = splsched();
340	stack_lock();
341	total = stack_total;
342	hiwat = stack_hiwat;
343	free = stack_free_count;
344	stack_unlock();
345	splx(s);
346
347	*count      = total - free;
348	*cur_size   = KERNEL_STACK_SIZE * total;
349	*max_size   = KERNEL_STACK_SIZE * hiwat;
350	*elem_size  = KERNEL_STACK_SIZE;
351	*alloc_size = KERNEL_STACK_SIZE;
352	*collectable = 1;
353	*exhaustable = 0;
354}
355
356/* OBSOLETE */
357void	stack_privilege(
358			thread_t	thread);
359
360void
361stack_privilege(
362	__unused thread_t	thread)
363{
364	/* OBSOLETE */
365}
366
367/*
368 * Return info on stack usage for threads in a specific processor set
369 */
370kern_return_t
371processor_set_stack_usage(
372	processor_set_t	pset,
373	unsigned int	*totalp,
374	vm_size_t	*spacep,
375	vm_size_t	*residentp,
376	vm_size_t	*maxusagep,
377	vm_offset_t	*maxstackp)
378{
379#if !MACH_DEBUG
380        return KERN_NOT_SUPPORTED;
381#else
382	unsigned int total;
383	vm_size_t maxusage;
384	vm_offset_t maxstack;
385
386	register thread_t *thread_list;
387	register thread_t thread;
388
389	unsigned int actual;	/* this many things */
390	unsigned int i;
391
392	vm_size_t size, size_needed;
393	void *addr;
394
395	if (pset == PROCESSOR_SET_NULL || pset != &pset0)
396		return KERN_INVALID_ARGUMENT;
397
398	size = 0;
399	addr = NULL;
400
401	for (;;) {
402		mutex_lock(&tasks_threads_lock);
403
404		actual = threads_count;
405
406		/* do we have the memory we need? */
407
408		size_needed = actual * sizeof(thread_t);
409		if (size_needed <= size)
410			break;
411
412		mutex_unlock(&tasks_threads_lock);
413
414		if (size != 0)
415			kfree(addr, size);
416
417		assert(size_needed > 0);
418		size = size_needed;
419
420		addr = kalloc(size);
421		if (addr == 0)
422			return KERN_RESOURCE_SHORTAGE;
423	}
424
425	/* OK, have memory and list is locked */
426	thread_list = (thread_t *) addr;
427	for (i = 0, thread = (thread_t) queue_first(&threads);
428					!queue_end(&threads, (queue_entry_t) thread);
429					thread = (thread_t) queue_next(&thread->threads)) {
430		thread_reference_internal(thread);
431		thread_list[i++] = thread;
432	}
433	assert(i <= actual);
434
435	mutex_unlock(&tasks_threads_lock);
436
437	/* calculate maxusage and free thread references */
438
439	total = 0;
440	maxusage = 0;
441	maxstack = 0;
442	while (i > 0) {
443		thread_t threadref = thread_list[--i];
444
445		if (threadref->kernel_stack != 0)
446			total++;
447
448		thread_deallocate(threadref);
449	}
450
451	if (size != 0)
452		kfree(addr, size);
453
454	*totalp = total;
455	*residentp = *spacep = total * round_page(KERNEL_STACK_SIZE);
456	*maxusagep = maxusage;
457	*maxstackp = maxstack;
458	return KERN_SUCCESS;
459
460#endif	/* MACH_DEBUG */
461}
462
463vm_offset_t min_valid_stack_address(void)
464{
465	return vm_map_min(kernel_map);
466}
467
468vm_offset_t max_valid_stack_address(void)
469{
470	return vm_map_max(kernel_map);
471}
472