1/* 2 * Copyright (c) 2003-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * Kernel stack management routines. 30 */ 31 32#include <mach/mach_host.h> 33#include <mach/mach_types.h> 34#include <mach/processor_set.h> 35 36#include <kern/kern_types.h> 37#include <kern/mach_param.h> 38#include <kern/processor.h> 39#include <kern/thread.h> 40#include <kern/zalloc.h> 41#include <kern/kalloc.h> 42 43#include <vm/vm_map.h> 44#include <vm/vm_kern.h> 45 46#include <mach_debug.h> 47 48/* 49 * We allocate stacks from generic kernel VM. 50 * 51 * The stack_free_list can only be accessed at splsched, 52 * because stack_alloc_try/thread_invoke operate at splsched. 53 */ 54 55decl_simple_lock_data(static,stack_lock_data) 56#define stack_lock() simple_lock(&stack_lock_data) 57#define stack_unlock() simple_unlock(&stack_lock_data) 58 59#define STACK_CACHE_SIZE 2 60 61static vm_offset_t stack_free_list; 62 63static unsigned int stack_free_count, stack_free_hiwat; /* free list count */ 64static unsigned int stack_hiwat; 65unsigned int stack_total; /* current total count */ 66 67static unsigned int stack_free_target; 68static int stack_free_delta; 69 70static unsigned int stack_new_count; /* total new stack allocations */ 71 72static vm_offset_t stack_addr_mask; 73 74/* 75 * The next field is at the base of the stack, 76 * so the low end is left unsullied. 77 */ 78#define stack_next(stack) \ 79 (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1)) 80 81void 82stack_init(void) 83{ 84 simple_lock_init(&stack_lock_data, 0); 85 86 if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE)) 87 panic("stack_init: stack size %d not a multiple of page size %d\n", KERNEL_STACK_SIZE, PAGE_SIZE); 88 89 stack_addr_mask = KERNEL_STACK_SIZE - 1; 90} 91 92/* 93 * stack_alloc: 94 * 95 * Allocate a stack for a thread, may 96 * block. 97 */ 98void 99stack_alloc( 100 thread_t thread) 101{ 102 vm_offset_t stack; 103 spl_t s; 104 int guard_flags; 105 106 assert(thread->kernel_stack == 0); 107 108 s = splsched(); 109 stack_lock(); 110 stack = stack_free_list; 111 if (stack != 0) { 112 stack_free_list = stack_next(stack); 113 stack_free_count--; 114 } 115 else { 116 if (++stack_total > stack_hiwat) 117 stack_hiwat = stack_total; 118 stack_new_count++; 119 } 120 stack_free_delta--; 121 stack_unlock(); 122 splx(s); 123 124 if (stack == 0) { 125 126 /* 127 * Request guard pages on either side of the stack. Ask 128 * kernel_memory_allocate() for two extra pages to account 129 * for these. 130 */ 131 132 guard_flags = KMA_GUARD_FIRST | KMA_GUARD_LAST; 133 if (kernel_memory_allocate(kernel_map, &stack, 134 KERNEL_STACK_SIZE + (2*PAGE_SIZE), 135 stack_addr_mask, 136 KMA_KOBJECT | guard_flags) 137 != KERN_SUCCESS) 138 panic("stack_alloc: kernel_memory_allocate"); 139 140 /* 141 * The stack address that comes back is the address of the lower 142 * guard page. Skip past it to get the actual stack base address. 143 */ 144 145 stack += PAGE_SIZE; 146 } 147 148 machine_stack_attach(thread, stack); 149} 150 151/* 152 * stack_free: 153 * 154 * Detach and free the stack for a thread. 155 */ 156void 157stack_free( 158 thread_t thread) 159{ 160 vm_offset_t stack = machine_stack_detach(thread); 161 162 assert(stack); 163 if (stack != thread->reserved_stack) 164 stack_free_stack(stack); 165} 166 167void 168stack_free_stack( 169 vm_offset_t stack) 170{ 171 struct stack_cache *cache; 172 spl_t s; 173 174 s = splsched(); 175 cache = &PROCESSOR_DATA(current_processor(), stack_cache); 176 if (cache->count < STACK_CACHE_SIZE) { 177 stack_next(stack) = cache->free; 178 cache->free = stack; 179 cache->count++; 180 } 181 else { 182 stack_lock(); 183 stack_next(stack) = stack_free_list; 184 stack_free_list = stack; 185 if (++stack_free_count > stack_free_hiwat) 186 stack_free_hiwat = stack_free_count; 187 stack_free_delta++; 188 stack_unlock(); 189 } 190 splx(s); 191} 192 193/* 194 * stack_alloc_try: 195 * 196 * Non-blocking attempt to allocate a 197 * stack for a thread. 198 * 199 * Returns TRUE on success. 200 * 201 * Called at splsched. 202 */ 203boolean_t 204stack_alloc_try( 205 thread_t thread) 206{ 207 struct stack_cache *cache; 208 vm_offset_t stack; 209 210 cache = &PROCESSOR_DATA(current_processor(), stack_cache); 211 stack = cache->free; 212 if (stack != 0) { 213 cache->free = stack_next(stack); 214 cache->count--; 215 } 216 else { 217 if (stack_free_list != 0) { 218 stack_lock(); 219 stack = stack_free_list; 220 if (stack != 0) { 221 stack_free_list = stack_next(stack); 222 stack_free_count--; 223 stack_free_delta--; 224 } 225 stack_unlock(); 226 } 227 } 228 229 if (stack != 0 || (stack = thread->reserved_stack) != 0) { 230 machine_stack_attach(thread, stack); 231 return (TRUE); 232 } 233 234 return (FALSE); 235} 236 237static unsigned int stack_collect_tick, last_stack_tick; 238 239/* 240 * stack_collect: 241 * 242 * Free excess kernel stacks, may 243 * block. 244 */ 245void 246stack_collect(void) 247{ 248 if (stack_collect_tick != last_stack_tick) { 249 unsigned int target; 250 vm_offset_t stack; 251 spl_t s; 252 253 s = splsched(); 254 stack_lock(); 255 256 target = stack_free_target + (STACK_CACHE_SIZE * processor_count); 257 target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta; 258 259 while (stack_free_count > target) { 260 stack = stack_free_list; 261 stack_free_list = stack_next(stack); 262 stack_free_count--; stack_total--; 263 stack_unlock(); 264 splx(s); 265 266 /* 267 * Get the stack base address, then decrement by one page 268 * to account for the lower guard page. Add two extra pages 269 * to the size to account for the guard pages on both ends 270 * that were originally requested when the stack was allocated 271 * back in stack_alloc(). 272 */ 273 274 stack = vm_map_trunc_page(stack); 275 stack -= PAGE_SIZE; 276 if (vm_map_remove( 277 kernel_map, 278 stack, 279 stack + KERNEL_STACK_SIZE+(2*PAGE_SIZE), 280 VM_MAP_REMOVE_KUNWIRE) 281 != KERN_SUCCESS) 282 panic("stack_collect: vm_map_remove"); 283 stack = 0; 284 285 s = splsched(); 286 stack_lock(); 287 288 target = stack_free_target + (STACK_CACHE_SIZE * processor_count); 289 target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta; 290 } 291 292 last_stack_tick = stack_collect_tick; 293 294 stack_unlock(); 295 splx(s); 296 } 297} 298 299/* 300 * compute_stack_target: 301 * 302 * Computes a new target free list count 303 * based on recent alloc / free activity. 304 * 305 * Limits stack collection to once per 306 * computation period. 307 */ 308void 309compute_stack_target( 310__unused void *arg) 311{ 312 spl_t s; 313 314 s = splsched(); 315 stack_lock(); 316 317 if (stack_free_target > 5) 318 stack_free_target = (4 * stack_free_target) / 5; 319 else 320 if (stack_free_target > 0) 321 stack_free_target--; 322 323 stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta; 324 325 stack_free_delta = 0; 326 stack_collect_tick++; 327 328 stack_unlock(); 329 splx(s); 330} 331 332void 333stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, 334 vm_size_t *alloc_size, int *collectable, int *exhaustable) 335{ 336 unsigned int total, hiwat, free; 337 spl_t s; 338 339 s = splsched(); 340 stack_lock(); 341 total = stack_total; 342 hiwat = stack_hiwat; 343 free = stack_free_count; 344 stack_unlock(); 345 splx(s); 346 347 *count = total - free; 348 *cur_size = KERNEL_STACK_SIZE * total; 349 *max_size = KERNEL_STACK_SIZE * hiwat; 350 *elem_size = KERNEL_STACK_SIZE; 351 *alloc_size = KERNEL_STACK_SIZE; 352 *collectable = 1; 353 *exhaustable = 0; 354} 355 356/* OBSOLETE */ 357void stack_privilege( 358 thread_t thread); 359 360void 361stack_privilege( 362 __unused thread_t thread) 363{ 364 /* OBSOLETE */ 365} 366 367/* 368 * Return info on stack usage for threads in a specific processor set 369 */ 370kern_return_t 371processor_set_stack_usage( 372 processor_set_t pset, 373 unsigned int *totalp, 374 vm_size_t *spacep, 375 vm_size_t *residentp, 376 vm_size_t *maxusagep, 377 vm_offset_t *maxstackp) 378{ 379#if !MACH_DEBUG 380 return KERN_NOT_SUPPORTED; 381#else 382 unsigned int total; 383 vm_size_t maxusage; 384 vm_offset_t maxstack; 385 386 register thread_t *thread_list; 387 register thread_t thread; 388 389 unsigned int actual; /* this many things */ 390 unsigned int i; 391 392 vm_size_t size, size_needed; 393 void *addr; 394 395 if (pset == PROCESSOR_SET_NULL || pset != &pset0) 396 return KERN_INVALID_ARGUMENT; 397 398 size = 0; 399 addr = NULL; 400 401 for (;;) { 402 mutex_lock(&tasks_threads_lock); 403 404 actual = threads_count; 405 406 /* do we have the memory we need? */ 407 408 size_needed = actual * sizeof(thread_t); 409 if (size_needed <= size) 410 break; 411 412 mutex_unlock(&tasks_threads_lock); 413 414 if (size != 0) 415 kfree(addr, size); 416 417 assert(size_needed > 0); 418 size = size_needed; 419 420 addr = kalloc(size); 421 if (addr == 0) 422 return KERN_RESOURCE_SHORTAGE; 423 } 424 425 /* OK, have memory and list is locked */ 426 thread_list = (thread_t *) addr; 427 for (i = 0, thread = (thread_t) queue_first(&threads); 428 !queue_end(&threads, (queue_entry_t) thread); 429 thread = (thread_t) queue_next(&thread->threads)) { 430 thread_reference_internal(thread); 431 thread_list[i++] = thread; 432 } 433 assert(i <= actual); 434 435 mutex_unlock(&tasks_threads_lock); 436 437 /* calculate maxusage and free thread references */ 438 439 total = 0; 440 maxusage = 0; 441 maxstack = 0; 442 while (i > 0) { 443 thread_t threadref = thread_list[--i]; 444 445 if (threadref->kernel_stack != 0) 446 total++; 447 448 thread_deallocate(threadref); 449 } 450 451 if (size != 0) 452 kfree(addr, size); 453 454 *totalp = total; 455 *residentp = *spacep = total * round_page(KERNEL_STACK_SIZE); 456 *maxusagep = maxusage; 457 *maxstackp = maxstack; 458 return KERN_SUCCESS; 459 460#endif /* MACH_DEBUG */ 461} 462 463vm_offset_t min_valid_stack_address(void) 464{ 465 return vm_map_min(kernel_map); 466} 467 468vm_offset_t max_valid_stack_address(void) 469{ 470 return vm_map_max(kernel_map); 471} 472