1/* 2 * Copyright (c) 2000-2010 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58/* 59 * File: zalloc.h 60 * Author: Avadis Tevanian, Jr. 61 * Date: 1985 62 * 63 */ 64 65#ifdef KERNEL_PRIVATE 66 67#ifndef _KERN_ZALLOC_H_ 68#define _KERN_ZALLOC_H_ 69 70#include <mach/machine/vm_types.h> 71#include <kern/kern_types.h> 72#include <sys/cdefs.h> 73 74#ifdef MACH_KERNEL_PRIVATE 75 76#include <zone_debug.h> 77#include <kern/lock.h> 78#include <kern/locks.h> 79#include <kern/queue.h> 80#include <kern/thread_call.h> 81 82#if CONFIG_GZALLOC 83typedef struct gzalloc_data { 84 uint32_t gzfc_index; 85 vm_offset_t *gzfc; 86} gzalloc_data_t; 87#endif 88 89/* 90 * A zone is a collection of fixed size blocks for which there 91 * is fast allocation/deallocation access. Kernel routines can 92 * use zones to manage data structures dynamically, creating a zone 93 * for each type of data structure to be managed. 94 * 95 */ 96 97struct zone_free_element; 98struct zone_page_metadata; 99 100struct zone { 101 struct zone_free_element *free_elements; /* free elements directly linked */ 102 struct { 103 queue_head_t any_free_foreign; /* foreign pages crammed into zone */ 104 queue_head_t all_free; 105 queue_head_t intermediate; 106 queue_head_t all_used; 107 } pages; /* list of zone_page_metadata structs, which maintain per-page free element lists */ 108 int count; /* Number of elements used now */ 109 int countfree; /* Number of free elements */ 110 lck_attr_t lock_attr; /* zone lock attribute */ 111 decl_lck_mtx_data(,lock) /* zone lock */ 112 lck_mtx_ext_t lock_ext; /* placeholder for indirect mutex */ 113 vm_size_t cur_size; /* current memory utilization */ 114 vm_size_t max_size; /* how large can this zone grow */ 115 vm_size_t elem_size; /* size of an element */ 116 vm_size_t alloc_size; /* size used for more memory */ 117 uint64_t page_count __attribute__((aligned(8))); /* number of pages used by this zone */ 118 uint64_t sum_count; /* count of allocs (life of zone) */ 119 uint32_t 120 /* boolean_t */ exhaustible :1, /* (F) merely return if empty? */ 121 /* boolean_t */ collectable :1, /* (F) garbage collect empty pages */ 122 /* boolean_t */ expandable :1, /* (T) expand zone (with message)? */ 123 /* boolean_t */ allows_foreign :1, /* (F) allow non-zalloc space */ 124 /* boolean_t */ doing_alloc :1, /* is zone expanding now? */ 125 /* boolean_t */ waiting :1, /* is thread waiting for expansion? */ 126 /* boolean_t */ async_pending :1, /* asynchronous allocation pending? */ 127 /* boolean_t */ zleak_on :1, /* Are we collecting allocation information? */ 128 /* boolean_t */ caller_acct :1, /* do we account allocation/free to the caller? */ 129 /* boolean_t */ doing_gc :1, /* garbage collect in progress? */ 130 /* boolean_t */ noencrypt :1, 131 /* boolean_t */ no_callout :1, 132 /* boolean_t */ async_prio_refill :1, 133 /* boolean_t */ gzalloc_exempt :1, 134 /* boolean_t */ alignment_required :1, 135 /* boolean_t */ use_page_list :1, 136 /* future */ _reserved :16; 137 138 int index; /* index into zone_info arrays for this zone */ 139 struct zone *next_zone; /* Link for all-zones list */ 140 const char *zone_name; /* a name for the zone */ 141#if ZONE_DEBUG 142 queue_head_t active_zones; /* active elements */ 143#endif /* ZONE_DEBUG */ 144 145#if CONFIG_ZLEAKS 146 uint32_t zleak_capture; /* per-zone counter for capturing every N allocations */ 147#endif /* CONFIG_ZLEAKS */ 148 uint32_t zp_count; /* counter for poisoning every N frees */ 149 vm_size_t prio_refill_watermark; 150 thread_t zone_replenish_thread; 151#if CONFIG_GZALLOC 152 gzalloc_data_t gz; 153#endif /* CONFIG_GZALLOC */ 154}; 155 156/* 157 * structure for tracking zone usage 158 * Used either one per task/thread for all zones or <per-task,per-zone>. 159 */ 160typedef struct zinfo_usage_store_t { 161 /* These fields may be updated atomically, and so must be 8 byte aligned */ 162 uint64_t alloc __attribute__((aligned(8))); /* allocation counter */ 163 uint64_t free __attribute__((aligned(8))); /* free counter */ 164} zinfo_usage_store_t; 165typedef zinfo_usage_store_t *zinfo_usage_t; 166 167extern void zone_gc(boolean_t); 168extern void consider_zone_gc(boolean_t); 169 170/* Steal memory for zone module */ 171extern void zone_steal_memory(void); 172 173/* Bootstrap zone module (create zone zone) */ 174extern void zone_bootstrap(void); 175 176/* Init zone module */ 177extern void zone_init( 178 vm_size_t map_size); 179 180/* Handle per-task zone info */ 181extern void zinfo_task_init(task_t task); 182extern void zinfo_task_free(task_t task); 183 184 185/* Stack use statistics */ 186extern void stack_fake_zone_init(int zone_index); 187extern void stack_fake_zone_info( 188 int *count, 189 vm_size_t *cur_size, 190 vm_size_t *max_size, 191 vm_size_t *elem_size, 192 vm_size_t *alloc_size, 193 uint64_t *sum_size, 194 int *collectable, 195 int *exhaustable, 196 int *caller_acct); 197 198#if ZONE_DEBUG 199 200extern void zone_debug_enable( 201 zone_t z); 202 203extern void zone_debug_disable( 204 zone_t z); 205 206#define zone_debug_enabled(z) z->active_zones.next 207#define ROUNDUP(x,y) ((((x)+(y)-1)/(y))*(y)) 208#define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16) 209#endif /* ZONE_DEBUG */ 210 211#endif /* MACH_KERNEL_PRIVATE */ 212 213__BEGIN_DECLS 214 215#ifdef XNU_KERNEL_PRIVATE 216 217/* Allocate from zone */ 218extern void * zalloc( 219 zone_t zone); 220 221/* Free zone element */ 222extern void zfree( 223 zone_t zone, 224 void *elem); 225 226/* Create zone */ 227extern zone_t zinit( 228 vm_size_t size, /* the size of an element */ 229 vm_size_t maxmem, /* maximum memory to use */ 230 vm_size_t alloc, /* allocation size */ 231 const char *name); /* a name for the zone */ 232 233 234/* Non-blocking version of zalloc */ 235extern void * zalloc_noblock( 236 zone_t zone); 237 238/* direct (non-wrappered) interface */ 239extern void * zalloc_canblock( 240 zone_t zone, 241 boolean_t canblock); 242 243/* Get from zone free list */ 244extern void * zget( 245 zone_t zone); 246 247/* Fill zone with memory */ 248extern void zcram( 249 zone_t zone, 250 vm_offset_t newmem, 251 vm_size_t size); 252 253/* Initially fill zone with specified number of elements */ 254extern int zfill( 255 zone_t zone, 256 int nelem); 257 258/* Change zone parameters */ 259extern void zone_change( 260 zone_t zone, 261 unsigned int item, 262 boolean_t value); 263extern void zone_prio_refill_configure(zone_t, vm_size_t); 264/* Item definitions */ 265#define Z_EXHAUST 1 /* Make zone exhaustible */ 266#define Z_COLLECT 2 /* Make zone collectable */ 267#define Z_EXPAND 3 /* Make zone expandable */ 268#define Z_FOREIGN 4 /* Allow collectable zone to contain foreign elements */ 269#define Z_CALLERACCT 5 /* Account alloc/free against the caller */ 270#define Z_NOENCRYPT 6 /* Don't encrypt zone during hibernation */ 271#define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via 272 * callouts 273 */ 274#define Z_ALIGNMENT_REQUIRED 8 275#define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */ 276 277/* Preallocate space for zone from zone map */ 278extern void zprealloc( 279 zone_t zone, 280 vm_size_t size); 281 282extern integer_t zone_free_count( 283 zone_t zone); 284 285/* 286 * MAX_ZTRACE_DEPTH configures how deep of a stack trace is taken on each zalloc in the zone of interest. 15 287 * levels is usually enough to get past all the layers of code in kalloc and IOKit and see who the actual 288 * caller is up above these lower levels. 289 * 290 * This is used both for the zone leak detector and the zone corruption log. 291 */ 292 293#define MAX_ZTRACE_DEPTH 15 294 295/* 296 * Structure for keeping track of a backtrace, used for leak detection. 297 * This is in the .h file because it is used during panic, see kern/debug.c 298 * A non-zero size indicates that the trace is in use. 299 */ 300struct ztrace { 301 vm_size_t zt_size; /* How much memory are all the allocations referring to this trace taking up? */ 302 uint32_t zt_depth; /* depth of stack (0 to MAX_ZTRACE_DEPTH) */ 303 void* zt_stack[MAX_ZTRACE_DEPTH]; /* series of return addresses from OSBacktrace */ 304 uint32_t zt_collisions; /* How many times did a different stack land here while it was occupied? */ 305 uint32_t zt_hit_count; /* for determining effectiveness of hash function */ 306}; 307 308#if CONFIG_ZLEAKS 309 310/* support for the kern.zleak.* sysctls */ 311 312extern kern_return_t zleak_activate(void); 313extern vm_size_t zleak_max_zonemap_size; 314extern vm_size_t zleak_global_tracking_threshold; 315extern vm_size_t zleak_per_zone_tracking_threshold; 316 317extern int get_zleak_state(void); 318 319#endif /* CONFIG_ZLEAKS */ 320 321/* These functions used for leak detection both in zalloc.c and mbuf.c */ 322extern uint32_t fastbacktrace(uintptr_t* bt, uint32_t max_frames) __attribute__((noinline)); 323extern uintptr_t hash_mix(uintptr_t); 324extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t); 325extern uint32_t hashaddr(uintptr_t, uint32_t); 326 327#define lock_zone(zone) \ 328MACRO_BEGIN \ 329 lck_mtx_lock_spin(&(zone)->lock); \ 330MACRO_END 331 332#define unlock_zone(zone) \ 333MACRO_BEGIN \ 334 lck_mtx_unlock(&(zone)->lock); \ 335MACRO_END 336 337#if CONFIG_GZALLOC 338void gzalloc_init(vm_size_t); 339void gzalloc_zone_init(zone_t); 340void gzalloc_configure(void); 341void gzalloc_reconfigure(zone_t); 342boolean_t gzalloc_enabled(void); 343 344vm_offset_t gzalloc_alloc(zone_t, boolean_t); 345boolean_t gzalloc_free(zone_t, void *); 346#endif /* CONFIG_GZALLOC */ 347 348#endif /* XNU_KERNEL_PRIVATE */ 349 350__END_DECLS 351 352#endif /* _KERN_ZALLOC_H_ */ 353 354#endif /* KERNEL_PRIVATE */ 355