1/* 2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58/* 59 * File: zalloc.h 60 * Author: Avadis Tevanian, Jr. 61 * Date: 1985 62 * 63 */ 64 65#ifdef KERNEL_PRIVATE 66 67#ifndef _KERN_ZALLOC_H_ 68#define _KERN_ZALLOC_H_ 69 70#include <mach/machine/vm_types.h> 71#include <kern/kern_types.h> 72#include <sys/cdefs.h> 73 74#ifdef MACH_KERNEL_PRIVATE 75 76#include <zone_debug.h> 77#include <kern/locks.h> 78#include <kern/queue.h> 79#include <kern/thread_call.h> 80 81#if CONFIG_GZALLOC 82typedef struct gzalloc_data { 83 uint32_t gzfc_index; 84 vm_offset_t *gzfc; 85} gzalloc_data_t; 86#endif 87 88/* 89 * A zone is a collection of fixed size blocks for which there 90 * is fast allocation/deallocation access. Kernel routines can 91 * use zones to manage data structures dynamically, creating a zone 92 * for each type of data structure to be managed. 93 * 94 */ 95 96struct zone_free_element; 97struct zone_page_metadata; 98 99struct zone { 100 struct zone_free_element *free_elements; /* free elements directly linked */ 101 struct { 102 queue_head_t any_free_foreign; /* foreign pages crammed into zone */ 103 queue_head_t all_free; 104 queue_head_t intermediate; 105 queue_head_t all_used; 106 } pages; /* list of zone_page_metadata structs, which maintain per-page free element lists */ 107 int count; /* Number of elements used now */ 108 int countfree; /* Number of free elements */ 109 lck_attr_t lock_attr; /* zone lock attribute */ 110 decl_lck_mtx_data(,lock) /* zone lock */ 111 lck_mtx_ext_t lock_ext; /* placeholder for indirect mutex */ 112 vm_size_t cur_size; /* current memory utilization */ 113 vm_size_t max_size; /* how large can this zone grow */ 114 vm_size_t elem_size; /* size of an element */ 115 vm_size_t alloc_size; /* size used for more memory */ 116 uint64_t page_count __attribute__((aligned(8))); /* number of pages used by this zone */ 117 uint64_t sum_count; /* count of allocs (life of zone) */ 118 uint32_t 119 /* boolean_t */ exhaustible :1, /* (F) merely return if empty? */ 120 /* boolean_t */ collectable :1, /* (F) garbage collect empty pages */ 121 /* boolean_t */ expandable :1, /* (T) expand zone (with message)? */ 122 /* boolean_t */ allows_foreign :1, /* (F) allow non-zalloc space */ 123 /* boolean_t */ doing_alloc :1, /* is zone expanding now? */ 124 /* boolean_t */ waiting :1, /* is thread waiting for expansion? */ 125 /* boolean_t */ async_pending :1, /* asynchronous allocation pending? */ 126 /* boolean_t */ zleak_on :1, /* Are we collecting allocation information? */ 127 /* boolean_t */ caller_acct :1, /* do we account allocation/free to the caller? */ 128 /* boolean_t */ doing_gc :1, /* garbage collect in progress? */ 129 /* boolean_t */ noencrypt :1, 130 /* boolean_t */ no_callout :1, 131 /* boolean_t */ async_prio_refill :1, 132 /* boolean_t */ gzalloc_exempt :1, 133 /* boolean_t */ alignment_required :1, 134 /* boolean_t */ use_page_list :1, 135 /* future */ _reserved :16; 136 137 int index; /* index into zone_info arrays for this zone */ 138 struct zone *next_zone; /* Link for all-zones list */ 139 const char *zone_name; /* a name for the zone */ 140#if ZONE_DEBUG 141 queue_head_t active_zones; /* active elements */ 142#endif /* ZONE_DEBUG */ 143 144#if CONFIG_ZLEAKS 145 uint32_t zleak_capture; /* per-zone counter for capturing every N allocations */ 146#endif /* CONFIG_ZLEAKS */ 147 uint32_t zp_count; /* counter for poisoning every N frees */ 148 vm_size_t prio_refill_watermark; 149 thread_t zone_replenish_thread; 150#if CONFIG_GZALLOC 151 gzalloc_data_t gz; 152#endif /* CONFIG_GZALLOC */ 153}; 154 155/* 156 * structure for tracking zone usage 157 * Used either one per task/thread for all zones or <per-task,per-zone>. 158 */ 159typedef struct zinfo_usage_store_t { 160 /* These fields may be updated atomically, and so must be 8 byte aligned */ 161 uint64_t alloc __attribute__((aligned(8))); /* allocation counter */ 162 uint64_t free __attribute__((aligned(8))); /* free counter */ 163} zinfo_usage_store_t; 164typedef zinfo_usage_store_t *zinfo_usage_t; 165 166extern void zone_gc(boolean_t); 167extern void consider_zone_gc(boolean_t); 168 169/* Steal memory for zone module */ 170extern void zone_steal_memory(void); 171 172/* Bootstrap zone module (create zone zone) */ 173extern void zone_bootstrap(void); 174 175/* Init zone module */ 176extern void zone_init( 177 vm_size_t map_size); 178 179/* Handle per-task zone info */ 180extern void zinfo_task_init(task_t task); 181extern void zinfo_task_free(task_t task); 182 183 184/* Stack use statistics */ 185extern void stack_fake_zone_init(int zone_index); 186extern void stack_fake_zone_info( 187 int *count, 188 vm_size_t *cur_size, 189 vm_size_t *max_size, 190 vm_size_t *elem_size, 191 vm_size_t *alloc_size, 192 uint64_t *sum_size, 193 int *collectable, 194 int *exhaustable, 195 int *caller_acct); 196 197#if ZONE_DEBUG 198 199extern void zone_debug_enable( 200 zone_t z); 201 202extern void zone_debug_disable( 203 zone_t z); 204 205#define zone_debug_enabled(z) z->active_zones.next 206#define ROUNDUP(x,y) ((((x)+(y)-1)/(y))*(y)) 207#define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16) 208#endif /* ZONE_DEBUG */ 209 210#endif /* MACH_KERNEL_PRIVATE */ 211 212__BEGIN_DECLS 213 214#ifdef XNU_KERNEL_PRIVATE 215 216extern vm_offset_t zone_map_min_address; 217extern vm_offset_t zone_map_max_address; 218 219 220/* Allocate from zone */ 221extern void * zalloc( 222 zone_t zone); 223 224/* Free zone element */ 225extern void zfree( 226 zone_t zone, 227 void *elem); 228 229/* Create zone */ 230extern zone_t zinit( 231 vm_size_t size, /* the size of an element */ 232 vm_size_t maxmem, /* maximum memory to use */ 233 vm_size_t alloc, /* allocation size */ 234 const char *name); /* a name for the zone */ 235 236 237/* Non-waiting for memory version of zalloc */ 238extern void * zalloc_nopagewait( 239 zone_t zone); 240 241/* Non-blocking version of zalloc */ 242extern void * zalloc_noblock( 243 zone_t zone); 244 245/* selective version of zalloc */ 246extern void * zalloc_canblock( 247 zone_t zone, 248 boolean_t canblock); 249 250/* Get from zone free list */ 251extern void * zget( 252 zone_t zone); 253 254/* Fill zone with memory */ 255extern void zcram( 256 zone_t zone, 257 vm_offset_t newmem, 258 vm_size_t size); 259 260/* Initially fill zone with specified number of elements */ 261extern int zfill( 262 zone_t zone, 263 int nelem); 264 265/* Change zone parameters */ 266extern void zone_change( 267 zone_t zone, 268 unsigned int item, 269 boolean_t value); 270extern void zone_prio_refill_configure(zone_t, vm_size_t); 271/* Item definitions */ 272#define Z_EXHAUST 1 /* Make zone exhaustible */ 273#define Z_COLLECT 2 /* Make zone collectable */ 274#define Z_EXPAND 3 /* Make zone expandable */ 275#define Z_FOREIGN 4 /* Allow collectable zone to contain foreign elements */ 276#define Z_CALLERACCT 5 /* Account alloc/free against the caller */ 277#define Z_NOENCRYPT 6 /* Don't encrypt zone during hibernation */ 278#define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via 279 * callouts 280 */ 281#define Z_ALIGNMENT_REQUIRED 8 282#define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */ 283 284/* Preallocate space for zone from zone map */ 285extern void zprealloc( 286 zone_t zone, 287 vm_size_t size); 288 289extern integer_t zone_free_count( 290 zone_t zone); 291 292/* 293 * MAX_ZTRACE_DEPTH configures how deep of a stack trace is taken on each zalloc in the zone of interest. 15 294 * levels is usually enough to get past all the layers of code in kalloc and IOKit and see who the actual 295 * caller is up above these lower levels. 296 * 297 * This is used both for the zone leak detector and the zone corruption log. 298 */ 299 300#define MAX_ZTRACE_DEPTH 15 301 302/* 303 * Structure for keeping track of a backtrace, used for leak detection. 304 * This is in the .h file because it is used during panic, see kern/debug.c 305 * A non-zero size indicates that the trace is in use. 306 */ 307struct ztrace { 308 vm_size_t zt_size; /* How much memory are all the allocations referring to this trace taking up? */ 309 uint32_t zt_depth; /* depth of stack (0 to MAX_ZTRACE_DEPTH) */ 310 void* zt_stack[MAX_ZTRACE_DEPTH]; /* series of return addresses from OSBacktrace */ 311 uint32_t zt_collisions; /* How many times did a different stack land here while it was occupied? */ 312 uint32_t zt_hit_count; /* for determining effectiveness of hash function */ 313}; 314 315#if CONFIG_ZLEAKS 316 317/* support for the kern.zleak.* sysctls */ 318 319extern kern_return_t zleak_activate(void); 320extern vm_size_t zleak_max_zonemap_size; 321extern vm_size_t zleak_global_tracking_threshold; 322extern vm_size_t zleak_per_zone_tracking_threshold; 323 324extern int get_zleak_state(void); 325 326#endif /* CONFIG_ZLEAKS */ 327 328/* These functions used for leak detection both in zalloc.c and mbuf.c */ 329extern uint32_t fastbacktrace(uintptr_t* bt, uint32_t max_frames) __attribute__((noinline)); 330extern uintptr_t hash_mix(uintptr_t); 331extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t); 332extern uint32_t hashaddr(uintptr_t, uint32_t); 333 334#define lock_zone(zone) \ 335MACRO_BEGIN \ 336 lck_mtx_lock_spin(&(zone)->lock); \ 337MACRO_END 338 339#define unlock_zone(zone) \ 340MACRO_BEGIN \ 341 lck_mtx_unlock(&(zone)->lock); \ 342MACRO_END 343 344#if CONFIG_GZALLOC 345void gzalloc_init(vm_size_t); 346void gzalloc_zone_init(zone_t); 347void gzalloc_configure(void); 348void gzalloc_reconfigure(zone_t); 349boolean_t gzalloc_enabled(void); 350 351vm_offset_t gzalloc_alloc(zone_t, boolean_t); 352boolean_t gzalloc_free(zone_t, void *); 353#endif /* CONFIG_GZALLOC */ 354 355#endif /* XNU_KERNEL_PRIVATE */ 356 357__END_DECLS 358 359#endif /* _KERN_ZALLOC_H_ */ 360 361#endif /* KERNEL_PRIVATE */ 362