1/* 2 * Copyright (c) 2006-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28#ifndef _SYS_MCACHE_H 29#define _SYS_MCACHE_H 30 31#ifdef KERNEL_PRIVATE 32 33#ifdef __cplusplus 34extern "C" { 35#endif 36 37#include <sys/types.h> 38#include <sys/queue.h> 39#include <mach/boolean.h> 40#include <kern/locks.h> 41 42#ifdef ASSERT 43#undef ASSERT 44#endif 45 46#ifdef VERIFY 47#undef VERIFY 48#endif 49 50/* 51 * Unlike VERIFY(), ASSERT() is evaluated only in DEBUG build. 52 */ 53#define VERIFY(EX) ((void)((EX) || assfail(#EX, __FILE__, __LINE__))) 54#if DEBUG 55#define ASSERT(EX) VERIFY(EX) 56#else 57#define ASSERT(EX) ((void)0) 58#endif 59 60#if defined(__ppc__) 61#define CPU_CACHE_SIZE 128 62#else 63#define CPU_CACHE_SIZE 64 64#endif 65 66#ifndef IS_P2ALIGNED 67#define IS_P2ALIGNED(v, a) \ 68 ((((uintptr_t)(v)) & ((uintptr_t)(a) - 1)) == 0) 69#endif /* IS_P2ALIGNED */ 70 71#ifndef P2ROUNDUP 72#define P2ROUNDUP(x, align) \ 73 (-(-((uintptr_t)(x)) & -(align))) 74#endif /* P2ROUNDUP */ 75 76#ifndef P2ROUNDDOWN 77#define P2ROUNDDOWN(x, align) \ 78 (((uintptr_t)(x)) & ~((uintptr_t)(align) - 1)) 79#endif /* P2ROUNDDOWN */ 80 81#define MCACHE_FREE_PATTERN 0xdeadbeefdeadbeefULL 82#define MCACHE_UNINITIALIZED_PATTERN 0xbaddcafebaddcafeULL 83 84/* 85 * mcache allocation request flags. 86 * 87 * MCR_NOSLEEP and MCR_FAILOK are mutually exclusive. The latter is used 88 * by the mbuf allocator to handle the implementation of several caches that 89 * involve multiple layers of mcache. It implies a best effort blocking 90 * allocation request; if the request cannot be satisfied, the caller will 91 * be blocked until further notice, similar to MCR_SLEEP, except that upon 92 * a wake up it will return immediately to the caller regardless of whether 93 * the request can been fulfilled. 94 * 95 * MCR_TRYHARD implies a non-blocking allocation request, regardless of 96 * whether MCR_NOSLEEP is set. It informs the allocator that the request 97 * should not cause the calling thread to block, and that it must have 98 * exhausted all possible schemes to fulfill the request, including doing 99 * reclaims and/or purges, before returning to the caller. 100 * 101 * Regular mcache clients should only use MCR_SLEEP or MCR_NOSLEEP. 102 */ 103#define MCR_SLEEP 0x0000 /* same as M_WAITOK */ 104#define MCR_NOSLEEP 0x0001 /* same as M_NOWAIT */ 105#define MCR_FAILOK 0x0100 /* private, for internal use only */ 106#define MCR_TRYHARD 0x0200 /* private, for internal use only */ 107#define MCR_USR1 0x1000 /* private, for internal use only */ 108 109#define MCR_NONBLOCKING (MCR_NOSLEEP | MCR_FAILOK | MCR_TRYHARD) 110 111/* 112 * Generic one-way linked list element structure. This is used to handle 113 * mcache_alloc_ext() requests in order to chain the allocated objects 114 * together before returning them to the caller. 115 */ 116typedef struct mcache_obj { 117 struct mcache_obj *obj_next; 118} mcache_obj_t; 119 120typedef struct mcache_bkt { 121 void *bkt_next; /* next bucket in list */ 122 void *bkt_obj[1]; /* one or more objects */ 123} mcache_bkt_t; 124 125typedef struct mcache_bktlist { 126 mcache_bkt_t *bl_list; /* bucket list */ 127 u_int32_t bl_total; /* number of buckets */ 128 u_int32_t bl_min; /* min since last update */ 129 u_int32_t bl_reaplimit; /* max reapable buckets */ 130 u_int64_t bl_alloc; /* allocations from this list */ 131} mcache_bktlist_t; 132 133typedef struct mcache_bkttype { 134 int bt_bktsize; /* bucket size (number of elements) */ 135 size_t bt_minbuf; /* all smaller buffers qualify */ 136 size_t bt_maxbuf; /* no larger bfufers qualify */ 137 struct mcache *bt_cache; /* bucket cache */ 138} mcache_bkttype_t; 139 140typedef struct mcache_cpu { 141 decl_lck_mtx_data(, cc_lock); 142 mcache_bkt_t *cc_filled; /* the currently filled bucket */ 143 mcache_bkt_t *cc_pfilled; /* the previously filled bucket */ 144 u_int64_t cc_alloc; /* allocations from this cpu */ 145 u_int64_t cc_free; /* frees to this cpu */ 146 int cc_objs; /* number of objects in filled bkt */ 147 int cc_pobjs; /* number of objects in previous bkt */ 148 int cc_bktsize; /* number of elements in a full bkt */ 149} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mcache_cpu_t; 150 151typedef unsigned int (*mcache_allocfn_t)(void *, mcache_obj_t ***, 152 unsigned int, int); 153typedef void (*mcache_freefn_t)(void *, mcache_obj_t *, boolean_t); 154typedef void (*mcache_auditfn_t)(void *, mcache_obj_t *, boolean_t); 155typedef void (*mcache_notifyfn_t)(void *, u_int32_t); 156 157typedef struct mcache { 158 /* 159 * Cache properties 160 */ 161 LIST_ENTRY(mcache) mc_list; /* cache linkage */ 162 char mc_name[32]; /* cache name */ 163 struct zone *mc_slab_zone; /* backend zone allocator */ 164 mcache_allocfn_t mc_slab_alloc; /* slab layer allocate callback */ 165 mcache_freefn_t mc_slab_free; /* slab layer free callback */ 166 mcache_auditfn_t mc_slab_audit; /* slab layer audit callback */ 167 mcache_notifyfn_t mc_slab_notify; /* slab layer notify callback */ 168 void *mc_private; /* opaque arg to callbacks */ 169 size_t mc_bufsize; /* object size */ 170 size_t mc_align; /* object alignment */ 171 u_int32_t mc_flags; /* cache creation flags */ 172 u_int32_t mc_purge_cnt; /* # of purges requested by slab */ 173 u_int32_t mc_enable_cnt; /* # of reenables due to purges */ 174 u_int32_t mc_waiter_cnt; /* # of slab layer waiters */ 175 u_int32_t mc_wretry_cnt; /* # of wait retries */ 176 u_int32_t mc_nwretry_cnt; /* # of no-wait retry attempts */ 177 u_int32_t mc_nwfail_cnt; /* # of no-wait retries that failed */ 178 decl_lck_mtx_data(, mc_sync_lock); /* protects purges and reenables */ 179 lck_attr_t *mc_sync_lock_attr; 180 lck_grp_t *mc_sync_lock_grp; 181 lck_grp_attr_t *mc_sync_lock_grp_attr; 182 /* 183 * Keep CPU and buckets layers lock statistics separate. 184 */ 185 lck_attr_t *mc_cpu_lock_attr; 186 lck_grp_t *mc_cpu_lock_grp; 187 lck_grp_attr_t *mc_cpu_lock_grp_attr; 188 189 /* 190 * Bucket layer common to all CPUs 191 */ 192 decl_lck_mtx_data(, mc_bkt_lock); 193 lck_attr_t *mc_bkt_lock_attr; 194 lck_grp_t *mc_bkt_lock_grp; 195 lck_grp_attr_t *mc_bkt_lock_grp_attr; 196 mcache_bkttype_t *cache_bkttype; /* bucket type */ 197 mcache_bktlist_t mc_full; /* full buckets */ 198 mcache_bktlist_t mc_empty; /* empty buckets */ 199 size_t mc_chunksize; /* bufsize + alignment */ 200 u_int32_t mc_bkt_contention; /* lock contention count */ 201 u_int32_t mc_bkt_contention_prev; /* previous snapshot */ 202 203 /* 204 * Per-CPU layer, aligned at cache line boundary 205 */ 206 mcache_cpu_t mc_cpu[1]; 207} mcache_t; 208 209#define MCACHE_ALIGN 8 /* default guaranteed alignment */ 210 211/* Valid values for mc_flags */ 212#define MCF_VERIFY 0x00000001 /* enable verification */ 213#define MCF_AUDIT 0x00000002 /* enable transaction auditing */ 214#define MCF_NOCPUCACHE 0x00000010 /* disable CPU layer caching */ 215 216#define MCF_DEBUG (MCF_VERIFY | MCF_AUDIT) 217#define MCF_FLAGS_MASK (MCF_DEBUG | MCF_NOCPUCACHE) 218 219/* Valid values for notify callback */ 220#define MCN_RETRYALLOC 0x00000001 /* Allocation should be retried */ 221 222#define MCACHE_STACK_DEPTH 16 223 224typedef struct mcache_audit { 225 struct mcache_audit *mca_next; /* next audit struct */ 226 void *mca_addr; /* address of buffer */ 227 mcache_t *mca_cache; /* parent cache of the buffer */ 228 struct thread *mca_thread; /* thread doing transaction */ 229 struct thread *mca_pthread; /* previous transaction thread */ 230 size_t mca_contents_size; /* size of contents */ 231 void *mca_contents; /* contents at last free */ 232 uint16_t mca_depth; /* pc stack depth */ 233 uint16_t mca_pdepth; /* previous transaction pc stack */ 234 void *mca_stack[MCACHE_STACK_DEPTH]; 235 void *mca_pstack[MCACHE_STACK_DEPTH]; 236 void *mca_uptr; /* user-specific pointer */ 237 uint32_t mca_uflags; /* user-specific flags */ 238} mcache_audit_t; 239 240__private_extern__ int assfail(const char *, const char *, int); 241__private_extern__ void mcache_init(void); 242__private_extern__ unsigned int mcache_getflags(void); 243__private_extern__ mcache_t *mcache_create(const char *, size_t, 244 size_t, u_int32_t, int); 245__private_extern__ void *mcache_alloc(mcache_t *, int); 246__private_extern__ void mcache_free(mcache_t *, void *); 247__private_extern__ mcache_t *mcache_create_ext(const char *, size_t, 248 mcache_allocfn_t, mcache_freefn_t, mcache_auditfn_t, mcache_notifyfn_t, 249 void *, u_int32_t, int); 250__private_extern__ void mcache_destroy(mcache_t *); 251__private_extern__ unsigned int mcache_alloc_ext(mcache_t *, mcache_obj_t **, 252 unsigned int, int); 253__private_extern__ void mcache_free_ext(mcache_t *, mcache_obj_t *); 254__private_extern__ void mcache_reap(void); 255__private_extern__ boolean_t mcache_purge_cache(mcache_t *); 256__private_extern__ void mcache_waiter_inc(mcache_t *); 257__private_extern__ void mcache_waiter_dec(mcache_t *); 258__private_extern__ boolean_t mcache_bkt_isempty(mcache_t *); 259 260__private_extern__ void mcache_buffer_log(mcache_audit_t *, void *, mcache_t *); 261__private_extern__ void mcache_set_pattern(u_int64_t, void *, size_t); 262__private_extern__ void *mcache_verify_pattern(u_int64_t, void *, size_t); 263__private_extern__ void *mcache_verify_set_pattern(u_int64_t, u_int64_t, 264 void *, size_t); 265__private_extern__ void mcache_audit_free_verify(mcache_audit_t *, 266 void *, size_t, size_t); 267__private_extern__ void mcache_audit_free_verify_set(mcache_audit_t *, 268 void *, size_t, size_t); 269__private_extern__ char *mcache_dump_mca(mcache_audit_t *); 270__private_extern__ void mcache_audit_panic(mcache_audit_t *, void *, size_t, 271 int64_t, int64_t); 272 273__private_extern__ mcache_t *mcache_audit_cache; 274 275#ifdef __cplusplus 276} 277#endif 278 279#endif /* KERNEL_PRIVATE */ 280 281#endif /* _SYS_MCACHE_H */ 282