1/* 2 * Copyright (c) 2006-2007 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/types.h> 28#include <sys/param.h> 29#include <sys/byteorder.h> 30#include <sys/kernel.h> 31#include <sys/systm.h> 32#include <sys/malloc.h> 33#include <sys/kmem.h> 34#include <sys/kmem_cache.h> 35#include <sys/debug.h> 36#include <sys/mutex.h> 37#include <sys/vmmeter.h> 38 39 40#include <vm/vm_page.h> 41#include <vm/vm_object.h> 42#include <vm/vm_kern.h> 43#include <vm/vm_map.h> 44 45#ifdef KMEM_DEBUG 46#include <sys/queue.h> 47#include <sys/stack.h> 48#endif 49 50#ifdef _KERNEL 51MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris"); 52#else 53#define malloc(size, type, flags) malloc(size) 54#define free(addr, type) free(addr) 55#endif 56 57#ifdef KMEM_DEBUG 58struct kmem_item { 59 struct stack stack; 60 LIST_ENTRY(kmem_item) next; 61}; 62static LIST_HEAD(, kmem_item) kmem_items; 63static struct mtx kmem_items_mtx; 64MTX_SYSINIT(kmem_items_mtx, &kmem_items_mtx, "kmem_items", MTX_DEF); 65#endif /* KMEM_DEBUG */ 66 67#include <sys/vmem.h> 68 69void * 70zfs_kmem_alloc(size_t size, int kmflags) 71{ 72 void *p; 73#ifdef KMEM_DEBUG 74 struct kmem_item *i; 75 76 size += sizeof (struct kmem_item); 77#endif 78 p = malloc(MAX(size, 16), M_SOLARIS, kmflags); 79#ifndef _KERNEL 80 if (kmflags & KM_SLEEP) 81 assert(p != NULL); 82#endif 83#ifdef KMEM_DEBUG 84 if (p != NULL) { 85 i = p; 86 p = (uint8_t *)p + sizeof (struct kmem_item); 87 stack_save(&i->stack); 88 mtx_lock(&kmem_items_mtx); 89 LIST_INSERT_HEAD(&kmem_items, i, next); 90 mtx_unlock(&kmem_items_mtx); 91 } 92#endif 93 return (p); 94} 95 96void 97zfs_kmem_free(void *buf, size_t size __unused) 98{ 99#ifdef KMEM_DEBUG 100 if (buf == NULL) { 101 printf("%s: attempt to free NULL\n", __func__); 102 return; 103 } 104 struct kmem_item *i; 105 106 buf = (uint8_t *)buf - sizeof (struct kmem_item); 107 mtx_lock(&kmem_items_mtx); 108 LIST_FOREACH(i, &kmem_items, next) { 109 if (i == buf) 110 break; 111 } 112 ASSERT3P(i, !=, NULL); 113 LIST_REMOVE(i, next); 114 mtx_unlock(&kmem_items_mtx); 115 memset(buf, 0xDC, MAX(size, 16)); 116#endif 117 free(buf, M_SOLARIS); 118} 119 120static uint64_t kmem_size_val; 121 122static void 123kmem_size_init(void *unused __unused) 124{ 125 126 kmem_size_val = (uint64_t)vm_cnt.v_page_count * PAGE_SIZE; 127 if (kmem_size_val > vm_kmem_size) 128 kmem_size_val = vm_kmem_size; 129} 130SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL); 131 132uint64_t 133kmem_size(void) 134{ 135 136 return (kmem_size_val); 137} 138 139static int 140kmem_std_constructor(void *mem, int size __unused, void *private, int flags) 141{ 142 struct kmem_cache *cache = private; 143 144 return (cache->kc_constructor(mem, cache->kc_private, flags)); 145} 146 147static void 148kmem_std_destructor(void *mem, int size __unused, void *private) 149{ 150 struct kmem_cache *cache = private; 151 152 cache->kc_destructor(mem, cache->kc_private); 153} 154 155kmem_cache_t * 156kmem_cache_create(const char *name, size_t bufsize, size_t align, 157 int (*constructor)(void *, void *, int), void (*destructor)(void *, void *), 158 void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags) 159{ 160 kmem_cache_t *cache; 161 162 ASSERT3P(vmp, ==, NULL); 163 164 cache = kmem_alloc(sizeof (*cache), KM_SLEEP); 165 strlcpy(cache->kc_name, name, sizeof (cache->kc_name)); 166 cache->kc_constructor = constructor; 167 cache->kc_destructor = destructor; 168 cache->kc_private = private; 169#if defined(_KERNEL) && !defined(KMEM_DEBUG) 170 cache->kc_zone = uma_zcreate(cache->kc_name, bufsize, 171 constructor != NULL ? kmem_std_constructor : NULL, 172 destructor != NULL ? kmem_std_destructor : NULL, 173 NULL, NULL, align > 0 ? align - 1 : 0, cflags); 174#else 175 cache->kc_size = bufsize; 176#endif 177 178 return (cache); 179} 180 181void 182kmem_cache_destroy(kmem_cache_t *cache) 183{ 184#if defined(_KERNEL) && !defined(KMEM_DEBUG) 185 uma_zdestroy(cache->kc_zone); 186#endif 187 kmem_free(cache, sizeof (*cache)); 188} 189 190void * 191kmem_cache_alloc(kmem_cache_t *cache, int flags) 192{ 193#if defined(_KERNEL) && !defined(KMEM_DEBUG) 194 return (uma_zalloc_arg(cache->kc_zone, cache, flags)); 195#else 196 void *p; 197 198 p = kmem_alloc(cache->kc_size, flags); 199 if (p != NULL && cache->kc_constructor != NULL) 200 kmem_std_constructor(p, cache->kc_size, cache, flags); 201 return (p); 202#endif 203} 204 205void 206kmem_cache_free(kmem_cache_t *cache, void *buf) 207{ 208#if defined(_KERNEL) && !defined(KMEM_DEBUG) 209 uma_zfree_arg(cache->kc_zone, buf, cache); 210#else 211 if (cache->kc_destructor != NULL) 212 kmem_std_destructor(buf, cache->kc_size, cache); 213 kmem_free(buf, cache->kc_size); 214#endif 215} 216 217/* 218 * Allow our caller to determine if there are running reaps. 219 * 220 * This call is very conservative and may return B_TRUE even when 221 * reaping activity isn't active. If it returns B_FALSE, then reaping 222 * activity is definitely inactive. 223 */ 224boolean_t 225kmem_cache_reap_active(void) 226{ 227 228 return (B_FALSE); 229} 230 231/* 232 * Reap (almost) everything soon. 233 * 234 * Note: this does not wait for the reap-tasks to complete. Caller 235 * should use kmem_cache_reap_active() (above) and/or moderation to 236 * avoid scheduling too many reap-tasks. 237 */ 238#ifdef _KERNEL 239void 240kmem_cache_reap_soon(kmem_cache_t *cache) 241{ 242#ifndef KMEM_DEBUG 243#if __FreeBSD_version >= 1300043 244 uma_zone_reclaim(cache->kc_zone, UMA_RECLAIM_DRAIN); 245#else 246 zone_drain(cache->kc_zone); 247#endif 248#endif 249} 250 251void 252kmem_reap(void) 253{ 254#if __FreeBSD_version >= 1300043 255 uma_reclaim(UMA_RECLAIM_TRIM); 256#else 257 uma_reclaim(); 258#endif 259} 260#else 261void 262kmem_cache_reap_soon(kmem_cache_t *cache __unused) 263{ 264} 265 266void 267kmem_reap(void) 268{ 269} 270#endif 271 272int 273kmem_debugging(void) 274{ 275 return (0); 276} 277 278void * 279calloc(size_t n, size_t s) 280{ 281 return (kmem_zalloc(n * s, KM_NOSLEEP)); 282} 283 284char * 285kmem_vasprintf(const char *fmt, va_list adx) 286{ 287 char *msg; 288 va_list adx2; 289 290 va_copy(adx2, adx); 291 msg = kmem_alloc(vsnprintf(NULL, 0, fmt, adx) + 1, KM_SLEEP); 292 (void) vsprintf(msg, fmt, adx2); 293 va_end(adx2); 294 295 return (msg); 296} 297 298#include <vm/uma.h> 299#include <vm/uma_int.h> 300#ifdef KMEM_DEBUG 301#error "KMEM_DEBUG not currently supported" 302#endif 303 304uint64_t 305spl_kmem_cache_inuse(kmem_cache_t *cache) 306{ 307 return (uma_zone_get_cur(cache->kc_zone)); 308} 309 310uint64_t 311spl_kmem_cache_entry_size(kmem_cache_t *cache) 312{ 313 return (cache->kc_zone->uz_size); 314} 315 316/* 317 * Register a move callback for cache defragmentation. 318 * XXX: Unimplemented but harmless to stub out for now. 319 */ 320void 321spl_kmem_cache_set_move(kmem_cache_t *skc, 322 kmem_cbrc_t (move)(void *, void *, size_t, void *)) 323{ 324 ASSERT3P(move, !=, NULL); 325} 326 327#ifdef KMEM_DEBUG 328void kmem_show(void *); 329void 330kmem_show(void *dummy __unused) 331{ 332 struct kmem_item *i; 333 334 mtx_lock(&kmem_items_mtx); 335 if (LIST_EMPTY(&kmem_items)) 336 printf("KMEM_DEBUG: No leaked elements.\n"); 337 else { 338 printf("KMEM_DEBUG: Leaked elements:\n\n"); 339 LIST_FOREACH(i, &kmem_items, next) { 340 printf("address=%p\n", i); 341 stack_print_ddb(&i->stack); 342 printf("\n"); 343 } 344 } 345 mtx_unlock(&kmem_items_mtx); 346} 347 348SYSUNINIT(sol_kmem, SI_SUB_CPU, SI_ORDER_FIRST, kmem_show, NULL); 349#endif /* KMEM_DEBUG */ 350