1/* 2 * Copyright (c) 2006-2007 Pawel Jakub Dawidek <pjd@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD$"); 29 30#include <sys/types.h> 31#include <sys/param.h> 32#include <sys/byteorder.h> 33#include <sys/kernel.h> 34#include <sys/systm.h> 35#include <sys/malloc.h> 36#include <sys/kmem.h> 37#include <sys/kmem_cache.h> 38#include <sys/debug.h> 39#include <sys/mutex.h> 40#include <sys/vmmeter.h> 41 42 43#include <vm/vm_page.h> 44#include <vm/vm_object.h> 45#include <vm/vm_kern.h> 46#include <vm/vm_map.h> 47 48#ifdef KMEM_DEBUG 49#include <sys/queue.h> 50#include <sys/stack.h> 51#endif 52 53#ifdef _KERNEL 54MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris"); 55#else 56#define malloc(size, type, flags) malloc(size) 57#define free(addr, type) free(addr) 58#endif 59 60#ifdef KMEM_DEBUG 61struct kmem_item { 62 struct stack stack; 63 LIST_ENTRY(kmem_item) next; 64}; 65static LIST_HEAD(, kmem_item) kmem_items; 66static struct mtx kmem_items_mtx; 67MTX_SYSINIT(kmem_items_mtx, &kmem_items_mtx, "kmem_items", MTX_DEF); 68#endif /* KMEM_DEBUG */ 69 70#include <sys/vmem.h> 71 72void * 73zfs_kmem_alloc(size_t size, int kmflags) 74{ 75 void *p; 76#ifdef KMEM_DEBUG 77 struct kmem_item *i; 78 79 size += sizeof (struct kmem_item); 80#endif 81 p = malloc(MAX(size, 16), M_SOLARIS, kmflags); 82#ifndef _KERNEL 83 if (kmflags & KM_SLEEP) 84 assert(p != NULL); 85#endif 86#ifdef KMEM_DEBUG 87 if (p != NULL) { 88 i = p; 89 p = (uint8_t *)p + sizeof (struct kmem_item); 90 stack_save(&i->stack); 91 mtx_lock(&kmem_items_mtx); 92 LIST_INSERT_HEAD(&kmem_items, i, next); 93 mtx_unlock(&kmem_items_mtx); 94 } 95#endif 96 return (p); 97} 98 99void 100zfs_kmem_free(void *buf, size_t size __unused) 101{ 102#ifdef KMEM_DEBUG 103 if (buf == NULL) { 104 printf("%s: attempt to free NULL\n", __func__); 105 return; 106 } 107 struct kmem_item *i; 108 109 buf = (uint8_t *)buf - sizeof (struct kmem_item); 110 mtx_lock(&kmem_items_mtx); 111 LIST_FOREACH(i, &kmem_items, next) { 112 if (i == buf) 113 break; 114 } 115 ASSERT(i != NULL); 116 LIST_REMOVE(i, next); 117 mtx_unlock(&kmem_items_mtx); 118 memset(buf, 0xDC, MAX(size, 16)); 119#endif 120 free(buf, M_SOLARIS); 121} 122 123static uint64_t kmem_size_val; 124 125static void 126kmem_size_init(void *unused __unused) 127{ 128 129 kmem_size_val = (uint64_t)vm_cnt.v_page_count * PAGE_SIZE; 130 if (kmem_size_val > vm_kmem_size) 131 kmem_size_val = vm_kmem_size; 132} 133SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL); 134 135uint64_t 136kmem_size(void) 137{ 138 139 return (kmem_size_val); 140} 141 142static int 143kmem_std_constructor(void *mem, int size __unused, void *private, int flags) 144{ 145 struct kmem_cache *cache = private; 146 147 return (cache->kc_constructor(mem, cache->kc_private, flags)); 148} 149 150static void 151kmem_std_destructor(void *mem, int size __unused, void *private) 152{ 153 struct kmem_cache *cache = private; 154 155 cache->kc_destructor(mem, cache->kc_private); 156} 157 158kmem_cache_t * 159kmem_cache_create(char *name, size_t bufsize, size_t align, 160 int (*constructor)(void *, void *, int), void (*destructor)(void *, void *), 161 void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags) 162{ 163 kmem_cache_t *cache; 164 165 ASSERT(vmp == NULL); 166 167 cache = kmem_alloc(sizeof (*cache), KM_SLEEP); 168 strlcpy(cache->kc_name, name, sizeof (cache->kc_name)); 169 cache->kc_constructor = constructor; 170 cache->kc_destructor = destructor; 171 cache->kc_private = private; 172#if defined(_KERNEL) && !defined(KMEM_DEBUG) 173 cache->kc_zone = uma_zcreate(cache->kc_name, bufsize, 174 constructor != NULL ? kmem_std_constructor : NULL, 175 destructor != NULL ? kmem_std_destructor : NULL, 176 NULL, NULL, align > 0 ? align - 1 : 0, cflags); 177#else 178 cache->kc_size = bufsize; 179#endif 180 181 return (cache); 182} 183 184void 185kmem_cache_destroy(kmem_cache_t *cache) 186{ 187#if defined(_KERNEL) && !defined(KMEM_DEBUG) 188 uma_zdestroy(cache->kc_zone); 189#endif 190 kmem_free(cache, sizeof (*cache)); 191} 192 193void * 194kmem_cache_alloc(kmem_cache_t *cache, int flags) 195{ 196#if defined(_KERNEL) && !defined(KMEM_DEBUG) 197 return (uma_zalloc_arg(cache->kc_zone, cache, flags)); 198#else 199 void *p; 200 201 p = kmem_alloc(cache->kc_size, flags); 202 if (p != NULL && cache->kc_constructor != NULL) 203 kmem_std_constructor(p, cache->kc_size, cache, flags); 204 return (p); 205#endif 206} 207 208void 209kmem_cache_free(kmem_cache_t *cache, void *buf) 210{ 211#if defined(_KERNEL) && !defined(KMEM_DEBUG) 212 uma_zfree_arg(cache->kc_zone, buf, cache); 213#else 214 if (cache->kc_destructor != NULL) 215 kmem_std_destructor(buf, cache->kc_size, cache); 216 kmem_free(buf, cache->kc_size); 217#endif 218} 219 220/* 221 * Allow our caller to determine if there are running reaps. 222 * 223 * This call is very conservative and may return B_TRUE even when 224 * reaping activity isn't active. If it returns B_FALSE, then reaping 225 * activity is definitely inactive. 226 */ 227boolean_t 228kmem_cache_reap_active(void) 229{ 230 231 return (B_FALSE); 232} 233 234/* 235 * Reap (almost) everything soon. 236 * 237 * Note: this does not wait for the reap-tasks to complete. Caller 238 * should use kmem_cache_reap_active() (above) and/or moderation to 239 * avoid scheduling too many reap-tasks. 240 */ 241#ifdef _KERNEL 242void 243kmem_cache_reap_soon(kmem_cache_t *cache) 244{ 245#ifndef KMEM_DEBUG 246#if __FreeBSD_version >= 1300043 247 uma_zone_reclaim(cache->kc_zone, UMA_RECLAIM_DRAIN); 248#else 249 zone_drain(cache->kc_zone); 250#endif 251#endif 252} 253 254void 255kmem_reap(void) 256{ 257#if __FreeBSD_version >= 1300043 258 uma_reclaim(UMA_RECLAIM_TRIM); 259#else 260 uma_reclaim(); 261#endif 262} 263#else 264void 265kmem_cache_reap_soon(kmem_cache_t *cache __unused) 266{ 267} 268 269void 270kmem_reap(void) 271{ 272} 273#endif 274 275int 276kmem_debugging(void) 277{ 278 return (0); 279} 280 281void * 282calloc(size_t n, size_t s) 283{ 284 return (kmem_zalloc(n * s, KM_NOSLEEP)); 285} 286 287char * 288kmem_vasprintf(const char *fmt, va_list adx) 289{ 290 char *msg; 291 va_list adx2; 292 293 va_copy(adx2, adx); 294 msg = kmem_alloc(vsnprintf(NULL, 0, fmt, adx) + 1, KM_SLEEP); 295 (void) vsprintf(msg, fmt, adx2); 296 va_end(adx2); 297 298 return (msg); 299} 300 301#include <vm/uma.h> 302#include <vm/uma_int.h> 303#ifdef KMEM_DEBUG 304#error "KMEM_DEBUG not currently supported" 305#endif 306 307uint64_t 308spl_kmem_cache_inuse(kmem_cache_t *cache) 309{ 310 return (uma_zone_get_cur(cache->kc_zone)); 311} 312 313uint64_t 314spl_kmem_cache_entry_size(kmem_cache_t *cache) 315{ 316 return (cache->kc_zone->uz_size); 317} 318 319/* 320 * Register a move callback for cache defragmentation. 321 * XXX: Unimplemented but harmless to stub out for now. 322 */ 323void 324spl_kmem_cache_set_move(kmem_cache_t *skc, 325 kmem_cbrc_t (move)(void *, void *, size_t, void *)) 326{ 327 ASSERT(move != NULL); 328} 329 330#ifdef KMEM_DEBUG 331void kmem_show(void *); 332void 333kmem_show(void *dummy __unused) 334{ 335 struct kmem_item *i; 336 337 mtx_lock(&kmem_items_mtx); 338 if (LIST_EMPTY(&kmem_items)) 339 printf("KMEM_DEBUG: No leaked elements.\n"); 340 else { 341 printf("KMEM_DEBUG: Leaked elements:\n\n"); 342 LIST_FOREACH(i, &kmem_items, next) { 343 printf("address=%p\n", i); 344 stack_print_ddb(&i->stack); 345 printf("\n"); 346 } 347 } 348 mtx_unlock(&kmem_items_mtx); 349} 350 351SYSUNINIT(sol_kmem, SI_SUB_CPU, SI_ORDER_FIRST, kmem_show, NULL); 352#endif /* KMEM_DEBUG */ 353