1/*
2 * Copyright (c) 2006-2007 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/types.h>
28#include <sys/param.h>
29#include <sys/byteorder.h>
30#include <sys/kernel.h>
31#include <sys/systm.h>
32#include <sys/malloc.h>
33#include <sys/kmem.h>
34#include <sys/kmem_cache.h>
35#include <sys/debug.h>
36#include <sys/mutex.h>
37#include <sys/vmmeter.h>
38
39
40#include <vm/vm_page.h>
41#include <vm/vm_object.h>
42#include <vm/vm_kern.h>
43#include <vm/vm_map.h>
44
45#ifdef KMEM_DEBUG
46#include <sys/queue.h>
47#include <sys/stack.h>
48#endif
49
50#ifdef _KERNEL
51MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris");
52#else
53#define	malloc(size, type, flags)	malloc(size)
54#define	free(addr, type)		free(addr)
55#endif
56
57#ifdef KMEM_DEBUG
58struct kmem_item {
59	struct stack	stack;
60	LIST_ENTRY(kmem_item) next;
61};
62static LIST_HEAD(, kmem_item) kmem_items;
63static struct mtx kmem_items_mtx;
64MTX_SYSINIT(kmem_items_mtx, &kmem_items_mtx, "kmem_items", MTX_DEF);
65#endif	/* KMEM_DEBUG */
66
67#include <sys/vmem.h>
68
69void *
70zfs_kmem_alloc(size_t size, int kmflags)
71{
72	void *p;
73#ifdef KMEM_DEBUG
74	struct kmem_item *i;
75
76	size += sizeof (struct kmem_item);
77#endif
78	p = malloc(MAX(size, 16), M_SOLARIS, kmflags);
79#ifndef _KERNEL
80	if (kmflags & KM_SLEEP)
81		assert(p != NULL);
82#endif
83#ifdef KMEM_DEBUG
84	if (p != NULL) {
85		i = p;
86		p = (uint8_t *)p + sizeof (struct kmem_item);
87		stack_save(&i->stack);
88		mtx_lock(&kmem_items_mtx);
89		LIST_INSERT_HEAD(&kmem_items, i, next);
90		mtx_unlock(&kmem_items_mtx);
91	}
92#endif
93	return (p);
94}
95
96void
97zfs_kmem_free(void *buf, size_t size __unused)
98{
99#ifdef KMEM_DEBUG
100	if (buf == NULL) {
101		printf("%s: attempt to free NULL\n", __func__);
102		return;
103	}
104	struct kmem_item *i;
105
106	buf = (uint8_t *)buf - sizeof (struct kmem_item);
107	mtx_lock(&kmem_items_mtx);
108	LIST_FOREACH(i, &kmem_items, next) {
109		if (i == buf)
110			break;
111	}
112	ASSERT3P(i, !=, NULL);
113	LIST_REMOVE(i, next);
114	mtx_unlock(&kmem_items_mtx);
115	memset(buf, 0xDC, MAX(size, 16));
116#endif
117	free(buf, M_SOLARIS);
118}
119
120static uint64_t kmem_size_val;
121
122static void
123kmem_size_init(void *unused __unused)
124{
125
126	kmem_size_val = (uint64_t)vm_cnt.v_page_count * PAGE_SIZE;
127	if (kmem_size_val > vm_kmem_size)
128		kmem_size_val = vm_kmem_size;
129}
130SYSINIT(kmem_size_init, SI_SUB_KMEM, SI_ORDER_ANY, kmem_size_init, NULL);
131
132uint64_t
133kmem_size(void)
134{
135
136	return (kmem_size_val);
137}
138
139static int
140kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
141{
142	struct kmem_cache *cache = private;
143
144	return (cache->kc_constructor(mem, cache->kc_private, flags));
145}
146
147static void
148kmem_std_destructor(void *mem, int size __unused, void *private)
149{
150	struct kmem_cache *cache = private;
151
152	cache->kc_destructor(mem, cache->kc_private);
153}
154
155kmem_cache_t *
156kmem_cache_create(const char *name, size_t bufsize, size_t align,
157    int (*constructor)(void *, void *, int), void (*destructor)(void *, void *),
158    void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags)
159{
160	kmem_cache_t *cache;
161
162	ASSERT3P(vmp, ==, NULL);
163
164	cache = kmem_alloc(sizeof (*cache), KM_SLEEP);
165	strlcpy(cache->kc_name, name, sizeof (cache->kc_name));
166	cache->kc_constructor = constructor;
167	cache->kc_destructor = destructor;
168	cache->kc_private = private;
169#if defined(_KERNEL) && !defined(KMEM_DEBUG)
170	cache->kc_zone = uma_zcreate(cache->kc_name, bufsize,
171	    constructor != NULL ? kmem_std_constructor : NULL,
172	    destructor != NULL ? kmem_std_destructor : NULL,
173	    NULL, NULL, align > 0 ? align - 1 : 0, cflags);
174#else
175	cache->kc_size = bufsize;
176#endif
177
178	return (cache);
179}
180
181void
182kmem_cache_destroy(kmem_cache_t *cache)
183{
184#if defined(_KERNEL) && !defined(KMEM_DEBUG)
185	uma_zdestroy(cache->kc_zone);
186#endif
187	kmem_free(cache, sizeof (*cache));
188}
189
190void *
191kmem_cache_alloc(kmem_cache_t *cache, int flags)
192{
193#if defined(_KERNEL) && !defined(KMEM_DEBUG)
194	return (uma_zalloc_arg(cache->kc_zone, cache, flags));
195#else
196	void *p;
197
198	p = kmem_alloc(cache->kc_size, flags);
199	if (p != NULL && cache->kc_constructor != NULL)
200		kmem_std_constructor(p, cache->kc_size, cache, flags);
201	return (p);
202#endif
203}
204
205void
206kmem_cache_free(kmem_cache_t *cache, void *buf)
207{
208#if defined(_KERNEL) && !defined(KMEM_DEBUG)
209	uma_zfree_arg(cache->kc_zone, buf, cache);
210#else
211	if (cache->kc_destructor != NULL)
212		kmem_std_destructor(buf, cache->kc_size, cache);
213	kmem_free(buf, cache->kc_size);
214#endif
215}
216
217/*
218 * Allow our caller to determine if there are running reaps.
219 *
220 * This call is very conservative and may return B_TRUE even when
221 * reaping activity isn't active. If it returns B_FALSE, then reaping
222 * activity is definitely inactive.
223 */
224boolean_t
225kmem_cache_reap_active(void)
226{
227
228	return (B_FALSE);
229}
230
231/*
232 * Reap (almost) everything soon.
233 *
234 * Note: this does not wait for the reap-tasks to complete. Caller
235 * should use kmem_cache_reap_active() (above) and/or moderation to
236 * avoid scheduling too many reap-tasks.
237 */
238#ifdef _KERNEL
239void
240kmem_cache_reap_soon(kmem_cache_t *cache)
241{
242#ifndef KMEM_DEBUG
243#if __FreeBSD_version >= 1300043
244	uma_zone_reclaim(cache->kc_zone, UMA_RECLAIM_DRAIN);
245#else
246	zone_drain(cache->kc_zone);
247#endif
248#endif
249}
250
251void
252kmem_reap(void)
253{
254#if __FreeBSD_version >= 1300043
255	uma_reclaim(UMA_RECLAIM_TRIM);
256#else
257	uma_reclaim();
258#endif
259}
260#else
261void
262kmem_cache_reap_soon(kmem_cache_t *cache __unused)
263{
264}
265
266void
267kmem_reap(void)
268{
269}
270#endif
271
272int
273kmem_debugging(void)
274{
275	return (0);
276}
277
278void *
279calloc(size_t n, size_t s)
280{
281	return (kmem_zalloc(n * s, KM_NOSLEEP));
282}
283
284char *
285kmem_vasprintf(const char *fmt, va_list adx)
286{
287	char *msg;
288	va_list adx2;
289
290	va_copy(adx2, adx);
291	msg = kmem_alloc(vsnprintf(NULL, 0, fmt, adx) + 1, KM_SLEEP);
292	(void) vsprintf(msg, fmt, adx2);
293	va_end(adx2);
294
295	return (msg);
296}
297
298#include <vm/uma.h>
299#include <vm/uma_int.h>
300#ifdef KMEM_DEBUG
301#error "KMEM_DEBUG not currently supported"
302#endif
303
304uint64_t
305spl_kmem_cache_inuse(kmem_cache_t *cache)
306{
307	return (uma_zone_get_cur(cache->kc_zone));
308}
309
310uint64_t
311spl_kmem_cache_entry_size(kmem_cache_t *cache)
312{
313	return (cache->kc_zone->uz_size);
314}
315
316/*
317 * Register a move callback for cache defragmentation.
318 * XXX: Unimplemented but harmless to stub out for now.
319 */
320void
321spl_kmem_cache_set_move(kmem_cache_t *skc,
322    kmem_cbrc_t (move)(void *, void *, size_t, void *))
323{
324	ASSERT3P(move, !=, NULL);
325}
326
327#ifdef KMEM_DEBUG
328void kmem_show(void *);
329void
330kmem_show(void *dummy __unused)
331{
332	struct kmem_item *i;
333
334	mtx_lock(&kmem_items_mtx);
335	if (LIST_EMPTY(&kmem_items))
336		printf("KMEM_DEBUG: No leaked elements.\n");
337	else {
338		printf("KMEM_DEBUG: Leaked elements:\n\n");
339		LIST_FOREACH(i, &kmem_items, next) {
340			printf("address=%p\n", i);
341			stack_print_ddb(&i->stack);
342			printf("\n");
343		}
344	}
345	mtx_unlock(&kmem_items_mtx);
346}
347
348SYSUNINIT(sol_kmem, SI_SUB_CPU, SI_ORDER_FIRST, kmem_show, NULL);
349#endif	/* KMEM_DEBUG */
350