1/*-
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * uma_core.c Implementation of the Universal Memory allocator
31 *
32 * This allocator is intended to replace the multitude of similar object caches
33 * in the standard FreeBSD kernel. The intent is to be flexible as well as
34 * effecient. A primary design goal is to return unused memory to the rest of
35 * the system. This will make the system as a whole more flexible due to the
36 * ability to move memory to subsystems which most need it instead of leaving
37 * pools of reserved memory unused.
38 *
39 * The basic ideas stem from similar slab/zone based allocators whose algorithms
40 * are well known.
41 *
42 */
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
| 1/*-
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * uma_core.c Implementation of the Universal Memory allocator
31 *
32 * This allocator is intended to replace the multitude of similar object caches
33 * in the standard FreeBSD kernel. The intent is to be flexible as well as
34 * effecient. A primary design goal is to return unused memory to the rest of
35 * the system. This will make the system as a whole more flexible due to the
36 * ability to move memory to subsystems which most need it instead of leaving
37 * pools of reserved memory unused.
38 *
39 * The basic ideas stem from similar slab/zone based allocators whose algorithms
40 * are well known.
41 *
42 */
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
|
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251709 2013-06-13 21:05:38Z jeff $");
| 51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $");
|
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
60#include "opt_ddb.h"
61#include "opt_param.h"
62#include "opt_vm.h"
63
64#include <sys/param.h>
65#include <sys/systm.h>
66#include <sys/bitset.h>
67#include <sys/kernel.h>
68#include <sys/types.h>
69#include <sys/queue.h>
70#include <sys/malloc.h>
71#include <sys/ktr.h>
72#include <sys/lock.h>
73#include <sys/sysctl.h>
74#include <sys/mutex.h>
75#include <sys/proc.h>
76#include <sys/rwlock.h>
77#include <sys/sbuf.h>
78#include <sys/smp.h>
79#include <sys/vmmeter.h>
80
81#include <vm/vm.h>
82#include <vm/vm_object.h>
83#include <vm/vm_page.h>
84#include <vm/vm_pageout.h>
85#include <vm/vm_param.h>
86#include <vm/vm_map.h>
87#include <vm/vm_kern.h>
88#include <vm/vm_extern.h>
89#include <vm/uma.h>
90#include <vm/uma_int.h>
91#include <vm/uma_dbg.h>
92
93#include <ddb/ddb.h>
94
95#ifdef DEBUG_MEMGUARD
96#include <vm/memguard.h>
97#endif
98
99/*
100 * This is the zone and keg from which all zones are spawned. The idea is that
101 * even the zone & keg heads are allocated from the allocator, so we use the
102 * bss section to bootstrap us.
103 */
104static struct uma_keg masterkeg;
105static struct uma_zone masterzone_k;
106static struct uma_zone masterzone_z;
107static uma_zone_t kegs = &masterzone_k;
108static uma_zone_t zones = &masterzone_z;
109
110/* This is the zone from which all of uma_slab_t's are allocated. */
111static uma_zone_t slabzone;
112static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */
113
114/*
115 * The initial hash tables come out of this zone so they can be allocated
116 * prior to malloc coming up.
117 */
118static uma_zone_t hashzone;
119
120/* The boot-time adjusted value for cache line alignment. */
121int uma_align_cache = 64 - 1;
122
123static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
124
125/*
126 * Are we allowed to allocate buckets?
127 */
128static int bucketdisable = 1;
129
130/* Linked list of all kegs in the system */
131static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
132
133/* This mutex protects the keg list */
| 52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
60#include "opt_ddb.h"
61#include "opt_param.h"
62#include "opt_vm.h"
63
64#include <sys/param.h>
65#include <sys/systm.h>
66#include <sys/bitset.h>
67#include <sys/kernel.h>
68#include <sys/types.h>
69#include <sys/queue.h>
70#include <sys/malloc.h>
71#include <sys/ktr.h>
72#include <sys/lock.h>
73#include <sys/sysctl.h>
74#include <sys/mutex.h>
75#include <sys/proc.h>
76#include <sys/rwlock.h>
77#include <sys/sbuf.h>
78#include <sys/smp.h>
79#include <sys/vmmeter.h>
80
81#include <vm/vm.h>
82#include <vm/vm_object.h>
83#include <vm/vm_page.h>
84#include <vm/vm_pageout.h>
85#include <vm/vm_param.h>
86#include <vm/vm_map.h>
87#include <vm/vm_kern.h>
88#include <vm/vm_extern.h>
89#include <vm/uma.h>
90#include <vm/uma_int.h>
91#include <vm/uma_dbg.h>
92
93#include <ddb/ddb.h>
94
95#ifdef DEBUG_MEMGUARD
96#include <vm/memguard.h>
97#endif
98
99/*
100 * This is the zone and keg from which all zones are spawned. The idea is that
101 * even the zone & keg heads are allocated from the allocator, so we use the
102 * bss section to bootstrap us.
103 */
104static struct uma_keg masterkeg;
105static struct uma_zone masterzone_k;
106static struct uma_zone masterzone_z;
107static uma_zone_t kegs = &masterzone_k;
108static uma_zone_t zones = &masterzone_z;
109
110/* This is the zone from which all of uma_slab_t's are allocated. */
111static uma_zone_t slabzone;
112static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */
113
114/*
115 * The initial hash tables come out of this zone so they can be allocated
116 * prior to malloc coming up.
117 */
118static uma_zone_t hashzone;
119
120/* The boot-time adjusted value for cache line alignment. */
121int uma_align_cache = 64 - 1;
122
123static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets");
124
125/*
126 * Are we allowed to allocate buckets?
127 */
128static int bucketdisable = 1;
129
130/* Linked list of all kegs in the system */
131static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
132
133/* This mutex protects the keg list */
|
134static struct mtx uma_mtx;
| 134static struct mtx_padalign uma_mtx;
|
135
136/* Linked list of boot time pages */
137static LIST_HEAD(,uma_slab) uma_boot_pages =
138 LIST_HEAD_INITIALIZER(uma_boot_pages);
139
140/* This mutex protects the boot time pages list */
| 135
136/* Linked list of boot time pages */
137static LIST_HEAD(,uma_slab) uma_boot_pages =
138 LIST_HEAD_INITIALIZER(uma_boot_pages);
139
140/* This mutex protects the boot time pages list */
|
141static struct mtx uma_boot_pages_mtx;
| 141static struct mtx_padalign uma_boot_pages_mtx;
|
142
143/* Is the VM done starting up? */
144static int booted = 0;
145#define UMA_STARTUP 1
146#define UMA_STARTUP2 2
147
148/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */
149static const u_int uma_max_ipers = SLAB_SETSIZE;
150
151/*
152 * Only mbuf clusters use ref zones. Just provide enough references
153 * to support the one user. New code should not use the ref facility.
154 */
155static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
156
157/*
158 * This is the handle used to schedule events that need to happen
159 * outside of the allocation fast path.
160 */
161static struct callout uma_callout;
162#define UMA_TIMEOUT 20 /* Seconds for callout interval. */
163
164/*
165 * This structure is passed as the zone ctor arg so that I don't have to create
166 * a special allocation function just for zones.
167 */
168struct uma_zctor_args {
169 const char *name;
170 size_t size;
171 uma_ctor ctor;
172 uma_dtor dtor;
173 uma_init uminit;
174 uma_fini fini;
| 142
143/* Is the VM done starting up? */
144static int booted = 0;
145#define UMA_STARTUP 1
146#define UMA_STARTUP2 2
147
148/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */
149static const u_int uma_max_ipers = SLAB_SETSIZE;
150
151/*
152 * Only mbuf clusters use ref zones. Just provide enough references
153 * to support the one user. New code should not use the ref facility.
154 */
155static const u_int uma_max_ipers_ref = PAGE_SIZE / MCLBYTES;
156
157/*
158 * This is the handle used to schedule events that need to happen
159 * outside of the allocation fast path.
160 */
161static struct callout uma_callout;
162#define UMA_TIMEOUT 20 /* Seconds for callout interval. */
163
164/*
165 * This structure is passed as the zone ctor arg so that I don't have to create
166 * a special allocation function just for zones.
167 */
168struct uma_zctor_args {
169 const char *name;
170 size_t size;
171 uma_ctor ctor;
172 uma_dtor dtor;
173 uma_init uminit;
174 uma_fini fini;
|
| 175 uma_import import;
176 uma_release release;
177 void *arg;
|
175 uma_keg_t keg;
176 int align;
177 uint32_t flags;
178};
179
180struct uma_kctor_args {
181 uma_zone_t zone;
182 size_t size;
183 uma_init uminit;
184 uma_fini fini;
185 int align;
186 uint32_t flags;
187};
188
189struct uma_bucket_zone {
190 uma_zone_t ubz_zone;
191 char *ubz_name;
192 int ubz_entries;
193};
194
195#define BUCKET_MAX 128
196
197struct uma_bucket_zone bucket_zones[] = {
198 { NULL, "16 Bucket", 16 },
199 { NULL, "32 Bucket", 32 },
200 { NULL, "64 Bucket", 64 },
201 { NULL, "128 Bucket", 128 },
202 { NULL, NULL, 0}
203};
204
205#define BUCKET_SHIFT 4
206#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
207
208/*
209 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
210 * of approximately the right size.
211 */
212static uint8_t bucket_size[BUCKET_ZONES];
213
214/*
215 * Flags and enumerations to be passed to internal functions.
216 */
217enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
218
| 178 uma_keg_t keg;
179 int align;
180 uint32_t flags;
181};
182
183struct uma_kctor_args {
184 uma_zone_t zone;
185 size_t size;
186 uma_init uminit;
187 uma_fini fini;
188 int align;
189 uint32_t flags;
190};
191
192struct uma_bucket_zone {
193 uma_zone_t ubz_zone;
194 char *ubz_name;
195 int ubz_entries;
196};
197
198#define BUCKET_MAX 128
199
200struct uma_bucket_zone bucket_zones[] = {
201 { NULL, "16 Bucket", 16 },
202 { NULL, "32 Bucket", 32 },
203 { NULL, "64 Bucket", 64 },
204 { NULL, "128 Bucket", 128 },
205 { NULL, NULL, 0}
206};
207
208#define BUCKET_SHIFT 4
209#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
210
211/*
212 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
213 * of approximately the right size.
214 */
215static uint8_t bucket_size[BUCKET_ZONES];
216
217/*
218 * Flags and enumerations to be passed to internal functions.
219 */
220enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
221
|
219#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
220#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
221
| |
222/* Prototypes.. */
223
224static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
225static void *page_alloc(uma_zone_t, int, uint8_t *, int);
226static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
227static void page_free(void *, int, uint8_t);
228static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
229static void cache_drain(uma_zone_t);
230static void bucket_drain(uma_zone_t, uma_bucket_t);
231static void bucket_cache_drain(uma_zone_t zone);
232static int keg_ctor(void *, int, void *, int);
233static void keg_dtor(void *, int, void *);
234static int zone_ctor(void *, int, void *, int);
235static void zone_dtor(void *, int, void *);
236static int zero_init(void *, int, int);
237static void keg_small_init(uma_keg_t keg);
238static void keg_large_init(uma_keg_t keg);
239static void zone_foreach(void (*zfunc)(uma_zone_t));
240static void zone_timeout(uma_zone_t zone);
241static int hash_alloc(struct uma_hash *);
242static int hash_expand(struct uma_hash *, struct uma_hash *);
243static void hash_free(struct uma_hash *hash);
244static void uma_timeout(void *);
245static void uma_startup3(void);
246static void *zone_alloc_item(uma_zone_t, void *, int);
| 222/* Prototypes.. */
223
224static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
225static void *page_alloc(uma_zone_t, int, uint8_t *, int);
226static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
227static void page_free(void *, int, uint8_t);
228static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
229static void cache_drain(uma_zone_t);
230static void bucket_drain(uma_zone_t, uma_bucket_t);
231static void bucket_cache_drain(uma_zone_t zone);
232static int keg_ctor(void *, int, void *, int);
233static void keg_dtor(void *, int, void *);
234static int zone_ctor(void *, int, void *, int);
235static void zone_dtor(void *, int, void *);
236static int zero_init(void *, int, int);
237static void keg_small_init(uma_keg_t keg);
238static void keg_large_init(uma_keg_t keg);
239static void zone_foreach(void (*zfunc)(uma_zone_t));
240static void zone_timeout(uma_zone_t zone);
241static int hash_alloc(struct uma_hash *);
242static int hash_expand(struct uma_hash *, struct uma_hash *);
243static void hash_free(struct uma_hash *hash);
244static void uma_timeout(void *);
245static void uma_startup3(void);
246static void *zone_alloc_item(uma_zone_t, void *, int);
|
247static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip,
248 int);
| 247static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
|
249static void bucket_enable(void);
250static void bucket_init(void);
251static uma_bucket_t bucket_alloc(int, int);
252static void bucket_free(uma_bucket_t);
253static void bucket_zone_drain(void);
254static int zone_alloc_bucket(uma_zone_t zone, int flags);
255static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
256static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
| 248static void bucket_enable(void);
249static void bucket_init(void);
250static uma_bucket_t bucket_alloc(int, int);
251static void bucket_free(uma_bucket_t);
252static void bucket_zone_drain(void);
253static int zone_alloc_bucket(uma_zone_t zone, int flags);
254static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
255static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
|
257static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
| 256static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
257static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
|
258static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
259 uma_fini fini, int align, uint32_t flags);
260static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
261static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
| 258static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
259 uma_fini fini, int align, uint32_t flags);
260static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
261static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
|
| 262static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
263static void zone_release(uma_zone_t zone, void **bucket, int cnt);
|
262
263void uma_print_zone(uma_zone_t);
264void uma_print_stats(void);
265static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
266static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
267
268SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
269
270SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
271 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
272
273SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
274 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
275
276static int zone_warnings = 1;
277TUNABLE_INT("vm.zone_warnings", &zone_warnings);
278SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RW, &zone_warnings, 0,
279 "Warn when UMA zones becomes full");
280
281/*
282 * This routine checks to see whether or not it's safe to enable buckets.
283 */
284
285static void
286bucket_enable(void)
287{
288 bucketdisable = vm_page_count_min();
289}
290
291/*
292 * Initialize bucket_zones, the array of zones of buckets of various sizes.
293 *
294 * For each zone, calculate the memory required for each bucket, consisting
295 * of the header and an array of pointers. Initialize bucket_size[] to point
296 * the range of appropriate bucket sizes at the zone.
297 */
298static void
299bucket_init(void)
300{
301 struct uma_bucket_zone *ubz;
302 int i;
303 int j;
304
305 for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) {
306 int size;
307
308 ubz = &bucket_zones[j];
309 size = roundup(sizeof(struct uma_bucket), sizeof(void *));
310 size += sizeof(void *) * ubz->ubz_entries;
311 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
312 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
313 UMA_ZFLAG_INTERNAL | UMA_ZFLAG_BUCKET);
314 for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT))
315 bucket_size[i >> BUCKET_SHIFT] = j;
316 }
317}
318
319/*
320 * Given a desired number of entries for a bucket, return the zone from which
321 * to allocate the bucket.
322 */
323static struct uma_bucket_zone *
324bucket_zone_lookup(int entries)
325{
326 int idx;
327
328 idx = howmany(entries, 1 << BUCKET_SHIFT);
329 return (&bucket_zones[bucket_size[idx]]);
330}
331
332static uma_bucket_t
333bucket_alloc(int entries, int bflags)
334{
335 struct uma_bucket_zone *ubz;
336 uma_bucket_t bucket;
337
338 /*
339 * This is to stop us from allocating per cpu buckets while we're
340 * running out of vm.boot_pages. Otherwise, we would exhaust the
341 * boot pages. This also prevents us from allocating buckets in
342 * low memory situations.
343 */
344 if (bucketdisable)
345 return (NULL);
346
347 ubz = bucket_zone_lookup(entries);
348 bucket = zone_alloc_item(ubz->ubz_zone, NULL, bflags);
349 if (bucket) {
350#ifdef INVARIANTS
351 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
352#endif
353 bucket->ub_cnt = 0;
354 bucket->ub_entries = ubz->ubz_entries;
355 }
356
357 return (bucket);
358}
359
360static void
361bucket_free(uma_bucket_t bucket)
362{
363 struct uma_bucket_zone *ubz;
364
365 ubz = bucket_zone_lookup(bucket->ub_entries);
| 264
265void uma_print_zone(uma_zone_t);
266void uma_print_stats(void);
267static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
268static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
269
270SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
271
272SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT,
273 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones");
274
275SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT,
276 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats");
277
278static int zone_warnings = 1;
279TUNABLE_INT("vm.zone_warnings", &zone_warnings);
280SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RW, &zone_warnings, 0,
281 "Warn when UMA zones becomes full");
282
283/*
284 * This routine checks to see whether or not it's safe to enable buckets.
285 */
286
287static void
288bucket_enable(void)
289{
290 bucketdisable = vm_page_count_min();
291}
292
293/*
294 * Initialize bucket_zones, the array of zones of buckets of various sizes.
295 *
296 * For each zone, calculate the memory required for each bucket, consisting
297 * of the header and an array of pointers. Initialize bucket_size[] to point
298 * the range of appropriate bucket sizes at the zone.
299 */
300static void
301bucket_init(void)
302{
303 struct uma_bucket_zone *ubz;
304 int i;
305 int j;
306
307 for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) {
308 int size;
309
310 ubz = &bucket_zones[j];
311 size = roundup(sizeof(struct uma_bucket), sizeof(void *));
312 size += sizeof(void *) * ubz->ubz_entries;
313 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
314 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
315 UMA_ZFLAG_INTERNAL | UMA_ZFLAG_BUCKET);
316 for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT))
317 bucket_size[i >> BUCKET_SHIFT] = j;
318 }
319}
320
321/*
322 * Given a desired number of entries for a bucket, return the zone from which
323 * to allocate the bucket.
324 */
325static struct uma_bucket_zone *
326bucket_zone_lookup(int entries)
327{
328 int idx;
329
330 idx = howmany(entries, 1 << BUCKET_SHIFT);
331 return (&bucket_zones[bucket_size[idx]]);
332}
333
334static uma_bucket_t
335bucket_alloc(int entries, int bflags)
336{
337 struct uma_bucket_zone *ubz;
338 uma_bucket_t bucket;
339
340 /*
341 * This is to stop us from allocating per cpu buckets while we're
342 * running out of vm.boot_pages. Otherwise, we would exhaust the
343 * boot pages. This also prevents us from allocating buckets in
344 * low memory situations.
345 */
346 if (bucketdisable)
347 return (NULL);
348
349 ubz = bucket_zone_lookup(entries);
350 bucket = zone_alloc_item(ubz->ubz_zone, NULL, bflags);
351 if (bucket) {
352#ifdef INVARIANTS
353 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
354#endif
355 bucket->ub_cnt = 0;
356 bucket->ub_entries = ubz->ubz_entries;
357 }
358
359 return (bucket);
360}
361
362static void
363bucket_free(uma_bucket_t bucket)
364{
365 struct uma_bucket_zone *ubz;
366
367 ubz = bucket_zone_lookup(bucket->ub_entries);
|
366 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
367 ZFREE_STATFREE);
| 368 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
|
368}
369
370static void
371bucket_zone_drain(void)
372{
373 struct uma_bucket_zone *ubz;
374
375 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
376 zone_drain(ubz->ubz_zone);
377}
378
379static void
380zone_log_warning(uma_zone_t zone)
381{
382 static const struct timeval warninterval = { 300, 0 };
383
384 if (!zone_warnings || zone->uz_warning == NULL)
385 return;
386
387 if (ratecheck(&zone->uz_ratecheck, &warninterval))
388 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
389}
390
| 369}
370
371static void
372bucket_zone_drain(void)
373{
374 struct uma_bucket_zone *ubz;
375
376 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
377 zone_drain(ubz->ubz_zone);
378}
379
380static void
381zone_log_warning(uma_zone_t zone)
382{
383 static const struct timeval warninterval = { 300, 0 };
384
385 if (!zone_warnings || zone->uz_warning == NULL)
386 return;
387
388 if (ratecheck(&zone->uz_ratecheck, &warninterval))
389 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
390}
391
|
391static inline uma_keg_t
392zone_first_keg(uma_zone_t zone)
393{
394
395 return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
396}
397
| |
398static void
399zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
400{
401 uma_klink_t klink;
402
403 LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
404 kegfn(klink->kl_keg);
405}
406
407/*
408 * Routine called by timeout which is used to fire off some time interval
409 * based calculations. (stats, hash size, etc.)
410 *
411 * Arguments:
412 * arg Unused
413 *
414 * Returns:
415 * Nothing
416 */
417static void
418uma_timeout(void *unused)
419{
420 bucket_enable();
421 zone_foreach(zone_timeout);
422
423 /* Reschedule this event */
424 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
425}
426
427/*
428 * Routine to perform timeout driven calculations. This expands the
429 * hashes and does per cpu statistics aggregation.
430 *
431 * Returns nothing.
432 */
433static void
434keg_timeout(uma_keg_t keg)
435{
436
437 KEG_LOCK(keg);
438 /*
439 * Expand the keg hash table.
440 *
441 * This is done if the number of slabs is larger than the hash size.
442 * What I'm trying to do here is completely reduce collisions. This
443 * may be a little aggressive. Should I allow for two collisions max?
444 */
445 if (keg->uk_flags & UMA_ZONE_HASH &&
446 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
447 struct uma_hash newhash;
448 struct uma_hash oldhash;
449 int ret;
450
451 /*
452 * This is so involved because allocating and freeing
453 * while the keg lock is held will lead to deadlock.
454 * I have to do everything in stages and check for
455 * races.
456 */
457 newhash = keg->uk_hash;
458 KEG_UNLOCK(keg);
459 ret = hash_alloc(&newhash);
460 KEG_LOCK(keg);
461 if (ret) {
462 if (hash_expand(&keg->uk_hash, &newhash)) {
463 oldhash = keg->uk_hash;
464 keg->uk_hash = newhash;
465 } else
466 oldhash = newhash;
467
468 KEG_UNLOCK(keg);
469 hash_free(&oldhash);
470 KEG_LOCK(keg);
471 }
472 }
473 KEG_UNLOCK(keg);
474}
475
476static void
477zone_timeout(uma_zone_t zone)
478{
479
480 zone_foreach_keg(zone, &keg_timeout);
481}
482
483/*
484 * Allocate and zero fill the next sized hash table from the appropriate
485 * backing store.
486 *
487 * Arguments:
488 * hash A new hash structure with the old hash size in uh_hashsize
489 *
490 * Returns:
491 * 1 on sucess and 0 on failure.
492 */
493static int
494hash_alloc(struct uma_hash *hash)
495{
496 int oldsize;
497 int alloc;
498
499 oldsize = hash->uh_hashsize;
500
501 /* We're just going to go to a power of two greater */
502 if (oldsize) {
503 hash->uh_hashsize = oldsize * 2;
504 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
505 hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
506 M_UMAHASH, M_NOWAIT);
507 } else {
508 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
509 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
510 M_WAITOK);
511 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
512 }
513 if (hash->uh_slab_hash) {
514 bzero(hash->uh_slab_hash, alloc);
515 hash->uh_hashmask = hash->uh_hashsize - 1;
516 return (1);
517 }
518
519 return (0);
520}
521
522/*
523 * Expands the hash table for HASH zones. This is done from zone_timeout
524 * to reduce collisions. This must not be done in the regular allocation
525 * path, otherwise, we can recurse on the vm while allocating pages.
526 *
527 * Arguments:
528 * oldhash The hash you want to expand
529 * newhash The hash structure for the new table
530 *
531 * Returns:
532 * Nothing
533 *
534 * Discussion:
535 */
536static int
537hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
538{
539 uma_slab_t slab;
540 int hval;
541 int i;
542
543 if (!newhash->uh_slab_hash)
544 return (0);
545
546 if (oldhash->uh_hashsize >= newhash->uh_hashsize)
547 return (0);
548
549 /*
550 * I need to investigate hash algorithms for resizing without a
551 * full rehash.
552 */
553
554 for (i = 0; i < oldhash->uh_hashsize; i++)
555 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
556 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
557 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
558 hval = UMA_HASH(newhash, slab->us_data);
559 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
560 slab, us_hlink);
561 }
562
563 return (1);
564}
565
566/*
567 * Free the hash bucket to the appropriate backing store.
568 *
569 * Arguments:
570 * slab_hash The hash bucket we're freeing
571 * hashsize The number of entries in that hash bucket
572 *
573 * Returns:
574 * Nothing
575 */
576static void
577hash_free(struct uma_hash *hash)
578{
579 if (hash->uh_slab_hash == NULL)
580 return;
581 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
| 392static void
393zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
394{
395 uma_klink_t klink;
396
397 LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
398 kegfn(klink->kl_keg);
399}
400
401/*
402 * Routine called by timeout which is used to fire off some time interval
403 * based calculations. (stats, hash size, etc.)
404 *
405 * Arguments:
406 * arg Unused
407 *
408 * Returns:
409 * Nothing
410 */
411static void
412uma_timeout(void *unused)
413{
414 bucket_enable();
415 zone_foreach(zone_timeout);
416
417 /* Reschedule this event */
418 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
419}
420
421/*
422 * Routine to perform timeout driven calculations. This expands the
423 * hashes and does per cpu statistics aggregation.
424 *
425 * Returns nothing.
426 */
427static void
428keg_timeout(uma_keg_t keg)
429{
430
431 KEG_LOCK(keg);
432 /*
433 * Expand the keg hash table.
434 *
435 * This is done if the number of slabs is larger than the hash size.
436 * What I'm trying to do here is completely reduce collisions. This
437 * may be a little aggressive. Should I allow for two collisions max?
438 */
439 if (keg->uk_flags & UMA_ZONE_HASH &&
440 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) {
441 struct uma_hash newhash;
442 struct uma_hash oldhash;
443 int ret;
444
445 /*
446 * This is so involved because allocating and freeing
447 * while the keg lock is held will lead to deadlock.
448 * I have to do everything in stages and check for
449 * races.
450 */
451 newhash = keg->uk_hash;
452 KEG_UNLOCK(keg);
453 ret = hash_alloc(&newhash);
454 KEG_LOCK(keg);
455 if (ret) {
456 if (hash_expand(&keg->uk_hash, &newhash)) {
457 oldhash = keg->uk_hash;
458 keg->uk_hash = newhash;
459 } else
460 oldhash = newhash;
461
462 KEG_UNLOCK(keg);
463 hash_free(&oldhash);
464 KEG_LOCK(keg);
465 }
466 }
467 KEG_UNLOCK(keg);
468}
469
470static void
471zone_timeout(uma_zone_t zone)
472{
473
474 zone_foreach_keg(zone, &keg_timeout);
475}
476
477/*
478 * Allocate and zero fill the next sized hash table from the appropriate
479 * backing store.
480 *
481 * Arguments:
482 * hash A new hash structure with the old hash size in uh_hashsize
483 *
484 * Returns:
485 * 1 on sucess and 0 on failure.
486 */
487static int
488hash_alloc(struct uma_hash *hash)
489{
490 int oldsize;
491 int alloc;
492
493 oldsize = hash->uh_hashsize;
494
495 /* We're just going to go to a power of two greater */
496 if (oldsize) {
497 hash->uh_hashsize = oldsize * 2;
498 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize;
499 hash->uh_slab_hash = (struct slabhead *)malloc(alloc,
500 M_UMAHASH, M_NOWAIT);
501 } else {
502 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
503 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
504 M_WAITOK);
505 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
506 }
507 if (hash->uh_slab_hash) {
508 bzero(hash->uh_slab_hash, alloc);
509 hash->uh_hashmask = hash->uh_hashsize - 1;
510 return (1);
511 }
512
513 return (0);
514}
515
516/*
517 * Expands the hash table for HASH zones. This is done from zone_timeout
518 * to reduce collisions. This must not be done in the regular allocation
519 * path, otherwise, we can recurse on the vm while allocating pages.
520 *
521 * Arguments:
522 * oldhash The hash you want to expand
523 * newhash The hash structure for the new table
524 *
525 * Returns:
526 * Nothing
527 *
528 * Discussion:
529 */
530static int
531hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash)
532{
533 uma_slab_t slab;
534 int hval;
535 int i;
536
537 if (!newhash->uh_slab_hash)
538 return (0);
539
540 if (oldhash->uh_hashsize >= newhash->uh_hashsize)
541 return (0);
542
543 /*
544 * I need to investigate hash algorithms for resizing without a
545 * full rehash.
546 */
547
548 for (i = 0; i < oldhash->uh_hashsize; i++)
549 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) {
550 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]);
551 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink);
552 hval = UMA_HASH(newhash, slab->us_data);
553 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval],
554 slab, us_hlink);
555 }
556
557 return (1);
558}
559
560/*
561 * Free the hash bucket to the appropriate backing store.
562 *
563 * Arguments:
564 * slab_hash The hash bucket we're freeing
565 * hashsize The number of entries in that hash bucket
566 *
567 * Returns:
568 * Nothing
569 */
570static void
571hash_free(struct uma_hash *hash)
572{
573 if (hash->uh_slab_hash == NULL)
574 return;
575 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
|
582 zone_free_item(hashzone,
583 hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE);
| 576 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
|
584 else
585 free(hash->uh_slab_hash, M_UMAHASH);
586}
587
588/*
589 * Frees all outstanding items in a bucket
590 *
591 * Arguments:
592 * zone The zone to free to, must be unlocked.
593 * bucket The free/alloc bucket with items, cpu queue must be locked.
594 *
595 * Returns:
596 * Nothing
597 */
598
599static void
600bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
601{
| 577 else
578 free(hash->uh_slab_hash, M_UMAHASH);
579}
580
581/*
582 * Frees all outstanding items in a bucket
583 *
584 * Arguments:
585 * zone The zone to free to, must be unlocked.
586 * bucket The free/alloc bucket with items, cpu queue must be locked.
587 *
588 * Returns:
589 * Nothing
590 */
591
592static void
593bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
594{
|
602 void *item;
| 595 int i;
|
603
604 if (bucket == NULL)
605 return;
606
| 596
597 if (bucket == NULL)
598 return;
599
|
607 while (bucket->ub_cnt > 0) {
608 bucket->ub_cnt--;
609 item = bucket->ub_bucket[bucket->ub_cnt];
610#ifdef INVARIANTS
611 bucket->ub_bucket[bucket->ub_cnt] = NULL;
612 KASSERT(item != NULL,
613 ("bucket_drain: botched ptr, item is NULL"));
614#endif
615 zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
616 }
| 600 if (zone->uz_fini)
601 for (i = 0; i < bucket->ub_cnt; i++)
602 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
603 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
604 bucket->ub_cnt = 0;
|
617}
618
619/*
620 * Drains the per cpu caches for a zone.
621 *
622 * NOTE: This may only be called while the zone is being turn down, and not
623 * during normal operation. This is necessary in order that we do not have
624 * to migrate CPUs to drain the per-CPU caches.
625 *
626 * Arguments:
627 * zone The zone to drain, must be unlocked.
628 *
629 * Returns:
630 * Nothing
631 */
632static void
633cache_drain(uma_zone_t zone)
634{
635 uma_cache_t cache;
636 int cpu;
637
638 /*
639 * XXX: It is safe to not lock the per-CPU caches, because we're
640 * tearing down the zone anyway. I.e., there will be no further use
641 * of the caches at this point.
642 *
643 * XXX: It would good to be able to assert that the zone is being
644 * torn down to prevent improper use of cache_drain().
645 *
646 * XXX: We lock the zone before passing into bucket_cache_drain() as
647 * it is used elsewhere. Should the tear-down path be made special
648 * there in some form?
649 */
650 CPU_FOREACH(cpu) {
651 cache = &zone->uz_cpu[cpu];
652 bucket_drain(zone, cache->uc_allocbucket);
653 bucket_drain(zone, cache->uc_freebucket);
654 if (cache->uc_allocbucket != NULL)
655 bucket_free(cache->uc_allocbucket);
656 if (cache->uc_freebucket != NULL)
657 bucket_free(cache->uc_freebucket);
658 cache->uc_allocbucket = cache->uc_freebucket = NULL;
659 }
660 ZONE_LOCK(zone);
661 bucket_cache_drain(zone);
662 ZONE_UNLOCK(zone);
663}
664
665/*
666 * Drain the cached buckets from a zone. Expects a locked zone on entry.
667 */
668static void
669bucket_cache_drain(uma_zone_t zone)
670{
671 uma_bucket_t bucket;
672
673 /*
674 * Drain the bucket queues and free the buckets, we just keep two per
675 * cpu (alloc/free).
676 */
677 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) {
678 LIST_REMOVE(bucket, ub_link);
679 ZONE_UNLOCK(zone);
680 bucket_drain(zone, bucket);
681 bucket_free(bucket);
682 ZONE_LOCK(zone);
683 }
684
685 /* Now we do the free queue.. */
686 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
687 LIST_REMOVE(bucket, ub_link);
688 bucket_free(bucket);
689 }
690}
691
692/*
693 * Frees pages from a keg back to the system. This is done on demand from
694 * the pageout daemon.
695 *
696 * Returns nothing.
697 */
698static void
699keg_drain(uma_keg_t keg)
700{
701 struct slabhead freeslabs = { 0 };
702 uma_slab_t slab;
703 uma_slab_t n;
704 uint8_t flags;
705 uint8_t *mem;
706 int i;
707
708 /*
709 * We don't want to take pages from statically allocated kegs at this
710 * time
711 */
712 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
713 return;
714
715#ifdef UMA_DEBUG
716 printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
717#endif
718 KEG_LOCK(keg);
719 if (keg->uk_free == 0)
720 goto finished;
721
722 slab = LIST_FIRST(&keg->uk_free_slab);
723 while (slab) {
724 n = LIST_NEXT(slab, us_link);
725
726 /* We have no where to free these to */
727 if (slab->us_flags & UMA_SLAB_BOOT) {
728 slab = n;
729 continue;
730 }
731
732 LIST_REMOVE(slab, us_link);
733 keg->uk_pages -= keg->uk_ppera;
734 keg->uk_free -= keg->uk_ipers;
735
736 if (keg->uk_flags & UMA_ZONE_HASH)
737 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
738
739 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
740
741 slab = n;
742 }
743finished:
744 KEG_UNLOCK(keg);
745
746 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
747 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
748 if (keg->uk_fini)
749 for (i = 0; i < keg->uk_ipers; i++)
750 keg->uk_fini(
751 slab->us_data + (keg->uk_rsize * i),
752 keg->uk_size);
753 flags = slab->us_flags;
754 mem = slab->us_data;
755
756 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
757 vm_object_t obj;
758
759 if (flags & UMA_SLAB_KMEM)
760 obj = kmem_object;
761 else if (flags & UMA_SLAB_KERNEL)
762 obj = kernel_object;
763 else
764 obj = NULL;
765 for (i = 0; i < keg->uk_ppera; i++)
766 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
767 obj);
768 }
769 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
| 605}
606
607/*
608 * Drains the per cpu caches for a zone.
609 *
610 * NOTE: This may only be called while the zone is being turn down, and not
611 * during normal operation. This is necessary in order that we do not have
612 * to migrate CPUs to drain the per-CPU caches.
613 *
614 * Arguments:
615 * zone The zone to drain, must be unlocked.
616 *
617 * Returns:
618 * Nothing
619 */
620static void
621cache_drain(uma_zone_t zone)
622{
623 uma_cache_t cache;
624 int cpu;
625
626 /*
627 * XXX: It is safe to not lock the per-CPU caches, because we're
628 * tearing down the zone anyway. I.e., there will be no further use
629 * of the caches at this point.
630 *
631 * XXX: It would good to be able to assert that the zone is being
632 * torn down to prevent improper use of cache_drain().
633 *
634 * XXX: We lock the zone before passing into bucket_cache_drain() as
635 * it is used elsewhere. Should the tear-down path be made special
636 * there in some form?
637 */
638 CPU_FOREACH(cpu) {
639 cache = &zone->uz_cpu[cpu];
640 bucket_drain(zone, cache->uc_allocbucket);
641 bucket_drain(zone, cache->uc_freebucket);
642 if (cache->uc_allocbucket != NULL)
643 bucket_free(cache->uc_allocbucket);
644 if (cache->uc_freebucket != NULL)
645 bucket_free(cache->uc_freebucket);
646 cache->uc_allocbucket = cache->uc_freebucket = NULL;
647 }
648 ZONE_LOCK(zone);
649 bucket_cache_drain(zone);
650 ZONE_UNLOCK(zone);
651}
652
653/*
654 * Drain the cached buckets from a zone. Expects a locked zone on entry.
655 */
656static void
657bucket_cache_drain(uma_zone_t zone)
658{
659 uma_bucket_t bucket;
660
661 /*
662 * Drain the bucket queues and free the buckets, we just keep two per
663 * cpu (alloc/free).
664 */
665 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) {
666 LIST_REMOVE(bucket, ub_link);
667 ZONE_UNLOCK(zone);
668 bucket_drain(zone, bucket);
669 bucket_free(bucket);
670 ZONE_LOCK(zone);
671 }
672
673 /* Now we do the free queue.. */
674 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
675 LIST_REMOVE(bucket, ub_link);
676 bucket_free(bucket);
677 }
678}
679
680/*
681 * Frees pages from a keg back to the system. This is done on demand from
682 * the pageout daemon.
683 *
684 * Returns nothing.
685 */
686static void
687keg_drain(uma_keg_t keg)
688{
689 struct slabhead freeslabs = { 0 };
690 uma_slab_t slab;
691 uma_slab_t n;
692 uint8_t flags;
693 uint8_t *mem;
694 int i;
695
696 /*
697 * We don't want to take pages from statically allocated kegs at this
698 * time
699 */
700 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL)
701 return;
702
703#ifdef UMA_DEBUG
704 printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
705#endif
706 KEG_LOCK(keg);
707 if (keg->uk_free == 0)
708 goto finished;
709
710 slab = LIST_FIRST(&keg->uk_free_slab);
711 while (slab) {
712 n = LIST_NEXT(slab, us_link);
713
714 /* We have no where to free these to */
715 if (slab->us_flags & UMA_SLAB_BOOT) {
716 slab = n;
717 continue;
718 }
719
720 LIST_REMOVE(slab, us_link);
721 keg->uk_pages -= keg->uk_ppera;
722 keg->uk_free -= keg->uk_ipers;
723
724 if (keg->uk_flags & UMA_ZONE_HASH)
725 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data);
726
727 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink);
728
729 slab = n;
730 }
731finished:
732 KEG_UNLOCK(keg);
733
734 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) {
735 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink);
736 if (keg->uk_fini)
737 for (i = 0; i < keg->uk_ipers; i++)
738 keg->uk_fini(
739 slab->us_data + (keg->uk_rsize * i),
740 keg->uk_size);
741 flags = slab->us_flags;
742 mem = slab->us_data;
743
744 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
745 vm_object_t obj;
746
747 if (flags & UMA_SLAB_KMEM)
748 obj = kmem_object;
749 else if (flags & UMA_SLAB_KERNEL)
750 obj = kernel_object;
751 else
752 obj = NULL;
753 for (i = 0; i < keg->uk_ppera; i++)
754 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
755 obj);
756 }
757 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
|
770 zone_free_item(keg->uk_slabzone, slab, NULL,
771 SKIP_NONE, ZFREE_STATFREE);
| 758 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
|
772#ifdef UMA_DEBUG
773 printf("%s: Returning %d bytes.\n",
774 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
775#endif
776 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
777 }
778}
779
780static void
781zone_drain_wait(uma_zone_t zone, int waitok)
782{
783
784 /*
785 * Set draining to interlock with zone_dtor() so we can release our
786 * locks as we go. Only dtor() should do a WAITOK call since it
787 * is the only call that knows the structure will still be available
788 * when it wakes up.
789 */
790 ZONE_LOCK(zone);
791 while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
792 if (waitok == M_NOWAIT)
793 goto out;
794 mtx_unlock(&uma_mtx);
795 msleep(zone, zone->uz_lock, PVM, "zonedrain", 1);
796 mtx_lock(&uma_mtx);
797 }
798 zone->uz_flags |= UMA_ZFLAG_DRAINING;
799 bucket_cache_drain(zone);
800 ZONE_UNLOCK(zone);
801 /*
802 * The DRAINING flag protects us from being freed while
803 * we're running. Normally the uma_mtx would protect us but we
804 * must be able to release and acquire the right lock for each keg.
805 */
806 zone_foreach_keg(zone, &keg_drain);
807 ZONE_LOCK(zone);
808 zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
809 wakeup(zone);
810out:
811 ZONE_UNLOCK(zone);
812}
813
814void
815zone_drain(uma_zone_t zone)
816{
817
818 zone_drain_wait(zone, M_NOWAIT);
819}
820
821/*
822 * Allocate a new slab for a keg. This does not insert the slab onto a list.
823 *
824 * Arguments:
825 * wait Shall we wait?
826 *
827 * Returns:
828 * The slab that was allocated or NULL if there is no memory and the
829 * caller specified M_NOWAIT.
830 */
831static uma_slab_t
832keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
833{
834 uma_slabrefcnt_t slabref;
835 uma_alloc allocf;
836 uma_slab_t slab;
837 uint8_t *mem;
838 uint8_t flags;
839 int i;
840
841 mtx_assert(&keg->uk_lock, MA_OWNED);
842 slab = NULL;
843
844#ifdef UMA_DEBUG
| 759#ifdef UMA_DEBUG
760 printf("%s: Returning %d bytes.\n",
761 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
762#endif
763 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
764 }
765}
766
767static void
768zone_drain_wait(uma_zone_t zone, int waitok)
769{
770
771 /*
772 * Set draining to interlock with zone_dtor() so we can release our
773 * locks as we go. Only dtor() should do a WAITOK call since it
774 * is the only call that knows the structure will still be available
775 * when it wakes up.
776 */
777 ZONE_LOCK(zone);
778 while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
779 if (waitok == M_NOWAIT)
780 goto out;
781 mtx_unlock(&uma_mtx);
782 msleep(zone, zone->uz_lock, PVM, "zonedrain", 1);
783 mtx_lock(&uma_mtx);
784 }
785 zone->uz_flags |= UMA_ZFLAG_DRAINING;
786 bucket_cache_drain(zone);
787 ZONE_UNLOCK(zone);
788 /*
789 * The DRAINING flag protects us from being freed while
790 * we're running. Normally the uma_mtx would protect us but we
791 * must be able to release and acquire the right lock for each keg.
792 */
793 zone_foreach_keg(zone, &keg_drain);
794 ZONE_LOCK(zone);
795 zone->uz_flags &= ~UMA_ZFLAG_DRAINING;
796 wakeup(zone);
797out:
798 ZONE_UNLOCK(zone);
799}
800
801void
802zone_drain(uma_zone_t zone)
803{
804
805 zone_drain_wait(zone, M_NOWAIT);
806}
807
808/*
809 * Allocate a new slab for a keg. This does not insert the slab onto a list.
810 *
811 * Arguments:
812 * wait Shall we wait?
813 *
814 * Returns:
815 * The slab that was allocated or NULL if there is no memory and the
816 * caller specified M_NOWAIT.
817 */
818static uma_slab_t
819keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
820{
821 uma_slabrefcnt_t slabref;
822 uma_alloc allocf;
823 uma_slab_t slab;
824 uint8_t *mem;
825 uint8_t flags;
826 int i;
827
828 mtx_assert(&keg->uk_lock, MA_OWNED);
829 slab = NULL;
830
831#ifdef UMA_DEBUG
|
845 printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
| 832 printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
|
846#endif
847 allocf = keg->uk_allocf;
848 KEG_UNLOCK(keg);
849
850 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
851 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
852 if (slab == NULL) {
853 KEG_LOCK(keg);
854 return NULL;
855 }
856 }
857
858 /*
859 * This reproduces the old vm_zone behavior of zero filling pages the
860 * first time they are added to a zone.
861 *
862 * Malloced items are zeroed in uma_zalloc.
863 */
864
865 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
866 wait |= M_ZERO;
867 else
868 wait &= ~M_ZERO;
869
870 if (keg->uk_flags & UMA_ZONE_NODUMP)
871 wait |= M_NODUMP;
872
873 /* zone is passed for legacy reasons. */
874 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
875 if (mem == NULL) {
876 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
| 833#endif
834 allocf = keg->uk_allocf;
835 KEG_UNLOCK(keg);
836
837 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
838 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
839 if (slab == NULL) {
840 KEG_LOCK(keg);
841 return NULL;
842 }
843 }
844
845 /*
846 * This reproduces the old vm_zone behavior of zero filling pages the
847 * first time they are added to a zone.
848 *
849 * Malloced items are zeroed in uma_zalloc.
850 */
851
852 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
853 wait |= M_ZERO;
854 else
855 wait &= ~M_ZERO;
856
857 if (keg->uk_flags & UMA_ZONE_NODUMP)
858 wait |= M_NODUMP;
859
860 /* zone is passed for legacy reasons. */
861 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
862 if (mem == NULL) {
863 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
|
877 zone_free_item(keg->uk_slabzone, slab, NULL,
878 SKIP_NONE, ZFREE_STATFREE);
| 864 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
|
879 KEG_LOCK(keg);
880 return (NULL);
881 }
882
883 /* Point the slab into the allocated memory */
884 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
885 slab = (uma_slab_t )(mem + keg->uk_pgoff);
886
887 if (keg->uk_flags & UMA_ZONE_VTOSLAB)
888 for (i = 0; i < keg->uk_ppera; i++)
889 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
890
891 slab->us_keg = keg;
892 slab->us_data = mem;
893 slab->us_freecount = keg->uk_ipers;
894 slab->us_flags = flags;
895 BIT_FILL(SLAB_SETSIZE, &slab->us_free);
896#ifdef INVARIANTS
897 BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
898#endif
899 if (keg->uk_flags & UMA_ZONE_REFCNT) {
900 slabref = (uma_slabrefcnt_t)slab;
901 for (i = 0; i < keg->uk_ipers; i++)
902 slabref->us_refcnt[i] = 0;
903 }
904
905 if (keg->uk_init != NULL) {
906 for (i = 0; i < keg->uk_ipers; i++)
907 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
908 keg->uk_size, wait) != 0)
909 break;
910 if (i != keg->uk_ipers) {
911 if (keg->uk_fini != NULL) {
912 for (i--; i > -1; i--)
913 keg->uk_fini(slab->us_data +
914 (keg->uk_rsize * i),
915 keg->uk_size);
916 }
917 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
918 vm_object_t obj;
919
920 if (flags & UMA_SLAB_KMEM)
921 obj = kmem_object;
922 else if (flags & UMA_SLAB_KERNEL)
923 obj = kernel_object;
924 else
925 obj = NULL;
926 for (i = 0; i < keg->uk_ppera; i++)
927 vsetobj((vm_offset_t)mem +
928 (i * PAGE_SIZE), obj);
929 }
930 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
931 zone_free_item(keg->uk_slabzone, slab,
| 865 KEG_LOCK(keg);
866 return (NULL);
867 }
868
869 /* Point the slab into the allocated memory */
870 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
871 slab = (uma_slab_t )(mem + keg->uk_pgoff);
872
873 if (keg->uk_flags & UMA_ZONE_VTOSLAB)
874 for (i = 0; i < keg->uk_ppera; i++)
875 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab);
876
877 slab->us_keg = keg;
878 slab->us_data = mem;
879 slab->us_freecount = keg->uk_ipers;
880 slab->us_flags = flags;
881 BIT_FILL(SLAB_SETSIZE, &slab->us_free);
882#ifdef INVARIANTS
883 BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree);
884#endif
885 if (keg->uk_flags & UMA_ZONE_REFCNT) {
886 slabref = (uma_slabrefcnt_t)slab;
887 for (i = 0; i < keg->uk_ipers; i++)
888 slabref->us_refcnt[i] = 0;
889 }
890
891 if (keg->uk_init != NULL) {
892 for (i = 0; i < keg->uk_ipers; i++)
893 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i),
894 keg->uk_size, wait) != 0)
895 break;
896 if (i != keg->uk_ipers) {
897 if (keg->uk_fini != NULL) {
898 for (i--; i > -1; i--)
899 keg->uk_fini(slab->us_data +
900 (keg->uk_rsize * i),
901 keg->uk_size);
902 }
903 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
904 vm_object_t obj;
905
906 if (flags & UMA_SLAB_KMEM)
907 obj = kmem_object;
908 else if (flags & UMA_SLAB_KERNEL)
909 obj = kernel_object;
910 else
911 obj = NULL;
912 for (i = 0; i < keg->uk_ppera; i++)
913 vsetobj((vm_offset_t)mem +
914 (i * PAGE_SIZE), obj);
915 }
916 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
917 zone_free_item(keg->uk_slabzone, slab,
|
932 NULL, SKIP_NONE, ZFREE_STATFREE);
| 918 NULL, SKIP_NONE);
|
933 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
934 flags);
935 KEG_LOCK(keg);
936 return (NULL);
937 }
938 }
939 KEG_LOCK(keg);
940
941 if (keg->uk_flags & UMA_ZONE_HASH)
942 UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
943
944 keg->uk_pages += keg->uk_ppera;
945 keg->uk_free += keg->uk_ipers;
946
947 return (slab);
948}
949
950/*
951 * This function is intended to be used early on in place of page_alloc() so
952 * that we may use the boot time page cache to satisfy allocations before
953 * the VM is ready.
954 */
955static void *
956startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
957{
958 uma_keg_t keg;
959 uma_slab_t tmps;
960 int pages, check_pages;
961
962 keg = zone_first_keg(zone);
963 pages = howmany(bytes, PAGE_SIZE);
964 check_pages = pages - 1;
965 KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
966
967 /*
968 * Check our small startup cache to see if it has pages remaining.
969 */
970 mtx_lock(&uma_boot_pages_mtx);
971
972 /* First check if we have enough room. */
973 tmps = LIST_FIRST(&uma_boot_pages);
974 while (tmps != NULL && check_pages-- > 0)
975 tmps = LIST_NEXT(tmps, us_link);
976 if (tmps != NULL) {
977 /*
978 * It's ok to lose tmps references. The last one will
979 * have tmps->us_data pointing to the start address of
980 * "pages" contiguous pages of memory.
981 */
982 while (pages-- > 0) {
983 tmps = LIST_FIRST(&uma_boot_pages);
984 LIST_REMOVE(tmps, us_link);
985 }
986 mtx_unlock(&uma_boot_pages_mtx);
987 *pflag = tmps->us_flags;
988 return (tmps->us_data);
989 }
990 mtx_unlock(&uma_boot_pages_mtx);
991 if (booted < UMA_STARTUP2)
992 panic("UMA: Increase vm.boot_pages");
993 /*
994 * Now that we've booted reset these users to their real allocator.
995 */
996#ifdef UMA_MD_SMALL_ALLOC
997 keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
998#else
999 keg->uk_allocf = page_alloc;
1000#endif
1001 return keg->uk_allocf(zone, bytes, pflag, wait);
1002}
1003
1004/*
1005 * Allocates a number of pages from the system
1006 *
1007 * Arguments:
1008 * bytes The number of bytes requested
1009 * wait Shall we wait?
1010 *
1011 * Returns:
1012 * A pointer to the alloced memory or possibly
1013 * NULL if M_NOWAIT is set.
1014 */
1015static void *
1016page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
1017{
1018 void *p; /* Returned page */
1019
1020 *pflag = UMA_SLAB_KMEM;
1021 p = (void *) kmem_malloc(kmem_map, bytes, wait);
1022
1023 return (p);
1024}
1025
1026/*
1027 * Allocates a number of pages from within an object
1028 *
1029 * Arguments:
1030 * bytes The number of bytes requested
1031 * wait Shall we wait?
1032 *
1033 * Returns:
1034 * A pointer to the alloced memory or possibly
1035 * NULL if M_NOWAIT is set.
1036 */
1037static void *
1038noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
1039{
1040 TAILQ_HEAD(, vm_page) alloctail;
1041 u_long npages;
1042 vm_offset_t retkva, zkva;
1043 vm_page_t p, p_next;
1044 uma_keg_t keg;
1045
1046 TAILQ_INIT(&alloctail);
1047 keg = zone_first_keg(zone);
1048
1049 npages = howmany(bytes, PAGE_SIZE);
1050 while (npages > 0) {
1051 p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
1052 VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
1053 if (p != NULL) {
1054 /*
1055 * Since the page does not belong to an object, its
1056 * listq is unused.
1057 */
1058 TAILQ_INSERT_TAIL(&alloctail, p, listq);
1059 npages--;
1060 continue;
1061 }
1062 if (wait & M_WAITOK) {
1063 VM_WAIT;
1064 continue;
1065 }
1066
1067 /*
1068 * Page allocation failed, free intermediate pages and
1069 * exit.
1070 */
1071 TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1072 vm_page_unwire(p, 0);
1073 vm_page_free(p);
1074 }
1075 return (NULL);
1076 }
1077 *flags = UMA_SLAB_PRIV;
1078 zkva = keg->uk_kva +
1079 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1080 retkva = zkva;
1081 TAILQ_FOREACH(p, &alloctail, listq) {
1082 pmap_qenter(zkva, &p, 1);
1083 zkva += PAGE_SIZE;
1084 }
1085
1086 return ((void *)retkva);
1087}
1088
1089/*
1090 * Frees a number of pages to the system
1091 *
1092 * Arguments:
1093 * mem A pointer to the memory to be freed
1094 * size The size of the memory being freed
1095 * flags The original p->us_flags field
1096 *
1097 * Returns:
1098 * Nothing
1099 */
1100static void
1101page_free(void *mem, int size, uint8_t flags)
1102{
1103 vm_map_t map;
1104
1105 if (flags & UMA_SLAB_KMEM)
1106 map = kmem_map;
1107 else if (flags & UMA_SLAB_KERNEL)
1108 map = kernel_map;
1109 else
1110 panic("UMA: page_free used with invalid flags %d", flags);
1111
1112 kmem_free(map, (vm_offset_t)mem, size);
1113}
1114
1115/*
1116 * Zero fill initializer
1117 *
1118 * Arguments/Returns follow uma_init specifications
1119 */
1120static int
1121zero_init(void *mem, int size, int flags)
1122{
1123 bzero(mem, size);
1124 return (0);
1125}
1126
1127/*
1128 * Finish creating a small uma keg. This calculates ipers, and the keg size.
1129 *
1130 * Arguments
1131 * keg The zone we should initialize
1132 *
1133 * Returns
1134 * Nothing
1135 */
1136static void
1137keg_small_init(uma_keg_t keg)
1138{
1139 u_int rsize;
1140 u_int memused;
1141 u_int wastedspace;
1142 u_int shsize;
1143
1144 if (keg->uk_flags & UMA_ZONE_PCPU) {
1145 KASSERT(mp_ncpus > 0, ("%s: ncpus %d\n", __func__, mp_ncpus));
1146 keg->uk_slabsize = sizeof(struct pcpu);
1147 keg->uk_ppera = howmany(mp_ncpus * sizeof(struct pcpu),
1148 PAGE_SIZE);
1149 } else {
1150 keg->uk_slabsize = UMA_SLAB_SIZE;
1151 keg->uk_ppera = 1;
1152 }
1153
1154 /*
1155 * Calculate the size of each allocation (rsize) according to
1156 * alignment. If the requested size is smaller than we have
1157 * allocation bits for we round it up.
1158 */
1159 rsize = keg->uk_size;
1160 if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
1161 rsize = keg->uk_slabsize / SLAB_SETSIZE;
1162 if (rsize & keg->uk_align)
1163 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1164 keg->uk_rsize = rsize;
1165
1166 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1167 keg->uk_rsize < sizeof(struct pcpu),
1168 ("%s: size %u too large", __func__, keg->uk_rsize));
1169
1170 if (keg->uk_flags & UMA_ZONE_REFCNT)
1171 rsize += sizeof(uint32_t);
1172
1173 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
1174 shsize = 0;
1175 else
1176 shsize = sizeof(struct uma_slab);
1177
1178 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1179 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1180 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1181
1182 memused = keg->uk_ipers * rsize + shsize;
1183 wastedspace = keg->uk_slabsize - memused;
1184
1185 /*
1186 * We can't do OFFPAGE if we're internal or if we've been
1187 * asked to not go to the VM for buckets. If we do this we
1188 * may end up going to the VM (kmem_map) for slabs which we
1189 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
1190 * result of UMA_ZONE_VM, which clearly forbids it.
1191 */
1192 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1193 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1194 return;
1195
1196 /*
1197 * See if using an OFFPAGE slab will limit our waste. Only do
1198 * this if it permits more items per-slab.
1199 *
1200 * XXX We could try growing slabsize to limit max waste as well.
1201 * Historically this was not done because the VM could not
1202 * efficiently handle contiguous allocations.
1203 */
1204 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1205 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1206 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1207 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1208 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1209#ifdef UMA_DEBUG
1210 printf("UMA decided we need offpage slab headers for "
1211 "keg: %s, calculated wastedspace = %d, "
1212 "maximum wasted space allowed = %d, "
1213 "calculated ipers = %d, "
1214 "new wasted space = %d\n", keg->uk_name, wastedspace,
1215 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1216 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1217#endif
1218 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1219 }
1220
1221 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1222 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1223 keg->uk_flags |= UMA_ZONE_HASH;
1224}
1225
1226/*
1227 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1228 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1229 * more complicated.
1230 *
1231 * Arguments
1232 * keg The keg we should initialize
1233 *
1234 * Returns
1235 * Nothing
1236 */
1237static void
1238keg_large_init(uma_keg_t keg)
1239{
1240
1241 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1242 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1243 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1244 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1245 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
1246
1247 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1248 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1249 keg->uk_ipers = 1;
1250 keg->uk_rsize = keg->uk_size;
1251
1252 /* We can't do OFFPAGE if we're internal, bail out here. */
1253 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1254 return;
1255
1256 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1257 if ((keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1258 keg->uk_flags |= UMA_ZONE_HASH;
1259}
1260
1261static void
1262keg_cachespread_init(uma_keg_t keg)
1263{
1264 int alignsize;
1265 int trailer;
1266 int pages;
1267 int rsize;
1268
1269 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1270 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1271
1272 alignsize = keg->uk_align + 1;
1273 rsize = keg->uk_size;
1274 /*
1275 * We want one item to start on every align boundary in a page. To
1276 * do this we will span pages. We will also extend the item by the
1277 * size of align if it is an even multiple of align. Otherwise, it
1278 * would fall on the same boundary every time.
1279 */
1280 if (rsize & keg->uk_align)
1281 rsize = (rsize & ~keg->uk_align) + alignsize;
1282 if ((rsize & alignsize) == 0)
1283 rsize += alignsize;
1284 trailer = rsize - keg->uk_size;
1285 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1286 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1287 keg->uk_rsize = rsize;
1288 keg->uk_ppera = pages;
1289 keg->uk_slabsize = UMA_SLAB_SIZE;
1290 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1291 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1292 KASSERT(keg->uk_ipers <= uma_max_ipers,
1293 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1294 keg->uk_ipers));
1295}
1296
1297/*
1298 * Keg header ctor. This initializes all fields, locks, etc. And inserts
1299 * the keg onto the global keg list.
1300 *
1301 * Arguments/Returns follow uma_ctor specifications
1302 * udata Actually uma_kctor_args
1303 */
1304static int
1305keg_ctor(void *mem, int size, void *udata, int flags)
1306{
1307 struct uma_kctor_args *arg = udata;
1308 uma_keg_t keg = mem;
1309 uma_zone_t zone;
1310
1311 bzero(keg, size);
1312 keg->uk_size = arg->size;
1313 keg->uk_init = arg->uminit;
1314 keg->uk_fini = arg->fini;
1315 keg->uk_align = arg->align;
1316 keg->uk_free = 0;
1317 keg->uk_pages = 0;
1318 keg->uk_flags = arg->flags;
1319 keg->uk_allocf = page_alloc;
1320 keg->uk_freef = page_free;
1321 keg->uk_recurse = 0;
1322 keg->uk_slabzone = NULL;
1323
1324 /*
1325 * The master zone is passed to us at keg-creation time.
1326 */
1327 zone = arg->zone;
1328 keg->uk_name = zone->uz_name;
1329
1330 if (arg->flags & UMA_ZONE_VM)
1331 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1332
1333 if (arg->flags & UMA_ZONE_ZINIT)
1334 keg->uk_init = zero_init;
1335
1336 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1337 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1338
1339 if (arg->flags & UMA_ZONE_PCPU)
1340#ifdef SMP
1341 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1342#else
1343 keg->uk_flags &= ~UMA_ZONE_PCPU;
1344#endif
1345
1346 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
1347 keg_cachespread_init(keg);
1348 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1349 if (keg->uk_size >
1350 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1351 sizeof(uint32_t)))
1352 keg_large_init(keg);
1353 else
1354 keg_small_init(keg);
1355 } else {
1356 if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
1357 keg_large_init(keg);
1358 else
1359 keg_small_init(keg);
1360 }
1361
1362 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1363 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1364 if (keg->uk_ipers > uma_max_ipers_ref)
1365 panic("Too many ref items per zone: %d > %d\n",
1366 keg->uk_ipers, uma_max_ipers_ref);
1367 keg->uk_slabzone = slabrefzone;
1368 } else
1369 keg->uk_slabzone = slabzone;
1370 }
1371
1372 /*
1373 * If we haven't booted yet we need allocations to go through the
1374 * startup cache until the vm is ready.
1375 */
1376 if (keg->uk_ppera == 1) {
1377#ifdef UMA_MD_SMALL_ALLOC
1378 keg->uk_allocf = uma_small_alloc;
1379 keg->uk_freef = uma_small_free;
1380
1381 if (booted < UMA_STARTUP)
1382 keg->uk_allocf = startup_alloc;
1383#else
1384 if (booted < UMA_STARTUP2)
1385 keg->uk_allocf = startup_alloc;
1386#endif
1387 } else if (booted < UMA_STARTUP2 &&
1388 (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1389 keg->uk_allocf = startup_alloc;
1390
1391 /*
1392 * Initialize keg's lock (shared among zones).
1393 */
1394 if (arg->flags & UMA_ZONE_MTXCLASS)
1395 KEG_LOCK_INIT(keg, 1);
1396 else
1397 KEG_LOCK_INIT(keg, 0);
1398
1399 /*
1400 * If we're putting the slab header in the actual page we need to
1401 * figure out where in each page it goes. This calculates a right
1402 * justified offset into the memory on an ALIGN_PTR boundary.
1403 */
1404 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1405 u_int totsize;
1406
1407 /* Size of the slab struct and free list */
1408 totsize = sizeof(struct uma_slab);
1409
1410 /* Size of the reference counts. */
1411 if (keg->uk_flags & UMA_ZONE_REFCNT)
1412 totsize += keg->uk_ipers * sizeof(uint32_t);
1413
1414 if (totsize & UMA_ALIGN_PTR)
1415 totsize = (totsize & ~UMA_ALIGN_PTR) +
1416 (UMA_ALIGN_PTR + 1);
1417 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1418
1419 /*
1420 * The only way the following is possible is if with our
1421 * UMA_ALIGN_PTR adjustments we are now bigger than
1422 * UMA_SLAB_SIZE. I haven't checked whether this is
1423 * mathematically possible for all cases, so we make
1424 * sure here anyway.
1425 */
1426 totsize = keg->uk_pgoff + sizeof(struct uma_slab);
1427 if (keg->uk_flags & UMA_ZONE_REFCNT)
1428 totsize += keg->uk_ipers * sizeof(uint32_t);
1429 if (totsize > PAGE_SIZE * keg->uk_ppera) {
1430 printf("zone %s ipers %d rsize %d size %d\n",
1431 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1432 keg->uk_size);
1433 panic("UMA slab won't fit.");
1434 }
1435 }
1436
1437 if (keg->uk_flags & UMA_ZONE_HASH)
1438 hash_alloc(&keg->uk_hash);
1439
1440#ifdef UMA_DEBUG
1441 printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
1442 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
1443 keg->uk_ipers, keg->uk_ppera,
1444 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
1445#endif
1446
1447 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
1448
1449 mtx_lock(&uma_mtx);
1450 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
1451 mtx_unlock(&uma_mtx);
1452 return (0);
1453}
1454
1455/*
1456 * Zone header ctor. This initializes all fields, locks, etc.
1457 *
1458 * Arguments/Returns follow uma_ctor specifications
1459 * udata Actually uma_zctor_args
1460 */
1461static int
1462zone_ctor(void *mem, int size, void *udata, int flags)
1463{
1464 struct uma_zctor_args *arg = udata;
1465 uma_zone_t zone = mem;
1466 uma_zone_t z;
1467 uma_keg_t keg;
1468
1469 bzero(zone, size);
1470 zone->uz_name = arg->name;
1471 zone->uz_ctor = arg->ctor;
1472 zone->uz_dtor = arg->dtor;
1473 zone->uz_slab = zone_fetch_slab;
1474 zone->uz_init = NULL;
1475 zone->uz_fini = NULL;
1476 zone->uz_allocs = 0;
1477 zone->uz_frees = 0;
1478 zone->uz_fails = 0;
1479 zone->uz_sleeps = 0;
1480 zone->uz_fills = zone->uz_count = 0;
1481 zone->uz_flags = 0;
1482 zone->uz_warning = NULL;
1483 timevalclear(&zone->uz_ratecheck);
1484 keg = arg->keg;
1485
| 919 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
920 flags);
921 KEG_LOCK(keg);
922 return (NULL);
923 }
924 }
925 KEG_LOCK(keg);
926
927 if (keg->uk_flags & UMA_ZONE_HASH)
928 UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
929
930 keg->uk_pages += keg->uk_ppera;
931 keg->uk_free += keg->uk_ipers;
932
933 return (slab);
934}
935
936/*
937 * This function is intended to be used early on in place of page_alloc() so
938 * that we may use the boot time page cache to satisfy allocations before
939 * the VM is ready.
940 */
941static void *
942startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
943{
944 uma_keg_t keg;
945 uma_slab_t tmps;
946 int pages, check_pages;
947
948 keg = zone_first_keg(zone);
949 pages = howmany(bytes, PAGE_SIZE);
950 check_pages = pages - 1;
951 KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n"));
952
953 /*
954 * Check our small startup cache to see if it has pages remaining.
955 */
956 mtx_lock(&uma_boot_pages_mtx);
957
958 /* First check if we have enough room. */
959 tmps = LIST_FIRST(&uma_boot_pages);
960 while (tmps != NULL && check_pages-- > 0)
961 tmps = LIST_NEXT(tmps, us_link);
962 if (tmps != NULL) {
963 /*
964 * It's ok to lose tmps references. The last one will
965 * have tmps->us_data pointing to the start address of
966 * "pages" contiguous pages of memory.
967 */
968 while (pages-- > 0) {
969 tmps = LIST_FIRST(&uma_boot_pages);
970 LIST_REMOVE(tmps, us_link);
971 }
972 mtx_unlock(&uma_boot_pages_mtx);
973 *pflag = tmps->us_flags;
974 return (tmps->us_data);
975 }
976 mtx_unlock(&uma_boot_pages_mtx);
977 if (booted < UMA_STARTUP2)
978 panic("UMA: Increase vm.boot_pages");
979 /*
980 * Now that we've booted reset these users to their real allocator.
981 */
982#ifdef UMA_MD_SMALL_ALLOC
983 keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc;
984#else
985 keg->uk_allocf = page_alloc;
986#endif
987 return keg->uk_allocf(zone, bytes, pflag, wait);
988}
989
990/*
991 * Allocates a number of pages from the system
992 *
993 * Arguments:
994 * bytes The number of bytes requested
995 * wait Shall we wait?
996 *
997 * Returns:
998 * A pointer to the alloced memory or possibly
999 * NULL if M_NOWAIT is set.
1000 */
1001static void *
1002page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait)
1003{
1004 void *p; /* Returned page */
1005
1006 *pflag = UMA_SLAB_KMEM;
1007 p = (void *) kmem_malloc(kmem_map, bytes, wait);
1008
1009 return (p);
1010}
1011
1012/*
1013 * Allocates a number of pages from within an object
1014 *
1015 * Arguments:
1016 * bytes The number of bytes requested
1017 * wait Shall we wait?
1018 *
1019 * Returns:
1020 * A pointer to the alloced memory or possibly
1021 * NULL if M_NOWAIT is set.
1022 */
1023static void *
1024noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait)
1025{
1026 TAILQ_HEAD(, vm_page) alloctail;
1027 u_long npages;
1028 vm_offset_t retkva, zkva;
1029 vm_page_t p, p_next;
1030 uma_keg_t keg;
1031
1032 TAILQ_INIT(&alloctail);
1033 keg = zone_first_keg(zone);
1034
1035 npages = howmany(bytes, PAGE_SIZE);
1036 while (npages > 0) {
1037 p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
1038 VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
1039 if (p != NULL) {
1040 /*
1041 * Since the page does not belong to an object, its
1042 * listq is unused.
1043 */
1044 TAILQ_INSERT_TAIL(&alloctail, p, listq);
1045 npages--;
1046 continue;
1047 }
1048 if (wait & M_WAITOK) {
1049 VM_WAIT;
1050 continue;
1051 }
1052
1053 /*
1054 * Page allocation failed, free intermediate pages and
1055 * exit.
1056 */
1057 TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
1058 vm_page_unwire(p, 0);
1059 vm_page_free(p);
1060 }
1061 return (NULL);
1062 }
1063 *flags = UMA_SLAB_PRIV;
1064 zkva = keg->uk_kva +
1065 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
1066 retkva = zkva;
1067 TAILQ_FOREACH(p, &alloctail, listq) {
1068 pmap_qenter(zkva, &p, 1);
1069 zkva += PAGE_SIZE;
1070 }
1071
1072 return ((void *)retkva);
1073}
1074
1075/*
1076 * Frees a number of pages to the system
1077 *
1078 * Arguments:
1079 * mem A pointer to the memory to be freed
1080 * size The size of the memory being freed
1081 * flags The original p->us_flags field
1082 *
1083 * Returns:
1084 * Nothing
1085 */
1086static void
1087page_free(void *mem, int size, uint8_t flags)
1088{
1089 vm_map_t map;
1090
1091 if (flags & UMA_SLAB_KMEM)
1092 map = kmem_map;
1093 else if (flags & UMA_SLAB_KERNEL)
1094 map = kernel_map;
1095 else
1096 panic("UMA: page_free used with invalid flags %d", flags);
1097
1098 kmem_free(map, (vm_offset_t)mem, size);
1099}
1100
1101/*
1102 * Zero fill initializer
1103 *
1104 * Arguments/Returns follow uma_init specifications
1105 */
1106static int
1107zero_init(void *mem, int size, int flags)
1108{
1109 bzero(mem, size);
1110 return (0);
1111}
1112
1113/*
1114 * Finish creating a small uma keg. This calculates ipers, and the keg size.
1115 *
1116 * Arguments
1117 * keg The zone we should initialize
1118 *
1119 * Returns
1120 * Nothing
1121 */
1122static void
1123keg_small_init(uma_keg_t keg)
1124{
1125 u_int rsize;
1126 u_int memused;
1127 u_int wastedspace;
1128 u_int shsize;
1129
1130 if (keg->uk_flags & UMA_ZONE_PCPU) {
1131 KASSERT(mp_ncpus > 0, ("%s: ncpus %d\n", __func__, mp_ncpus));
1132 keg->uk_slabsize = sizeof(struct pcpu);
1133 keg->uk_ppera = howmany(mp_ncpus * sizeof(struct pcpu),
1134 PAGE_SIZE);
1135 } else {
1136 keg->uk_slabsize = UMA_SLAB_SIZE;
1137 keg->uk_ppera = 1;
1138 }
1139
1140 /*
1141 * Calculate the size of each allocation (rsize) according to
1142 * alignment. If the requested size is smaller than we have
1143 * allocation bits for we round it up.
1144 */
1145 rsize = keg->uk_size;
1146 if (rsize < keg->uk_slabsize / SLAB_SETSIZE)
1147 rsize = keg->uk_slabsize / SLAB_SETSIZE;
1148 if (rsize & keg->uk_align)
1149 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1150 keg->uk_rsize = rsize;
1151
1152 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1153 keg->uk_rsize < sizeof(struct pcpu),
1154 ("%s: size %u too large", __func__, keg->uk_rsize));
1155
1156 if (keg->uk_flags & UMA_ZONE_REFCNT)
1157 rsize += sizeof(uint32_t);
1158
1159 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
1160 shsize = 0;
1161 else
1162 shsize = sizeof(struct uma_slab);
1163
1164 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1165 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1166 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1167
1168 memused = keg->uk_ipers * rsize + shsize;
1169 wastedspace = keg->uk_slabsize - memused;
1170
1171 /*
1172 * We can't do OFFPAGE if we're internal or if we've been
1173 * asked to not go to the VM for buckets. If we do this we
1174 * may end up going to the VM (kmem_map) for slabs which we
1175 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
1176 * result of UMA_ZONE_VM, which clearly forbids it.
1177 */
1178 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1179 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1180 return;
1181
1182 /*
1183 * See if using an OFFPAGE slab will limit our waste. Only do
1184 * this if it permits more items per-slab.
1185 *
1186 * XXX We could try growing slabsize to limit max waste as well.
1187 * Historically this was not done because the VM could not
1188 * efficiently handle contiguous allocations.
1189 */
1190 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1191 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1192 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1193 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE,
1194 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1195#ifdef UMA_DEBUG
1196 printf("UMA decided we need offpage slab headers for "
1197 "keg: %s, calculated wastedspace = %d, "
1198 "maximum wasted space allowed = %d, "
1199 "calculated ipers = %d, "
1200 "new wasted space = %d\n", keg->uk_name, wastedspace,
1201 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1202 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1203#endif
1204 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1205 }
1206
1207 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1208 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1209 keg->uk_flags |= UMA_ZONE_HASH;
1210}
1211
1212/*
1213 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1214 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1215 * more complicated.
1216 *
1217 * Arguments
1218 * keg The keg we should initialize
1219 *
1220 * Returns
1221 * Nothing
1222 */
1223static void
1224keg_large_init(uma_keg_t keg)
1225{
1226
1227 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1228 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1229 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1230 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1231 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
1232
1233 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1234 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1235 keg->uk_ipers = 1;
1236 keg->uk_rsize = keg->uk_size;
1237
1238 /* We can't do OFFPAGE if we're internal, bail out here. */
1239 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1240 return;
1241
1242 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1243 if ((keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1244 keg->uk_flags |= UMA_ZONE_HASH;
1245}
1246
1247static void
1248keg_cachespread_init(uma_keg_t keg)
1249{
1250 int alignsize;
1251 int trailer;
1252 int pages;
1253 int rsize;
1254
1255 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1256 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1257
1258 alignsize = keg->uk_align + 1;
1259 rsize = keg->uk_size;
1260 /*
1261 * We want one item to start on every align boundary in a page. To
1262 * do this we will span pages. We will also extend the item by the
1263 * size of align if it is an even multiple of align. Otherwise, it
1264 * would fall on the same boundary every time.
1265 */
1266 if (rsize & keg->uk_align)
1267 rsize = (rsize & ~keg->uk_align) + alignsize;
1268 if ((rsize & alignsize) == 0)
1269 rsize += alignsize;
1270 trailer = rsize - keg->uk_size;
1271 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1272 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1273 keg->uk_rsize = rsize;
1274 keg->uk_ppera = pages;
1275 keg->uk_slabsize = UMA_SLAB_SIZE;
1276 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1277 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1278 KASSERT(keg->uk_ipers <= uma_max_ipers,
1279 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1280 keg->uk_ipers));
1281}
1282
1283/*
1284 * Keg header ctor. This initializes all fields, locks, etc. And inserts
1285 * the keg onto the global keg list.
1286 *
1287 * Arguments/Returns follow uma_ctor specifications
1288 * udata Actually uma_kctor_args
1289 */
1290static int
1291keg_ctor(void *mem, int size, void *udata, int flags)
1292{
1293 struct uma_kctor_args *arg = udata;
1294 uma_keg_t keg = mem;
1295 uma_zone_t zone;
1296
1297 bzero(keg, size);
1298 keg->uk_size = arg->size;
1299 keg->uk_init = arg->uminit;
1300 keg->uk_fini = arg->fini;
1301 keg->uk_align = arg->align;
1302 keg->uk_free = 0;
1303 keg->uk_pages = 0;
1304 keg->uk_flags = arg->flags;
1305 keg->uk_allocf = page_alloc;
1306 keg->uk_freef = page_free;
1307 keg->uk_recurse = 0;
1308 keg->uk_slabzone = NULL;
1309
1310 /*
1311 * The master zone is passed to us at keg-creation time.
1312 */
1313 zone = arg->zone;
1314 keg->uk_name = zone->uz_name;
1315
1316 if (arg->flags & UMA_ZONE_VM)
1317 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1318
1319 if (arg->flags & UMA_ZONE_ZINIT)
1320 keg->uk_init = zero_init;
1321
1322 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1323 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1324
1325 if (arg->flags & UMA_ZONE_PCPU)
1326#ifdef SMP
1327 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1328#else
1329 keg->uk_flags &= ~UMA_ZONE_PCPU;
1330#endif
1331
1332 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) {
1333 keg_cachespread_init(keg);
1334 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1335 if (keg->uk_size >
1336 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1337 sizeof(uint32_t)))
1338 keg_large_init(keg);
1339 else
1340 keg_small_init(keg);
1341 } else {
1342 if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
1343 keg_large_init(keg);
1344 else
1345 keg_small_init(keg);
1346 }
1347
1348 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1349 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1350 if (keg->uk_ipers > uma_max_ipers_ref)
1351 panic("Too many ref items per zone: %d > %d\n",
1352 keg->uk_ipers, uma_max_ipers_ref);
1353 keg->uk_slabzone = slabrefzone;
1354 } else
1355 keg->uk_slabzone = slabzone;
1356 }
1357
1358 /*
1359 * If we haven't booted yet we need allocations to go through the
1360 * startup cache until the vm is ready.
1361 */
1362 if (keg->uk_ppera == 1) {
1363#ifdef UMA_MD_SMALL_ALLOC
1364 keg->uk_allocf = uma_small_alloc;
1365 keg->uk_freef = uma_small_free;
1366
1367 if (booted < UMA_STARTUP)
1368 keg->uk_allocf = startup_alloc;
1369#else
1370 if (booted < UMA_STARTUP2)
1371 keg->uk_allocf = startup_alloc;
1372#endif
1373 } else if (booted < UMA_STARTUP2 &&
1374 (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1375 keg->uk_allocf = startup_alloc;
1376
1377 /*
1378 * Initialize keg's lock (shared among zones).
1379 */
1380 if (arg->flags & UMA_ZONE_MTXCLASS)
1381 KEG_LOCK_INIT(keg, 1);
1382 else
1383 KEG_LOCK_INIT(keg, 0);
1384
1385 /*
1386 * If we're putting the slab header in the actual page we need to
1387 * figure out where in each page it goes. This calculates a right
1388 * justified offset into the memory on an ALIGN_PTR boundary.
1389 */
1390 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1391 u_int totsize;
1392
1393 /* Size of the slab struct and free list */
1394 totsize = sizeof(struct uma_slab);
1395
1396 /* Size of the reference counts. */
1397 if (keg->uk_flags & UMA_ZONE_REFCNT)
1398 totsize += keg->uk_ipers * sizeof(uint32_t);
1399
1400 if (totsize & UMA_ALIGN_PTR)
1401 totsize = (totsize & ~UMA_ALIGN_PTR) +
1402 (UMA_ALIGN_PTR + 1);
1403 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1404
1405 /*
1406 * The only way the following is possible is if with our
1407 * UMA_ALIGN_PTR adjustments we are now bigger than
1408 * UMA_SLAB_SIZE. I haven't checked whether this is
1409 * mathematically possible for all cases, so we make
1410 * sure here anyway.
1411 */
1412 totsize = keg->uk_pgoff + sizeof(struct uma_slab);
1413 if (keg->uk_flags & UMA_ZONE_REFCNT)
1414 totsize += keg->uk_ipers * sizeof(uint32_t);
1415 if (totsize > PAGE_SIZE * keg->uk_ppera) {
1416 printf("zone %s ipers %d rsize %d size %d\n",
1417 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1418 keg->uk_size);
1419 panic("UMA slab won't fit.");
1420 }
1421 }
1422
1423 if (keg->uk_flags & UMA_ZONE_HASH)
1424 hash_alloc(&keg->uk_hash);
1425
1426#ifdef UMA_DEBUG
1427 printf("UMA: %s(%p) size %d(%d) flags %#x ipers %d ppera %d out %d free %d\n",
1428 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
1429 keg->uk_ipers, keg->uk_ppera,
1430 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
1431#endif
1432
1433 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
1434
1435 mtx_lock(&uma_mtx);
1436 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
1437 mtx_unlock(&uma_mtx);
1438 return (0);
1439}
1440
1441/*
1442 * Zone header ctor. This initializes all fields, locks, etc.
1443 *
1444 * Arguments/Returns follow uma_ctor specifications
1445 * udata Actually uma_zctor_args
1446 */
1447static int
1448zone_ctor(void *mem, int size, void *udata, int flags)
1449{
1450 struct uma_zctor_args *arg = udata;
1451 uma_zone_t zone = mem;
1452 uma_zone_t z;
1453 uma_keg_t keg;
1454
1455 bzero(zone, size);
1456 zone->uz_name = arg->name;
1457 zone->uz_ctor = arg->ctor;
1458 zone->uz_dtor = arg->dtor;
1459 zone->uz_slab = zone_fetch_slab;
1460 zone->uz_init = NULL;
1461 zone->uz_fini = NULL;
1462 zone->uz_allocs = 0;
1463 zone->uz_frees = 0;
1464 zone->uz_fails = 0;
1465 zone->uz_sleeps = 0;
1466 zone->uz_fills = zone->uz_count = 0;
1467 zone->uz_flags = 0;
1468 zone->uz_warning = NULL;
1469 timevalclear(&zone->uz_ratecheck);
1470 keg = arg->keg;
1471
|
| 1472 /*
1473 * This is a pure cache zone, no kegs.
1474 */
1475 if (arg->import) {
1476 zone->uz_import = arg->import;
1477 zone->uz_release = arg->release;
1478 zone->uz_arg = arg->arg;
1479 zone->uz_count = BUCKET_MAX;
1480 return (0);
1481 }
1482
1483 /*
1484 * Use the regular zone/keg/slab allocator.
1485 */
1486 zone->uz_import = (uma_import)zone_import;
1487 zone->uz_release = (uma_release)zone_release;
1488 zone->uz_arg = zone;
1489
|
1486 if (arg->flags & UMA_ZONE_SECONDARY) {
1487 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1488 zone->uz_init = arg->uminit;
1489 zone->uz_fini = arg->fini;
1490 zone->uz_lock = &keg->uk_lock;
1491 zone->uz_flags |= UMA_ZONE_SECONDARY;
1492 mtx_lock(&uma_mtx);
1493 ZONE_LOCK(zone);
1494 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1495 if (LIST_NEXT(z, uz_link) == NULL) {
1496 LIST_INSERT_AFTER(z, zone, uz_link);
1497 break;
1498 }
1499 }
1500 ZONE_UNLOCK(zone);
1501 mtx_unlock(&uma_mtx);
1502 } else if (keg == NULL) {
1503 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
1504 arg->align, arg->flags)) == NULL)
1505 return (ENOMEM);
1506 } else {
1507 struct uma_kctor_args karg;
1508 int error;
1509
1510 /* We should only be here from uma_startup() */
1511 karg.size = arg->size;
1512 karg.uminit = arg->uminit;
1513 karg.fini = arg->fini;
1514 karg.align = arg->align;
1515 karg.flags = arg->flags;
1516 karg.zone = zone;
1517 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1518 flags);
1519 if (error)
1520 return (error);
1521 }
| 1490 if (arg->flags & UMA_ZONE_SECONDARY) {
1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1492 zone->uz_init = arg->uminit;
1493 zone->uz_fini = arg->fini;
1494 zone->uz_lock = &keg->uk_lock;
1495 zone->uz_flags |= UMA_ZONE_SECONDARY;
1496 mtx_lock(&uma_mtx);
1497 ZONE_LOCK(zone);
1498 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1499 if (LIST_NEXT(z, uz_link) == NULL) {
1500 LIST_INSERT_AFTER(z, zone, uz_link);
1501 break;
1502 }
1503 }
1504 ZONE_UNLOCK(zone);
1505 mtx_unlock(&uma_mtx);
1506 } else if (keg == NULL) {
1507 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
1508 arg->align, arg->flags)) == NULL)
1509 return (ENOMEM);
1510 } else {
1511 struct uma_kctor_args karg;
1512 int error;
1513
1514 /* We should only be here from uma_startup() */
1515 karg.size = arg->size;
1516 karg.uminit = arg->uminit;
1517 karg.fini = arg->fini;
1518 karg.align = arg->align;
1519 karg.flags = arg->flags;
1520 karg.zone = zone;
1521 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1522 flags);
1523 if (error)
1524 return (error);
1525 }
|
| 1526
|
1522 /*
1523 * Link in the first keg.
1524 */
1525 zone->uz_klink.kl_keg = keg;
1526 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1527 zone->uz_lock = &keg->uk_lock;
1528 zone->uz_size = keg->uk_size;
1529 zone->uz_flags |= (keg->uk_flags &
1530 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
1531
1532 /*
1533 * Some internal zones don't have room allocated for the per cpu
1534 * caches. If we're internal, bail out here.
1535 */
1536 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1537 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1538 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1539 return (0);
1540 }
1541
1542 if (keg->uk_flags & UMA_ZONE_MAXBUCKET)
1543 zone->uz_count = BUCKET_MAX;
1544 else if (keg->uk_ipers <= BUCKET_MAX)
1545 zone->uz_count = keg->uk_ipers;
1546 else
1547 zone->uz_count = BUCKET_MAX;
1548 return (0);
1549}
1550
1551/*
1552 * Keg header dtor. This frees all data, destroys locks, frees the hash
1553 * table and removes the keg from the global list.
1554 *
1555 * Arguments/Returns follow uma_dtor specifications
1556 * udata unused
1557 */
1558static void
1559keg_dtor(void *arg, int size, void *udata)
1560{
1561 uma_keg_t keg;
1562
1563 keg = (uma_keg_t)arg;
1564 KEG_LOCK(keg);
1565 if (keg->uk_free != 0) {
1566 printf("Freed UMA keg was not empty (%d items). "
1567 " Lost %d pages of memory.\n",
1568 keg->uk_free, keg->uk_pages);
1569 }
1570 KEG_UNLOCK(keg);
1571
1572 hash_free(&keg->uk_hash);
1573
1574 KEG_LOCK_FINI(keg);
1575}
1576
1577/*
1578 * Zone header dtor.
1579 *
1580 * Arguments/Returns follow uma_dtor specifications
1581 * udata unused
1582 */
1583static void
1584zone_dtor(void *arg, int size, void *udata)
1585{
1586 uma_klink_t klink;
1587 uma_zone_t zone;
1588 uma_keg_t keg;
1589
1590 zone = (uma_zone_t)arg;
1591 keg = zone_first_keg(zone);
1592
1593 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
1594 cache_drain(zone);
1595
1596 mtx_lock(&uma_mtx);
1597 LIST_REMOVE(zone, uz_link);
1598 mtx_unlock(&uma_mtx);
1599 /*
1600 * XXX there are some races here where
1601 * the zone can be drained but zone lock
1602 * released and then refilled before we
1603 * remove it... we dont care for now
1604 */
1605 zone_drain_wait(zone, M_WAITOK);
1606 /*
1607 * Unlink all of our kegs.
1608 */
1609 while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
1610 klink->kl_keg = NULL;
1611 LIST_REMOVE(klink, kl_link);
1612 if (klink == &zone->uz_klink)
1613 continue;
1614 free(klink, M_TEMP);
1615 }
1616 /*
1617 * We only destroy kegs from non secondary zones.
1618 */
| 1527 /*
1528 * Link in the first keg.
1529 */
1530 zone->uz_klink.kl_keg = keg;
1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1532 zone->uz_lock = &keg->uk_lock;
1533 zone->uz_size = keg->uk_size;
1534 zone->uz_flags |= (keg->uk_flags &
1535 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
1536
1537 /*
1538 * Some internal zones don't have room allocated for the per cpu
1539 * caches. If we're internal, bail out here.
1540 */
1541 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1542 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1543 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1544 return (0);
1545 }
1546
1547 if (keg->uk_flags & UMA_ZONE_MAXBUCKET)
1548 zone->uz_count = BUCKET_MAX;
1549 else if (keg->uk_ipers <= BUCKET_MAX)
1550 zone->uz_count = keg->uk_ipers;
1551 else
1552 zone->uz_count = BUCKET_MAX;
1553 return (0);
1554}
1555
1556/*
1557 * Keg header dtor. This frees all data, destroys locks, frees the hash
1558 * table and removes the keg from the global list.
1559 *
1560 * Arguments/Returns follow uma_dtor specifications
1561 * udata unused
1562 */
1563static void
1564keg_dtor(void *arg, int size, void *udata)
1565{
1566 uma_keg_t keg;
1567
1568 keg = (uma_keg_t)arg;
1569 KEG_LOCK(keg);
1570 if (keg->uk_free != 0) {
1571 printf("Freed UMA keg was not empty (%d items). "
1572 " Lost %d pages of memory.\n",
1573 keg->uk_free, keg->uk_pages);
1574 }
1575 KEG_UNLOCK(keg);
1576
1577 hash_free(&keg->uk_hash);
1578
1579 KEG_LOCK_FINI(keg);
1580}
1581
1582/*
1583 * Zone header dtor.
1584 *
1585 * Arguments/Returns follow uma_dtor specifications
1586 * udata unused
1587 */
1588static void
1589zone_dtor(void *arg, int size, void *udata)
1590{
1591 uma_klink_t klink;
1592 uma_zone_t zone;
1593 uma_keg_t keg;
1594
1595 zone = (uma_zone_t)arg;
1596 keg = zone_first_keg(zone);
1597
1598 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
1599 cache_drain(zone);
1600
1601 mtx_lock(&uma_mtx);
1602 LIST_REMOVE(zone, uz_link);
1603 mtx_unlock(&uma_mtx);
1604 /*
1605 * XXX there are some races here where
1606 * the zone can be drained but zone lock
1607 * released and then refilled before we
1608 * remove it... we dont care for now
1609 */
1610 zone_drain_wait(zone, M_WAITOK);
1611 /*
1612 * Unlink all of our kegs.
1613 */
1614 while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) {
1615 klink->kl_keg = NULL;
1616 LIST_REMOVE(klink, kl_link);
1617 if (klink == &zone->uz_klink)
1618 continue;
1619 free(klink, M_TEMP);
1620 }
1621 /*
1622 * We only destroy kegs from non secondary zones.
1623 */
|
1619 if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
| 1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
|
1620 mtx_lock(&uma_mtx);
1621 LIST_REMOVE(keg, uk_link);
1622 mtx_unlock(&uma_mtx);
| 1625 mtx_lock(&uma_mtx);
1626 LIST_REMOVE(keg, uk_link);
1627 mtx_unlock(&uma_mtx);
|
1623 zone_free_item(kegs, keg, NULL, SKIP_NONE,
1624 ZFREE_STATFREE);
| 1628 zone_free_item(kegs, keg, NULL, SKIP_NONE);
|
1625 }
1626}
1627
1628/*
1629 * Traverses every zone in the system and calls a callback
1630 *
1631 * Arguments:
1632 * zfunc A pointer to a function which accepts a zone
1633 * as an argument.
1634 *
1635 * Returns:
1636 * Nothing
1637 */
1638static void
1639zone_foreach(void (*zfunc)(uma_zone_t))
1640{
1641 uma_keg_t keg;
1642 uma_zone_t zone;
1643
1644 mtx_lock(&uma_mtx);
1645 LIST_FOREACH(keg, &uma_kegs, uk_link) {
1646 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
1647 zfunc(zone);
1648 }
1649 mtx_unlock(&uma_mtx);
1650}
1651
1652/* Public functions */
1653/* See uma.h */
1654void
1655uma_startup(void *bootmem, int boot_pages)
1656{
1657 struct uma_zctor_args args;
1658 uma_slab_t slab;
1659 u_int slabsize;
1660 int i;
1661
1662#ifdef UMA_DEBUG
1663 printf("Creating uma keg headers zone and keg.\n");
1664#endif
1665 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1666
1667 /* "manually" create the initial zone */
| 1629 }
1630}
1631
1632/*
1633 * Traverses every zone in the system and calls a callback
1634 *
1635 * Arguments:
1636 * zfunc A pointer to a function which accepts a zone
1637 * as an argument.
1638 *
1639 * Returns:
1640 * Nothing
1641 */
1642static void
1643zone_foreach(void (*zfunc)(uma_zone_t))
1644{
1645 uma_keg_t keg;
1646 uma_zone_t zone;
1647
1648 mtx_lock(&uma_mtx);
1649 LIST_FOREACH(keg, &uma_kegs, uk_link) {
1650 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
1651 zfunc(zone);
1652 }
1653 mtx_unlock(&uma_mtx);
1654}
1655
1656/* Public functions */
1657/* See uma.h */
1658void
1659uma_startup(void *bootmem, int boot_pages)
1660{
1661 struct uma_zctor_args args;
1662 uma_slab_t slab;
1663 u_int slabsize;
1664 int i;
1665
1666#ifdef UMA_DEBUG
1667 printf("Creating uma keg headers zone and keg.\n");
1668#endif
1669 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1670
1671 /* "manually" create the initial zone */
|
| 1672 memset(&args, 0, sizeof(args));
|
1668 args.name = "UMA Kegs";
1669 args.size = sizeof(struct uma_keg);
1670 args.ctor = keg_ctor;
1671 args.dtor = keg_dtor;
1672 args.uminit = zero_init;
1673 args.fini = NULL;
1674 args.keg = &masterkeg;
1675 args.align = 32 - 1;
1676 args.flags = UMA_ZFLAG_INTERNAL;
1677 /* The initial zone has no Per cpu queues so it's smaller */
1678 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
1679
1680#ifdef UMA_DEBUG
1681 printf("Filling boot free list.\n");
1682#endif
1683 for (i = 0; i < boot_pages; i++) {
1684 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
1685 slab->us_data = (uint8_t *)slab;
1686 slab->us_flags = UMA_SLAB_BOOT;
1687 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
1688 }
1689 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
1690
1691#ifdef UMA_DEBUG
1692 printf("Creating uma zone headers zone and keg.\n");
1693#endif
1694 args.name = "UMA Zones";
1695 args.size = sizeof(struct uma_zone) +
1696 (sizeof(struct uma_cache) * (mp_maxid + 1));
1697 args.ctor = zone_ctor;
1698 args.dtor = zone_dtor;
1699 args.uminit = zero_init;
1700 args.fini = NULL;
1701 args.keg = NULL;
1702 args.align = 32 - 1;
1703 args.flags = UMA_ZFLAG_INTERNAL;
1704 /* The initial zone has no Per cpu queues so it's smaller */
1705 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
1706
1707#ifdef UMA_DEBUG
1708 printf("Initializing pcpu cache locks.\n");
1709#endif
1710#ifdef UMA_DEBUG
1711 printf("Creating slab and hash zones.\n");
1712#endif
1713
1714 /* Now make a zone for slab headers */
1715 slabzone = uma_zcreate("UMA Slabs",
1716 sizeof(struct uma_slab),
1717 NULL, NULL, NULL, NULL,
1718 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1719
1720 /*
1721 * We also create a zone for the bigger slabs with reference
1722 * counts in them, to accomodate UMA_ZONE_REFCNT zones.
1723 */
1724 slabsize = sizeof(struct uma_slab_refcnt);
1725 slabsize += uma_max_ipers_ref * sizeof(uint32_t);
1726 slabrefzone = uma_zcreate("UMA RCntSlabs",
1727 slabsize,
1728 NULL, NULL, NULL, NULL,
1729 UMA_ALIGN_PTR,
1730 UMA_ZFLAG_INTERNAL);
1731
1732 hashzone = uma_zcreate("UMA Hash",
1733 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
1734 NULL, NULL, NULL, NULL,
1735 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1736
1737 bucket_init();
1738
1739 booted = UMA_STARTUP;
1740
1741#ifdef UMA_DEBUG
1742 printf("UMA startup complete.\n");
1743#endif
1744}
1745
1746/* see uma.h */
1747void
1748uma_startup2(void)
1749{
1750 booted = UMA_STARTUP2;
1751 bucket_enable();
1752#ifdef UMA_DEBUG
1753 printf("UMA startup2 complete.\n");
1754#endif
1755}
1756
1757/*
1758 * Initialize our callout handle
1759 *
1760 */
1761
1762static void
1763uma_startup3(void)
1764{
1765#ifdef UMA_DEBUG
1766 printf("Starting callout.\n");
1767#endif
1768 callout_init(&uma_callout, CALLOUT_MPSAFE);
1769 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
1770#ifdef UMA_DEBUG
1771 printf("UMA startup3 complete.\n");
1772#endif
1773}
1774
1775static uma_keg_t
1776uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
1777 int align, uint32_t flags)
1778{
1779 struct uma_kctor_args args;
1780
1781 args.size = size;
1782 args.uminit = uminit;
1783 args.fini = fini;
1784 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1785 args.flags = flags;
1786 args.zone = zone;
1787 return (zone_alloc_item(kegs, &args, M_WAITOK));
1788}
1789
1790/* See uma.h */
1791void
1792uma_set_align(int align)
1793{
1794
1795 if (align != UMA_ALIGN_CACHE)
1796 uma_align_cache = align;
1797}
1798
1799/* See uma.h */
1800uma_zone_t
1801uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1802 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1803
1804{
1805 struct uma_zctor_args args;
1806
1807 /* This stuff is essential for the zone ctor */
| 1673 args.name = "UMA Kegs";
1674 args.size = sizeof(struct uma_keg);
1675 args.ctor = keg_ctor;
1676 args.dtor = keg_dtor;
1677 args.uminit = zero_init;
1678 args.fini = NULL;
1679 args.keg = &masterkeg;
1680 args.align = 32 - 1;
1681 args.flags = UMA_ZFLAG_INTERNAL;
1682 /* The initial zone has no Per cpu queues so it's smaller */
1683 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK);
1684
1685#ifdef UMA_DEBUG
1686 printf("Filling boot free list.\n");
1687#endif
1688 for (i = 0; i < boot_pages; i++) {
1689 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE));
1690 slab->us_data = (uint8_t *)slab;
1691 slab->us_flags = UMA_SLAB_BOOT;
1692 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link);
1693 }
1694 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF);
1695
1696#ifdef UMA_DEBUG
1697 printf("Creating uma zone headers zone and keg.\n");
1698#endif
1699 args.name = "UMA Zones";
1700 args.size = sizeof(struct uma_zone) +
1701 (sizeof(struct uma_cache) * (mp_maxid + 1));
1702 args.ctor = zone_ctor;
1703 args.dtor = zone_dtor;
1704 args.uminit = zero_init;
1705 args.fini = NULL;
1706 args.keg = NULL;
1707 args.align = 32 - 1;
1708 args.flags = UMA_ZFLAG_INTERNAL;
1709 /* The initial zone has no Per cpu queues so it's smaller */
1710 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK);
1711
1712#ifdef UMA_DEBUG
1713 printf("Initializing pcpu cache locks.\n");
1714#endif
1715#ifdef UMA_DEBUG
1716 printf("Creating slab and hash zones.\n");
1717#endif
1718
1719 /* Now make a zone for slab headers */
1720 slabzone = uma_zcreate("UMA Slabs",
1721 sizeof(struct uma_slab),
1722 NULL, NULL, NULL, NULL,
1723 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1724
1725 /*
1726 * We also create a zone for the bigger slabs with reference
1727 * counts in them, to accomodate UMA_ZONE_REFCNT zones.
1728 */
1729 slabsize = sizeof(struct uma_slab_refcnt);
1730 slabsize += uma_max_ipers_ref * sizeof(uint32_t);
1731 slabrefzone = uma_zcreate("UMA RCntSlabs",
1732 slabsize,
1733 NULL, NULL, NULL, NULL,
1734 UMA_ALIGN_PTR,
1735 UMA_ZFLAG_INTERNAL);
1736
1737 hashzone = uma_zcreate("UMA Hash",
1738 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT,
1739 NULL, NULL, NULL, NULL,
1740 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
1741
1742 bucket_init();
1743
1744 booted = UMA_STARTUP;
1745
1746#ifdef UMA_DEBUG
1747 printf("UMA startup complete.\n");
1748#endif
1749}
1750
1751/* see uma.h */
1752void
1753uma_startup2(void)
1754{
1755 booted = UMA_STARTUP2;
1756 bucket_enable();
1757#ifdef UMA_DEBUG
1758 printf("UMA startup2 complete.\n");
1759#endif
1760}
1761
1762/*
1763 * Initialize our callout handle
1764 *
1765 */
1766
1767static void
1768uma_startup3(void)
1769{
1770#ifdef UMA_DEBUG
1771 printf("Starting callout.\n");
1772#endif
1773 callout_init(&uma_callout, CALLOUT_MPSAFE);
1774 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL);
1775#ifdef UMA_DEBUG
1776 printf("UMA startup3 complete.\n");
1777#endif
1778}
1779
1780static uma_keg_t
1781uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini,
1782 int align, uint32_t flags)
1783{
1784 struct uma_kctor_args args;
1785
1786 args.size = size;
1787 args.uminit = uminit;
1788 args.fini = fini;
1789 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align;
1790 args.flags = flags;
1791 args.zone = zone;
1792 return (zone_alloc_item(kegs, &args, M_WAITOK));
1793}
1794
1795/* See uma.h */
1796void
1797uma_set_align(int align)
1798{
1799
1800 if (align != UMA_ALIGN_CACHE)
1801 uma_align_cache = align;
1802}
1803
1804/* See uma.h */
1805uma_zone_t
1806uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1807 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1808
1809{
1810 struct uma_zctor_args args;
1811
1812 /* This stuff is essential for the zone ctor */
|
| 1813 memset(&args, 0, sizeof(args));
|
1808 args.name = name;
1809 args.size = size;
1810 args.ctor = ctor;
1811 args.dtor = dtor;
1812 args.uminit = uminit;
1813 args.fini = fini;
1814 args.align = align;
1815 args.flags = flags;
1816 args.keg = NULL;
1817
1818 return (zone_alloc_item(zones, &args, M_WAITOK));
1819}
1820
1821/* See uma.h */
1822uma_zone_t
1823uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1824 uma_init zinit, uma_fini zfini, uma_zone_t master)
1825{
1826 struct uma_zctor_args args;
1827 uma_keg_t keg;
1828
1829 keg = zone_first_keg(master);
| 1814 args.name = name;
1815 args.size = size;
1816 args.ctor = ctor;
1817 args.dtor = dtor;
1818 args.uminit = uminit;
1819 args.fini = fini;
1820 args.align = align;
1821 args.flags = flags;
1822 args.keg = NULL;
1823
1824 return (zone_alloc_item(zones, &args, M_WAITOK));
1825}
1826
1827/* See uma.h */
1828uma_zone_t
1829uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1830 uma_init zinit, uma_fini zfini, uma_zone_t master)
1831{
1832 struct uma_zctor_args args;
1833 uma_keg_t keg;
1834
1835 keg = zone_first_keg(master);
|
| 1836 memset(&args, 0, sizeof(args));
|
1830 args.name = name;
1831 args.size = keg->uk_size;
1832 args.ctor = ctor;
1833 args.dtor = dtor;
1834 args.uminit = zinit;
1835 args.fini = zfini;
1836 args.align = keg->uk_align;
1837 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
1838 args.keg = keg;
1839
1840 /* XXX Attaches only one keg of potentially many. */
1841 return (zone_alloc_item(zones, &args, M_WAITOK));
1842}
1843
| 1837 args.name = name;
1838 args.size = keg->uk_size;
1839 args.ctor = ctor;
1840 args.dtor = dtor;
1841 args.uminit = zinit;
1842 args.fini = zfini;
1843 args.align = keg->uk_align;
1844 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
1845 args.keg = keg;
1846
1847 /* XXX Attaches only one keg of potentially many. */
1848 return (zone_alloc_item(zones, &args, M_WAITOK));
1849}
1850
|
| 1851/* See uma.h */
1852uma_zone_t
1853uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
1854 uma_fini zfini, uma_import zimport, uma_release zrelease,
1855 void *arg, int flags)
1856{
1857 struct uma_zctor_args args;
1858
1859 memset(&args, 0, sizeof(args));
1860 args.name = name;
1861 args.size = 0;
1862 args.ctor = ctor;
1863 args.dtor = dtor;
1864 args.uminit = zinit;
1865 args.fini = zfini;
1866 args.import = zimport;
1867 args.release = zrelease;
1868 args.arg = arg;
1869 args.align = 0;
1870 args.flags = flags;
1871
1872 return (zone_alloc_item(zones, &args, M_WAITOK));
1873}
1874
|
1844static void
1845zone_lock_pair(uma_zone_t a, uma_zone_t b)
1846{
1847 if (a < b) {
1848 ZONE_LOCK(a);
1849 mtx_lock_flags(b->uz_lock, MTX_DUPOK);
1850 } else {
1851 ZONE_LOCK(b);
1852 mtx_lock_flags(a->uz_lock, MTX_DUPOK);
1853 }
1854}
1855
1856static void
1857zone_unlock_pair(uma_zone_t a, uma_zone_t b)
1858{
1859
1860 ZONE_UNLOCK(a);
1861 ZONE_UNLOCK(b);
1862}
1863
1864int
1865uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
1866{
1867 uma_klink_t klink;
1868 uma_klink_t kl;
1869 int error;
1870
1871 error = 0;
1872 klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
1873
1874 zone_lock_pair(zone, master);
1875 /*
1876 * zone must use vtoslab() to resolve objects and must already be
1877 * a secondary.
1878 */
1879 if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
1880 != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
1881 error = EINVAL;
1882 goto out;
1883 }
1884 /*
1885 * The new master must also use vtoslab().
1886 */
1887 if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
1888 error = EINVAL;
1889 goto out;
1890 }
1891 /*
1892 * Both must either be refcnt, or not be refcnt.
1893 */
1894 if ((zone->uz_flags & UMA_ZONE_REFCNT) !=
1895 (master->uz_flags & UMA_ZONE_REFCNT)) {
1896 error = EINVAL;
1897 goto out;
1898 }
1899 /*
1900 * The underlying object must be the same size. rsize
1901 * may be different.
1902 */
1903 if (master->uz_size != zone->uz_size) {
1904 error = E2BIG;
1905 goto out;
1906 }
1907 /*
1908 * Put it at the end of the list.
1909 */
1910 klink->kl_keg = zone_first_keg(master);
1911 LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
1912 if (LIST_NEXT(kl, kl_link) == NULL) {
1913 LIST_INSERT_AFTER(kl, klink, kl_link);
1914 break;
1915 }
1916 }
1917 klink = NULL;
1918 zone->uz_flags |= UMA_ZFLAG_MULTI;
1919 zone->uz_slab = zone_fetch_slab_multi;
1920
1921out:
1922 zone_unlock_pair(zone, master);
1923 if (klink != NULL)
1924 free(klink, M_TEMP);
1925
1926 return (error);
1927}
1928
1929
1930/* See uma.h */
1931void
1932uma_zdestroy(uma_zone_t zone)
1933{
1934
| 1875static void
1876zone_lock_pair(uma_zone_t a, uma_zone_t b)
1877{
1878 if (a < b) {
1879 ZONE_LOCK(a);
1880 mtx_lock_flags(b->uz_lock, MTX_DUPOK);
1881 } else {
1882 ZONE_LOCK(b);
1883 mtx_lock_flags(a->uz_lock, MTX_DUPOK);
1884 }
1885}
1886
1887static void
1888zone_unlock_pair(uma_zone_t a, uma_zone_t b)
1889{
1890
1891 ZONE_UNLOCK(a);
1892 ZONE_UNLOCK(b);
1893}
1894
1895int
1896uma_zsecond_add(uma_zone_t zone, uma_zone_t master)
1897{
1898 uma_klink_t klink;
1899 uma_klink_t kl;
1900 int error;
1901
1902 error = 0;
1903 klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO);
1904
1905 zone_lock_pair(zone, master);
1906 /*
1907 * zone must use vtoslab() to resolve objects and must already be
1908 * a secondary.
1909 */
1910 if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY))
1911 != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) {
1912 error = EINVAL;
1913 goto out;
1914 }
1915 /*
1916 * The new master must also use vtoslab().
1917 */
1918 if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) {
1919 error = EINVAL;
1920 goto out;
1921 }
1922 /*
1923 * Both must either be refcnt, or not be refcnt.
1924 */
1925 if ((zone->uz_flags & UMA_ZONE_REFCNT) !=
1926 (master->uz_flags & UMA_ZONE_REFCNT)) {
1927 error = EINVAL;
1928 goto out;
1929 }
1930 /*
1931 * The underlying object must be the same size. rsize
1932 * may be different.
1933 */
1934 if (master->uz_size != zone->uz_size) {
1935 error = E2BIG;
1936 goto out;
1937 }
1938 /*
1939 * Put it at the end of the list.
1940 */
1941 klink->kl_keg = zone_first_keg(master);
1942 LIST_FOREACH(kl, &zone->uz_kegs, kl_link) {
1943 if (LIST_NEXT(kl, kl_link) == NULL) {
1944 LIST_INSERT_AFTER(kl, klink, kl_link);
1945 break;
1946 }
1947 }
1948 klink = NULL;
1949 zone->uz_flags |= UMA_ZFLAG_MULTI;
1950 zone->uz_slab = zone_fetch_slab_multi;
1951
1952out:
1953 zone_unlock_pair(zone, master);
1954 if (klink != NULL)
1955 free(klink, M_TEMP);
1956
1957 return (error);
1958}
1959
1960
1961/* See uma.h */
1962void
1963uma_zdestroy(uma_zone_t zone)
1964{
1965
|
1935 zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
| 1966 zone_free_item(zones, zone, NULL, SKIP_NONE);
|
1936}
1937
1938/* See uma.h */
1939void *
1940uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1941{
1942 void *item;
1943 uma_cache_t cache;
1944 uma_bucket_t bucket;
1945 int cpu;
1946
1947 /* This is the fast path allocation */
1948#ifdef UMA_DEBUG_ALLOC_1
1949 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
1950#endif
1951 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
1952 zone->uz_name, flags);
1953
1954 if (flags & M_WAITOK) {
1955 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1956 "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
1957 }
1958#ifdef DEBUG_MEMGUARD
1959 if (memguard_cmp_zone(zone)) {
1960 item = memguard_alloc(zone->uz_size, flags);
1961 if (item != NULL) {
1962 /*
1963 * Avoid conflict with the use-after-free
1964 * protecting infrastructure from INVARIANTS.
1965 */
1966 if (zone->uz_init != NULL &&
1967 zone->uz_init != mtrash_init &&
1968 zone->uz_init(item, zone->uz_size, flags) != 0)
1969 return (NULL);
1970 if (zone->uz_ctor != NULL &&
1971 zone->uz_ctor != mtrash_ctor &&
1972 zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
1973 zone->uz_fini(item, zone->uz_size);
1974 return (NULL);
1975 }
1976 return (item);
1977 }
1978 /* This is unfortunate but should not be fatal. */
1979 }
1980#endif
1981 /*
1982 * If possible, allocate from the per-CPU cache. There are two
1983 * requirements for safe access to the per-CPU cache: (1) the thread
1984 * accessing the cache must not be preempted or yield during access,
1985 * and (2) the thread must not migrate CPUs without switching which
1986 * cache it accesses. We rely on a critical section to prevent
1987 * preemption and migration. We release the critical section in
1988 * order to acquire the zone mutex if we are unable to allocate from
1989 * the current cache; when we re-acquire the critical section, we
1990 * must detect and handle migration if it has occurred.
1991 */
1992zalloc_restart:
1993 critical_enter();
1994 cpu = curcpu;
1995 cache = &zone->uz_cpu[cpu];
1996
1997zalloc_start:
1998 bucket = cache->uc_allocbucket;
1999
2000 if (bucket) {
2001 if (bucket->ub_cnt > 0) {
2002 bucket->ub_cnt--;
2003 item = bucket->ub_bucket[bucket->ub_cnt];
2004#ifdef INVARIANTS
2005 bucket->ub_bucket[bucket->ub_cnt] = NULL;
2006#endif
2007 KASSERT(item != NULL,
2008 ("uma_zalloc: Bucket pointer mangled."));
2009 cache->uc_allocs++;
2010 critical_exit();
2011 if (zone->uz_ctor != NULL) {
2012 if (zone->uz_ctor(item, zone->uz_size,
2013 udata, flags) != 0) {
| 1967}
1968
1969/* See uma.h */
1970void *
1971uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1972{
1973 void *item;
1974 uma_cache_t cache;
1975 uma_bucket_t bucket;
1976 int cpu;
1977
1978 /* This is the fast path allocation */
1979#ifdef UMA_DEBUG_ALLOC_1
1980 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone);
1981#endif
1982 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread,
1983 zone->uz_name, flags);
1984
1985 if (flags & M_WAITOK) {
1986 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1987 "uma_zalloc_arg: zone \"%s\"", zone->uz_name);
1988 }
1989#ifdef DEBUG_MEMGUARD
1990 if (memguard_cmp_zone(zone)) {
1991 item = memguard_alloc(zone->uz_size, flags);
1992 if (item != NULL) {
1993 /*
1994 * Avoid conflict with the use-after-free
1995 * protecting infrastructure from INVARIANTS.
1996 */
1997 if (zone->uz_init != NULL &&
1998 zone->uz_init != mtrash_init &&
1999 zone->uz_init(item, zone->uz_size, flags) != 0)
2000 return (NULL);
2001 if (zone->uz_ctor != NULL &&
2002 zone->uz_ctor != mtrash_ctor &&
2003 zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2004 zone->uz_fini(item, zone->uz_size);
2005 return (NULL);
2006 }
2007 return (item);
2008 }
2009 /* This is unfortunate but should not be fatal. */
2010 }
2011#endif
2012 /*
2013 * If possible, allocate from the per-CPU cache. There are two
2014 * requirements for safe access to the per-CPU cache: (1) the thread
2015 * accessing the cache must not be preempted or yield during access,
2016 * and (2) the thread must not migrate CPUs without switching which
2017 * cache it accesses. We rely on a critical section to prevent
2018 * preemption and migration. We release the critical section in
2019 * order to acquire the zone mutex if we are unable to allocate from
2020 * the current cache; when we re-acquire the critical section, we
2021 * must detect and handle migration if it has occurred.
2022 */
2023zalloc_restart:
2024 critical_enter();
2025 cpu = curcpu;
2026 cache = &zone->uz_cpu[cpu];
2027
2028zalloc_start:
2029 bucket = cache->uc_allocbucket;
2030
2031 if (bucket) {
2032 if (bucket->ub_cnt > 0) {
2033 bucket->ub_cnt--;
2034 item = bucket->ub_bucket[bucket->ub_cnt];
2035#ifdef INVARIANTS
2036 bucket->ub_bucket[bucket->ub_cnt] = NULL;
2037#endif
2038 KASSERT(item != NULL,
2039 ("uma_zalloc: Bucket pointer mangled."));
2040 cache->uc_allocs++;
2041 critical_exit();
2042 if (zone->uz_ctor != NULL) {
2043 if (zone->uz_ctor(item, zone->uz_size,
2044 udata, flags) != 0) {
|
| 2045 atomic_add_long(&zone->uz_fails, 1);
|
2014 zone_free_item(zone, item, udata,
| 2046 zone_free_item(zone, item, udata,
|
2015 SKIP_DTOR, ZFREE_STATFAIL |
2016 ZFREE_STATFREE);
| 2047 SKIP_DTOR);
|
2017 return (NULL);
2018 }
2019 }
2020#ifdef INVARIANTS
2021 uma_dbg_alloc(zone, NULL, item);
2022#endif
2023 if (flags & M_ZERO)
2024 bzero(item, zone->uz_size);
2025 return (item);
2026 } else if (cache->uc_freebucket) {
2027 /*
2028 * We have run out of items in our allocbucket.
2029 * See if we can switch with our free bucket.
2030 */
2031 if (cache->uc_freebucket->ub_cnt > 0) {
2032#ifdef UMA_DEBUG_ALLOC
2033 printf("uma_zalloc: Swapping empty with"
2034 " alloc.\n");
2035#endif
2036 bucket = cache->uc_freebucket;
2037 cache->uc_freebucket = cache->uc_allocbucket;
2038 cache->uc_allocbucket = bucket;
2039
2040 goto zalloc_start;
2041 }
2042 }
2043 }
2044 /*
2045 * Attempt to retrieve the item from the per-CPU cache has failed, so
2046 * we must go back to the zone. This requires the zone lock, so we
2047 * must drop the critical section, then re-acquire it when we go back
2048 * to the cache. Since the critical section is released, we may be
2049 * preempted or migrate. As such, make sure not to maintain any
2050 * thread-local state specific to the cache from prior to releasing
2051 * the critical section.
2052 */
2053 critical_exit();
2054 ZONE_LOCK(zone);
2055 critical_enter();
2056 cpu = curcpu;
2057 cache = &zone->uz_cpu[cpu];
2058 bucket = cache->uc_allocbucket;
2059 if (bucket != NULL) {
2060 if (bucket->ub_cnt > 0) {
2061 ZONE_UNLOCK(zone);
2062 goto zalloc_start;
2063 }
2064 bucket = cache->uc_freebucket;
2065 if (bucket != NULL && bucket->ub_cnt > 0) {
2066 ZONE_UNLOCK(zone);
2067 goto zalloc_start;
2068 }
2069 }
2070
2071 /* Since we have locked the zone we may as well send back our stats */
| 2048 return (NULL);
2049 }
2050 }
2051#ifdef INVARIANTS
2052 uma_dbg_alloc(zone, NULL, item);
2053#endif
2054 if (flags & M_ZERO)
2055 bzero(item, zone->uz_size);
2056 return (item);
2057 } else if (cache->uc_freebucket) {
2058 /*
2059 * We have run out of items in our allocbucket.
2060 * See if we can switch with our free bucket.
2061 */
2062 if (cache->uc_freebucket->ub_cnt > 0) {
2063#ifdef UMA_DEBUG_ALLOC
2064 printf("uma_zalloc: Swapping empty with"
2065 " alloc.\n");
2066#endif
2067 bucket = cache->uc_freebucket;
2068 cache->uc_freebucket = cache->uc_allocbucket;
2069 cache->uc_allocbucket = bucket;
2070
2071 goto zalloc_start;
2072 }
2073 }
2074 }
2075 /*
2076 * Attempt to retrieve the item from the per-CPU cache has failed, so
2077 * we must go back to the zone. This requires the zone lock, so we
2078 * must drop the critical section, then re-acquire it when we go back
2079 * to the cache. Since the critical section is released, we may be
2080 * preempted or migrate. As such, make sure not to maintain any
2081 * thread-local state specific to the cache from prior to releasing
2082 * the critical section.
2083 */
2084 critical_exit();
2085 ZONE_LOCK(zone);
2086 critical_enter();
2087 cpu = curcpu;
2088 cache = &zone->uz_cpu[cpu];
2089 bucket = cache->uc_allocbucket;
2090 if (bucket != NULL) {
2091 if (bucket->ub_cnt > 0) {
2092 ZONE_UNLOCK(zone);
2093 goto zalloc_start;
2094 }
2095 bucket = cache->uc_freebucket;
2096 if (bucket != NULL && bucket->ub_cnt > 0) {
2097 ZONE_UNLOCK(zone);
2098 goto zalloc_start;
2099 }
2100 }
2101
2102 /* Since we have locked the zone we may as well send back our stats */
|
2072 zone->uz_allocs += cache->uc_allocs;
| 2103 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2104 atomic_add_long(&zone->uz_frees, cache->uc_frees);
|
2073 cache->uc_allocs = 0;
| 2105 cache->uc_allocs = 0;
|
2074 zone->uz_frees += cache->uc_frees;
| |
2075 cache->uc_frees = 0;
2076
2077 /* Our old one is now a free bucket */
2078 if (cache->uc_allocbucket) {
2079 KASSERT(cache->uc_allocbucket->ub_cnt == 0,
2080 ("uma_zalloc_arg: Freeing a non free bucket."));
2081 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2082 cache->uc_allocbucket, ub_link);
2083 cache->uc_allocbucket = NULL;
2084 }
2085
2086 /* Check the free list for a new alloc bucket */
2087 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) {
2088 KASSERT(bucket->ub_cnt != 0,
2089 ("uma_zalloc_arg: Returning an empty bucket."));
2090
2091 LIST_REMOVE(bucket, ub_link);
2092 cache->uc_allocbucket = bucket;
2093 ZONE_UNLOCK(zone);
2094 goto zalloc_start;
2095 }
2096 /* We are no longer associated with this CPU. */
2097 critical_exit();
2098
2099 /* Bump up our uz_count so we get here less */
2100 if (zone->uz_count < BUCKET_MAX)
2101 zone->uz_count++;
2102
2103 /*
2104 * Now lets just fill a bucket and put it on the free list. If that
2105 * works we'll restart the allocation from the begining.
2106 */
2107 if (zone_alloc_bucket(zone, flags)) {
2108 ZONE_UNLOCK(zone);
2109 goto zalloc_restart;
2110 }
2111 ZONE_UNLOCK(zone);
2112 /*
2113 * We may not be able to get a bucket so return an actual item.
2114 */
2115#ifdef UMA_DEBUG
2116 printf("uma_zalloc_arg: Bucketzone returned NULL\n");
2117#endif
2118
2119 item = zone_alloc_item(zone, udata, flags);
2120 return (item);
2121}
2122
2123static uma_slab_t
2124keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
2125{
2126 uma_slab_t slab;
2127
2128 mtx_assert(&keg->uk_lock, MA_OWNED);
2129 slab = NULL;
2130
2131 for (;;) {
2132 /*
2133 * Find a slab with some space. Prefer slabs that are partially
2134 * used over those that are totally full. This helps to reduce
2135 * fragmentation.
2136 */
2137 if (keg->uk_free != 0) {
2138 if (!LIST_EMPTY(&keg->uk_part_slab)) {
2139 slab = LIST_FIRST(&keg->uk_part_slab);
2140 } else {
2141 slab = LIST_FIRST(&keg->uk_free_slab);
2142 LIST_REMOVE(slab, us_link);
2143 LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
2144 us_link);
2145 }
2146 MPASS(slab->us_keg == keg);
2147 return (slab);
2148 }
2149
2150 /*
2151 * M_NOVM means don't ask at all!
2152 */
2153 if (flags & M_NOVM)
2154 break;
2155
2156 if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
2157 keg->uk_flags |= UMA_ZFLAG_FULL;
2158 /*
2159 * If this is not a multi-zone, set the FULL bit.
2160 * Otherwise slab_multi() takes care of it.
2161 */
2162 if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
2163 zone->uz_flags |= UMA_ZFLAG_FULL;
2164 zone_log_warning(zone);
2165 }
2166 if (flags & M_NOWAIT)
2167 break;
2168 zone->uz_sleeps++;
2169 msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
2170 continue;
2171 }
2172 keg->uk_recurse++;
2173 slab = keg_alloc_slab(keg, zone, flags);
2174 keg->uk_recurse--;
2175 /*
2176 * If we got a slab here it's safe to mark it partially used
2177 * and return. We assume that the caller is going to remove
2178 * at least one item.
2179 */
2180 if (slab) {
2181 MPASS(slab->us_keg == keg);
2182 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2183 return (slab);
2184 }
2185 /*
2186 * We might not have been able to get a slab but another cpu
2187 * could have while we were unlocked. Check again before we
2188 * fail.
2189 */
2190 flags |= M_NOVM;
2191 }
2192 return (slab);
2193}
2194
2195static inline void
2196zone_relock(uma_zone_t zone, uma_keg_t keg)
2197{
2198 if (zone->uz_lock != &keg->uk_lock) {
2199 KEG_UNLOCK(keg);
2200 ZONE_LOCK(zone);
2201 }
2202}
2203
2204static inline void
2205keg_relock(uma_keg_t keg, uma_zone_t zone)
2206{
2207 if (zone->uz_lock != &keg->uk_lock) {
2208 ZONE_UNLOCK(zone);
2209 KEG_LOCK(keg);
2210 }
2211}
2212
2213static uma_slab_t
2214zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2215{
2216 uma_slab_t slab;
2217
2218 if (keg == NULL)
2219 keg = zone_first_keg(zone);
2220 /*
2221 * This is to prevent us from recursively trying to allocate
2222 * buckets. The problem is that if an allocation forces us to
2223 * grab a new bucket we will call page_alloc, which will go off
2224 * and cause the vm to allocate vm_map_entries. If we need new
2225 * buckets there too we will recurse in kmem_alloc and bad
2226 * things happen. So instead we return a NULL bucket, and make
2227 * the code that allocates buckets smart enough to deal with it
2228 */
2229 if (keg->uk_flags & UMA_ZFLAG_BUCKET && keg->uk_recurse != 0)
2230 return (NULL);
2231
2232 for (;;) {
2233 slab = keg_fetch_slab(keg, zone, flags);
2234 if (slab)
2235 return (slab);
2236 if (flags & (M_NOWAIT | M_NOVM))
2237 break;
2238 }
2239 return (NULL);
2240}
2241
2242/*
2243 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
2244 * with the keg locked. Caller must call zone_relock() afterwards if the
2245 * zone lock is required. On NULL the zone lock is held.
2246 *
2247 * The last pointer is used to seed the search. It is not required.
2248 */
2249static uma_slab_t
2250zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2251{
2252 uma_klink_t klink;
2253 uma_slab_t slab;
2254 uma_keg_t keg;
2255 int flags;
2256 int empty;
2257 int full;
2258
2259 /*
2260 * Don't wait on the first pass. This will skip limit tests
2261 * as well. We don't want to block if we can find a provider
2262 * without blocking.
2263 */
2264 flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2265 /*
2266 * Use the last slab allocated as a hint for where to start
2267 * the search.
2268 */
2269 if (last) {
2270 slab = keg_fetch_slab(last, zone, flags);
2271 if (slab)
2272 return (slab);
2273 zone_relock(zone, last);
2274 last = NULL;
2275 }
2276 /*
2277 * Loop until we have a slab incase of transient failures
2278 * while M_WAITOK is specified. I'm not sure this is 100%
2279 * required but we've done it for so long now.
2280 */
2281 for (;;) {
2282 empty = 0;
2283 full = 0;
2284 /*
2285 * Search the available kegs for slabs. Be careful to hold the
2286 * correct lock while calling into the keg layer.
2287 */
2288 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2289 keg = klink->kl_keg;
2290 keg_relock(keg, zone);
2291 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2292 slab = keg_fetch_slab(keg, zone, flags);
2293 if (slab)
2294 return (slab);
2295 }
2296 if (keg->uk_flags & UMA_ZFLAG_FULL)
2297 full++;
2298 else
2299 empty++;
2300 zone_relock(zone, keg);
2301 }
2302 if (rflags & (M_NOWAIT | M_NOVM))
2303 break;
2304 flags = rflags;
2305 /*
2306 * All kegs are full. XXX We can't atomically check all kegs
2307 * and sleep so just sleep for a short period and retry.
2308 */
2309 if (full && !empty) {
2310 zone->uz_flags |= UMA_ZFLAG_FULL;
2311 zone->uz_sleeps++;
2312 zone_log_warning(zone);
2313 msleep(zone, zone->uz_lock, PVM, "zonelimit", hz/100);
2314 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2315 continue;
2316 }
2317 }
2318 return (NULL);
2319}
2320
2321static void *
| 2106 cache->uc_frees = 0;
2107
2108 /* Our old one is now a free bucket */
2109 if (cache->uc_allocbucket) {
2110 KASSERT(cache->uc_allocbucket->ub_cnt == 0,
2111 ("uma_zalloc_arg: Freeing a non free bucket."));
2112 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2113 cache->uc_allocbucket, ub_link);
2114 cache->uc_allocbucket = NULL;
2115 }
2116
2117 /* Check the free list for a new alloc bucket */
2118 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) {
2119 KASSERT(bucket->ub_cnt != 0,
2120 ("uma_zalloc_arg: Returning an empty bucket."));
2121
2122 LIST_REMOVE(bucket, ub_link);
2123 cache->uc_allocbucket = bucket;
2124 ZONE_UNLOCK(zone);
2125 goto zalloc_start;
2126 }
2127 /* We are no longer associated with this CPU. */
2128 critical_exit();
2129
2130 /* Bump up our uz_count so we get here less */
2131 if (zone->uz_count < BUCKET_MAX)
2132 zone->uz_count++;
2133
2134 /*
2135 * Now lets just fill a bucket and put it on the free list. If that
2136 * works we'll restart the allocation from the begining.
2137 */
2138 if (zone_alloc_bucket(zone, flags)) {
2139 ZONE_UNLOCK(zone);
2140 goto zalloc_restart;
2141 }
2142 ZONE_UNLOCK(zone);
2143 /*
2144 * We may not be able to get a bucket so return an actual item.
2145 */
2146#ifdef UMA_DEBUG
2147 printf("uma_zalloc_arg: Bucketzone returned NULL\n");
2148#endif
2149
2150 item = zone_alloc_item(zone, udata, flags);
2151 return (item);
2152}
2153
2154static uma_slab_t
2155keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
2156{
2157 uma_slab_t slab;
2158
2159 mtx_assert(&keg->uk_lock, MA_OWNED);
2160 slab = NULL;
2161
2162 for (;;) {
2163 /*
2164 * Find a slab with some space. Prefer slabs that are partially
2165 * used over those that are totally full. This helps to reduce
2166 * fragmentation.
2167 */
2168 if (keg->uk_free != 0) {
2169 if (!LIST_EMPTY(&keg->uk_part_slab)) {
2170 slab = LIST_FIRST(&keg->uk_part_slab);
2171 } else {
2172 slab = LIST_FIRST(&keg->uk_free_slab);
2173 LIST_REMOVE(slab, us_link);
2174 LIST_INSERT_HEAD(&keg->uk_part_slab, slab,
2175 us_link);
2176 }
2177 MPASS(slab->us_keg == keg);
2178 return (slab);
2179 }
2180
2181 /*
2182 * M_NOVM means don't ask at all!
2183 */
2184 if (flags & M_NOVM)
2185 break;
2186
2187 if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
2188 keg->uk_flags |= UMA_ZFLAG_FULL;
2189 /*
2190 * If this is not a multi-zone, set the FULL bit.
2191 * Otherwise slab_multi() takes care of it.
2192 */
2193 if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) {
2194 zone->uz_flags |= UMA_ZFLAG_FULL;
2195 zone_log_warning(zone);
2196 }
2197 if (flags & M_NOWAIT)
2198 break;
2199 zone->uz_sleeps++;
2200 msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
2201 continue;
2202 }
2203 keg->uk_recurse++;
2204 slab = keg_alloc_slab(keg, zone, flags);
2205 keg->uk_recurse--;
2206 /*
2207 * If we got a slab here it's safe to mark it partially used
2208 * and return. We assume that the caller is going to remove
2209 * at least one item.
2210 */
2211 if (slab) {
2212 MPASS(slab->us_keg == keg);
2213 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2214 return (slab);
2215 }
2216 /*
2217 * We might not have been able to get a slab but another cpu
2218 * could have while we were unlocked. Check again before we
2219 * fail.
2220 */
2221 flags |= M_NOVM;
2222 }
2223 return (slab);
2224}
2225
2226static inline void
2227zone_relock(uma_zone_t zone, uma_keg_t keg)
2228{
2229 if (zone->uz_lock != &keg->uk_lock) {
2230 KEG_UNLOCK(keg);
2231 ZONE_LOCK(zone);
2232 }
2233}
2234
2235static inline void
2236keg_relock(uma_keg_t keg, uma_zone_t zone)
2237{
2238 if (zone->uz_lock != &keg->uk_lock) {
2239 ZONE_UNLOCK(zone);
2240 KEG_LOCK(keg);
2241 }
2242}
2243
2244static uma_slab_t
2245zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2246{
2247 uma_slab_t slab;
2248
2249 if (keg == NULL)
2250 keg = zone_first_keg(zone);
2251 /*
2252 * This is to prevent us from recursively trying to allocate
2253 * buckets. The problem is that if an allocation forces us to
2254 * grab a new bucket we will call page_alloc, which will go off
2255 * and cause the vm to allocate vm_map_entries. If we need new
2256 * buckets there too we will recurse in kmem_alloc and bad
2257 * things happen. So instead we return a NULL bucket, and make
2258 * the code that allocates buckets smart enough to deal with it
2259 */
2260 if (keg->uk_flags & UMA_ZFLAG_BUCKET && keg->uk_recurse != 0)
2261 return (NULL);
2262
2263 for (;;) {
2264 slab = keg_fetch_slab(keg, zone, flags);
2265 if (slab)
2266 return (slab);
2267 if (flags & (M_NOWAIT | M_NOVM))
2268 break;
2269 }
2270 return (NULL);
2271}
2272
2273/*
2274 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
2275 * with the keg locked. Caller must call zone_relock() afterwards if the
2276 * zone lock is required. On NULL the zone lock is held.
2277 *
2278 * The last pointer is used to seed the search. It is not required.
2279 */
2280static uma_slab_t
2281zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2282{
2283 uma_klink_t klink;
2284 uma_slab_t slab;
2285 uma_keg_t keg;
2286 int flags;
2287 int empty;
2288 int full;
2289
2290 /*
2291 * Don't wait on the first pass. This will skip limit tests
2292 * as well. We don't want to block if we can find a provider
2293 * without blocking.
2294 */
2295 flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2296 /*
2297 * Use the last slab allocated as a hint for where to start
2298 * the search.
2299 */
2300 if (last) {
2301 slab = keg_fetch_slab(last, zone, flags);
2302 if (slab)
2303 return (slab);
2304 zone_relock(zone, last);
2305 last = NULL;
2306 }
2307 /*
2308 * Loop until we have a slab incase of transient failures
2309 * while M_WAITOK is specified. I'm not sure this is 100%
2310 * required but we've done it for so long now.
2311 */
2312 for (;;) {
2313 empty = 0;
2314 full = 0;
2315 /*
2316 * Search the available kegs for slabs. Be careful to hold the
2317 * correct lock while calling into the keg layer.
2318 */
2319 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2320 keg = klink->kl_keg;
2321 keg_relock(keg, zone);
2322 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2323 slab = keg_fetch_slab(keg, zone, flags);
2324 if (slab)
2325 return (slab);
2326 }
2327 if (keg->uk_flags & UMA_ZFLAG_FULL)
2328 full++;
2329 else
2330 empty++;
2331 zone_relock(zone, keg);
2332 }
2333 if (rflags & (M_NOWAIT | M_NOVM))
2334 break;
2335 flags = rflags;
2336 /*
2337 * All kegs are full. XXX We can't atomically check all kegs
2338 * and sleep so just sleep for a short period and retry.
2339 */
2340 if (full && !empty) {
2341 zone->uz_flags |= UMA_ZFLAG_FULL;
2342 zone->uz_sleeps++;
2343 zone_log_warning(zone);
2344 msleep(zone, zone->uz_lock, PVM, "zonelimit", hz/100);
2345 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2346 continue;
2347 }
2348 }
2349 return (NULL);
2350}
2351
2352static void *
|
2322slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
| 2353slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
|
2323{
| 2354{
|
2324 uma_keg_t keg;
| |
2325 void *item;
2326 uint8_t freei;
2327
| 2355 void *item;
2356 uint8_t freei;
2357
|
2328 keg = slab->us_keg;
| 2358 MPASS(keg == slab->us_keg);
|
2329 mtx_assert(&keg->uk_lock, MA_OWNED);
2330
2331 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2332 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2333 item = slab->us_data + (keg->uk_rsize * freei);
2334 slab->us_freecount--;
2335 keg->uk_free--;
2336
2337 /* Move this slab to the full list */
2338 if (slab->us_freecount == 0) {
2339 LIST_REMOVE(slab, us_link);
2340 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2341 }
2342
2343 return (item);
2344}
2345
2346static int
| 2359 mtx_assert(&keg->uk_lock, MA_OWNED);
2360
2361 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2362 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2363 item = slab->us_data + (keg->uk_rsize * freei);
2364 slab->us_freecount--;
2365 keg->uk_free--;
2366
2367 /* Move this slab to the full list */
2368 if (slab->us_freecount == 0) {
2369 LIST_REMOVE(slab, us_link);
2370 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2371 }
2372
2373 return (item);
2374}
2375
2376static int
|
2347zone_alloc_bucket(uma_zone_t zone, int flags)
| 2377zone_import(uma_zone_t zone, void **bucket, int max, int flags)
|
2348{
| 2378{
|
2349 uma_bucket_t bucket;
| |
2350 uma_slab_t slab;
2351 uma_keg_t keg;
| 2379 uma_slab_t slab;
2380 uma_keg_t keg;
|
2352 int16_t saved;
2353 int max, origflags = flags;
| 2381 int i;
|
2354
| 2382
|
2355 /*
2356 * Try this zone's free list first so we don't allocate extra buckets.
2357 */
2358 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2359 KASSERT(bucket->ub_cnt == 0,
2360 ("zone_alloc_bucket: Bucket on free list is not empty."));
2361 LIST_REMOVE(bucket, ub_link);
2362 } else {
2363 int bflags;
| 2383 ZONE_LOCK(zone);
2384 /* Try to keep the buckets totally full */
2385 slab = NULL;
2386 keg = NULL;
2387 for (i = 0; i < max; ) {
2388 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2389 break;
2390 keg = slab->us_keg;
2391 while (slab->us_freecount && i < max)
2392 bucket[i++] = slab_alloc_item(keg, slab);
|
2364
| 2393
|
2365 bflags = (flags & ~M_ZERO);
2366 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2367 bflags |= M_NOVM;
2368
2369 ZONE_UNLOCK(zone);
2370 bucket = bucket_alloc(zone->uz_count, bflags);
2371 ZONE_LOCK(zone);
| 2394 /* Don't block on the next fill */
2395 flags &= ~M_WAITOK;
2396 flags |= M_NOWAIT;
|
2372 }
| 2397 }
|
| 2398 if (slab != NULL)
2399 KEG_UNLOCK(keg);
2400 else
2401 ZONE_UNLOCK(zone);
|
2373
| 2402
|
2374 if (bucket == NULL) {
2375 return (0);
2376 }
| 2403 return i;
2404}
|
2377
| 2405
|
| 2406static int
2407zone_alloc_bucket(uma_zone_t zone, int flags)
2408{
2409 uma_bucket_t bucket;
2410 int bflags;
2411 int max;
2412
|
2378#ifdef SMP
2379 /*
2380 * This code is here to limit the number of simultaneous bucket fills
2381 * for any given zone to the number of per cpu caches in this zone. This
2382 * is done so that we don't allocate more memory than we really need.
2383 */
2384 if (zone->uz_fills >= mp_ncpus)
| 2413#ifdef SMP
2414 /*
2415 * This code is here to limit the number of simultaneous bucket fills
2416 * for any given zone to the number of per cpu caches in this zone. This
2417 * is done so that we don't allocate more memory than we really need.
2418 */
2419 if (zone->uz_fills >= mp_ncpus)
|
2385 goto done;
| 2420 return (0);
|
2386
2387#endif
2388 zone->uz_fills++;
| 2421
2422#endif
2423 zone->uz_fills++;
|
| 2424 max = zone->uz_count;
|
2389
| 2425
|
2390 max = MIN(bucket->ub_entries, zone->uz_count);
2391 /* Try to keep the buckets totally full */
2392 saved = bucket->ub_cnt;
2393 slab = NULL;
2394 keg = NULL;
2395 while (bucket->ub_cnt < max &&
2396 (slab = zone->uz_slab(zone, keg, flags)) != NULL) {
2397 keg = slab->us_keg;
2398 while (slab->us_freecount && bucket->ub_cnt < max) {
2399 bucket->ub_bucket[bucket->ub_cnt++] =
2400 slab_alloc_item(zone, slab);
2401 }
2402
2403 /* Don't block on the next fill */
2404 flags |= M_NOWAIT;
| 2426 /*
2427 * Try this zone's free list first so we don't allocate extra buckets.
2428 */
2429 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2430 KASSERT(bucket->ub_cnt == 0,
2431 ("zone_alloc_bucket: Bucket on free list is not empty."));
2432 LIST_REMOVE(bucket, ub_link);
2433 ZONE_UNLOCK(zone);
2434 } else {
2435 bflags = (flags & ~M_ZERO);
2436 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2437 bflags |= M_NOVM;
2438 ZONE_UNLOCK(zone);
2439 bucket = bucket_alloc(zone->uz_count, bflags);
2440 if (bucket == NULL)
2441 goto out;
|
2405 }
| 2442 }
|
2406 if (slab)
2407 zone_relock(zone, keg);
| |
2408
| 2443
|
| 2444 max = MIN(bucket->ub_entries, max);
2445 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2446 max, flags);
2447
|
2409 /*
| 2448 /*
|
2410 * We unlock here because we need to call the zone's init.
2411 * It should be safe to unlock because the slab dealt with
2412 * above is already on the appropriate list within the keg
2413 * and the bucket we filled is not yet on any list, so we
2414 * own it.
| 2449 * Initialize the memory if necessary.
|
2415 */
| 2450 */
|
2416 if (zone->uz_init != NULL) {
| 2451 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
|
2417 int i;
2418
| 2452 int i;
2453
|
2419 ZONE_UNLOCK(zone);
2420 for (i = saved; i < bucket->ub_cnt; i++)
| 2454 for (i = 0; i < bucket->ub_cnt; i++)
|
2421 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
| 2455 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
|
2422 origflags) != 0)
| 2456 flags) != 0)
|
2423 break;
2424 /*
2425 * If we couldn't initialize the whole bucket, put the
2426 * rest back onto the freelist.
2427 */
2428 if (i != bucket->ub_cnt) {
| 2457 break;
2458 /*
2459 * If we couldn't initialize the whole bucket, put the
2460 * rest back onto the freelist.
2461 */
2462 if (i != bucket->ub_cnt) {
|
2429 int j;
2430
2431 for (j = i; j < bucket->ub_cnt; j++) {
2432 zone_free_item(zone, bucket->ub_bucket[j],
2433 NULL, SKIP_FINI, 0);
| 2463 zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
2464 bucket->ub_cnt - i);
|
2434#ifdef INVARIANTS
| 2465#ifdef INVARIANTS
|
2435 bucket->ub_bucket[j] = NULL;
| 2466 bzero(&bucket->ub_bucket[i],
2467 sizeof(void *) * (bucket->ub_cnt - i));
|
2436#endif
| 2468#endif
|
2437 }
| |
2438 bucket->ub_cnt = i;
2439 }
| 2469 bucket->ub_cnt = i;
2470 }
|
2440 ZONE_LOCK(zone);
| |
2441 }
2442
| 2471 }
2472
|
| 2473out:
2474 ZONE_LOCK(zone);
|
2443 zone->uz_fills--;
| 2475 zone->uz_fills--;
|
2444 if (bucket->ub_cnt != 0) {
| 2476 if (bucket != NULL && bucket->ub_cnt != 0) {
|
2445 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2446 bucket, ub_link);
2447 return (1);
2448 }
| 2477 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2478 bucket, ub_link);
2479 return (1);
2480 }
|
2449#ifdef SMP
2450done:
2451#endif
2452 bucket_free(bucket);
| 2481 atomic_add_long(&zone->uz_fails, 1);
2482 if (bucket != NULL)
2483 bucket_free(bucket);
|
2453
2454 return (0);
2455}
2456/*
| 2484
2485 return (0);
2486}
2487/*
|
2457 * Allocates an item for an internal zone
| 2488 * Allocates a single item from a zone.
|
2458 *
2459 * Arguments
2460 * zone The zone to alloc for.
2461 * udata The data to be passed to the constructor.
2462 * flags M_WAITOK, M_NOWAIT, M_ZERO.
2463 *
2464 * Returns
2465 * NULL if there is no memory and M_NOWAIT is set
2466 * An item if successful
2467 */
2468
2469static void *
2470zone_alloc_item(uma_zone_t zone, void *udata, int flags)
2471{
| 2489 *
2490 * Arguments
2491 * zone The zone to alloc for.
2492 * udata The data to be passed to the constructor.
2493 * flags M_WAITOK, M_NOWAIT, M_ZERO.
2494 *
2495 * Returns
2496 * NULL if there is no memory and M_NOWAIT is set
2497 * An item if successful
2498 */
2499
2500static void *
2501zone_alloc_item(uma_zone_t zone, void *udata, int flags)
2502{
|
2472 uma_slab_t slab;
| |
2473 void *item;
2474
2475 item = NULL;
2476
2477#ifdef UMA_DEBUG_ALLOC
2478 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2479#endif
| 2503 void *item;
2504
2505 item = NULL;
2506
2507#ifdef UMA_DEBUG_ALLOC
2508 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2509#endif
|
2480 ZONE_LOCK(zone);
| 2510 if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
2511 goto fail;
2512 atomic_add_long(&zone->uz_allocs, 1);
|
2481
| 2513
|
2482 slab = zone->uz_slab(zone, NULL, flags);
2483 if (slab == NULL) {
2484 zone->uz_fails++;
2485 ZONE_UNLOCK(zone);
2486 return (NULL);
2487 }
2488
2489 item = slab_alloc_item(zone, slab);
2490
2491 zone_relock(zone, slab->us_keg);
2492 zone->uz_allocs++;
2493 ZONE_UNLOCK(zone);
2494
| |
2495 /*
2496 * We have to call both the zone's init (not the keg's init)
2497 * and the zone's ctor. This is because the item is going from
2498 * a keg slab directly to the user, and the user is expecting it
2499 * to be both zone-init'd as well as zone-ctor'd.
2500 */
2501 if (zone->uz_init != NULL) {
2502 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
| 2514 /*
2515 * We have to call both the zone's init (not the keg's init)
2516 * and the zone's ctor. This is because the item is going from
2517 * a keg slab directly to the user, and the user is expecting it
2518 * to be both zone-init'd as well as zone-ctor'd.
2519 */
2520 if (zone->uz_init != NULL) {
2521 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
|
2503 zone_free_item(zone, item, udata, SKIP_FINI,
2504 ZFREE_STATFAIL | ZFREE_STATFREE);
2505 return (NULL);
| 2522 zone_free_item(zone, item, udata, SKIP_FINI);
2523 goto fail;
|
2506 }
2507 }
2508 if (zone->uz_ctor != NULL) {
2509 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
| 2524 }
2525 }
2526 if (zone->uz_ctor != NULL) {
2527 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
|
2510 zone_free_item(zone, item, udata, SKIP_DTOR,
2511 ZFREE_STATFAIL | ZFREE_STATFREE);
2512 return (NULL);
| 2528 zone_free_item(zone, item, udata, SKIP_DTOR);
2529 goto fail;
|
2513 }
2514 }
2515#ifdef INVARIANTS
| 2530 }
2531 }
2532#ifdef INVARIANTS
|
2516 uma_dbg_alloc(zone, slab, item);
| 2533 uma_dbg_alloc(zone, NULL, item);
|
2517#endif
2518 if (flags & M_ZERO)
2519 bzero(item, zone->uz_size);
2520
2521 return (item);
| 2534#endif
2535 if (flags & M_ZERO)
2536 bzero(item, zone->uz_size);
2537
2538 return (item);
|
| 2539
2540fail:
2541 atomic_add_long(&zone->uz_fails, 1);
2542 return (NULL);
|
2522}
2523
2524/* See uma.h */
2525void
2526uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2527{
2528 uma_cache_t cache;
2529 uma_bucket_t bucket;
2530 int bflags;
2531 int cpu;
2532
2533#ifdef UMA_DEBUG_ALLOC_1
2534 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
2535#endif
2536 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
2537 zone->uz_name);
2538
2539 /* uma_zfree(..., NULL) does nothing, to match free(9). */
2540 if (item == NULL)
2541 return;
2542#ifdef DEBUG_MEMGUARD
2543 if (is_memguard_addr(item)) {
2544 if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
2545 zone->uz_dtor(item, zone->uz_size, udata);
2546 if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
2547 zone->uz_fini(item, zone->uz_size);
2548 memguard_free(item);
2549 return;
2550 }
2551#endif
2552#ifdef INVARIANTS
2553 if (zone->uz_flags & UMA_ZONE_MALLOC)
2554 uma_dbg_free(zone, udata, item);
2555 else
2556 uma_dbg_free(zone, NULL, item);
2557#endif
2558 if (zone->uz_dtor)
2559 zone->uz_dtor(item, zone->uz_size, udata);
2560
2561 /*
2562 * The race here is acceptable. If we miss it we'll just have to wait
2563 * a little longer for the limits to be reset.
2564 */
2565 if (zone->uz_flags & UMA_ZFLAG_FULL)
2566 goto zfree_internal;
2567
2568 /*
2569 * If possible, free to the per-CPU cache. There are two
2570 * requirements for safe access to the per-CPU cache: (1) the thread
2571 * accessing the cache must not be preempted or yield during access,
2572 * and (2) the thread must not migrate CPUs without switching which
2573 * cache it accesses. We rely on a critical section to prevent
2574 * preemption and migration. We release the critical section in
2575 * order to acquire the zone mutex if we are unable to free to the
2576 * current cache; when we re-acquire the critical section, we must
2577 * detect and handle migration if it has occurred.
2578 */
2579zfree_restart:
2580 critical_enter();
2581 cpu = curcpu;
2582 cache = &zone->uz_cpu[cpu];
2583
2584zfree_start:
2585 bucket = cache->uc_freebucket;
2586
2587 if (bucket) {
2588 /*
2589 * Do we have room in our bucket? It is OK for this uz count
2590 * check to be slightly out of sync.
2591 */
2592
2593 if (bucket->ub_cnt < bucket->ub_entries) {
2594 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
2595 ("uma_zfree: Freeing to non free bucket index."));
2596 bucket->ub_bucket[bucket->ub_cnt] = item;
2597 bucket->ub_cnt++;
2598 cache->uc_frees++;
2599 critical_exit();
2600 return;
2601 } else if (cache->uc_allocbucket) {
2602#ifdef UMA_DEBUG_ALLOC
2603 printf("uma_zfree: Swapping buckets.\n");
2604#endif
2605 /*
2606 * We have run out of space in our freebucket.
2607 * See if we can switch with our alloc bucket.
2608 */
2609 if (cache->uc_allocbucket->ub_cnt <
2610 cache->uc_freebucket->ub_cnt) {
2611 bucket = cache->uc_freebucket;
2612 cache->uc_freebucket = cache->uc_allocbucket;
2613 cache->uc_allocbucket = bucket;
2614 goto zfree_start;
2615 }
2616 }
2617 }
2618 /*
2619 * We can get here for two reasons:
2620 *
2621 * 1) The buckets are NULL
2622 * 2) The alloc and free buckets are both somewhat full.
2623 *
2624 * We must go back the zone, which requires acquiring the zone lock,
2625 * which in turn means we must release and re-acquire the critical
2626 * section. Since the critical section is released, we may be
2627 * preempted or migrate. As such, make sure not to maintain any
2628 * thread-local state specific to the cache from prior to releasing
2629 * the critical section.
2630 */
2631 critical_exit();
2632 ZONE_LOCK(zone);
2633 critical_enter();
2634 cpu = curcpu;
2635 cache = &zone->uz_cpu[cpu];
2636 if (cache->uc_freebucket != NULL) {
2637 if (cache->uc_freebucket->ub_cnt <
2638 cache->uc_freebucket->ub_entries) {
2639 ZONE_UNLOCK(zone);
2640 goto zfree_start;
2641 }
2642 if (cache->uc_allocbucket != NULL &&
2643 (cache->uc_allocbucket->ub_cnt <
2644 cache->uc_freebucket->ub_cnt)) {
2645 ZONE_UNLOCK(zone);
2646 goto zfree_start;
2647 }
2648 }
2649
2650 /* Since we have locked the zone we may as well send back our stats */
| 2543}
2544
2545/* See uma.h */
2546void
2547uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2548{
2549 uma_cache_t cache;
2550 uma_bucket_t bucket;
2551 int bflags;
2552 int cpu;
2553
2554#ifdef UMA_DEBUG_ALLOC_1
2555 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
2556#endif
2557 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
2558 zone->uz_name);
2559
2560 /* uma_zfree(..., NULL) does nothing, to match free(9). */
2561 if (item == NULL)
2562 return;
2563#ifdef DEBUG_MEMGUARD
2564 if (is_memguard_addr(item)) {
2565 if (zone->uz_dtor != NULL && zone->uz_dtor != mtrash_dtor)
2566 zone->uz_dtor(item, zone->uz_size, udata);
2567 if (zone->uz_fini != NULL && zone->uz_fini != mtrash_fini)
2568 zone->uz_fini(item, zone->uz_size);
2569 memguard_free(item);
2570 return;
2571 }
2572#endif
2573#ifdef INVARIANTS
2574 if (zone->uz_flags & UMA_ZONE_MALLOC)
2575 uma_dbg_free(zone, udata, item);
2576 else
2577 uma_dbg_free(zone, NULL, item);
2578#endif
2579 if (zone->uz_dtor)
2580 zone->uz_dtor(item, zone->uz_size, udata);
2581
2582 /*
2583 * The race here is acceptable. If we miss it we'll just have to wait
2584 * a little longer for the limits to be reset.
2585 */
2586 if (zone->uz_flags & UMA_ZFLAG_FULL)
2587 goto zfree_internal;
2588
2589 /*
2590 * If possible, free to the per-CPU cache. There are two
2591 * requirements for safe access to the per-CPU cache: (1) the thread
2592 * accessing the cache must not be preempted or yield during access,
2593 * and (2) the thread must not migrate CPUs without switching which
2594 * cache it accesses. We rely on a critical section to prevent
2595 * preemption and migration. We release the critical section in
2596 * order to acquire the zone mutex if we are unable to free to the
2597 * current cache; when we re-acquire the critical section, we must
2598 * detect and handle migration if it has occurred.
2599 */
2600zfree_restart:
2601 critical_enter();
2602 cpu = curcpu;
2603 cache = &zone->uz_cpu[cpu];
2604
2605zfree_start:
2606 bucket = cache->uc_freebucket;
2607
2608 if (bucket) {
2609 /*
2610 * Do we have room in our bucket? It is OK for this uz count
2611 * check to be slightly out of sync.
2612 */
2613
2614 if (bucket->ub_cnt < bucket->ub_entries) {
2615 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL,
2616 ("uma_zfree: Freeing to non free bucket index."));
2617 bucket->ub_bucket[bucket->ub_cnt] = item;
2618 bucket->ub_cnt++;
2619 cache->uc_frees++;
2620 critical_exit();
2621 return;
2622 } else if (cache->uc_allocbucket) {
2623#ifdef UMA_DEBUG_ALLOC
2624 printf("uma_zfree: Swapping buckets.\n");
2625#endif
2626 /*
2627 * We have run out of space in our freebucket.
2628 * See if we can switch with our alloc bucket.
2629 */
2630 if (cache->uc_allocbucket->ub_cnt <
2631 cache->uc_freebucket->ub_cnt) {
2632 bucket = cache->uc_freebucket;
2633 cache->uc_freebucket = cache->uc_allocbucket;
2634 cache->uc_allocbucket = bucket;
2635 goto zfree_start;
2636 }
2637 }
2638 }
2639 /*
2640 * We can get here for two reasons:
2641 *
2642 * 1) The buckets are NULL
2643 * 2) The alloc and free buckets are both somewhat full.
2644 *
2645 * We must go back the zone, which requires acquiring the zone lock,
2646 * which in turn means we must release and re-acquire the critical
2647 * section. Since the critical section is released, we may be
2648 * preempted or migrate. As such, make sure not to maintain any
2649 * thread-local state specific to the cache from prior to releasing
2650 * the critical section.
2651 */
2652 critical_exit();
2653 ZONE_LOCK(zone);
2654 critical_enter();
2655 cpu = curcpu;
2656 cache = &zone->uz_cpu[cpu];
2657 if (cache->uc_freebucket != NULL) {
2658 if (cache->uc_freebucket->ub_cnt <
2659 cache->uc_freebucket->ub_entries) {
2660 ZONE_UNLOCK(zone);
2661 goto zfree_start;
2662 }
2663 if (cache->uc_allocbucket != NULL &&
2664 (cache->uc_allocbucket->ub_cnt <
2665 cache->uc_freebucket->ub_cnt)) {
2666 ZONE_UNLOCK(zone);
2667 goto zfree_start;
2668 }
2669 }
2670
2671 /* Since we have locked the zone we may as well send back our stats */
|
2651 zone->uz_allocs += cache->uc_allocs;
| 2672 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2673 atomic_add_long(&zone->uz_frees, cache->uc_frees);
|
2652 cache->uc_allocs = 0;
| 2674 cache->uc_allocs = 0;
|
2653 zone->uz_frees += cache->uc_frees;
| |
2654 cache->uc_frees = 0;
2655
2656 bucket = cache->uc_freebucket;
2657 cache->uc_freebucket = NULL;
2658
2659 /* Can we throw this on the zone full list? */
2660 if (bucket != NULL) {
2661#ifdef UMA_DEBUG_ALLOC
2662 printf("uma_zfree: Putting old bucket on the free list.\n");
2663#endif
2664 /* ub_cnt is pointing to the last free item */
2665 KASSERT(bucket->ub_cnt != 0,
2666 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
2667 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2668 bucket, ub_link);
2669 }
2670 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2671 LIST_REMOVE(bucket, ub_link);
2672 ZONE_UNLOCK(zone);
2673 cache->uc_freebucket = bucket;
2674 goto zfree_start;
2675 }
2676 /* We are no longer associated with this CPU. */
2677 critical_exit();
2678
2679 /* And the zone.. */
2680 ZONE_UNLOCK(zone);
2681
2682#ifdef UMA_DEBUG_ALLOC
2683 printf("uma_zfree: Allocating new free bucket.\n");
2684#endif
2685 bflags = M_NOWAIT;
2686
2687 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2688 bflags |= M_NOVM;
2689 bucket = bucket_alloc(zone->uz_count, bflags);
2690 if (bucket) {
2691 ZONE_LOCK(zone);
2692 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2693 bucket, ub_link);
2694 ZONE_UNLOCK(zone);
2695 goto zfree_restart;
2696 }
2697
2698 /*
2699 * If nothing else caught this, we'll just do an internal free.
2700 */
2701zfree_internal:
| 2675 cache->uc_frees = 0;
2676
2677 bucket = cache->uc_freebucket;
2678 cache->uc_freebucket = NULL;
2679
2680 /* Can we throw this on the zone full list? */
2681 if (bucket != NULL) {
2682#ifdef UMA_DEBUG_ALLOC
2683 printf("uma_zfree: Putting old bucket on the free list.\n");
2684#endif
2685 /* ub_cnt is pointing to the last free item */
2686 KASSERT(bucket->ub_cnt != 0,
2687 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n"));
2688 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2689 bucket, ub_link);
2690 }
2691 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2692 LIST_REMOVE(bucket, ub_link);
2693 ZONE_UNLOCK(zone);
2694 cache->uc_freebucket = bucket;
2695 goto zfree_start;
2696 }
2697 /* We are no longer associated with this CPU. */
2698 critical_exit();
2699
2700 /* And the zone.. */
2701 ZONE_UNLOCK(zone);
2702
2703#ifdef UMA_DEBUG_ALLOC
2704 printf("uma_zfree: Allocating new free bucket.\n");
2705#endif
2706 bflags = M_NOWAIT;
2707
2708 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2709 bflags |= M_NOVM;
2710 bucket = bucket_alloc(zone->uz_count, bflags);
2711 if (bucket) {
2712 ZONE_LOCK(zone);
2713 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2714 bucket, ub_link);
2715 ZONE_UNLOCK(zone);
2716 goto zfree_restart;
2717 }
2718
2719 /*
2720 * If nothing else caught this, we'll just do an internal free.
2721 */
2722zfree_internal:
|
2702 zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
| 2723 zone_free_item(zone, item, udata, SKIP_DTOR);
|
2703
2704 return;
2705}
2706
| 2724
2725 return;
2726}
2727
|
2707/*
2708 * Frees an item to an INTERNAL zone or allocates a free bucket
2709 *
2710 * Arguments:
2711 * zone The zone to free to
2712 * item The item we're freeing
2713 * udata User supplied data for the dtor
2714 * skip Skip dtors and finis
2715 */
| |
2716static void
| 2728static void
|
2717zone_free_item(uma_zone_t zone, void *item, void *udata,
2718 enum zfreeskip skip, int flags)
| 2729slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
|
2719{
| 2730{
|
2720 uma_slab_t slab;
2721 uma_keg_t keg;
2722 uint8_t *mem;
| |
2723 uint8_t freei;
| 2731 uint8_t freei;
|
2724 int clearfull;
| |
2725
| 2732
|
2726#ifdef INVARIANTS
2727 if (skip == SKIP_NONE) {
2728 if (zone->uz_flags & UMA_ZONE_MALLOC)
2729 uma_dbg_free(zone, udata, item);
2730 else
2731 uma_dbg_free(zone, NULL, item);
2732 }
2733#endif
2734 if (skip < SKIP_DTOR && zone->uz_dtor)
2735 zone->uz_dtor(item, zone->uz_size, udata);
2736
2737 if (skip < SKIP_FINI && zone->uz_fini)
2738 zone->uz_fini(item, zone->uz_size);
2739
2740 ZONE_LOCK(zone);
2741
2742 if (flags & ZFREE_STATFAIL)
2743 zone->uz_fails++;
2744 if (flags & ZFREE_STATFREE)
2745 zone->uz_frees++;
2746
2747 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2748 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2749 keg = zone_first_keg(zone); /* Must only be one. */
2750 if (zone->uz_flags & UMA_ZONE_HASH) {
2751 slab = hash_sfind(&keg->uk_hash, mem);
2752 } else {
2753 mem += keg->uk_pgoff;
2754 slab = (uma_slab_t)mem;
2755 }
2756 } else {
2757 /* This prevents redundant lookups via free(). */
2758 if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL)
2759 slab = (uma_slab_t)udata;
2760 else
2761 slab = vtoslab((vm_offset_t)item);
2762 keg = slab->us_keg;
2763 keg_relock(keg, zone);
2764 }
| 2733 mtx_assert(&keg->uk_lock, MA_OWNED);
|
2765 MPASS(keg == slab->us_keg);
2766
2767 /* Do we need to remove from any lists? */
2768 if (slab->us_freecount+1 == keg->uk_ipers) {
2769 LIST_REMOVE(slab, us_link);
2770 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2771 } else if (slab->us_freecount == 0) {
2772 LIST_REMOVE(slab, us_link);
2773 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2774 }
2775
2776 /* Slab management. */
2777 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2778 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
2779 slab->us_freecount++;
2780
2781 /* Keg statistics. */
2782 keg->uk_free++;
| 2734 MPASS(keg == slab->us_keg);
2735
2736 /* Do we need to remove from any lists? */
2737 if (slab->us_freecount+1 == keg->uk_ipers) {
2738 LIST_REMOVE(slab, us_link);
2739 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2740 } else if (slab->us_freecount == 0) {
2741 LIST_REMOVE(slab, us_link);
2742 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2743 }
2744
2745 /* Slab management. */
2746 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2747 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
2748 slab->us_freecount++;
2749
2750 /* Keg statistics. */
2751 keg->uk_free++;
|
| 2752}
|
2783
| 2753
|
| 2754static void
2755zone_release(uma_zone_t zone, void **bucket, int cnt)
2756{
2757 void *item;
2758 uma_slab_t slab;
2759 uma_keg_t keg;
2760 uint8_t *mem;
2761 int clearfull;
2762 int i;
2763
|
2784 clearfull = 0;
| 2764 clearfull = 0;
|
2785 if (keg->uk_flags & UMA_ZFLAG_FULL) {
2786 if (keg->uk_pages < keg->uk_maxpages) {
2787 keg->uk_flags &= ~UMA_ZFLAG_FULL;
2788 clearfull = 1;
| 2765 ZONE_LOCK(zone);
2766 keg = zone_first_keg(zone);
2767 for (i = 0; i < cnt; i++) {
2768 item = bucket[i];
2769 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2770 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2771 if (zone->uz_flags & UMA_ZONE_HASH) {
2772 slab = hash_sfind(&keg->uk_hash, mem);
2773 } else {
2774 mem += keg->uk_pgoff;
2775 slab = (uma_slab_t)mem;
2776 }
2777 } else {
2778 slab = vtoslab((vm_offset_t)item);
2779 if (slab->us_keg != keg) {
2780 KEG_UNLOCK(keg);
2781 keg = slab->us_keg;
2782 KEG_LOCK(keg);
2783 }
|
2789 }
| 2784 }
|
| 2785 slab_free_item(keg, slab, item);
2786 if (keg->uk_flags & UMA_ZFLAG_FULL) {
2787 if (keg->uk_pages < keg->uk_maxpages) {
2788 keg->uk_flags &= ~UMA_ZFLAG_FULL;
2789 clearfull = 1;
2790 }
|
2790
| 2791
|
2791 /*
2792 * We can handle one more allocation. Since we're
2793 * clearing ZFLAG_FULL, wake up all procs blocked
2794 * on pages. This should be uncommon, so keeping this
2795 * simple for now (rather than adding count of blocked
2796 * threads etc).
2797 */
2798 wakeup(keg);
| 2792 /*
2793 * We can handle one more allocation. Since we're
2794 * clearing ZFLAG_FULL, wake up all procs blocked
2795 * on pages. This should be uncommon, so keeping this
2796 * simple for now (rather than adding count of blocked
2797 * threads etc).
2798 */
2799 wakeup(keg);
2800 }
|
2799 }
| 2801 }
|
| 2802 zone_relock(zone, keg);
|
2800 if (clearfull) {
| 2803 if (clearfull) {
|
2801 zone_relock(zone, keg);
| |
2802 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2803 wakeup(zone);
| 2804 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2805 wakeup(zone);
|
2804 ZONE_UNLOCK(zone);
2805 } else
2806 KEG_UNLOCK(keg);
| 2806 }
2807 ZONE_UNLOCK(zone);
|
2807
2808}
2809
| 2808
2809}
2810
|
| 2811/*
2812 * Frees a single item to any zone.
2813 *
2814 * Arguments:
2815 * zone The zone to free to
2816 * item The item we're freeing
2817 * udata User supplied data for the dtor
2818 * skip Skip dtors and finis
2819 */
2820static void
2821zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
2822{
2823
2824#ifdef INVARIANTS
2825 if (skip == SKIP_NONE) {
2826 if (zone->uz_flags & UMA_ZONE_MALLOC)
2827 uma_dbg_free(zone, udata, item);
2828 else
2829 uma_dbg_free(zone, NULL, item);
2830 }
2831#endif
2832 if (skip < SKIP_DTOR && zone->uz_dtor)
2833 zone->uz_dtor(item, zone->uz_size, udata);
2834
2835 if (skip < SKIP_FINI && zone->uz_fini)
2836 zone->uz_fini(item, zone->uz_size);
2837
2838 atomic_add_long(&zone->uz_frees, 1);
2839 zone->uz_release(zone->uz_arg, &item, 1);
2840}
2841
|
2810/* See uma.h */
2811int
2812uma_zone_set_max(uma_zone_t zone, int nitems)
2813{
2814 uma_keg_t keg;
2815
| 2842/* See uma.h */
2843int
2844uma_zone_set_max(uma_zone_t zone, int nitems)
2845{
2846 uma_keg_t keg;
2847
|
2816 ZONE_LOCK(zone);
| |
2817 keg = zone_first_keg(zone);
| 2848 keg = zone_first_keg(zone);
|
| 2849 if (keg == NULL)
2850 return (0);
2851 ZONE_LOCK(zone);
|
2818 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2819 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2820 keg->uk_maxpages += keg->uk_ppera;
2821 nitems = keg->uk_maxpages * keg->uk_ipers;
2822 ZONE_UNLOCK(zone);
2823
2824 return (nitems);
2825}
2826
2827/* See uma.h */
2828int
2829uma_zone_get_max(uma_zone_t zone)
2830{
2831 int nitems;
2832 uma_keg_t keg;
2833
| 2852 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2853 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2854 keg->uk_maxpages += keg->uk_ppera;
2855 nitems = keg->uk_maxpages * keg->uk_ipers;
2856 ZONE_UNLOCK(zone);
2857
2858 return (nitems);
2859}
2860
2861/* See uma.h */
2862int
2863uma_zone_get_max(uma_zone_t zone)
2864{
2865 int nitems;
2866 uma_keg_t keg;
2867
|
2834 ZONE_LOCK(zone);
| |
2835 keg = zone_first_keg(zone);
| 2868 keg = zone_first_keg(zone);
|
| 2869 if (keg == NULL)
2870 return (0);
2871 ZONE_LOCK(zone);
|
2836 nitems = keg->uk_maxpages * keg->uk_ipers;
2837 ZONE_UNLOCK(zone);
2838
2839 return (nitems);
2840}
2841
2842/* See uma.h */
2843void
2844uma_zone_set_warning(uma_zone_t zone, const char *warning)
2845{
2846
2847 ZONE_LOCK(zone);
2848 zone->uz_warning = warning;
2849 ZONE_UNLOCK(zone);
2850}
2851
2852/* See uma.h */
2853int
2854uma_zone_get_cur(uma_zone_t zone)
2855{
2856 int64_t nitems;
2857 u_int i;
2858
2859 ZONE_LOCK(zone);
2860 nitems = zone->uz_allocs - zone->uz_frees;
2861 CPU_FOREACH(i) {
2862 /*
2863 * See the comment in sysctl_vm_zone_stats() regarding the
2864 * safety of accessing the per-cpu caches. With the zone lock
2865 * held, it is safe, but can potentially result in stale data.
2866 */
2867 nitems += zone->uz_cpu[i].uc_allocs -
2868 zone->uz_cpu[i].uc_frees;
2869 }
2870 ZONE_UNLOCK(zone);
2871
2872 return (nitems < 0 ? 0 : nitems);
2873}
2874
2875/* See uma.h */
2876void
2877uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2878{
2879 uma_keg_t keg;
2880
2881 ZONE_LOCK(zone);
2882 keg = zone_first_keg(zone);
| 2872 nitems = keg->uk_maxpages * keg->uk_ipers;
2873 ZONE_UNLOCK(zone);
2874
2875 return (nitems);
2876}
2877
2878/* See uma.h */
2879void
2880uma_zone_set_warning(uma_zone_t zone, const char *warning)
2881{
2882
2883 ZONE_LOCK(zone);
2884 zone->uz_warning = warning;
2885 ZONE_UNLOCK(zone);
2886}
2887
2888/* See uma.h */
2889int
2890uma_zone_get_cur(uma_zone_t zone)
2891{
2892 int64_t nitems;
2893 u_int i;
2894
2895 ZONE_LOCK(zone);
2896 nitems = zone->uz_allocs - zone->uz_frees;
2897 CPU_FOREACH(i) {
2898 /*
2899 * See the comment in sysctl_vm_zone_stats() regarding the
2900 * safety of accessing the per-cpu caches. With the zone lock
2901 * held, it is safe, but can potentially result in stale data.
2902 */
2903 nitems += zone->uz_cpu[i].uc_allocs -
2904 zone->uz_cpu[i].uc_frees;
2905 }
2906 ZONE_UNLOCK(zone);
2907
2908 return (nitems < 0 ? 0 : nitems);
2909}
2910
2911/* See uma.h */
2912void
2913uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2914{
2915 uma_keg_t keg;
2916
2917 ZONE_LOCK(zone);
2918 keg = zone_first_keg(zone);
|
| 2919 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
|
2883 KASSERT(keg->uk_pages == 0,
2884 ("uma_zone_set_init on non-empty keg"));
2885 keg->uk_init = uminit;
2886 ZONE_UNLOCK(zone);
2887}
2888
2889/* See uma.h */
2890void
2891uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2892{
2893 uma_keg_t keg;
2894
2895 ZONE_LOCK(zone);
2896 keg = zone_first_keg(zone);
| 2920 KASSERT(keg->uk_pages == 0,
2921 ("uma_zone_set_init on non-empty keg"));
2922 keg->uk_init = uminit;
2923 ZONE_UNLOCK(zone);
2924}
2925
2926/* See uma.h */
2927void
2928uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2929{
2930 uma_keg_t keg;
2931
2932 ZONE_LOCK(zone);
2933 keg = zone_first_keg(zone);
|
| 2934 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
|
2897 KASSERT(keg->uk_pages == 0,
2898 ("uma_zone_set_fini on non-empty keg"));
2899 keg->uk_fini = fini;
2900 ZONE_UNLOCK(zone);
2901}
2902
2903/* See uma.h */
2904void
2905uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
2906{
2907 ZONE_LOCK(zone);
2908 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2909 ("uma_zone_set_zinit on non-empty keg"));
2910 zone->uz_init = zinit;
2911 ZONE_UNLOCK(zone);
2912}
2913
2914/* See uma.h */
2915void
2916uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
2917{
2918 ZONE_LOCK(zone);
2919 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2920 ("uma_zone_set_zfini on non-empty keg"));
2921 zone->uz_fini = zfini;
2922 ZONE_UNLOCK(zone);
2923}
2924
2925/* See uma.h */
2926/* XXX uk_freef is not actually used with the zone locked */
2927void
2928uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2929{
| 2935 KASSERT(keg->uk_pages == 0,
2936 ("uma_zone_set_fini on non-empty keg"));
2937 keg->uk_fini = fini;
2938 ZONE_UNLOCK(zone);
2939}
2940
2941/* See uma.h */
2942void
2943uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
2944{
2945 ZONE_LOCK(zone);
2946 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2947 ("uma_zone_set_zinit on non-empty keg"));
2948 zone->uz_init = zinit;
2949 ZONE_UNLOCK(zone);
2950}
2951
2952/* See uma.h */
2953void
2954uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
2955{
2956 ZONE_LOCK(zone);
2957 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2958 ("uma_zone_set_zfini on non-empty keg"));
2959 zone->uz_fini = zfini;
2960 ZONE_UNLOCK(zone);
2961}
2962
2963/* See uma.h */
2964/* XXX uk_freef is not actually used with the zone locked */
2965void
2966uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2967{
|
| 2968 uma_keg_t keg;
|
2930
2931 ZONE_LOCK(zone);
| 2969
2970 ZONE_LOCK(zone);
|
2932 zone_first_keg(zone)->uk_freef = freef;
| 2971 keg = zone_first_keg(zone);
2972 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2973 keg->uk_freef = freef;
|
2933 ZONE_UNLOCK(zone);
2934}
2935
2936/* See uma.h */
2937/* XXX uk_allocf is not actually used with the zone locked */
2938void
2939uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2940{
2941 uma_keg_t keg;
2942
2943 ZONE_LOCK(zone);
2944 keg = zone_first_keg(zone);
2945 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
2946 keg->uk_allocf = allocf;
2947 ZONE_UNLOCK(zone);
2948}
2949
2950/* See uma.h */
2951int
2952uma_zone_reserve_kva(uma_zone_t zone, int count)
2953{
2954 uma_keg_t keg;
2955 vm_offset_t kva;
2956 int pages;
2957
2958 keg = zone_first_keg(zone);
| 2974 ZONE_UNLOCK(zone);
2975}
2976
2977/* See uma.h */
2978/* XXX uk_allocf is not actually used with the zone locked */
2979void
2980uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2981{
2982 uma_keg_t keg;
2983
2984 ZONE_LOCK(zone);
2985 keg = zone_first_keg(zone);
2986 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
2987 keg->uk_allocf = allocf;
2988 ZONE_UNLOCK(zone);
2989}
2990
2991/* See uma.h */
2992int
2993uma_zone_reserve_kva(uma_zone_t zone, int count)
2994{
2995 uma_keg_t keg;
2996 vm_offset_t kva;
2997 int pages;
2998
2999 keg = zone_first_keg(zone);
|
| 3000 if (keg == NULL)
3001 return (0);
|
2959 pages = count / keg->uk_ipers;
2960
2961 if (pages * keg->uk_ipers < count)
2962 pages++;
2963
2964#ifdef UMA_MD_SMALL_ALLOC
2965 if (keg->uk_ppera > 1) {
2966#else
2967 if (1) {
2968#endif
2969 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
2970 if (kva == 0)
2971 return (0);
2972 } else
2973 kva = 0;
2974 ZONE_LOCK(zone);
2975 keg->uk_kva = kva;
2976 keg->uk_offset = 0;
2977 keg->uk_maxpages = pages;
2978#ifdef UMA_MD_SMALL_ALLOC
2979 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
2980#else
2981 keg->uk_allocf = noobj_alloc;
2982#endif
2983 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
2984 ZONE_UNLOCK(zone);
2985 return (1);
2986}
2987
2988/* See uma.h */
2989void
2990uma_prealloc(uma_zone_t zone, int items)
2991{
2992 int slabs;
2993 uma_slab_t slab;
2994 uma_keg_t keg;
2995
2996 keg = zone_first_keg(zone);
| 3002 pages = count / keg->uk_ipers;
3003
3004 if (pages * keg->uk_ipers < count)
3005 pages++;
3006
3007#ifdef UMA_MD_SMALL_ALLOC
3008 if (keg->uk_ppera > 1) {
3009#else
3010 if (1) {
3011#endif
3012 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
3013 if (kva == 0)
3014 return (0);
3015 } else
3016 kva = 0;
3017 ZONE_LOCK(zone);
3018 keg->uk_kva = kva;
3019 keg->uk_offset = 0;
3020 keg->uk_maxpages = pages;
3021#ifdef UMA_MD_SMALL_ALLOC
3022 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
3023#else
3024 keg->uk_allocf = noobj_alloc;
3025#endif
3026 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
3027 ZONE_UNLOCK(zone);
3028 return (1);
3029}
3030
3031/* See uma.h */
3032void
3033uma_prealloc(uma_zone_t zone, int items)
3034{
3035 int slabs;
3036 uma_slab_t slab;
3037 uma_keg_t keg;
3038
3039 keg = zone_first_keg(zone);
|
| 3040 if (keg == NULL)
3041 return;
|
2997 ZONE_LOCK(zone);
2998 slabs = items / keg->uk_ipers;
2999 if (slabs * keg->uk_ipers < items)
3000 slabs++;
3001 while (slabs > 0) {
3002 slab = keg_alloc_slab(keg, zone, M_WAITOK);
3003 if (slab == NULL)
3004 break;
3005 MPASS(slab->us_keg == keg);
3006 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
3007 slabs--;
3008 }
3009 ZONE_UNLOCK(zone);
3010}
3011
3012/* See uma.h */
3013uint32_t *
3014uma_find_refcnt(uma_zone_t zone, void *item)
3015{
3016 uma_slabrefcnt_t slabref;
3017 uma_slab_t slab;
3018 uma_keg_t keg;
3019 uint32_t *refcnt;
3020 int idx;
3021
3022 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
3023 slabref = (uma_slabrefcnt_t)slab;
3024 keg = slab->us_keg;
3025 KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
3026 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3027 idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
3028 refcnt = &slabref->us_refcnt[idx];
3029 return refcnt;
3030}
3031
3032/* See uma.h */
3033void
3034uma_reclaim(void)
3035{
3036#ifdef UMA_DEBUG
3037 printf("UMA: vm asked us to release pages!\n");
3038#endif
3039 bucket_enable();
3040 zone_foreach(zone_drain);
3041 /*
3042 * Some slabs may have been freed but this zone will be visited early
3043 * we visit again so that we can free pages that are empty once other
3044 * zones are drained. We have to do the same for buckets.
3045 */
3046 zone_drain(slabzone);
3047 zone_drain(slabrefzone);
3048 bucket_zone_drain();
3049}
3050
3051/* See uma.h */
3052int
3053uma_zone_exhausted(uma_zone_t zone)
3054{
3055 int full;
3056
3057 ZONE_LOCK(zone);
3058 full = (zone->uz_flags & UMA_ZFLAG_FULL);
3059 ZONE_UNLOCK(zone);
3060 return (full);
3061}
3062
3063int
3064uma_zone_exhausted_nolock(uma_zone_t zone)
3065{
3066 return (zone->uz_flags & UMA_ZFLAG_FULL);
3067}
3068
3069void *
3070uma_large_malloc(int size, int wait)
3071{
3072 void *mem;
3073 uma_slab_t slab;
3074 uint8_t flags;
3075
3076 slab = zone_alloc_item(slabzone, NULL, wait);
3077 if (slab == NULL)
3078 return (NULL);
3079 mem = page_alloc(NULL, size, &flags, wait);
3080 if (mem) {
3081 vsetslab((vm_offset_t)mem, slab);
3082 slab->us_data = mem;
3083 slab->us_flags = flags | UMA_SLAB_MALLOC;
3084 slab->us_size = size;
3085 } else {
| 3042 ZONE_LOCK(zone);
3043 slabs = items / keg->uk_ipers;
3044 if (slabs * keg->uk_ipers < items)
3045 slabs++;
3046 while (slabs > 0) {
3047 slab = keg_alloc_slab(keg, zone, M_WAITOK);
3048 if (slab == NULL)
3049 break;
3050 MPASS(slab->us_keg == keg);
3051 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
3052 slabs--;
3053 }
3054 ZONE_UNLOCK(zone);
3055}
3056
3057/* See uma.h */
3058uint32_t *
3059uma_find_refcnt(uma_zone_t zone, void *item)
3060{
3061 uma_slabrefcnt_t slabref;
3062 uma_slab_t slab;
3063 uma_keg_t keg;
3064 uint32_t *refcnt;
3065 int idx;
3066
3067 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
3068 slabref = (uma_slabrefcnt_t)slab;
3069 keg = slab->us_keg;
3070 KASSERT(keg->uk_flags & UMA_ZONE_REFCNT,
3071 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT"));
3072 idx = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
3073 refcnt = &slabref->us_refcnt[idx];
3074 return refcnt;
3075}
3076
3077/* See uma.h */
3078void
3079uma_reclaim(void)
3080{
3081#ifdef UMA_DEBUG
3082 printf("UMA: vm asked us to release pages!\n");
3083#endif
3084 bucket_enable();
3085 zone_foreach(zone_drain);
3086 /*
3087 * Some slabs may have been freed but this zone will be visited early
3088 * we visit again so that we can free pages that are empty once other
3089 * zones are drained. We have to do the same for buckets.
3090 */
3091 zone_drain(slabzone);
3092 zone_drain(slabrefzone);
3093 bucket_zone_drain();
3094}
3095
3096/* See uma.h */
3097int
3098uma_zone_exhausted(uma_zone_t zone)
3099{
3100 int full;
3101
3102 ZONE_LOCK(zone);
3103 full = (zone->uz_flags & UMA_ZFLAG_FULL);
3104 ZONE_UNLOCK(zone);
3105 return (full);
3106}
3107
3108int
3109uma_zone_exhausted_nolock(uma_zone_t zone)
3110{
3111 return (zone->uz_flags & UMA_ZFLAG_FULL);
3112}
3113
3114void *
3115uma_large_malloc(int size, int wait)
3116{
3117 void *mem;
3118 uma_slab_t slab;
3119 uint8_t flags;
3120
3121 slab = zone_alloc_item(slabzone, NULL, wait);
3122 if (slab == NULL)
3123 return (NULL);
3124 mem = page_alloc(NULL, size, &flags, wait);
3125 if (mem) {
3126 vsetslab((vm_offset_t)mem, slab);
3127 slab->us_data = mem;
3128 slab->us_flags = flags | UMA_SLAB_MALLOC;
3129 slab->us_size = size;
3130 } else {
|
3086 zone_free_item(slabzone, slab, NULL, SKIP_NONE,
3087 ZFREE_STATFAIL | ZFREE_STATFREE);
| 3131 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
|
3088 }
3089
3090 return (mem);
3091}
3092
3093void
3094uma_large_free(uma_slab_t slab)
3095{
3096 vsetobj((vm_offset_t)slab->us_data, kmem_object);
3097 page_free(slab->us_data, slab->us_size, slab->us_flags);
| 3132 }
3133
3134 return (mem);
3135}
3136
3137void
3138uma_large_free(uma_slab_t slab)
3139{
3140 vsetobj((vm_offset_t)slab->us_data, kmem_object);
3141 page_free(slab->us_data, slab->us_size, slab->us_flags);
|
3098 zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
| 3142 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
|
3099}
3100
3101void
3102uma_print_stats(void)
3103{
3104 zone_foreach(uma_print_zone);
3105}
3106
3107static void
3108slab_print(uma_slab_t slab)
3109{
3110 printf("slab: keg %p, data %p, freecount %d\n",
3111 slab->us_keg, slab->us_data, slab->us_freecount);
3112}
3113
3114static void
3115cache_print(uma_cache_t cache)
3116{
3117 printf("alloc: %p(%d), free: %p(%d)\n",
3118 cache->uc_allocbucket,
3119 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
3120 cache->uc_freebucket,
3121 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
3122}
3123
3124static void
3125uma_print_keg(uma_keg_t keg)
3126{
3127 uma_slab_t slab;
3128
3129 printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
3130 "out %d free %d limit %d\n",
3131 keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
3132 keg->uk_ipers, keg->uk_ppera,
3133 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free,
3134 (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
3135 printf("Part slabs:\n");
3136 LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
3137 slab_print(slab);
3138 printf("Free slabs:\n");
3139 LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
3140 slab_print(slab);
3141 printf("Full slabs:\n");
3142 LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
3143 slab_print(slab);
3144}
3145
3146void
3147uma_print_zone(uma_zone_t zone)
3148{
3149 uma_cache_t cache;
3150 uma_klink_t kl;
3151 int i;
3152
3153 printf("zone: %s(%p) size %d flags %#x\n",
3154 zone->uz_name, zone, zone->uz_size, zone->uz_flags);
3155 LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
3156 uma_print_keg(kl->kl_keg);
3157 CPU_FOREACH(i) {
3158 cache = &zone->uz_cpu[i];
3159 printf("CPU %d Cache:\n", i);
3160 cache_print(cache);
3161 }
3162}
3163
3164#ifdef DDB
3165/*
3166 * Generate statistics across both the zone and its per-cpu cache's. Return
3167 * desired statistics if the pointer is non-NULL for that statistic.
3168 *
3169 * Note: does not update the zone statistics, as it can't safely clear the
3170 * per-CPU cache statistic.
3171 *
3172 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
3173 * safe from off-CPU; we should modify the caches to track this information
3174 * directly so that we don't have to.
3175 */
3176static void
3177uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
3178 uint64_t *freesp, uint64_t *sleepsp)
3179{
3180 uma_cache_t cache;
3181 uint64_t allocs, frees, sleeps;
3182 int cachefree, cpu;
3183
3184 allocs = frees = sleeps = 0;
3185 cachefree = 0;
3186 CPU_FOREACH(cpu) {
3187 cache = &z->uz_cpu[cpu];
3188 if (cache->uc_allocbucket != NULL)
3189 cachefree += cache->uc_allocbucket->ub_cnt;
3190 if (cache->uc_freebucket != NULL)
3191 cachefree += cache->uc_freebucket->ub_cnt;
3192 allocs += cache->uc_allocs;
3193 frees += cache->uc_frees;
3194 }
3195 allocs += z->uz_allocs;
3196 frees += z->uz_frees;
3197 sleeps += z->uz_sleeps;
3198 if (cachefreep != NULL)
3199 *cachefreep = cachefree;
3200 if (allocsp != NULL)
3201 *allocsp = allocs;
3202 if (freesp != NULL)
3203 *freesp = frees;
3204 if (sleepsp != NULL)
3205 *sleepsp = sleeps;
3206}
3207#endif /* DDB */
3208
3209static int
3210sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
3211{
3212 uma_keg_t kz;
3213 uma_zone_t z;
3214 int count;
3215
3216 count = 0;
3217 mtx_lock(&uma_mtx);
3218 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3219 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3220 count++;
3221 }
3222 mtx_unlock(&uma_mtx);
3223 return (sysctl_handle_int(oidp, &count, 0, req));
3224}
3225
3226static int
3227sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
3228{
3229 struct uma_stream_header ush;
3230 struct uma_type_header uth;
3231 struct uma_percpu_stat ups;
3232 uma_bucket_t bucket;
3233 struct sbuf sbuf;
3234 uma_cache_t cache;
3235 uma_klink_t kl;
3236 uma_keg_t kz;
3237 uma_zone_t z;
3238 uma_keg_t k;
3239 int count, error, i;
3240
3241 error = sysctl_wire_old_buffer(req, 0);
3242 if (error != 0)
3243 return (error);
3244 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
3245
3246 count = 0;
3247 mtx_lock(&uma_mtx);
3248 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3249 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3250 count++;
3251 }
3252
3253 /*
3254 * Insert stream header.
3255 */
3256 bzero(&ush, sizeof(ush));
3257 ush.ush_version = UMA_STREAM_VERSION;
3258 ush.ush_maxcpus = (mp_maxid + 1);
3259 ush.ush_count = count;
3260 (void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
3261
3262 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3263 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3264 bzero(&uth, sizeof(uth));
3265 ZONE_LOCK(z);
3266 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
3267 uth.uth_align = kz->uk_align;
3268 uth.uth_size = kz->uk_size;
3269 uth.uth_rsize = kz->uk_rsize;
3270 LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
3271 k = kl->kl_keg;
3272 uth.uth_maxpages += k->uk_maxpages;
3273 uth.uth_pages += k->uk_pages;
3274 uth.uth_keg_free += k->uk_free;
3275 uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
3276 * k->uk_ipers;
3277 }
3278
3279 /*
3280 * A zone is secondary is it is not the first entry
3281 * on the keg's zone list.
3282 */
3283 if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
3284 (LIST_FIRST(&kz->uk_zones) != z))
3285 uth.uth_zone_flags = UTH_ZONE_SECONDARY;
3286
3287 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
3288 uth.uth_zone_free += bucket->ub_cnt;
3289 uth.uth_allocs = z->uz_allocs;
3290 uth.uth_frees = z->uz_frees;
3291 uth.uth_fails = z->uz_fails;
3292 uth.uth_sleeps = z->uz_sleeps;
3293 (void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
3294 /*
3295 * While it is not normally safe to access the cache
3296 * bucket pointers while not on the CPU that owns the
3297 * cache, we only allow the pointers to be exchanged
3298 * without the zone lock held, not invalidated, so
3299 * accept the possible race associated with bucket
3300 * exchange during monitoring.
3301 */
3302 for (i = 0; i < (mp_maxid + 1); i++) {
3303 bzero(&ups, sizeof(ups));
3304 if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
3305 goto skip;
3306 if (CPU_ABSENT(i))
3307 goto skip;
3308 cache = &z->uz_cpu[i];
3309 if (cache->uc_allocbucket != NULL)
3310 ups.ups_cache_free +=
3311 cache->uc_allocbucket->ub_cnt;
3312 if (cache->uc_freebucket != NULL)
3313 ups.ups_cache_free +=
3314 cache->uc_freebucket->ub_cnt;
3315 ups.ups_allocs = cache->uc_allocs;
3316 ups.ups_frees = cache->uc_frees;
3317skip:
3318 (void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
3319 }
3320 ZONE_UNLOCK(z);
3321 }
3322 }
3323 mtx_unlock(&uma_mtx);
3324 error = sbuf_finish(&sbuf);
3325 sbuf_delete(&sbuf);
3326 return (error);
3327}
3328
3329#ifdef DDB
3330DB_SHOW_COMMAND(uma, db_show_uma)
3331{
3332 uint64_t allocs, frees, sleeps;
3333 uma_bucket_t bucket;
3334 uma_keg_t kz;
3335 uma_zone_t z;
3336 int cachefree;
3337
3338 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
3339 "Requests", "Sleeps");
3340 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3341 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3342 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
3343 allocs = z->uz_allocs;
3344 frees = z->uz_frees;
3345 sleeps = z->uz_sleeps;
3346 cachefree = 0;
3347 } else
3348 uma_zone_sumstat(z, &cachefree, &allocs,
3349 &frees, &sleeps);
3350 if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
3351 (LIST_FIRST(&kz->uk_zones) != z)))
3352 cachefree += kz->uk_free;
3353 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
3354 cachefree += bucket->ub_cnt;
3355 db_printf("%18s %8ju %8jd %8d %12ju %8ju\n", z->uz_name,
3356 (uintmax_t)kz->uk_size,
3357 (intmax_t)(allocs - frees), cachefree,
3358 (uintmax_t)allocs, sleeps);
3359 if (db_pager_quit)
3360 return;
3361 }
3362 }
3363}
3364#endif
| 3143}
3144
3145void
3146uma_print_stats(void)
3147{
3148 zone_foreach(uma_print_zone);
3149}
3150
3151static void
3152slab_print(uma_slab_t slab)
3153{
3154 printf("slab: keg %p, data %p, freecount %d\n",
3155 slab->us_keg, slab->us_data, slab->us_freecount);
3156}
3157
3158static void
3159cache_print(uma_cache_t cache)
3160{
3161 printf("alloc: %p(%d), free: %p(%d)\n",
3162 cache->uc_allocbucket,
3163 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0,
3164 cache->uc_freebucket,
3165 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0);
3166}
3167
3168static void
3169uma_print_keg(uma_keg_t keg)
3170{
3171 uma_slab_t slab;
3172
3173 printf("keg: %s(%p) size %d(%d) flags %#x ipers %d ppera %d "
3174 "out %d free %d limit %d\n",
3175 keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
3176 keg->uk_ipers, keg->uk_ppera,
3177 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free,
3178 (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
3179 printf("Part slabs:\n");
3180 LIST_FOREACH(slab, &keg->uk_part_slab, us_link)
3181 slab_print(slab);
3182 printf("Free slabs:\n");
3183 LIST_FOREACH(slab, &keg->uk_free_slab, us_link)
3184 slab_print(slab);
3185 printf("Full slabs:\n");
3186 LIST_FOREACH(slab, &keg->uk_full_slab, us_link)
3187 slab_print(slab);
3188}
3189
3190void
3191uma_print_zone(uma_zone_t zone)
3192{
3193 uma_cache_t cache;
3194 uma_klink_t kl;
3195 int i;
3196
3197 printf("zone: %s(%p) size %d flags %#x\n",
3198 zone->uz_name, zone, zone->uz_size, zone->uz_flags);
3199 LIST_FOREACH(kl, &zone->uz_kegs, kl_link)
3200 uma_print_keg(kl->kl_keg);
3201 CPU_FOREACH(i) {
3202 cache = &zone->uz_cpu[i];
3203 printf("CPU %d Cache:\n", i);
3204 cache_print(cache);
3205 }
3206}
3207
3208#ifdef DDB
3209/*
3210 * Generate statistics across both the zone and its per-cpu cache's. Return
3211 * desired statistics if the pointer is non-NULL for that statistic.
3212 *
3213 * Note: does not update the zone statistics, as it can't safely clear the
3214 * per-CPU cache statistic.
3215 *
3216 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't
3217 * safe from off-CPU; we should modify the caches to track this information
3218 * directly so that we don't have to.
3219 */
3220static void
3221uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp,
3222 uint64_t *freesp, uint64_t *sleepsp)
3223{
3224 uma_cache_t cache;
3225 uint64_t allocs, frees, sleeps;
3226 int cachefree, cpu;
3227
3228 allocs = frees = sleeps = 0;
3229 cachefree = 0;
3230 CPU_FOREACH(cpu) {
3231 cache = &z->uz_cpu[cpu];
3232 if (cache->uc_allocbucket != NULL)
3233 cachefree += cache->uc_allocbucket->ub_cnt;
3234 if (cache->uc_freebucket != NULL)
3235 cachefree += cache->uc_freebucket->ub_cnt;
3236 allocs += cache->uc_allocs;
3237 frees += cache->uc_frees;
3238 }
3239 allocs += z->uz_allocs;
3240 frees += z->uz_frees;
3241 sleeps += z->uz_sleeps;
3242 if (cachefreep != NULL)
3243 *cachefreep = cachefree;
3244 if (allocsp != NULL)
3245 *allocsp = allocs;
3246 if (freesp != NULL)
3247 *freesp = frees;
3248 if (sleepsp != NULL)
3249 *sleepsp = sleeps;
3250}
3251#endif /* DDB */
3252
3253static int
3254sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS)
3255{
3256 uma_keg_t kz;
3257 uma_zone_t z;
3258 int count;
3259
3260 count = 0;
3261 mtx_lock(&uma_mtx);
3262 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3263 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3264 count++;
3265 }
3266 mtx_unlock(&uma_mtx);
3267 return (sysctl_handle_int(oidp, &count, 0, req));
3268}
3269
3270static int
3271sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS)
3272{
3273 struct uma_stream_header ush;
3274 struct uma_type_header uth;
3275 struct uma_percpu_stat ups;
3276 uma_bucket_t bucket;
3277 struct sbuf sbuf;
3278 uma_cache_t cache;
3279 uma_klink_t kl;
3280 uma_keg_t kz;
3281 uma_zone_t z;
3282 uma_keg_t k;
3283 int count, error, i;
3284
3285 error = sysctl_wire_old_buffer(req, 0);
3286 if (error != 0)
3287 return (error);
3288 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
3289
3290 count = 0;
3291 mtx_lock(&uma_mtx);
3292 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3293 LIST_FOREACH(z, &kz->uk_zones, uz_link)
3294 count++;
3295 }
3296
3297 /*
3298 * Insert stream header.
3299 */
3300 bzero(&ush, sizeof(ush));
3301 ush.ush_version = UMA_STREAM_VERSION;
3302 ush.ush_maxcpus = (mp_maxid + 1);
3303 ush.ush_count = count;
3304 (void)sbuf_bcat(&sbuf, &ush, sizeof(ush));
3305
3306 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3307 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3308 bzero(&uth, sizeof(uth));
3309 ZONE_LOCK(z);
3310 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME);
3311 uth.uth_align = kz->uk_align;
3312 uth.uth_size = kz->uk_size;
3313 uth.uth_rsize = kz->uk_rsize;
3314 LIST_FOREACH(kl, &z->uz_kegs, kl_link) {
3315 k = kl->kl_keg;
3316 uth.uth_maxpages += k->uk_maxpages;
3317 uth.uth_pages += k->uk_pages;
3318 uth.uth_keg_free += k->uk_free;
3319 uth.uth_limit = (k->uk_maxpages / k->uk_ppera)
3320 * k->uk_ipers;
3321 }
3322
3323 /*
3324 * A zone is secondary is it is not the first entry
3325 * on the keg's zone list.
3326 */
3327 if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
3328 (LIST_FIRST(&kz->uk_zones) != z))
3329 uth.uth_zone_flags = UTH_ZONE_SECONDARY;
3330
3331 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
3332 uth.uth_zone_free += bucket->ub_cnt;
3333 uth.uth_allocs = z->uz_allocs;
3334 uth.uth_frees = z->uz_frees;
3335 uth.uth_fails = z->uz_fails;
3336 uth.uth_sleeps = z->uz_sleeps;
3337 (void)sbuf_bcat(&sbuf, &uth, sizeof(uth));
3338 /*
3339 * While it is not normally safe to access the cache
3340 * bucket pointers while not on the CPU that owns the
3341 * cache, we only allow the pointers to be exchanged
3342 * without the zone lock held, not invalidated, so
3343 * accept the possible race associated with bucket
3344 * exchange during monitoring.
3345 */
3346 for (i = 0; i < (mp_maxid + 1); i++) {
3347 bzero(&ups, sizeof(ups));
3348 if (kz->uk_flags & UMA_ZFLAG_INTERNAL)
3349 goto skip;
3350 if (CPU_ABSENT(i))
3351 goto skip;
3352 cache = &z->uz_cpu[i];
3353 if (cache->uc_allocbucket != NULL)
3354 ups.ups_cache_free +=
3355 cache->uc_allocbucket->ub_cnt;
3356 if (cache->uc_freebucket != NULL)
3357 ups.ups_cache_free +=
3358 cache->uc_freebucket->ub_cnt;
3359 ups.ups_allocs = cache->uc_allocs;
3360 ups.ups_frees = cache->uc_frees;
3361skip:
3362 (void)sbuf_bcat(&sbuf, &ups, sizeof(ups));
3363 }
3364 ZONE_UNLOCK(z);
3365 }
3366 }
3367 mtx_unlock(&uma_mtx);
3368 error = sbuf_finish(&sbuf);
3369 sbuf_delete(&sbuf);
3370 return (error);
3371}
3372
3373#ifdef DDB
3374DB_SHOW_COMMAND(uma, db_show_uma)
3375{
3376 uint64_t allocs, frees, sleeps;
3377 uma_bucket_t bucket;
3378 uma_keg_t kz;
3379 uma_zone_t z;
3380 int cachefree;
3381
3382 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free",
3383 "Requests", "Sleeps");
3384 LIST_FOREACH(kz, &uma_kegs, uk_link) {
3385 LIST_FOREACH(z, &kz->uk_zones, uz_link) {
3386 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) {
3387 allocs = z->uz_allocs;
3388 frees = z->uz_frees;
3389 sleeps = z->uz_sleeps;
3390 cachefree = 0;
3391 } else
3392 uma_zone_sumstat(z, &cachefree, &allocs,
3393 &frees, &sleeps);
3394 if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
3395 (LIST_FIRST(&kz->uk_zones) != z)))
3396 cachefree += kz->uk_free;
3397 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
3398 cachefree += bucket->ub_cnt;
3399 db_printf("%18s %8ju %8jd %8d %12ju %8ju\n", z->uz_name,
3400 (uintmax_t)kz->uk_size,
3401 (intmax_t)(allocs - frees), cachefree,
3402 (uintmax_t)allocs, sleeps);
3403 if (db_pager_quit)
3404 return;
3405 }
3406 }
3407}
3408#endif
|