1/*-
| 1/*-
|
2 * Copyright (c) 2002, 2003, 2004, 2005 Jeffrey Roberson <jeff@FreeBSD.org>
| 2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
|
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * uma_core.c Implementation of the Universal Memory allocator 31 * 32 * This allocator is intended to replace the multitude of similar object caches 33 * in the standard FreeBSD kernel. The intent is to be flexible as well as 34 * effecient. A primary design goal is to return unused memory to the rest of 35 * the system. This will make the system as a whole more flexible due to the 36 * ability to move memory to subsystems which most need it instead of leaving 37 * pools of reserved memory unused. 38 * 39 * The basic ideas stem from similar slab/zone based allocators whose algorithms 40 * are well known. 41 * 42 */ 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h>
| 3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29/* 30 * uma_core.c Implementation of the Universal Memory allocator 31 * 32 * This allocator is intended to replace the multitude of similar object caches 33 * in the standard FreeBSD kernel. The intent is to be flexible as well as 34 * effecient. A primary design goal is to return unused memory to the rest of 35 * the system. This will make the system as a whole more flexible due to the 36 * ability to move memory to subsystems which most need it instead of leaving 37 * pools of reserved memory unused. 38 * 39 * The basic ideas stem from similar slab/zone based allocators whose algorithms 40 * are well known. 41 * 42 */ 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h>
|
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 182047 2008-08-23 12:40:07Z antoine $");
| 51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 187681 2009-01-25 09:11:24Z jeff $");
|
52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 60#include "opt_ddb.h" 61#include "opt_param.h" 62 63#include <sys/param.h> 64#include <sys/systm.h> 65#include <sys/kernel.h> 66#include <sys/types.h> 67#include <sys/queue.h> 68#include <sys/malloc.h> 69#include <sys/ktr.h> 70#include <sys/lock.h> 71#include <sys/sysctl.h> 72#include <sys/mutex.h> 73#include <sys/proc.h> 74#include <sys/sbuf.h> 75#include <sys/smp.h> 76#include <sys/vmmeter.h> 77 78#include <vm/vm.h> 79#include <vm/vm_object.h> 80#include <vm/vm_page.h> 81#include <vm/vm_param.h> 82#include <vm/vm_map.h> 83#include <vm/vm_kern.h> 84#include <vm/vm_extern.h> 85#include <vm/uma.h> 86#include <vm/uma_int.h> 87#include <vm/uma_dbg.h> 88 89#include <machine/vmparam.h> 90 91#include <ddb/ddb.h> 92 93/* 94 * This is the zone and keg from which all zones are spawned. The idea is that 95 * even the zone & keg heads are allocated from the allocator, so we use the 96 * bss section to bootstrap us. 97 */ 98static struct uma_keg masterkeg; 99static struct uma_zone masterzone_k; 100static struct uma_zone masterzone_z; 101static uma_zone_t kegs = &masterzone_k; 102static uma_zone_t zones = &masterzone_z; 103 104/* This is the zone from which all of uma_slab_t's are allocated. */ 105static uma_zone_t slabzone; 106static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */ 107 108/* 109 * The initial hash tables come out of this zone so they can be allocated 110 * prior to malloc coming up. 111 */ 112static uma_zone_t hashzone; 113 114/* The boot-time adjusted value for cache line alignment. */
| 52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 60#include "opt_ddb.h" 61#include "opt_param.h" 62 63#include <sys/param.h> 64#include <sys/systm.h> 65#include <sys/kernel.h> 66#include <sys/types.h> 67#include <sys/queue.h> 68#include <sys/malloc.h> 69#include <sys/ktr.h> 70#include <sys/lock.h> 71#include <sys/sysctl.h> 72#include <sys/mutex.h> 73#include <sys/proc.h> 74#include <sys/sbuf.h> 75#include <sys/smp.h> 76#include <sys/vmmeter.h> 77 78#include <vm/vm.h> 79#include <vm/vm_object.h> 80#include <vm/vm_page.h> 81#include <vm/vm_param.h> 82#include <vm/vm_map.h> 83#include <vm/vm_kern.h> 84#include <vm/vm_extern.h> 85#include <vm/uma.h> 86#include <vm/uma_int.h> 87#include <vm/uma_dbg.h> 88 89#include <machine/vmparam.h> 90 91#include <ddb/ddb.h> 92 93/* 94 * This is the zone and keg from which all zones are spawned. The idea is that 95 * even the zone & keg heads are allocated from the allocator, so we use the 96 * bss section to bootstrap us. 97 */ 98static struct uma_keg masterkeg; 99static struct uma_zone masterzone_k; 100static struct uma_zone masterzone_z; 101static uma_zone_t kegs = &masterzone_k; 102static uma_zone_t zones = &masterzone_z; 103 104/* This is the zone from which all of uma_slab_t's are allocated. */ 105static uma_zone_t slabzone; 106static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */ 107 108/* 109 * The initial hash tables come out of this zone so they can be allocated 110 * prior to malloc coming up. 111 */ 112static uma_zone_t hashzone; 113 114/* The boot-time adjusted value for cache line alignment. */
|
115static int uma_align_cache = 16 - 1;
| 115static int uma_align_cache = 64 - 1;
|
116 117static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 118 119/* 120 * Are we allowed to allocate buckets? 121 */ 122static int bucketdisable = 1; 123 124/* Linked list of all kegs in the system */ 125static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(&uma_kegs); 126 127/* This mutex protects the keg list */ 128static struct mtx uma_mtx; 129 130/* Linked list of boot time pages */ 131static LIST_HEAD(,uma_slab) uma_boot_pages = 132 LIST_HEAD_INITIALIZER(&uma_boot_pages); 133 134/* This mutex protects the boot time pages list */ 135static struct mtx uma_boot_pages_mtx; 136 137/* Is the VM done starting up? */ 138static int booted = 0; 139 140/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */ 141static u_int uma_max_ipers; 142static u_int uma_max_ipers_ref; 143 144/* 145 * This is the handle used to schedule events that need to happen 146 * outside of the allocation fast path. 147 */ 148static struct callout uma_callout; 149#define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 150 151/* 152 * This structure is passed as the zone ctor arg so that I don't have to create 153 * a special allocation function just for zones. 154 */ 155struct uma_zctor_args { 156 char *name; 157 size_t size; 158 uma_ctor ctor; 159 uma_dtor dtor; 160 uma_init uminit; 161 uma_fini fini; 162 uma_keg_t keg; 163 int align; 164 u_int32_t flags; 165}; 166 167struct uma_kctor_args { 168 uma_zone_t zone; 169 size_t size; 170 uma_init uminit; 171 uma_fini fini; 172 int align; 173 u_int32_t flags; 174}; 175 176struct uma_bucket_zone { 177 uma_zone_t ubz_zone; 178 char *ubz_name; 179 int ubz_entries; 180}; 181 182#define BUCKET_MAX 128 183 184struct uma_bucket_zone bucket_zones[] = { 185 { NULL, "16 Bucket", 16 }, 186 { NULL, "32 Bucket", 32 }, 187 { NULL, "64 Bucket", 64 }, 188 { NULL, "128 Bucket", 128 }, 189 { NULL, NULL, 0} 190}; 191 192#define BUCKET_SHIFT 4 193#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) 194 195/* 196 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket 197 * of approximately the right size. 198 */ 199static uint8_t bucket_size[BUCKET_ZONES]; 200 201/* 202 * Flags and enumerations to be passed to internal functions. 203 */ 204enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; 205 206#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */ 207#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */ 208 209/* Prototypes.. */ 210 211static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 212static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 213static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); 214static void page_free(void *, int, u_int8_t);
| 116 117static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 118 119/* 120 * Are we allowed to allocate buckets? 121 */ 122static int bucketdisable = 1; 123 124/* Linked list of all kegs in the system */ 125static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(&uma_kegs); 126 127/* This mutex protects the keg list */ 128static struct mtx uma_mtx; 129 130/* Linked list of boot time pages */ 131static LIST_HEAD(,uma_slab) uma_boot_pages = 132 LIST_HEAD_INITIALIZER(&uma_boot_pages); 133 134/* This mutex protects the boot time pages list */ 135static struct mtx uma_boot_pages_mtx; 136 137/* Is the VM done starting up? */ 138static int booted = 0; 139 140/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */ 141static u_int uma_max_ipers; 142static u_int uma_max_ipers_ref; 143 144/* 145 * This is the handle used to schedule events that need to happen 146 * outside of the allocation fast path. 147 */ 148static struct callout uma_callout; 149#define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 150 151/* 152 * This structure is passed as the zone ctor arg so that I don't have to create 153 * a special allocation function just for zones. 154 */ 155struct uma_zctor_args { 156 char *name; 157 size_t size; 158 uma_ctor ctor; 159 uma_dtor dtor; 160 uma_init uminit; 161 uma_fini fini; 162 uma_keg_t keg; 163 int align; 164 u_int32_t flags; 165}; 166 167struct uma_kctor_args { 168 uma_zone_t zone; 169 size_t size; 170 uma_init uminit; 171 uma_fini fini; 172 int align; 173 u_int32_t flags; 174}; 175 176struct uma_bucket_zone { 177 uma_zone_t ubz_zone; 178 char *ubz_name; 179 int ubz_entries; 180}; 181 182#define BUCKET_MAX 128 183 184struct uma_bucket_zone bucket_zones[] = { 185 { NULL, "16 Bucket", 16 }, 186 { NULL, "32 Bucket", 32 }, 187 { NULL, "64 Bucket", 64 }, 188 { NULL, "128 Bucket", 128 }, 189 { NULL, NULL, 0} 190}; 191 192#define BUCKET_SHIFT 4 193#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) 194 195/* 196 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket 197 * of approximately the right size. 198 */ 199static uint8_t bucket_size[BUCKET_ZONES]; 200 201/* 202 * Flags and enumerations to be passed to internal functions. 203 */ 204enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; 205 206#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */ 207#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */ 208 209/* Prototypes.. */ 210 211static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 212static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 213static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); 214static void page_free(void *, int, u_int8_t);
|
215static uma_slab_t slab_zalloc(uma_zone_t, int);
| 215static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
|
216static void cache_drain(uma_zone_t); 217static void bucket_drain(uma_zone_t, uma_bucket_t); 218static void bucket_cache_drain(uma_zone_t zone); 219static int keg_ctor(void *, int, void *, int); 220static void keg_dtor(void *, int, void *); 221static int zone_ctor(void *, int, void *, int); 222static void zone_dtor(void *, int, void *); 223static int zero_init(void *, int, int);
| 216static void cache_drain(uma_zone_t); 217static void bucket_drain(uma_zone_t, uma_bucket_t); 218static void bucket_cache_drain(uma_zone_t zone); 219static int keg_ctor(void *, int, void *, int); 220static void keg_dtor(void *, int, void *); 221static int zone_ctor(void *, int, void *, int); 222static void zone_dtor(void *, int, void *); 223static int zero_init(void *, int, int);
|
224static void zone_small_init(uma_zone_t zone); 225static void zone_large_init(uma_zone_t zone);
| 224static void keg_small_init(uma_keg_t keg); 225static void keg_large_init(uma_keg_t keg);
|
226static void zone_foreach(void (*zfunc)(uma_zone_t)); 227static void zone_timeout(uma_zone_t zone); 228static int hash_alloc(struct uma_hash *); 229static int hash_expand(struct uma_hash *, struct uma_hash *); 230static void hash_free(struct uma_hash *hash); 231static void uma_timeout(void *); 232static void uma_startup3(void);
| 226static void zone_foreach(void (*zfunc)(uma_zone_t)); 227static void zone_timeout(uma_zone_t zone); 228static int hash_alloc(struct uma_hash *); 229static int hash_expand(struct uma_hash *, struct uma_hash *); 230static void hash_free(struct uma_hash *hash); 231static void uma_timeout(void *); 232static void uma_startup3(void);
|
233static void *uma_zalloc_internal(uma_zone_t, void *, int); 234static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip,
| 233static void *zone_alloc_item(uma_zone_t, void *, int); 234static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip,
|
235 int); 236static void bucket_enable(void); 237static void bucket_init(void); 238static uma_bucket_t bucket_alloc(int, int); 239static void bucket_free(uma_bucket_t); 240static void bucket_zone_drain(void);
| 235 int); 236static void bucket_enable(void); 237static void bucket_init(void); 238static uma_bucket_t bucket_alloc(int, int); 239static void bucket_free(uma_bucket_t); 240static void bucket_zone_drain(void);
|
241static int uma_zalloc_bucket(uma_zone_t zone, int flags); 242static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); 243static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); 244static uma_zone_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
| 241static int zone_alloc_bucket(uma_zone_t zone, int flags); 242static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags); 243static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags); 244static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab); 245static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
|
245 uma_fini fini, int align, u_int32_t flags);
| 246 uma_fini fini, int align, u_int32_t flags);
|
| 247static inline void zone_relock(uma_zone_t zone, uma_keg_t keg); 248static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
|
246 247void uma_print_zone(uma_zone_t); 248void uma_print_stats(void); 249static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 250static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 251 252SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 253 254SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT, 255 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 256 257SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT, 258 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 259 260/* 261 * This routine checks to see whether or not it's safe to enable buckets. 262 */ 263 264static void 265bucket_enable(void) 266{ 267 if (cnt.v_free_count < cnt.v_free_min) 268 bucketdisable = 1; 269 else 270 bucketdisable = 0; 271} 272 273/* 274 * Initialize bucket_zones, the array of zones of buckets of various sizes. 275 * 276 * For each zone, calculate the memory required for each bucket, consisting 277 * of the header and an array of pointers. Initialize bucket_size[] to point 278 * the range of appropriate bucket sizes at the zone. 279 */ 280static void 281bucket_init(void) 282{ 283 struct uma_bucket_zone *ubz; 284 int i; 285 int j; 286 287 for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) { 288 int size; 289 290 ubz = &bucket_zones[j]; 291 size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 292 size += sizeof(void *) * ubz->ubz_entries; 293 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
| 249 250void uma_print_zone(uma_zone_t); 251void uma_print_stats(void); 252static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 253static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 254 255SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 256 257SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLTYPE_INT, 258 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 259 260SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLTYPE_STRUCT, 261 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 262 263/* 264 * This routine checks to see whether or not it's safe to enable buckets. 265 */ 266 267static void 268bucket_enable(void) 269{ 270 if (cnt.v_free_count < cnt.v_free_min) 271 bucketdisable = 1; 272 else 273 bucketdisable = 0; 274} 275 276/* 277 * Initialize bucket_zones, the array of zones of buckets of various sizes. 278 * 279 * For each zone, calculate the memory required for each bucket, consisting 280 * of the header and an array of pointers. Initialize bucket_size[] to point 281 * the range of appropriate bucket sizes at the zone. 282 */ 283static void 284bucket_init(void) 285{ 286 struct uma_bucket_zone *ubz; 287 int i; 288 int j; 289 290 for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) { 291 int size; 292 293 ubz = &bucket_zones[j]; 294 size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 295 size += sizeof(void *) * ubz->ubz_entries; 296 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size,
|
294 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL);
| 297 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 298 UMA_ZFLAG_INTERNAL | UMA_ZFLAG_BUCKET);
|
295 for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT)) 296 bucket_size[i >> BUCKET_SHIFT] = j; 297 } 298} 299 300/* 301 * Given a desired number of entries for a bucket, return the zone from which 302 * to allocate the bucket. 303 */ 304static struct uma_bucket_zone * 305bucket_zone_lookup(int entries) 306{ 307 int idx; 308 309 idx = howmany(entries, 1 << BUCKET_SHIFT); 310 return (&bucket_zones[bucket_size[idx]]); 311} 312 313static uma_bucket_t 314bucket_alloc(int entries, int bflags) 315{ 316 struct uma_bucket_zone *ubz; 317 uma_bucket_t bucket; 318 319 /* 320 * This is to stop us from allocating per cpu buckets while we're 321 * running out of vm.boot_pages. Otherwise, we would exhaust the 322 * boot pages. This also prevents us from allocating buckets in 323 * low memory situations. 324 */ 325 if (bucketdisable) 326 return (NULL); 327 328 ubz = bucket_zone_lookup(entries);
| 299 for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT)) 300 bucket_size[i >> BUCKET_SHIFT] = j; 301 } 302} 303 304/* 305 * Given a desired number of entries for a bucket, return the zone from which 306 * to allocate the bucket. 307 */ 308static struct uma_bucket_zone * 309bucket_zone_lookup(int entries) 310{ 311 int idx; 312 313 idx = howmany(entries, 1 << BUCKET_SHIFT); 314 return (&bucket_zones[bucket_size[idx]]); 315} 316 317static uma_bucket_t 318bucket_alloc(int entries, int bflags) 319{ 320 struct uma_bucket_zone *ubz; 321 uma_bucket_t bucket; 322 323 /* 324 * This is to stop us from allocating per cpu buckets while we're 325 * running out of vm.boot_pages. Otherwise, we would exhaust the 326 * boot pages. This also prevents us from allocating buckets in 327 * low memory situations. 328 */ 329 if (bucketdisable) 330 return (NULL); 331 332 ubz = bucket_zone_lookup(entries);
|
329 bucket = uma_zalloc_internal(ubz->ubz_zone, NULL, bflags);
| 333 bucket = zone_alloc_item(ubz->ubz_zone, NULL, bflags);
|
330 if (bucket) { 331#ifdef INVARIANTS 332 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 333#endif 334 bucket->ub_cnt = 0; 335 bucket->ub_entries = ubz->ubz_entries; 336 } 337 338 return (bucket); 339} 340 341static void 342bucket_free(uma_bucket_t bucket) 343{ 344 struct uma_bucket_zone *ubz; 345 346 ubz = bucket_zone_lookup(bucket->ub_entries);
| 334 if (bucket) { 335#ifdef INVARIANTS 336 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 337#endif 338 bucket->ub_cnt = 0; 339 bucket->ub_entries = ubz->ubz_entries; 340 } 341 342 return (bucket); 343} 344 345static void 346bucket_free(uma_bucket_t bucket) 347{ 348 struct uma_bucket_zone *ubz; 349 350 ubz = bucket_zone_lookup(bucket->ub_entries);
|
347 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
| 351 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
|
348 ZFREE_STATFREE); 349} 350 351static void 352bucket_zone_drain(void) 353{ 354 struct uma_bucket_zone *ubz; 355 356 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 357 zone_drain(ubz->ubz_zone); 358} 359
| 352 ZFREE_STATFREE); 353} 354 355static void 356bucket_zone_drain(void) 357{ 358 struct uma_bucket_zone *ubz; 359 360 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 361 zone_drain(ubz->ubz_zone); 362} 363
|
| 364static inline uma_keg_t 365zone_first_keg(uma_zone_t zone) 366{
|
360
| 367
|
| 368 return (LIST_FIRST(&zone->uz_kegs)->kl_keg); 369} 370 371static void 372zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t)) 373{ 374 uma_klink_t klink; 375 376 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) 377 kegfn(klink->kl_keg); 378} 379
|
361/* 362 * Routine called by timeout which is used to fire off some time interval 363 * based calculations. (stats, hash size, etc.) 364 * 365 * Arguments: 366 * arg Unused 367 * 368 * Returns: 369 * Nothing 370 */ 371static void 372uma_timeout(void *unused) 373{ 374 bucket_enable(); 375 zone_foreach(zone_timeout); 376 377 /* Reschedule this event */ 378 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 379} 380 381/* 382 * Routine to perform timeout driven calculations. This expands the 383 * hashes and does per cpu statistics aggregation. 384 *
| 380/* 381 * Routine called by timeout which is used to fire off some time interval 382 * based calculations. (stats, hash size, etc.) 383 * 384 * Arguments: 385 * arg Unused 386 * 387 * Returns: 388 * Nothing 389 */ 390static void 391uma_timeout(void *unused) 392{ 393 bucket_enable(); 394 zone_foreach(zone_timeout); 395 396 /* Reschedule this event */ 397 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 398} 399 400/* 401 * Routine to perform timeout driven calculations. This expands the 402 * hashes and does per cpu statistics aggregation. 403 *
|
385 * Arguments: 386 * zone The zone to operate on 387 * 388 * Returns: 389 * Nothing
| 404 * Returns nothing.
|
390 */ 391static void
| 405 */ 406static void
|
392zone_timeout(uma_zone_t zone)
| 407keg_timeout(uma_keg_t keg)
|
393{
| 408{
|
394 uma_keg_t keg; 395 u_int64_t alloc;
| |
396
| 409
|
397 keg = zone->uz_keg; 398 alloc = 0; 399
| 410 KEG_LOCK(keg);
|
400 /*
| 411 /*
|
401 * Expand the zone hash table.
| 412 * Expand the keg hash table.
|
402 * 403 * This is done if the number of slabs is larger than the hash size. 404 * What I'm trying to do here is completely reduce collisions. This 405 * may be a little aggressive. Should I allow for two collisions max? 406 */
| 413 * 414 * This is done if the number of slabs is larger than the hash size. 415 * What I'm trying to do here is completely reduce collisions. This 416 * may be a little aggressive. Should I allow for two collisions max? 417 */
|
407 ZONE_LOCK(zone);
| |
408 if (keg->uk_flags & UMA_ZONE_HASH && 409 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { 410 struct uma_hash newhash; 411 struct uma_hash oldhash; 412 int ret; 413 414 /* 415 * This is so involved because allocating and freeing
| 418 if (keg->uk_flags & UMA_ZONE_HASH && 419 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { 420 struct uma_hash newhash; 421 struct uma_hash oldhash; 422 int ret; 423 424 /* 425 * This is so involved because allocating and freeing
|
416 * while the zone lock is held will lead to deadlock.
| 426 * while the keg lock is held will lead to deadlock.
|
417 * I have to do everything in stages and check for 418 * races. 419 */ 420 newhash = keg->uk_hash;
| 427 * I have to do everything in stages and check for 428 * races. 429 */ 430 newhash = keg->uk_hash;
|
421 ZONE_UNLOCK(zone);
| 431 KEG_UNLOCK(keg);
|
422 ret = hash_alloc(&newhash);
| 432 ret = hash_alloc(&newhash);
|
423 ZONE_LOCK(zone);
| 433 KEG_LOCK(keg);
|
424 if (ret) { 425 if (hash_expand(&keg->uk_hash, &newhash)) { 426 oldhash = keg->uk_hash; 427 keg->uk_hash = newhash; 428 } else 429 oldhash = newhash; 430
| 434 if (ret) { 435 if (hash_expand(&keg->uk_hash, &newhash)) { 436 oldhash = keg->uk_hash; 437 keg->uk_hash = newhash; 438 } else 439 oldhash = newhash; 440
|
431 ZONE_UNLOCK(zone);
| 441 KEG_UNLOCK(keg);
|
432 hash_free(&oldhash);
| 442 hash_free(&oldhash);
|
433 ZONE_LOCK(zone);
| 443 KEG_LOCK(keg);
|
434 } 435 }
| 444 } 445 }
|
436 ZONE_UNLOCK(zone);
| 446 KEG_UNLOCK(keg);
|
437} 438
| 447} 448
|
| 449static void 450zone_timeout(uma_zone_t zone) 451{ 452 453 zone_foreach_keg(zone, &keg_timeout); 454} 455
|
439/* 440 * Allocate and zero fill the next sized hash table from the appropriate 441 * backing store. 442 * 443 * Arguments: 444 * hash A new hash structure with the old hash size in uh_hashsize 445 * 446 * Returns: 447 * 1 on sucess and 0 on failure. 448 */ 449static int 450hash_alloc(struct uma_hash *hash) 451{ 452 int oldsize; 453 int alloc; 454 455 oldsize = hash->uh_hashsize; 456 457 /* We're just going to go to a power of two greater */ 458 if (oldsize) { 459 hash->uh_hashsize = oldsize * 2; 460 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 461 hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 462 M_UMAHASH, M_NOWAIT); 463 } else { 464 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
| 456/* 457 * Allocate and zero fill the next sized hash table from the appropriate 458 * backing store. 459 * 460 * Arguments: 461 * hash A new hash structure with the old hash size in uh_hashsize 462 * 463 * Returns: 464 * 1 on sucess and 0 on failure. 465 */ 466static int 467hash_alloc(struct uma_hash *hash) 468{ 469 int oldsize; 470 int alloc; 471 472 oldsize = hash->uh_hashsize; 473 474 /* We're just going to go to a power of two greater */ 475 if (oldsize) { 476 hash->uh_hashsize = oldsize * 2; 477 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 478 hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 479 M_UMAHASH, M_NOWAIT); 480 } else { 481 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
|
465 hash->uh_slab_hash = uma_zalloc_internal(hashzone, NULL,
| 482 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL,
|
466 M_WAITOK); 467 hash->uh_hashsize = UMA_HASH_SIZE_INIT; 468 } 469 if (hash->uh_slab_hash) { 470 bzero(hash->uh_slab_hash, alloc); 471 hash->uh_hashmask = hash->uh_hashsize - 1; 472 return (1); 473 } 474 475 return (0); 476} 477 478/* 479 * Expands the hash table for HASH zones. This is done from zone_timeout 480 * to reduce collisions. This must not be done in the regular allocation 481 * path, otherwise, we can recurse on the vm while allocating pages. 482 * 483 * Arguments: 484 * oldhash The hash you want to expand 485 * newhash The hash structure for the new table 486 * 487 * Returns: 488 * Nothing 489 * 490 * Discussion: 491 */ 492static int 493hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 494{ 495 uma_slab_t slab; 496 int hval; 497 int i; 498 499 if (!newhash->uh_slab_hash) 500 return (0); 501 502 if (oldhash->uh_hashsize >= newhash->uh_hashsize) 503 return (0); 504 505 /* 506 * I need to investigate hash algorithms for resizing without a 507 * full rehash. 508 */ 509 510 for (i = 0; i < oldhash->uh_hashsize; i++) 511 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 512 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 513 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 514 hval = UMA_HASH(newhash, slab->us_data); 515 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 516 slab, us_hlink); 517 } 518 519 return (1); 520} 521 522/* 523 * Free the hash bucket to the appropriate backing store. 524 * 525 * Arguments: 526 * slab_hash The hash bucket we're freeing 527 * hashsize The number of entries in that hash bucket 528 * 529 * Returns: 530 * Nothing 531 */ 532static void 533hash_free(struct uma_hash *hash) 534{ 535 if (hash->uh_slab_hash == NULL) 536 return; 537 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
| 483 M_WAITOK); 484 hash->uh_hashsize = UMA_HASH_SIZE_INIT; 485 } 486 if (hash->uh_slab_hash) { 487 bzero(hash->uh_slab_hash, alloc); 488 hash->uh_hashmask = hash->uh_hashsize - 1; 489 return (1); 490 } 491 492 return (0); 493} 494 495/* 496 * Expands the hash table for HASH zones. This is done from zone_timeout 497 * to reduce collisions. This must not be done in the regular allocation 498 * path, otherwise, we can recurse on the vm while allocating pages. 499 * 500 * Arguments: 501 * oldhash The hash you want to expand 502 * newhash The hash structure for the new table 503 * 504 * Returns: 505 * Nothing 506 * 507 * Discussion: 508 */ 509static int 510hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 511{ 512 uma_slab_t slab; 513 int hval; 514 int i; 515 516 if (!newhash->uh_slab_hash) 517 return (0); 518 519 if (oldhash->uh_hashsize >= newhash->uh_hashsize) 520 return (0); 521 522 /* 523 * I need to investigate hash algorithms for resizing without a 524 * full rehash. 525 */ 526 527 for (i = 0; i < oldhash->uh_hashsize; i++) 528 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 529 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 530 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 531 hval = UMA_HASH(newhash, slab->us_data); 532 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 533 slab, us_hlink); 534 } 535 536 return (1); 537} 538 539/* 540 * Free the hash bucket to the appropriate backing store. 541 * 542 * Arguments: 543 * slab_hash The hash bucket we're freeing 544 * hashsize The number of entries in that hash bucket 545 * 546 * Returns: 547 * Nothing 548 */ 549static void 550hash_free(struct uma_hash *hash) 551{ 552 if (hash->uh_slab_hash == NULL) 553 return; 554 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
|
538 uma_zfree_internal(hashzone,
| 555 zone_free_item(hashzone,
|
539 hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE); 540 else 541 free(hash->uh_slab_hash, M_UMAHASH); 542} 543 544/* 545 * Frees all outstanding items in a bucket 546 * 547 * Arguments: 548 * zone The zone to free to, must be unlocked. 549 * bucket The free/alloc bucket with items, cpu queue must be locked. 550 * 551 * Returns: 552 * Nothing 553 */ 554 555static void 556bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 557{
| 556 hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE); 557 else 558 free(hash->uh_slab_hash, M_UMAHASH); 559} 560 561/* 562 * Frees all outstanding items in a bucket 563 * 564 * Arguments: 565 * zone The zone to free to, must be unlocked. 566 * bucket The free/alloc bucket with items, cpu queue must be locked. 567 * 568 * Returns: 569 * Nothing 570 */ 571 572static void 573bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 574{
|
558 uma_slab_t slab; 559 int mzone;
| |
560 void *item; 561 562 if (bucket == NULL) 563 return; 564
| 575 void *item; 576 577 if (bucket == NULL) 578 return; 579
|
565 slab = NULL; 566 mzone = 0; 567 568 /* We have to lookup the slab again for malloc.. */ 569 if (zone->uz_keg->uk_flags & UMA_ZONE_MALLOC) 570 mzone = 1; 571
| |
572 while (bucket->ub_cnt > 0) { 573 bucket->ub_cnt--; 574 item = bucket->ub_bucket[bucket->ub_cnt]; 575#ifdef INVARIANTS 576 bucket->ub_bucket[bucket->ub_cnt] = NULL; 577 KASSERT(item != NULL, 578 ("bucket_drain: botched ptr, item is NULL")); 579#endif
| 580 while (bucket->ub_cnt > 0) { 581 bucket->ub_cnt--; 582 item = bucket->ub_bucket[bucket->ub_cnt]; 583#ifdef INVARIANTS 584 bucket->ub_bucket[bucket->ub_cnt] = NULL; 585 KASSERT(item != NULL, 586 ("bucket_drain: botched ptr, item is NULL")); 587#endif
|
580 /* 581 * This is extremely inefficient. The slab pointer was passed 582 * to uma_zfree_arg, but we lost it because the buckets don't 583 * hold them. This will go away when free() gets a size passed 584 * to it. 585 */ 586 if (mzone) 587 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); 588 uma_zfree_internal(zone, item, slab, SKIP_DTOR, 0);
| 588 zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
|
589 } 590} 591 592/* 593 * Drains the per cpu caches for a zone. 594 * 595 * NOTE: This may only be called while the zone is being turn down, and not 596 * during normal operation. This is necessary in order that we do not have 597 * to migrate CPUs to drain the per-CPU caches. 598 * 599 * Arguments: 600 * zone The zone to drain, must be unlocked. 601 * 602 * Returns: 603 * Nothing 604 */ 605static void 606cache_drain(uma_zone_t zone) 607{ 608 uma_cache_t cache; 609 int cpu; 610 611 /* 612 * XXX: It is safe to not lock the per-CPU caches, because we're 613 * tearing down the zone anyway. I.e., there will be no further use 614 * of the caches at this point. 615 * 616 * XXX: It would good to be able to assert that the zone is being 617 * torn down to prevent improper use of cache_drain(). 618 * 619 * XXX: We lock the zone before passing into bucket_cache_drain() as 620 * it is used elsewhere. Should the tear-down path be made special 621 * there in some form? 622 */ 623 for (cpu = 0; cpu <= mp_maxid; cpu++) { 624 if (CPU_ABSENT(cpu)) 625 continue; 626 cache = &zone->uz_cpu[cpu]; 627 bucket_drain(zone, cache->uc_allocbucket); 628 bucket_drain(zone, cache->uc_freebucket); 629 if (cache->uc_allocbucket != NULL) 630 bucket_free(cache->uc_allocbucket); 631 if (cache->uc_freebucket != NULL) 632 bucket_free(cache->uc_freebucket); 633 cache->uc_allocbucket = cache->uc_freebucket = NULL; 634 } 635 ZONE_LOCK(zone); 636 bucket_cache_drain(zone); 637 ZONE_UNLOCK(zone); 638} 639 640/* 641 * Drain the cached buckets from a zone. Expects a locked zone on entry. 642 */ 643static void 644bucket_cache_drain(uma_zone_t zone) 645{ 646 uma_bucket_t bucket; 647 648 /* 649 * Drain the bucket queues and free the buckets, we just keep two per 650 * cpu (alloc/free). 651 */ 652 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 653 LIST_REMOVE(bucket, ub_link); 654 ZONE_UNLOCK(zone); 655 bucket_drain(zone, bucket); 656 bucket_free(bucket); 657 ZONE_LOCK(zone); 658 } 659 660 /* Now we do the free queue.. */ 661 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 662 LIST_REMOVE(bucket, ub_link); 663 bucket_free(bucket); 664 } 665} 666 667/*
| 589 } 590} 591 592/* 593 * Drains the per cpu caches for a zone. 594 * 595 * NOTE: This may only be called while the zone is being turn down, and not 596 * during normal operation. This is necessary in order that we do not have 597 * to migrate CPUs to drain the per-CPU caches. 598 * 599 * Arguments: 600 * zone The zone to drain, must be unlocked. 601 * 602 * Returns: 603 * Nothing 604 */ 605static void 606cache_drain(uma_zone_t zone) 607{ 608 uma_cache_t cache; 609 int cpu; 610 611 /* 612 * XXX: It is safe to not lock the per-CPU caches, because we're 613 * tearing down the zone anyway. I.e., there will be no further use 614 * of the caches at this point. 615 * 616 * XXX: It would good to be able to assert that the zone is being 617 * torn down to prevent improper use of cache_drain(). 618 * 619 * XXX: We lock the zone before passing into bucket_cache_drain() as 620 * it is used elsewhere. Should the tear-down path be made special 621 * there in some form? 622 */ 623 for (cpu = 0; cpu <= mp_maxid; cpu++) { 624 if (CPU_ABSENT(cpu)) 625 continue; 626 cache = &zone->uz_cpu[cpu]; 627 bucket_drain(zone, cache->uc_allocbucket); 628 bucket_drain(zone, cache->uc_freebucket); 629 if (cache->uc_allocbucket != NULL) 630 bucket_free(cache->uc_allocbucket); 631 if (cache->uc_freebucket != NULL) 632 bucket_free(cache->uc_freebucket); 633 cache->uc_allocbucket = cache->uc_freebucket = NULL; 634 } 635 ZONE_LOCK(zone); 636 bucket_cache_drain(zone); 637 ZONE_UNLOCK(zone); 638} 639 640/* 641 * Drain the cached buckets from a zone. Expects a locked zone on entry. 642 */ 643static void 644bucket_cache_drain(uma_zone_t zone) 645{ 646 uma_bucket_t bucket; 647 648 /* 649 * Drain the bucket queues and free the buckets, we just keep two per 650 * cpu (alloc/free). 651 */ 652 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 653 LIST_REMOVE(bucket, ub_link); 654 ZONE_UNLOCK(zone); 655 bucket_drain(zone, bucket); 656 bucket_free(bucket); 657 ZONE_LOCK(zone); 658 } 659 660 /* Now we do the free queue.. */ 661 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 662 LIST_REMOVE(bucket, ub_link); 663 bucket_free(bucket); 664 } 665} 666 667/*
|
668 * Frees pages from a zone back to the system. This is done on demand from
| 668 * Frees pages from a keg back to the system. This is done on demand from
|
669 * the pageout daemon. 670 *
| 669 * the pageout daemon. 670 *
|
671 * Arguments: 672 * zone The zone to free pages from 673 * all Should we drain all items? 674 * 675 * Returns: 676 * Nothing.
| 671 * Returns nothing.
|
677 */
| 672 */
|
678void 679zone_drain(uma_zone_t zone)
| 673static void 674keg_drain(uma_keg_t keg)
|
680{ 681 struct slabhead freeslabs = { 0 };
| 675{ 676 struct slabhead freeslabs = { 0 };
|
682 uma_keg_t keg;
| |
683 uma_slab_t slab; 684 uma_slab_t n; 685 u_int8_t flags; 686 u_int8_t *mem; 687 int i; 688
| 677 uma_slab_t slab; 678 uma_slab_t n; 679 u_int8_t flags; 680 u_int8_t *mem; 681 int i; 682
|
689 keg = zone->uz_keg; 690
| |
691 /*
| 683 /*
|
692 * We don't want to take pages from statically allocated zones at this
| 684 * We don't want to take pages from statically allocated kegs at this
|
693 * time 694 */ 695 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 696 return; 697
| 685 * time 686 */ 687 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 688 return; 689
|
698 ZONE_LOCK(zone); 699
| |
700#ifdef UMA_DEBUG
| 690#ifdef UMA_DEBUG
|
701 printf("%s free items: %u\n", zone->uz_name, keg->uk_free);
| 691 printf("%s free items: %u\n", keg->uk_name, keg->uk_free);
|
702#endif
| 692#endif
|
703 bucket_cache_drain(zone);
| 693 KEG_LOCK(keg);
|
704 if (keg->uk_free == 0) 705 goto finished; 706 707 slab = LIST_FIRST(&keg->uk_free_slab); 708 while (slab) { 709 n = LIST_NEXT(slab, us_link); 710 711 /* We have no where to free these to */ 712 if (slab->us_flags & UMA_SLAB_BOOT) { 713 slab = n; 714 continue; 715 } 716 717 LIST_REMOVE(slab, us_link); 718 keg->uk_pages -= keg->uk_ppera; 719 keg->uk_free -= keg->uk_ipers; 720 721 if (keg->uk_flags & UMA_ZONE_HASH) 722 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data); 723 724 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 725 726 slab = n; 727 } 728finished:
| 694 if (keg->uk_free == 0) 695 goto finished; 696 697 slab = LIST_FIRST(&keg->uk_free_slab); 698 while (slab) { 699 n = LIST_NEXT(slab, us_link); 700 701 /* We have no where to free these to */ 702 if (slab->us_flags & UMA_SLAB_BOOT) { 703 slab = n; 704 continue; 705 } 706 707 LIST_REMOVE(slab, us_link); 708 keg->uk_pages -= keg->uk_ppera; 709 keg->uk_free -= keg->uk_ipers; 710 711 if (keg->uk_flags & UMA_ZONE_HASH) 712 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data); 713 714 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 715 716 slab = n; 717 } 718finished:
|
729 ZONE_UNLOCK(zone);
| 719 KEG_UNLOCK(keg);
|
730 731 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 732 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 733 if (keg->uk_fini) 734 for (i = 0; i < keg->uk_ipers; i++) 735 keg->uk_fini( 736 slab->us_data + (keg->uk_rsize * i), 737 keg->uk_size); 738 flags = slab->us_flags; 739 mem = slab->us_data; 740
| 720 721 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 722 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 723 if (keg->uk_fini) 724 for (i = 0; i < keg->uk_ipers; i++) 725 keg->uk_fini( 726 slab->us_data + (keg->uk_rsize * i), 727 keg->uk_size); 728 flags = slab->us_flags; 729 mem = slab->us_data; 730
|
741 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 742 (keg->uk_flags & UMA_ZONE_REFCNT)) {
| 731 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
|
743 vm_object_t obj; 744 745 if (flags & UMA_SLAB_KMEM) 746 obj = kmem_object; 747 else if (flags & UMA_SLAB_KERNEL) 748 obj = kernel_object; 749 else 750 obj = NULL; 751 for (i = 0; i < keg->uk_ppera; i++) 752 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 753 obj); 754 } 755 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
| 732 vm_object_t obj; 733 734 if (flags & UMA_SLAB_KMEM) 735 obj = kmem_object; 736 else if (flags & UMA_SLAB_KERNEL) 737 obj = kernel_object; 738 else 739 obj = NULL; 740 for (i = 0; i < keg->uk_ppera; i++) 741 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 742 obj); 743 } 744 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
|
756 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
| 745 zone_free_item(keg->uk_slabzone, slab, NULL,
|
757 SKIP_NONE, ZFREE_STATFREE); 758#ifdef UMA_DEBUG 759 printf("%s: Returning %d bytes.\n",
| 746 SKIP_NONE, ZFREE_STATFREE); 747#ifdef UMA_DEBUG 748 printf("%s: Returning %d bytes.\n",
|
760 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
| 749 keg->uk_name, UMA_SLAB_SIZE * keg->uk_ppera);
|
761#endif 762 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); 763 } 764} 765
| 750#endif 751 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); 752 } 753} 754
|
| 755static void 756zone_drain_wait(uma_zone_t zone, int waitok) 757{ 758 759 /* 760 * Set draining to interlock with zone_dtor() so we can release our 761 * locks as we go. Only dtor() should do a WAITOK call since it 762 * is the only call that knows the structure will still be available 763 * when it wakes up. 764 */ 765 ZONE_LOCK(zone); 766 while (zone->uz_flags & UMA_ZFLAG_DRAINING) { 767 if (waitok == M_NOWAIT) 768 goto out; 769 mtx_unlock(&uma_mtx); 770 msleep(zone, zone->uz_lock, PVM, "zonedrain", 1); 771 mtx_lock(&uma_mtx); 772 } 773 zone->uz_flags |= UMA_ZFLAG_DRAINING; 774 bucket_cache_drain(zone); 775 ZONE_UNLOCK(zone); 776 /* 777 * The DRAINING flag protects us from being freed while 778 * we're running. Normally the uma_mtx would protect us but we 779 * must be able to release and acquire the right lock for each keg. 780 */ 781 zone_foreach_keg(zone, &keg_drain); 782 ZONE_LOCK(zone); 783 zone->uz_flags &= ~UMA_ZFLAG_DRAINING; 784 wakeup(zone); 785out: 786 ZONE_UNLOCK(zone); 787} 788 789void 790zone_drain(uma_zone_t zone) 791{ 792 793 zone_drain_wait(zone, M_NOWAIT); 794} 795
|
766/*
| 796/*
|
767 * Allocate a new slab for a zone. This does not insert the slab onto a list.
| 797 * Allocate a new slab for a keg. This does not insert the slab onto a list.
|
768 * 769 * Arguments:
| 798 * 799 * Arguments:
|
770 * zone The zone to allocate slabs for
| |
771 * wait Shall we wait? 772 * 773 * Returns: 774 * The slab that was allocated or NULL if there is no memory and the 775 * caller specified M_NOWAIT. 776 */ 777static uma_slab_t
| 800 * wait Shall we wait? 801 * 802 * Returns: 803 * The slab that was allocated or NULL if there is no memory and the 804 * caller specified M_NOWAIT. 805 */ 806static uma_slab_t
|
778slab_zalloc(uma_zone_t zone, int wait)
| 807keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait)
|
779{ 780 uma_slabrefcnt_t slabref;
| 808{ 809 uma_slabrefcnt_t slabref;
|
| 810 uma_alloc allocf;
|
781 uma_slab_t slab;
| 811 uma_slab_t slab;
|
782 uma_keg_t keg;
| |
783 u_int8_t *mem; 784 u_int8_t flags; 785 int i; 786
| 812 u_int8_t *mem; 813 u_int8_t flags; 814 int i; 815
|
| 816 mtx_assert(&keg->uk_lock, MA_OWNED);
|
787 slab = NULL;
| 817 slab = NULL;
|
788 keg = zone->uz_keg;
| |
789 790#ifdef UMA_DEBUG
| 818 819#ifdef UMA_DEBUG
|
791 printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name);
| 820 printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
|
792#endif
| 821#endif
|
793 ZONE_UNLOCK(zone);
| 822 allocf = keg->uk_allocf; 823 KEG_UNLOCK(keg);
|
794 795 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
| 824 825 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
|
796 slab = uma_zalloc_internal(keg->uk_slabzone, NULL, wait);
| 826 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
|
797 if (slab == NULL) {
| 827 if (slab == NULL) {
|
798 ZONE_LOCK(zone);
| 828 KEG_LOCK(keg);
|
799 return NULL; 800 } 801 } 802 803 /* 804 * This reproduces the old vm_zone behavior of zero filling pages the 805 * first time they are added to a zone. 806 * 807 * Malloced items are zeroed in uma_zalloc. 808 */ 809 810 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 811 wait |= M_ZERO; 812 else 813 wait &= ~M_ZERO; 814
| 829 return NULL; 830 } 831 } 832 833 /* 834 * This reproduces the old vm_zone behavior of zero filling pages the 835 * first time they are added to a zone. 836 * 837 * Malloced items are zeroed in uma_zalloc. 838 */ 839 840 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 841 wait |= M_ZERO; 842 else 843 wait &= ~M_ZERO; 844
|
815 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, 816 &flags, wait);
| 845 /* zone is passed for legacy reasons. */ 846 mem = allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, &flags, wait);
|
817 if (mem == NULL) { 818 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
| 847 if (mem == NULL) { 848 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
|
819 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
| 849 zone_free_item(keg->uk_slabzone, slab, NULL,
|
820 SKIP_NONE, ZFREE_STATFREE);
| 850 SKIP_NONE, ZFREE_STATFREE);
|
821 ZONE_LOCK(zone);
| 851 KEG_LOCK(keg);
|
822 return (NULL); 823 } 824 825 /* Point the slab into the allocated memory */ 826 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 827 slab = (uma_slab_t )(mem + keg->uk_pgoff); 828
| 852 return (NULL); 853 } 854 855 /* Point the slab into the allocated memory */ 856 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 857 slab = (uma_slab_t )(mem + keg->uk_pgoff); 858
|
829 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 830 (keg->uk_flags & UMA_ZONE_REFCNT))
| 859 if (keg->uk_flags & UMA_ZONE_VTOSLAB)
|
831 for (i = 0; i < keg->uk_ppera; i++) 832 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 833 834 slab->us_keg = keg; 835 slab->us_data = mem; 836 slab->us_freecount = keg->uk_ipers; 837 slab->us_firstfree = 0; 838 slab->us_flags = flags; 839 840 if (keg->uk_flags & UMA_ZONE_REFCNT) { 841 slabref = (uma_slabrefcnt_t)slab; 842 for (i = 0; i < keg->uk_ipers; i++) { 843 slabref->us_freelist[i].us_refcnt = 0; 844 slabref->us_freelist[i].us_item = i+1; 845 } 846 } else { 847 for (i = 0; i < keg->uk_ipers; i++) 848 slab->us_freelist[i].us_item = i+1; 849 } 850 851 if (keg->uk_init != NULL) { 852 for (i = 0; i < keg->uk_ipers; i++) 853 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 854 keg->uk_size, wait) != 0) 855 break; 856 if (i != keg->uk_ipers) { 857 if (keg->uk_fini != NULL) { 858 for (i--; i > -1; i--) 859 keg->uk_fini(slab->us_data + 860 (keg->uk_rsize * i), 861 keg->uk_size); 862 }
| 860 for (i = 0; i < keg->uk_ppera; i++) 861 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 862 863 slab->us_keg = keg; 864 slab->us_data = mem; 865 slab->us_freecount = keg->uk_ipers; 866 slab->us_firstfree = 0; 867 slab->us_flags = flags; 868 869 if (keg->uk_flags & UMA_ZONE_REFCNT) { 870 slabref = (uma_slabrefcnt_t)slab; 871 for (i = 0; i < keg->uk_ipers; i++) { 872 slabref->us_freelist[i].us_refcnt = 0; 873 slabref->us_freelist[i].us_item = i+1; 874 } 875 } else { 876 for (i = 0; i < keg->uk_ipers; i++) 877 slab->us_freelist[i].us_item = i+1; 878 } 879 880 if (keg->uk_init != NULL) { 881 for (i = 0; i < keg->uk_ipers; i++) 882 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 883 keg->uk_size, wait) != 0) 884 break; 885 if (i != keg->uk_ipers) { 886 if (keg->uk_fini != NULL) { 887 for (i--; i > -1; i--) 888 keg->uk_fini(slab->us_data + 889 (keg->uk_rsize * i), 890 keg->uk_size); 891 }
|
863 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 864 (keg->uk_flags & UMA_ZONE_REFCNT)) {
| 892 if (keg->uk_flags & UMA_ZONE_VTOSLAB) {
|
865 vm_object_t obj; 866 867 if (flags & UMA_SLAB_KMEM) 868 obj = kmem_object; 869 else if (flags & UMA_SLAB_KERNEL) 870 obj = kernel_object; 871 else 872 obj = NULL; 873 for (i = 0; i < keg->uk_ppera; i++) 874 vsetobj((vm_offset_t)mem + 875 (i * PAGE_SIZE), obj); 876 } 877 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
| 893 vm_object_t obj; 894 895 if (flags & UMA_SLAB_KMEM) 896 obj = kmem_object; 897 else if (flags & UMA_SLAB_KERNEL) 898 obj = kernel_object; 899 else 900 obj = NULL; 901 for (i = 0; i < keg->uk_ppera; i++) 902 vsetobj((vm_offset_t)mem + 903 (i * PAGE_SIZE), obj); 904 } 905 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
|
878 uma_zfree_internal(keg->uk_slabzone, slab,
| 906 zone_free_item(keg->uk_slabzone, slab,
|
879 NULL, SKIP_NONE, ZFREE_STATFREE); 880 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, 881 flags);
| 907 NULL, SKIP_NONE, ZFREE_STATFREE); 908 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, 909 flags);
|
882 ZONE_LOCK(zone);
| 910 KEG_LOCK(keg);
|
883 return (NULL); 884 } 885 }
| 911 return (NULL); 912 } 913 }
|
886 ZONE_LOCK(zone);
| 914 KEG_LOCK(keg);
|
887 888 if (keg->uk_flags & UMA_ZONE_HASH) 889 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 890 891 keg->uk_pages += keg->uk_ppera; 892 keg->uk_free += keg->uk_ipers; 893 894 return (slab); 895} 896 897/* 898 * This function is intended to be used early on in place of page_alloc() so 899 * that we may use the boot time page cache to satisfy allocations before 900 * the VM is ready. 901 */ 902static void * 903startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 904{ 905 uma_keg_t keg; 906 uma_slab_t tmps; 907
| 915 916 if (keg->uk_flags & UMA_ZONE_HASH) 917 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 918 919 keg->uk_pages += keg->uk_ppera; 920 keg->uk_free += keg->uk_ipers; 921 922 return (slab); 923} 924 925/* 926 * This function is intended to be used early on in place of page_alloc() so 927 * that we may use the boot time page cache to satisfy allocations before 928 * the VM is ready. 929 */ 930static void * 931startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 932{ 933 uma_keg_t keg; 934 uma_slab_t tmps; 935
|
908 keg = zone->uz_keg;
| 936 keg = zone_first_keg(zone);
|
909 910 /* 911 * Check our small startup cache to see if it has pages remaining. 912 */ 913 mtx_lock(&uma_boot_pages_mtx); 914 if ((tmps = LIST_FIRST(&uma_boot_pages)) != NULL) { 915 LIST_REMOVE(tmps, us_link); 916 mtx_unlock(&uma_boot_pages_mtx); 917 *pflag = tmps->us_flags; 918 return (tmps->us_data); 919 } 920 mtx_unlock(&uma_boot_pages_mtx); 921 if (booted == 0) 922 panic("UMA: Increase vm.boot_pages"); 923 /* 924 * Now that we've booted reset these users to their real allocator. 925 */ 926#ifdef UMA_MD_SMALL_ALLOC 927 keg->uk_allocf = uma_small_alloc; 928#else 929 keg->uk_allocf = page_alloc; 930#endif 931 return keg->uk_allocf(zone, bytes, pflag, wait); 932} 933 934/* 935 * Allocates a number of pages from the system 936 * 937 * Arguments:
| 937 938 /* 939 * Check our small startup cache to see if it has pages remaining. 940 */ 941 mtx_lock(&uma_boot_pages_mtx); 942 if ((tmps = LIST_FIRST(&uma_boot_pages)) != NULL) { 943 LIST_REMOVE(tmps, us_link); 944 mtx_unlock(&uma_boot_pages_mtx); 945 *pflag = tmps->us_flags; 946 return (tmps->us_data); 947 } 948 mtx_unlock(&uma_boot_pages_mtx); 949 if (booted == 0) 950 panic("UMA: Increase vm.boot_pages"); 951 /* 952 * Now that we've booted reset these users to their real allocator. 953 */ 954#ifdef UMA_MD_SMALL_ALLOC 955 keg->uk_allocf = uma_small_alloc; 956#else 957 keg->uk_allocf = page_alloc; 958#endif 959 return keg->uk_allocf(zone, bytes, pflag, wait); 960} 961 962/* 963 * Allocates a number of pages from the system 964 * 965 * Arguments:
|
938 * zone Unused
| |
939 * bytes The number of bytes requested 940 * wait Shall we wait? 941 * 942 * Returns: 943 * A pointer to the alloced memory or possibly 944 * NULL if M_NOWAIT is set. 945 */ 946static void * 947page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 948{ 949 void *p; /* Returned page */ 950 951 *pflag = UMA_SLAB_KMEM; 952 p = (void *) kmem_malloc(kmem_map, bytes, wait); 953 954 return (p); 955} 956 957/* 958 * Allocates a number of pages from within an object 959 * 960 * Arguments:
| 966 * bytes The number of bytes requested 967 * wait Shall we wait? 968 * 969 * Returns: 970 * A pointer to the alloced memory or possibly 971 * NULL if M_NOWAIT is set. 972 */ 973static void * 974page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 975{ 976 void *p; /* Returned page */ 977 978 *pflag = UMA_SLAB_KMEM; 979 p = (void *) kmem_malloc(kmem_map, bytes, wait); 980 981 return (p); 982} 983 984/* 985 * Allocates a number of pages from within an object 986 * 987 * Arguments:
|
961 * zone Unused
| |
962 * bytes The number of bytes requested 963 * wait Shall we wait? 964 * 965 * Returns: 966 * A pointer to the alloced memory or possibly 967 * NULL if M_NOWAIT is set. 968 */ 969static void * 970obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 971{ 972 vm_object_t object; 973 vm_offset_t retkva, zkva; 974 vm_page_t p; 975 int pages, startpages;
| 988 * bytes The number of bytes requested 989 * wait Shall we wait? 990 * 991 * Returns: 992 * A pointer to the alloced memory or possibly 993 * NULL if M_NOWAIT is set. 994 */ 995static void * 996obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 997{ 998 vm_object_t object; 999 vm_offset_t retkva, zkva; 1000 vm_page_t p; 1001 int pages, startpages;
|
| 1002 uma_keg_t keg;
|
976
| 1003
|
977 object = zone->uz_keg->uk_obj;
| 1004 keg = zone_first_keg(zone); 1005 object = keg->uk_obj;
|
978 retkva = 0; 979 980 /* 981 * This looks a little weird since we're getting one page at a time. 982 */ 983 VM_OBJECT_LOCK(object); 984 p = TAILQ_LAST(&object->memq, pglist); 985 pages = p != NULL ? p->pindex + 1 : 0; 986 startpages = pages;
| 1006 retkva = 0; 1007 1008 /* 1009 * This looks a little weird since we're getting one page at a time. 1010 */ 1011 VM_OBJECT_LOCK(object); 1012 p = TAILQ_LAST(&object->memq, pglist); 1013 pages = p != NULL ? p->pindex + 1 : 0; 1014 startpages = pages;
|
987 zkva = zone->uz_keg->uk_kva + pages * PAGE_SIZE;
| 1015 zkva = keg->uk_kva + pages * PAGE_SIZE;
|
988 for (; bytes > 0; bytes -= PAGE_SIZE) { 989 p = vm_page_alloc(object, pages, 990 VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 991 if (p == NULL) { 992 if (pages != startpages) 993 pmap_qremove(retkva, pages - startpages); 994 while (pages != startpages) { 995 pages--; 996 p = TAILQ_LAST(&object->memq, pglist); 997 vm_page_lock_queues(); 998 vm_page_unwire(p, 0); 999 vm_page_free(p); 1000 vm_page_unlock_queues(); 1001 } 1002 retkva = 0; 1003 goto done; 1004 } 1005 pmap_qenter(zkva, &p, 1); 1006 if (retkva == 0) 1007 retkva = zkva; 1008 zkva += PAGE_SIZE; 1009 pages += 1; 1010 } 1011done: 1012 VM_OBJECT_UNLOCK(object); 1013 *flags = UMA_SLAB_PRIV; 1014 1015 return ((void *)retkva); 1016} 1017 1018/* 1019 * Frees a number of pages to the system 1020 * 1021 * Arguments: 1022 * mem A pointer to the memory to be freed 1023 * size The size of the memory being freed 1024 * flags The original p->us_flags field 1025 * 1026 * Returns: 1027 * Nothing 1028 */ 1029static void 1030page_free(void *mem, int size, u_int8_t flags) 1031{ 1032 vm_map_t map; 1033 1034 if (flags & UMA_SLAB_KMEM) 1035 map = kmem_map; 1036 else 1037 panic("UMA: page_free used with invalid flags %d\n", flags); 1038 1039 kmem_free(map, (vm_offset_t)mem, size); 1040} 1041 1042/* 1043 * Zero fill initializer 1044 * 1045 * Arguments/Returns follow uma_init specifications 1046 */ 1047static int 1048zero_init(void *mem, int size, int flags) 1049{ 1050 bzero(mem, size); 1051 return (0); 1052} 1053 1054/*
| 1016 for (; bytes > 0; bytes -= PAGE_SIZE) { 1017 p = vm_page_alloc(object, pages, 1018 VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 1019 if (p == NULL) { 1020 if (pages != startpages) 1021 pmap_qremove(retkva, pages - startpages); 1022 while (pages != startpages) { 1023 pages--; 1024 p = TAILQ_LAST(&object->memq, pglist); 1025 vm_page_lock_queues(); 1026 vm_page_unwire(p, 0); 1027 vm_page_free(p); 1028 vm_page_unlock_queues(); 1029 } 1030 retkva = 0; 1031 goto done; 1032 } 1033 pmap_qenter(zkva, &p, 1); 1034 if (retkva == 0) 1035 retkva = zkva; 1036 zkva += PAGE_SIZE; 1037 pages += 1; 1038 } 1039done: 1040 VM_OBJECT_UNLOCK(object); 1041 *flags = UMA_SLAB_PRIV; 1042 1043 return ((void *)retkva); 1044} 1045 1046/* 1047 * Frees a number of pages to the system 1048 * 1049 * Arguments: 1050 * mem A pointer to the memory to be freed 1051 * size The size of the memory being freed 1052 * flags The original p->us_flags field 1053 * 1054 * Returns: 1055 * Nothing 1056 */ 1057static void 1058page_free(void *mem, int size, u_int8_t flags) 1059{ 1060 vm_map_t map; 1061 1062 if (flags & UMA_SLAB_KMEM) 1063 map = kmem_map; 1064 else 1065 panic("UMA: page_free used with invalid flags %d\n", flags); 1066 1067 kmem_free(map, (vm_offset_t)mem, size); 1068} 1069 1070/* 1071 * Zero fill initializer 1072 * 1073 * Arguments/Returns follow uma_init specifications 1074 */ 1075static int 1076zero_init(void *mem, int size, int flags) 1077{ 1078 bzero(mem, size); 1079 return (0); 1080} 1081 1082/*
|
1055 * Finish creating a small uma zone. This calculates ipers, and the zone size.
| 1083 * Finish creating a small uma keg. This calculates ipers, and the keg size.
|
1056 * 1057 * Arguments
| 1084 * 1085 * Arguments
|
1058 * zone The zone we should initialize
| 1086 * keg The zone we should initialize
|
1059 * 1060 * Returns 1061 * Nothing 1062 */ 1063static void
| 1087 * 1088 * Returns 1089 * Nothing 1090 */ 1091static void
|
1064zone_small_init(uma_zone_t zone)
| 1092keg_small_init(uma_keg_t keg)
|
1065{
| 1093{
|
1066 uma_keg_t keg;
| |
1067 u_int rsize; 1068 u_int memused; 1069 u_int wastedspace; 1070 u_int shsize; 1071
| 1094 u_int rsize; 1095 u_int memused; 1096 u_int wastedspace; 1097 u_int shsize; 1098
|
1072 keg = zone->uz_keg; 1073 KASSERT(keg != NULL, ("Keg is null in zone_small_init"));
| 1099 KASSERT(keg != NULL, ("Keg is null in keg_small_init"));
|
1074 rsize = keg->uk_size; 1075 1076 if (rsize < UMA_SMALLEST_UNIT) 1077 rsize = UMA_SMALLEST_UNIT; 1078 if (rsize & keg->uk_align) 1079 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1080 1081 keg->uk_rsize = rsize; 1082 keg->uk_ppera = 1; 1083 1084 if (keg->uk_flags & UMA_ZONE_REFCNT) { 1085 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */ 1086 shsize = sizeof(struct uma_slab_refcnt); 1087 } else { 1088 rsize += UMA_FRITM_SZ; /* Account for linkage */ 1089 shsize = sizeof(struct uma_slab); 1090 } 1091 1092 keg->uk_ipers = (UMA_SLAB_SIZE - shsize) / rsize;
| 1100 rsize = keg->uk_size; 1101 1102 if (rsize < UMA_SMALLEST_UNIT) 1103 rsize = UMA_SMALLEST_UNIT; 1104 if (rsize & keg->uk_align) 1105 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1106 1107 keg->uk_rsize = rsize; 1108 keg->uk_ppera = 1; 1109 1110 if (keg->uk_flags & UMA_ZONE_REFCNT) { 1111 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */ 1112 shsize = sizeof(struct uma_slab_refcnt); 1113 } else { 1114 rsize += UMA_FRITM_SZ; /* Account for linkage */ 1115 shsize = sizeof(struct uma_slab); 1116 } 1117 1118 keg->uk_ipers = (UMA_SLAB_SIZE - shsize) / rsize;
|
1093 KASSERT(keg->uk_ipers != 0, ("zone_small_init: ipers is 0"));
| 1119 KASSERT(keg->uk_ipers != 0, ("keg_small_init: ipers is 0"));
|
1094 memused = keg->uk_ipers * rsize + shsize; 1095 wastedspace = UMA_SLAB_SIZE - memused; 1096 1097 /* 1098 * We can't do OFFPAGE if we're internal or if we've been 1099 * asked to not go to the VM for buckets. If we do this we 1100 * may end up going to the VM (kmem_map) for slabs which we 1101 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a 1102 * result of UMA_ZONE_VM, which clearly forbids it. 1103 */ 1104 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1105 (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 1106 return; 1107 1108 if ((wastedspace >= UMA_MAX_WASTE) && 1109 (keg->uk_ipers < (UMA_SLAB_SIZE / keg->uk_rsize))) { 1110 keg->uk_ipers = UMA_SLAB_SIZE / keg->uk_rsize; 1111 KASSERT(keg->uk_ipers <= 255,
| 1120 memused = keg->uk_ipers * rsize + shsize; 1121 wastedspace = UMA_SLAB_SIZE - memused; 1122 1123 /* 1124 * We can't do OFFPAGE if we're internal or if we've been 1125 * asked to not go to the VM for buckets. If we do this we 1126 * may end up going to the VM (kmem_map) for slabs which we 1127 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a 1128 * result of UMA_ZONE_VM, which clearly forbids it. 1129 */ 1130 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1131 (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 1132 return; 1133 1134 if ((wastedspace >= UMA_MAX_WASTE) && 1135 (keg->uk_ipers < (UMA_SLAB_SIZE / keg->uk_rsize))) { 1136 keg->uk_ipers = UMA_SLAB_SIZE / keg->uk_rsize; 1137 KASSERT(keg->uk_ipers <= 255,
|
1112 ("zone_small_init: keg->uk_ipers too high!"));
| 1138 ("keg_small_init: keg->uk_ipers too high!"));
|
1113#ifdef UMA_DEBUG 1114 printf("UMA decided we need offpage slab headers for "
| 1139#ifdef UMA_DEBUG 1140 printf("UMA decided we need offpage slab headers for "
|
1115 "zone: %s, calculated wastedspace = %d, "
| 1141 "keg: %s, calculated wastedspace = %d, "
|
1116 "maximum wasted space allowed = %d, " 1117 "calculated ipers = %d, "
| 1142 "maximum wasted space allowed = %d, " 1143 "calculated ipers = %d, "
|
1118 "new wasted space = %d\n", zone->uz_name, wastedspace,
| 1144 "new wasted space = %d\n", keg->uk_name, wastedspace,
|
1119 UMA_MAX_WASTE, keg->uk_ipers, 1120 UMA_SLAB_SIZE - keg->uk_ipers * keg->uk_rsize); 1121#endif 1122 keg->uk_flags |= UMA_ZONE_OFFPAGE;
| 1145 UMA_MAX_WASTE, keg->uk_ipers, 1146 UMA_SLAB_SIZE - keg->uk_ipers * keg->uk_rsize); 1147#endif 1148 keg->uk_flags |= UMA_ZONE_OFFPAGE;
|
1123 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
| 1149 if ((keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
|
1124 keg->uk_flags |= UMA_ZONE_HASH; 1125 } 1126} 1127 1128/*
| 1150 keg->uk_flags |= UMA_ZONE_HASH; 1151 } 1152} 1153 1154/*
|
1129 * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do
| 1155 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
|
1130 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 1131 * more complicated. 1132 * 1133 * Arguments
| 1156 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 1157 * more complicated. 1158 * 1159 * Arguments
|
1134 * zone The zone we should initialize
| 1160 * keg The keg we should initialize
|
1135 * 1136 * Returns 1137 * Nothing 1138 */ 1139static void
| 1161 * 1162 * Returns 1163 * Nothing 1164 */ 1165static void
|
1140zone_large_init(uma_zone_t zone)
| 1166keg_large_init(uma_keg_t keg)
|
1141{
| 1167{
|
1142 uma_keg_t keg;
| |
1143 int pages; 1144
| 1168 int pages; 1169
|
1145 keg = zone->uz_keg; 1146 1147 KASSERT(keg != NULL, ("Keg is null in zone_large_init"));
| 1170 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
|
1148 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
| 1171 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
|
1149 ("zone_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY zone"));
| 1172 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
|
1150 1151 pages = keg->uk_size / UMA_SLAB_SIZE; 1152 1153 /* Account for remainder */ 1154 if ((pages * UMA_SLAB_SIZE) < keg->uk_size) 1155 pages++; 1156 1157 keg->uk_ppera = pages; 1158 keg->uk_ipers = 1; 1159 1160 keg->uk_flags |= UMA_ZONE_OFFPAGE;
| 1173 1174 pages = keg->uk_size / UMA_SLAB_SIZE; 1175 1176 /* Account for remainder */ 1177 if ((pages * UMA_SLAB_SIZE) < keg->uk_size) 1178 pages++; 1179 1180 keg->uk_ppera = pages; 1181 keg->uk_ipers = 1; 1182 1183 keg->uk_flags |= UMA_ZONE_OFFPAGE;
|
1161 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
| 1184 if ((keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
|
1162 keg->uk_flags |= UMA_ZONE_HASH; 1163 1164 keg->uk_rsize = keg->uk_size; 1165} 1166
| 1185 keg->uk_flags |= UMA_ZONE_HASH; 1186 1187 keg->uk_rsize = keg->uk_size; 1188} 1189
|
| 1190static void 1191keg_cachespread_init(uma_keg_t keg) 1192{ 1193 int alignsize; 1194 int trailer; 1195 int pages; 1196 int rsize; 1197 1198 alignsize = keg->uk_align + 1; 1199 rsize = keg->uk_size; 1200 /* 1201 * We want one item to start on every align boundary in a page. To 1202 * do this we will span pages. We will also extend the item by the 1203 * size of align if it is an even multiple of align. Otherwise, it 1204 * would fall on the same boundary every time. 1205 */ 1206 if (rsize & keg->uk_align) 1207 rsize = (rsize & ~keg->uk_align) + alignsize; 1208 if ((rsize & alignsize) == 0) 1209 rsize += alignsize; 1210 trailer = rsize - keg->uk_size; 1211 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; 1212 pages = MIN(pages, (128 * 1024) / PAGE_SIZE); 1213 keg->uk_rsize = rsize; 1214 keg->uk_ppera = pages; 1215 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; 1216 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; 1217 KASSERT(keg->uk_ipers <= uma_max_ipers, 1218 ("keg_small_init: keg->uk_ipers too high(%d) increase max_ipers", 1219 keg->uk_ipers)); 1220} 1221
|
1167/* 1168 * Keg header ctor. This initializes all fields, locks, etc. And inserts 1169 * the keg onto the global keg list. 1170 * 1171 * Arguments/Returns follow uma_ctor specifications 1172 * udata Actually uma_kctor_args 1173 */ 1174static int 1175keg_ctor(void *mem, int size, void *udata, int flags) 1176{ 1177 struct uma_kctor_args *arg = udata; 1178 uma_keg_t keg = mem; 1179 uma_zone_t zone; 1180 1181 bzero(keg, size); 1182 keg->uk_size = arg->size; 1183 keg->uk_init = arg->uminit; 1184 keg->uk_fini = arg->fini; 1185 keg->uk_align = arg->align; 1186 keg->uk_free = 0; 1187 keg->uk_pages = 0; 1188 keg->uk_flags = arg->flags; 1189 keg->uk_allocf = page_alloc; 1190 keg->uk_freef = page_free; 1191 keg->uk_recurse = 0; 1192 keg->uk_slabzone = NULL; 1193 1194 /* 1195 * The master zone is passed to us at keg-creation time. 1196 */ 1197 zone = arg->zone;
| 1222/* 1223 * Keg header ctor. This initializes all fields, locks, etc. And inserts 1224 * the keg onto the global keg list. 1225 * 1226 * Arguments/Returns follow uma_ctor specifications 1227 * udata Actually uma_kctor_args 1228 */ 1229static int 1230keg_ctor(void *mem, int size, void *udata, int flags) 1231{ 1232 struct uma_kctor_args *arg = udata; 1233 uma_keg_t keg = mem; 1234 uma_zone_t zone; 1235 1236 bzero(keg, size); 1237 keg->uk_size = arg->size; 1238 keg->uk_init = arg->uminit; 1239 keg->uk_fini = arg->fini; 1240 keg->uk_align = arg->align; 1241 keg->uk_free = 0; 1242 keg->uk_pages = 0; 1243 keg->uk_flags = arg->flags; 1244 keg->uk_allocf = page_alloc; 1245 keg->uk_freef = page_free; 1246 keg->uk_recurse = 0; 1247 keg->uk_slabzone = NULL; 1248 1249 /* 1250 * The master zone is passed to us at keg-creation time. 1251 */ 1252 zone = arg->zone;
|
1198 zone->uz_keg = keg;
| 1253 keg->uk_name = zone->uz_name;
|
1199 1200 if (arg->flags & UMA_ZONE_VM) 1201 keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1202 1203 if (arg->flags & UMA_ZONE_ZINIT) 1204 keg->uk_init = zero_init; 1205
| 1254 1255 if (arg->flags & UMA_ZONE_VM) 1256 keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1257 1258 if (arg->flags & UMA_ZONE_ZINIT) 1259 keg->uk_init = zero_init; 1260
|
| 1261 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC) 1262 keg->uk_flags |= UMA_ZONE_VTOSLAB; 1263
|
1206 /* 1207 * The +UMA_FRITM_SZ added to uk_size is to account for the
| 1264 /* 1265 * The +UMA_FRITM_SZ added to uk_size is to account for the
|
1208 * linkage that is added to the size in zone_small_init(). If
| 1266 * linkage that is added to the size in keg_small_init(). If
|
1209 * we don't account for this here then we may end up in
| 1267 * we don't account for this here then we may end up in
|
1210 * zone_small_init() with a calculated 'ipers' of 0.
| 1268 * keg_small_init() with a calculated 'ipers' of 0.
|
1211 */ 1212 if (keg->uk_flags & UMA_ZONE_REFCNT) {
| 1269 */ 1270 if (keg->uk_flags & UMA_ZONE_REFCNT) {
|
1213 if ((keg->uk_size+UMA_FRITMREF_SZ) >
| 1271 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) 1272 keg_cachespread_init(keg); 1273 else if ((keg->uk_size+UMA_FRITMREF_SZ) >
|
1214 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)))
| 1274 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)))
|
1215 zone_large_init(zone);
| 1275 keg_large_init(keg);
|
1216 else
| 1276 else
|
1217 zone_small_init(zone);
| 1277 keg_small_init(keg);
|
1218 } else {
| 1278 } else {
|
1219 if ((keg->uk_size+UMA_FRITM_SZ) >
| 1279 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) 1280 keg_cachespread_init(keg); 1281 else if ((keg->uk_size+UMA_FRITM_SZ) >
|
1220 (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
| 1282 (UMA_SLAB_SIZE - sizeof(struct uma_slab)))
|
1221 zone_large_init(zone);
| 1283 keg_large_init(keg);
|
1222 else
| 1284 else
|
1223 zone_small_init(zone);
| 1285 keg_small_init(keg);
|
1224 } 1225 1226 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 1227 if (keg->uk_flags & UMA_ZONE_REFCNT) 1228 keg->uk_slabzone = slabrefzone; 1229 else 1230 keg->uk_slabzone = slabzone; 1231 } 1232 1233 /* 1234 * If we haven't booted yet we need allocations to go through the 1235 * startup cache until the vm is ready. 1236 */ 1237 if (keg->uk_ppera == 1) { 1238#ifdef UMA_MD_SMALL_ALLOC 1239 keg->uk_allocf = uma_small_alloc; 1240 keg->uk_freef = uma_small_free; 1241#endif 1242 if (booted == 0) 1243 keg->uk_allocf = startup_alloc; 1244 } 1245 1246 /*
| 1286 } 1287 1288 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 1289 if (keg->uk_flags & UMA_ZONE_REFCNT) 1290 keg->uk_slabzone = slabrefzone; 1291 else 1292 keg->uk_slabzone = slabzone; 1293 } 1294 1295 /* 1296 * If we haven't booted yet we need allocations to go through the 1297 * startup cache until the vm is ready. 1298 */ 1299 if (keg->uk_ppera == 1) { 1300#ifdef UMA_MD_SMALL_ALLOC 1301 keg->uk_allocf = uma_small_alloc; 1302 keg->uk_freef = uma_small_free; 1303#endif 1304 if (booted == 0) 1305 keg->uk_allocf = startup_alloc; 1306 } 1307 1308 /*
|
1247 * Initialize keg's lock (shared among zones) through 1248 * Master zone
| 1309 * Initialize keg's lock (shared among zones).
|
1249 */
| 1310 */
|
1250 zone->uz_lock = &keg->uk_lock;
| |
1251 if (arg->flags & UMA_ZONE_MTXCLASS)
| 1311 if (arg->flags & UMA_ZONE_MTXCLASS)
|
1252 ZONE_LOCK_INIT(zone, 1);
| 1312 KEG_LOCK_INIT(keg, 1);
|
1253 else
| 1313 else
|
1254 ZONE_LOCK_INIT(zone, 0);
| 1314 KEG_LOCK_INIT(keg, 0);
|
1255 1256 /* 1257 * If we're putting the slab header in the actual page we need to 1258 * figure out where in each page it goes. This calculates a right 1259 * justified offset into the memory on an ALIGN_PTR boundary. 1260 */ 1261 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 1262 u_int totsize; 1263 1264 /* Size of the slab struct and free list */ 1265 if (keg->uk_flags & UMA_ZONE_REFCNT) 1266 totsize = sizeof(struct uma_slab_refcnt) + 1267 keg->uk_ipers * UMA_FRITMREF_SZ; 1268 else 1269 totsize = sizeof(struct uma_slab) + 1270 keg->uk_ipers * UMA_FRITM_SZ; 1271 1272 if (totsize & UMA_ALIGN_PTR) 1273 totsize = (totsize & ~UMA_ALIGN_PTR) + 1274 (UMA_ALIGN_PTR + 1); 1275 keg->uk_pgoff = UMA_SLAB_SIZE - totsize; 1276 1277 if (keg->uk_flags & UMA_ZONE_REFCNT) 1278 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt) 1279 + keg->uk_ipers * UMA_FRITMREF_SZ; 1280 else 1281 totsize = keg->uk_pgoff + sizeof(struct uma_slab) 1282 + keg->uk_ipers * UMA_FRITM_SZ; 1283 1284 /* 1285 * The only way the following is possible is if with our 1286 * UMA_ALIGN_PTR adjustments we are now bigger than 1287 * UMA_SLAB_SIZE. I haven't checked whether this is 1288 * mathematically possible for all cases, so we make 1289 * sure here anyway. 1290 */ 1291 if (totsize > UMA_SLAB_SIZE) { 1292 printf("zone %s ipers %d rsize %d size %d\n", 1293 zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1294 keg->uk_size); 1295 panic("UMA slab won't fit.\n"); 1296 } 1297 } 1298 1299 if (keg->uk_flags & UMA_ZONE_HASH) 1300 hash_alloc(&keg->uk_hash); 1301 1302#ifdef UMA_DEBUG
| 1315 1316 /* 1317 * If we're putting the slab header in the actual page we need to 1318 * figure out where in each page it goes. This calculates a right 1319 * justified offset into the memory on an ALIGN_PTR boundary. 1320 */ 1321 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 1322 u_int totsize; 1323 1324 /* Size of the slab struct and free list */ 1325 if (keg->uk_flags & UMA_ZONE_REFCNT) 1326 totsize = sizeof(struct uma_slab_refcnt) + 1327 keg->uk_ipers * UMA_FRITMREF_SZ; 1328 else 1329 totsize = sizeof(struct uma_slab) + 1330 keg->uk_ipers * UMA_FRITM_SZ; 1331 1332 if (totsize & UMA_ALIGN_PTR) 1333 totsize = (totsize & ~UMA_ALIGN_PTR) + 1334 (UMA_ALIGN_PTR + 1); 1335 keg->uk_pgoff = UMA_SLAB_SIZE - totsize; 1336 1337 if (keg->uk_flags & UMA_ZONE_REFCNT) 1338 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt) 1339 + keg->uk_ipers * UMA_FRITMREF_SZ; 1340 else 1341 totsize = keg->uk_pgoff + sizeof(struct uma_slab) 1342 + keg->uk_ipers * UMA_FRITM_SZ; 1343 1344 /* 1345 * The only way the following is possible is if with our 1346 * UMA_ALIGN_PTR adjustments we are now bigger than 1347 * UMA_SLAB_SIZE. I haven't checked whether this is 1348 * mathematically possible for all cases, so we make 1349 * sure here anyway. 1350 */ 1351 if (totsize > UMA_SLAB_SIZE) { 1352 printf("zone %s ipers %d rsize %d size %d\n", 1353 zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1354 keg->uk_size); 1355 panic("UMA slab won't fit.\n"); 1356 } 1357 } 1358 1359 if (keg->uk_flags & UMA_ZONE_HASH) 1360 hash_alloc(&keg->uk_hash); 1361 1362#ifdef UMA_DEBUG
|
1303 printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 1304 zone->uz_name, zone, 1305 keg->uk_size, keg->uk_ipers, 1306 keg->uk_ppera, keg->uk_pgoff);
| 1363 printf("UMA: %s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 1364 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags, 1365 keg->uk_ipers, keg->uk_ppera, 1366 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
|
1307#endif 1308 1309 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 1310 1311 mtx_lock(&uma_mtx); 1312 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 1313 mtx_unlock(&uma_mtx); 1314 return (0); 1315} 1316 1317/* 1318 * Zone header ctor. This initializes all fields, locks, etc. 1319 * 1320 * Arguments/Returns follow uma_ctor specifications 1321 * udata Actually uma_zctor_args 1322 */
| 1367#endif 1368 1369 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 1370 1371 mtx_lock(&uma_mtx); 1372 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 1373 mtx_unlock(&uma_mtx); 1374 return (0); 1375} 1376 1377/* 1378 * Zone header ctor. This initializes all fields, locks, etc. 1379 * 1380 * Arguments/Returns follow uma_ctor specifications 1381 * udata Actually uma_zctor_args 1382 */
|
1323
| |
1324static int 1325zone_ctor(void *mem, int size, void *udata, int flags) 1326{ 1327 struct uma_zctor_args *arg = udata; 1328 uma_zone_t zone = mem; 1329 uma_zone_t z; 1330 uma_keg_t keg; 1331 1332 bzero(zone, size); 1333 zone->uz_name = arg->name; 1334 zone->uz_ctor = arg->ctor; 1335 zone->uz_dtor = arg->dtor;
| 1383static int 1384zone_ctor(void *mem, int size, void *udata, int flags) 1385{ 1386 struct uma_zctor_args *arg = udata; 1387 uma_zone_t zone = mem; 1388 uma_zone_t z; 1389 uma_keg_t keg; 1390 1391 bzero(zone, size); 1392 zone->uz_name = arg->name; 1393 zone->uz_ctor = arg->ctor; 1394 zone->uz_dtor = arg->dtor;
|
| 1395 zone->uz_slab = zone_fetch_slab;
|
1336 zone->uz_init = NULL; 1337 zone->uz_fini = NULL; 1338 zone->uz_allocs = 0; 1339 zone->uz_frees = 0; 1340 zone->uz_fails = 0; 1341 zone->uz_fills = zone->uz_count = 0;
| 1396 zone->uz_init = NULL; 1397 zone->uz_fini = NULL; 1398 zone->uz_allocs = 0; 1399 zone->uz_frees = 0; 1400 zone->uz_fails = 0; 1401 zone->uz_fills = zone->uz_count = 0;
|
| 1402 zone->uz_flags = 0; 1403 keg = arg->keg;
|
1342 1343 if (arg->flags & UMA_ZONE_SECONDARY) { 1344 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
| 1404 1405 if (arg->flags & UMA_ZONE_SECONDARY) { 1406 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
|
1345 keg = arg->keg; 1346 zone->uz_keg = keg;
| |
1347 zone->uz_init = arg->uminit; 1348 zone->uz_fini = arg->fini; 1349 zone->uz_lock = &keg->uk_lock;
| 1407 zone->uz_init = arg->uminit; 1408 zone->uz_fini = arg->fini; 1409 zone->uz_lock = &keg->uk_lock;
|
| 1410 zone->uz_flags |= UMA_ZONE_SECONDARY;
|
1350 mtx_lock(&uma_mtx); 1351 ZONE_LOCK(zone);
| 1411 mtx_lock(&uma_mtx); 1412 ZONE_LOCK(zone);
|
1352 keg->uk_flags |= UMA_ZONE_SECONDARY;
| |
1353 LIST_FOREACH(z, &keg->uk_zones, uz_link) { 1354 if (LIST_NEXT(z, uz_link) == NULL) { 1355 LIST_INSERT_AFTER(z, zone, uz_link); 1356 break; 1357 } 1358 } 1359 ZONE_UNLOCK(zone); 1360 mtx_unlock(&uma_mtx);
| 1413 LIST_FOREACH(z, &keg->uk_zones, uz_link) { 1414 if (LIST_NEXT(z, uz_link) == NULL) { 1415 LIST_INSERT_AFTER(z, zone, uz_link); 1416 break; 1417 } 1418 } 1419 ZONE_UNLOCK(zone); 1420 mtx_unlock(&uma_mtx);
|
1361 } else if (arg->keg == NULL) { 1362 if (uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 1363 arg->align, arg->flags) == NULL)
| 1421 } else if (keg == NULL) { 1422 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 1423 arg->align, arg->flags)) == NULL)
|
1364 return (ENOMEM); 1365 } else { 1366 struct uma_kctor_args karg; 1367 int error; 1368 1369 /* We should only be here from uma_startup() */ 1370 karg.size = arg->size; 1371 karg.uminit = arg->uminit; 1372 karg.fini = arg->fini; 1373 karg.align = arg->align; 1374 karg.flags = arg->flags; 1375 karg.zone = zone; 1376 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1377 flags); 1378 if (error) 1379 return (error); 1380 }
| 1424 return (ENOMEM); 1425 } else { 1426 struct uma_kctor_args karg; 1427 int error; 1428 1429 /* We should only be here from uma_startup() */ 1430 karg.size = arg->size; 1431 karg.uminit = arg->uminit; 1432 karg.fini = arg->fini; 1433 karg.align = arg->align; 1434 karg.flags = arg->flags; 1435 karg.zone = zone; 1436 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1437 flags); 1438 if (error) 1439 return (error); 1440 }
|
1381 keg = zone->uz_keg;
| 1441 /* 1442 * Link in the first keg. 1443 */ 1444 zone->uz_klink.kl_keg = keg; 1445 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
|
1382 zone->uz_lock = &keg->uk_lock;
| 1446 zone->uz_lock = &keg->uk_lock;
|
| 1447 zone->uz_size = keg->uk_size; 1448 zone->uz_flags |= (keg->uk_flags & 1449 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
|
1383 1384 /* 1385 * Some internal zones don't have room allocated for the per cpu 1386 * caches. If we're internal, bail out here. 1387 */ 1388 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
| 1450 1451 /* 1452 * Some internal zones don't have room allocated for the per cpu 1453 * caches. If we're internal, bail out here. 1454 */ 1455 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
|
1389 KASSERT((keg->uk_flags & UMA_ZONE_SECONDARY) == 0,
| 1456 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
|
1390 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 1391 return (0); 1392 } 1393 1394 if (keg->uk_flags & UMA_ZONE_MAXBUCKET) 1395 zone->uz_count = BUCKET_MAX; 1396 else if (keg->uk_ipers <= BUCKET_MAX) 1397 zone->uz_count = keg->uk_ipers; 1398 else 1399 zone->uz_count = BUCKET_MAX; 1400 return (0); 1401} 1402 1403/* 1404 * Keg header dtor. This frees all data, destroys locks, frees the hash 1405 * table and removes the keg from the global list. 1406 * 1407 * Arguments/Returns follow uma_dtor specifications 1408 * udata unused 1409 */ 1410static void 1411keg_dtor(void *arg, int size, void *udata) 1412{ 1413 uma_keg_t keg; 1414 1415 keg = (uma_keg_t)arg;
| 1457 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 1458 return (0); 1459 } 1460 1461 if (keg->uk_flags & UMA_ZONE_MAXBUCKET) 1462 zone->uz_count = BUCKET_MAX; 1463 else if (keg->uk_ipers <= BUCKET_MAX) 1464 zone->uz_count = keg->uk_ipers; 1465 else 1466 zone->uz_count = BUCKET_MAX; 1467 return (0); 1468} 1469 1470/* 1471 * Keg header dtor. This frees all data, destroys locks, frees the hash 1472 * table and removes the keg from the global list. 1473 * 1474 * Arguments/Returns follow uma_dtor specifications 1475 * udata unused 1476 */ 1477static void 1478keg_dtor(void *arg, int size, void *udata) 1479{ 1480 uma_keg_t keg; 1481 1482 keg = (uma_keg_t)arg;
|
1416 mtx_lock(&keg->uk_lock);
| 1483 KEG_LOCK(keg);
|
1417 if (keg->uk_free != 0) { 1418 printf("Freed UMA keg was not empty (%d items). " 1419 " Lost %d pages of memory.\n", 1420 keg->uk_free, keg->uk_pages); 1421 }
| 1484 if (keg->uk_free != 0) { 1485 printf("Freed UMA keg was not empty (%d items). " 1486 " Lost %d pages of memory.\n", 1487 keg->uk_free, keg->uk_pages); 1488 }
|
1422 mtx_unlock(&keg->uk_lock);
| 1489 KEG_UNLOCK(keg);
|
1423
| 1490
|
1424 if (keg->uk_flags & UMA_ZONE_HASH) 1425 hash_free(&keg->uk_hash);
| 1491 hash_free(&keg->uk_hash);
|
1426
| 1492
|
1427 mtx_destroy(&keg->uk_lock);
| 1493 KEG_LOCK_FINI(keg);
|
1428} 1429 1430/* 1431 * Zone header dtor. 1432 * 1433 * Arguments/Returns follow uma_dtor specifications 1434 * udata unused 1435 */ 1436static void 1437zone_dtor(void *arg, int size, void *udata) 1438{
| 1494} 1495 1496/* 1497 * Zone header dtor. 1498 * 1499 * Arguments/Returns follow uma_dtor specifications 1500 * udata unused 1501 */ 1502static void 1503zone_dtor(void *arg, int size, void *udata) 1504{
|
| 1505 uma_klink_t klink;
|
1439 uma_zone_t zone; 1440 uma_keg_t keg; 1441 1442 zone = (uma_zone_t)arg;
| 1506 uma_zone_t zone; 1507 uma_keg_t keg; 1508 1509 zone = (uma_zone_t)arg;
|
1443 keg = zone->uz_keg;
| 1510 keg = zone_first_keg(zone);
|
1444
| 1511
|
1445 if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL))
| 1512 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL))
|
1446 cache_drain(zone); 1447 1448 mtx_lock(&uma_mtx);
| 1513 cache_drain(zone); 1514 1515 mtx_lock(&uma_mtx);
|
1449 zone_drain(zone); 1450 if (keg->uk_flags & UMA_ZONE_SECONDARY) { 1451 LIST_REMOVE(zone, uz_link); 1452 /* 1453 * XXX there are some races here where 1454 * the zone can be drained but zone lock 1455 * released and then refilled before we 1456 * remove it... we dont care for now 1457 */ 1458 ZONE_LOCK(zone); 1459 if (LIST_EMPTY(&keg->uk_zones)) 1460 keg->uk_flags &= ~UMA_ZONE_SECONDARY; 1461 ZONE_UNLOCK(zone); 1462 mtx_unlock(&uma_mtx); 1463 } else {
| 1516 LIST_REMOVE(zone, uz_link); 1517 mtx_unlock(&uma_mtx); 1518 /* 1519 * XXX there are some races here where 1520 * the zone can be drained but zone lock 1521 * released and then refilled before we 1522 * remove it... we dont care for now 1523 */ 1524 zone_drain_wait(zone, M_WAITOK); 1525 /* 1526 * Unlink all of our kegs. 1527 */ 1528 while ((klink = LIST_FIRST(&zone->uz_kegs)) != NULL) { 1529 klink->kl_keg = NULL; 1530 LIST_REMOVE(klink, kl_link); 1531 if (klink == &zone->uz_klink) 1532 continue; 1533 free(klink, M_TEMP); 1534 } 1535 /* 1536 * We only destroy kegs from non secondary zones. 1537 */ 1538 if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) { 1539 mtx_lock(&uma_mtx);
|
1464 LIST_REMOVE(keg, uk_link);
| 1540 LIST_REMOVE(keg, uk_link);
|
1465 LIST_REMOVE(zone, uz_link);
| |
1466 mtx_unlock(&uma_mtx);
| 1541 mtx_unlock(&uma_mtx);
|
1467 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE,
| 1542 zone_free_item(kegs, keg, NULL, SKIP_NONE,
|
1468 ZFREE_STATFREE); 1469 }
| 1543 ZFREE_STATFREE); 1544 }
|
1470 zone->uz_keg = NULL;
| |
1471} 1472 1473/* 1474 * Traverses every zone in the system and calls a callback 1475 * 1476 * Arguments: 1477 * zfunc A pointer to a function which accepts a zone 1478 * as an argument. 1479 * 1480 * Returns: 1481 * Nothing 1482 */ 1483static void 1484zone_foreach(void (*zfunc)(uma_zone_t)) 1485{ 1486 uma_keg_t keg; 1487 uma_zone_t zone; 1488 1489 mtx_lock(&uma_mtx); 1490 LIST_FOREACH(keg, &uma_kegs, uk_link) { 1491 LIST_FOREACH(zone, &keg->uk_zones, uz_link) 1492 zfunc(zone); 1493 } 1494 mtx_unlock(&uma_mtx); 1495} 1496 1497/* Public functions */ 1498/* See uma.h */ 1499void 1500uma_startup(void *bootmem, int boot_pages) 1501{ 1502 struct uma_zctor_args args; 1503 uma_slab_t slab; 1504 u_int slabsize; 1505 u_int objsize, totsize, wsize; 1506 int i; 1507 1508#ifdef UMA_DEBUG 1509 printf("Creating uma keg headers zone and keg.\n"); 1510#endif 1511 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 1512 1513 /* 1514 * Figure out the maximum number of items-per-slab we'll have if 1515 * we're using the OFFPAGE slab header to track free items, given 1516 * all possible object sizes and the maximum desired wastage 1517 * (UMA_MAX_WASTE). 1518 * 1519 * We iterate until we find an object size for
| 1545} 1546 1547/* 1548 * Traverses every zone in the system and calls a callback 1549 * 1550 * Arguments: 1551 * zfunc A pointer to a function which accepts a zone 1552 * as an argument. 1553 * 1554 * Returns: 1555 * Nothing 1556 */ 1557static void 1558zone_foreach(void (*zfunc)(uma_zone_t)) 1559{ 1560 uma_keg_t keg; 1561 uma_zone_t zone; 1562 1563 mtx_lock(&uma_mtx); 1564 LIST_FOREACH(keg, &uma_kegs, uk_link) { 1565 LIST_FOREACH(zone, &keg->uk_zones, uz_link) 1566 zfunc(zone); 1567 } 1568 mtx_unlock(&uma_mtx); 1569} 1570 1571/* Public functions */ 1572/* See uma.h */ 1573void 1574uma_startup(void *bootmem, int boot_pages) 1575{ 1576 struct uma_zctor_args args; 1577 uma_slab_t slab; 1578 u_int slabsize; 1579 u_int objsize, totsize, wsize; 1580 int i; 1581 1582#ifdef UMA_DEBUG 1583 printf("Creating uma keg headers zone and keg.\n"); 1584#endif 1585 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 1586 1587 /* 1588 * Figure out the maximum number of items-per-slab we'll have if 1589 * we're using the OFFPAGE slab header to track free items, given 1590 * all possible object sizes and the maximum desired wastage 1591 * (UMA_MAX_WASTE). 1592 * 1593 * We iterate until we find an object size for
|
1520 * which the calculated wastage in zone_small_init() will be
| 1594 * which the calculated wastage in keg_small_init() will be
|
1521 * enough to warrant OFFPAGE. Since wastedspace versus objsize 1522 * is an overall increasing see-saw function, we find the smallest 1523 * objsize such that the wastage is always acceptable for objects 1524 * with that objsize or smaller. Since a smaller objsize always 1525 * generates a larger possible uma_max_ipers, we use this computed 1526 * objsize to calculate the largest ipers possible. Since the 1527 * ipers calculated for OFFPAGE slab headers is always larger than
| 1595 * enough to warrant OFFPAGE. Since wastedspace versus objsize 1596 * is an overall increasing see-saw function, we find the smallest 1597 * objsize such that the wastage is always acceptable for objects 1598 * with that objsize or smaller. Since a smaller objsize always 1599 * generates a larger possible uma_max_ipers, we use this computed 1600 * objsize to calculate the largest ipers possible. Since the 1601 * ipers calculated for OFFPAGE slab headers is always larger than
|
1528 * the ipers initially calculated in zone_small_init(), we use
| 1602 * the ipers initially calculated in keg_small_init(), we use
|
1529 * the former's equation (UMA_SLAB_SIZE / keg->uk_rsize) to 1530 * obtain the maximum ipers possible for offpage slab headers. 1531 * 1532 * It should be noted that ipers versus objsize is an inversly 1533 * proportional function which drops off rather quickly so as 1534 * long as our UMA_MAX_WASTE is such that the objsize we calculate 1535 * falls into the portion of the inverse relation AFTER the steep 1536 * falloff, then uma_max_ipers shouldn't be too high (~10 on i386). 1537 * 1538 * Note that we have 8-bits (1 byte) to use as a freelist index 1539 * inside the actual slab header itself and this is enough to 1540 * accomodate us. In the worst case, a UMA_SMALLEST_UNIT sized 1541 * object with offpage slab header would have ipers = 1542 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is 1543 * 1 greater than what our byte-integer freelist index can 1544 * accomodate, but we know that this situation never occurs as 1545 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate 1546 * that we need to go to offpage slab headers. Or, if we do, 1547 * then we trap that condition below and panic in the INVARIANTS case. 1548 */ 1549 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) - UMA_MAX_WASTE; 1550 totsize = wsize; 1551 objsize = UMA_SMALLEST_UNIT; 1552 while (totsize >= wsize) { 1553 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / 1554 (objsize + UMA_FRITM_SZ); 1555 totsize *= (UMA_FRITM_SZ + objsize); 1556 objsize++; 1557 } 1558 if (objsize > UMA_SMALLEST_UNIT) 1559 objsize--;
| 1603 * the former's equation (UMA_SLAB_SIZE / keg->uk_rsize) to 1604 * obtain the maximum ipers possible for offpage slab headers. 1605 * 1606 * It should be noted that ipers versus objsize is an inversly 1607 * proportional function which drops off rather quickly so as 1608 * long as our UMA_MAX_WASTE is such that the objsize we calculate 1609 * falls into the portion of the inverse relation AFTER the steep 1610 * falloff, then uma_max_ipers shouldn't be too high (~10 on i386). 1611 * 1612 * Note that we have 8-bits (1 byte) to use as a freelist index 1613 * inside the actual slab header itself and this is enough to 1614 * accomodate us. In the worst case, a UMA_SMALLEST_UNIT sized 1615 * object with offpage slab header would have ipers = 1616 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is 1617 * 1 greater than what our byte-integer freelist index can 1618 * accomodate, but we know that this situation never occurs as 1619 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate 1620 * that we need to go to offpage slab headers. Or, if we do, 1621 * then we trap that condition below and panic in the INVARIANTS case. 1622 */ 1623 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) - UMA_MAX_WASTE; 1624 totsize = wsize; 1625 objsize = UMA_SMALLEST_UNIT; 1626 while (totsize >= wsize) { 1627 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / 1628 (objsize + UMA_FRITM_SZ); 1629 totsize *= (UMA_FRITM_SZ + objsize); 1630 objsize++; 1631 } 1632 if (objsize > UMA_SMALLEST_UNIT) 1633 objsize--;
|
1560 uma_max_ipers = UMA_SLAB_SIZE / objsize;
| 1634 uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64);
|
1561 1562 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - UMA_MAX_WASTE; 1563 totsize = wsize; 1564 objsize = UMA_SMALLEST_UNIT; 1565 while (totsize >= wsize) { 1566 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) / 1567 (objsize + UMA_FRITMREF_SZ); 1568 totsize *= (UMA_FRITMREF_SZ + objsize); 1569 objsize++; 1570 } 1571 if (objsize > UMA_SMALLEST_UNIT) 1572 objsize--;
| 1635 1636 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - UMA_MAX_WASTE; 1637 totsize = wsize; 1638 objsize = UMA_SMALLEST_UNIT; 1639 while (totsize >= wsize) { 1640 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) / 1641 (objsize + UMA_FRITMREF_SZ); 1642 totsize *= (UMA_FRITMREF_SZ + objsize); 1643 objsize++; 1644 } 1645 if (objsize > UMA_SMALLEST_UNIT) 1646 objsize--;
|
1573 uma_max_ipers_ref = UMA_SLAB_SIZE / objsize;
| 1647 uma_max_ipers_ref = MAX(UMA_SLAB_SIZE / objsize, 64);
|
1574 1575 KASSERT((uma_max_ipers_ref <= 255) && (uma_max_ipers <= 255), 1576 ("uma_startup: calculated uma_max_ipers values too large!")); 1577 1578#ifdef UMA_DEBUG 1579 printf("Calculated uma_max_ipers (for OFFPAGE) is %d\n", uma_max_ipers); 1580 printf("Calculated uma_max_ipers_slab (for OFFPAGE) is %d\n", 1581 uma_max_ipers_ref); 1582#endif 1583 1584 /* "manually" create the initial zone */ 1585 args.name = "UMA Kegs"; 1586 args.size = sizeof(struct uma_keg); 1587 args.ctor = keg_ctor; 1588 args.dtor = keg_dtor; 1589 args.uminit = zero_init; 1590 args.fini = NULL; 1591 args.keg = &masterkeg; 1592 args.align = 32 - 1; 1593 args.flags = UMA_ZFLAG_INTERNAL; 1594 /* The initial zone has no Per cpu queues so it's smaller */ 1595 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK); 1596 1597#ifdef UMA_DEBUG 1598 printf("Filling boot free list.\n"); 1599#endif 1600 for (i = 0; i < boot_pages; i++) { 1601 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 1602 slab->us_data = (u_int8_t *)slab; 1603 slab->us_flags = UMA_SLAB_BOOT; 1604 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 1605 } 1606 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); 1607 1608#ifdef UMA_DEBUG 1609 printf("Creating uma zone headers zone and keg.\n"); 1610#endif 1611 args.name = "UMA Zones"; 1612 args.size = sizeof(struct uma_zone) + 1613 (sizeof(struct uma_cache) * (mp_maxid + 1)); 1614 args.ctor = zone_ctor; 1615 args.dtor = zone_dtor; 1616 args.uminit = zero_init; 1617 args.fini = NULL; 1618 args.keg = NULL; 1619 args.align = 32 - 1; 1620 args.flags = UMA_ZFLAG_INTERNAL; 1621 /* The initial zone has no Per cpu queues so it's smaller */ 1622 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK); 1623 1624#ifdef UMA_DEBUG 1625 printf("Initializing pcpu cache locks.\n"); 1626#endif 1627#ifdef UMA_DEBUG 1628 printf("Creating slab and hash zones.\n"); 1629#endif 1630 1631 /* 1632 * This is the max number of free list items we'll have with 1633 * offpage slabs. 1634 */ 1635 slabsize = uma_max_ipers * UMA_FRITM_SZ; 1636 slabsize += sizeof(struct uma_slab); 1637 1638 /* Now make a zone for slab headers */ 1639 slabzone = uma_zcreate("UMA Slabs", 1640 slabsize, 1641 NULL, NULL, NULL, NULL, 1642 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1643 1644 /* 1645 * We also create a zone for the bigger slabs with reference 1646 * counts in them, to accomodate UMA_ZONE_REFCNT zones. 1647 */ 1648 slabsize = uma_max_ipers_ref * UMA_FRITMREF_SZ; 1649 slabsize += sizeof(struct uma_slab_refcnt); 1650 slabrefzone = uma_zcreate("UMA RCntSlabs", 1651 slabsize, 1652 NULL, NULL, NULL, NULL, 1653 UMA_ALIGN_PTR, 1654 UMA_ZFLAG_INTERNAL); 1655 1656 hashzone = uma_zcreate("UMA Hash", 1657 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 1658 NULL, NULL, NULL, NULL, 1659 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1660 1661 bucket_init(); 1662 1663#if defined(UMA_MD_SMALL_ALLOC) && !defined(UMA_MD_SMALL_ALLOC_NEEDS_VM) 1664 booted = 1; 1665#endif 1666 1667#ifdef UMA_DEBUG 1668 printf("UMA startup complete.\n"); 1669#endif 1670} 1671 1672/* see uma.h */ 1673void 1674uma_startup2(void) 1675{ 1676 booted = 1; 1677 bucket_enable(); 1678#ifdef UMA_DEBUG 1679 printf("UMA startup2 complete.\n"); 1680#endif 1681} 1682 1683/* 1684 * Initialize our callout handle 1685 * 1686 */ 1687 1688static void 1689uma_startup3(void) 1690{ 1691#ifdef UMA_DEBUG 1692 printf("Starting callout.\n"); 1693#endif 1694 callout_init(&uma_callout, CALLOUT_MPSAFE); 1695 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 1696#ifdef UMA_DEBUG 1697 printf("UMA startup3 complete.\n"); 1698#endif 1699} 1700
| 1648 1649 KASSERT((uma_max_ipers_ref <= 255) && (uma_max_ipers <= 255), 1650 ("uma_startup: calculated uma_max_ipers values too large!")); 1651 1652#ifdef UMA_DEBUG 1653 printf("Calculated uma_max_ipers (for OFFPAGE) is %d\n", uma_max_ipers); 1654 printf("Calculated uma_max_ipers_slab (for OFFPAGE) is %d\n", 1655 uma_max_ipers_ref); 1656#endif 1657 1658 /* "manually" create the initial zone */ 1659 args.name = "UMA Kegs"; 1660 args.size = sizeof(struct uma_keg); 1661 args.ctor = keg_ctor; 1662 args.dtor = keg_dtor; 1663 args.uminit = zero_init; 1664 args.fini = NULL; 1665 args.keg = &masterkeg; 1666 args.align = 32 - 1; 1667 args.flags = UMA_ZFLAG_INTERNAL; 1668 /* The initial zone has no Per cpu queues so it's smaller */ 1669 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK); 1670 1671#ifdef UMA_DEBUG 1672 printf("Filling boot free list.\n"); 1673#endif 1674 for (i = 0; i < boot_pages; i++) { 1675 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 1676 slab->us_data = (u_int8_t *)slab; 1677 slab->us_flags = UMA_SLAB_BOOT; 1678 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 1679 } 1680 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); 1681 1682#ifdef UMA_DEBUG 1683 printf("Creating uma zone headers zone and keg.\n"); 1684#endif 1685 args.name = "UMA Zones"; 1686 args.size = sizeof(struct uma_zone) + 1687 (sizeof(struct uma_cache) * (mp_maxid + 1)); 1688 args.ctor = zone_ctor; 1689 args.dtor = zone_dtor; 1690 args.uminit = zero_init; 1691 args.fini = NULL; 1692 args.keg = NULL; 1693 args.align = 32 - 1; 1694 args.flags = UMA_ZFLAG_INTERNAL; 1695 /* The initial zone has no Per cpu queues so it's smaller */ 1696 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK); 1697 1698#ifdef UMA_DEBUG 1699 printf("Initializing pcpu cache locks.\n"); 1700#endif 1701#ifdef UMA_DEBUG 1702 printf("Creating slab and hash zones.\n"); 1703#endif 1704 1705 /* 1706 * This is the max number of free list items we'll have with 1707 * offpage slabs. 1708 */ 1709 slabsize = uma_max_ipers * UMA_FRITM_SZ; 1710 slabsize += sizeof(struct uma_slab); 1711 1712 /* Now make a zone for slab headers */ 1713 slabzone = uma_zcreate("UMA Slabs", 1714 slabsize, 1715 NULL, NULL, NULL, NULL, 1716 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1717 1718 /* 1719 * We also create a zone for the bigger slabs with reference 1720 * counts in them, to accomodate UMA_ZONE_REFCNT zones. 1721 */ 1722 slabsize = uma_max_ipers_ref * UMA_FRITMREF_SZ; 1723 slabsize += sizeof(struct uma_slab_refcnt); 1724 slabrefzone = uma_zcreate("UMA RCntSlabs", 1725 slabsize, 1726 NULL, NULL, NULL, NULL, 1727 UMA_ALIGN_PTR, 1728 UMA_ZFLAG_INTERNAL); 1729 1730 hashzone = uma_zcreate("UMA Hash", 1731 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 1732 NULL, NULL, NULL, NULL, 1733 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1734 1735 bucket_init(); 1736 1737#if defined(UMA_MD_SMALL_ALLOC) && !defined(UMA_MD_SMALL_ALLOC_NEEDS_VM) 1738 booted = 1; 1739#endif 1740 1741#ifdef UMA_DEBUG 1742 printf("UMA startup complete.\n"); 1743#endif 1744} 1745 1746/* see uma.h */ 1747void 1748uma_startup2(void) 1749{ 1750 booted = 1; 1751 bucket_enable(); 1752#ifdef UMA_DEBUG 1753 printf("UMA startup2 complete.\n"); 1754#endif 1755} 1756 1757/* 1758 * Initialize our callout handle 1759 * 1760 */ 1761 1762static void 1763uma_startup3(void) 1764{ 1765#ifdef UMA_DEBUG 1766 printf("Starting callout.\n"); 1767#endif 1768 callout_init(&uma_callout, CALLOUT_MPSAFE); 1769 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 1770#ifdef UMA_DEBUG 1771 printf("UMA startup3 complete.\n"); 1772#endif 1773} 1774
|
1701static uma_zone_t
| 1775static uma_keg_t
|
1702uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 1703 int align, u_int32_t flags) 1704{ 1705 struct uma_kctor_args args; 1706 1707 args.size = size; 1708 args.uminit = uminit; 1709 args.fini = fini; 1710 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 1711 args.flags = flags; 1712 args.zone = zone;
| 1776uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 1777 int align, u_int32_t flags) 1778{ 1779 struct uma_kctor_args args; 1780 1781 args.size = size; 1782 args.uminit = uminit; 1783 args.fini = fini; 1784 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 1785 args.flags = flags; 1786 args.zone = zone;
|
1713 return (uma_zalloc_internal(kegs, &args, M_WAITOK));
| 1787 return (zone_alloc_item(kegs, &args, M_WAITOK));
|
1714} 1715 1716/* See uma.h */ 1717void 1718uma_set_align(int align) 1719{ 1720 1721 if (align != UMA_ALIGN_CACHE) 1722 uma_align_cache = align; 1723} 1724 1725/* See uma.h */ 1726uma_zone_t 1727uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1728 uma_init uminit, uma_fini fini, int align, u_int32_t flags) 1729 1730{ 1731 struct uma_zctor_args args; 1732 1733 /* This stuff is essential for the zone ctor */ 1734 args.name = name; 1735 args.size = size; 1736 args.ctor = ctor; 1737 args.dtor = dtor; 1738 args.uminit = uminit; 1739 args.fini = fini; 1740 args.align = align; 1741 args.flags = flags; 1742 args.keg = NULL; 1743
| 1788} 1789 1790/* See uma.h */ 1791void 1792uma_set_align(int align) 1793{ 1794 1795 if (align != UMA_ALIGN_CACHE) 1796 uma_align_cache = align; 1797} 1798 1799/* See uma.h */ 1800uma_zone_t 1801uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1802 uma_init uminit, uma_fini fini, int align, u_int32_t flags) 1803 1804{ 1805 struct uma_zctor_args args; 1806 1807 /* This stuff is essential for the zone ctor */ 1808 args.name = name; 1809 args.size = size; 1810 args.ctor = ctor; 1811 args.dtor = dtor; 1812 args.uminit = uminit; 1813 args.fini = fini; 1814 args.align = align; 1815 args.flags = flags; 1816 args.keg = NULL; 1817
|
1744 return (uma_zalloc_internal(zones, &args, M_WAITOK));
| 1818 return (zone_alloc_item(zones, &args, M_WAITOK));
|
1745} 1746 1747/* See uma.h */ 1748uma_zone_t 1749uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 1750 uma_init zinit, uma_fini zfini, uma_zone_t master) 1751{ 1752 struct uma_zctor_args args;
| 1819} 1820 1821/* See uma.h */ 1822uma_zone_t 1823uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 1824 uma_init zinit, uma_fini zfini, uma_zone_t master) 1825{ 1826 struct uma_zctor_args args;
|
| 1827 uma_keg_t keg;
|
1753
| 1828
|
| 1829 keg = zone_first_keg(master);
|
1754 args.name = name;
| 1830 args.name = name;
|
1755 args.size = master->uz_keg->uk_size;
| 1831 args.size = keg->uk_size;
|
1756 args.ctor = ctor; 1757 args.dtor = dtor; 1758 args.uminit = zinit; 1759 args.fini = zfini;
| 1832 args.ctor = ctor; 1833 args.dtor = dtor; 1834 args.uminit = zinit; 1835 args.fini = zfini;
|
1760 args.align = master->uz_keg->uk_align; 1761 args.flags = master->uz_keg->uk_flags | UMA_ZONE_SECONDARY; 1762 args.keg = master->uz_keg;
| 1836 args.align = keg->uk_align; 1837 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 1838 args.keg = keg;
|
1763
| 1839
|
1764 return (uma_zalloc_internal(zones, &args, M_WAITOK));
| 1840 /* XXX Attaches only one keg of potentially many. */ 1841 return (zone_alloc_item(zones, &args, M_WAITOK));
|
1765} 1766
| 1842} 1843
|
| 1844static void 1845zone_lock_pair(uma_zone_t a, uma_zone_t b) 1846{ 1847 if (a < b) { 1848 ZONE_LOCK(a); 1849 mtx_lock_flags(b->uz_lock, MTX_DUPOK); 1850 } else { 1851 ZONE_LOCK(b); 1852 mtx_lock_flags(a->uz_lock, MTX_DUPOK); 1853 } 1854} 1855 1856static void 1857zone_unlock_pair(uma_zone_t a, uma_zone_t b) 1858{ 1859 1860 ZONE_UNLOCK(a); 1861 ZONE_UNLOCK(b); 1862} 1863 1864int 1865uma_zsecond_add(uma_zone_t zone, uma_zone_t master) 1866{ 1867 uma_klink_t klink; 1868 uma_klink_t kl; 1869 int error; 1870 1871 error = 0; 1872 klink = malloc(sizeof(*klink), M_TEMP, M_WAITOK | M_ZERO); 1873 1874 zone_lock_pair(zone, master); 1875 /* 1876 * zone must use vtoslab() to resolve objects and must already be 1877 * a secondary. 1878 */ 1879 if ((zone->uz_flags & (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) 1880 != (UMA_ZONE_VTOSLAB | UMA_ZONE_SECONDARY)) { 1881 error = EINVAL; 1882 goto out; 1883 } 1884 /* 1885 * The new master must also use vtoslab(). 1886 */ 1887 if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) { 1888 error = EINVAL; 1889 goto out; 1890 } 1891 /* 1892 * Both must either be refcnt, or not be refcnt. 1893 */ 1894 if ((zone->uz_flags & UMA_ZONE_REFCNT) != 1895 (master->uz_flags & UMA_ZONE_REFCNT)) { 1896 error = EINVAL; 1897 goto out; 1898 } 1899 /* 1900 * The underlying object must be the same size. rsize 1901 * may be different. 1902 */ 1903 if (master->uz_size != zone->uz_size) { 1904 error = E2BIG; 1905 goto out; 1906 } 1907 /* 1908 * Put it at the end of the list. 1909 */ 1910 klink->kl_keg = zone_first_keg(master); 1911 LIST_FOREACH(kl, &zone->uz_kegs, kl_link) { 1912 if (LIST_NEXT(kl, kl_link) == NULL) { 1913 LIST_INSERT_AFTER(kl, klink, kl_link); 1914 break; 1915 } 1916 } 1917 klink = NULL; 1918 zone->uz_flags |= UMA_ZFLAG_MULTI; 1919 zone->uz_slab = zone_fetch_slab_multi; 1920 1921out: 1922 zone_unlock_pair(zone, master); 1923 if (klink != NULL) 1924 free(klink, M_TEMP); 1925 1926 return (error); 1927} 1928 1929
|
1767/* See uma.h */ 1768void 1769uma_zdestroy(uma_zone_t zone) 1770{ 1771
| 1930/* See uma.h */ 1931void 1932uma_zdestroy(uma_zone_t zone) 1933{ 1934
|
1772 uma_zfree_internal(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
| 1935 zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
|
1773} 1774 1775/* See uma.h */ 1776void * 1777uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1778{ 1779 void *item; 1780 uma_cache_t cache; 1781 uma_bucket_t bucket; 1782 int cpu; 1783 1784 /* This is the fast path allocation */ 1785#ifdef UMA_DEBUG_ALLOC_1 1786 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 1787#endif 1788 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread, 1789 zone->uz_name, flags); 1790 1791 if (flags & M_WAITOK) { 1792 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 1793 "uma_zalloc_arg: zone \"%s\"", zone->uz_name); 1794 } 1795 1796 /* 1797 * If possible, allocate from the per-CPU cache. There are two 1798 * requirements for safe access to the per-CPU cache: (1) the thread 1799 * accessing the cache must not be preempted or yield during access, 1800 * and (2) the thread must not migrate CPUs without switching which 1801 * cache it accesses. We rely on a critical section to prevent 1802 * preemption and migration. We release the critical section in 1803 * order to acquire the zone mutex if we are unable to allocate from 1804 * the current cache; when we re-acquire the critical section, we 1805 * must detect and handle migration if it has occurred. 1806 */ 1807zalloc_restart: 1808 critical_enter(); 1809 cpu = curcpu; 1810 cache = &zone->uz_cpu[cpu]; 1811 1812zalloc_start: 1813 bucket = cache->uc_allocbucket; 1814 1815 if (bucket) { 1816 if (bucket->ub_cnt > 0) { 1817 bucket->ub_cnt--; 1818 item = bucket->ub_bucket[bucket->ub_cnt]; 1819#ifdef INVARIANTS 1820 bucket->ub_bucket[bucket->ub_cnt] = NULL; 1821#endif 1822 KASSERT(item != NULL, 1823 ("uma_zalloc: Bucket pointer mangled.")); 1824 cache->uc_allocs++; 1825 critical_exit(); 1826#ifdef INVARIANTS 1827 ZONE_LOCK(zone); 1828 uma_dbg_alloc(zone, NULL, item); 1829 ZONE_UNLOCK(zone); 1830#endif 1831 if (zone->uz_ctor != NULL) {
| 1936} 1937 1938/* See uma.h */ 1939void * 1940uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1941{ 1942 void *item; 1943 uma_cache_t cache; 1944 uma_bucket_t bucket; 1945 int cpu; 1946 1947 /* This is the fast path allocation */ 1948#ifdef UMA_DEBUG_ALLOC_1 1949 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 1950#endif 1951 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread, 1952 zone->uz_name, flags); 1953 1954 if (flags & M_WAITOK) { 1955 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 1956 "uma_zalloc_arg: zone \"%s\"", zone->uz_name); 1957 } 1958 1959 /* 1960 * If possible, allocate from the per-CPU cache. There are two 1961 * requirements for safe access to the per-CPU cache: (1) the thread 1962 * accessing the cache must not be preempted or yield during access, 1963 * and (2) the thread must not migrate CPUs without switching which 1964 * cache it accesses. We rely on a critical section to prevent 1965 * preemption and migration. We release the critical section in 1966 * order to acquire the zone mutex if we are unable to allocate from 1967 * the current cache; when we re-acquire the critical section, we 1968 * must detect and handle migration if it has occurred. 1969 */ 1970zalloc_restart: 1971 critical_enter(); 1972 cpu = curcpu; 1973 cache = &zone->uz_cpu[cpu]; 1974 1975zalloc_start: 1976 bucket = cache->uc_allocbucket; 1977 1978 if (bucket) { 1979 if (bucket->ub_cnt > 0) { 1980 bucket->ub_cnt--; 1981 item = bucket->ub_bucket[bucket->ub_cnt]; 1982#ifdef INVARIANTS 1983 bucket->ub_bucket[bucket->ub_cnt] = NULL; 1984#endif 1985 KASSERT(item != NULL, 1986 ("uma_zalloc: Bucket pointer mangled.")); 1987 cache->uc_allocs++; 1988 critical_exit(); 1989#ifdef INVARIANTS 1990 ZONE_LOCK(zone); 1991 uma_dbg_alloc(zone, NULL, item); 1992 ZONE_UNLOCK(zone); 1993#endif 1994 if (zone->uz_ctor != NULL) {
|
1832 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
| 1995 if (zone->uz_ctor(item, zone->uz_size,
|
1833 udata, flags) != 0) {
| 1996 udata, flags) != 0) {
|
1834 uma_zfree_internal(zone, item, udata,
| 1997 zone_free_item(zone, item, udata,
|
1835 SKIP_DTOR, ZFREE_STATFAIL | 1836 ZFREE_STATFREE); 1837 return (NULL); 1838 } 1839 } 1840 if (flags & M_ZERO)
| 1998 SKIP_DTOR, ZFREE_STATFAIL | 1999 ZFREE_STATFREE); 2000 return (NULL); 2001 } 2002 } 2003 if (flags & M_ZERO)
|
1841 bzero(item, zone->uz_keg->uk_size);
| 2004 bzero(item, zone->uz_size);
|
1842 return (item); 1843 } else if (cache->uc_freebucket) { 1844 /* 1845 * We have run out of items in our allocbucket. 1846 * See if we can switch with our free bucket. 1847 */ 1848 if (cache->uc_freebucket->ub_cnt > 0) { 1849#ifdef UMA_DEBUG_ALLOC 1850 printf("uma_zalloc: Swapping empty with" 1851 " alloc.\n"); 1852#endif 1853 bucket = cache->uc_freebucket; 1854 cache->uc_freebucket = cache->uc_allocbucket; 1855 cache->uc_allocbucket = bucket; 1856 1857 goto zalloc_start; 1858 } 1859 } 1860 } 1861 /* 1862 * Attempt to retrieve the item from the per-CPU cache has failed, so 1863 * we must go back to the zone. This requires the zone lock, so we 1864 * must drop the critical section, then re-acquire it when we go back 1865 * to the cache. Since the critical section is released, we may be 1866 * preempted or migrate. As such, make sure not to maintain any 1867 * thread-local state specific to the cache from prior to releasing 1868 * the critical section. 1869 */ 1870 critical_exit(); 1871 ZONE_LOCK(zone); 1872 critical_enter(); 1873 cpu = curcpu; 1874 cache = &zone->uz_cpu[cpu]; 1875 bucket = cache->uc_allocbucket; 1876 if (bucket != NULL) { 1877 if (bucket->ub_cnt > 0) { 1878 ZONE_UNLOCK(zone); 1879 goto zalloc_start; 1880 } 1881 bucket = cache->uc_freebucket; 1882 if (bucket != NULL && bucket->ub_cnt > 0) { 1883 ZONE_UNLOCK(zone); 1884 goto zalloc_start; 1885 } 1886 } 1887 1888 /* Since we have locked the zone we may as well send back our stats */ 1889 zone->uz_allocs += cache->uc_allocs; 1890 cache->uc_allocs = 0; 1891 zone->uz_frees += cache->uc_frees; 1892 cache->uc_frees = 0; 1893 1894 /* Our old one is now a free bucket */ 1895 if (cache->uc_allocbucket) { 1896 KASSERT(cache->uc_allocbucket->ub_cnt == 0, 1897 ("uma_zalloc_arg: Freeing a non free bucket.")); 1898 LIST_INSERT_HEAD(&zone->uz_free_bucket, 1899 cache->uc_allocbucket, ub_link); 1900 cache->uc_allocbucket = NULL; 1901 } 1902 1903 /* Check the free list for a new alloc bucket */ 1904 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 1905 KASSERT(bucket->ub_cnt != 0, 1906 ("uma_zalloc_arg: Returning an empty bucket.")); 1907 1908 LIST_REMOVE(bucket, ub_link); 1909 cache->uc_allocbucket = bucket; 1910 ZONE_UNLOCK(zone); 1911 goto zalloc_start; 1912 } 1913 /* We are no longer associated with this CPU. */ 1914 critical_exit(); 1915 1916 /* Bump up our uz_count so we get here less */ 1917 if (zone->uz_count < BUCKET_MAX) 1918 zone->uz_count++; 1919 1920 /* 1921 * Now lets just fill a bucket and put it on the free list. If that 1922 * works we'll restart the allocation from the begining. 1923 */
| 2005 return (item); 2006 } else if (cache->uc_freebucket) { 2007 /* 2008 * We have run out of items in our allocbucket. 2009 * See if we can switch with our free bucket. 2010 */ 2011 if (cache->uc_freebucket->ub_cnt > 0) { 2012#ifdef UMA_DEBUG_ALLOC 2013 printf("uma_zalloc: Swapping empty with" 2014 " alloc.\n"); 2015#endif 2016 bucket = cache->uc_freebucket; 2017 cache->uc_freebucket = cache->uc_allocbucket; 2018 cache->uc_allocbucket = bucket; 2019 2020 goto zalloc_start; 2021 } 2022 } 2023 } 2024 /* 2025 * Attempt to retrieve the item from the per-CPU cache has failed, so 2026 * we must go back to the zone. This requires the zone lock, so we 2027 * must drop the critical section, then re-acquire it when we go back 2028 * to the cache. Since the critical section is released, we may be 2029 * preempted or migrate. As such, make sure not to maintain any 2030 * thread-local state specific to the cache from prior to releasing 2031 * the critical section. 2032 */ 2033 critical_exit(); 2034 ZONE_LOCK(zone); 2035 critical_enter(); 2036 cpu = curcpu; 2037 cache = &zone->uz_cpu[cpu]; 2038 bucket = cache->uc_allocbucket; 2039 if (bucket != NULL) { 2040 if (bucket->ub_cnt > 0) { 2041 ZONE_UNLOCK(zone); 2042 goto zalloc_start; 2043 } 2044 bucket = cache->uc_freebucket; 2045 if (bucket != NULL && bucket->ub_cnt > 0) { 2046 ZONE_UNLOCK(zone); 2047 goto zalloc_start; 2048 } 2049 } 2050 2051 /* Since we have locked the zone we may as well send back our stats */ 2052 zone->uz_allocs += cache->uc_allocs; 2053 cache->uc_allocs = 0; 2054 zone->uz_frees += cache->uc_frees; 2055 cache->uc_frees = 0; 2056 2057 /* Our old one is now a free bucket */ 2058 if (cache->uc_allocbucket) { 2059 KASSERT(cache->uc_allocbucket->ub_cnt == 0, 2060 ("uma_zalloc_arg: Freeing a non free bucket.")); 2061 LIST_INSERT_HEAD(&zone->uz_free_bucket, 2062 cache->uc_allocbucket, ub_link); 2063 cache->uc_allocbucket = NULL; 2064 } 2065 2066 /* Check the free list for a new alloc bucket */ 2067 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 2068 KASSERT(bucket->ub_cnt != 0, 2069 ("uma_zalloc_arg: Returning an empty bucket.")); 2070 2071 LIST_REMOVE(bucket, ub_link); 2072 cache->uc_allocbucket = bucket; 2073 ZONE_UNLOCK(zone); 2074 goto zalloc_start; 2075 } 2076 /* We are no longer associated with this CPU. */ 2077 critical_exit(); 2078 2079 /* Bump up our uz_count so we get here less */ 2080 if (zone->uz_count < BUCKET_MAX) 2081 zone->uz_count++; 2082 2083 /* 2084 * Now lets just fill a bucket and put it on the free list. If that 2085 * works we'll restart the allocation from the begining. 2086 */
|
1924 if (uma_zalloc_bucket(zone, flags)) {
| 2087 if (zone_alloc_bucket(zone, flags)) {
|
1925 ZONE_UNLOCK(zone); 1926 goto zalloc_restart; 1927 } 1928 ZONE_UNLOCK(zone); 1929 /* 1930 * We may not be able to get a bucket so return an actual item. 1931 */ 1932#ifdef UMA_DEBUG 1933 printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 1934#endif 1935
| 2088 ZONE_UNLOCK(zone); 2089 goto zalloc_restart; 2090 } 2091 ZONE_UNLOCK(zone); 2092 /* 2093 * We may not be able to get a bucket so return an actual item. 2094 */ 2095#ifdef UMA_DEBUG 2096 printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 2097#endif 2098
|
1936 return (uma_zalloc_internal(zone, udata, flags));
| 2099 item = zone_alloc_item(zone, udata, flags); 2100 return (item);
|
1937} 1938 1939static uma_slab_t
| 2101} 2102 2103static uma_slab_t
|
1940uma_zone_slab(uma_zone_t zone, int flags)
| 2104keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int flags)
|
1941{ 1942 uma_slab_t slab;
| 2105{ 2106 uma_slab_t slab;
|
1943 uma_keg_t keg;
| |
1944
| 2107
|
1945 keg = zone->uz_keg; 1946 1947 /* 1948 * This is to prevent us from recursively trying to allocate 1949 * buckets. The problem is that if an allocation forces us to 1950 * grab a new bucket we will call page_alloc, which will go off 1951 * and cause the vm to allocate vm_map_entries. If we need new 1952 * buckets there too we will recurse in kmem_alloc and bad 1953 * things happen. So instead we return a NULL bucket, and make 1954 * the code that allocates buckets smart enough to deal with it 1955 * 1956 * XXX: While we want this protection for the bucket zones so that 1957 * recursion from the VM is handled (and the calling code that 1958 * allocates buckets knows how to deal with it), we do not want 1959 * to prevent allocation from the slab header zones (slabzone 1960 * and slabrefzone) if uk_recurse is not zero for them. The 1961 * reason is that it could lead to NULL being returned for 1962 * slab header allocations even in the M_WAITOK case, and the 1963 * caller can't handle that. 1964 */ 1965 if (keg->uk_flags & UMA_ZFLAG_INTERNAL && keg->uk_recurse != 0) 1966 if (zone != slabzone && zone != slabrefzone && zone != zones) 1967 return (NULL); 1968
| 2108 mtx_assert(&keg->uk_lock, MA_OWNED);
|
1969 slab = NULL; 1970 1971 for (;;) { 1972 /* 1973 * Find a slab with some space. Prefer slabs that are partially 1974 * used over those that are totally full. This helps to reduce 1975 * fragmentation. 1976 */ 1977 if (keg->uk_free != 0) { 1978 if (!LIST_EMPTY(&keg->uk_part_slab)) { 1979 slab = LIST_FIRST(&keg->uk_part_slab); 1980 } else { 1981 slab = LIST_FIRST(&keg->uk_free_slab); 1982 LIST_REMOVE(slab, us_link); 1983 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, 1984 us_link); 1985 }
| 2109 slab = NULL; 2110 2111 for (;;) { 2112 /* 2113 * Find a slab with some space. Prefer slabs that are partially 2114 * used over those that are totally full. This helps to reduce 2115 * fragmentation. 2116 */ 2117 if (keg->uk_free != 0) { 2118 if (!LIST_EMPTY(&keg->uk_part_slab)) { 2119 slab = LIST_FIRST(&keg->uk_part_slab); 2120 } else { 2121 slab = LIST_FIRST(&keg->uk_free_slab); 2122 LIST_REMOVE(slab, us_link); 2123 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, 2124 us_link); 2125 }
|
| 2126 MPASS(slab->us_keg == keg);
|
1986 return (slab); 1987 } 1988 1989 /* 1990 * M_NOVM means don't ask at all! 1991 */ 1992 if (flags & M_NOVM) 1993 break; 1994
| 2127 return (slab); 2128 } 2129 2130 /* 2131 * M_NOVM means don't ask at all! 2132 */ 2133 if (flags & M_NOVM) 2134 break; 2135
|
1995 if (keg->uk_maxpages && 1996 keg->uk_pages >= keg->uk_maxpages) {
| 2136 if (keg->uk_maxpages && keg->uk_pages >= keg->uk_maxpages) {
|
1997 keg->uk_flags |= UMA_ZFLAG_FULL;
| 2137 keg->uk_flags |= UMA_ZFLAG_FULL;
|
1998
| 2138 /* 2139 * If this is not a multi-zone, set the FULL bit. 2140 * Otherwise slab_multi() takes care of it. 2141 */ 2142 if ((zone->uz_flags & UMA_ZFLAG_MULTI) == 0) 2143 zone->uz_flags |= UMA_ZFLAG_FULL;
|
1999 if (flags & M_NOWAIT) 2000 break;
| 2144 if (flags & M_NOWAIT) 2145 break;
|
2001 else 2002 msleep(keg, &keg->uk_lock, PVM, 2003 "zonelimit", 0);
| 2146 msleep(keg, &keg->uk_lock, PVM, "keglimit", 0);
|
2004 continue; 2005 } 2006 keg->uk_recurse++;
| 2147 continue; 2148 } 2149 keg->uk_recurse++;
|
2007 slab = slab_zalloc(zone, flags);
| 2150 slab = keg_alloc_slab(keg, zone, flags);
|
2008 keg->uk_recurse--;
| 2151 keg->uk_recurse--;
|
2009
| |
2010 /* 2011 * If we got a slab here it's safe to mark it partially used 2012 * and return. We assume that the caller is going to remove 2013 * at least one item. 2014 */ 2015 if (slab) {
| 2152 /* 2153 * If we got a slab here it's safe to mark it partially used 2154 * and return. We assume that the caller is going to remove 2155 * at least one item. 2156 */ 2157 if (slab) {
|
| 2158 MPASS(slab->us_keg == keg);
|
2016 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2017 return (slab); 2018 } 2019 /* 2020 * We might not have been able to get a slab but another cpu 2021 * could have while we were unlocked. Check again before we 2022 * fail. 2023 */
| 2159 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2160 return (slab); 2161 } 2162 /* 2163 * We might not have been able to get a slab but another cpu 2164 * could have while we were unlocked. Check again before we 2165 * fail. 2166 */
|
2024 if (flags & M_NOWAIT) 2025 flags |= M_NOVM;
| 2167 flags |= M_NOVM;
|
2026 } 2027 return (slab); 2028} 2029
| 2168 } 2169 return (slab); 2170} 2171
|
| 2172static inline void 2173zone_relock(uma_zone_t zone, uma_keg_t keg) 2174{ 2175 if (zone->uz_lock != &keg->uk_lock) { 2176 KEG_UNLOCK(keg); 2177 ZONE_LOCK(zone); 2178 } 2179} 2180 2181static inline void 2182keg_relock(uma_keg_t keg, uma_zone_t zone) 2183{ 2184 if (zone->uz_lock != &keg->uk_lock) { 2185 ZONE_UNLOCK(zone); 2186 KEG_LOCK(keg); 2187 } 2188} 2189 2190static uma_slab_t 2191zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags) 2192{ 2193 uma_slab_t slab; 2194 2195 if (keg == NULL) 2196 keg = zone_first_keg(zone); 2197 /* 2198 * This is to prevent us from recursively trying to allocate 2199 * buckets. The problem is that if an allocation forces us to 2200 * grab a new bucket we will call page_alloc, which will go off 2201 * and cause the vm to allocate vm_map_entries. If we need new 2202 * buckets there too we will recurse in kmem_alloc and bad 2203 * things happen. So instead we return a NULL bucket, and make 2204 * the code that allocates buckets smart enough to deal with it 2205 */ 2206 if (keg->uk_flags & UMA_ZFLAG_BUCKET && keg->uk_recurse != 0) 2207 return (NULL); 2208 2209 for (;;) { 2210 slab = keg_fetch_slab(keg, zone, flags); 2211 if (slab) 2212 return (slab); 2213 if (flags & (M_NOWAIT | M_NOVM)) 2214 break; 2215 } 2216 return (NULL); 2217} 2218 2219/* 2220 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns 2221 * with the keg locked. Caller must call zone_relock() afterwards if the 2222 * zone lock is required. On NULL the zone lock is held. 2223 * 2224 * The last pointer is used to seed the search. It is not required. 2225 */ 2226static uma_slab_t 2227zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags) 2228{ 2229 uma_klink_t klink; 2230 uma_slab_t slab; 2231 uma_keg_t keg; 2232 int flags; 2233 int empty; 2234 int full; 2235 2236 /* 2237 * Don't wait on the first pass. This will skip limit tests 2238 * as well. We don't want to block if we can find a provider 2239 * without blocking. 2240 */ 2241 flags = (rflags & ~M_WAITOK) | M_NOWAIT; 2242 /* 2243 * Use the last slab allocated as a hint for where to start 2244 * the search. 2245 */ 2246 if (last) { 2247 slab = keg_fetch_slab(last, zone, flags); 2248 if (slab) 2249 return (slab); 2250 zone_relock(zone, last); 2251 last = NULL; 2252 } 2253 /* 2254 * Loop until we have a slab incase of transient failures 2255 * while M_WAITOK is specified. I'm not sure this is 100% 2256 * required but we've done it for so long now. 2257 */ 2258 for (;;) { 2259 empty = 0; 2260 full = 0; 2261 /* 2262 * Search the available kegs for slabs. Be careful to hold the 2263 * correct lock while calling into the keg layer. 2264 */ 2265 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) { 2266 keg = klink->kl_keg; 2267 keg_relock(keg, zone); 2268 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) { 2269 slab = keg_fetch_slab(keg, zone, flags); 2270 if (slab) 2271 return (slab); 2272 } 2273 if (keg->uk_flags & UMA_ZFLAG_FULL) 2274 full++; 2275 else 2276 empty++; 2277 zone_relock(zone, keg); 2278 } 2279 if (rflags & (M_NOWAIT | M_NOVM)) 2280 break; 2281 flags = rflags; 2282 /* 2283 * All kegs are full. XXX We can't atomically check all kegs 2284 * and sleep so just sleep for a short period and retry. 2285 */ 2286 if (full && !empty) { 2287 zone->uz_flags |= UMA_ZFLAG_FULL; 2288 msleep(zone, zone->uz_lock, PVM, "zonelimit", hz/100); 2289 zone->uz_flags &= ~UMA_ZFLAG_FULL; 2290 continue; 2291 } 2292 } 2293 return (NULL); 2294} 2295
|
2030static void *
| 2296static void *
|
2031uma_slab_alloc(uma_zone_t zone, uma_slab_t slab)
| 2297slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
|
2032{ 2033 uma_keg_t keg; 2034 uma_slabrefcnt_t slabref; 2035 void *item; 2036 u_int8_t freei; 2037
| 2298{ 2299 uma_keg_t keg; 2300 uma_slabrefcnt_t slabref; 2301 void *item; 2302 u_int8_t freei; 2303
|
2038 keg = zone->uz_keg;
| 2304 keg = slab->us_keg; 2305 mtx_assert(&keg->uk_lock, MA_OWNED);
|
2039 2040 freei = slab->us_firstfree; 2041 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2042 slabref = (uma_slabrefcnt_t)slab; 2043 slab->us_firstfree = slabref->us_freelist[freei].us_item; 2044 } else { 2045 slab->us_firstfree = slab->us_freelist[freei].us_item; 2046 } 2047 item = slab->us_data + (keg->uk_rsize * freei); 2048 2049 slab->us_freecount--; 2050 keg->uk_free--; 2051#ifdef INVARIANTS 2052 uma_dbg_alloc(zone, slab, item); 2053#endif 2054 /* Move this slab to the full list */ 2055 if (slab->us_freecount == 0) { 2056 LIST_REMOVE(slab, us_link); 2057 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link); 2058 } 2059 2060 return (item); 2061} 2062 2063static int
| 2306 2307 freei = slab->us_firstfree; 2308 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2309 slabref = (uma_slabrefcnt_t)slab; 2310 slab->us_firstfree = slabref->us_freelist[freei].us_item; 2311 } else { 2312 slab->us_firstfree = slab->us_freelist[freei].us_item; 2313 } 2314 item = slab->us_data + (keg->uk_rsize * freei); 2315 2316 slab->us_freecount--; 2317 keg->uk_free--; 2318#ifdef INVARIANTS 2319 uma_dbg_alloc(zone, slab, item); 2320#endif 2321 /* Move this slab to the full list */ 2322 if (slab->us_freecount == 0) { 2323 LIST_REMOVE(slab, us_link); 2324 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link); 2325 } 2326 2327 return (item); 2328} 2329 2330static int
|
2064uma_zalloc_bucket(uma_zone_t zone, int flags)
| 2331zone_alloc_bucket(uma_zone_t zone, int flags)
|
2065{ 2066 uma_bucket_t bucket; 2067 uma_slab_t slab;
| 2332{ 2333 uma_bucket_t bucket; 2334 uma_slab_t slab;
|
| 2335 uma_keg_t keg;
|
2068 int16_t saved; 2069 int max, origflags = flags; 2070 2071 /* 2072 * Try this zone's free list first so we don't allocate extra buckets. 2073 */ 2074 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2075 KASSERT(bucket->ub_cnt == 0,
| 2336 int16_t saved; 2337 int max, origflags = flags; 2338 2339 /* 2340 * Try this zone's free list first so we don't allocate extra buckets. 2341 */ 2342 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2343 KASSERT(bucket->ub_cnt == 0,
|
2076 ("uma_zalloc_bucket: Bucket on free list is not empty."));
| 2344 ("zone_alloc_bucket: Bucket on free list is not empty."));
|
2077 LIST_REMOVE(bucket, ub_link); 2078 } else { 2079 int bflags; 2080 2081 bflags = (flags & ~M_ZERO);
| 2345 LIST_REMOVE(bucket, ub_link); 2346 } else { 2347 int bflags; 2348 2349 bflags = (flags & ~M_ZERO);
|
2082 if (zone->uz_keg->uk_flags & UMA_ZFLAG_CACHEONLY)
| 2350 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
|
2083 bflags |= M_NOVM; 2084 2085 ZONE_UNLOCK(zone); 2086 bucket = bucket_alloc(zone->uz_count, bflags); 2087 ZONE_LOCK(zone); 2088 } 2089
| 2351 bflags |= M_NOVM; 2352 2353 ZONE_UNLOCK(zone); 2354 bucket = bucket_alloc(zone->uz_count, bflags); 2355 ZONE_LOCK(zone); 2356 } 2357
|
2090 if (bucket == NULL)
| 2358 if (bucket == NULL) {
|
2091 return (0);
| 2359 return (0);
|
| 2360 }
|
2092 2093#ifdef SMP 2094 /* 2095 * This code is here to limit the number of simultaneous bucket fills 2096 * for any given zone to the number of per cpu caches in this zone. This 2097 * is done so that we don't allocate more memory than we really need. 2098 */ 2099 if (zone->uz_fills >= mp_ncpus) 2100 goto done; 2101 2102#endif 2103 zone->uz_fills++; 2104 2105 max = MIN(bucket->ub_entries, zone->uz_count); 2106 /* Try to keep the buckets totally full */ 2107 saved = bucket->ub_cnt;
| 2361 2362#ifdef SMP 2363 /* 2364 * This code is here to limit the number of simultaneous bucket fills 2365 * for any given zone to the number of per cpu caches in this zone. This 2366 * is done so that we don't allocate more memory than we really need. 2367 */ 2368 if (zone->uz_fills >= mp_ncpus) 2369 goto done; 2370 2371#endif 2372 zone->uz_fills++; 2373 2374 max = MIN(bucket->ub_entries, zone->uz_count); 2375 /* Try to keep the buckets totally full */ 2376 saved = bucket->ub_cnt;
|
| 2377 slab = NULL; 2378 keg = NULL;
|
2108 while (bucket->ub_cnt < max &&
| 2379 while (bucket->ub_cnt < max &&
|
2109 (slab = uma_zone_slab(zone, flags)) != NULL) {
| 2380 (slab = zone->uz_slab(zone, keg, flags)) != NULL) { 2381 keg = slab->us_keg;
|
2110 while (slab->us_freecount && bucket->ub_cnt < max) { 2111 bucket->ub_bucket[bucket->ub_cnt++] =
| 2382 while (slab->us_freecount && bucket->ub_cnt < max) { 2383 bucket->ub_bucket[bucket->ub_cnt++] =
|
2112 uma_slab_alloc(zone, slab);
| 2384 slab_alloc_item(zone, slab);
|
2113 } 2114 2115 /* Don't block on the next fill */ 2116 flags |= M_NOWAIT; 2117 }
| 2385 } 2386 2387 /* Don't block on the next fill */ 2388 flags |= M_NOWAIT; 2389 }
|
| 2390 if (slab) 2391 zone_relock(zone, keg);
|
2118 2119 /* 2120 * We unlock here because we need to call the zone's init. 2121 * It should be safe to unlock because the slab dealt with 2122 * above is already on the appropriate list within the keg 2123 * and the bucket we filled is not yet on any list, so we 2124 * own it. 2125 */ 2126 if (zone->uz_init != NULL) { 2127 int i; 2128 2129 ZONE_UNLOCK(zone); 2130 for (i = saved; i < bucket->ub_cnt; i++)
| 2392 2393 /* 2394 * We unlock here because we need to call the zone's init. 2395 * It should be safe to unlock because the slab dealt with 2396 * above is already on the appropriate list within the keg 2397 * and the bucket we filled is not yet on any list, so we 2398 * own it. 2399 */ 2400 if (zone->uz_init != NULL) { 2401 int i; 2402 2403 ZONE_UNLOCK(zone); 2404 for (i = saved; i < bucket->ub_cnt; i++)
|
2131 if (zone->uz_init(bucket->ub_bucket[i], 2132 zone->uz_keg->uk_size, origflags) != 0)
| 2405 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, 2406 origflags) != 0)
|
2133 break; 2134 /* 2135 * If we couldn't initialize the whole bucket, put the 2136 * rest back onto the freelist. 2137 */ 2138 if (i != bucket->ub_cnt) { 2139 int j; 2140 2141 for (j = i; j < bucket->ub_cnt; j++) {
| 2407 break; 2408 /* 2409 * If we couldn't initialize the whole bucket, put the 2410 * rest back onto the freelist. 2411 */ 2412 if (i != bucket->ub_cnt) { 2413 int j; 2414 2415 for (j = i; j < bucket->ub_cnt; j++) {
|
2142 uma_zfree_internal(zone, bucket->ub_bucket[j],
| 2416 zone_free_item(zone, bucket->ub_bucket[j],
|
2143 NULL, SKIP_FINI, 0); 2144#ifdef INVARIANTS 2145 bucket->ub_bucket[j] = NULL; 2146#endif 2147 } 2148 bucket->ub_cnt = i; 2149 } 2150 ZONE_LOCK(zone); 2151 } 2152 2153 zone->uz_fills--; 2154 if (bucket->ub_cnt != 0) { 2155 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2156 bucket, ub_link); 2157 return (1); 2158 } 2159#ifdef SMP 2160done: 2161#endif 2162 bucket_free(bucket); 2163 2164 return (0); 2165} 2166/* 2167 * Allocates an item for an internal zone 2168 * 2169 * Arguments 2170 * zone The zone to alloc for. 2171 * udata The data to be passed to the constructor. 2172 * flags M_WAITOK, M_NOWAIT, M_ZERO. 2173 * 2174 * Returns 2175 * NULL if there is no memory and M_NOWAIT is set 2176 * An item if successful 2177 */ 2178 2179static void *
| 2417 NULL, SKIP_FINI, 0); 2418#ifdef INVARIANTS 2419 bucket->ub_bucket[j] = NULL; 2420#endif 2421 } 2422 bucket->ub_cnt = i; 2423 } 2424 ZONE_LOCK(zone); 2425 } 2426 2427 zone->uz_fills--; 2428 if (bucket->ub_cnt != 0) { 2429 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2430 bucket, ub_link); 2431 return (1); 2432 } 2433#ifdef SMP 2434done: 2435#endif 2436 bucket_free(bucket); 2437 2438 return (0); 2439} 2440/* 2441 * Allocates an item for an internal zone 2442 * 2443 * Arguments 2444 * zone The zone to alloc for. 2445 * udata The data to be passed to the constructor. 2446 * flags M_WAITOK, M_NOWAIT, M_ZERO. 2447 * 2448 * Returns 2449 * NULL if there is no memory and M_NOWAIT is set 2450 * An item if successful 2451 */ 2452 2453static void *
|
2180uma_zalloc_internal(uma_zone_t zone, void *udata, int flags)
| 2454zone_alloc_item(uma_zone_t zone, void *udata, int flags)
|
2181{
| 2455{
|
2182 uma_keg_t keg;
| |
2183 uma_slab_t slab; 2184 void *item; 2185 2186 item = NULL;
| 2456 uma_slab_t slab; 2457 void *item; 2458 2459 item = NULL;
|
2187 keg = zone->uz_keg;
| |
2188 2189#ifdef UMA_DEBUG_ALLOC 2190 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 2191#endif 2192 ZONE_LOCK(zone); 2193
| 2460 2461#ifdef UMA_DEBUG_ALLOC 2462 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 2463#endif 2464 ZONE_LOCK(zone); 2465
|
2194 slab = uma_zone_slab(zone, flags);
| 2466 slab = zone->uz_slab(zone, NULL, flags);
|
2195 if (slab == NULL) { 2196 zone->uz_fails++; 2197 ZONE_UNLOCK(zone); 2198 return (NULL); 2199 } 2200
| 2467 if (slab == NULL) { 2468 zone->uz_fails++; 2469 ZONE_UNLOCK(zone); 2470 return (NULL); 2471 } 2472
|
2201 item = uma_slab_alloc(zone, slab);
| 2473 item = slab_alloc_item(zone, slab);
|
2202
| 2474
|
| 2475 zone_relock(zone, slab->us_keg);
|
2203 zone->uz_allocs++;
| 2476 zone->uz_allocs++;
|
2204
| |
2205 ZONE_UNLOCK(zone); 2206 2207 /* 2208 * We have to call both the zone's init (not the keg's init) 2209 * and the zone's ctor. This is because the item is going from 2210 * a keg slab directly to the user, and the user is expecting it 2211 * to be both zone-init'd as well as zone-ctor'd. 2212 */ 2213 if (zone->uz_init != NULL) {
| 2477 ZONE_UNLOCK(zone); 2478 2479 /* 2480 * We have to call both the zone's init (not the keg's init) 2481 * and the zone's ctor. This is because the item is going from 2482 * a keg slab directly to the user, and the user is expecting it 2483 * to be both zone-init'd as well as zone-ctor'd. 2484 */ 2485 if (zone->uz_init != NULL) {
|
2214 if (zone->uz_init(item, keg->uk_size, flags) != 0) { 2215 uma_zfree_internal(zone, item, udata, SKIP_FINI,
| 2486 if (zone->uz_init(item, zone->uz_size, flags) != 0) { 2487 zone_free_item(zone, item, udata, SKIP_FINI,
|
2216 ZFREE_STATFAIL | ZFREE_STATFREE); 2217 return (NULL); 2218 } 2219 } 2220 if (zone->uz_ctor != NULL) {
| 2488 ZFREE_STATFAIL | ZFREE_STATFREE); 2489 return (NULL); 2490 } 2491 } 2492 if (zone->uz_ctor != NULL) {
|
2221 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) { 2222 uma_zfree_internal(zone, item, udata, SKIP_DTOR,
| 2493 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { 2494 zone_free_item(zone, item, udata, SKIP_DTOR,
|
2223 ZFREE_STATFAIL | ZFREE_STATFREE); 2224 return (NULL); 2225 } 2226 } 2227 if (flags & M_ZERO)
| 2495 ZFREE_STATFAIL | ZFREE_STATFREE); 2496 return (NULL); 2497 } 2498 } 2499 if (flags & M_ZERO)
|
2228 bzero(item, keg->uk_size);
| 2500 bzero(item, zone->uz_size);
|
2229 2230 return (item); 2231} 2232 2233/* See uma.h */ 2234void 2235uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 2236{
| 2501 2502 return (item); 2503} 2504 2505/* See uma.h */ 2506void 2507uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 2508{
|
2237 uma_keg_t keg;
| |
2238 uma_cache_t cache; 2239 uma_bucket_t bucket; 2240 int bflags; 2241 int cpu; 2242
| 2509 uma_cache_t cache; 2510 uma_bucket_t bucket; 2511 int bflags; 2512 int cpu; 2513
|
2243 keg = zone->uz_keg; 2244
| |
2245#ifdef UMA_DEBUG_ALLOC_1 2246 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 2247#endif 2248 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, 2249 zone->uz_name); 2250 2251 if (zone->uz_dtor)
| 2514#ifdef UMA_DEBUG_ALLOC_1 2515 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 2516#endif 2517 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, 2518 zone->uz_name); 2519 2520 if (zone->uz_dtor)
|
2252 zone->uz_dtor(item, keg->uk_size, udata);
| 2521 zone->uz_dtor(item, zone->uz_size, udata); 2522
|
2253#ifdef INVARIANTS 2254 ZONE_LOCK(zone);
| 2523#ifdef INVARIANTS 2524 ZONE_LOCK(zone);
|
2255 if (keg->uk_flags & UMA_ZONE_MALLOC)
| 2525 if (zone->uz_flags & UMA_ZONE_MALLOC)
|
2256 uma_dbg_free(zone, udata, item); 2257 else 2258 uma_dbg_free(zone, NULL, item); 2259 ZONE_UNLOCK(zone); 2260#endif 2261 /* 2262 * The race here is acceptable. If we miss it we'll just have to wait 2263 * a little longer for the limits to be reset. 2264 */
| 2526 uma_dbg_free(zone, udata, item); 2527 else 2528 uma_dbg_free(zone, NULL, item); 2529 ZONE_UNLOCK(zone); 2530#endif 2531 /* 2532 * The race here is acceptable. If we miss it we'll just have to wait 2533 * a little longer for the limits to be reset. 2534 */
|
2265 if (keg->uk_flags & UMA_ZFLAG_FULL)
| 2535 if (zone->uz_flags & UMA_ZFLAG_FULL)
|
2266 goto zfree_internal; 2267 2268 /* 2269 * If possible, free to the per-CPU cache. There are two 2270 * requirements for safe access to the per-CPU cache: (1) the thread 2271 * accessing the cache must not be preempted or yield during access, 2272 * and (2) the thread must not migrate CPUs without switching which 2273 * cache it accesses. We rely on a critical section to prevent 2274 * preemption and migration. We release the critical section in 2275 * order to acquire the zone mutex if we are unable to free to the 2276 * current cache; when we re-acquire the critical section, we must 2277 * detect and handle migration if it has occurred. 2278 */ 2279zfree_restart: 2280 critical_enter(); 2281 cpu = curcpu; 2282 cache = &zone->uz_cpu[cpu]; 2283 2284zfree_start: 2285 bucket = cache->uc_freebucket; 2286 2287 if (bucket) { 2288 /* 2289 * Do we have room in our bucket? It is OK for this uz count 2290 * check to be slightly out of sync. 2291 */ 2292 2293 if (bucket->ub_cnt < bucket->ub_entries) { 2294 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 2295 ("uma_zfree: Freeing to non free bucket index.")); 2296 bucket->ub_bucket[bucket->ub_cnt] = item; 2297 bucket->ub_cnt++; 2298 cache->uc_frees++; 2299 critical_exit(); 2300 return; 2301 } else if (cache->uc_allocbucket) { 2302#ifdef UMA_DEBUG_ALLOC 2303 printf("uma_zfree: Swapping buckets.\n"); 2304#endif 2305 /* 2306 * We have run out of space in our freebucket. 2307 * See if we can switch with our alloc bucket. 2308 */ 2309 if (cache->uc_allocbucket->ub_cnt < 2310 cache->uc_freebucket->ub_cnt) { 2311 bucket = cache->uc_freebucket; 2312 cache->uc_freebucket = cache->uc_allocbucket; 2313 cache->uc_allocbucket = bucket; 2314 goto zfree_start; 2315 } 2316 } 2317 } 2318 /* 2319 * We can get here for two reasons: 2320 * 2321 * 1) The buckets are NULL 2322 * 2) The alloc and free buckets are both somewhat full. 2323 * 2324 * We must go back the zone, which requires acquiring the zone lock, 2325 * which in turn means we must release and re-acquire the critical 2326 * section. Since the critical section is released, we may be 2327 * preempted or migrate. As such, make sure not to maintain any 2328 * thread-local state specific to the cache from prior to releasing 2329 * the critical section. 2330 */ 2331 critical_exit(); 2332 ZONE_LOCK(zone); 2333 critical_enter(); 2334 cpu = curcpu; 2335 cache = &zone->uz_cpu[cpu]; 2336 if (cache->uc_freebucket != NULL) { 2337 if (cache->uc_freebucket->ub_cnt < 2338 cache->uc_freebucket->ub_entries) { 2339 ZONE_UNLOCK(zone); 2340 goto zfree_start; 2341 } 2342 if (cache->uc_allocbucket != NULL && 2343 (cache->uc_allocbucket->ub_cnt < 2344 cache->uc_freebucket->ub_cnt)) { 2345 ZONE_UNLOCK(zone); 2346 goto zfree_start; 2347 } 2348 } 2349 2350 /* Since we have locked the zone we may as well send back our stats */ 2351 zone->uz_allocs += cache->uc_allocs; 2352 cache->uc_allocs = 0; 2353 zone->uz_frees += cache->uc_frees; 2354 cache->uc_frees = 0; 2355 2356 bucket = cache->uc_freebucket; 2357 cache->uc_freebucket = NULL; 2358 2359 /* Can we throw this on the zone full list? */ 2360 if (bucket != NULL) { 2361#ifdef UMA_DEBUG_ALLOC 2362 printf("uma_zfree: Putting old bucket on the free list.\n"); 2363#endif 2364 /* ub_cnt is pointing to the last free item */ 2365 KASSERT(bucket->ub_cnt != 0, 2366 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 2367 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2368 bucket, ub_link); 2369 } 2370 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2371 LIST_REMOVE(bucket, ub_link); 2372 ZONE_UNLOCK(zone); 2373 cache->uc_freebucket = bucket; 2374 goto zfree_start; 2375 } 2376 /* We are no longer associated with this CPU. */ 2377 critical_exit(); 2378 2379 /* And the zone.. */ 2380 ZONE_UNLOCK(zone); 2381 2382#ifdef UMA_DEBUG_ALLOC 2383 printf("uma_zfree: Allocating new free bucket.\n"); 2384#endif 2385 bflags = M_NOWAIT; 2386
| 2536 goto zfree_internal; 2537 2538 /* 2539 * If possible, free to the per-CPU cache. There are two 2540 * requirements for safe access to the per-CPU cache: (1) the thread 2541 * accessing the cache must not be preempted or yield during access, 2542 * and (2) the thread must not migrate CPUs without switching which 2543 * cache it accesses. We rely on a critical section to prevent 2544 * preemption and migration. We release the critical section in 2545 * order to acquire the zone mutex if we are unable to free to the 2546 * current cache; when we re-acquire the critical section, we must 2547 * detect and handle migration if it has occurred. 2548 */ 2549zfree_restart: 2550 critical_enter(); 2551 cpu = curcpu; 2552 cache = &zone->uz_cpu[cpu]; 2553 2554zfree_start: 2555 bucket = cache->uc_freebucket; 2556 2557 if (bucket) { 2558 /* 2559 * Do we have room in our bucket? It is OK for this uz count 2560 * check to be slightly out of sync. 2561 */ 2562 2563 if (bucket->ub_cnt < bucket->ub_entries) { 2564 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 2565 ("uma_zfree: Freeing to non free bucket index.")); 2566 bucket->ub_bucket[bucket->ub_cnt] = item; 2567 bucket->ub_cnt++; 2568 cache->uc_frees++; 2569 critical_exit(); 2570 return; 2571 } else if (cache->uc_allocbucket) { 2572#ifdef UMA_DEBUG_ALLOC 2573 printf("uma_zfree: Swapping buckets.\n"); 2574#endif 2575 /* 2576 * We have run out of space in our freebucket. 2577 * See if we can switch with our alloc bucket. 2578 */ 2579 if (cache->uc_allocbucket->ub_cnt < 2580 cache->uc_freebucket->ub_cnt) { 2581 bucket = cache->uc_freebucket; 2582 cache->uc_freebucket = cache->uc_allocbucket; 2583 cache->uc_allocbucket = bucket; 2584 goto zfree_start; 2585 } 2586 } 2587 } 2588 /* 2589 * We can get here for two reasons: 2590 * 2591 * 1) The buckets are NULL 2592 * 2) The alloc and free buckets are both somewhat full. 2593 * 2594 * We must go back the zone, which requires acquiring the zone lock, 2595 * which in turn means we must release and re-acquire the critical 2596 * section. Since the critical section is released, we may be 2597 * preempted or migrate. As such, make sure not to maintain any 2598 * thread-local state specific to the cache from prior to releasing 2599 * the critical section. 2600 */ 2601 critical_exit(); 2602 ZONE_LOCK(zone); 2603 critical_enter(); 2604 cpu = curcpu; 2605 cache = &zone->uz_cpu[cpu]; 2606 if (cache->uc_freebucket != NULL) { 2607 if (cache->uc_freebucket->ub_cnt < 2608 cache->uc_freebucket->ub_entries) { 2609 ZONE_UNLOCK(zone); 2610 goto zfree_start; 2611 } 2612 if (cache->uc_allocbucket != NULL && 2613 (cache->uc_allocbucket->ub_cnt < 2614 cache->uc_freebucket->ub_cnt)) { 2615 ZONE_UNLOCK(zone); 2616 goto zfree_start; 2617 } 2618 } 2619 2620 /* Since we have locked the zone we may as well send back our stats */ 2621 zone->uz_allocs += cache->uc_allocs; 2622 cache->uc_allocs = 0; 2623 zone->uz_frees += cache->uc_frees; 2624 cache->uc_frees = 0; 2625 2626 bucket = cache->uc_freebucket; 2627 cache->uc_freebucket = NULL; 2628 2629 /* Can we throw this on the zone full list? */ 2630 if (bucket != NULL) { 2631#ifdef UMA_DEBUG_ALLOC 2632 printf("uma_zfree: Putting old bucket on the free list.\n"); 2633#endif 2634 /* ub_cnt is pointing to the last free item */ 2635 KASSERT(bucket->ub_cnt != 0, 2636 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 2637 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2638 bucket, ub_link); 2639 } 2640 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2641 LIST_REMOVE(bucket, ub_link); 2642 ZONE_UNLOCK(zone); 2643 cache->uc_freebucket = bucket; 2644 goto zfree_start; 2645 } 2646 /* We are no longer associated with this CPU. */ 2647 critical_exit(); 2648 2649 /* And the zone.. */ 2650 ZONE_UNLOCK(zone); 2651 2652#ifdef UMA_DEBUG_ALLOC 2653 printf("uma_zfree: Allocating new free bucket.\n"); 2654#endif 2655 bflags = M_NOWAIT; 2656
|
2387 if (keg->uk_flags & UMA_ZFLAG_CACHEONLY)
| 2657 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
|
2388 bflags |= M_NOVM; 2389 bucket = bucket_alloc(zone->uz_count, bflags); 2390 if (bucket) { 2391 ZONE_LOCK(zone); 2392 LIST_INSERT_HEAD(&zone->uz_free_bucket, 2393 bucket, ub_link); 2394 ZONE_UNLOCK(zone); 2395 goto zfree_restart; 2396 } 2397 2398 /* 2399 * If nothing else caught this, we'll just do an internal free. 2400 */ 2401zfree_internal:
| 2658 bflags |= M_NOVM; 2659 bucket = bucket_alloc(zone->uz_count, bflags); 2660 if (bucket) { 2661 ZONE_LOCK(zone); 2662 LIST_INSERT_HEAD(&zone->uz_free_bucket, 2663 bucket, ub_link); 2664 ZONE_UNLOCK(zone); 2665 goto zfree_restart; 2666 } 2667 2668 /* 2669 * If nothing else caught this, we'll just do an internal free. 2670 */ 2671zfree_internal:
|
2402 uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
| 2672 zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
|
2403 2404 return; 2405} 2406 2407/* 2408 * Frees an item to an INTERNAL zone or allocates a free bucket 2409 * 2410 * Arguments: 2411 * zone The zone to free to 2412 * item The item we're freeing 2413 * udata User supplied data for the dtor 2414 * skip Skip dtors and finis 2415 */ 2416static void
| 2673 2674 return; 2675} 2676 2677/* 2678 * Frees an item to an INTERNAL zone or allocates a free bucket 2679 * 2680 * Arguments: 2681 * zone The zone to free to 2682 * item The item we're freeing 2683 * udata User supplied data for the dtor 2684 * skip Skip dtors and finis 2685 */ 2686static void
|
2417uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
| 2687zone_free_item(uma_zone_t zone, void *item, void *udata,
|
2418 enum zfreeskip skip, int flags) 2419{ 2420 uma_slab_t slab; 2421 uma_slabrefcnt_t slabref; 2422 uma_keg_t keg; 2423 u_int8_t *mem; 2424 u_int8_t freei;
| 2688 enum zfreeskip skip, int flags) 2689{ 2690 uma_slab_t slab; 2691 uma_slabrefcnt_t slabref; 2692 uma_keg_t keg; 2693 u_int8_t *mem; 2694 u_int8_t freei;
|
| 2695 int clearfull;
|
2425
| 2696
|
2426 keg = zone->uz_keg; 2427
| |
2428 if (skip < SKIP_DTOR && zone->uz_dtor)
| 2697 if (skip < SKIP_DTOR && zone->uz_dtor)
|
2429 zone->uz_dtor(item, keg->uk_size, udata);
| 2698 zone->uz_dtor(item, zone->uz_size, udata); 2699
|
2430 if (skip < SKIP_FINI && zone->uz_fini)
| 2700 if (skip < SKIP_FINI && zone->uz_fini)
|
2431 zone->uz_fini(item, keg->uk_size);
| 2701 zone->uz_fini(item, zone->uz_size);
|
2432 2433 ZONE_LOCK(zone); 2434 2435 if (flags & ZFREE_STATFAIL) 2436 zone->uz_fails++; 2437 if (flags & ZFREE_STATFREE) 2438 zone->uz_frees++; 2439
| 2702 2703 ZONE_LOCK(zone); 2704 2705 if (flags & ZFREE_STATFAIL) 2706 zone->uz_fails++; 2707 if (flags & ZFREE_STATFREE) 2708 zone->uz_frees++; 2709
|
2440 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
| 2710 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
|
2441 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
| 2711 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
|
2442 if (keg->uk_flags & UMA_ZONE_HASH)
| 2712 keg = zone_first_keg(zone); /* Must only be one. */ 2713 if (zone->uz_flags & UMA_ZONE_HASH) {
|
2443 slab = hash_sfind(&keg->uk_hash, mem);
| 2714 slab = hash_sfind(&keg->uk_hash, mem);
|
2444 else {
| 2715 } else {
|
2445 mem += keg->uk_pgoff; 2446 slab = (uma_slab_t)mem; 2447 } 2448 } else {
| 2716 mem += keg->uk_pgoff; 2717 slab = (uma_slab_t)mem; 2718 } 2719 } else {
|
2449 slab = (uma_slab_t)udata;
| 2720 /* This prevents redundant lookups via free(). */ 2721 if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL) 2722 slab = (uma_slab_t)udata; 2723 else 2724 slab = vtoslab((vm_offset_t)item); 2725 keg = slab->us_keg; 2726 keg_relock(keg, zone);
|
2450 }
| 2727 }
|
| 2728 MPASS(keg == slab->us_keg);
|
2451 2452 /* Do we need to remove from any lists? */ 2453 if (slab->us_freecount+1 == keg->uk_ipers) { 2454 LIST_REMOVE(slab, us_link); 2455 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2456 } else if (slab->us_freecount == 0) { 2457 LIST_REMOVE(slab, us_link); 2458 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2459 } 2460 2461 /* Slab management stuff */ 2462 freei = ((unsigned long)item - (unsigned long)slab->us_data) 2463 / keg->uk_rsize; 2464 2465#ifdef INVARIANTS 2466 if (!skip) 2467 uma_dbg_free(zone, slab, item); 2468#endif 2469 2470 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2471 slabref = (uma_slabrefcnt_t)slab; 2472 slabref->us_freelist[freei].us_item = slab->us_firstfree; 2473 } else { 2474 slab->us_freelist[freei].us_item = slab->us_firstfree; 2475 } 2476 slab->us_firstfree = freei; 2477 slab->us_freecount++; 2478 2479 /* Zone statistics */ 2480 keg->uk_free++; 2481
| 2729 2730 /* Do we need to remove from any lists? */ 2731 if (slab->us_freecount+1 == keg->uk_ipers) { 2732 LIST_REMOVE(slab, us_link); 2733 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2734 } else if (slab->us_freecount == 0) { 2735 LIST_REMOVE(slab, us_link); 2736 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2737 } 2738 2739 /* Slab management stuff */ 2740 freei = ((unsigned long)item - (unsigned long)slab->us_data) 2741 / keg->uk_rsize; 2742 2743#ifdef INVARIANTS 2744 if (!skip) 2745 uma_dbg_free(zone, slab, item); 2746#endif 2747 2748 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2749 slabref = (uma_slabrefcnt_t)slab; 2750 slabref->us_freelist[freei].us_item = slab->us_firstfree; 2751 } else { 2752 slab->us_freelist[freei].us_item = slab->us_firstfree; 2753 } 2754 slab->us_firstfree = freei; 2755 slab->us_freecount++; 2756 2757 /* Zone statistics */ 2758 keg->uk_free++; 2759
|
| 2760 clearfull = 0;
|
2482 if (keg->uk_flags & UMA_ZFLAG_FULL) {
| 2761 if (keg->uk_flags & UMA_ZFLAG_FULL) {
|
2483 if (keg->uk_pages < keg->uk_maxpages)
| 2762 if (keg->uk_pages < keg->uk_maxpages) {
|
2484 keg->uk_flags &= ~UMA_ZFLAG_FULL;
| 2763 keg->uk_flags &= ~UMA_ZFLAG_FULL;
|
| 2764 clearfull = 1; 2765 }
|
2485 2486 /* 2487 * We can handle one more allocation. Since we're clearing ZFLAG_FULL, 2488 * wake up all procs blocked on pages. This should be uncommon, so 2489 * keeping this simple for now (rather than adding count of blocked 2490 * threads etc). 2491 */ 2492 wakeup(keg); 2493 }
| 2766 2767 /* 2768 * We can handle one more allocation. Since we're clearing ZFLAG_FULL, 2769 * wake up all procs blocked on pages. This should be uncommon, so 2770 * keeping this simple for now (rather than adding count of blocked 2771 * threads etc). 2772 */ 2773 wakeup(keg); 2774 }
|
2494 2495 ZONE_UNLOCK(zone);
| 2775 if (clearfull) { 2776 zone_relock(zone, keg); 2777 zone->uz_flags &= ~UMA_ZFLAG_FULL; 2778 wakeup(zone); 2779 ZONE_UNLOCK(zone); 2780 } else 2781 KEG_UNLOCK(keg);
|
2496} 2497 2498/* See uma.h */ 2499void 2500uma_zone_set_max(uma_zone_t zone, int nitems) 2501{ 2502 uma_keg_t keg; 2503
| 2782} 2783 2784/* See uma.h */ 2785void 2786uma_zone_set_max(uma_zone_t zone, int nitems) 2787{ 2788 uma_keg_t keg; 2789
|
2504 keg = zone->uz_keg;
| |
2505 ZONE_LOCK(zone);
| 2790 ZONE_LOCK(zone);
|
2506 if (keg->uk_ppera > 1) 2507 keg->uk_maxpages = nitems * keg->uk_ppera; 2508 else 2509 keg->uk_maxpages = nitems / keg->uk_ipers; 2510
| 2791 keg = zone_first_keg(zone); 2792 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
|
2511 if (keg->uk_maxpages * keg->uk_ipers < nitems)
| 2793 if (keg->uk_maxpages * keg->uk_ipers < nitems)
|
2512 keg->uk_maxpages++;
| 2794 keg->uk_maxpages += keg->uk_ppera;
|
2513 2514 ZONE_UNLOCK(zone); 2515} 2516 2517/* See uma.h */ 2518void 2519uma_zone_set_init(uma_zone_t zone, uma_init uminit) 2520{
| 2795 2796 ZONE_UNLOCK(zone); 2797} 2798 2799/* See uma.h */ 2800void 2801uma_zone_set_init(uma_zone_t zone, uma_init uminit) 2802{
|
| 2803 uma_keg_t keg; 2804
|
2521 ZONE_LOCK(zone);
| 2805 ZONE_LOCK(zone);
|
2522 KASSERT(zone->uz_keg->uk_pages == 0,
| 2806 keg = zone_first_keg(zone); 2807 KASSERT(keg->uk_pages == 0,
|
2523 ("uma_zone_set_init on non-empty keg"));
| 2808 ("uma_zone_set_init on non-empty keg"));
|
2524 zone->uz_keg->uk_init = uminit;
| 2809 keg->uk_init = uminit;
|
2525 ZONE_UNLOCK(zone); 2526} 2527 2528/* See uma.h */ 2529void 2530uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 2531{
| 2810 ZONE_UNLOCK(zone); 2811} 2812 2813/* See uma.h */ 2814void 2815uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 2816{
|
| 2817 uma_keg_t keg; 2818
|
2532 ZONE_LOCK(zone);
| 2819 ZONE_LOCK(zone);
|
2533 KASSERT(zone->uz_keg->uk_pages == 0,
| 2820 keg = zone_first_keg(zone); 2821 KASSERT(keg->uk_pages == 0,
|
2534 ("uma_zone_set_fini on non-empty keg"));
| 2822 ("uma_zone_set_fini on non-empty keg"));
|
2535 zone->uz_keg->uk_fini = fini;
| 2823 keg->uk_fini = fini;
|
2536 ZONE_UNLOCK(zone); 2537} 2538 2539/* See uma.h */ 2540void 2541uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 2542{ 2543 ZONE_LOCK(zone);
| 2824 ZONE_UNLOCK(zone); 2825} 2826 2827/* See uma.h */ 2828void 2829uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 2830{ 2831 ZONE_LOCK(zone);
|
2544 KASSERT(zone->uz_keg->uk_pages == 0,
| 2832 KASSERT(zone_first_keg(zone)->uk_pages == 0,
|
2545 ("uma_zone_set_zinit on non-empty keg")); 2546 zone->uz_init = zinit; 2547 ZONE_UNLOCK(zone); 2548} 2549 2550/* See uma.h */ 2551void 2552uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 2553{ 2554 ZONE_LOCK(zone);
| 2833 ("uma_zone_set_zinit on non-empty keg")); 2834 zone->uz_init = zinit; 2835 ZONE_UNLOCK(zone); 2836} 2837 2838/* See uma.h */ 2839void 2840uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 2841{ 2842 ZONE_LOCK(zone);
|
2555 KASSERT(zone->uz_keg->uk_pages == 0,
| 2843 KASSERT(zone_first_keg(zone)->uk_pages == 0,
|
2556 ("uma_zone_set_zfini on non-empty keg")); 2557 zone->uz_fini = zfini; 2558 ZONE_UNLOCK(zone); 2559} 2560 2561/* See uma.h */ 2562/* XXX uk_freef is not actually used with the zone locked */ 2563void 2564uma_zone_set_freef(uma_zone_t zone, uma_free freef) 2565{
| 2844 ("uma_zone_set_zfini on non-empty keg")); 2845 zone->uz_fini = zfini; 2846 ZONE_UNLOCK(zone); 2847} 2848 2849/* See uma.h */ 2850/* XXX uk_freef is not actually used with the zone locked */ 2851void 2852uma_zone_set_freef(uma_zone_t zone, uma_free freef) 2853{
|
| 2854
|
2566 ZONE_LOCK(zone);
| 2855 ZONE_LOCK(zone);
|
2567 zone->uz_keg->uk_freef = freef;
| 2856 zone_first_keg(zone)->uk_freef = freef;
|
2568 ZONE_UNLOCK(zone); 2569} 2570 2571/* See uma.h */ 2572/* XXX uk_allocf is not actually used with the zone locked */ 2573void 2574uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 2575{
| 2857 ZONE_UNLOCK(zone); 2858} 2859 2860/* See uma.h */ 2861/* XXX uk_allocf is not actually used with the zone locked */ 2862void 2863uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 2864{
|
| 2865 uma_keg_t keg; 2866
|
2576 ZONE_LOCK(zone);
| 2867 ZONE_LOCK(zone);
|
2577 zone->uz_keg->uk_flags |= UMA_ZFLAG_PRIVALLOC; 2578 zone->uz_keg->uk_allocf = allocf;
| 2868 keg = zone_first_keg(zone); 2869 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC; 2870 keg->uk_allocf = allocf;
|
2579 ZONE_UNLOCK(zone); 2580} 2581 2582/* See uma.h */ 2583int 2584uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 2585{ 2586 uma_keg_t keg; 2587 vm_offset_t kva; 2588 int pages; 2589
| 2871 ZONE_UNLOCK(zone); 2872} 2873 2874/* See uma.h */ 2875int 2876uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 2877{ 2878 uma_keg_t keg; 2879 vm_offset_t kva; 2880 int pages; 2881
|
2590 keg = zone->uz_keg;
| 2882 keg = zone_first_keg(zone);
|
2591 pages = count / keg->uk_ipers; 2592 2593 if (pages * keg->uk_ipers < count) 2594 pages++; 2595 2596 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE); 2597 2598 if (kva == 0) 2599 return (0); 2600 if (obj == NULL) { 2601 obj = vm_object_allocate(OBJT_DEFAULT, 2602 pages); 2603 } else { 2604 VM_OBJECT_LOCK_INIT(obj, "uma object"); 2605 _vm_object_allocate(OBJT_DEFAULT, 2606 pages, obj); 2607 } 2608 ZONE_LOCK(zone); 2609 keg->uk_kva = kva; 2610 keg->uk_obj = obj; 2611 keg->uk_maxpages = pages; 2612 keg->uk_allocf = obj_alloc; 2613 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC; 2614 ZONE_UNLOCK(zone); 2615 return (1); 2616} 2617 2618/* See uma.h */ 2619void 2620uma_prealloc(uma_zone_t zone, int items) 2621{ 2622 int slabs; 2623 uma_slab_t slab; 2624 uma_keg_t keg; 2625
| 2883 pages = count / keg->uk_ipers; 2884 2885 if (pages * keg->uk_ipers < count) 2886 pages++; 2887 2888 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE); 2889 2890 if (kva == 0) 2891 return (0); 2892 if (obj == NULL) { 2893 obj = vm_object_allocate(OBJT_DEFAULT, 2894 pages); 2895 } else { 2896 VM_OBJECT_LOCK_INIT(obj, "uma object"); 2897 _vm_object_allocate(OBJT_DEFAULT, 2898 pages, obj); 2899 } 2900 ZONE_LOCK(zone); 2901 keg->uk_kva = kva; 2902 keg->uk_obj = obj; 2903 keg->uk_maxpages = pages; 2904 keg->uk_allocf = obj_alloc; 2905 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC; 2906 ZONE_UNLOCK(zone); 2907 return (1); 2908} 2909 2910/* See uma.h */ 2911void 2912uma_prealloc(uma_zone_t zone, int items) 2913{ 2914 int slabs; 2915 uma_slab_t slab; 2916 uma_keg_t keg; 2917
|
2626 keg = zone->uz_keg;
| 2918 keg = zone_first_keg(zone);
|
2627 ZONE_LOCK(zone); 2628 slabs = items / keg->uk_ipers; 2629 if (slabs * keg->uk_ipers < items) 2630 slabs++; 2631 while (slabs > 0) {
| 2919 ZONE_LOCK(zone); 2920 slabs = items / keg->uk_ipers; 2921 if (slabs * keg->uk_ipers < items) 2922 slabs++; 2923 while (slabs > 0) {
|
2632 slab = slab_zalloc(zone, M_WAITOK);
| 2924 slab = keg_alloc_slab(keg, zone, M_WAITOK); 2925 if (slab == NULL) 2926 break; 2927 MPASS(slab->us_keg == keg);
|
2633 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2634 slabs--; 2635 } 2636 ZONE_UNLOCK(zone); 2637} 2638 2639/* See uma.h */ 2640u_int32_t * 2641uma_find_refcnt(uma_zone_t zone, void *item) 2642{ 2643 uma_slabrefcnt_t slabref; 2644 uma_keg_t keg; 2645 u_int32_t *refcnt; 2646 int idx; 2647
| 2928 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2929 slabs--; 2930 } 2931 ZONE_UNLOCK(zone); 2932} 2933 2934/* See uma.h */ 2935u_int32_t * 2936uma_find_refcnt(uma_zone_t zone, void *item) 2937{ 2938 uma_slabrefcnt_t slabref; 2939 uma_keg_t keg; 2940 u_int32_t *refcnt; 2941 int idx; 2942
|
2648 keg = zone->uz_keg;
| |
2649 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item & 2650 (~UMA_SLAB_MASK));
| 2943 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item & 2944 (~UMA_SLAB_MASK));
|
| 2945 keg = slabref->us_keg;
|
2651 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT, 2652 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); 2653 idx = ((unsigned long)item - (unsigned long)slabref->us_data) 2654 / keg->uk_rsize; 2655 refcnt = &slabref->us_freelist[idx].us_refcnt; 2656 return refcnt; 2657} 2658 2659/* See uma.h */ 2660void 2661uma_reclaim(void) 2662{ 2663#ifdef UMA_DEBUG 2664 printf("UMA: vm asked us to release pages!\n"); 2665#endif 2666 bucket_enable(); 2667 zone_foreach(zone_drain); 2668 /* 2669 * Some slabs may have been freed but this zone will be visited early 2670 * we visit again so that we can free pages that are empty once other 2671 * zones are drained. We have to do the same for buckets. 2672 */ 2673 zone_drain(slabzone); 2674 zone_drain(slabrefzone); 2675 bucket_zone_drain(); 2676} 2677 2678/* See uma.h */ 2679int 2680uma_zone_exhausted(uma_zone_t zone) 2681{ 2682 int full; 2683 2684 ZONE_LOCK(zone);
| 2946 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT, 2947 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); 2948 idx = ((unsigned long)item - (unsigned long)slabref->us_data) 2949 / keg->uk_rsize; 2950 refcnt = &slabref->us_freelist[idx].us_refcnt; 2951 return refcnt; 2952} 2953 2954/* See uma.h */ 2955void 2956uma_reclaim(void) 2957{ 2958#ifdef UMA_DEBUG 2959 printf("UMA: vm asked us to release pages!\n"); 2960#endif 2961 bucket_enable(); 2962 zone_foreach(zone_drain); 2963 /* 2964 * Some slabs may have been freed but this zone will be visited early 2965 * we visit again so that we can free pages that are empty once other 2966 * zones are drained. We have to do the same for buckets. 2967 */ 2968 zone_drain(slabzone); 2969 zone_drain(slabrefzone); 2970 bucket_zone_drain(); 2971} 2972 2973/* See uma.h */ 2974int 2975uma_zone_exhausted(uma_zone_t zone) 2976{ 2977 int full; 2978 2979 ZONE_LOCK(zone);
|
2685 full = (zone->uz_keg->uk_flags & UMA_ZFLAG_FULL);
| 2980 full = (zone->uz_flags & UMA_ZFLAG_FULL);
|
2686 ZONE_UNLOCK(zone); 2687 return (full); 2688} 2689 2690int 2691uma_zone_exhausted_nolock(uma_zone_t zone) 2692{
| 2981 ZONE_UNLOCK(zone); 2982 return (full); 2983} 2984 2985int 2986uma_zone_exhausted_nolock(uma_zone_t zone) 2987{
|
2693 return (zone->uz_keg->uk_flags & UMA_ZFLAG_FULL);
| 2988 return (zone->uz_flags & UMA_ZFLAG_FULL);
|
2694} 2695 2696void * 2697uma_large_malloc(int size, int wait) 2698{ 2699 void *mem; 2700 uma_slab_t slab; 2701 u_int8_t flags; 2702
| 2989} 2990 2991void * 2992uma_large_malloc(int size, int wait) 2993{ 2994 void *mem; 2995 uma_slab_t slab; 2996 u_int8_t flags; 2997
|
2703 slab = uma_zalloc_internal(slabzone, NULL, wait);
| 2998 slab = zone_alloc_item(slabzone, NULL, wait);
|
2704 if (slab == NULL) 2705 return (NULL); 2706 mem = page_alloc(NULL, size, &flags, wait); 2707 if (mem) { 2708 vsetslab((vm_offset_t)mem, slab); 2709 slab->us_data = mem; 2710 slab->us_flags = flags | UMA_SLAB_MALLOC; 2711 slab->us_size = size; 2712 } else {
| 2999 if (slab == NULL) 3000 return (NULL); 3001 mem = page_alloc(NULL, size, &flags, wait); 3002 if (mem) { 3003 vsetslab((vm_offset_t)mem, slab); 3004 slab->us_data = mem; 3005 slab->us_flags = flags | UMA_SLAB_MALLOC; 3006 slab->us_size = size; 3007 } else {
|
2713 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE,
| 3008 zone_free_item(slabzone, slab, NULL, SKIP_NONE,
|
2714 ZFREE_STATFAIL | ZFREE_STATFREE); 2715 } 2716 2717 return (mem); 2718} 2719 2720void 2721uma_large_free(uma_slab_t slab) 2722{ 2723 vsetobj((vm_offset_t)slab->us_data, kmem_object); 2724 page_free(slab->us_data, slab->us_size, slab->us_flags);
| 3009 ZFREE_STATFAIL | ZFREE_STATFREE); 3010 } 3011 3012 return (mem); 3013} 3014 3015void 3016uma_large_free(uma_slab_t slab) 3017{ 3018 vsetobj((vm_offset_t)slab->us_data, kmem_object); 3019 page_free(slab->us_data, slab->us_size, slab->us_flags);
|
2725 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
| 3020 zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
|
2726} 2727 2728void 2729uma_print_stats(void) 2730{ 2731 zone_foreach(uma_print_zone); 2732} 2733 2734static void 2735slab_print(uma_slab_t slab) 2736{ 2737 printf("slab: keg %p, data %p, freecount %d, firstfree %d\n", 2738 slab->us_keg, slab->us_data, slab->us_freecount, 2739 slab->us_firstfree); 2740} 2741 2742static void 2743cache_print(uma_cache_t cache) 2744{ 2745 printf("alloc: %p(%d), free: %p(%d)\n", 2746 cache->uc_allocbucket, 2747 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 2748 cache->uc_freebucket, 2749 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 2750} 2751
| 3021} 3022 3023void 3024uma_print_stats(void) 3025{ 3026 zone_foreach(uma_print_zone); 3027} 3028 3029static void 3030slab_print(uma_slab_t slab) 3031{ 3032 printf("slab: keg %p, data %p, freecount %d, firstfree %d\n", 3033 slab->us_keg, slab->us_data, slab->us_freecount, 3034 slab->us_firstfree); 3035} 3036 3037static void 3038cache_print(uma_cache_t cache) 3039{ 3040 printf("alloc: %p(%d), free: %p(%d)\n", 3041 cache->uc_allocbucket, 3042 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 3043 cache->uc_freebucket, 3044 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 3045} 3046
|
2752void 2753uma_print_zone(uma_zone_t zone)
| 3047static void 3048uma_print_keg(uma_keg_t keg)
|
2754{
| 3049{
|
2755 uma_cache_t cache; 2756 uma_keg_t keg;
| |
2757 uma_slab_t slab;
| 3050 uma_slab_t slab;
|
2758 int i;
| |
2759
| 3051
|
2760 keg = zone->uz_keg; 2761 printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 2762 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags,
| 3052 printf("keg: %s(%p) size %d(%d) flags %d ipers %d ppera %d " 3053 "out %d free %d limit %d\n", 3054 keg->uk_name, keg, keg->uk_size, keg->uk_rsize, keg->uk_flags,
|
2763 keg->uk_ipers, keg->uk_ppera,
| 3055 keg->uk_ipers, keg->uk_ppera,
|
2764 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free);
| 3056 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free, 3057 (keg->uk_maxpages / keg->uk_ppera) * keg->uk_ipers);
|
2765 printf("Part slabs:\n"); 2766 LIST_FOREACH(slab, &keg->uk_part_slab, us_link) 2767 slab_print(slab); 2768 printf("Free slabs:\n"); 2769 LIST_FOREACH(slab, &keg->uk_free_slab, us_link) 2770 slab_print(slab); 2771 printf("Full slabs:\n"); 2772 LIST_FOREACH(slab, &keg->uk_full_slab, us_link) 2773 slab_print(slab);
| 3058 printf("Part slabs:\n"); 3059 LIST_FOREACH(slab, &keg->uk_part_slab, us_link) 3060 slab_print(slab); 3061 printf("Free slabs:\n"); 3062 LIST_FOREACH(slab, &keg->uk_free_slab, us_link) 3063 slab_print(slab); 3064 printf("Full slabs:\n"); 3065 LIST_FOREACH(slab, &keg->uk_full_slab, us_link) 3066 slab_print(slab);
|
| 3067} 3068 3069void 3070uma_print_zone(uma_zone_t zone) 3071{ 3072 uma_cache_t cache; 3073 uma_klink_t kl; 3074 int i; 3075 3076 printf("zone: %s(%p) size %d flags %d\n", 3077 zone->uz_name, zone, zone->uz_size, zone->uz_flags); 3078 LIST_FOREACH(kl, &zone->uz_kegs, kl_link) 3079 uma_print_keg(kl->kl_keg);
|
2774 for (i = 0; i <= mp_maxid; i++) { 2775 if (CPU_ABSENT(i)) 2776 continue; 2777 cache = &zone->uz_cpu[i]; 2778 printf("CPU %d Cache:\n", i); 2779 cache_print(cache); 2780 } 2781} 2782 2783#ifdef DDB 2784/* 2785 * Generate statistics across both the zone and its per-cpu cache's. Return 2786 * desired statistics if the pointer is non-NULL for that statistic. 2787 * 2788 * Note: does not update the zone statistics, as it can't safely clear the 2789 * per-CPU cache statistic. 2790 * 2791 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't 2792 * safe from off-CPU; we should modify the caches to track this information 2793 * directly so that we don't have to. 2794 */ 2795static void 2796uma_zone_sumstat(uma_zone_t z, int *cachefreep, u_int64_t *allocsp, 2797 u_int64_t *freesp) 2798{ 2799 uma_cache_t cache; 2800 u_int64_t allocs, frees; 2801 int cachefree, cpu; 2802 2803 allocs = frees = 0; 2804 cachefree = 0; 2805 for (cpu = 0; cpu <= mp_maxid; cpu++) { 2806 if (CPU_ABSENT(cpu)) 2807 continue; 2808 cache = &z->uz_cpu[cpu]; 2809 if (cache->uc_allocbucket != NULL) 2810 cachefree += cache->uc_allocbucket->ub_cnt; 2811 if (cache->uc_freebucket != NULL) 2812 cachefree += cache->uc_freebucket->ub_cnt; 2813 allocs += cache->uc_allocs; 2814 frees += cache->uc_frees; 2815 } 2816 allocs += z->uz_allocs; 2817 frees += z->uz_frees; 2818 if (cachefreep != NULL) 2819 *cachefreep = cachefree; 2820 if (allocsp != NULL) 2821 *allocsp = allocs; 2822 if (freesp != NULL) 2823 *freesp = frees; 2824} 2825#endif /* DDB */ 2826 2827static int 2828sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 2829{ 2830 uma_keg_t kz; 2831 uma_zone_t z; 2832 int count; 2833 2834 count = 0; 2835 mtx_lock(&uma_mtx); 2836 LIST_FOREACH(kz, &uma_kegs, uk_link) { 2837 LIST_FOREACH(z, &kz->uk_zones, uz_link) 2838 count++; 2839 } 2840 mtx_unlock(&uma_mtx); 2841 return (sysctl_handle_int(oidp, &count, 0, req)); 2842} 2843 2844static int 2845sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 2846{ 2847 struct uma_stream_header ush; 2848 struct uma_type_header uth; 2849 struct uma_percpu_stat ups; 2850 uma_bucket_t bucket; 2851 struct sbuf sbuf; 2852 uma_cache_t cache;
| 3080 for (i = 0; i <= mp_maxid; i++) { 3081 if (CPU_ABSENT(i)) 3082 continue; 3083 cache = &zone->uz_cpu[i]; 3084 printf("CPU %d Cache:\n", i); 3085 cache_print(cache); 3086 } 3087} 3088 3089#ifdef DDB 3090/* 3091 * Generate statistics across both the zone and its per-cpu cache's. Return 3092 * desired statistics if the pointer is non-NULL for that statistic. 3093 * 3094 * Note: does not update the zone statistics, as it can't safely clear the 3095 * per-CPU cache statistic. 3096 * 3097 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't 3098 * safe from off-CPU; we should modify the caches to track this information 3099 * directly so that we don't have to. 3100 */ 3101static void 3102uma_zone_sumstat(uma_zone_t z, int *cachefreep, u_int64_t *allocsp, 3103 u_int64_t *freesp) 3104{ 3105 uma_cache_t cache; 3106 u_int64_t allocs, frees; 3107 int cachefree, cpu; 3108 3109 allocs = frees = 0; 3110 cachefree = 0; 3111 for (cpu = 0; cpu <= mp_maxid; cpu++) { 3112 if (CPU_ABSENT(cpu)) 3113 continue; 3114 cache = &z->uz_cpu[cpu]; 3115 if (cache->uc_allocbucket != NULL) 3116 cachefree += cache->uc_allocbucket->ub_cnt; 3117 if (cache->uc_freebucket != NULL) 3118 cachefree += cache->uc_freebucket->ub_cnt; 3119 allocs += cache->uc_allocs; 3120 frees += cache->uc_frees; 3121 } 3122 allocs += z->uz_allocs; 3123 frees += z->uz_frees; 3124 if (cachefreep != NULL) 3125 *cachefreep = cachefree; 3126 if (allocsp != NULL) 3127 *allocsp = allocs; 3128 if (freesp != NULL) 3129 *freesp = frees; 3130} 3131#endif /* DDB */ 3132 3133static int 3134sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 3135{ 3136 uma_keg_t kz; 3137 uma_zone_t z; 3138 int count; 3139 3140 count = 0; 3141 mtx_lock(&uma_mtx); 3142 LIST_FOREACH(kz, &uma_kegs, uk_link) { 3143 LIST_FOREACH(z, &kz->uk_zones, uz_link) 3144 count++; 3145 } 3146 mtx_unlock(&uma_mtx); 3147 return (sysctl_handle_int(oidp, &count, 0, req)); 3148} 3149 3150static int 3151sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 3152{ 3153 struct uma_stream_header ush; 3154 struct uma_type_header uth; 3155 struct uma_percpu_stat ups; 3156 uma_bucket_t bucket; 3157 struct sbuf sbuf; 3158 uma_cache_t cache;
|
| 3159 uma_klink_t kl;
|
2853 uma_keg_t kz; 2854 uma_zone_t z;
| 3160 uma_keg_t kz; 3161 uma_zone_t z;
|
| 3162 uma_keg_t k;
|
2855 char *buffer; 2856 int buflen, count, error, i; 2857 2858 mtx_lock(&uma_mtx); 2859restart: 2860 mtx_assert(&uma_mtx, MA_OWNED); 2861 count = 0; 2862 LIST_FOREACH(kz, &uma_kegs, uk_link) { 2863 LIST_FOREACH(z, &kz->uk_zones, uz_link) 2864 count++; 2865 } 2866 mtx_unlock(&uma_mtx); 2867 2868 buflen = sizeof(ush) + count * (sizeof(uth) + sizeof(ups) * 2869 (mp_maxid + 1)) + 1; 2870 buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO); 2871 2872 mtx_lock(&uma_mtx); 2873 i = 0; 2874 LIST_FOREACH(kz, &uma_kegs, uk_link) { 2875 LIST_FOREACH(z, &kz->uk_zones, uz_link) 2876 i++; 2877 } 2878 if (i > count) { 2879 free(buffer, M_TEMP); 2880 goto restart; 2881 } 2882 count = i; 2883 2884 sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN); 2885 2886 /* 2887 * Insert stream header. 2888 */ 2889 bzero(&ush, sizeof(ush)); 2890 ush.ush_version = UMA_STREAM_VERSION; 2891 ush.ush_maxcpus = (mp_maxid + 1); 2892 ush.ush_count = count; 2893 if (sbuf_bcat(&sbuf, &ush, sizeof(ush)) < 0) { 2894 mtx_unlock(&uma_mtx); 2895 error = ENOMEM; 2896 goto out; 2897 } 2898 2899 LIST_FOREACH(kz, &uma_kegs, uk_link) { 2900 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 2901 bzero(&uth, sizeof(uth)); 2902 ZONE_LOCK(z); 2903 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 2904 uth.uth_align = kz->uk_align;
| 3163 char *buffer; 3164 int buflen, count, error, i; 3165 3166 mtx_lock(&uma_mtx); 3167restart: 3168 mtx_assert(&uma_mtx, MA_OWNED); 3169 count = 0; 3170 LIST_FOREACH(kz, &uma_kegs, uk_link) { 3171 LIST_FOREACH(z, &kz->uk_zones, uz_link) 3172 count++; 3173 } 3174 mtx_unlock(&uma_mtx); 3175 3176 buflen = sizeof(ush) + count * (sizeof(uth) + sizeof(ups) * 3177 (mp_maxid + 1)) + 1; 3178 buffer = malloc(buflen, M_TEMP, M_WAITOK | M_ZERO); 3179 3180 mtx_lock(&uma_mtx); 3181 i = 0; 3182 LIST_FOREACH(kz, &uma_kegs, uk_link) { 3183 LIST_FOREACH(z, &kz->uk_zones, uz_link) 3184 i++; 3185 } 3186 if (i > count) { 3187 free(buffer, M_TEMP); 3188 goto restart; 3189 } 3190 count = i; 3191 3192 sbuf_new(&sbuf, buffer, buflen, SBUF_FIXEDLEN); 3193 3194 /* 3195 * Insert stream header. 3196 */ 3197 bzero(&ush, sizeof(ush)); 3198 ush.ush_version = UMA_STREAM_VERSION; 3199 ush.ush_maxcpus = (mp_maxid + 1); 3200 ush.ush_count = count; 3201 if (sbuf_bcat(&sbuf, &ush, sizeof(ush)) < 0) { 3202 mtx_unlock(&uma_mtx); 3203 error = ENOMEM; 3204 goto out; 3205 } 3206 3207 LIST_FOREACH(kz, &uma_kegs, uk_link) { 3208 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 3209 bzero(&uth, sizeof(uth)); 3210 ZONE_LOCK(z); 3211 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 3212 uth.uth_align = kz->uk_align;
|
2905 uth.uth_pages = kz->uk_pages; 2906 uth.uth_keg_free = kz->uk_free;
| |
2907 uth.uth_size = kz->uk_size; 2908 uth.uth_rsize = kz->uk_rsize;
| 3213 uth.uth_size = kz->uk_size; 3214 uth.uth_rsize = kz->uk_rsize;
|
2909 uth.uth_maxpages = kz->uk_maxpages; 2910 if (kz->uk_ppera > 1) 2911 uth.uth_limit = kz->uk_maxpages / 2912 kz->uk_ppera; 2913 else 2914 uth.uth_limit = kz->uk_maxpages * 2915 kz->uk_ipers;
| 3215 LIST_FOREACH(kl, &z->uz_kegs, kl_link) { 3216 k = kl->kl_keg; 3217 uth.uth_maxpages += k->uk_maxpages; 3218 uth.uth_pages += k->uk_pages; 3219 uth.uth_keg_free += k->uk_free; 3220 uth.uth_limit = (k->uk_maxpages / k->uk_ppera) 3221 * k->uk_ipers; 3222 }
|
2916 2917 /* 2918 * A zone is secondary is it is not the first entry 2919 * on the keg's zone list. 2920 */
| 3223 3224 /* 3225 * A zone is secondary is it is not the first entry 3226 * on the keg's zone list. 3227 */
|
2921 if ((kz->uk_flags & UMA_ZONE_SECONDARY) &&
| 3228 if ((z->uz_flags & UMA_ZONE_SECONDARY) &&
|
2922 (LIST_FIRST(&kz->uk_zones) != z)) 2923 uth.uth_zone_flags = UTH_ZONE_SECONDARY; 2924 2925 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) 2926 uth.uth_zone_free += bucket->ub_cnt; 2927 uth.uth_allocs = z->uz_allocs; 2928 uth.uth_frees = z->uz_frees; 2929 uth.uth_fails = z->uz_fails; 2930 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) { 2931 ZONE_UNLOCK(z); 2932 mtx_unlock(&uma_mtx); 2933 error = ENOMEM; 2934 goto out; 2935 } 2936 /* 2937 * While it is not normally safe to access the cache 2938 * bucket pointers while not on the CPU that owns the 2939 * cache, we only allow the pointers to be exchanged 2940 * without the zone lock held, not invalidated, so 2941 * accept the possible race associated with bucket 2942 * exchange during monitoring. 2943 */ 2944 for (i = 0; i < (mp_maxid + 1); i++) { 2945 bzero(&ups, sizeof(ups)); 2946 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) 2947 goto skip; 2948 if (CPU_ABSENT(i)) 2949 goto skip; 2950 cache = &z->uz_cpu[i]; 2951 if (cache->uc_allocbucket != NULL) 2952 ups.ups_cache_free += 2953 cache->uc_allocbucket->ub_cnt; 2954 if (cache->uc_freebucket != NULL) 2955 ups.ups_cache_free += 2956 cache->uc_freebucket->ub_cnt; 2957 ups.ups_allocs = cache->uc_allocs; 2958 ups.ups_frees = cache->uc_frees; 2959skip: 2960 if (sbuf_bcat(&sbuf, &ups, sizeof(ups)) < 0) { 2961 ZONE_UNLOCK(z); 2962 mtx_unlock(&uma_mtx); 2963 error = ENOMEM; 2964 goto out; 2965 } 2966 } 2967 ZONE_UNLOCK(z); 2968 } 2969 } 2970 mtx_unlock(&uma_mtx); 2971 sbuf_finish(&sbuf); 2972 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 2973out: 2974 free(buffer, M_TEMP); 2975 return (error); 2976} 2977 2978#ifdef DDB 2979DB_SHOW_COMMAND(uma, db_show_uma) 2980{ 2981 u_int64_t allocs, frees; 2982 uma_bucket_t bucket; 2983 uma_keg_t kz; 2984 uma_zone_t z; 2985 int cachefree; 2986 2987 db_printf("%18s %8s %8s %8s %12s\n", "Zone", "Size", "Used", "Free", 2988 "Requests"); 2989 LIST_FOREACH(kz, &uma_kegs, uk_link) { 2990 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 2991 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 2992 allocs = z->uz_allocs; 2993 frees = z->uz_frees; 2994 cachefree = 0; 2995 } else 2996 uma_zone_sumstat(z, &cachefree, &allocs, 2997 &frees);
| 3229 (LIST_FIRST(&kz->uk_zones) != z)) 3230 uth.uth_zone_flags = UTH_ZONE_SECONDARY; 3231 3232 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) 3233 uth.uth_zone_free += bucket->ub_cnt; 3234 uth.uth_allocs = z->uz_allocs; 3235 uth.uth_frees = z->uz_frees; 3236 uth.uth_fails = z->uz_fails; 3237 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) { 3238 ZONE_UNLOCK(z); 3239 mtx_unlock(&uma_mtx); 3240 error = ENOMEM; 3241 goto out; 3242 } 3243 /* 3244 * While it is not normally safe to access the cache 3245 * bucket pointers while not on the CPU that owns the 3246 * cache, we only allow the pointers to be exchanged 3247 * without the zone lock held, not invalidated, so 3248 * accept the possible race associated with bucket 3249 * exchange during monitoring. 3250 */ 3251 for (i = 0; i < (mp_maxid + 1); i++) { 3252 bzero(&ups, sizeof(ups)); 3253 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) 3254 goto skip; 3255 if (CPU_ABSENT(i)) 3256 goto skip; 3257 cache = &z->uz_cpu[i]; 3258 if (cache->uc_allocbucket != NULL) 3259 ups.ups_cache_free += 3260 cache->uc_allocbucket->ub_cnt; 3261 if (cache->uc_freebucket != NULL) 3262 ups.ups_cache_free += 3263 cache->uc_freebucket->ub_cnt; 3264 ups.ups_allocs = cache->uc_allocs; 3265 ups.ups_frees = cache->uc_frees; 3266skip: 3267 if (sbuf_bcat(&sbuf, &ups, sizeof(ups)) < 0) { 3268 ZONE_UNLOCK(z); 3269 mtx_unlock(&uma_mtx); 3270 error = ENOMEM; 3271 goto out; 3272 } 3273 } 3274 ZONE_UNLOCK(z); 3275 } 3276 } 3277 mtx_unlock(&uma_mtx); 3278 sbuf_finish(&sbuf); 3279 error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 3280out: 3281 free(buffer, M_TEMP); 3282 return (error); 3283} 3284 3285#ifdef DDB 3286DB_SHOW_COMMAND(uma, db_show_uma) 3287{ 3288 u_int64_t allocs, frees; 3289 uma_bucket_t bucket; 3290 uma_keg_t kz; 3291 uma_zone_t z; 3292 int cachefree; 3293 3294 db_printf("%18s %8s %8s %8s %12s\n", "Zone", "Size", "Used", "Free", 3295 "Requests"); 3296 LIST_FOREACH(kz, &uma_kegs, uk_link) { 3297 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 3298 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 3299 allocs = z->uz_allocs; 3300 frees = z->uz_frees; 3301 cachefree = 0; 3302 } else 3303 uma_zone_sumstat(z, &cachefree, &allocs, 3304 &frees);
|
2998 if (!((kz->uk_flags & UMA_ZONE_SECONDARY) &&
| 3305 if (!((z->uz_flags & UMA_ZONE_SECONDARY) &&
|
2999 (LIST_FIRST(&kz->uk_zones) != z))) 3000 cachefree += kz->uk_free; 3001 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) 3002 cachefree += bucket->ub_cnt; 3003 db_printf("%18s %8ju %8jd %8d %12ju\n", z->uz_name, 3004 (uintmax_t)kz->uk_size, 3005 (intmax_t)(allocs - frees), cachefree, 3006 (uintmax_t)allocs); 3007 } 3008 } 3009} 3010#endif
| 3306 (LIST_FIRST(&kz->uk_zones) != z))) 3307 cachefree += kz->uk_free; 3308 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) 3309 cachefree += bucket->ub_cnt; 3310 db_printf("%18s %8ju %8jd %8d %12ju\n", z->uz_name, 3311 (uintmax_t)kz->uk_size, 3312 (intmax_t)(allocs - frees), cachefree, 3313 (uintmax_t)allocs); 3314 } 3315 } 3316} 3317#endif
|