1/*-
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
--- 35 unchanged lines hidden (view full) ---
44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 148070 2005-07-15 23:34:39Z rwatson $");
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60
--- 126 unchanged lines hidden (view full) ---
187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
195/*
196 * Flags and enumerations to be passed to internal functions.
197 */
198enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
199
200#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
201
202/* Prototypes.. */
203
204static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
205static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
206static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
207static void page_free(void *, int, u_int8_t);
208static uma_slab_t slab_zalloc(uma_zone_t, int);
209static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
219static void zone_foreach(void (*zfunc)(uma_zone_t));
220static void zone_timeout(uma_zone_t zone);
221static int hash_alloc(struct uma_hash *);
222static int hash_expand(struct uma_hash *, struct uma_hash *);
223static void hash_free(struct uma_hash *hash);
224static void uma_timeout(void *);
225static void uma_startup3(void);
226static void *uma_zalloc_internal(uma_zone_t, void *, int);
227static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip,
228 int);
229static void bucket_enable(void);
230static void bucket_init(void);
231static uma_bucket_t bucket_alloc(int, int);
232static void bucket_free(uma_bucket_t);
233static void bucket_zone_drain(void);
234static int uma_zalloc_bucket(uma_zone_t zone, int flags);
235static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
236static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);
--- 108 unchanged lines hidden (view full) ---
345}
346
347static void
348bucket_free(uma_bucket_t bucket)
349{
350 struct uma_bucket_zone *ubz;
351
352 ubz = bucket_zone_lookup(bucket->ub_entries);
353 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE, 0);
354}
355
356static void
357bucket_zone_drain(void)
358{
359 struct uma_bucket_zone *ubz;
360
361 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 174 unchanged lines hidden (view full) ---
536 */
537static void
538hash_free(struct uma_hash *hash)
539{
540 if (hash->uh_slab_hash == NULL)
541 return;
542 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
543 uma_zfree_internal(hashzone,
544 hash->uh_slab_hash, NULL, SKIP_NONE, 0);
545 else
546 free(hash->uh_slab_hash, M_UMAHASH);
547}
548
549/*
550 * Frees all outstanding items in a bucket
551 *
552 * Arguments:
--- 32 unchanged lines hidden (view full) ---
585 /*
586 * This is extremely inefficient. The slab pointer was passed
587 * to uma_zfree_arg, but we lost it because the buckets don't
588 * hold them. This will go away when free() gets a size passed
589 * to it.
590 */
591 if (mzone)
592 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
593 uma_zfree_internal(zone, item, slab, SKIP_DTOR, 0);
594 }
595}
596
597/*
598 * Drains the per cpu caches for a zone.
599 *
600 * NOTE: This may only be called while the zone is being turn down, and not
601 * during normal operation. This is necessary in order that we do not have
--- 150 unchanged lines hidden (view full) ---
752 else
753 obj = NULL;
754 for (i = 0; i < keg->uk_ppera; i++)
755 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
756 obj);
757 }
758 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
759 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
760 SKIP_NONE, 0);
761#ifdef UMA_DEBUG
762 printf("%s: Returning %d bytes.\n",
763 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
764#endif
765 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
766 }
767}
768
--- 46 unchanged lines hidden (view full) ---
815 else
816 wait &= ~M_ZERO;
817
818 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
819 &flags, wait);
820 if (mem == NULL) {
821 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
822 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
823 SKIP_NONE, 0);
824 ZONE_LOCK(zone);
825 return (NULL);
826 }
827
828 /* Point the slab into the allocated memory */
829 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
830 slab = (uma_slab_t )(mem + keg->uk_pgoff);
831
--- 40 unchanged lines hidden (view full) ---
872 else
873 obj = NULL;
874 for (i = 0; i < keg->uk_ppera; i++)
875 vsetobj((vm_offset_t)mem +
876 (i * PAGE_SIZE), obj);
877 }
878 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
879 uma_zfree_internal(keg->uk_slabzone, slab,
880 NULL, SKIP_NONE, 0);
881 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
882 flags);
883 ZONE_LOCK(zone);
884 return (NULL);
885 }
886 }
887 ZONE_LOCK(zone);
888
--- 447 unchanged lines hidden (view full) ---
1336 bzero(zone, size);
1337 zone->uz_name = arg->name;
1338 zone->uz_ctor = arg->ctor;
1339 zone->uz_dtor = arg->dtor;
1340 zone->uz_init = NULL;
1341 zone->uz_fini = NULL;
1342 zone->uz_allocs = 0;
1343 zone->uz_frees = 0;
1344 zone->uz_fails = 0;
1345 zone->uz_fills = zone->uz_count = 0;
1346
1347 if (arg->flags & UMA_ZONE_SECONDARY) {
1348 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1349 keg = arg->keg;
1350 zone->uz_keg = keg;
1351 zone->uz_init = arg->uminit;
1352 zone->uz_fini = arg->fini;
--- 110 unchanged lines hidden (view full) ---
1463 if (LIST_EMPTY(&keg->uk_zones))
1464 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1465 ZONE_UNLOCK(zone);
1466 mtx_unlock(&uma_mtx);
1467 } else {
1468 LIST_REMOVE(keg, uk_link);
1469 LIST_REMOVE(zone, uz_link);
1470 mtx_unlock(&uma_mtx);
1471 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE, 0);
1472 }
1473 zone->uz_keg = NULL;
1474}
1475
1476/*
1477 * Traverses every zone in the system and calls a callback
1478 *
1479 * Arguments:
--- 287 unchanged lines hidden (view full) ---
1767
1768 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1769}
1770
1771/* See uma.h */
1772void
1773uma_zdestroy(uma_zone_t zone)
1774{
1775 uma_zfree_internal(zones, zone, NULL, SKIP_NONE, 0);
1776}
1777
1778/* See uma.h */
1779void *
1780uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1781{
1782 void *item;
1783 uma_cache_t cache;
--- 67 unchanged lines hidden (view full) ---
1851 ZONE_LOCK(zone);
1852 uma_dbg_alloc(zone, NULL, item);
1853 ZONE_UNLOCK(zone);
1854#endif
1855 if (zone->uz_ctor != NULL) {
1856 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1857 udata, flags) != 0) {
1858 uma_zfree_internal(zone, item, udata,
1859 SKIP_DTOR, ZFREE_STATFAIL);
1860 return (NULL);
1861 }
1862 }
1863 if (flags & M_ZERO)
1864 bzero(item, zone->uz_keg->uk_size);
1865 return (item);
1866 } else if (cache->uc_freebucket) {
1867 /*
--- 290 unchanged lines hidden (view full) ---
2158 * If we couldn't initialize the whole bucket, put the
2159 * rest back onto the freelist.
2160 */
2161 if (i != bucket->ub_cnt) {
2162 int j;
2163
2164 for (j = i; j < bucket->ub_cnt; j++) {
2165 uma_zfree_internal(zone, bucket->ub_bucket[j],
2166 NULL, SKIP_FINI, 0);
2167#ifdef INVARIANTS
2168 bucket->ub_bucket[j] = NULL;
2169#endif
2170 }
2171 bucket->ub_cnt = i;
2172 }
2173 ZONE_LOCK(zone);
2174 }
--- 36 unchanged lines hidden (view full) ---
2211
2212#ifdef UMA_DEBUG_ALLOC
2213 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2214#endif
2215 ZONE_LOCK(zone);
2216
2217 slab = uma_zone_slab(zone, flags);
2218 if (slab == NULL) {
2219 zone->uz_fails++;
2220 ZONE_UNLOCK(zone);
2221 return (NULL);
2222 }
2223
2224 item = uma_slab_alloc(zone, slab);
2225
2226 zone->uz_allocs++;
2227
2228 ZONE_UNLOCK(zone);
2229
2230 /*
2231 * We have to call both the zone's init (not the keg's init)
2232 * and the zone's ctor. This is because the item is going from
2233 * a keg slab directly to the user, and the user is expecting it
2234 * to be both zone-init'd as well as zone-ctor'd.
2235 */
2236 if (zone->uz_init != NULL) {
2237 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2238 uma_zfree_internal(zone, item, udata, SKIP_FINI,
2239 ZFREE_STATFAIL);
2240 return (NULL);
2241 }
2242 }
2243 if (zone->uz_ctor != NULL) {
2244 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2245 uma_zfree_internal(zone, item, udata, SKIP_DTOR,
2246 ZFREE_STATFAIL);
2247 return (NULL);
2248 }
2249 }
2250 if (flags & M_ZERO)
2251 bzero(item, keg->uk_size);
2252
2253 return (item);
2254}
--- 156 unchanged lines hidden (view full) ---
2411 ZONE_UNLOCK(zone);
2412 goto zfree_restart;
2413 }
2414
2415 /*
2416 * If nothing else caught this, we'll just do an internal free.
2417 */
2418zfree_internal:
2419 uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL);
2420
2421 return;
2422}
2423
2424/*
2425 * Frees an item to an INTERNAL zone or allocates a free bucket
2426 *
2427 * Arguments:
2428 * zone The zone to free to
2429 * item The item we're freeing
2430 * udata User supplied data for the dtor
2431 * skip Skip dtors and finis
2432 */
2433static void
2434uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2435 enum zfreeskip skip, int flags)
2436{
2437 uma_slab_t slab;
2438 uma_slabrefcnt_t slabref;
2439 uma_keg_t keg;
2440 u_int8_t *mem;
2441 u_int8_t freei;
2442
2443 keg = zone->uz_keg;
2444
2445 if (skip < SKIP_DTOR && zone->uz_dtor)
2446 zone->uz_dtor(item, keg->uk_size, udata);
2447 if (skip < SKIP_FINI && zone->uz_fini)
2448 zone->uz_fini(item, keg->uk_size);
2449
2450 ZONE_LOCK(zone);
2451
2452 if (flags & ZFREE_STATFAIL)
2453 zone->uz_fails++;
2454
2455 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2456 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2457 if (keg->uk_flags & UMA_ZONE_HASH)
2458 slab = hash_sfind(&keg->uk_hash, mem);
2459 else {
2460 mem += keg->uk_pgoff;
2461 slab = (uma_slab_t)mem;
2462 }
--- 235 unchanged lines hidden (view full) ---
2698 return (NULL);
2699 mem = page_alloc(NULL, size, &flags, wait);
2700 if (mem) {
2701 vsetslab((vm_offset_t)mem, slab);
2702 slab->us_data = mem;
2703 slab->us_flags = flags | UMA_SLAB_MALLOC;
2704 slab->us_size = size;
2705 } else {
2706 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE,
2707 ZFREE_STATFAIL);
2708 }
2709
2710 return (mem);
2711}
2712
2713void
2714uma_large_free(uma_slab_t slab)
2715{
2716 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2717 page_free(slab->us_data, slab->us_size, slab->us_flags);
2718 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, 0);
2719}
2720
2721void
2722uma_print_stats(void)
2723{
2724 zone_foreach(uma_print_zone);
2725}
2726
--- 253 unchanged lines hidden (view full) ---
2980 kz->uk_ppera;
2981 else
2982 uth.uth_limit = kz->uk_maxpages *
2983 kz->uk_ipers;
2984 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2985 uth.uth_zone_free += bucket->ub_cnt;
2986 uth.uth_allocs = z->uz_allocs;
2987 uth.uth_frees = z->uz_frees;
2988 uth.uth_fails = z->uz_fails;
2989 ZONE_UNLOCK(z);
2990 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2991 mtx_unlock(&uma_mtx);
2992 error = ENOMEM;
2993 goto out;
2994 }
2995 /*
2996 * XXXRW: Should not access bucket fields from
--- 33 unchanged lines hidden ---
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
--- 35 unchanged lines hidden (view full) ---
44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 148070 2005-07-15 23:34:39Z rwatson $");
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60
--- 126 unchanged lines hidden (view full) ---
187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
195/*
196 * Flags and enumerations to be passed to internal functions.
197 */
198enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
199
200#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
201
202/* Prototypes.. */
203
204static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
205static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
206static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
207static void page_free(void *, int, u_int8_t);
208static uma_slab_t slab_zalloc(uma_zone_t, int);
209static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
219static void zone_foreach(void (*zfunc)(uma_zone_t));
220static void zone_timeout(uma_zone_t zone);
221static int hash_alloc(struct uma_hash *);
222static int hash_expand(struct uma_hash *, struct uma_hash *);
223static void hash_free(struct uma_hash *hash);
224static void uma_timeout(void *);
225static void uma_startup3(void);
226static void *uma_zalloc_internal(uma_zone_t, void *, int);
227static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip,
228 int);
229static void bucket_enable(void);
230static void bucket_init(void);
231static uma_bucket_t bucket_alloc(int, int);
232static void bucket_free(uma_bucket_t);
233static void bucket_zone_drain(void);
234static int uma_zalloc_bucket(uma_zone_t zone, int flags);
235static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
236static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);
--- 108 unchanged lines hidden (view full) ---
345}
346
347static void
348bucket_free(uma_bucket_t bucket)
349{
350 struct uma_bucket_zone *ubz;
351
352 ubz = bucket_zone_lookup(bucket->ub_entries);
353 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE, 0);
354}
355
356static void
357bucket_zone_drain(void)
358{
359 struct uma_bucket_zone *ubz;
360
361 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 174 unchanged lines hidden (view full) ---
536 */
537static void
538hash_free(struct uma_hash *hash)
539{
540 if (hash->uh_slab_hash == NULL)
541 return;
542 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
543 uma_zfree_internal(hashzone,
544 hash->uh_slab_hash, NULL, SKIP_NONE, 0);
545 else
546 free(hash->uh_slab_hash, M_UMAHASH);
547}
548
549/*
550 * Frees all outstanding items in a bucket
551 *
552 * Arguments:
--- 32 unchanged lines hidden (view full) ---
585 /*
586 * This is extremely inefficient. The slab pointer was passed
587 * to uma_zfree_arg, but we lost it because the buckets don't
588 * hold them. This will go away when free() gets a size passed
589 * to it.
590 */
591 if (mzone)
592 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
593 uma_zfree_internal(zone, item, slab, SKIP_DTOR, 0);
594 }
595}
596
597/*
598 * Drains the per cpu caches for a zone.
599 *
600 * NOTE: This may only be called while the zone is being turn down, and not
601 * during normal operation. This is necessary in order that we do not have
--- 150 unchanged lines hidden (view full) ---
752 else
753 obj = NULL;
754 for (i = 0; i < keg->uk_ppera; i++)
755 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
756 obj);
757 }
758 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
759 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
760 SKIP_NONE, 0);
761#ifdef UMA_DEBUG
762 printf("%s: Returning %d bytes.\n",
763 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
764#endif
765 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
766 }
767}
768
--- 46 unchanged lines hidden (view full) ---
815 else
816 wait &= ~M_ZERO;
817
818 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
819 &flags, wait);
820 if (mem == NULL) {
821 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
822 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
823 SKIP_NONE, 0);
824 ZONE_LOCK(zone);
825 return (NULL);
826 }
827
828 /* Point the slab into the allocated memory */
829 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
830 slab = (uma_slab_t )(mem + keg->uk_pgoff);
831
--- 40 unchanged lines hidden (view full) ---
872 else
873 obj = NULL;
874 for (i = 0; i < keg->uk_ppera; i++)
875 vsetobj((vm_offset_t)mem +
876 (i * PAGE_SIZE), obj);
877 }
878 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
879 uma_zfree_internal(keg->uk_slabzone, slab,
880 NULL, SKIP_NONE, 0);
881 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
882 flags);
883 ZONE_LOCK(zone);
884 return (NULL);
885 }
886 }
887 ZONE_LOCK(zone);
888
--- 447 unchanged lines hidden (view full) ---
1336 bzero(zone, size);
1337 zone->uz_name = arg->name;
1338 zone->uz_ctor = arg->ctor;
1339 zone->uz_dtor = arg->dtor;
1340 zone->uz_init = NULL;
1341 zone->uz_fini = NULL;
1342 zone->uz_allocs = 0;
1343 zone->uz_frees = 0;
1344 zone->uz_fails = 0;
1345 zone->uz_fills = zone->uz_count = 0;
1346
1347 if (arg->flags & UMA_ZONE_SECONDARY) {
1348 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1349 keg = arg->keg;
1350 zone->uz_keg = keg;
1351 zone->uz_init = arg->uminit;
1352 zone->uz_fini = arg->fini;
--- 110 unchanged lines hidden (view full) ---
1463 if (LIST_EMPTY(&keg->uk_zones))
1464 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1465 ZONE_UNLOCK(zone);
1466 mtx_unlock(&uma_mtx);
1467 } else {
1468 LIST_REMOVE(keg, uk_link);
1469 LIST_REMOVE(zone, uz_link);
1470 mtx_unlock(&uma_mtx);
1471 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE, 0);
1472 }
1473 zone->uz_keg = NULL;
1474}
1475
1476/*
1477 * Traverses every zone in the system and calls a callback
1478 *
1479 * Arguments:
--- 287 unchanged lines hidden (view full) ---
1767
1768 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1769}
1770
1771/* See uma.h */
1772void
1773uma_zdestroy(uma_zone_t zone)
1774{
1775 uma_zfree_internal(zones, zone, NULL, SKIP_NONE, 0);
1776}
1777
1778/* See uma.h */
1779void *
1780uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1781{
1782 void *item;
1783 uma_cache_t cache;
--- 67 unchanged lines hidden (view full) ---
1851 ZONE_LOCK(zone);
1852 uma_dbg_alloc(zone, NULL, item);
1853 ZONE_UNLOCK(zone);
1854#endif
1855 if (zone->uz_ctor != NULL) {
1856 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1857 udata, flags) != 0) {
1858 uma_zfree_internal(zone, item, udata,
1859 SKIP_DTOR, ZFREE_STATFAIL);
1860 return (NULL);
1861 }
1862 }
1863 if (flags & M_ZERO)
1864 bzero(item, zone->uz_keg->uk_size);
1865 return (item);
1866 } else if (cache->uc_freebucket) {
1867 /*
--- 290 unchanged lines hidden (view full) ---
2158 * If we couldn't initialize the whole bucket, put the
2159 * rest back onto the freelist.
2160 */
2161 if (i != bucket->ub_cnt) {
2162 int j;
2163
2164 for (j = i; j < bucket->ub_cnt; j++) {
2165 uma_zfree_internal(zone, bucket->ub_bucket[j],
2166 NULL, SKIP_FINI, 0);
2167#ifdef INVARIANTS
2168 bucket->ub_bucket[j] = NULL;
2169#endif
2170 }
2171 bucket->ub_cnt = i;
2172 }
2173 ZONE_LOCK(zone);
2174 }
--- 36 unchanged lines hidden (view full) ---
2211
2212#ifdef UMA_DEBUG_ALLOC
2213 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2214#endif
2215 ZONE_LOCK(zone);
2216
2217 slab = uma_zone_slab(zone, flags);
2218 if (slab == NULL) {
2219 zone->uz_fails++;
2220 ZONE_UNLOCK(zone);
2221 return (NULL);
2222 }
2223
2224 item = uma_slab_alloc(zone, slab);
2225
2226 zone->uz_allocs++;
2227
2228 ZONE_UNLOCK(zone);
2229
2230 /*
2231 * We have to call both the zone's init (not the keg's init)
2232 * and the zone's ctor. This is because the item is going from
2233 * a keg slab directly to the user, and the user is expecting it
2234 * to be both zone-init'd as well as zone-ctor'd.
2235 */
2236 if (zone->uz_init != NULL) {
2237 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2238 uma_zfree_internal(zone, item, udata, SKIP_FINI,
2239 ZFREE_STATFAIL);
2240 return (NULL);
2241 }
2242 }
2243 if (zone->uz_ctor != NULL) {
2244 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2245 uma_zfree_internal(zone, item, udata, SKIP_DTOR,
2246 ZFREE_STATFAIL);
2247 return (NULL);
2248 }
2249 }
2250 if (flags & M_ZERO)
2251 bzero(item, keg->uk_size);
2252
2253 return (item);
2254}
--- 156 unchanged lines hidden (view full) ---
2411 ZONE_UNLOCK(zone);
2412 goto zfree_restart;
2413 }
2414
2415 /*
2416 * If nothing else caught this, we'll just do an internal free.
2417 */
2418zfree_internal:
2419 uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL);
2420
2421 return;
2422}
2423
2424/*
2425 * Frees an item to an INTERNAL zone or allocates a free bucket
2426 *
2427 * Arguments:
2428 * zone The zone to free to
2429 * item The item we're freeing
2430 * udata User supplied data for the dtor
2431 * skip Skip dtors and finis
2432 */
2433static void
2434uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2435 enum zfreeskip skip, int flags)
2436{
2437 uma_slab_t slab;
2438 uma_slabrefcnt_t slabref;
2439 uma_keg_t keg;
2440 u_int8_t *mem;
2441 u_int8_t freei;
2442
2443 keg = zone->uz_keg;
2444
2445 if (skip < SKIP_DTOR && zone->uz_dtor)
2446 zone->uz_dtor(item, keg->uk_size, udata);
2447 if (skip < SKIP_FINI && zone->uz_fini)
2448 zone->uz_fini(item, keg->uk_size);
2449
2450 ZONE_LOCK(zone);
2451
2452 if (flags & ZFREE_STATFAIL)
2453 zone->uz_fails++;
2454
2455 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2456 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2457 if (keg->uk_flags & UMA_ZONE_HASH)
2458 slab = hash_sfind(&keg->uk_hash, mem);
2459 else {
2460 mem += keg->uk_pgoff;
2461 slab = (uma_slab_t)mem;
2462 }
--- 235 unchanged lines hidden (view full) ---
2698 return (NULL);
2699 mem = page_alloc(NULL, size, &flags, wait);
2700 if (mem) {
2701 vsetslab((vm_offset_t)mem, slab);
2702 slab->us_data = mem;
2703 slab->us_flags = flags | UMA_SLAB_MALLOC;
2704 slab->us_size = size;
2705 } else {
2706 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE,
2707 ZFREE_STATFAIL);
2708 }
2709
2710 return (mem);
2711}
2712
2713void
2714uma_large_free(uma_slab_t slab)
2715{
2716 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2717 page_free(slab->us_data, slab->us_size, slab->us_flags);
2718 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, 0);
2719}
2720
2721void
2722uma_print_stats(void)
2723{
2724 zone_foreach(uma_print_zone);
2725}
2726
--- 253 unchanged lines hidden (view full) ---
2980 kz->uk_ppera;
2981 else
2982 uth.uth_limit = kz->uk_maxpages *
2983 kz->uk_ipers;
2984 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2985 uth.uth_zone_free += bucket->ub_cnt;
2986 uth.uth_allocs = z->uz_allocs;
2987 uth.uth_frees = z->uz_frees;
2988 uth.uth_fails = z->uz_fails;
2989 ZONE_UNLOCK(z);
2990 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2991 mtx_unlock(&uma_mtx);
2992 error = ENOMEM;
2993 goto out;
2994 }
2995 /*
2996 * XXXRW: Should not access bucket fields from
--- 33 unchanged lines hidden ---