1/*-
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
--- 35 unchanged lines hidden (view full) ---
44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 147996 2005-07-14 16:35:13Z rwatson $");
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60
--- 126 unchanged lines hidden (view full) ---
187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
195enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
196
197/* Prototypes.. */
198
199static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
200static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
201static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
202static void page_free(void *, int, u_int8_t);
203static uma_slab_t slab_zalloc(uma_zone_t, int);
204static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
214static void zone_foreach(void (*zfunc)(uma_zone_t));
215static void zone_timeout(uma_zone_t zone);
216static int hash_alloc(struct uma_hash *);
217static int hash_expand(struct uma_hash *, struct uma_hash *);
218static void hash_free(struct uma_hash *hash);
219static void uma_timeout(void *);
220static void uma_startup3(void);
221static void *uma_zalloc_internal(uma_zone_t, void *, int);
222static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip);
223static void bucket_enable(void);
224static void bucket_init(void);
225static uma_bucket_t bucket_alloc(int, int);
226static void bucket_free(uma_bucket_t);
227static void bucket_zone_drain(void);
228static int uma_zalloc_bucket(uma_zone_t zone, int flags);
229static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
230static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);
--- 108 unchanged lines hidden (view full) ---
339}
340
341static void
342bucket_free(uma_bucket_t bucket)
343{
344 struct uma_bucket_zone *ubz;
345
346 ubz = bucket_zone_lookup(bucket->ub_entries);
347 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
348}
349
350static void
351bucket_zone_drain(void)
352{
353 struct uma_bucket_zone *ubz;
354
355 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 174 unchanged lines hidden (view full) ---
530 */
531static void
532hash_free(struct uma_hash *hash)
533{
534 if (hash->uh_slab_hash == NULL)
535 return;
536 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
537 uma_zfree_internal(hashzone,
538 hash->uh_slab_hash, NULL, SKIP_NONE);
539 else
540 free(hash->uh_slab_hash, M_UMAHASH);
541}
542
543/*
544 * Frees all outstanding items in a bucket
545 *
546 * Arguments:
--- 32 unchanged lines hidden (view full) ---
579 /*
580 * This is extremely inefficient. The slab pointer was passed
581 * to uma_zfree_arg, but we lost it because the buckets don't
582 * hold them. This will go away when free() gets a size passed
583 * to it.
584 */
585 if (mzone)
586 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
587 uma_zfree_internal(zone, item, slab, SKIP_DTOR);
588 }
589}
590
591/*
592 * Drains the per cpu caches for a zone.
593 *
594 * NOTE: This may only be called while the zone is being turn down, and not
595 * during normal operation. This is necessary in order that we do not have
--- 150 unchanged lines hidden (view full) ---
746 else
747 obj = NULL;
748 for (i = 0; i < keg->uk_ppera; i++)
749 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
750 obj);
751 }
752 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
753 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
754 SKIP_NONE);
755#ifdef UMA_DEBUG
756 printf("%s: Returning %d bytes.\n",
757 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
758#endif
759 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
760 }
761}
762
--- 46 unchanged lines hidden (view full) ---
809 else
810 wait &= ~M_ZERO;
811
812 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
813 &flags, wait);
814 if (mem == NULL) {
815 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
816 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
817 SKIP_NONE);
818 ZONE_LOCK(zone);
819 return (NULL);
820 }
821
822 /* Point the slab into the allocated memory */
823 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
824 slab = (uma_slab_t )(mem + keg->uk_pgoff);
825
--- 40 unchanged lines hidden (view full) ---
866 else
867 obj = NULL;
868 for (i = 0; i < keg->uk_ppera; i++)
869 vsetobj((vm_offset_t)mem +
870 (i * PAGE_SIZE), obj);
871 }
872 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
873 uma_zfree_internal(keg->uk_slabzone, slab,
874 NULL, SKIP_NONE);
875 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
876 flags);
877 ZONE_LOCK(zone);
878 return (NULL);
879 }
880 }
881 ZONE_LOCK(zone);
882
--- 447 unchanged lines hidden (view full) ---
1330 bzero(zone, size);
1331 zone->uz_name = arg->name;
1332 zone->uz_ctor = arg->ctor;
1333 zone->uz_dtor = arg->dtor;
1334 zone->uz_init = NULL;
1335 zone->uz_fini = NULL;
1336 zone->uz_allocs = 0;
1337 zone->uz_frees = 0;
1338 zone->uz_fills = zone->uz_count = 0;
1339
1340 if (arg->flags & UMA_ZONE_SECONDARY) {
1341 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1342 keg = arg->keg;
1343 zone->uz_keg = keg;
1344 zone->uz_init = arg->uminit;
1345 zone->uz_fini = arg->fini;
--- 110 unchanged lines hidden (view full) ---
1456 if (LIST_EMPTY(&keg->uk_zones))
1457 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1458 ZONE_UNLOCK(zone);
1459 mtx_unlock(&uma_mtx);
1460 } else {
1461 LIST_REMOVE(keg, uk_link);
1462 LIST_REMOVE(zone, uz_link);
1463 mtx_unlock(&uma_mtx);
1464 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE);
1465 }
1466 zone->uz_keg = NULL;
1467}
1468
1469/*
1470 * Traverses every zone in the system and calls a callback
1471 *
1472 * Arguments:
--- 287 unchanged lines hidden (view full) ---
1760
1761 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1762}
1763
1764/* See uma.h */
1765void
1766uma_zdestroy(uma_zone_t zone)
1767{
1768 uma_zfree_internal(zones, zone, NULL, SKIP_NONE);
1769}
1770
1771/* See uma.h */
1772void *
1773uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1774{
1775 void *item;
1776 uma_cache_t cache;
--- 67 unchanged lines hidden (view full) ---
1844 ZONE_LOCK(zone);
1845 uma_dbg_alloc(zone, NULL, item);
1846 ZONE_UNLOCK(zone);
1847#endif
1848 if (zone->uz_ctor != NULL) {
1849 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1850 udata, flags) != 0) {
1851 uma_zfree_internal(zone, item, udata,
1852 SKIP_DTOR);
1853 return (NULL);
1854 }
1855 }
1856 if (flags & M_ZERO)
1857 bzero(item, zone->uz_keg->uk_size);
1858 return (item);
1859 } else if (cache->uc_freebucket) {
1860 /*
--- 290 unchanged lines hidden (view full) ---
2151 * If we couldn't initialize the whole bucket, put the
2152 * rest back onto the freelist.
2153 */
2154 if (i != bucket->ub_cnt) {
2155 int j;
2156
2157 for (j = i; j < bucket->ub_cnt; j++) {
2158 uma_zfree_internal(zone, bucket->ub_bucket[j],
2159 NULL, SKIP_FINI);
2160#ifdef INVARIANTS
2161 bucket->ub_bucket[j] = NULL;
2162#endif
2163 }
2164 bucket->ub_cnt = i;
2165 }
2166 ZONE_LOCK(zone);
2167 }
--- 36 unchanged lines hidden (view full) ---
2204
2205#ifdef UMA_DEBUG_ALLOC
2206 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2207#endif
2208 ZONE_LOCK(zone);
2209
2210 slab = uma_zone_slab(zone, flags);
2211 if (slab == NULL) {
2212 ZONE_UNLOCK(zone);
2213 return (NULL);
2214 }
2215
2216 item = uma_slab_alloc(zone, slab);
2217
2218 zone->uz_allocs++;
2219
2220 ZONE_UNLOCK(zone);
2221
2222 /*
2223 * We have to call both the zone's init (not the keg's init)
2224 * and the zone's ctor. This is because the item is going from
2225 * a keg slab directly to the user, and the user is expecting it
2226 * to be both zone-init'd as well as zone-ctor'd.
2227 */
2228 if (zone->uz_init != NULL) {
2229 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2230 uma_zfree_internal(zone, item, udata, SKIP_FINI);
2231 return (NULL);
2232 }
2233 }
2234 if (zone->uz_ctor != NULL) {
2235 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2236 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2237 return (NULL);
2238 }
2239 }
2240 if (flags & M_ZERO)
2241 bzero(item, keg->uk_size);
2242
2243 return (item);
2244}
--- 156 unchanged lines hidden (view full) ---
2401 ZONE_UNLOCK(zone);
2402 goto zfree_restart;
2403 }
2404
2405 /*
2406 * If nothing else caught this, we'll just do an internal free.
2407 */
2408zfree_internal:
2409 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2410
2411 return;
2412}
2413
2414/*
2415 * Frees an item to an INTERNAL zone or allocates a free bucket
2416 *
2417 * Arguments:
2418 * zone The zone to free to
2419 * item The item we're freeing
2420 * udata User supplied data for the dtor
2421 * skip Skip dtors and finis
2422 */
2423static void
2424uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2425 enum zfreeskip skip)
2426{
2427 uma_slab_t slab;
2428 uma_slabrefcnt_t slabref;
2429 uma_keg_t keg;
2430 u_int8_t *mem;
2431 u_int8_t freei;
2432
2433 keg = zone->uz_keg;
2434
2435 if (skip < SKIP_DTOR && zone->uz_dtor)
2436 zone->uz_dtor(item, keg->uk_size, udata);
2437 if (skip < SKIP_FINI && zone->uz_fini)
2438 zone->uz_fini(item, keg->uk_size);
2439
2440 ZONE_LOCK(zone);
2441
2442 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2443 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2444 if (keg->uk_flags & UMA_ZONE_HASH)
2445 slab = hash_sfind(&keg->uk_hash, mem);
2446 else {
2447 mem += keg->uk_pgoff;
2448 slab = (uma_slab_t)mem;
2449 }
--- 235 unchanged lines hidden (view full) ---
2685 return (NULL);
2686 mem = page_alloc(NULL, size, &flags, wait);
2687 if (mem) {
2688 vsetslab((vm_offset_t)mem, slab);
2689 slab->us_data = mem;
2690 slab->us_flags = flags | UMA_SLAB_MALLOC;
2691 slab->us_size = size;
2692 } else {
2693 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2694 }
2695
2696 return (mem);
2697}
2698
2699void
2700uma_large_free(uma_slab_t slab)
2701{
2702 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2703 page_free(slab->us_data, slab->us_size, slab->us_flags);
2704 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2705}
2706
2707void
2708uma_print_stats(void)
2709{
2710 zone_foreach(uma_print_zone);
2711}
2712
--- 253 unchanged lines hidden (view full) ---
2966 kz->uk_ppera;
2967 else
2968 uth.uth_limit = kz->uk_maxpages *
2969 kz->uk_ipers;
2970 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2971 uth.uth_zone_free += bucket->ub_cnt;
2972 uth.uth_allocs = z->uz_allocs;
2973 uth.uth_frees = z->uz_frees;
2974 ZONE_UNLOCK(z);
2975 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2976 mtx_unlock(&uma_mtx);
2977 error = ENOMEM;
2978 goto out;
2979 }
2980 /*
2981 * XXXRW: Should not access bucket fields from
--- 33 unchanged lines hidden ---
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
--- 35 unchanged lines hidden (view full) ---
44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 147996 2005-07-14 16:35:13Z rwatson $");
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60
--- 126 unchanged lines hidden (view full) ---
187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
195enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
196
197/* Prototypes.. */
198
199static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
200static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
201static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
202static void page_free(void *, int, u_int8_t);
203static uma_slab_t slab_zalloc(uma_zone_t, int);
204static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
214static void zone_foreach(void (*zfunc)(uma_zone_t));
215static void zone_timeout(uma_zone_t zone);
216static int hash_alloc(struct uma_hash *);
217static int hash_expand(struct uma_hash *, struct uma_hash *);
218static void hash_free(struct uma_hash *hash);
219static void uma_timeout(void *);
220static void uma_startup3(void);
221static void *uma_zalloc_internal(uma_zone_t, void *, int);
222static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip);
223static void bucket_enable(void);
224static void bucket_init(void);
225static uma_bucket_t bucket_alloc(int, int);
226static void bucket_free(uma_bucket_t);
227static void bucket_zone_drain(void);
228static int uma_zalloc_bucket(uma_zone_t zone, int flags);
229static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
230static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);
--- 108 unchanged lines hidden (view full) ---
339}
340
341static void
342bucket_free(uma_bucket_t bucket)
343{
344 struct uma_bucket_zone *ubz;
345
346 ubz = bucket_zone_lookup(bucket->ub_entries);
347 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
348}
349
350static void
351bucket_zone_drain(void)
352{
353 struct uma_bucket_zone *ubz;
354
355 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 174 unchanged lines hidden (view full) ---
530 */
531static void
532hash_free(struct uma_hash *hash)
533{
534 if (hash->uh_slab_hash == NULL)
535 return;
536 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
537 uma_zfree_internal(hashzone,
538 hash->uh_slab_hash, NULL, SKIP_NONE);
539 else
540 free(hash->uh_slab_hash, M_UMAHASH);
541}
542
543/*
544 * Frees all outstanding items in a bucket
545 *
546 * Arguments:
--- 32 unchanged lines hidden (view full) ---
579 /*
580 * This is extremely inefficient. The slab pointer was passed
581 * to uma_zfree_arg, but we lost it because the buckets don't
582 * hold them. This will go away when free() gets a size passed
583 * to it.
584 */
585 if (mzone)
586 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
587 uma_zfree_internal(zone, item, slab, SKIP_DTOR);
588 }
589}
590
591/*
592 * Drains the per cpu caches for a zone.
593 *
594 * NOTE: This may only be called while the zone is being turn down, and not
595 * during normal operation. This is necessary in order that we do not have
--- 150 unchanged lines hidden (view full) ---
746 else
747 obj = NULL;
748 for (i = 0; i < keg->uk_ppera; i++)
749 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
750 obj);
751 }
752 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
753 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
754 SKIP_NONE);
755#ifdef UMA_DEBUG
756 printf("%s: Returning %d bytes.\n",
757 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
758#endif
759 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
760 }
761}
762
--- 46 unchanged lines hidden (view full) ---
809 else
810 wait &= ~M_ZERO;
811
812 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
813 &flags, wait);
814 if (mem == NULL) {
815 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
816 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
817 SKIP_NONE);
818 ZONE_LOCK(zone);
819 return (NULL);
820 }
821
822 /* Point the slab into the allocated memory */
823 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
824 slab = (uma_slab_t )(mem + keg->uk_pgoff);
825
--- 40 unchanged lines hidden (view full) ---
866 else
867 obj = NULL;
868 for (i = 0; i < keg->uk_ppera; i++)
869 vsetobj((vm_offset_t)mem +
870 (i * PAGE_SIZE), obj);
871 }
872 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
873 uma_zfree_internal(keg->uk_slabzone, slab,
874 NULL, SKIP_NONE);
875 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
876 flags);
877 ZONE_LOCK(zone);
878 return (NULL);
879 }
880 }
881 ZONE_LOCK(zone);
882
--- 447 unchanged lines hidden (view full) ---
1330 bzero(zone, size);
1331 zone->uz_name = arg->name;
1332 zone->uz_ctor = arg->ctor;
1333 zone->uz_dtor = arg->dtor;
1334 zone->uz_init = NULL;
1335 zone->uz_fini = NULL;
1336 zone->uz_allocs = 0;
1337 zone->uz_frees = 0;
1338 zone->uz_fills = zone->uz_count = 0;
1339
1340 if (arg->flags & UMA_ZONE_SECONDARY) {
1341 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1342 keg = arg->keg;
1343 zone->uz_keg = keg;
1344 zone->uz_init = arg->uminit;
1345 zone->uz_fini = arg->fini;
--- 110 unchanged lines hidden (view full) ---
1456 if (LIST_EMPTY(&keg->uk_zones))
1457 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1458 ZONE_UNLOCK(zone);
1459 mtx_unlock(&uma_mtx);
1460 } else {
1461 LIST_REMOVE(keg, uk_link);
1462 LIST_REMOVE(zone, uz_link);
1463 mtx_unlock(&uma_mtx);
1464 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE);
1465 }
1466 zone->uz_keg = NULL;
1467}
1468
1469/*
1470 * Traverses every zone in the system and calls a callback
1471 *
1472 * Arguments:
--- 287 unchanged lines hidden (view full) ---
1760
1761 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1762}
1763
1764/* See uma.h */
1765void
1766uma_zdestroy(uma_zone_t zone)
1767{
1768 uma_zfree_internal(zones, zone, NULL, SKIP_NONE);
1769}
1770
1771/* See uma.h */
1772void *
1773uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1774{
1775 void *item;
1776 uma_cache_t cache;
--- 67 unchanged lines hidden (view full) ---
1844 ZONE_LOCK(zone);
1845 uma_dbg_alloc(zone, NULL, item);
1846 ZONE_UNLOCK(zone);
1847#endif
1848 if (zone->uz_ctor != NULL) {
1849 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1850 udata, flags) != 0) {
1851 uma_zfree_internal(zone, item, udata,
1852 SKIP_DTOR);
1853 return (NULL);
1854 }
1855 }
1856 if (flags & M_ZERO)
1857 bzero(item, zone->uz_keg->uk_size);
1858 return (item);
1859 } else if (cache->uc_freebucket) {
1860 /*
--- 290 unchanged lines hidden (view full) ---
2151 * If we couldn't initialize the whole bucket, put the
2152 * rest back onto the freelist.
2153 */
2154 if (i != bucket->ub_cnt) {
2155 int j;
2156
2157 for (j = i; j < bucket->ub_cnt; j++) {
2158 uma_zfree_internal(zone, bucket->ub_bucket[j],
2159 NULL, SKIP_FINI);
2160#ifdef INVARIANTS
2161 bucket->ub_bucket[j] = NULL;
2162#endif
2163 }
2164 bucket->ub_cnt = i;
2165 }
2166 ZONE_LOCK(zone);
2167 }
--- 36 unchanged lines hidden (view full) ---
2204
2205#ifdef UMA_DEBUG_ALLOC
2206 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2207#endif
2208 ZONE_LOCK(zone);
2209
2210 slab = uma_zone_slab(zone, flags);
2211 if (slab == NULL) {
2212 ZONE_UNLOCK(zone);
2213 return (NULL);
2214 }
2215
2216 item = uma_slab_alloc(zone, slab);
2217
2218 zone->uz_allocs++;
2219
2220 ZONE_UNLOCK(zone);
2221
2222 /*
2223 * We have to call both the zone's init (not the keg's init)
2224 * and the zone's ctor. This is because the item is going from
2225 * a keg slab directly to the user, and the user is expecting it
2226 * to be both zone-init'd as well as zone-ctor'd.
2227 */
2228 if (zone->uz_init != NULL) {
2229 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2230 uma_zfree_internal(zone, item, udata, SKIP_FINI);
2231 return (NULL);
2232 }
2233 }
2234 if (zone->uz_ctor != NULL) {
2235 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2236 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2237 return (NULL);
2238 }
2239 }
2240 if (flags & M_ZERO)
2241 bzero(item, keg->uk_size);
2242
2243 return (item);
2244}
--- 156 unchanged lines hidden (view full) ---
2401 ZONE_UNLOCK(zone);
2402 goto zfree_restart;
2403 }
2404
2405 /*
2406 * If nothing else caught this, we'll just do an internal free.
2407 */
2408zfree_internal:
2409 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2410
2411 return;
2412}
2413
2414/*
2415 * Frees an item to an INTERNAL zone or allocates a free bucket
2416 *
2417 * Arguments:
2418 * zone The zone to free to
2419 * item The item we're freeing
2420 * udata User supplied data for the dtor
2421 * skip Skip dtors and finis
2422 */
2423static void
2424uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2425 enum zfreeskip skip)
2426{
2427 uma_slab_t slab;
2428 uma_slabrefcnt_t slabref;
2429 uma_keg_t keg;
2430 u_int8_t *mem;
2431 u_int8_t freei;
2432
2433 keg = zone->uz_keg;
2434
2435 if (skip < SKIP_DTOR && zone->uz_dtor)
2436 zone->uz_dtor(item, keg->uk_size, udata);
2437 if (skip < SKIP_FINI && zone->uz_fini)
2438 zone->uz_fini(item, keg->uk_size);
2439
2440 ZONE_LOCK(zone);
2441
2442 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2443 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2444 if (keg->uk_flags & UMA_ZONE_HASH)
2445 slab = hash_sfind(&keg->uk_hash, mem);
2446 else {
2447 mem += keg->uk_pgoff;
2448 slab = (uma_slab_t)mem;
2449 }
--- 235 unchanged lines hidden (view full) ---
2685 return (NULL);
2686 mem = page_alloc(NULL, size, &flags, wait);
2687 if (mem) {
2688 vsetslab((vm_offset_t)mem, slab);
2689 slab->us_data = mem;
2690 slab->us_flags = flags | UMA_SLAB_MALLOC;
2691 slab->us_size = size;
2692 } else {
2693 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2694 }
2695
2696 return (mem);
2697}
2698
2699void
2700uma_large_free(uma_slab_t slab)
2701{
2702 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2703 page_free(slab->us_data, slab->us_size, slab->us_flags);
2704 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2705}
2706
2707void
2708uma_print_stats(void)
2709{
2710 zone_foreach(uma_print_zone);
2711}
2712
--- 253 unchanged lines hidden (view full) ---
2966 kz->uk_ppera;
2967 else
2968 uth.uth_limit = kz->uk_maxpages *
2969 kz->uk_ipers;
2970 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2971 uth.uth_zone_free += bucket->ub_cnt;
2972 uth.uth_allocs = z->uz_allocs;
2973 uth.uth_frees = z->uz_frees;
2974 ZONE_UNLOCK(z);
2975 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2976 mtx_unlock(&uma_mtx);
2977 error = ENOMEM;
2978 goto out;
2979 }
2980 /*
2981 * XXXRW: Should not access bucket fields from
--- 33 unchanged lines hidden ---