Deleted Added
sdiff udiff text old ( 147996 ) new ( 148070 )
full compact
uma_core.c (147996) uma_core.c (148070)
1/*-
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 35 unchanged lines hidden (view full) ---

44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2004-2005 Robert N. M. Watson
3 * Copyright (c) 2004, 2005,
4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved.
5 * Copyright (c) 2002, 2003, 2004, 2005,
6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without

--- 35 unchanged lines hidden (view full) ---

44
45/*
46 * TODO:
47 * - Improve memory usage for large allocations
48 * - Investigate cache size adjustments
49 */
50
51#include <sys/cdefs.h>
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 147996 2005-07-14 16:35:13Z rwatson $");
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 148070 2005-07-15 23:34:39Z rwatson $");
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60

--- 126 unchanged lines hidden (view full) ---

187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
53
54/* I should really use ktr.. */
55/*
56#define UMA_DEBUG 1
57#define UMA_DEBUG_ALLOC 1
58#define UMA_DEBUG_ALLOC_1 1
59*/
60

--- 126 unchanged lines hidden (view full) ---

187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1)
188
189/*
190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket
191 * of approximately the right size.
192 */
193static uint8_t bucket_size[BUCKET_ZONES];
194
195/*
196 * Flags and enumerations to be passed to internal functions.
197 */
195enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
196
198enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI };
199
200#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
201
197/* Prototypes.. */
198
199static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
200static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
201static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
202static void page_free(void *, int, u_int8_t);
203static uma_slab_t slab_zalloc(uma_zone_t, int);
204static void cache_drain(uma_zone_t);

--- 9 unchanged lines hidden (view full) ---

214static void zone_foreach(void (*zfunc)(uma_zone_t));
215static void zone_timeout(uma_zone_t zone);
216static int hash_alloc(struct uma_hash *);
217static int hash_expand(struct uma_hash *, struct uma_hash *);
218static void hash_free(struct uma_hash *hash);
219static void uma_timeout(void *);
220static void uma_startup3(void);
221static void *uma_zalloc_internal(uma_zone_t, void *, int);
202/* Prototypes.. */
203
204static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
205static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
206static void *startup_alloc(uma_zone_t, int, u_int8_t *, int);
207static void page_free(void *, int, u_int8_t);
208static uma_slab_t slab_zalloc(uma_zone_t, int);
209static void cache_drain(uma_zone_t);

--- 9 unchanged lines hidden (view full) ---

219static void zone_foreach(void (*zfunc)(uma_zone_t));
220static void zone_timeout(uma_zone_t zone);
221static int hash_alloc(struct uma_hash *);
222static int hash_expand(struct uma_hash *, struct uma_hash *);
223static void hash_free(struct uma_hash *hash);
224static void uma_timeout(void *);
225static void uma_startup3(void);
226static void *uma_zalloc_internal(uma_zone_t, void *, int);
222static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip);
227static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip,
228 int);
223static void bucket_enable(void);
224static void bucket_init(void);
225static uma_bucket_t bucket_alloc(int, int);
226static void bucket_free(uma_bucket_t);
227static void bucket_zone_drain(void);
228static int uma_zalloc_bucket(uma_zone_t zone, int flags);
229static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
230static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);

--- 108 unchanged lines hidden (view full) ---

339}
340
341static void
342bucket_free(uma_bucket_t bucket)
343{
344 struct uma_bucket_zone *ubz;
345
346 ubz = bucket_zone_lookup(bucket->ub_entries);
229static void bucket_enable(void);
230static void bucket_init(void);
231static uma_bucket_t bucket_alloc(int, int);
232static void bucket_free(uma_bucket_t);
233static void bucket_zone_drain(void);
234static int uma_zalloc_bucket(uma_zone_t zone, int flags);
235static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags);
236static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab);

--- 108 unchanged lines hidden (view full) ---

345}
346
347static void
348bucket_free(uma_bucket_t bucket)
349{
350 struct uma_bucket_zone *ubz;
351
352 ubz = bucket_zone_lookup(bucket->ub_entries);
347 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
353 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE, 0);
348}
349
350static void
351bucket_zone_drain(void)
352{
353 struct uma_bucket_zone *ubz;
354
355 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)

--- 174 unchanged lines hidden (view full) ---

530 */
531static void
532hash_free(struct uma_hash *hash)
533{
534 if (hash->uh_slab_hash == NULL)
535 return;
536 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
537 uma_zfree_internal(hashzone,
354}
355
356static void
357bucket_zone_drain(void)
358{
359 struct uma_bucket_zone *ubz;
360
361 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)

--- 174 unchanged lines hidden (view full) ---

536 */
537static void
538hash_free(struct uma_hash *hash)
539{
540 if (hash->uh_slab_hash == NULL)
541 return;
542 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
543 uma_zfree_internal(hashzone,
538 hash->uh_slab_hash, NULL, SKIP_NONE);
544 hash->uh_slab_hash, NULL, SKIP_NONE, 0);
539 else
540 free(hash->uh_slab_hash, M_UMAHASH);
541}
542
543/*
544 * Frees all outstanding items in a bucket
545 *
546 * Arguments:

--- 32 unchanged lines hidden (view full) ---

579 /*
580 * This is extremely inefficient. The slab pointer was passed
581 * to uma_zfree_arg, but we lost it because the buckets don't
582 * hold them. This will go away when free() gets a size passed
583 * to it.
584 */
585 if (mzone)
586 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
545 else
546 free(hash->uh_slab_hash, M_UMAHASH);
547}
548
549/*
550 * Frees all outstanding items in a bucket
551 *
552 * Arguments:

--- 32 unchanged lines hidden (view full) ---

585 /*
586 * This is extremely inefficient. The slab pointer was passed
587 * to uma_zfree_arg, but we lost it because the buckets don't
588 * hold them. This will go away when free() gets a size passed
589 * to it.
590 */
591 if (mzone)
592 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK));
587 uma_zfree_internal(zone, item, slab, SKIP_DTOR);
593 uma_zfree_internal(zone, item, slab, SKIP_DTOR, 0);
588 }
589}
590
591/*
592 * Drains the per cpu caches for a zone.
593 *
594 * NOTE: This may only be called while the zone is being turn down, and not
595 * during normal operation. This is necessary in order that we do not have

--- 150 unchanged lines hidden (view full) ---

746 else
747 obj = NULL;
748 for (i = 0; i < keg->uk_ppera; i++)
749 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
750 obj);
751 }
752 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
753 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
594 }
595}
596
597/*
598 * Drains the per cpu caches for a zone.
599 *
600 * NOTE: This may only be called while the zone is being turn down, and not
601 * during normal operation. This is necessary in order that we do not have

--- 150 unchanged lines hidden (view full) ---

752 else
753 obj = NULL;
754 for (i = 0; i < keg->uk_ppera; i++)
755 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
756 obj);
757 }
758 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
759 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
754 SKIP_NONE);
760 SKIP_NONE, 0);
755#ifdef UMA_DEBUG
756 printf("%s: Returning %d bytes.\n",
757 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
758#endif
759 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
760 }
761}
762

--- 46 unchanged lines hidden (view full) ---

809 else
810 wait &= ~M_ZERO;
811
812 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
813 &flags, wait);
814 if (mem == NULL) {
815 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
816 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
761#ifdef UMA_DEBUG
762 printf("%s: Returning %d bytes.\n",
763 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera);
764#endif
765 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
766 }
767}
768

--- 46 unchanged lines hidden (view full) ---

815 else
816 wait &= ~M_ZERO;
817
818 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE,
819 &flags, wait);
820 if (mem == NULL) {
821 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
822 uma_zfree_internal(keg->uk_slabzone, slab, NULL,
817 SKIP_NONE);
823 SKIP_NONE, 0);
818 ZONE_LOCK(zone);
819 return (NULL);
820 }
821
822 /* Point the slab into the allocated memory */
823 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
824 slab = (uma_slab_t )(mem + keg->uk_pgoff);
825

--- 40 unchanged lines hidden (view full) ---

866 else
867 obj = NULL;
868 for (i = 0; i < keg->uk_ppera; i++)
869 vsetobj((vm_offset_t)mem +
870 (i * PAGE_SIZE), obj);
871 }
872 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
873 uma_zfree_internal(keg->uk_slabzone, slab,
824 ZONE_LOCK(zone);
825 return (NULL);
826 }
827
828 /* Point the slab into the allocated memory */
829 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
830 slab = (uma_slab_t )(mem + keg->uk_pgoff);
831

--- 40 unchanged lines hidden (view full) ---

872 else
873 obj = NULL;
874 for (i = 0; i < keg->uk_ppera; i++)
875 vsetobj((vm_offset_t)mem +
876 (i * PAGE_SIZE), obj);
877 }
878 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
879 uma_zfree_internal(keg->uk_slabzone, slab,
874 NULL, SKIP_NONE);
880 NULL, SKIP_NONE, 0);
875 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
876 flags);
877 ZONE_LOCK(zone);
878 return (NULL);
879 }
880 }
881 ZONE_LOCK(zone);
882

--- 447 unchanged lines hidden (view full) ---

1330 bzero(zone, size);
1331 zone->uz_name = arg->name;
1332 zone->uz_ctor = arg->ctor;
1333 zone->uz_dtor = arg->dtor;
1334 zone->uz_init = NULL;
1335 zone->uz_fini = NULL;
1336 zone->uz_allocs = 0;
1337 zone->uz_frees = 0;
881 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
882 flags);
883 ZONE_LOCK(zone);
884 return (NULL);
885 }
886 }
887 ZONE_LOCK(zone);
888

--- 447 unchanged lines hidden (view full) ---

1336 bzero(zone, size);
1337 zone->uz_name = arg->name;
1338 zone->uz_ctor = arg->ctor;
1339 zone->uz_dtor = arg->dtor;
1340 zone->uz_init = NULL;
1341 zone->uz_fini = NULL;
1342 zone->uz_allocs = 0;
1343 zone->uz_frees = 0;
1344 zone->uz_fails = 0;
1338 zone->uz_fills = zone->uz_count = 0;
1339
1340 if (arg->flags & UMA_ZONE_SECONDARY) {
1341 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1342 keg = arg->keg;
1343 zone->uz_keg = keg;
1344 zone->uz_init = arg->uminit;
1345 zone->uz_fini = arg->fini;

--- 110 unchanged lines hidden (view full) ---

1456 if (LIST_EMPTY(&keg->uk_zones))
1457 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1458 ZONE_UNLOCK(zone);
1459 mtx_unlock(&uma_mtx);
1460 } else {
1461 LIST_REMOVE(keg, uk_link);
1462 LIST_REMOVE(zone, uz_link);
1463 mtx_unlock(&uma_mtx);
1345 zone->uz_fills = zone->uz_count = 0;
1346
1347 if (arg->flags & UMA_ZONE_SECONDARY) {
1348 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1349 keg = arg->keg;
1350 zone->uz_keg = keg;
1351 zone->uz_init = arg->uminit;
1352 zone->uz_fini = arg->fini;

--- 110 unchanged lines hidden (view full) ---

1463 if (LIST_EMPTY(&keg->uk_zones))
1464 keg->uk_flags &= ~UMA_ZONE_SECONDARY;
1465 ZONE_UNLOCK(zone);
1466 mtx_unlock(&uma_mtx);
1467 } else {
1468 LIST_REMOVE(keg, uk_link);
1469 LIST_REMOVE(zone, uz_link);
1470 mtx_unlock(&uma_mtx);
1464 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE);
1471 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE, 0);
1465 }
1466 zone->uz_keg = NULL;
1467}
1468
1469/*
1470 * Traverses every zone in the system and calls a callback
1471 *
1472 * Arguments:

--- 287 unchanged lines hidden (view full) ---

1760
1761 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1762}
1763
1764/* See uma.h */
1765void
1766uma_zdestroy(uma_zone_t zone)
1767{
1472 }
1473 zone->uz_keg = NULL;
1474}
1475
1476/*
1477 * Traverses every zone in the system and calls a callback
1478 *
1479 * Arguments:

--- 287 unchanged lines hidden (view full) ---

1767
1768 return (uma_zalloc_internal(zones, &args, M_WAITOK));
1769}
1770
1771/* See uma.h */
1772void
1773uma_zdestroy(uma_zone_t zone)
1774{
1768 uma_zfree_internal(zones, zone, NULL, SKIP_NONE);
1775 uma_zfree_internal(zones, zone, NULL, SKIP_NONE, 0);
1769}
1770
1771/* See uma.h */
1772void *
1773uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1774{
1775 void *item;
1776 uma_cache_t cache;

--- 67 unchanged lines hidden (view full) ---

1844 ZONE_LOCK(zone);
1845 uma_dbg_alloc(zone, NULL, item);
1846 ZONE_UNLOCK(zone);
1847#endif
1848 if (zone->uz_ctor != NULL) {
1849 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1850 udata, flags) != 0) {
1851 uma_zfree_internal(zone, item, udata,
1776}
1777
1778/* See uma.h */
1779void *
1780uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1781{
1782 void *item;
1783 uma_cache_t cache;

--- 67 unchanged lines hidden (view full) ---

1851 ZONE_LOCK(zone);
1852 uma_dbg_alloc(zone, NULL, item);
1853 ZONE_UNLOCK(zone);
1854#endif
1855 if (zone->uz_ctor != NULL) {
1856 if (zone->uz_ctor(item, zone->uz_keg->uk_size,
1857 udata, flags) != 0) {
1858 uma_zfree_internal(zone, item, udata,
1852 SKIP_DTOR);
1859 SKIP_DTOR, ZFREE_STATFAIL);
1853 return (NULL);
1854 }
1855 }
1856 if (flags & M_ZERO)
1857 bzero(item, zone->uz_keg->uk_size);
1858 return (item);
1859 } else if (cache->uc_freebucket) {
1860 /*

--- 290 unchanged lines hidden (view full) ---

2151 * If we couldn't initialize the whole bucket, put the
2152 * rest back onto the freelist.
2153 */
2154 if (i != bucket->ub_cnt) {
2155 int j;
2156
2157 for (j = i; j < bucket->ub_cnt; j++) {
2158 uma_zfree_internal(zone, bucket->ub_bucket[j],
1860 return (NULL);
1861 }
1862 }
1863 if (flags & M_ZERO)
1864 bzero(item, zone->uz_keg->uk_size);
1865 return (item);
1866 } else if (cache->uc_freebucket) {
1867 /*

--- 290 unchanged lines hidden (view full) ---

2158 * If we couldn't initialize the whole bucket, put the
2159 * rest back onto the freelist.
2160 */
2161 if (i != bucket->ub_cnt) {
2162 int j;
2163
2164 for (j = i; j < bucket->ub_cnt; j++) {
2165 uma_zfree_internal(zone, bucket->ub_bucket[j],
2159 NULL, SKIP_FINI);
2166 NULL, SKIP_FINI, 0);
2160#ifdef INVARIANTS
2161 bucket->ub_bucket[j] = NULL;
2162#endif
2163 }
2164 bucket->ub_cnt = i;
2165 }
2166 ZONE_LOCK(zone);
2167 }

--- 36 unchanged lines hidden (view full) ---

2204
2205#ifdef UMA_DEBUG_ALLOC
2206 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2207#endif
2208 ZONE_LOCK(zone);
2209
2210 slab = uma_zone_slab(zone, flags);
2211 if (slab == NULL) {
2167#ifdef INVARIANTS
2168 bucket->ub_bucket[j] = NULL;
2169#endif
2170 }
2171 bucket->ub_cnt = i;
2172 }
2173 ZONE_LOCK(zone);
2174 }

--- 36 unchanged lines hidden (view full) ---

2211
2212#ifdef UMA_DEBUG_ALLOC
2213 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2214#endif
2215 ZONE_LOCK(zone);
2216
2217 slab = uma_zone_slab(zone, flags);
2218 if (slab == NULL) {
2219 zone->uz_fails++;
2212 ZONE_UNLOCK(zone);
2213 return (NULL);
2214 }
2215
2216 item = uma_slab_alloc(zone, slab);
2217
2218 zone->uz_allocs++;
2219
2220 ZONE_UNLOCK(zone);
2221
2222 /*
2223 * We have to call both the zone's init (not the keg's init)
2224 * and the zone's ctor. This is because the item is going from
2225 * a keg slab directly to the user, and the user is expecting it
2226 * to be both zone-init'd as well as zone-ctor'd.
2227 */
2228 if (zone->uz_init != NULL) {
2229 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2220 ZONE_UNLOCK(zone);
2221 return (NULL);
2222 }
2223
2224 item = uma_slab_alloc(zone, slab);
2225
2226 zone->uz_allocs++;
2227
2228 ZONE_UNLOCK(zone);
2229
2230 /*
2231 * We have to call both the zone's init (not the keg's init)
2232 * and the zone's ctor. This is because the item is going from
2233 * a keg slab directly to the user, and the user is expecting it
2234 * to be both zone-init'd as well as zone-ctor'd.
2235 */
2236 if (zone->uz_init != NULL) {
2237 if (zone->uz_init(item, keg->uk_size, flags) != 0) {
2230 uma_zfree_internal(zone, item, udata, SKIP_FINI);
2238 uma_zfree_internal(zone, item, udata, SKIP_FINI,
2239 ZFREE_STATFAIL);
2231 return (NULL);
2232 }
2233 }
2234 if (zone->uz_ctor != NULL) {
2235 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2240 return (NULL);
2241 }
2242 }
2243 if (zone->uz_ctor != NULL) {
2244 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) {
2236 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2245 uma_zfree_internal(zone, item, udata, SKIP_DTOR,
2246 ZFREE_STATFAIL);
2237 return (NULL);
2238 }
2239 }
2240 if (flags & M_ZERO)
2241 bzero(item, keg->uk_size);
2242
2243 return (item);
2244}

--- 156 unchanged lines hidden (view full) ---

2401 ZONE_UNLOCK(zone);
2402 goto zfree_restart;
2403 }
2404
2405 /*
2406 * If nothing else caught this, we'll just do an internal free.
2407 */
2408zfree_internal:
2247 return (NULL);
2248 }
2249 }
2250 if (flags & M_ZERO)
2251 bzero(item, keg->uk_size);
2252
2253 return (item);
2254}

--- 156 unchanged lines hidden (view full) ---

2411 ZONE_UNLOCK(zone);
2412 goto zfree_restart;
2413 }
2414
2415 /*
2416 * If nothing else caught this, we'll just do an internal free.
2417 */
2418zfree_internal:
2409 uma_zfree_internal(zone, item, udata, SKIP_DTOR);
2419 uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL);
2410
2411 return;
2412}
2413
2414/*
2415 * Frees an item to an INTERNAL zone or allocates a free bucket
2416 *
2417 * Arguments:
2418 * zone The zone to free to
2419 * item The item we're freeing
2420 * udata User supplied data for the dtor
2421 * skip Skip dtors and finis
2422 */
2423static void
2424uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2420
2421 return;
2422}
2423
2424/*
2425 * Frees an item to an INTERNAL zone or allocates a free bucket
2426 *
2427 * Arguments:
2428 * zone The zone to free to
2429 * item The item we're freeing
2430 * udata User supplied data for the dtor
2431 * skip Skip dtors and finis
2432 */
2433static void
2434uma_zfree_internal(uma_zone_t zone, void *item, void *udata,
2425 enum zfreeskip skip)
2435 enum zfreeskip skip, int flags)
2426{
2427 uma_slab_t slab;
2428 uma_slabrefcnt_t slabref;
2429 uma_keg_t keg;
2430 u_int8_t *mem;
2431 u_int8_t freei;
2432
2433 keg = zone->uz_keg;
2434
2435 if (skip < SKIP_DTOR && zone->uz_dtor)
2436 zone->uz_dtor(item, keg->uk_size, udata);
2437 if (skip < SKIP_FINI && zone->uz_fini)
2438 zone->uz_fini(item, keg->uk_size);
2439
2440 ZONE_LOCK(zone);
2441
2436{
2437 uma_slab_t slab;
2438 uma_slabrefcnt_t slabref;
2439 uma_keg_t keg;
2440 u_int8_t *mem;
2441 u_int8_t freei;
2442
2443 keg = zone->uz_keg;
2444
2445 if (skip < SKIP_DTOR && zone->uz_dtor)
2446 zone->uz_dtor(item, keg->uk_size, udata);
2447 if (skip < SKIP_FINI && zone->uz_fini)
2448 zone->uz_fini(item, keg->uk_size);
2449
2450 ZONE_LOCK(zone);
2451
2452 if (flags & ZFREE_STATFAIL)
2453 zone->uz_fails++;
2454
2442 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2443 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2444 if (keg->uk_flags & UMA_ZONE_HASH)
2445 slab = hash_sfind(&keg->uk_hash, mem);
2446 else {
2447 mem += keg->uk_pgoff;
2448 slab = (uma_slab_t)mem;
2449 }

--- 235 unchanged lines hidden (view full) ---

2685 return (NULL);
2686 mem = page_alloc(NULL, size, &flags, wait);
2687 if (mem) {
2688 vsetslab((vm_offset_t)mem, slab);
2689 slab->us_data = mem;
2690 slab->us_flags = flags | UMA_SLAB_MALLOC;
2691 slab->us_size = size;
2692 } else {
2455 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) {
2456 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK));
2457 if (keg->uk_flags & UMA_ZONE_HASH)
2458 slab = hash_sfind(&keg->uk_hash, mem);
2459 else {
2460 mem += keg->uk_pgoff;
2461 slab = (uma_slab_t)mem;
2462 }

--- 235 unchanged lines hidden (view full) ---

2698 return (NULL);
2699 mem = page_alloc(NULL, size, &flags, wait);
2700 if (mem) {
2701 vsetslab((vm_offset_t)mem, slab);
2702 slab->us_data = mem;
2703 slab->us_flags = flags | UMA_SLAB_MALLOC;
2704 slab->us_size = size;
2705 } else {
2693 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2706 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE,
2707 ZFREE_STATFAIL);
2694 }
2695
2696 return (mem);
2697}
2698
2699void
2700uma_large_free(uma_slab_t slab)
2701{
2702 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2703 page_free(slab->us_data, slab->us_size, slab->us_flags);
2708 }
2709
2710 return (mem);
2711}
2712
2713void
2714uma_large_free(uma_slab_t slab)
2715{
2716 vsetobj((vm_offset_t)slab->us_data, kmem_object);
2717 page_free(slab->us_data, slab->us_size, slab->us_flags);
2704 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE);
2718 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, 0);
2705}
2706
2707void
2708uma_print_stats(void)
2709{
2710 zone_foreach(uma_print_zone);
2711}
2712

--- 253 unchanged lines hidden (view full) ---

2966 kz->uk_ppera;
2967 else
2968 uth.uth_limit = kz->uk_maxpages *
2969 kz->uk_ipers;
2970 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2971 uth.uth_zone_free += bucket->ub_cnt;
2972 uth.uth_allocs = z->uz_allocs;
2973 uth.uth_frees = z->uz_frees;
2719}
2720
2721void
2722uma_print_stats(void)
2723{
2724 zone_foreach(uma_print_zone);
2725}
2726

--- 253 unchanged lines hidden (view full) ---

2980 kz->uk_ppera;
2981 else
2982 uth.uth_limit = kz->uk_maxpages *
2983 kz->uk_ipers;
2984 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link)
2985 uth.uth_zone_free += bucket->ub_cnt;
2986 uth.uth_allocs = z->uz_allocs;
2987 uth.uth_frees = z->uz_frees;
2988 uth.uth_fails = z->uz_fails;
2974 ZONE_UNLOCK(z);
2975 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2976 mtx_unlock(&uma_mtx);
2977 error = ENOMEM;
2978 goto out;
2979 }
2980 /*
2981 * XXXRW: Should not access bucket fields from

--- 33 unchanged lines hidden --- 		2989 ZONE_UNLOCK(z);
2990 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) {
2991 mtx_unlock(&uma_mtx);
2992 error = ENOMEM;
2993 goto out;
2994 }
2995 /*
2996 * XXXRW: Should not access bucket fields from

--- 33 unchanged lines hidden ---