51c51
< __FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251709 2013-06-13 21:05:38Z jeff $");
---
> __FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $");
134c134
< static struct mtx uma_mtx;
---
> static struct mtx_padalign uma_mtx;
141c141
< static struct mtx uma_boot_pages_mtx;
---
> static struct mtx_padalign uma_boot_pages_mtx;
174a175,177
> uma_import import;
> uma_release release;
> void *arg;
219,221d221
< #define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
< #define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
<
247,248c247
< static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip,
< int);
---
> static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
257c256,257
< static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
---
> static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
> static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
261a262,263
> static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
> static void zone_release(uma_zone_t zone, void **bucket, int cnt);
366,367c368
< zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
< ZFREE_STATFREE);
---
> zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
391,397d391
< static inline uma_keg_t
< zone_first_keg(uma_zone_t zone)
< {
<
< return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
< }
<
582,583c576
< zone_free_item(hashzone,
< hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
602c595
< void *item;
---
> int i;
607,616c600,604
< while (bucket->ub_cnt > 0) {
< bucket->ub_cnt--;
< item = bucket->ub_bucket[bucket->ub_cnt];
< #ifdef INVARIANTS
< bucket->ub_bucket[bucket->ub_cnt] = NULL;
< KASSERT(item != NULL,
< ("bucket_drain: botched ptr, item is NULL"));
< #endif
< zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
< }
---
> if (zone->uz_fini)
> for (i = 0; i < bucket->ub_cnt; i++)
> zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
> zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
> bucket->ub_cnt = 0;
770,771c758
< zone_free_item(keg->uk_slabzone, slab, NULL,
< SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
845c832
< printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
---
> printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
877,878c864
< zone_free_item(keg->uk_slabzone, slab, NULL,
< SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
932c918
< NULL, SKIP_NONE, ZFREE_STATFREE);
---
> NULL, SKIP_NONE);
1485a1472,1489
> /*
> * This is a pure cache zone, no kegs.
> */
> if (arg->import) {
> zone->uz_import = arg->import;
> zone->uz_release = arg->release;
> zone->uz_arg = arg->arg;
> zone->uz_count = BUCKET_MAX;
> return (0);
> }
>
> /*
> * Use the regular zone/keg/slab allocator.
> */
> zone->uz_import = (uma_import)zone_import;
> zone->uz_release = (uma_release)zone_release;
> zone->uz_arg = zone;
>
1521a1526
>
1619c1624
< if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
---
> if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1623,1624c1628
< zone_free_item(kegs, keg, NULL, SKIP_NONE,
< ZFREE_STATFREE);
---
> zone_free_item(kegs, keg, NULL, SKIP_NONE);
1667a1672
> memset(&args, 0, sizeof(args));
1807a1813
> memset(&args, 0, sizeof(args));
1829a1836
> memset(&args, 0, sizeof(args));
1843a1851,1874
> /* See uma.h */
> uma_zone_t
> uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
> uma_fini zfini, uma_import zimport, uma_release zrelease,
> void *arg, int flags)
> {
> struct uma_zctor_args args;
>
> memset(&args, 0, sizeof(args));
> args.name = name;
> args.size = 0;
> args.ctor = ctor;
> args.dtor = dtor;
> args.uminit = zinit;
> args.fini = zfini;
> args.import = zimport;
> args.release = zrelease;
> args.arg = arg;
> args.align = 0;
> args.flags = flags;
>
> return (zone_alloc_item(zones, &args, M_WAITOK));
> }
>
1935c1966
< zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(zones, zone, NULL, SKIP_NONE);
2013a2045
> atomic_add_long(&zone->uz_fails, 1);
2015,2016c2047
< SKIP_DTOR, ZFREE_STATFAIL |
< ZFREE_STATFREE);
---
> SKIP_DTOR);
2072c2103,2104
< zone->uz_allocs += cache->uc_allocs;
---
> atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
> atomic_add_long(&zone->uz_frees, cache->uc_frees);
2074d2105
< zone->uz_frees += cache->uc_frees;
2322c2353
< slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
---
> slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2324d2354
< uma_keg_t keg;
2328c2358
< keg = slab->us_keg;
---
> MPASS(keg == slab->us_keg);
2347c2377
< zone_alloc_bucket(uma_zone_t zone, int flags)
---
> zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2349d2378
< uma_bucket_t bucket;
2352,2353c2381
< int16_t saved;
< int max, origflags = flags;
---
> int i;
2355,2363c2383,2392
< /*
< * Try this zone's free list first so we don't allocate extra buckets.
< */
< if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
< KASSERT(bucket->ub_cnt == 0,
< ("zone_alloc_bucket: Bucket on free list is not empty."));
< LIST_REMOVE(bucket, ub_link);
< } else {
< int bflags;
---
> ZONE_LOCK(zone);
> /* Try to keep the buckets totally full */
> slab = NULL;
> keg = NULL;
> for (i = 0; i < max; ) {
> if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
> break;
> keg = slab->us_keg;
> while (slab->us_freecount && i < max)
> bucket[i++] = slab_alloc_item(keg, slab);
2365,2371c2394,2396
< bflags = (flags & ~M_ZERO);
< if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
< bflags |= M_NOVM;
<
< ZONE_UNLOCK(zone);
< bucket = bucket_alloc(zone->uz_count, bflags);
< ZONE_LOCK(zone);
---
> /* Don't block on the next fill */
> flags &= ~M_WAITOK;
> flags |= M_NOWAIT;
2372a2398,2401
> if (slab != NULL)
> KEG_UNLOCK(keg);
> else
> ZONE_UNLOCK(zone);
2374,2376c2403,2404
< if (bucket == NULL) {
< return (0);
< }
---
> return i;
> }
2377a2406,2412
> static int
> zone_alloc_bucket(uma_zone_t zone, int flags)
> {
> uma_bucket_t bucket;
> int bflags;
> int max;
>
2385c2420
< goto done;
---
> return (0);
2388a2424
> max = zone->uz_count;
2390,2404c2426,2441
< max = MIN(bucket->ub_entries, zone->uz_count);
< /* Try to keep the buckets totally full */
< saved = bucket->ub_cnt;
< slab = NULL;
< keg = NULL;
< while (bucket->ub_cnt < max &&
< (slab = zone->uz_slab(zone, keg, flags)) != NULL) {
< keg = slab->us_keg;
< while (slab->us_freecount && bucket->ub_cnt < max) {
< bucket->ub_bucket[bucket->ub_cnt++] =
< slab_alloc_item(zone, slab);
< }
<
< /* Don't block on the next fill */
< flags |= M_NOWAIT;
---
> /*
> * Try this zone's free list first so we don't allocate extra buckets.
> */
> if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
> KASSERT(bucket->ub_cnt == 0,
> ("zone_alloc_bucket: Bucket on free list is not empty."));
> LIST_REMOVE(bucket, ub_link);
> ZONE_UNLOCK(zone);
> } else {
> bflags = (flags & ~M_ZERO);
> if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
> bflags |= M_NOVM;
> ZONE_UNLOCK(zone);
> bucket = bucket_alloc(zone->uz_count, bflags);
> if (bucket == NULL)
> goto out;
2406,2407d2442
< if (slab)
< zone_relock(zone, keg);
2408a2444,2447
> max = MIN(bucket->ub_entries, max);
> bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
> max, flags);
>
2410,2414c2449
< * We unlock here because we need to call the zone's init.
< * It should be safe to unlock because the slab dealt with
< * above is already on the appropriate list within the keg
< * and the bucket we filled is not yet on any list, so we
< * own it.
---
> * Initialize the memory if necessary.
2416c2451
< if (zone->uz_init != NULL) {
---
> if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2419,2420c2454
< ZONE_UNLOCK(zone);
< for (i = saved; i < bucket->ub_cnt; i++)
---
> for (i = 0; i < bucket->ub_cnt; i++)
2422c2456
< origflags) != 0)
---
> flags) != 0)
2429,2433c2463,2464
< int j;
<
< for (j = i; j < bucket->ub_cnt; j++) {
< zone_free_item(zone, bucket->ub_bucket[j],
< NULL, SKIP_FINI, 0);
---
> zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
> bucket->ub_cnt - i);
2435c2466,2467
< bucket->ub_bucket[j] = NULL;
---
> bzero(&bucket->ub_bucket[i],
> sizeof(void *) * (bucket->ub_cnt - i));
2437d2468
< }
2440d2470
< ZONE_LOCK(zone);
2442a2473,2474
> out:
> ZONE_LOCK(zone);
2444c2476
< if (bucket->ub_cnt != 0) {
---
> if (bucket != NULL && bucket->ub_cnt != 0) {
2449,2452c2481,2483
< #ifdef SMP
< done:
< #endif
< bucket_free(bucket);
---
> atomic_add_long(&zone->uz_fails, 1);
> if (bucket != NULL)
> bucket_free(bucket);
2457c2488
< * Allocates an item for an internal zone
---
> * Allocates a single item from a zone.
2472d2502
< uma_slab_t slab;
2480c2510,2512
< ZONE_LOCK(zone);
---
> if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
> goto fail;
> atomic_add_long(&zone->uz_allocs, 1);
2482,2494d2513
< slab = zone->uz_slab(zone, NULL, flags);
< if (slab == NULL) {
< zone->uz_fails++;
< ZONE_UNLOCK(zone);
< return (NULL);
< }
<
< item = slab_alloc_item(zone, slab);
<
< zone_relock(zone, slab->us_keg);
< zone->uz_allocs++;
< ZONE_UNLOCK(zone);
<
2503,2505c2522,2523
< zone_free_item(zone, item, udata, SKIP_FINI,
< ZFREE_STATFAIL | ZFREE_STATFREE);
< return (NULL);
---
> zone_free_item(zone, item, udata, SKIP_FINI);
> goto fail;
2510,2512c2528,2529
< zone_free_item(zone, item, udata, SKIP_DTOR,
< ZFREE_STATFAIL | ZFREE_STATFREE);
< return (NULL);
---
> zone_free_item(zone, item, udata, SKIP_DTOR);
> goto fail;
2516c2533
< uma_dbg_alloc(zone, slab, item);
---
> uma_dbg_alloc(zone, NULL, item);
2521a2539,2542
>
> fail:
> atomic_add_long(&zone->uz_fails, 1);
> return (NULL);
2651c2672,2673
< zone->uz_allocs += cache->uc_allocs;
---
> atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
> atomic_add_long(&zone->uz_frees, cache->uc_frees);
2653d2674
< zone->uz_frees += cache->uc_frees;
2702c2723
< zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
---
> zone_free_item(zone, item, udata, SKIP_DTOR);
2707,2715d2727
< /*
< * Frees an item to an INTERNAL zone or allocates a free bucket
< *
< * Arguments:
< * zone The zone to free to
< * item The item we're freeing
< * udata User supplied data for the dtor
< * skip Skip dtors and finis
< */
2717,2718c2729
< zone_free_item(uma_zone_t zone, void *item, void *udata,
< enum zfreeskip skip, int flags)
---
> slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
2720,2722d2730
< uma_slab_t slab;
< uma_keg_t keg;
< uint8_t *mem;
2724d2731
< int clearfull;
2726,2764c2733
< #ifdef INVARIANTS
< if (skip == SKIP_NONE) {
< if (zone->uz_flags & UMA_ZONE_MALLOC)
< uma_dbg_free(zone, udata, item);
< else
< uma_dbg_free(zone, NULL, item);
< }
< #endif
< if (skip < SKIP_DTOR && zone->uz_dtor)
< zone->uz_dtor(item, zone->uz_size, udata);
<
< if (skip < SKIP_FINI && zone->uz_fini)
< zone->uz_fini(item, zone->uz_size);
<
< ZONE_LOCK(zone);
<
< if (flags & ZFREE_STATFAIL)
< zone->uz_fails++;
< if (flags & ZFREE_STATFREE)
< zone->uz_frees++;
<
< if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
< mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
< keg = zone_first_keg(zone); /* Must only be one. */
< if (zone->uz_flags & UMA_ZONE_HASH) {
< slab = hash_sfind(&keg->uk_hash, mem);
< } else {
< mem += keg->uk_pgoff;
< slab = (uma_slab_t)mem;
< }
< } else {
< /* This prevents redundant lookups via free(). */
< if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL)
< slab = (uma_slab_t)udata;
< else
< slab = vtoslab((vm_offset_t)item);
< keg = slab->us_keg;
< keg_relock(keg, zone);
< }
---
> mtx_assert(&keg->uk_lock, MA_OWNED);
2782a2752
> }
2783a2754,2763
> static void
> zone_release(uma_zone_t zone, void **bucket, int cnt)
> {
> void *item;
> uma_slab_t slab;
> uma_keg_t keg;
> uint8_t *mem;
> int clearfull;
> int i;
>
2785,2788c2765,2783
< if (keg->uk_flags & UMA_ZFLAG_FULL) {
< if (keg->uk_pages < keg->uk_maxpages) {
< keg->uk_flags &= ~UMA_ZFLAG_FULL;
< clearfull = 1;
---
> ZONE_LOCK(zone);
> keg = zone_first_keg(zone);
> for (i = 0; i < cnt; i++) {
> item = bucket[i];
> if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
> mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
> if (zone->uz_flags & UMA_ZONE_HASH) {
> slab = hash_sfind(&keg->uk_hash, mem);
> } else {
> mem += keg->uk_pgoff;
> slab = (uma_slab_t)mem;
> }
> } else {
> slab = vtoslab((vm_offset_t)item);
> if (slab->us_keg != keg) {
> KEG_UNLOCK(keg);
> keg = slab->us_keg;
> KEG_LOCK(keg);
> }
2789a2785,2790
> slab_free_item(keg, slab, item);
> if (keg->uk_flags & UMA_ZFLAG_FULL) {
> if (keg->uk_pages < keg->uk_maxpages) {
> keg->uk_flags &= ~UMA_ZFLAG_FULL;
> clearfull = 1;
> }
2791,2798c2792,2800
< /*
< * We can handle one more allocation. Since we're
< * clearing ZFLAG_FULL, wake up all procs blocked
< * on pages. This should be uncommon, so keeping this
< * simple for now (rather than adding count of blocked
< * threads etc).
< */
< wakeup(keg);
---
> /*
> * We can handle one more allocation. Since we're
> * clearing ZFLAG_FULL, wake up all procs blocked
> * on pages. This should be uncommon, so keeping this
> * simple for now (rather than adding count of blocked
> * threads etc).
> */
> wakeup(keg);
> }
2799a2802
> zone_relock(zone, keg);
2801d2803
< zone_relock(zone, keg);
2804,2806c2806,2807
< ZONE_UNLOCK(zone);
< } else
< KEG_UNLOCK(keg);
---
> }
> ZONE_UNLOCK(zone);
2809a2811,2841
> /*
> * Frees a single item to any zone.
> *
> * Arguments:
> * zone The zone to free to
> * item The item we're freeing
> * udata User supplied data for the dtor
> * skip Skip dtors and finis
> */
> static void
> zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
> {
>
> #ifdef INVARIANTS
> if (skip == SKIP_NONE) {
> if (zone->uz_flags & UMA_ZONE_MALLOC)
> uma_dbg_free(zone, udata, item);
> else
> uma_dbg_free(zone, NULL, item);
> }
> #endif
> if (skip < SKIP_DTOR && zone->uz_dtor)
> zone->uz_dtor(item, zone->uz_size, udata);
>
> if (skip < SKIP_FINI && zone->uz_fini)
> zone->uz_fini(item, zone->uz_size);
>
> atomic_add_long(&zone->uz_frees, 1);
> zone->uz_release(zone->uz_arg, &item, 1);
> }
>
2816d2847
< ZONE_LOCK(zone);
2817a2849,2851
> if (keg == NULL)
> return (0);
> ZONE_LOCK(zone);
2834d2867
< ZONE_LOCK(zone);
2835a2869,2871
> if (keg == NULL)
> return (0);
> ZONE_LOCK(zone);
2882a2919
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2896a2934
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2929a2968
> uma_keg_t keg;
2932c2971,2973
< zone_first_keg(zone)->uk_freef = freef;
---
> keg = zone_first_keg(zone);
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
> keg->uk_freef = freef;
2958a3000,3001
> if (keg == NULL)
> return (0);
2996a3040,3041
> if (keg == NULL)
> return;
3086,3087c3131
< zone_free_item(slabzone, slab, NULL, SKIP_NONE,
< ZFREE_STATFAIL | ZFREE_STATFREE);
---
> zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3098c3142
< zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(slabzone, slab, NULL, SKIP_NONE);
< __FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251709 2013-06-13 21:05:38Z jeff $");
---
> __FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $");
134c134
< static struct mtx uma_mtx;
---
> static struct mtx_padalign uma_mtx;
141c141
< static struct mtx uma_boot_pages_mtx;
---
> static struct mtx_padalign uma_boot_pages_mtx;
174a175,177
> uma_import import;
> uma_release release;
> void *arg;
219,221d221
< #define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */
< #define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */
<
247,248c247
< static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip,
< int);
---
> static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
257c256,257
< static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
---
> static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
> static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
261a262,263
> static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
> static void zone_release(uma_zone_t zone, void **bucket, int cnt);
366,367c368
< zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
< ZFREE_STATFREE);
---
> zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
391,397d391
< static inline uma_keg_t
< zone_first_keg(uma_zone_t zone)
< {
<
< return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
< }
<
582,583c576
< zone_free_item(hashzone,
< hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
602c595
< void *item;
---
> int i;
607,616c600,604
< while (bucket->ub_cnt > 0) {
< bucket->ub_cnt--;
< item = bucket->ub_bucket[bucket->ub_cnt];
< #ifdef INVARIANTS
< bucket->ub_bucket[bucket->ub_cnt] = NULL;
< KASSERT(item != NULL,
< ("bucket_drain: botched ptr, item is NULL"));
< #endif
< zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
< }
---
> if (zone->uz_fini)
> for (i = 0; i < bucket->ub_cnt; i++)
> zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
> zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
> bucket->ub_cnt = 0;
770,771c758
< zone_free_item(keg->uk_slabzone, slab, NULL,
< SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
845c832
< printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
---
> printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
877,878c864
< zone_free_item(keg->uk_slabzone, slab, NULL,
< SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
932c918
< NULL, SKIP_NONE, ZFREE_STATFREE);
---
> NULL, SKIP_NONE);
1485a1472,1489
> /*
> * This is a pure cache zone, no kegs.
> */
> if (arg->import) {
> zone->uz_import = arg->import;
> zone->uz_release = arg->release;
> zone->uz_arg = arg->arg;
> zone->uz_count = BUCKET_MAX;
> return (0);
> }
>
> /*
> * Use the regular zone/keg/slab allocator.
> */
> zone->uz_import = (uma_import)zone_import;
> zone->uz_release = (uma_release)zone_release;
> zone->uz_arg = zone;
>
1521a1526
>
1619c1624
< if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
---
> if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1623,1624c1628
< zone_free_item(kegs, keg, NULL, SKIP_NONE,
< ZFREE_STATFREE);
---
> zone_free_item(kegs, keg, NULL, SKIP_NONE);
1667a1672
> memset(&args, 0, sizeof(args));
1807a1813
> memset(&args, 0, sizeof(args));
1829a1836
> memset(&args, 0, sizeof(args));
1843a1851,1874
> /* See uma.h */
> uma_zone_t
> uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
> uma_fini zfini, uma_import zimport, uma_release zrelease,
> void *arg, int flags)
> {
> struct uma_zctor_args args;
>
> memset(&args, 0, sizeof(args));
> args.name = name;
> args.size = 0;
> args.ctor = ctor;
> args.dtor = dtor;
> args.uminit = zinit;
> args.fini = zfini;
> args.import = zimport;
> args.release = zrelease;
> args.arg = arg;
> args.align = 0;
> args.flags = flags;
>
> return (zone_alloc_item(zones, &args, M_WAITOK));
> }
>
1935c1966
< zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(zones, zone, NULL, SKIP_NONE);
2013a2045
> atomic_add_long(&zone->uz_fails, 1);
2015,2016c2047
< SKIP_DTOR, ZFREE_STATFAIL |
< ZFREE_STATFREE);
---
> SKIP_DTOR);
2072c2103,2104
< zone->uz_allocs += cache->uc_allocs;
---
> atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
> atomic_add_long(&zone->uz_frees, cache->uc_frees);
2074d2105
< zone->uz_frees += cache->uc_frees;
2322c2353
< slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
---
> slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2324d2354
< uma_keg_t keg;
2328c2358
< keg = slab->us_keg;
---
> MPASS(keg == slab->us_keg);
2347c2377
< zone_alloc_bucket(uma_zone_t zone, int flags)
---
> zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2349d2378
< uma_bucket_t bucket;
2352,2353c2381
< int16_t saved;
< int max, origflags = flags;
---
> int i;
2355,2363c2383,2392
< /*
< * Try this zone's free list first so we don't allocate extra buckets.
< */
< if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
< KASSERT(bucket->ub_cnt == 0,
< ("zone_alloc_bucket: Bucket on free list is not empty."));
< LIST_REMOVE(bucket, ub_link);
< } else {
< int bflags;
---
> ZONE_LOCK(zone);
> /* Try to keep the buckets totally full */
> slab = NULL;
> keg = NULL;
> for (i = 0; i < max; ) {
> if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
> break;
> keg = slab->us_keg;
> while (slab->us_freecount && i < max)
> bucket[i++] = slab_alloc_item(keg, slab);
2365,2371c2394,2396
< bflags = (flags & ~M_ZERO);
< if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
< bflags |= M_NOVM;
<
< ZONE_UNLOCK(zone);
< bucket = bucket_alloc(zone->uz_count, bflags);
< ZONE_LOCK(zone);
---
> /* Don't block on the next fill */
> flags &= ~M_WAITOK;
> flags |= M_NOWAIT;
2372a2398,2401
> if (slab != NULL)
> KEG_UNLOCK(keg);
> else
> ZONE_UNLOCK(zone);
2374,2376c2403,2404
< if (bucket == NULL) {
< return (0);
< }
---
> return i;
> }
2377a2406,2412
> static int
> zone_alloc_bucket(uma_zone_t zone, int flags)
> {
> uma_bucket_t bucket;
> int bflags;
> int max;
>
2385c2420
< goto done;
---
> return (0);
2388a2424
> max = zone->uz_count;
2390,2404c2426,2441
< max = MIN(bucket->ub_entries, zone->uz_count);
< /* Try to keep the buckets totally full */
< saved = bucket->ub_cnt;
< slab = NULL;
< keg = NULL;
< while (bucket->ub_cnt < max &&
< (slab = zone->uz_slab(zone, keg, flags)) != NULL) {
< keg = slab->us_keg;
< while (slab->us_freecount && bucket->ub_cnt < max) {
< bucket->ub_bucket[bucket->ub_cnt++] =
< slab_alloc_item(zone, slab);
< }
<
< /* Don't block on the next fill */
< flags |= M_NOWAIT;
---
> /*
> * Try this zone's free list first so we don't allocate extra buckets.
> */
> if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
> KASSERT(bucket->ub_cnt == 0,
> ("zone_alloc_bucket: Bucket on free list is not empty."));
> LIST_REMOVE(bucket, ub_link);
> ZONE_UNLOCK(zone);
> } else {
> bflags = (flags & ~M_ZERO);
> if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
> bflags |= M_NOVM;
> ZONE_UNLOCK(zone);
> bucket = bucket_alloc(zone->uz_count, bflags);
> if (bucket == NULL)
> goto out;
2406,2407d2442
< if (slab)
< zone_relock(zone, keg);
2408a2444,2447
> max = MIN(bucket->ub_entries, max);
> bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
> max, flags);
>
2410,2414c2449
< * We unlock here because we need to call the zone's init.
< * It should be safe to unlock because the slab dealt with
< * above is already on the appropriate list within the keg
< * and the bucket we filled is not yet on any list, so we
< * own it.
---
> * Initialize the memory if necessary.
2416c2451
< if (zone->uz_init != NULL) {
---
> if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2419,2420c2454
< ZONE_UNLOCK(zone);
< for (i = saved; i < bucket->ub_cnt; i++)
---
> for (i = 0; i < bucket->ub_cnt; i++)
2422c2456
< origflags) != 0)
---
> flags) != 0)
2429,2433c2463,2464
< int j;
<
< for (j = i; j < bucket->ub_cnt; j++) {
< zone_free_item(zone, bucket->ub_bucket[j],
< NULL, SKIP_FINI, 0);
---
> zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
> bucket->ub_cnt - i);
2435c2466,2467
< bucket->ub_bucket[j] = NULL;
---
> bzero(&bucket->ub_bucket[i],
> sizeof(void *) * (bucket->ub_cnt - i));
2437d2468
< }
2440d2470
< ZONE_LOCK(zone);
2442a2473,2474
> out:
> ZONE_LOCK(zone);
2444c2476
< if (bucket->ub_cnt != 0) {
---
> if (bucket != NULL && bucket->ub_cnt != 0) {
2449,2452c2481,2483
< #ifdef SMP
< done:
< #endif
< bucket_free(bucket);
---
> atomic_add_long(&zone->uz_fails, 1);
> if (bucket != NULL)
> bucket_free(bucket);
2457c2488
< * Allocates an item for an internal zone
---
> * Allocates a single item from a zone.
2472d2502
< uma_slab_t slab;
2480c2510,2512
< ZONE_LOCK(zone);
---
> if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
> goto fail;
> atomic_add_long(&zone->uz_allocs, 1);
2482,2494d2513
< slab = zone->uz_slab(zone, NULL, flags);
< if (slab == NULL) {
< zone->uz_fails++;
< ZONE_UNLOCK(zone);
< return (NULL);
< }
<
< item = slab_alloc_item(zone, slab);
<
< zone_relock(zone, slab->us_keg);
< zone->uz_allocs++;
< ZONE_UNLOCK(zone);
<
2503,2505c2522,2523
< zone_free_item(zone, item, udata, SKIP_FINI,
< ZFREE_STATFAIL | ZFREE_STATFREE);
< return (NULL);
---
> zone_free_item(zone, item, udata, SKIP_FINI);
> goto fail;
2510,2512c2528,2529
< zone_free_item(zone, item, udata, SKIP_DTOR,
< ZFREE_STATFAIL | ZFREE_STATFREE);
< return (NULL);
---
> zone_free_item(zone, item, udata, SKIP_DTOR);
> goto fail;
2516c2533
< uma_dbg_alloc(zone, slab, item);
---
> uma_dbg_alloc(zone, NULL, item);
2521a2539,2542
>
> fail:
> atomic_add_long(&zone->uz_fails, 1);
> return (NULL);
2651c2672,2673
< zone->uz_allocs += cache->uc_allocs;
---
> atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
> atomic_add_long(&zone->uz_frees, cache->uc_frees);
2653d2674
< zone->uz_frees += cache->uc_frees;
2702c2723
< zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
---
> zone_free_item(zone, item, udata, SKIP_DTOR);
2707,2715d2727
< /*
< * Frees an item to an INTERNAL zone or allocates a free bucket
< *
< * Arguments:
< * zone The zone to free to
< * item The item we're freeing
< * udata User supplied data for the dtor
< * skip Skip dtors and finis
< */
2717,2718c2729
< zone_free_item(uma_zone_t zone, void *item, void *udata,
< enum zfreeskip skip, int flags)
---
> slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
2720,2722d2730
< uma_slab_t slab;
< uma_keg_t keg;
< uint8_t *mem;
2724d2731
< int clearfull;
2726,2764c2733
< #ifdef INVARIANTS
< if (skip == SKIP_NONE) {
< if (zone->uz_flags & UMA_ZONE_MALLOC)
< uma_dbg_free(zone, udata, item);
< else
< uma_dbg_free(zone, NULL, item);
< }
< #endif
< if (skip < SKIP_DTOR && zone->uz_dtor)
< zone->uz_dtor(item, zone->uz_size, udata);
<
< if (skip < SKIP_FINI && zone->uz_fini)
< zone->uz_fini(item, zone->uz_size);
<
< ZONE_LOCK(zone);
<
< if (flags & ZFREE_STATFAIL)
< zone->uz_fails++;
< if (flags & ZFREE_STATFREE)
< zone->uz_frees++;
<
< if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
< mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
< keg = zone_first_keg(zone); /* Must only be one. */
< if (zone->uz_flags & UMA_ZONE_HASH) {
< slab = hash_sfind(&keg->uk_hash, mem);
< } else {
< mem += keg->uk_pgoff;
< slab = (uma_slab_t)mem;
< }
< } else {
< /* This prevents redundant lookups via free(). */
< if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL)
< slab = (uma_slab_t)udata;
< else
< slab = vtoslab((vm_offset_t)item);
< keg = slab->us_keg;
< keg_relock(keg, zone);
< }
---
> mtx_assert(&keg->uk_lock, MA_OWNED);
2782a2752
> }
2783a2754,2763
> static void
> zone_release(uma_zone_t zone, void **bucket, int cnt)
> {
> void *item;
> uma_slab_t slab;
> uma_keg_t keg;
> uint8_t *mem;
> int clearfull;
> int i;
>
2785,2788c2765,2783
< if (keg->uk_flags & UMA_ZFLAG_FULL) {
< if (keg->uk_pages < keg->uk_maxpages) {
< keg->uk_flags &= ~UMA_ZFLAG_FULL;
< clearfull = 1;
---
> ZONE_LOCK(zone);
> keg = zone_first_keg(zone);
> for (i = 0; i < cnt; i++) {
> item = bucket[i];
> if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
> mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
> if (zone->uz_flags & UMA_ZONE_HASH) {
> slab = hash_sfind(&keg->uk_hash, mem);
> } else {
> mem += keg->uk_pgoff;
> slab = (uma_slab_t)mem;
> }
> } else {
> slab = vtoslab((vm_offset_t)item);
> if (slab->us_keg != keg) {
> KEG_UNLOCK(keg);
> keg = slab->us_keg;
> KEG_LOCK(keg);
> }
2789a2785,2790
> slab_free_item(keg, slab, item);
> if (keg->uk_flags & UMA_ZFLAG_FULL) {
> if (keg->uk_pages < keg->uk_maxpages) {
> keg->uk_flags &= ~UMA_ZFLAG_FULL;
> clearfull = 1;
> }
2791,2798c2792,2800
< /*
< * We can handle one more allocation. Since we're
< * clearing ZFLAG_FULL, wake up all procs blocked
< * on pages. This should be uncommon, so keeping this
< * simple for now (rather than adding count of blocked
< * threads etc).
< */
< wakeup(keg);
---
> /*
> * We can handle one more allocation. Since we're
> * clearing ZFLAG_FULL, wake up all procs blocked
> * on pages. This should be uncommon, so keeping this
> * simple for now (rather than adding count of blocked
> * threads etc).
> */
> wakeup(keg);
> }
2799a2802
> zone_relock(zone, keg);
2801d2803
< zone_relock(zone, keg);
2804,2806c2806,2807
< ZONE_UNLOCK(zone);
< } else
< KEG_UNLOCK(keg);
---
> }
> ZONE_UNLOCK(zone);
2809a2811,2841
> /*
> * Frees a single item to any zone.
> *
> * Arguments:
> * zone The zone to free to
> * item The item we're freeing
> * udata User supplied data for the dtor
> * skip Skip dtors and finis
> */
> static void
> zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
> {
>
> #ifdef INVARIANTS
> if (skip == SKIP_NONE) {
> if (zone->uz_flags & UMA_ZONE_MALLOC)
> uma_dbg_free(zone, udata, item);
> else
> uma_dbg_free(zone, NULL, item);
> }
> #endif
> if (skip < SKIP_DTOR && zone->uz_dtor)
> zone->uz_dtor(item, zone->uz_size, udata);
>
> if (skip < SKIP_FINI && zone->uz_fini)
> zone->uz_fini(item, zone->uz_size);
>
> atomic_add_long(&zone->uz_frees, 1);
> zone->uz_release(zone->uz_arg, &item, 1);
> }
>
2816d2847
< ZONE_LOCK(zone);
2817a2849,2851
> if (keg == NULL)
> return (0);
> ZONE_LOCK(zone);
2834d2867
< ZONE_LOCK(zone);
2835a2869,2871
> if (keg == NULL)
> return (0);
> ZONE_LOCK(zone);
2882a2919
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2896a2934
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2929a2968
> uma_keg_t keg;
2932c2971,2973
< zone_first_keg(zone)->uk_freef = freef;
---
> keg = zone_first_keg(zone);
> KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
> keg->uk_freef = freef;
2958a3000,3001
> if (keg == NULL)
> return (0);
2996a3040,3041
> if (keg == NULL)
> return;
3086,3087c3131
< zone_free_item(slabzone, slab, NULL, SKIP_NONE,
< ZFREE_STATFAIL | ZFREE_STATFREE);
---
> zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3098c3142
< zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
---
> zone_free_item(slabzone, slab, NULL, SKIP_NONE);