1/*-
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 252040 2013-06-20 19:08:12Z jeff $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 181 unchanged lines hidden (view full) ---
241static int hash_expand(struct uma_hash *, struct uma_hash *);
242static void hash_free(struct uma_hash *hash);
243static void uma_timeout(void *);
244static void uma_startup3(void);
245static void *zone_alloc_item(uma_zone_t, void *, int);
246static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
247static void bucket_enable(void);
248static void bucket_init(void);
249static uma_bucket_t bucket_alloc(uma_zone_t zone, int);
250static void bucket_free(uma_zone_t zone, uma_bucket_t);
251static void bucket_zone_drain(void);
252static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, int flags);
253static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
254static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
255static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
256static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
257static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
258 uma_fini fini, int align, uint32_t flags);
259static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
260static void zone_release(uma_zone_t zone, void **bucket, int cnt);
261
262void uma_print_zone(uma_zone_t);
263void uma_print_stats(void);
264static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
265static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
266
--- 78 unchanged lines hidden (view full) ---
345 for (; ubz->ubz_entries != 0; ubz++)
346 if (ubz->ubz_maxsize < size)
347 break;
348 ubz--;
349 return (ubz->ubz_entries);
350}
351
352static uma_bucket_t
353bucket_alloc(uma_zone_t zone, int flags)
354{
355 struct uma_bucket_zone *ubz;
356 uma_bucket_t bucket;
357
358 /*
359 * This is to stop us from allocating per cpu buckets while we're
360 * running out of vm.boot_pages. Otherwise, we would exhaust the
361 * boot pages. This also prevents us from allocating buckets in
362 * low memory situations.
363 */
364 if (bucketdisable)
365 return (NULL);
366
367 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
368 flags |= M_NOVM;
369 ubz = bucket_zone_lookup(zone->uz_count);
370 bucket = uma_zalloc(ubz->ubz_zone, flags);
371 if (bucket) {
372#ifdef INVARIANTS
373 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
374#endif
375 bucket->ub_cnt = 0;
376 bucket->ub_entries = ubz->ubz_entries;
377 }
378
379 return (bucket);
380}
381
382static void
383bucket_free(uma_zone_t zone, uma_bucket_t bucket)
384{
385 struct uma_bucket_zone *ubz;
386
387 KASSERT(bucket->ub_cnt == 0,
388 ("bucket_free: Freeing a non free bucket."));
389 ubz = bucket_zone_lookup(bucket->ub_entries);
390 uma_zfree(ubz->ubz_zone, bucket);
391}
--- 265 unchanged lines hidden (view full) ---
657 * it is used elsewhere. Should the tear-down path be made special
658 * there in some form?
659 */
660 CPU_FOREACH(cpu) {
661 cache = &zone->uz_cpu[cpu];
662 bucket_drain(zone, cache->uc_allocbucket);
663 bucket_drain(zone, cache->uc_freebucket);
664 if (cache->uc_allocbucket != NULL)
665 bucket_free(zone, cache->uc_allocbucket);
666 if (cache->uc_freebucket != NULL)
667 bucket_free(zone, cache->uc_freebucket);
668 cache->uc_allocbucket = cache->uc_freebucket = NULL;
669 }
670 ZONE_LOCK(zone);
671 bucket_cache_drain(zone);
672 ZONE_UNLOCK(zone);
673}
674
675/*
--- 7 unchanged lines hidden (view full) ---
683 /*
684 * Drain the bucket queues and free the buckets, we just keep two per
685 * cpu (alloc/free).
686 */
687 while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
688 LIST_REMOVE(bucket, ub_link);
689 ZONE_UNLOCK(zone);
690 bucket_drain(zone, bucket);
691 bucket_free(zone, bucket);
692 ZONE_LOCK(zone);
693 }
694}
695
696static void
697keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
698{
699 uint8_t *mem;
--- 96 unchanged lines hidden (view full) ---
796 * is the only call that knows the structure will still be available
797 * when it wakes up.
798 */
799 ZONE_LOCK(zone);
800 while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
801 if (waitok == M_NOWAIT)
802 goto out;
803 mtx_unlock(&uma_mtx);
804 msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
805 mtx_lock(&uma_mtx);
806 }
807 zone->uz_flags |= UMA_ZFLAG_DRAINING;
808 bucket_cache_drain(zone);
809 ZONE_UNLOCK(zone);
810 /*
811 * The DRAINING flag protects us from being freed while
812 * we're running. Normally the uma_mtx would protect us but we
--- 557 unchanged lines hidden (view full) ---
1370 if (booted < UMA_STARTUP2)
1371 keg->uk_allocf = startup_alloc;
1372#endif
1373 } else if (booted < UMA_STARTUP2 &&
1374 (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1375 keg->uk_allocf = startup_alloc;
1376
1377 /*
1378 * Initialize keg's lock
1379 */
1380 KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
1381
1382 /*
1383 * If we're putting the slab header in the actual page we need to
1384 * figure out where in each page it goes. This calculates a right
1385 * justified offset into the memory on an ALIGN_PTR boundary.
1386 */
1387 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1388 u_int totsize;
--- 72 unchanged lines hidden (view full) ---
1461 zone->uz_fails = 0;
1462 zone->uz_sleeps = 0;
1463 zone->uz_count = 0;
1464 zone->uz_flags = 0;
1465 zone->uz_warning = NULL;
1466 timevalclear(&zone->uz_ratecheck);
1467 keg = arg->keg;
1468
1469 ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
1470
1471 /*
1472 * This is a pure cache zone, no kegs.
1473 */
1474 if (arg->import) {
1475 zone->uz_size = arg->size;
1476 zone->uz_import = arg->import;
1477 zone->uz_release = arg->release;
1478 zone->uz_arg = arg->arg;
1479 zone->uz_lockptr = &zone->uz_lock;
1480 goto out;
1481 }
1482
1483 /*
1484 * Use the regular zone/keg/slab allocator.
1485 */
1486 zone->uz_import = (uma_import)zone_import;
1487 zone->uz_release = (uma_release)zone_release;
1488 zone->uz_arg = zone;
1489
1490 if (arg->flags & UMA_ZONE_SECONDARY) {
1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1492 zone->uz_init = arg->uminit;
1493 zone->uz_fini = arg->fini;
1494 zone->uz_lockptr = &keg->uk_lock;
1495 zone->uz_flags |= UMA_ZONE_SECONDARY;
1496 mtx_lock(&uma_mtx);
1497 ZONE_LOCK(zone);
1498 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1499 if (LIST_NEXT(z, uz_link) == NULL) {
1500 LIST_INSERT_AFTER(z, zone, uz_link);
1501 break;
1502 }
--- 21 unchanged lines hidden (view full) ---
1524 return (error);
1525 }
1526
1527 /*
1528 * Link in the first keg.
1529 */
1530 zone->uz_klink.kl_keg = keg;
1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1532 zone->uz_lockptr = &keg->uk_lock;
1533 zone->uz_size = keg->uk_size;
1534 zone->uz_flags |= (keg->uk_flags &
1535 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
1536
1537 /*
1538 * Some internal zones don't have room allocated for the per cpu
1539 * caches. If we're internal, bail out here.
1540 */
1541 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1542 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1543 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1544 return (0);
1545 }
1546
1547out:
1548 if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
1549 zone->uz_count = bucket_select(zone->uz_size);
1550 else
1551 zone->uz_count = BUCKET_MAX;
1552
1553 return (0);
1554}
1555
1556/*
1557 * Keg header dtor. This frees all data, destroys locks, frees the hash
--- 64 unchanged lines hidden (view full) ---
1622 * We only destroy kegs from non secondary zones.
1623 */
1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1625 mtx_lock(&uma_mtx);
1626 LIST_REMOVE(keg, uk_link);
1627 mtx_unlock(&uma_mtx);
1628 zone_free_item(kegs, keg, NULL, SKIP_NONE);
1629 }
1630 ZONE_LOCK_FINI(zone);
1631}
1632
1633/*
1634 * Traverses every zone in the system and calls a callback
1635 *
1636 * Arguments:
1637 * zfunc A pointer to a function which accepts a zone
1638 * as an argument.
--- 207 unchanged lines hidden (view full) ---
1846 args.keg = keg;
1847
1848 /* XXX Attaches only one keg of potentially many. */
1849 return (zone_alloc_item(zones, &args, M_WAITOK));
1850}
1851
1852/* See uma.h */
1853uma_zone_t
1854uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
1855 uma_init zinit, uma_fini zfini, uma_import zimport,
1856 uma_release zrelease, void *arg, int flags)
1857{
1858 struct uma_zctor_args args;
1859
1860 memset(&args, 0, sizeof(args));
1861 args.name = name;
1862 args.size = size;
1863 args.ctor = ctor;
1864 args.dtor = dtor;
1865 args.uminit = zinit;
1866 args.fini = zfini;
1867 args.import = zimport;
1868 args.release = zrelease;
1869 args.arg = arg;
1870 args.align = 0;
1871 args.flags = flags;
1872
1873 return (zone_alloc_item(zones, &args, M_WAITOK));
1874}
1875
1876static void
1877zone_lock_pair(uma_zone_t a, uma_zone_t b)
1878{
1879 if (a < b) {
1880 ZONE_LOCK(a);
1881 mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
1882 } else {
1883 ZONE_LOCK(b);
1884 mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
1885 }
1886}
1887
1888static void
1889zone_unlock_pair(uma_zone_t a, uma_zone_t b)
1890{
1891
1892 ZONE_UNLOCK(a);
--- 175 unchanged lines hidden (view full) ---
2068
2069 /*
2070 * Discard any empty allocation bucket while we hold no locks.
2071 */
2072 bucket = cache->uc_allocbucket;
2073 cache->uc_allocbucket = NULL;
2074 critical_exit();
2075 if (bucket != NULL)
2076 bucket_free(zone, bucket);
2077
2078 /* Short-circuit for zones without buckets and low memory. */
2079 if (zone->uz_count == 0 || bucketdisable)
2080 goto zalloc_item;
2081
2082 /*
2083 * Attempt to retrieve the item from the per-CPU cache has failed, so
2084 * we must go back to the zone. This requires the zone lock, so we
--- 153 unchanged lines hidden (view full) ---
2238 * could have while we were unlocked. Check again before we
2239 * fail.
2240 */
2241 flags |= M_NOVM;
2242 }
2243 return (slab);
2244}
2245
2246static uma_slab_t
2247zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2248{
2249 uma_slab_t slab;
2250
2251 if (keg == NULL) {
2252 keg = zone_first_keg(zone);
2253 KEG_LOCK(keg);
2254 }
2255
2256 for (;;) {
2257 slab = keg_fetch_slab(keg, zone, flags);
2258 if (slab)
2259 return (slab);
2260 if (flags & (M_NOWAIT | M_NOVM))
2261 break;
2262 }
2263 KEG_UNLOCK(keg);
2264 return (NULL);
2265}
2266
2267/*
2268 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
2269 * with the keg locked. On NULL no lock is held.
2270 *
2271 * The last pointer is used to seed the search. It is not required.
2272 */
2273static uma_slab_t
2274zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2275{
2276 uma_klink_t klink;
2277 uma_slab_t slab;
--- 7 unchanged lines hidden (view full) ---
2285 * as well. We don't want to block if we can find a provider
2286 * without blocking.
2287 */
2288 flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2289 /*
2290 * Use the last slab allocated as a hint for where to start
2291 * the search.
2292 */
2293 if (last != NULL) {
2294 slab = keg_fetch_slab(last, zone, flags);
2295 if (slab)
2296 return (slab);
2297 KEG_UNLOCK(last);
2298 }
2299 /*
2300 * Loop until we have a slab incase of transient failures
2301 * while M_WAITOK is specified. I'm not sure this is 100%
2302 * required but we've done it for so long now.
2303 */
2304 for (;;) {
2305 empty = 0;
2306 full = 0;
2307 /*
2308 * Search the available kegs for slabs. Be careful to hold the
2309 * correct lock while calling into the keg layer.
2310 */
2311 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2312 keg = klink->kl_keg;
2313 KEG_LOCK(keg);
2314 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2315 slab = keg_fetch_slab(keg, zone, flags);
2316 if (slab)
2317 return (slab);
2318 }
2319 if (keg->uk_flags & UMA_ZFLAG_FULL)
2320 full++;
2321 else
2322 empty++;
2323 KEG_UNLOCK(keg);
2324 }
2325 if (rflags & (M_NOWAIT | M_NOVM))
2326 break;
2327 flags = rflags;
2328 /*
2329 * All kegs are full. XXX We can't atomically check all kegs
2330 * and sleep so just sleep for a short period and retry.
2331 */
2332 if (full && !empty) {
2333 ZONE_LOCK(zone);
2334 zone->uz_flags |= UMA_ZFLAG_FULL;
2335 zone->uz_sleeps++;
2336 zone_log_warning(zone);
2337 msleep(zone, zone->uz_lockptr, PVM,
2338 "zonelimit", hz/100);
2339 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2340 ZONE_UNLOCK(zone);
2341 continue;
2342 }
2343 }
2344 return (NULL);
2345}
2346
2347static void *
2348slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
--- 21 unchanged lines hidden (view full) ---
2370
2371static int
2372zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2373{
2374 uma_slab_t slab;
2375 uma_keg_t keg;
2376 int i;
2377
2378 slab = NULL;
2379 keg = NULL;
2380 /* Try to keep the buckets totally full */
2381 for (i = 0; i < max; ) {
2382 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2383 break;
2384 keg = slab->us_keg;
2385 while (slab->us_freecount && i < max)
2386 bucket[i++] = slab_alloc_item(keg, slab);
2387
2388 /* Don't block on the next fill */
2389 flags &= ~M_WAITOK;
2390 flags |= M_NOWAIT;
2391 }
2392 if (slab != NULL)
2393 KEG_UNLOCK(keg);
2394
2395 return i;
2396}
2397
2398static uma_bucket_t
2399zone_alloc_bucket(uma_zone_t zone, int flags)
2400{
2401 uma_bucket_t bucket;
2402 int max;
2403
2404 bucket = bucket_alloc(zone, M_NOWAIT | (flags & M_NOVM));
2405 if (bucket == NULL)
2406 goto out;
2407
2408 max = MIN(bucket->ub_entries, zone->uz_count);
2409 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2410 max, flags);
2411
2412 /*
2413 * Initialize the memory if necessary.
2414 */
2415 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2416 int i;
2417
2418 for (i = 0; i < bucket->ub_cnt; i++)
2419 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
2420 flags) != 0)
2421 break;
2422 /*
2423 * If we couldn't initialize the whole bucket, put the
2424 * rest back onto the freelist.
2425 */
2426 if (i != bucket->ub_cnt) {
2427 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
2428 bucket->ub_cnt - i);
2429#ifdef INVARIANTS
2430 bzero(&bucket->ub_bucket[i],
2431 sizeof(void *) * (bucket->ub_cnt - i));
2432#endif
2433 bucket->ub_cnt = i;
2434 }
2435 }
2436
2437out:
2438 if (bucket == NULL || bucket->ub_cnt == 0) {
2439 if (bucket != NULL)
2440 bucket_free(zone, bucket);
2441 atomic_add_long(&zone->uz_fails, 1);
2442 return (NULL);
2443 }
2444
2445 return (bucket);
2446}
2447
2448/*
--- 55 unchanged lines hidden (view full) ---
2504}
2505
2506/* See uma.h */
2507void
2508uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2509{
2510 uma_cache_t cache;
2511 uma_bucket_t bucket;
2512 int cpu;
2513
2514#ifdef UMA_DEBUG_ALLOC_1
2515 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
2516#endif
2517 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
2518 zone->uz_name);
2519
--- 108 unchanged lines hidden (view full) ---
2628 critical_exit();
2629
2630 /* And the zone.. */
2631 ZONE_UNLOCK(zone);
2632
2633#ifdef UMA_DEBUG_ALLOC
2634 printf("uma_zfree: Allocating new free bucket.\n");
2635#endif
2636 bucket = bucket_alloc(zone, M_NOWAIT);
2637 if (bucket) {
2638 critical_enter();
2639 cpu = curcpu;
2640 cache = &zone->uz_cpu[cpu];
2641 if (cache->uc_freebucket == NULL) {
2642 cache->uc_freebucket = bucket;
2643 goto zfree_start;
2644 }
2645 /*
2646 * We lost the race, start over. We have to drop our
2647 * critical section to free the bucket.
2648 */
2649 critical_exit();
2650 bucket_free(zone, bucket);
2651 goto zfree_restart;
2652 }
2653
2654 /*
2655 * If nothing else caught this, we'll just do an internal free.
2656 */
2657zfree_item:
2658 zone_free_item(zone, item, udata, SKIP_DTOR);
--- 33 unchanged lines hidden (view full) ---
2692 void *item;
2693 uma_slab_t slab;
2694 uma_keg_t keg;
2695 uint8_t *mem;
2696 int clearfull;
2697 int i;
2698
2699 clearfull = 0;
2700 keg = zone_first_keg(zone);
2701 KEG_LOCK(keg);
2702 for (i = 0; i < cnt; i++) {
2703 item = bucket[i];
2704 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2705 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2706 if (zone->uz_flags & UMA_ZONE_HASH) {
2707 slab = hash_sfind(&keg->uk_hash, mem);
2708 } else {
2709 mem += keg->uk_pgoff;
--- 19 unchanged lines hidden (view full) ---
2729 * clearing ZFLAG_FULL, wake up all procs blocked
2730 * on pages. This should be uncommon, so keeping this
2731 * simple for now (rather than adding count of blocked
2732 * threads etc).
2733 */
2734 wakeup(keg);
2735 }
2736 }
2737 KEG_UNLOCK(keg);
2738 if (clearfull) {
2739 ZONE_LOCK(zone);
2740 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2741 wakeup(zone);
2742 ZONE_UNLOCK(zone);
2743 }
2744
2745}
2746
2747/*
2748 * Frees a single item to any zone.
2749 *
2750 * Arguments:
2751 * zone The zone to free to
--- 27 unchanged lines hidden (view full) ---
2779int
2780uma_zone_set_max(uma_zone_t zone, int nitems)
2781{
2782 uma_keg_t keg;
2783
2784 keg = zone_first_keg(zone);
2785 if (keg == NULL)
2786 return (0);
2787 KEG_LOCK(keg);
2788 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2789 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2790 keg->uk_maxpages += keg->uk_ppera;
2791 nitems = keg->uk_maxpages * keg->uk_ipers;
2792 KEG_UNLOCK(keg);
2793
2794 return (nitems);
2795}
2796
2797/* See uma.h */
2798int
2799uma_zone_get_max(uma_zone_t zone)
2800{
2801 int nitems;
2802 uma_keg_t keg;
2803
2804 keg = zone_first_keg(zone);
2805 if (keg == NULL)
2806 return (0);
2807 KEG_LOCK(keg);
2808 nitems = keg->uk_maxpages * keg->uk_ipers;
2809 KEG_UNLOCK(keg);
2810
2811 return (nitems);
2812}
2813
2814/* See uma.h */
2815void
2816uma_zone_set_warning(uma_zone_t zone, const char *warning)
2817{
--- 27 unchanged lines hidden (view full) ---
2845}
2846
2847/* See uma.h */
2848void
2849uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2850{
2851 uma_keg_t keg;
2852
2853 keg = zone_first_keg(zone);
2854 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2855 KEG_LOCK(keg);
2856 KASSERT(keg->uk_pages == 0,
2857 ("uma_zone_set_init on non-empty keg"));
2858 keg->uk_init = uminit;
2859 KEG_UNLOCK(keg);
2860}
2861
2862/* See uma.h */
2863void
2864uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2865{
2866 uma_keg_t keg;
2867
2868 keg = zone_first_keg(zone);
2869 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2870 KEG_LOCK(keg);
2871 KASSERT(keg->uk_pages == 0,
2872 ("uma_zone_set_fini on non-empty keg"));
2873 keg->uk_fini = fini;
2874 KEG_UNLOCK(keg);
2875}
2876
2877/* See uma.h */
2878void
2879uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
2880{
2881
2882 ZONE_LOCK(zone);
2883 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2884 ("uma_zone_set_zinit on non-empty keg"));
2885 zone->uz_init = zinit;
2886 ZONE_UNLOCK(zone);
2887}
2888
2889/* See uma.h */
2890void
2891uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
2892{
2893
2894 ZONE_LOCK(zone);
2895 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2896 ("uma_zone_set_zfini on non-empty keg"));
2897 zone->uz_fini = zfini;
2898 ZONE_UNLOCK(zone);
2899}
2900
2901/* See uma.h */
2902/* XXX uk_freef is not actually used with the zone locked */
2903void
2904uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2905{
2906 uma_keg_t keg;
2907
2908 keg = zone_first_keg(zone);
2909 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2910 KEG_LOCK(keg);
2911 keg->uk_freef = freef;
2912 KEG_UNLOCK(keg);
2913}
2914
2915/* See uma.h */
2916/* XXX uk_allocf is not actually used with the zone locked */
2917void
2918uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2919{
2920 uma_keg_t keg;
2921
2922 keg = zone_first_keg(zone);
2923 KEG_LOCK(keg);
2924 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
2925 keg->uk_allocf = allocf;
2926 KEG_UNLOCK(keg);
2927}
2928
2929/* See uma.h */
2930int
2931uma_zone_reserve_kva(uma_zone_t zone, int count)
2932{
2933 uma_keg_t keg;
2934 vm_offset_t kva;
--- 12 unchanged lines hidden (view full) ---
2947#else
2948 if (1) {
2949#endif
2950 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
2951 if (kva == 0)
2952 return (0);
2953 } else
2954 kva = 0;
2955 KEG_LOCK(keg);
2956 keg->uk_kva = kva;
2957 keg->uk_offset = 0;
2958 keg->uk_maxpages = pages;
2959#ifdef UMA_MD_SMALL_ALLOC
2960 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
2961#else
2962 keg->uk_allocf = noobj_alloc;
2963#endif
2964 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
2965 KEG_UNLOCK(keg);
2966
2967 return (1);
2968}
2969
2970/* See uma.h */
2971void
2972uma_prealloc(uma_zone_t zone, int items)
2973{
2974 int slabs;
2975 uma_slab_t slab;
2976 uma_keg_t keg;
2977
2978 keg = zone_first_keg(zone);
2979 if (keg == NULL)
2980 return;
2981 KEG_LOCK(keg);
2982 slabs = items / keg->uk_ipers;
2983 if (slabs * keg->uk_ipers < items)
2984 slabs++;
2985 while (slabs > 0) {
2986 slab = keg_alloc_slab(keg, zone, M_WAITOK);
2987 if (slab == NULL)
2988 break;
2989 MPASS(slab->us_keg == keg);
2990 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2991 slabs--;
2992 }
2993 KEG_UNLOCK(keg);
2994}
2995
2996/* See uma.h */
2997uint32_t *
2998uma_find_refcnt(uma_zone_t zone, void *item)
2999{
3000 uma_slabrefcnt_t slabref;
3001 uma_slab_t slab;
--- 346 unchanged lines hidden ---
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 252040 2013-06-20 19:08:12Z jeff $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 181 unchanged lines hidden (view full) ---
241static int hash_expand(struct uma_hash *, struct uma_hash *);
242static void hash_free(struct uma_hash *hash);
243static void uma_timeout(void *);
244static void uma_startup3(void);
245static void *zone_alloc_item(uma_zone_t, void *, int);
246static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
247static void bucket_enable(void);
248static void bucket_init(void);
249static uma_bucket_t bucket_alloc(uma_zone_t zone, int);
250static void bucket_free(uma_zone_t zone, uma_bucket_t);
251static void bucket_zone_drain(void);
252static uma_bucket_t zone_alloc_bucket(uma_zone_t zone, int flags);
253static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
254static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
255static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
256static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
257static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
258 uma_fini fini, int align, uint32_t flags);
259static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
260static void zone_release(uma_zone_t zone, void **bucket, int cnt);
261
262void uma_print_zone(uma_zone_t);
263void uma_print_stats(void);
264static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
265static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
266
--- 78 unchanged lines hidden (view full) ---
345 for (; ubz->ubz_entries != 0; ubz++)
346 if (ubz->ubz_maxsize < size)
347 break;
348 ubz--;
349 return (ubz->ubz_entries);
350}
351
352static uma_bucket_t
353bucket_alloc(uma_zone_t zone, int flags)
354{
355 struct uma_bucket_zone *ubz;
356 uma_bucket_t bucket;
357
358 /*
359 * This is to stop us from allocating per cpu buckets while we're
360 * running out of vm.boot_pages. Otherwise, we would exhaust the
361 * boot pages. This also prevents us from allocating buckets in
362 * low memory situations.
363 */
364 if (bucketdisable)
365 return (NULL);
366
367 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
368 flags |= M_NOVM;
369 ubz = bucket_zone_lookup(zone->uz_count);
370 bucket = uma_zalloc(ubz->ubz_zone, flags);
371 if (bucket) {
372#ifdef INVARIANTS
373 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries);
374#endif
375 bucket->ub_cnt = 0;
376 bucket->ub_entries = ubz->ubz_entries;
377 }
378
379 return (bucket);
380}
381
382static void
383bucket_free(uma_zone_t zone, uma_bucket_t bucket)
384{
385 struct uma_bucket_zone *ubz;
386
387 KASSERT(bucket->ub_cnt == 0,
388 ("bucket_free: Freeing a non free bucket."));
389 ubz = bucket_zone_lookup(bucket->ub_entries);
390 uma_zfree(ubz->ubz_zone, bucket);
391}
--- 265 unchanged lines hidden (view full) ---
657 * it is used elsewhere. Should the tear-down path be made special
658 * there in some form?
659 */
660 CPU_FOREACH(cpu) {
661 cache = &zone->uz_cpu[cpu];
662 bucket_drain(zone, cache->uc_allocbucket);
663 bucket_drain(zone, cache->uc_freebucket);
664 if (cache->uc_allocbucket != NULL)
665 bucket_free(zone, cache->uc_allocbucket);
666 if (cache->uc_freebucket != NULL)
667 bucket_free(zone, cache->uc_freebucket);
668 cache->uc_allocbucket = cache->uc_freebucket = NULL;
669 }
670 ZONE_LOCK(zone);
671 bucket_cache_drain(zone);
672 ZONE_UNLOCK(zone);
673}
674
675/*
--- 7 unchanged lines hidden (view full) ---
683 /*
684 * Drain the bucket queues and free the buckets, we just keep two per
685 * cpu (alloc/free).
686 */
687 while ((bucket = LIST_FIRST(&zone->uz_buckets)) != NULL) {
688 LIST_REMOVE(bucket, ub_link);
689 ZONE_UNLOCK(zone);
690 bucket_drain(zone, bucket);
691 bucket_free(zone, bucket);
692 ZONE_LOCK(zone);
693 }
694}
695
696static void
697keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
698{
699 uint8_t *mem;
--- 96 unchanged lines hidden (view full) ---
796 * is the only call that knows the structure will still be available
797 * when it wakes up.
798 */
799 ZONE_LOCK(zone);
800 while (zone->uz_flags & UMA_ZFLAG_DRAINING) {
801 if (waitok == M_NOWAIT)
802 goto out;
803 mtx_unlock(&uma_mtx);
804 msleep(zone, zone->uz_lockptr, PVM, "zonedrain", 1);
805 mtx_lock(&uma_mtx);
806 }
807 zone->uz_flags |= UMA_ZFLAG_DRAINING;
808 bucket_cache_drain(zone);
809 ZONE_UNLOCK(zone);
810 /*
811 * The DRAINING flag protects us from being freed while
812 * we're running. Normally the uma_mtx would protect us but we
--- 557 unchanged lines hidden (view full) ---
1370 if (booted < UMA_STARTUP2)
1371 keg->uk_allocf = startup_alloc;
1372#endif
1373 } else if (booted < UMA_STARTUP2 &&
1374 (keg->uk_flags & UMA_ZFLAG_INTERNAL))
1375 keg->uk_allocf = startup_alloc;
1376
1377 /*
1378 * Initialize keg's lock
1379 */
1380 KEG_LOCK_INIT(keg, (arg->flags & UMA_ZONE_MTXCLASS));
1381
1382 /*
1383 * If we're putting the slab header in the actual page we need to
1384 * figure out where in each page it goes. This calculates a right
1385 * justified offset into the memory on an ALIGN_PTR boundary.
1386 */
1387 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) {
1388 u_int totsize;
--- 72 unchanged lines hidden (view full) ---
1461 zone->uz_fails = 0;
1462 zone->uz_sleeps = 0;
1463 zone->uz_count = 0;
1464 zone->uz_flags = 0;
1465 zone->uz_warning = NULL;
1466 timevalclear(&zone->uz_ratecheck);
1467 keg = arg->keg;
1468
1469 ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
1470
1471 /*
1472 * This is a pure cache zone, no kegs.
1473 */
1474 if (arg->import) {
1475 zone->uz_size = arg->size;
1476 zone->uz_import = arg->import;
1477 zone->uz_release = arg->release;
1478 zone->uz_arg = arg->arg;
1479 zone->uz_lockptr = &zone->uz_lock;
1480 goto out;
1481 }
1482
1483 /*
1484 * Use the regular zone/keg/slab allocator.
1485 */
1486 zone->uz_import = (uma_import)zone_import;
1487 zone->uz_release = (uma_release)zone_release;
1488 zone->uz_arg = zone;
1489
1490 if (arg->flags & UMA_ZONE_SECONDARY) {
1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1492 zone->uz_init = arg->uminit;
1493 zone->uz_fini = arg->fini;
1494 zone->uz_lockptr = &keg->uk_lock;
1495 zone->uz_flags |= UMA_ZONE_SECONDARY;
1496 mtx_lock(&uma_mtx);
1497 ZONE_LOCK(zone);
1498 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
1499 if (LIST_NEXT(z, uz_link) == NULL) {
1500 LIST_INSERT_AFTER(z, zone, uz_link);
1501 break;
1502 }
--- 21 unchanged lines hidden (view full) ---
1524 return (error);
1525 }
1526
1527 /*
1528 * Link in the first keg.
1529 */
1530 zone->uz_klink.kl_keg = keg;
1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1532 zone->uz_lockptr = &keg->uk_lock;
1533 zone->uz_size = keg->uk_size;
1534 zone->uz_flags |= (keg->uk_flags &
1535 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT));
1536
1537 /*
1538 * Some internal zones don't have room allocated for the per cpu
1539 * caches. If we're internal, bail out here.
1540 */
1541 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) {
1542 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0,
1543 ("Secondary zone requested UMA_ZFLAG_INTERNAL"));
1544 return (0);
1545 }
1546
1547out:
1548 if ((arg->flags & UMA_ZONE_MAXBUCKET) == 0)
1549 zone->uz_count = bucket_select(zone->uz_size);
1550 else
1551 zone->uz_count = BUCKET_MAX;
1552
1553 return (0);
1554}
1555
1556/*
1557 * Keg header dtor. This frees all data, destroys locks, frees the hash
--- 64 unchanged lines hidden (view full) ---
1622 * We only destroy kegs from non secondary zones.
1623 */
1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1625 mtx_lock(&uma_mtx);
1626 LIST_REMOVE(keg, uk_link);
1627 mtx_unlock(&uma_mtx);
1628 zone_free_item(kegs, keg, NULL, SKIP_NONE);
1629 }
1630 ZONE_LOCK_FINI(zone);
1631}
1632
1633/*
1634 * Traverses every zone in the system and calls a callback
1635 *
1636 * Arguments:
1637 * zfunc A pointer to a function which accepts a zone
1638 * as an argument.
--- 207 unchanged lines hidden (view full) ---
1846 args.keg = keg;
1847
1848 /* XXX Attaches only one keg of potentially many. */
1849 return (zone_alloc_item(zones, &args, M_WAITOK));
1850}
1851
1852/* See uma.h */
1853uma_zone_t
1854uma_zcache_create(char *name, int size, uma_ctor ctor, uma_dtor dtor,
1855 uma_init zinit, uma_fini zfini, uma_import zimport,
1856 uma_release zrelease, void *arg, int flags)
1857{
1858 struct uma_zctor_args args;
1859
1860 memset(&args, 0, sizeof(args));
1861 args.name = name;
1862 args.size = size;
1863 args.ctor = ctor;
1864 args.dtor = dtor;
1865 args.uminit = zinit;
1866 args.fini = zfini;
1867 args.import = zimport;
1868 args.release = zrelease;
1869 args.arg = arg;
1870 args.align = 0;
1871 args.flags = flags;
1872
1873 return (zone_alloc_item(zones, &args, M_WAITOK));
1874}
1875
1876static void
1877zone_lock_pair(uma_zone_t a, uma_zone_t b)
1878{
1879 if (a < b) {
1880 ZONE_LOCK(a);
1881 mtx_lock_flags(b->uz_lockptr, MTX_DUPOK);
1882 } else {
1883 ZONE_LOCK(b);
1884 mtx_lock_flags(a->uz_lockptr, MTX_DUPOK);
1885 }
1886}
1887
1888static void
1889zone_unlock_pair(uma_zone_t a, uma_zone_t b)
1890{
1891
1892 ZONE_UNLOCK(a);
--- 175 unchanged lines hidden (view full) ---
2068
2069 /*
2070 * Discard any empty allocation bucket while we hold no locks.
2071 */
2072 bucket = cache->uc_allocbucket;
2073 cache->uc_allocbucket = NULL;
2074 critical_exit();
2075 if (bucket != NULL)
2076 bucket_free(zone, bucket);
2077
2078 /* Short-circuit for zones without buckets and low memory. */
2079 if (zone->uz_count == 0 || bucketdisable)
2080 goto zalloc_item;
2081
2082 /*
2083 * Attempt to retrieve the item from the per-CPU cache has failed, so
2084 * we must go back to the zone. This requires the zone lock, so we
--- 153 unchanged lines hidden (view full) ---
2238 * could have while we were unlocked. Check again before we
2239 * fail.
2240 */
2241 flags |= M_NOVM;
2242 }
2243 return (slab);
2244}
2245
2246static uma_slab_t
2247zone_fetch_slab(uma_zone_t zone, uma_keg_t keg, int flags)
2248{
2249 uma_slab_t slab;
2250
2251 if (keg == NULL) {
2252 keg = zone_first_keg(zone);
2253 KEG_LOCK(keg);
2254 }
2255
2256 for (;;) {
2257 slab = keg_fetch_slab(keg, zone, flags);
2258 if (slab)
2259 return (slab);
2260 if (flags & (M_NOWAIT | M_NOVM))
2261 break;
2262 }
2263 KEG_UNLOCK(keg);
2264 return (NULL);
2265}
2266
2267/*
2268 * uma_zone_fetch_slab_multi: Fetches a slab from one available keg. Returns
2269 * with the keg locked. On NULL no lock is held.
2270 *
2271 * The last pointer is used to seed the search. It is not required.
2272 */
2273static uma_slab_t
2274zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int rflags)
2275{
2276 uma_klink_t klink;
2277 uma_slab_t slab;
--- 7 unchanged lines hidden (view full) ---
2285 * as well. We don't want to block if we can find a provider
2286 * without blocking.
2287 */
2288 flags = (rflags & ~M_WAITOK) | M_NOWAIT;
2289 /*
2290 * Use the last slab allocated as a hint for where to start
2291 * the search.
2292 */
2293 if (last != NULL) {
2294 slab = keg_fetch_slab(last, zone, flags);
2295 if (slab)
2296 return (slab);
2297 KEG_UNLOCK(last);
2298 }
2299 /*
2300 * Loop until we have a slab incase of transient failures
2301 * while M_WAITOK is specified. I'm not sure this is 100%
2302 * required but we've done it for so long now.
2303 */
2304 for (;;) {
2305 empty = 0;
2306 full = 0;
2307 /*
2308 * Search the available kegs for slabs. Be careful to hold the
2309 * correct lock while calling into the keg layer.
2310 */
2311 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) {
2312 keg = klink->kl_keg;
2313 KEG_LOCK(keg);
2314 if ((keg->uk_flags & UMA_ZFLAG_FULL) == 0) {
2315 slab = keg_fetch_slab(keg, zone, flags);
2316 if (slab)
2317 return (slab);
2318 }
2319 if (keg->uk_flags & UMA_ZFLAG_FULL)
2320 full++;
2321 else
2322 empty++;
2323 KEG_UNLOCK(keg);
2324 }
2325 if (rflags & (M_NOWAIT | M_NOVM))
2326 break;
2327 flags = rflags;
2328 /*
2329 * All kegs are full. XXX We can't atomically check all kegs
2330 * and sleep so just sleep for a short period and retry.
2331 */
2332 if (full && !empty) {
2333 ZONE_LOCK(zone);
2334 zone->uz_flags |= UMA_ZFLAG_FULL;
2335 zone->uz_sleeps++;
2336 zone_log_warning(zone);
2337 msleep(zone, zone->uz_lockptr, PVM,
2338 "zonelimit", hz/100);
2339 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2340 ZONE_UNLOCK(zone);
2341 continue;
2342 }
2343 }
2344 return (NULL);
2345}
2346
2347static void *
2348slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
--- 21 unchanged lines hidden (view full) ---
2370
2371static int
2372zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2373{
2374 uma_slab_t slab;
2375 uma_keg_t keg;
2376 int i;
2377
2378 slab = NULL;
2379 keg = NULL;
2380 /* Try to keep the buckets totally full */
2381 for (i = 0; i < max; ) {
2382 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2383 break;
2384 keg = slab->us_keg;
2385 while (slab->us_freecount && i < max)
2386 bucket[i++] = slab_alloc_item(keg, slab);
2387
2388 /* Don't block on the next fill */
2389 flags &= ~M_WAITOK;
2390 flags |= M_NOWAIT;
2391 }
2392 if (slab != NULL)
2393 KEG_UNLOCK(keg);
2394
2395 return i;
2396}
2397
2398static uma_bucket_t
2399zone_alloc_bucket(uma_zone_t zone, int flags)
2400{
2401 uma_bucket_t bucket;
2402 int max;
2403
2404 bucket = bucket_alloc(zone, M_NOWAIT | (flags & M_NOVM));
2405 if (bucket == NULL)
2406 goto out;
2407
2408 max = MIN(bucket->ub_entries, zone->uz_count);
2409 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2410 max, flags);
2411
2412 /*
2413 * Initialize the memory if necessary.
2414 */
2415 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2416 int i;
2417
2418 for (i = 0; i < bucket->ub_cnt; i++)
2419 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
2420 flags) != 0)
2421 break;
2422 /*
2423 * If we couldn't initialize the whole bucket, put the
2424 * rest back onto the freelist.
2425 */
2426 if (i != bucket->ub_cnt) {
2427 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i],
2428 bucket->ub_cnt - i);
2429#ifdef INVARIANTS
2430 bzero(&bucket->ub_bucket[i],
2431 sizeof(void *) * (bucket->ub_cnt - i));
2432#endif
2433 bucket->ub_cnt = i;
2434 }
2435 }
2436
2437out:
2438 if (bucket == NULL || bucket->ub_cnt == 0) {
2439 if (bucket != NULL)
2440 bucket_free(zone, bucket);
2441 atomic_add_long(&zone->uz_fails, 1);
2442 return (NULL);
2443 }
2444
2445 return (bucket);
2446}
2447
2448/*
--- 55 unchanged lines hidden (view full) ---
2504}
2505
2506/* See uma.h */
2507void
2508uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2509{
2510 uma_cache_t cache;
2511 uma_bucket_t bucket;
2512 int cpu;
2513
2514#ifdef UMA_DEBUG_ALLOC_1
2515 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone);
2516#endif
2517 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread,
2518 zone->uz_name);
2519
--- 108 unchanged lines hidden (view full) ---
2628 critical_exit();
2629
2630 /* And the zone.. */
2631 ZONE_UNLOCK(zone);
2632
2633#ifdef UMA_DEBUG_ALLOC
2634 printf("uma_zfree: Allocating new free bucket.\n");
2635#endif
2636 bucket = bucket_alloc(zone, M_NOWAIT);
2637 if (bucket) {
2638 critical_enter();
2639 cpu = curcpu;
2640 cache = &zone->uz_cpu[cpu];
2641 if (cache->uc_freebucket == NULL) {
2642 cache->uc_freebucket = bucket;
2643 goto zfree_start;
2644 }
2645 /*
2646 * We lost the race, start over. We have to drop our
2647 * critical section to free the bucket.
2648 */
2649 critical_exit();
2650 bucket_free(zone, bucket);
2651 goto zfree_restart;
2652 }
2653
2654 /*
2655 * If nothing else caught this, we'll just do an internal free.
2656 */
2657zfree_item:
2658 zone_free_item(zone, item, udata, SKIP_DTOR);
--- 33 unchanged lines hidden (view full) ---
2692 void *item;
2693 uma_slab_t slab;
2694 uma_keg_t keg;
2695 uint8_t *mem;
2696 int clearfull;
2697 int i;
2698
2699 clearfull = 0;
2700 keg = zone_first_keg(zone);
2701 KEG_LOCK(keg);
2702 for (i = 0; i < cnt; i++) {
2703 item = bucket[i];
2704 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2705 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2706 if (zone->uz_flags & UMA_ZONE_HASH) {
2707 slab = hash_sfind(&keg->uk_hash, mem);
2708 } else {
2709 mem += keg->uk_pgoff;
--- 19 unchanged lines hidden (view full) ---
2729 * clearing ZFLAG_FULL, wake up all procs blocked
2730 * on pages. This should be uncommon, so keeping this
2731 * simple for now (rather than adding count of blocked
2732 * threads etc).
2733 */
2734 wakeup(keg);
2735 }
2736 }
2737 KEG_UNLOCK(keg);
2738 if (clearfull) {
2739 ZONE_LOCK(zone);
2740 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2741 wakeup(zone);
2742 ZONE_UNLOCK(zone);
2743 }
2744
2745}
2746
2747/*
2748 * Frees a single item to any zone.
2749 *
2750 * Arguments:
2751 * zone The zone to free to
--- 27 unchanged lines hidden (view full) ---
2779int
2780uma_zone_set_max(uma_zone_t zone, int nitems)
2781{
2782 uma_keg_t keg;
2783
2784 keg = zone_first_keg(zone);
2785 if (keg == NULL)
2786 return (0);
2787 KEG_LOCK(keg);
2788 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2789 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2790 keg->uk_maxpages += keg->uk_ppera;
2791 nitems = keg->uk_maxpages * keg->uk_ipers;
2792 KEG_UNLOCK(keg);
2793
2794 return (nitems);
2795}
2796
2797/* See uma.h */
2798int
2799uma_zone_get_max(uma_zone_t zone)
2800{
2801 int nitems;
2802 uma_keg_t keg;
2803
2804 keg = zone_first_keg(zone);
2805 if (keg == NULL)
2806 return (0);
2807 KEG_LOCK(keg);
2808 nitems = keg->uk_maxpages * keg->uk_ipers;
2809 KEG_UNLOCK(keg);
2810
2811 return (nitems);
2812}
2813
2814/* See uma.h */
2815void
2816uma_zone_set_warning(uma_zone_t zone, const char *warning)
2817{
--- 27 unchanged lines hidden (view full) ---
2845}
2846
2847/* See uma.h */
2848void
2849uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2850{
2851 uma_keg_t keg;
2852
2853 keg = zone_first_keg(zone);
2854 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2855 KEG_LOCK(keg);
2856 KASSERT(keg->uk_pages == 0,
2857 ("uma_zone_set_init on non-empty keg"));
2858 keg->uk_init = uminit;
2859 KEG_UNLOCK(keg);
2860}
2861
2862/* See uma.h */
2863void
2864uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2865{
2866 uma_keg_t keg;
2867
2868 keg = zone_first_keg(zone);
2869 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2870 KEG_LOCK(keg);
2871 KASSERT(keg->uk_pages == 0,
2872 ("uma_zone_set_fini on non-empty keg"));
2873 keg->uk_fini = fini;
2874 KEG_UNLOCK(keg);
2875}
2876
2877/* See uma.h */
2878void
2879uma_zone_set_zinit(uma_zone_t zone, uma_init zinit)
2880{
2881
2882 ZONE_LOCK(zone);
2883 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2884 ("uma_zone_set_zinit on non-empty keg"));
2885 zone->uz_init = zinit;
2886 ZONE_UNLOCK(zone);
2887}
2888
2889/* See uma.h */
2890void
2891uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini)
2892{
2893
2894 ZONE_LOCK(zone);
2895 KASSERT(zone_first_keg(zone)->uk_pages == 0,
2896 ("uma_zone_set_zfini on non-empty keg"));
2897 zone->uz_fini = zfini;
2898 ZONE_UNLOCK(zone);
2899}
2900
2901/* See uma.h */
2902/* XXX uk_freef is not actually used with the zone locked */
2903void
2904uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2905{
2906 uma_keg_t keg;
2907
2908 keg = zone_first_keg(zone);
2909 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2910 KEG_LOCK(keg);
2911 keg->uk_freef = freef;
2912 KEG_UNLOCK(keg);
2913}
2914
2915/* See uma.h */
2916/* XXX uk_allocf is not actually used with the zone locked */
2917void
2918uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2919{
2920 uma_keg_t keg;
2921
2922 keg = zone_first_keg(zone);
2923 KEG_LOCK(keg);
2924 keg->uk_flags |= UMA_ZFLAG_PRIVALLOC;
2925 keg->uk_allocf = allocf;
2926 KEG_UNLOCK(keg);
2927}
2928
2929/* See uma.h */
2930int
2931uma_zone_reserve_kva(uma_zone_t zone, int count)
2932{
2933 uma_keg_t keg;
2934 vm_offset_t kva;
--- 12 unchanged lines hidden (view full) ---
2947#else
2948 if (1) {
2949#endif
2950 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE);
2951 if (kva == 0)
2952 return (0);
2953 } else
2954 kva = 0;
2955 KEG_LOCK(keg);
2956 keg->uk_kva = kva;
2957 keg->uk_offset = 0;
2958 keg->uk_maxpages = pages;
2959#ifdef UMA_MD_SMALL_ALLOC
2960 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
2961#else
2962 keg->uk_allocf = noobj_alloc;
2963#endif
2964 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC;
2965 KEG_UNLOCK(keg);
2966
2967 return (1);
2968}
2969
2970/* See uma.h */
2971void
2972uma_prealloc(uma_zone_t zone, int items)
2973{
2974 int slabs;
2975 uma_slab_t slab;
2976 uma_keg_t keg;
2977
2978 keg = zone_first_keg(zone);
2979 if (keg == NULL)
2980 return;
2981 KEG_LOCK(keg);
2982 slabs = items / keg->uk_ipers;
2983 if (slabs * keg->uk_ipers < items)
2984 slabs++;
2985 while (slabs > 0) {
2986 slab = keg_alloc_slab(keg, zone, M_WAITOK);
2987 if (slab == NULL)
2988 break;
2989 MPASS(slab->us_keg == keg);
2990 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2991 slabs--;
2992 }
2993 KEG_UNLOCK(keg);
2994}
2995
2996/* See uma.h */
2997uint32_t *
2998uma_find_refcnt(uma_zone_t zone, void *item)
2999{
3000 uma_slabrefcnt_t slabref;
3001 uma_slab_t slab;
--- 346 unchanged lines hidden ---