1/*-
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 66 unchanged lines hidden (view full) ---
126 * Are we allowed to allocate buckets?
127 */
128static int bucketdisable = 1;
129
130/* Linked list of all kegs in the system */
131static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
132
133/* This mutex protects the keg list */
134static struct mtx_padalign uma_mtx;
135
136/* Linked list of boot time pages */
137static LIST_HEAD(,uma_slab) uma_boot_pages =
138 LIST_HEAD_INITIALIZER(uma_boot_pages);
139
140/* This mutex protects the boot time pages list */
141static struct mtx_padalign uma_boot_pages_mtx;
142
143/* Is the VM done starting up? */
144static int booted = 0;
145#define UMA_STARTUP 1
146#define UMA_STARTUP2 2
147
148/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */
149static const u_int uma_max_ipers = SLAB_SETSIZE;
--- 17 unchanged lines hidden (view full) ---
167 */
168struct uma_zctor_args {
169 const char *name;
170 size_t size;
171 uma_ctor ctor;
172 uma_dtor dtor;
173 uma_init uminit;
174 uma_fini fini;
175 uma_import import;
176 uma_release release;
177 void *arg;
178 uma_keg_t keg;
179 int align;
180 uint32_t flags;
181};
182
183struct uma_kctor_args {
184 uma_zone_t zone;
185 size_t size;
--- 28 unchanged lines hidden (view full) ---
214 */
215static uint8_t bucket_size[BUCKET_ZONES];
216
217/*
218 * Flags and enumerations to be passed to internal functions.
219 */
220enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
221
222/* Prototypes.. */
223
224static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
225static void *page_alloc(uma_zone_t, int, uint8_t *, int);
226static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
227static void page_free(void *, int, uint8_t);
228static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
229static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
239static void zone_foreach(void (*zfunc)(uma_zone_t));
240static void zone_timeout(uma_zone_t zone);
241static int hash_alloc(struct uma_hash *);
242static int hash_expand(struct uma_hash *, struct uma_hash *);
243static void hash_free(struct uma_hash *hash);
244static void uma_timeout(void *);
245static void uma_startup3(void);
246static void *zone_alloc_item(uma_zone_t, void *, int);
247static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
248static void bucket_enable(void);
249static void bucket_init(void);
250static uma_bucket_t bucket_alloc(int, int);
251static void bucket_free(uma_bucket_t);
252static void bucket_zone_drain(void);
253static int zone_alloc_bucket(uma_zone_t zone, int flags);
254static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
255static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
256static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
257static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
258static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
259 uma_fini fini, int align, uint32_t flags);
260static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
261static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
262static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
263static void zone_release(uma_zone_t zone, void **bucket, int cnt);
264
265void uma_print_zone(uma_zone_t);
266void uma_print_stats(void);
267static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
268static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
269
270SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
271
--- 88 unchanged lines hidden (view full) ---
360}
361
362static void
363bucket_free(uma_bucket_t bucket)
364{
365 struct uma_bucket_zone *ubz;
366
367 ubz = bucket_zone_lookup(bucket->ub_entries);
368 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
369}
370
371static void
372bucket_zone_drain(void)
373{
374 struct uma_bucket_zone *ubz;
375
376 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 7 unchanged lines hidden (view full) ---
384
385 if (!zone_warnings || zone->uz_warning == NULL)
386 return;
387
388 if (ratecheck(&zone->uz_ratecheck, &warninterval))
389 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
390}
391
392static void
393zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
394{
395 uma_klink_t klink;
396
397 LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
398 kegfn(klink->kl_keg);
399}
--- 168 unchanged lines hidden (view full) ---
568 * Nothing
569 */
570static void
571hash_free(struct uma_hash *hash)
572{
573 if (hash->uh_slab_hash == NULL)
574 return;
575 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
576 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
577 else
578 free(hash->uh_slab_hash, M_UMAHASH);
579}
580
581/*
582 * Frees all outstanding items in a bucket
583 *
584 * Arguments:
585 * zone The zone to free to, must be unlocked.
586 * bucket The free/alloc bucket with items, cpu queue must be locked.
587 *
588 * Returns:
589 * Nothing
590 */
591
592static void
593bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
594{
595 int i;
596
597 if (bucket == NULL)
598 return;
599
600 if (zone->uz_fini)
601 for (i = 0; i < bucket->ub_cnt; i++)
602 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
603 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
604 bucket->ub_cnt = 0;
605}
606
607/*
608 * Drains the per cpu caches for a zone.
609 *
610 * NOTE: This may only be called while the zone is being turn down, and not
611 * during normal operation. This is necessary in order that we do not have
612 * to migrate CPUs to drain the per-CPU caches.
--- 137 unchanged lines hidden (view full) ---
750 obj = kernel_object;
751 else
752 obj = NULL;
753 for (i = 0; i < keg->uk_ppera; i++)
754 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
755 obj);
756 }
757 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
758 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
759#ifdef UMA_DEBUG
760 printf("%s: Returning %d bytes.\n",
761 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
762#endif
763 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
764 }
765}
766
--- 57 unchanged lines hidden (view full) ---
824 uint8_t *mem;
825 uint8_t flags;
826 int i;
827
828 mtx_assert(&keg->uk_lock, MA_OWNED);
829 slab = NULL;
830
831#ifdef UMA_DEBUG
832 printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
833#endif
834 allocf = keg->uk_allocf;
835 KEG_UNLOCK(keg);
836
837 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
838 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
839 if (slab == NULL) {
840 KEG_LOCK(keg);
--- 15 unchanged lines hidden (view full) ---
856
857 if (keg->uk_flags & UMA_ZONE_NODUMP)
858 wait |= M_NODUMP;
859
860 /* zone is passed for legacy reasons. */
861 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
862 if (mem == NULL) {
863 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
864 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
865 KEG_LOCK(keg);
866 return (NULL);
867 }
868
869 /* Point the slab into the allocated memory */
870 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
871 slab = (uma_slab_t )(mem + keg->uk_pgoff);
872
--- 37 unchanged lines hidden (view full) ---
910 else
911 obj = NULL;
912 for (i = 0; i < keg->uk_ppera; i++)
913 vsetobj((vm_offset_t)mem +
914 (i * PAGE_SIZE), obj);
915 }
916 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
917 zone_free_item(keg->uk_slabzone, slab,
918 NULL, SKIP_NONE);
919 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
920 flags);
921 KEG_LOCK(keg);
922 return (NULL);
923 }
924 }
925 KEG_LOCK(keg);
926
--- 537 unchanged lines hidden (view full) ---
1464 zone->uz_fails = 0;
1465 zone->uz_sleeps = 0;
1466 zone->uz_fills = zone->uz_count = 0;
1467 zone->uz_flags = 0;
1468 zone->uz_warning = NULL;
1469 timevalclear(&zone->uz_ratecheck);
1470 keg = arg->keg;
1471
1472 /*
1473 * This is a pure cache zone, no kegs.
1474 */
1475 if (arg->import) {
1476 zone->uz_import = arg->import;
1477 zone->uz_release = arg->release;
1478 zone->uz_arg = arg->arg;
1479 zone->uz_count = BUCKET_MAX;
1480 return (0);
1481 }
1482
1483 /*
1484 * Use the regular zone/keg/slab allocator.
1485 */
1486 zone->uz_import = (uma_import)zone_import;
1487 zone->uz_release = (uma_release)zone_release;
1488 zone->uz_arg = zone;
1489
1490 if (arg->flags & UMA_ZONE_SECONDARY) {
1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1492 zone->uz_init = arg->uminit;
1493 zone->uz_fini = arg->fini;
1494 zone->uz_lock = &keg->uk_lock;
1495 zone->uz_flags |= UMA_ZONE_SECONDARY;
1496 mtx_lock(&uma_mtx);
1497 ZONE_LOCK(zone);
--- 20 unchanged lines hidden (view full) ---
1518 karg.align = arg->align;
1519 karg.flags = arg->flags;
1520 karg.zone = zone;
1521 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1522 flags);
1523 if (error)
1524 return (error);
1525 }
1526
1527 /*
1528 * Link in the first keg.
1529 */
1530 zone->uz_klink.kl_keg = keg;
1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1532 zone->uz_lock = &keg->uk_lock;
1533 zone->uz_size = keg->uk_size;
1534 zone->uz_flags |= (keg->uk_flags &
--- 81 unchanged lines hidden (view full) ---
1616 LIST_REMOVE(klink, kl_link);
1617 if (klink == &zone->uz_klink)
1618 continue;
1619 free(klink, M_TEMP);
1620 }
1621 /*
1622 * We only destroy kegs from non secondary zones.
1623 */
1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1625 mtx_lock(&uma_mtx);
1626 LIST_REMOVE(keg, uk_link);
1627 mtx_unlock(&uma_mtx);
1628 zone_free_item(kegs, keg, NULL, SKIP_NONE);
1629 }
1630}
1631
1632/*
1633 * Traverses every zone in the system and calls a callback
1634 *
1635 * Arguments:
1636 * zfunc A pointer to a function which accepts a zone
--- 27 unchanged lines hidden (view full) ---
1664 int i;
1665
1666#ifdef UMA_DEBUG
1667 printf("Creating uma keg headers zone and keg.\n");
1668#endif
1669 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1670
1671 /* "manually" create the initial zone */
1672 memset(&args, 0, sizeof(args));
1673 args.name = "UMA Kegs";
1674 args.size = sizeof(struct uma_keg);
1675 args.ctor = keg_ctor;
1676 args.dtor = keg_dtor;
1677 args.uminit = zero_init;
1678 args.fini = NULL;
1679 args.keg = &masterkeg;
1680 args.align = 32 - 1;
--- 124 unchanged lines hidden (view full) ---
1805uma_zone_t
1806uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1807 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1808
1809{
1810 struct uma_zctor_args args;
1811
1812 /* This stuff is essential for the zone ctor */
1813 memset(&args, 0, sizeof(args));
1814 args.name = name;
1815 args.size = size;
1816 args.ctor = ctor;
1817 args.dtor = dtor;
1818 args.uminit = uminit;
1819 args.fini = fini;
1820 args.align = align;
1821 args.flags = flags;
--- 6 unchanged lines hidden (view full) ---
1828uma_zone_t
1829uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1830 uma_init zinit, uma_fini zfini, uma_zone_t master)
1831{
1832 struct uma_zctor_args args;
1833 uma_keg_t keg;
1834
1835 keg = zone_first_keg(master);
1836 memset(&args, 0, sizeof(args));
1837 args.name = name;
1838 args.size = keg->uk_size;
1839 args.ctor = ctor;
1840 args.dtor = dtor;
1841 args.uminit = zinit;
1842 args.fini = zfini;
1843 args.align = keg->uk_align;
1844 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
1845 args.keg = keg;
1846
1847 /* XXX Attaches only one keg of potentially many. */
1848 return (zone_alloc_item(zones, &args, M_WAITOK));
1849}
1850
1851/* See uma.h */
1852uma_zone_t
1853uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
1854 uma_fini zfini, uma_import zimport, uma_release zrelease,
1855 void *arg, int flags)
1856{
1857 struct uma_zctor_args args;
1858
1859 memset(&args, 0, sizeof(args));
1860 args.name = name;
1861 args.size = 0;
1862 args.ctor = ctor;
1863 args.dtor = dtor;
1864 args.uminit = zinit;
1865 args.fini = zfini;
1866 args.import = zimport;
1867 args.release = zrelease;
1868 args.arg = arg;
1869 args.align = 0;
1870 args.flags = flags;
1871
1872 return (zone_alloc_item(zones, &args, M_WAITOK));
1873}
1874
1875static void
1876zone_lock_pair(uma_zone_t a, uma_zone_t b)
1877{
1878 if (a < b) {
1879 ZONE_LOCK(a);
1880 mtx_lock_flags(b->uz_lock, MTX_DUPOK);
1881 } else {
1882 ZONE_LOCK(b);
--- 75 unchanged lines hidden (view full) ---
1958}
1959
1960
1961/* See uma.h */
1962void
1963uma_zdestroy(uma_zone_t zone)
1964{
1965
1966 zone_free_item(zones, zone, NULL, SKIP_NONE);
1967}
1968
1969/* See uma.h */
1970void *
1971uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1972{
1973 void *item;
1974 uma_cache_t cache;
--- 62 unchanged lines hidden (view full) ---
2037#endif
2038 KASSERT(item != NULL,
2039 ("uma_zalloc: Bucket pointer mangled."));
2040 cache->uc_allocs++;
2041 critical_exit();
2042 if (zone->uz_ctor != NULL) {
2043 if (zone->uz_ctor(item, zone->uz_size,
2044 udata, flags) != 0) {
2045 atomic_add_long(&zone->uz_fails, 1);
2046 zone_free_item(zone, item, udata,
2047 SKIP_DTOR);
2048 return (NULL);
2049 }
2050 }
2051#ifdef INVARIANTS
2052 uma_dbg_alloc(zone, NULL, item);
2053#endif
2054 if (flags & M_ZERO)
2055 bzero(item, zone->uz_size);
--- 39 unchanged lines hidden (view full) ---
2095 bucket = cache->uc_freebucket;
2096 if (bucket != NULL && bucket->ub_cnt > 0) {
2097 ZONE_UNLOCK(zone);
2098 goto zalloc_start;
2099 }
2100 }
2101
2102 /* Since we have locked the zone we may as well send back our stats */
2103 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2104 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2105 cache->uc_allocs = 0;
2106 cache->uc_frees = 0;
2107
2108 /* Our old one is now a free bucket */
2109 if (cache->uc_allocbucket) {
2110 KASSERT(cache->uc_allocbucket->ub_cnt == 0,
2111 ("uma_zalloc_arg: Freeing a non free bucket."));
2112 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2113 cache->uc_allocbucket, ub_link);
--- 231 unchanged lines hidden (view full) ---
2345 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2346 continue;
2347 }
2348 }
2349 return (NULL);
2350}
2351
2352static void *
2353slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2354{
2355 void *item;
2356 uint8_t freei;
2357
2358 MPASS(keg == slab->us_keg);
2359 mtx_assert(&keg->uk_lock, MA_OWNED);
2360
2361 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2362 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2363 item = slab->us_data + (keg->uk_rsize * freei);
2364 slab->us_freecount--;
2365 keg->uk_free--;
2366
2367 /* Move this slab to the full list */
2368 if (slab->us_freecount == 0) {
2369 LIST_REMOVE(slab, us_link);
2370 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2371 }
2372
2373 return (item);
2374}
2375
2376static int
2377zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2378{
2379 uma_slab_t slab;
2380 uma_keg_t keg;
2381 int i;
2382
2383 ZONE_LOCK(zone);
2384 /* Try to keep the buckets totally full */
2385 slab = NULL;
2386 keg = NULL;
2387 for (i = 0; i < max; ) {
2388 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2389 break;
2390 keg = slab->us_keg;
2391 while (slab->us_freecount && i < max)
2392 bucket[i++] = slab_alloc_item(keg, slab);
2393
2394 /* Don't block on the next fill */
2395 flags &= ~M_WAITOK;
2396 flags |= M_NOWAIT;
2397 }
2398 if (slab != NULL)
2399 KEG_UNLOCK(keg);
2400 else
2401 ZONE_UNLOCK(zone);
2402
2403 return i;
2404}
2405
2406static int
2407zone_alloc_bucket(uma_zone_t zone, int flags)
2408{
2409 uma_bucket_t bucket;
2410 int bflags;
2411 int max;
2412
2413#ifdef SMP
2414 /*
2415 * This code is here to limit the number of simultaneous bucket fills
2416 * for any given zone to the number of per cpu caches in this zone. This
2417 * is done so that we don't allocate more memory than we really need.
2418 */
2419 if (zone->uz_fills >= mp_ncpus)
2420 return (0);
2421
2422#endif
2423 zone->uz_fills++;
2424 max = zone->uz_count;
2425
2426 /*
2427 * Try this zone's free list first so we don't allocate extra buckets.
2428 */
2429 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2430 KASSERT(bucket->ub_cnt == 0,
2431 ("zone_alloc_bucket: Bucket on free list is not empty."));
2432 LIST_REMOVE(bucket, ub_link);
2433 ZONE_UNLOCK(zone);
2434 } else {
2435 bflags = (flags & ~M_ZERO);
2436 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2437 bflags |= M_NOVM;
2438 ZONE_UNLOCK(zone);
2439 bucket = bucket_alloc(zone->uz_count, bflags);
2440 if (bucket == NULL)
2441 goto out;
2442 }
2443
2444 max = MIN(bucket->ub_entries, max);
2445 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2446 max, flags);
2447
2448 /*
2449 * Initialize the memory if necessary.
2450 */
2451 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2452 int i;
2453
2454 for (i = 0; i < bucket->ub_cnt; i++)
2455 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
2456 flags) != 0)
2457 break;
2458 /*
2459 * If we couldn't initialize the whole bucket, put the
2460 * rest back onto the freelist.
2461 */
2462 if (i != bucket->ub_cnt) {
2463 zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
2464 bucket->ub_cnt - i);
2465#ifdef INVARIANTS
2466 bzero(&bucket->ub_bucket[i],
2467 sizeof(void *) * (bucket->ub_cnt - i));
2468#endif
2469 bucket->ub_cnt = i;
2470 }
2471 }
2472
2473out:
2474 ZONE_LOCK(zone);
2475 zone->uz_fills--;
2476 if (bucket != NULL && bucket->ub_cnt != 0) {
2477 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2478 bucket, ub_link);
2479 return (1);
2480 }
2481 atomic_add_long(&zone->uz_fails, 1);
2482 if (bucket != NULL)
2483 bucket_free(bucket);
2484
2485 return (0);
2486}
2487/*
2488 * Allocates a single item from a zone.
2489 *
2490 * Arguments
2491 * zone The zone to alloc for.
2492 * udata The data to be passed to the constructor.
2493 * flags M_WAITOK, M_NOWAIT, M_ZERO.
2494 *
2495 * Returns
2496 * NULL if there is no memory and M_NOWAIT is set
2497 * An item if successful
2498 */
2499
2500static void *
2501zone_alloc_item(uma_zone_t zone, void *udata, int flags)
2502{
2503 void *item;
2504
2505 item = NULL;
2506
2507#ifdef UMA_DEBUG_ALLOC
2508 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2509#endif
2510 if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
2511 goto fail;
2512 atomic_add_long(&zone->uz_allocs, 1);
2513
2514 /*
2515 * We have to call both the zone's init (not the keg's init)
2516 * and the zone's ctor. This is because the item is going from
2517 * a keg slab directly to the user, and the user is expecting it
2518 * to be both zone-init'd as well as zone-ctor'd.
2519 */
2520 if (zone->uz_init != NULL) {
2521 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
2522 zone_free_item(zone, item, udata, SKIP_FINI);
2523 goto fail;
2524 }
2525 }
2526 if (zone->uz_ctor != NULL) {
2527 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2528 zone_free_item(zone, item, udata, SKIP_DTOR);
2529 goto fail;
2530 }
2531 }
2532#ifdef INVARIANTS
2533 uma_dbg_alloc(zone, NULL, item);
2534#endif
2535 if (flags & M_ZERO)
2536 bzero(item, zone->uz_size);
2537
2538 return (item);
2539
2540fail:
2541 atomic_add_long(&zone->uz_fails, 1);
2542 return (NULL);
2543}
2544
2545/* See uma.h */
2546void
2547uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2548{
2549 uma_cache_t cache;
2550 uma_bucket_t bucket;
--- 113 unchanged lines hidden (view full) ---
2664 (cache->uc_allocbucket->ub_cnt <
2665 cache->uc_freebucket->ub_cnt)) {
2666 ZONE_UNLOCK(zone);
2667 goto zfree_start;
2668 }
2669 }
2670
2671 /* Since we have locked the zone we may as well send back our stats */
2672 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2673 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2674 cache->uc_allocs = 0;
2675 cache->uc_frees = 0;
2676
2677 bucket = cache->uc_freebucket;
2678 cache->uc_freebucket = NULL;
2679
2680 /* Can we throw this on the zone full list? */
2681 if (bucket != NULL) {
2682#ifdef UMA_DEBUG_ALLOC
--- 32 unchanged lines hidden (view full) ---
2715 ZONE_UNLOCK(zone);
2716 goto zfree_restart;
2717 }
2718
2719 /*
2720 * If nothing else caught this, we'll just do an internal free.
2721 */
2722zfree_internal:
2723 zone_free_item(zone, item, udata, SKIP_DTOR);
2724
2725 return;
2726}
2727
2728static void
2729slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
2730{
2731 uint8_t freei;
2732
2733 mtx_assert(&keg->uk_lock, MA_OWNED);
2734 MPASS(keg == slab->us_keg);
2735
2736 /* Do we need to remove from any lists? */
2737 if (slab->us_freecount+1 == keg->uk_ipers) {
2738 LIST_REMOVE(slab, us_link);
2739 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2740 } else if (slab->us_freecount == 0) {
2741 LIST_REMOVE(slab, us_link);
2742 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2743 }
2744
2745 /* Slab management. */
2746 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2747 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
2748 slab->us_freecount++;
2749
2750 /* Keg statistics. */
2751 keg->uk_free++;
2752}
2753
2754static void
2755zone_release(uma_zone_t zone, void **bucket, int cnt)
2756{
2757 void *item;
2758 uma_slab_t slab;
2759 uma_keg_t keg;
2760 uint8_t *mem;
2761 int clearfull;
2762 int i;
2763
2764 clearfull = 0;
2765 ZONE_LOCK(zone);
2766 keg = zone_first_keg(zone);
2767 for (i = 0; i < cnt; i++) {
2768 item = bucket[i];
2769 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2770 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2771 if (zone->uz_flags & UMA_ZONE_HASH) {
2772 slab = hash_sfind(&keg->uk_hash, mem);
2773 } else {
2774 mem += keg->uk_pgoff;
2775 slab = (uma_slab_t)mem;
2776 }
2777 } else {
2778 slab = vtoslab((vm_offset_t)item);
2779 if (slab->us_keg != keg) {
2780 KEG_UNLOCK(keg);
2781 keg = slab->us_keg;
2782 KEG_LOCK(keg);
2783 }
2784 }
2785 slab_free_item(keg, slab, item);
2786 if (keg->uk_flags & UMA_ZFLAG_FULL) {
2787 if (keg->uk_pages < keg->uk_maxpages) {
2788 keg->uk_flags &= ~UMA_ZFLAG_FULL;
2789 clearfull = 1;
2790 }
2791
2792 /*
2793 * We can handle one more allocation. Since we're
2794 * clearing ZFLAG_FULL, wake up all procs blocked
2795 * on pages. This should be uncommon, so keeping this
2796 * simple for now (rather than adding count of blocked
2797 * threads etc).
2798 */
2799 wakeup(keg);
2800 }
2801 }
2802 zone_relock(zone, keg);
2803 if (clearfull) {
2804 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2805 wakeup(zone);
2806 }
2807 ZONE_UNLOCK(zone);
2808
2809}
2810
2811/*
2812 * Frees a single item to any zone.
2813 *
2814 * Arguments:
2815 * zone The zone to free to
2816 * item The item we're freeing
2817 * udata User supplied data for the dtor
2818 * skip Skip dtors and finis
2819 */
2820static void
2821zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
2822{
2823
2824#ifdef INVARIANTS
2825 if (skip == SKIP_NONE) {
2826 if (zone->uz_flags & UMA_ZONE_MALLOC)
2827 uma_dbg_free(zone, udata, item);
2828 else
2829 uma_dbg_free(zone, NULL, item);
2830 }
2831#endif
2832 if (skip < SKIP_DTOR && zone->uz_dtor)
2833 zone->uz_dtor(item, zone->uz_size, udata);
2834
2835 if (skip < SKIP_FINI && zone->uz_fini)
2836 zone->uz_fini(item, zone->uz_size);
2837
2838 atomic_add_long(&zone->uz_frees, 1);
2839 zone->uz_release(zone->uz_arg, &item, 1);
2840}
2841
2842/* See uma.h */
2843int
2844uma_zone_set_max(uma_zone_t zone, int nitems)
2845{
2846 uma_keg_t keg;
2847
2848 keg = zone_first_keg(zone);
2849 if (keg == NULL)
2850 return (0);
2851 ZONE_LOCK(zone);
2852 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2853 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2854 keg->uk_maxpages += keg->uk_ppera;
2855 nitems = keg->uk_maxpages * keg->uk_ipers;
2856 ZONE_UNLOCK(zone);
2857
2858 return (nitems);
2859}
2860
2861/* See uma.h */
2862int
2863uma_zone_get_max(uma_zone_t zone)
2864{
2865 int nitems;
2866 uma_keg_t keg;
2867
2868 keg = zone_first_keg(zone);
2869 if (keg == NULL)
2870 return (0);
2871 ZONE_LOCK(zone);
2872 nitems = keg->uk_maxpages * keg->uk_ipers;
2873 ZONE_UNLOCK(zone);
2874
2875 return (nitems);
2876}
2877
2878/* See uma.h */
2879void
--- 31 unchanged lines hidden (view full) ---
2911/* See uma.h */
2912void
2913uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2914{
2915 uma_keg_t keg;
2916
2917 ZONE_LOCK(zone);
2918 keg = zone_first_keg(zone);
2919 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2920 KASSERT(keg->uk_pages == 0,
2921 ("uma_zone_set_init on non-empty keg"));
2922 keg->uk_init = uminit;
2923 ZONE_UNLOCK(zone);
2924}
2925
2926/* See uma.h */
2927void
2928uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2929{
2930 uma_keg_t keg;
2931
2932 ZONE_LOCK(zone);
2933 keg = zone_first_keg(zone);
2934 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2935 KASSERT(keg->uk_pages == 0,
2936 ("uma_zone_set_fini on non-empty keg"));
2937 keg->uk_fini = fini;
2938 ZONE_UNLOCK(zone);
2939}
2940
2941/* See uma.h */
2942void
--- 17 unchanged lines hidden (view full) ---
2960 ZONE_UNLOCK(zone);
2961}
2962
2963/* See uma.h */
2964/* XXX uk_freef is not actually used with the zone locked */
2965void
2966uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2967{
2968 uma_keg_t keg;
2969
2970 ZONE_LOCK(zone);
2971 keg = zone_first_keg(zone);
2972 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2973 keg->uk_freef = freef;
2974 ZONE_UNLOCK(zone);
2975}
2976
2977/* See uma.h */
2978/* XXX uk_allocf is not actually used with the zone locked */
2979void
2980uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2981{
--- 10 unchanged lines hidden (view full) ---
2992int
2993uma_zone_reserve_kva(uma_zone_t zone, int count)
2994{
2995 uma_keg_t keg;
2996 vm_offset_t kva;
2997 int pages;
2998
2999 keg = zone_first_keg(zone);
3000 if (keg == NULL)
3001 return (0);
3002 pages = count / keg->uk_ipers;
3003
3004 if (pages * keg->uk_ipers < count)
3005 pages++;
3006
3007#ifdef UMA_MD_SMALL_ALLOC
3008 if (keg->uk_ppera > 1) {
3009#else
--- 22 unchanged lines hidden (view full) ---
3032void
3033uma_prealloc(uma_zone_t zone, int items)
3034{
3035 int slabs;
3036 uma_slab_t slab;
3037 uma_keg_t keg;
3038
3039 keg = zone_first_keg(zone);
3040 if (keg == NULL)
3041 return;
3042 ZONE_LOCK(zone);
3043 slabs = items / keg->uk_ipers;
3044 if (slabs * keg->uk_ipers < items)
3045 slabs++;
3046 while (slabs > 0) {
3047 slab = keg_alloc_slab(keg, zone, M_WAITOK);
3048 if (slab == NULL)
3049 break;
--- 73 unchanged lines hidden (view full) ---
3123 return (NULL);
3124 mem = page_alloc(NULL, size, &flags, wait);
3125 if (mem) {
3126 vsetslab((vm_offset_t)mem, slab);
3127 slab->us_data = mem;
3128 slab->us_flags = flags | UMA_SLAB_MALLOC;
3129 slab->us_size = size;
3130 } else {
3131 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3132 }
3133
3134 return (mem);
3135}
3136
3137void
3138uma_large_free(uma_slab_t slab)
3139{
3140 vsetobj((vm_offset_t)slab->us_data, kmem_object);
3141 page_free(slab->us_data, slab->us_size, slab->us_flags);
3142 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3143}
3144
3145void
3146uma_print_stats(void)
3147{
3148 zone_foreach(uma_print_zone);
3149}
3150
--- 258 unchanged lines hidden ---
2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 66 unchanged lines hidden (view full) ---
126 * Are we allowed to allocate buckets?
127 */
128static int bucketdisable = 1;
129
130/* Linked list of all kegs in the system */
131static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
132
133/* This mutex protects the keg list */
134static struct mtx_padalign uma_mtx;
135
136/* Linked list of boot time pages */
137static LIST_HEAD(,uma_slab) uma_boot_pages =
138 LIST_HEAD_INITIALIZER(uma_boot_pages);
139
140/* This mutex protects the boot time pages list */
141static struct mtx_padalign uma_boot_pages_mtx;
142
143/* Is the VM done starting up? */
144static int booted = 0;
145#define UMA_STARTUP 1
146#define UMA_STARTUP2 2
147
148/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */
149static const u_int uma_max_ipers = SLAB_SETSIZE;
--- 17 unchanged lines hidden (view full) ---
167 */
168struct uma_zctor_args {
169 const char *name;
170 size_t size;
171 uma_ctor ctor;
172 uma_dtor dtor;
173 uma_init uminit;
174 uma_fini fini;
175 uma_import import;
176 uma_release release;
177 void *arg;
178 uma_keg_t keg;
179 int align;
180 uint32_t flags;
181};
182
183struct uma_kctor_args {
184 uma_zone_t zone;
185 size_t size;
--- 28 unchanged lines hidden (view full) ---
214 */
215static uint8_t bucket_size[BUCKET_ZONES];
216
217/*
218 * Flags and enumerations to be passed to internal functions.
219 */
220enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
221
222/* Prototypes.. */
223
224static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
225static void *page_alloc(uma_zone_t, int, uint8_t *, int);
226static void *startup_alloc(uma_zone_t, int, uint8_t *, int);
227static void page_free(void *, int, uint8_t);
228static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int);
229static void cache_drain(uma_zone_t);
--- 9 unchanged lines hidden (view full) ---
239static void zone_foreach(void (*zfunc)(uma_zone_t));
240static void zone_timeout(uma_zone_t zone);
241static int hash_alloc(struct uma_hash *);
242static int hash_expand(struct uma_hash *, struct uma_hash *);
243static void hash_free(struct uma_hash *hash);
244static void uma_timeout(void *);
245static void uma_startup3(void);
246static void *zone_alloc_item(uma_zone_t, void *, int);
247static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
248static void bucket_enable(void);
249static void bucket_init(void);
250static uma_bucket_t bucket_alloc(int, int);
251static void bucket_free(uma_bucket_t);
252static void bucket_zone_drain(void);
253static int zone_alloc_bucket(uma_zone_t zone, int flags);
254static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
255static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags);
256static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
257static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
258static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
259 uma_fini fini, int align, uint32_t flags);
260static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
261static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
262static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
263static void zone_release(uma_zone_t zone, void **bucket, int cnt);
264
265void uma_print_zone(uma_zone_t);
266void uma_print_stats(void);
267static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS);
268static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS);
269
270SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL);
271
--- 88 unchanged lines hidden (view full) ---
360}
361
362static void
363bucket_free(uma_bucket_t bucket)
364{
365 struct uma_bucket_zone *ubz;
366
367 ubz = bucket_zone_lookup(bucket->ub_entries);
368 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
369}
370
371static void
372bucket_zone_drain(void)
373{
374 struct uma_bucket_zone *ubz;
375
376 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++)
--- 7 unchanged lines hidden (view full) ---
384
385 if (!zone_warnings || zone->uz_warning == NULL)
386 return;
387
388 if (ratecheck(&zone->uz_ratecheck, &warninterval))
389 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
390}
391
392static void
393zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
394{
395 uma_klink_t klink;
396
397 LIST_FOREACH(klink, &zone->uz_kegs, kl_link)
398 kegfn(klink->kl_keg);
399}
--- 168 unchanged lines hidden (view full) ---
568 * Nothing
569 */
570static void
571hash_free(struct uma_hash *hash)
572{
573 if (hash->uh_slab_hash == NULL)
574 return;
575 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
576 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
577 else
578 free(hash->uh_slab_hash, M_UMAHASH);
579}
580
581/*
582 * Frees all outstanding items in a bucket
583 *
584 * Arguments:
585 * zone The zone to free to, must be unlocked.
586 * bucket The free/alloc bucket with items, cpu queue must be locked.
587 *
588 * Returns:
589 * Nothing
590 */
591
592static void
593bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
594{
595 int i;
596
597 if (bucket == NULL)
598 return;
599
600 if (zone->uz_fini)
601 for (i = 0; i < bucket->ub_cnt; i++)
602 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
603 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
604 bucket->ub_cnt = 0;
605}
606
607/*
608 * Drains the per cpu caches for a zone.
609 *
610 * NOTE: This may only be called while the zone is being turn down, and not
611 * during normal operation. This is necessary in order that we do not have
612 * to migrate CPUs to drain the per-CPU caches.
--- 137 unchanged lines hidden (view full) ---
750 obj = kernel_object;
751 else
752 obj = NULL;
753 for (i = 0; i < keg->uk_ppera; i++)
754 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
755 obj);
756 }
757 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
758 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
759#ifdef UMA_DEBUG
760 printf("%s: Returning %d bytes.\n",
761 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
762#endif
763 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
764 }
765}
766
--- 57 unchanged lines hidden (view full) ---
824 uint8_t *mem;
825 uint8_t flags;
826 int i;
827
828 mtx_assert(&keg->uk_lock, MA_OWNED);
829 slab = NULL;
830
831#ifdef UMA_DEBUG
832 printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
833#endif
834 allocf = keg->uk_allocf;
835 KEG_UNLOCK(keg);
836
837 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
838 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait);
839 if (slab == NULL) {
840 KEG_LOCK(keg);
--- 15 unchanged lines hidden (view full) ---
856
857 if (keg->uk_flags & UMA_ZONE_NODUMP)
858 wait |= M_NODUMP;
859
860 /* zone is passed for legacy reasons. */
861 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
862 if (mem == NULL) {
863 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
864 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
865 KEG_LOCK(keg);
866 return (NULL);
867 }
868
869 /* Point the slab into the allocated memory */
870 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE))
871 slab = (uma_slab_t )(mem + keg->uk_pgoff);
872
--- 37 unchanged lines hidden (view full) ---
910 else
911 obj = NULL;
912 for (i = 0; i < keg->uk_ppera; i++)
913 vsetobj((vm_offset_t)mem +
914 (i * PAGE_SIZE), obj);
915 }
916 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
917 zone_free_item(keg->uk_slabzone, slab,
918 NULL, SKIP_NONE);
919 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
920 flags);
921 KEG_LOCK(keg);
922 return (NULL);
923 }
924 }
925 KEG_LOCK(keg);
926
--- 537 unchanged lines hidden (view full) ---
1464 zone->uz_fails = 0;
1465 zone->uz_sleeps = 0;
1466 zone->uz_fills = zone->uz_count = 0;
1467 zone->uz_flags = 0;
1468 zone->uz_warning = NULL;
1469 timevalclear(&zone->uz_ratecheck);
1470 keg = arg->keg;
1471
1472 /*
1473 * This is a pure cache zone, no kegs.
1474 */
1475 if (arg->import) {
1476 zone->uz_import = arg->import;
1477 zone->uz_release = arg->release;
1478 zone->uz_arg = arg->arg;
1479 zone->uz_count = BUCKET_MAX;
1480 return (0);
1481 }
1482
1483 /*
1484 * Use the regular zone/keg/slab allocator.
1485 */
1486 zone->uz_import = (uma_import)zone_import;
1487 zone->uz_release = (uma_release)zone_release;
1488 zone->uz_arg = zone;
1489
1490 if (arg->flags & UMA_ZONE_SECONDARY) {
1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
1492 zone->uz_init = arg->uminit;
1493 zone->uz_fini = arg->fini;
1494 zone->uz_lock = &keg->uk_lock;
1495 zone->uz_flags |= UMA_ZONE_SECONDARY;
1496 mtx_lock(&uma_mtx);
1497 ZONE_LOCK(zone);
--- 20 unchanged lines hidden (view full) ---
1518 karg.align = arg->align;
1519 karg.flags = arg->flags;
1520 karg.zone = zone;
1521 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
1522 flags);
1523 if (error)
1524 return (error);
1525 }
1526
1527 /*
1528 * Link in the first keg.
1529 */
1530 zone->uz_klink.kl_keg = keg;
1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link);
1532 zone->uz_lock = &keg->uk_lock;
1533 zone->uz_size = keg->uk_size;
1534 zone->uz_flags |= (keg->uk_flags &
--- 81 unchanged lines hidden (view full) ---
1616 LIST_REMOVE(klink, kl_link);
1617 if (klink == &zone->uz_klink)
1618 continue;
1619 free(klink, M_TEMP);
1620 }
1621 /*
1622 * We only destroy kegs from non secondary zones.
1623 */
1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
1625 mtx_lock(&uma_mtx);
1626 LIST_REMOVE(keg, uk_link);
1627 mtx_unlock(&uma_mtx);
1628 zone_free_item(kegs, keg, NULL, SKIP_NONE);
1629 }
1630}
1631
1632/*
1633 * Traverses every zone in the system and calls a callback
1634 *
1635 * Arguments:
1636 * zfunc A pointer to a function which accepts a zone
--- 27 unchanged lines hidden (view full) ---
1664 int i;
1665
1666#ifdef UMA_DEBUG
1667 printf("Creating uma keg headers zone and keg.\n");
1668#endif
1669 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1670
1671 /* "manually" create the initial zone */
1672 memset(&args, 0, sizeof(args));
1673 args.name = "UMA Kegs";
1674 args.size = sizeof(struct uma_keg);
1675 args.ctor = keg_ctor;
1676 args.dtor = keg_dtor;
1677 args.uminit = zero_init;
1678 args.fini = NULL;
1679 args.keg = &masterkeg;
1680 args.align = 32 - 1;
--- 124 unchanged lines hidden (view full) ---
1805uma_zone_t
1806uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor,
1807 uma_init uminit, uma_fini fini, int align, uint32_t flags)
1808
1809{
1810 struct uma_zctor_args args;
1811
1812 /* This stuff is essential for the zone ctor */
1813 memset(&args, 0, sizeof(args));
1814 args.name = name;
1815 args.size = size;
1816 args.ctor = ctor;
1817 args.dtor = dtor;
1818 args.uminit = uminit;
1819 args.fini = fini;
1820 args.align = align;
1821 args.flags = flags;
--- 6 unchanged lines hidden (view full) ---
1828uma_zone_t
1829uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor,
1830 uma_init zinit, uma_fini zfini, uma_zone_t master)
1831{
1832 struct uma_zctor_args args;
1833 uma_keg_t keg;
1834
1835 keg = zone_first_keg(master);
1836 memset(&args, 0, sizeof(args));
1837 args.name = name;
1838 args.size = keg->uk_size;
1839 args.ctor = ctor;
1840 args.dtor = dtor;
1841 args.uminit = zinit;
1842 args.fini = zfini;
1843 args.align = keg->uk_align;
1844 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
1845 args.keg = keg;
1846
1847 /* XXX Attaches only one keg of potentially many. */
1848 return (zone_alloc_item(zones, &args, M_WAITOK));
1849}
1850
1851/* See uma.h */
1852uma_zone_t
1853uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
1854 uma_fini zfini, uma_import zimport, uma_release zrelease,
1855 void *arg, int flags)
1856{
1857 struct uma_zctor_args args;
1858
1859 memset(&args, 0, sizeof(args));
1860 args.name = name;
1861 args.size = 0;
1862 args.ctor = ctor;
1863 args.dtor = dtor;
1864 args.uminit = zinit;
1865 args.fini = zfini;
1866 args.import = zimport;
1867 args.release = zrelease;
1868 args.arg = arg;
1869 args.align = 0;
1870 args.flags = flags;
1871
1872 return (zone_alloc_item(zones, &args, M_WAITOK));
1873}
1874
1875static void
1876zone_lock_pair(uma_zone_t a, uma_zone_t b)
1877{
1878 if (a < b) {
1879 ZONE_LOCK(a);
1880 mtx_lock_flags(b->uz_lock, MTX_DUPOK);
1881 } else {
1882 ZONE_LOCK(b);
--- 75 unchanged lines hidden (view full) ---
1958}
1959
1960
1961/* See uma.h */
1962void
1963uma_zdestroy(uma_zone_t zone)
1964{
1965
1966 zone_free_item(zones, zone, NULL, SKIP_NONE);
1967}
1968
1969/* See uma.h */
1970void *
1971uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
1972{
1973 void *item;
1974 uma_cache_t cache;
--- 62 unchanged lines hidden (view full) ---
2037#endif
2038 KASSERT(item != NULL,
2039 ("uma_zalloc: Bucket pointer mangled."));
2040 cache->uc_allocs++;
2041 critical_exit();
2042 if (zone->uz_ctor != NULL) {
2043 if (zone->uz_ctor(item, zone->uz_size,
2044 udata, flags) != 0) {
2045 atomic_add_long(&zone->uz_fails, 1);
2046 zone_free_item(zone, item, udata,
2047 SKIP_DTOR);
2048 return (NULL);
2049 }
2050 }
2051#ifdef INVARIANTS
2052 uma_dbg_alloc(zone, NULL, item);
2053#endif
2054 if (flags & M_ZERO)
2055 bzero(item, zone->uz_size);
--- 39 unchanged lines hidden (view full) ---
2095 bucket = cache->uc_freebucket;
2096 if (bucket != NULL && bucket->ub_cnt > 0) {
2097 ZONE_UNLOCK(zone);
2098 goto zalloc_start;
2099 }
2100 }
2101
2102 /* Since we have locked the zone we may as well send back our stats */
2103 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2104 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2105 cache->uc_allocs = 0;
2106 cache->uc_frees = 0;
2107
2108 /* Our old one is now a free bucket */
2109 if (cache->uc_allocbucket) {
2110 KASSERT(cache->uc_allocbucket->ub_cnt == 0,
2111 ("uma_zalloc_arg: Freeing a non free bucket."));
2112 LIST_INSERT_HEAD(&zone->uz_free_bucket,
2113 cache->uc_allocbucket, ub_link);
--- 231 unchanged lines hidden (view full) ---
2345 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2346 continue;
2347 }
2348 }
2349 return (NULL);
2350}
2351
2352static void *
2353slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
2354{
2355 void *item;
2356 uint8_t freei;
2357
2358 MPASS(keg == slab->us_keg);
2359 mtx_assert(&keg->uk_lock, MA_OWNED);
2360
2361 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
2362 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free);
2363 item = slab->us_data + (keg->uk_rsize * freei);
2364 slab->us_freecount--;
2365 keg->uk_free--;
2366
2367 /* Move this slab to the full list */
2368 if (slab->us_freecount == 0) {
2369 LIST_REMOVE(slab, us_link);
2370 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link);
2371 }
2372
2373 return (item);
2374}
2375
2376static int
2377zone_import(uma_zone_t zone, void **bucket, int max, int flags)
2378{
2379 uma_slab_t slab;
2380 uma_keg_t keg;
2381 int i;
2382
2383 ZONE_LOCK(zone);
2384 /* Try to keep the buckets totally full */
2385 slab = NULL;
2386 keg = NULL;
2387 for (i = 0; i < max; ) {
2388 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
2389 break;
2390 keg = slab->us_keg;
2391 while (slab->us_freecount && i < max)
2392 bucket[i++] = slab_alloc_item(keg, slab);
2393
2394 /* Don't block on the next fill */
2395 flags &= ~M_WAITOK;
2396 flags |= M_NOWAIT;
2397 }
2398 if (slab != NULL)
2399 KEG_UNLOCK(keg);
2400 else
2401 ZONE_UNLOCK(zone);
2402
2403 return i;
2404}
2405
2406static int
2407zone_alloc_bucket(uma_zone_t zone, int flags)
2408{
2409 uma_bucket_t bucket;
2410 int bflags;
2411 int max;
2412
2413#ifdef SMP
2414 /*
2415 * This code is here to limit the number of simultaneous bucket fills
2416 * for any given zone to the number of per cpu caches in this zone. This
2417 * is done so that we don't allocate more memory than we really need.
2418 */
2419 if (zone->uz_fills >= mp_ncpus)
2420 return (0);
2421
2422#endif
2423 zone->uz_fills++;
2424 max = zone->uz_count;
2425
2426 /*
2427 * Try this zone's free list first so we don't allocate extra buckets.
2428 */
2429 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
2430 KASSERT(bucket->ub_cnt == 0,
2431 ("zone_alloc_bucket: Bucket on free list is not empty."));
2432 LIST_REMOVE(bucket, ub_link);
2433 ZONE_UNLOCK(zone);
2434 } else {
2435 bflags = (flags & ~M_ZERO);
2436 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
2437 bflags |= M_NOVM;
2438 ZONE_UNLOCK(zone);
2439 bucket = bucket_alloc(zone->uz_count, bflags);
2440 if (bucket == NULL)
2441 goto out;
2442 }
2443
2444 max = MIN(bucket->ub_entries, max);
2445 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
2446 max, flags);
2447
2448 /*
2449 * Initialize the memory if necessary.
2450 */
2451 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
2452 int i;
2453
2454 for (i = 0; i < bucket->ub_cnt; i++)
2455 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
2456 flags) != 0)
2457 break;
2458 /*
2459 * If we couldn't initialize the whole bucket, put the
2460 * rest back onto the freelist.
2461 */
2462 if (i != bucket->ub_cnt) {
2463 zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
2464 bucket->ub_cnt - i);
2465#ifdef INVARIANTS
2466 bzero(&bucket->ub_bucket[i],
2467 sizeof(void *) * (bucket->ub_cnt - i));
2468#endif
2469 bucket->ub_cnt = i;
2470 }
2471 }
2472
2473out:
2474 ZONE_LOCK(zone);
2475 zone->uz_fills--;
2476 if (bucket != NULL && bucket->ub_cnt != 0) {
2477 LIST_INSERT_HEAD(&zone->uz_full_bucket,
2478 bucket, ub_link);
2479 return (1);
2480 }
2481 atomic_add_long(&zone->uz_fails, 1);
2482 if (bucket != NULL)
2483 bucket_free(bucket);
2484
2485 return (0);
2486}
2487/*
2488 * Allocates a single item from a zone.
2489 *
2490 * Arguments
2491 * zone The zone to alloc for.
2492 * udata The data to be passed to the constructor.
2493 * flags M_WAITOK, M_NOWAIT, M_ZERO.
2494 *
2495 * Returns
2496 * NULL if there is no memory and M_NOWAIT is set
2497 * An item if successful
2498 */
2499
2500static void *
2501zone_alloc_item(uma_zone_t zone, void *udata, int flags)
2502{
2503 void *item;
2504
2505 item = NULL;
2506
2507#ifdef UMA_DEBUG_ALLOC
2508 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone);
2509#endif
2510 if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
2511 goto fail;
2512 atomic_add_long(&zone->uz_allocs, 1);
2513
2514 /*
2515 * We have to call both the zone's init (not the keg's init)
2516 * and the zone's ctor. This is because the item is going from
2517 * a keg slab directly to the user, and the user is expecting it
2518 * to be both zone-init'd as well as zone-ctor'd.
2519 */
2520 if (zone->uz_init != NULL) {
2521 if (zone->uz_init(item, zone->uz_size, flags) != 0) {
2522 zone_free_item(zone, item, udata, SKIP_FINI);
2523 goto fail;
2524 }
2525 }
2526 if (zone->uz_ctor != NULL) {
2527 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
2528 zone_free_item(zone, item, udata, SKIP_DTOR);
2529 goto fail;
2530 }
2531 }
2532#ifdef INVARIANTS
2533 uma_dbg_alloc(zone, NULL, item);
2534#endif
2535 if (flags & M_ZERO)
2536 bzero(item, zone->uz_size);
2537
2538 return (item);
2539
2540fail:
2541 atomic_add_long(&zone->uz_fails, 1);
2542 return (NULL);
2543}
2544
2545/* See uma.h */
2546void
2547uma_zfree_arg(uma_zone_t zone, void *item, void *udata)
2548{
2549 uma_cache_t cache;
2550 uma_bucket_t bucket;
--- 113 unchanged lines hidden (view full) ---
2664 (cache->uc_allocbucket->ub_cnt <
2665 cache->uc_freebucket->ub_cnt)) {
2666 ZONE_UNLOCK(zone);
2667 goto zfree_start;
2668 }
2669 }
2670
2671 /* Since we have locked the zone we may as well send back our stats */
2672 atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
2673 atomic_add_long(&zone->uz_frees, cache->uc_frees);
2674 cache->uc_allocs = 0;
2675 cache->uc_frees = 0;
2676
2677 bucket = cache->uc_freebucket;
2678 cache->uc_freebucket = NULL;
2679
2680 /* Can we throw this on the zone full list? */
2681 if (bucket != NULL) {
2682#ifdef UMA_DEBUG_ALLOC
--- 32 unchanged lines hidden (view full) ---
2715 ZONE_UNLOCK(zone);
2716 goto zfree_restart;
2717 }
2718
2719 /*
2720 * If nothing else caught this, we'll just do an internal free.
2721 */
2722zfree_internal:
2723 zone_free_item(zone, item, udata, SKIP_DTOR);
2724
2725 return;
2726}
2727
2728static void
2729slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
2730{
2731 uint8_t freei;
2732
2733 mtx_assert(&keg->uk_lock, MA_OWNED);
2734 MPASS(keg == slab->us_keg);
2735
2736 /* Do we need to remove from any lists? */
2737 if (slab->us_freecount+1 == keg->uk_ipers) {
2738 LIST_REMOVE(slab, us_link);
2739 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link);
2740 } else if (slab->us_freecount == 0) {
2741 LIST_REMOVE(slab, us_link);
2742 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link);
2743 }
2744
2745 /* Slab management. */
2746 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize;
2747 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free);
2748 slab->us_freecount++;
2749
2750 /* Keg statistics. */
2751 keg->uk_free++;
2752}
2753
2754static void
2755zone_release(uma_zone_t zone, void **bucket, int cnt)
2756{
2757 void *item;
2758 uma_slab_t slab;
2759 uma_keg_t keg;
2760 uint8_t *mem;
2761 int clearfull;
2762 int i;
2763
2764 clearfull = 0;
2765 ZONE_LOCK(zone);
2766 keg = zone_first_keg(zone);
2767 for (i = 0; i < cnt; i++) {
2768 item = bucket[i];
2769 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
2770 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
2771 if (zone->uz_flags & UMA_ZONE_HASH) {
2772 slab = hash_sfind(&keg->uk_hash, mem);
2773 } else {
2774 mem += keg->uk_pgoff;
2775 slab = (uma_slab_t)mem;
2776 }
2777 } else {
2778 slab = vtoslab((vm_offset_t)item);
2779 if (slab->us_keg != keg) {
2780 KEG_UNLOCK(keg);
2781 keg = slab->us_keg;
2782 KEG_LOCK(keg);
2783 }
2784 }
2785 slab_free_item(keg, slab, item);
2786 if (keg->uk_flags & UMA_ZFLAG_FULL) {
2787 if (keg->uk_pages < keg->uk_maxpages) {
2788 keg->uk_flags &= ~UMA_ZFLAG_FULL;
2789 clearfull = 1;
2790 }
2791
2792 /*
2793 * We can handle one more allocation. Since we're
2794 * clearing ZFLAG_FULL, wake up all procs blocked
2795 * on pages. This should be uncommon, so keeping this
2796 * simple for now (rather than adding count of blocked
2797 * threads etc).
2798 */
2799 wakeup(keg);
2800 }
2801 }
2802 zone_relock(zone, keg);
2803 if (clearfull) {
2804 zone->uz_flags &= ~UMA_ZFLAG_FULL;
2805 wakeup(zone);
2806 }
2807 ZONE_UNLOCK(zone);
2808
2809}
2810
2811/*
2812 * Frees a single item to any zone.
2813 *
2814 * Arguments:
2815 * zone The zone to free to
2816 * item The item we're freeing
2817 * udata User supplied data for the dtor
2818 * skip Skip dtors and finis
2819 */
2820static void
2821zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
2822{
2823
2824#ifdef INVARIANTS
2825 if (skip == SKIP_NONE) {
2826 if (zone->uz_flags & UMA_ZONE_MALLOC)
2827 uma_dbg_free(zone, udata, item);
2828 else
2829 uma_dbg_free(zone, NULL, item);
2830 }
2831#endif
2832 if (skip < SKIP_DTOR && zone->uz_dtor)
2833 zone->uz_dtor(item, zone->uz_size, udata);
2834
2835 if (skip < SKIP_FINI && zone->uz_fini)
2836 zone->uz_fini(item, zone->uz_size);
2837
2838 atomic_add_long(&zone->uz_frees, 1);
2839 zone->uz_release(zone->uz_arg, &item, 1);
2840}
2841
2842/* See uma.h */
2843int
2844uma_zone_set_max(uma_zone_t zone, int nitems)
2845{
2846 uma_keg_t keg;
2847
2848 keg = zone_first_keg(zone);
2849 if (keg == NULL)
2850 return (0);
2851 ZONE_LOCK(zone);
2852 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
2853 if (keg->uk_maxpages * keg->uk_ipers < nitems)
2854 keg->uk_maxpages += keg->uk_ppera;
2855 nitems = keg->uk_maxpages * keg->uk_ipers;
2856 ZONE_UNLOCK(zone);
2857
2858 return (nitems);
2859}
2860
2861/* See uma.h */
2862int
2863uma_zone_get_max(uma_zone_t zone)
2864{
2865 int nitems;
2866 uma_keg_t keg;
2867
2868 keg = zone_first_keg(zone);
2869 if (keg == NULL)
2870 return (0);
2871 ZONE_LOCK(zone);
2872 nitems = keg->uk_maxpages * keg->uk_ipers;
2873 ZONE_UNLOCK(zone);
2874
2875 return (nitems);
2876}
2877
2878/* See uma.h */
2879void
--- 31 unchanged lines hidden (view full) ---
2911/* See uma.h */
2912void
2913uma_zone_set_init(uma_zone_t zone, uma_init uminit)
2914{
2915 uma_keg_t keg;
2916
2917 ZONE_LOCK(zone);
2918 keg = zone_first_keg(zone);
2919 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2920 KASSERT(keg->uk_pages == 0,
2921 ("uma_zone_set_init on non-empty keg"));
2922 keg->uk_init = uminit;
2923 ZONE_UNLOCK(zone);
2924}
2925
2926/* See uma.h */
2927void
2928uma_zone_set_fini(uma_zone_t zone, uma_fini fini)
2929{
2930 uma_keg_t keg;
2931
2932 ZONE_LOCK(zone);
2933 keg = zone_first_keg(zone);
2934 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2935 KASSERT(keg->uk_pages == 0,
2936 ("uma_zone_set_fini on non-empty keg"));
2937 keg->uk_fini = fini;
2938 ZONE_UNLOCK(zone);
2939}
2940
2941/* See uma.h */
2942void
--- 17 unchanged lines hidden (view full) ---
2960 ZONE_UNLOCK(zone);
2961}
2962
2963/* See uma.h */
2964/* XXX uk_freef is not actually used with the zone locked */
2965void
2966uma_zone_set_freef(uma_zone_t zone, uma_free freef)
2967{
2968 uma_keg_t keg;
2969
2970 ZONE_LOCK(zone);
2971 keg = zone_first_keg(zone);
2972 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
2973 keg->uk_freef = freef;
2974 ZONE_UNLOCK(zone);
2975}
2976
2977/* See uma.h */
2978/* XXX uk_allocf is not actually used with the zone locked */
2979void
2980uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf)
2981{
--- 10 unchanged lines hidden (view full) ---
2992int
2993uma_zone_reserve_kva(uma_zone_t zone, int count)
2994{
2995 uma_keg_t keg;
2996 vm_offset_t kva;
2997 int pages;
2998
2999 keg = zone_first_keg(zone);
3000 if (keg == NULL)
3001 return (0);
3002 pages = count / keg->uk_ipers;
3003
3004 if (pages * keg->uk_ipers < count)
3005 pages++;
3006
3007#ifdef UMA_MD_SMALL_ALLOC
3008 if (keg->uk_ppera > 1) {
3009#else
--- 22 unchanged lines hidden (view full) ---
3032void
3033uma_prealloc(uma_zone_t zone, int items)
3034{
3035 int slabs;
3036 uma_slab_t slab;
3037 uma_keg_t keg;
3038
3039 keg = zone_first_keg(zone);
3040 if (keg == NULL)
3041 return;
3042 ZONE_LOCK(zone);
3043 slabs = items / keg->uk_ipers;
3044 if (slabs * keg->uk_ipers < items)
3045 slabs++;
3046 while (slabs > 0) {
3047 slab = keg_alloc_slab(keg, zone, M_WAITOK);
3048 if (slab == NULL)
3049 break;
--- 73 unchanged lines hidden (view full) ---
3123 return (NULL);
3124 mem = page_alloc(NULL, size, &flags, wait);
3125 if (mem) {
3126 vsetslab((vm_offset_t)mem, slab);
3127 slab->us_data = mem;
3128 slab->us_flags = flags | UMA_SLAB_MALLOC;
3129 slab->us_size = size;
3130 } else {
3131 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3132 }
3133
3134 return (mem);
3135}
3136
3137void
3138uma_large_free(uma_slab_t slab)
3139{
3140 vsetobj((vm_offset_t)slab->us_data, kmem_object);
3141 page_free(slab->us_data, slab->us_size, slab->us_flags);
3142 zone_free_item(slabzone, slab, NULL, SKIP_NONE);
3143}
3144
3145void
3146uma_print_stats(void)
3147{
3148 zone_foreach(uma_print_zone);
3149}
3150
--- 258 unchanged lines hidden ---