1/*- 2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org> 3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions --- 34 unchanged lines hidden (view full) --- 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h> |
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 251826 2013-06-17 03:43:47Z jeff $"); |
52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 --- 66 unchanged lines hidden (view full) --- 126 * Are we allowed to allocate buckets? 127 */ 128static int bucketdisable = 1; 129 130/* Linked list of all kegs in the system */ 131static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); 132 133/* This mutex protects the keg list */ |
134static struct mtx_padalign uma_mtx; |
135 136/* Linked list of boot time pages */ 137static LIST_HEAD(,uma_slab) uma_boot_pages = 138 LIST_HEAD_INITIALIZER(uma_boot_pages); 139 140/* This mutex protects the boot time pages list */ |
141static struct mtx_padalign uma_boot_pages_mtx; |
142 143/* Is the VM done starting up? */ 144static int booted = 0; 145#define UMA_STARTUP 1 146#define UMA_STARTUP2 2 147 148/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */ 149static const u_int uma_max_ipers = SLAB_SETSIZE; --- 17 unchanged lines hidden (view full) --- 167 */ 168struct uma_zctor_args { 169 const char *name; 170 size_t size; 171 uma_ctor ctor; 172 uma_dtor dtor; 173 uma_init uminit; 174 uma_fini fini; |
175 uma_import import; 176 uma_release release; 177 void *arg; |
178 uma_keg_t keg; 179 int align; 180 uint32_t flags; 181}; 182 183struct uma_kctor_args { 184 uma_zone_t zone; 185 size_t size; --- 28 unchanged lines hidden (view full) --- 214 */ 215static uint8_t bucket_size[BUCKET_ZONES]; 216 217/* 218 * Flags and enumerations to be passed to internal functions. 219 */ 220enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; 221 |
222/* Prototypes.. */ 223 224static void *noobj_alloc(uma_zone_t, int, uint8_t *, int); 225static void *page_alloc(uma_zone_t, int, uint8_t *, int); 226static void *startup_alloc(uma_zone_t, int, uint8_t *, int); 227static void page_free(void *, int, uint8_t); 228static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int); 229static void cache_drain(uma_zone_t); --- 9 unchanged lines hidden (view full) --- 239static void zone_foreach(void (*zfunc)(uma_zone_t)); 240static void zone_timeout(uma_zone_t zone); 241static int hash_alloc(struct uma_hash *); 242static int hash_expand(struct uma_hash *, struct uma_hash *); 243static void hash_free(struct uma_hash *hash); 244static void uma_timeout(void *); 245static void uma_startup3(void); 246static void *zone_alloc_item(uma_zone_t, void *, int); |
247static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); |
248static void bucket_enable(void); 249static void bucket_init(void); 250static uma_bucket_t bucket_alloc(int, int); 251static void bucket_free(uma_bucket_t); 252static void bucket_zone_drain(void); 253static int zone_alloc_bucket(uma_zone_t zone, int flags); 254static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags); 255static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags); |
256static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); 257static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item); |
258static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 259 uma_fini fini, int align, uint32_t flags); 260static inline void zone_relock(uma_zone_t zone, uma_keg_t keg); 261static inline void keg_relock(uma_keg_t keg, uma_zone_t zone); |
262static int zone_import(uma_zone_t zone, void **bucket, int max, int flags); 263static void zone_release(uma_zone_t zone, void **bucket, int cnt); |
264 265void uma_print_zone(uma_zone_t); 266void uma_print_stats(void); 267static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 268static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 269 270SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 271 --- 88 unchanged lines hidden (view full) --- 360} 361 362static void 363bucket_free(uma_bucket_t bucket) 364{ 365 struct uma_bucket_zone *ubz; 366 367 ubz = bucket_zone_lookup(bucket->ub_entries); |
368 zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE); |
369} 370 371static void 372bucket_zone_drain(void) 373{ 374 struct uma_bucket_zone *ubz; 375 376 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) --- 7 unchanged lines hidden (view full) --- 384 385 if (!zone_warnings || zone->uz_warning == NULL) 386 return; 387 388 if (ratecheck(&zone->uz_ratecheck, &warninterval)) 389 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); 390} 391 |
392static void 393zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t)) 394{ 395 uma_klink_t klink; 396 397 LIST_FOREACH(klink, &zone->uz_kegs, kl_link) 398 kegfn(klink->kl_keg); 399} --- 168 unchanged lines hidden (view full) --- 568 * Nothing 569 */ 570static void 571hash_free(struct uma_hash *hash) 572{ 573 if (hash->uh_slab_hash == NULL) 574 return; 575 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) |
576 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE); |
577 else 578 free(hash->uh_slab_hash, M_UMAHASH); 579} 580 581/* 582 * Frees all outstanding items in a bucket 583 * 584 * Arguments: 585 * zone The zone to free to, must be unlocked. 586 * bucket The free/alloc bucket with items, cpu queue must be locked. 587 * 588 * Returns: 589 * Nothing 590 */ 591 592static void 593bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 594{ |
595 int i; |
596 597 if (bucket == NULL) 598 return; 599 |
600 if (zone->uz_fini) 601 for (i = 0; i < bucket->ub_cnt; i++) 602 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); 603 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); 604 bucket->ub_cnt = 0; |
605} 606 607/* 608 * Drains the per cpu caches for a zone. 609 * 610 * NOTE: This may only be called while the zone is being turn down, and not 611 * during normal operation. This is necessary in order that we do not have 612 * to migrate CPUs to drain the per-CPU caches. --- 137 unchanged lines hidden (view full) --- 750 obj = kernel_object; 751 else 752 obj = NULL; 753 for (i = 0; i < keg->uk_ppera; i++) 754 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 755 obj); 756 } 757 if (keg->uk_flags & UMA_ZONE_OFFPAGE) |
758 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); |
759#ifdef UMA_DEBUG 760 printf("%s: Returning %d bytes.\n", 761 keg->uk_name, PAGE_SIZE * keg->uk_ppera); 762#endif 763 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); 764 } 765} 766 --- 57 unchanged lines hidden (view full) --- 824 uint8_t *mem; 825 uint8_t flags; 826 int i; 827 828 mtx_assert(&keg->uk_lock, MA_OWNED); 829 slab = NULL; 830 831#ifdef UMA_DEBUG |
832 printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name); |
833#endif 834 allocf = keg->uk_allocf; 835 KEG_UNLOCK(keg); 836 837 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 838 slab = zone_alloc_item(keg->uk_slabzone, NULL, wait); 839 if (slab == NULL) { 840 KEG_LOCK(keg); --- 15 unchanged lines hidden (view full) --- 856 857 if (keg->uk_flags & UMA_ZONE_NODUMP) 858 wait |= M_NODUMP; 859 860 /* zone is passed for legacy reasons. */ 861 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait); 862 if (mem == NULL) { 863 if (keg->uk_flags & UMA_ZONE_OFFPAGE) |
864 zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE); |
865 KEG_LOCK(keg); 866 return (NULL); 867 } 868 869 /* Point the slab into the allocated memory */ 870 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 871 slab = (uma_slab_t )(mem + keg->uk_pgoff); 872 --- 37 unchanged lines hidden (view full) --- 910 else 911 obj = NULL; 912 for (i = 0; i < keg->uk_ppera; i++) 913 vsetobj((vm_offset_t)mem + 914 (i * PAGE_SIZE), obj); 915 } 916 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 917 zone_free_item(keg->uk_slabzone, slab, |
918 NULL, SKIP_NONE); |
919 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, 920 flags); 921 KEG_LOCK(keg); 922 return (NULL); 923 } 924 } 925 KEG_LOCK(keg); 926 --- 537 unchanged lines hidden (view full) --- 1464 zone->uz_fails = 0; 1465 zone->uz_sleeps = 0; 1466 zone->uz_fills = zone->uz_count = 0; 1467 zone->uz_flags = 0; 1468 zone->uz_warning = NULL; 1469 timevalclear(&zone->uz_ratecheck); 1470 keg = arg->keg; 1471 |
1472 /* 1473 * This is a pure cache zone, no kegs. 1474 */ 1475 if (arg->import) { 1476 zone->uz_import = arg->import; 1477 zone->uz_release = arg->release; 1478 zone->uz_arg = arg->arg; 1479 zone->uz_count = BUCKET_MAX; 1480 return (0); 1481 } 1482 1483 /* 1484 * Use the regular zone/keg/slab allocator. 1485 */ 1486 zone->uz_import = (uma_import)zone_import; 1487 zone->uz_release = (uma_release)zone_release; 1488 zone->uz_arg = zone; 1489 |
1490 if (arg->flags & UMA_ZONE_SECONDARY) { 1491 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 1492 zone->uz_init = arg->uminit; 1493 zone->uz_fini = arg->fini; 1494 zone->uz_lock = &keg->uk_lock; 1495 zone->uz_flags |= UMA_ZONE_SECONDARY; 1496 mtx_lock(&uma_mtx); 1497 ZONE_LOCK(zone); --- 20 unchanged lines hidden (view full) --- 1518 karg.align = arg->align; 1519 karg.flags = arg->flags; 1520 karg.zone = zone; 1521 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1522 flags); 1523 if (error) 1524 return (error); 1525 } |
1526 |
1527 /* 1528 * Link in the first keg. 1529 */ 1530 zone->uz_klink.kl_keg = keg; 1531 LIST_INSERT_HEAD(&zone->uz_kegs, &zone->uz_klink, kl_link); 1532 zone->uz_lock = &keg->uk_lock; 1533 zone->uz_size = keg->uk_size; 1534 zone->uz_flags |= (keg->uk_flags & --- 81 unchanged lines hidden (view full) --- 1616 LIST_REMOVE(klink, kl_link); 1617 if (klink == &zone->uz_klink) 1618 continue; 1619 free(klink, M_TEMP); 1620 } 1621 /* 1622 * We only destroy kegs from non secondary zones. 1623 */ |
1624 if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) { |
1625 mtx_lock(&uma_mtx); 1626 LIST_REMOVE(keg, uk_link); 1627 mtx_unlock(&uma_mtx); |
1628 zone_free_item(kegs, keg, NULL, SKIP_NONE); |
1629 } 1630} 1631 1632/* 1633 * Traverses every zone in the system and calls a callback 1634 * 1635 * Arguments: 1636 * zfunc A pointer to a function which accepts a zone --- 27 unchanged lines hidden (view full) --- 1664 int i; 1665 1666#ifdef UMA_DEBUG 1667 printf("Creating uma keg headers zone and keg.\n"); 1668#endif 1669 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 1670 1671 /* "manually" create the initial zone */ |
1672 memset(&args, 0, sizeof(args)); |
1673 args.name = "UMA Kegs"; 1674 args.size = sizeof(struct uma_keg); 1675 args.ctor = keg_ctor; 1676 args.dtor = keg_dtor; 1677 args.uminit = zero_init; 1678 args.fini = NULL; 1679 args.keg = &masterkeg; 1680 args.align = 32 - 1; --- 124 unchanged lines hidden (view full) --- 1805uma_zone_t 1806uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1807 uma_init uminit, uma_fini fini, int align, uint32_t flags) 1808 1809{ 1810 struct uma_zctor_args args; 1811 1812 /* This stuff is essential for the zone ctor */ |
1813 memset(&args, 0, sizeof(args)); |
1814 args.name = name; 1815 args.size = size; 1816 args.ctor = ctor; 1817 args.dtor = dtor; 1818 args.uminit = uminit; 1819 args.fini = fini; 1820 args.align = align; 1821 args.flags = flags; --- 6 unchanged lines hidden (view full) --- 1828uma_zone_t 1829uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 1830 uma_init zinit, uma_fini zfini, uma_zone_t master) 1831{ 1832 struct uma_zctor_args args; 1833 uma_keg_t keg; 1834 1835 keg = zone_first_keg(master); |
1836 memset(&args, 0, sizeof(args)); |
1837 args.name = name; 1838 args.size = keg->uk_size; 1839 args.ctor = ctor; 1840 args.dtor = dtor; 1841 args.uminit = zinit; 1842 args.fini = zfini; 1843 args.align = keg->uk_align; 1844 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 1845 args.keg = keg; 1846 1847 /* XXX Attaches only one keg of potentially many. */ 1848 return (zone_alloc_item(zones, &args, M_WAITOK)); 1849} 1850 |
1851/* See uma.h */ 1852uma_zone_t 1853uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit, 1854 uma_fini zfini, uma_import zimport, uma_release zrelease, 1855 void *arg, int flags) 1856{ 1857 struct uma_zctor_args args; 1858 1859 memset(&args, 0, sizeof(args)); 1860 args.name = name; 1861 args.size = 0; 1862 args.ctor = ctor; 1863 args.dtor = dtor; 1864 args.uminit = zinit; 1865 args.fini = zfini; 1866 args.import = zimport; 1867 args.release = zrelease; 1868 args.arg = arg; 1869 args.align = 0; 1870 args.flags = flags; 1871 1872 return (zone_alloc_item(zones, &args, M_WAITOK)); 1873} 1874 |
1875static void 1876zone_lock_pair(uma_zone_t a, uma_zone_t b) 1877{ 1878 if (a < b) { 1879 ZONE_LOCK(a); 1880 mtx_lock_flags(b->uz_lock, MTX_DUPOK); 1881 } else { 1882 ZONE_LOCK(b); --- 75 unchanged lines hidden (view full) --- 1958} 1959 1960 1961/* See uma.h */ 1962void 1963uma_zdestroy(uma_zone_t zone) 1964{ 1965 |
1966 zone_free_item(zones, zone, NULL, SKIP_NONE); |
1967} 1968 1969/* See uma.h */ 1970void * 1971uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1972{ 1973 void *item; 1974 uma_cache_t cache; --- 62 unchanged lines hidden (view full) --- 2037#endif 2038 KASSERT(item != NULL, 2039 ("uma_zalloc: Bucket pointer mangled.")); 2040 cache->uc_allocs++; 2041 critical_exit(); 2042 if (zone->uz_ctor != NULL) { 2043 if (zone->uz_ctor(item, zone->uz_size, 2044 udata, flags) != 0) { |
2045 atomic_add_long(&zone->uz_fails, 1); |
2046 zone_free_item(zone, item, udata, |
2047 SKIP_DTOR); |
2048 return (NULL); 2049 } 2050 } 2051#ifdef INVARIANTS 2052 uma_dbg_alloc(zone, NULL, item); 2053#endif 2054 if (flags & M_ZERO) 2055 bzero(item, zone->uz_size); --- 39 unchanged lines hidden (view full) --- 2095 bucket = cache->uc_freebucket; 2096 if (bucket != NULL && bucket->ub_cnt > 0) { 2097 ZONE_UNLOCK(zone); 2098 goto zalloc_start; 2099 } 2100 } 2101 2102 /* Since we have locked the zone we may as well send back our stats */ |
2103 atomic_add_long(&zone->uz_allocs, cache->uc_allocs); 2104 atomic_add_long(&zone->uz_frees, cache->uc_frees); |
2105 cache->uc_allocs = 0; |
2106 cache->uc_frees = 0; 2107 2108 /* Our old one is now a free bucket */ 2109 if (cache->uc_allocbucket) { 2110 KASSERT(cache->uc_allocbucket->ub_cnt == 0, 2111 ("uma_zalloc_arg: Freeing a non free bucket.")); 2112 LIST_INSERT_HEAD(&zone->uz_free_bucket, 2113 cache->uc_allocbucket, ub_link); --- 231 unchanged lines hidden (view full) --- 2345 zone->uz_flags &= ~UMA_ZFLAG_FULL; 2346 continue; 2347 } 2348 } 2349 return (NULL); 2350} 2351 2352static void * |
2353slab_alloc_item(uma_keg_t keg, uma_slab_t slab) |
2354{ |
2355 void *item; 2356 uint8_t freei; 2357 |
2358 MPASS(keg == slab->us_keg); |
2359 mtx_assert(&keg->uk_lock, MA_OWNED); 2360 2361 freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1; 2362 BIT_CLR(SLAB_SETSIZE, freei, &slab->us_free); 2363 item = slab->us_data + (keg->uk_rsize * freei); 2364 slab->us_freecount--; 2365 keg->uk_free--; 2366 2367 /* Move this slab to the full list */ 2368 if (slab->us_freecount == 0) { 2369 LIST_REMOVE(slab, us_link); 2370 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link); 2371 } 2372 2373 return (item); 2374} 2375 2376static int |
2377zone_import(uma_zone_t zone, void **bucket, int max, int flags) |
2378{ |
2379 uma_slab_t slab; 2380 uma_keg_t keg; |
2381 int i; |
2382 |
2383 ZONE_LOCK(zone); 2384 /* Try to keep the buckets totally full */ 2385 slab = NULL; 2386 keg = NULL; 2387 for (i = 0; i < max; ) { 2388 if ((slab = zone->uz_slab(zone, keg, flags)) == NULL) 2389 break; 2390 keg = slab->us_keg; 2391 while (slab->us_freecount && i < max) 2392 bucket[i++] = slab_alloc_item(keg, slab); |
2393 |
2394 /* Don't block on the next fill */ 2395 flags &= ~M_WAITOK; 2396 flags |= M_NOWAIT; |
2397 } |
2398 if (slab != NULL) 2399 KEG_UNLOCK(keg); 2400 else 2401 ZONE_UNLOCK(zone); |
2402 |
2403 return i; 2404} |
2405 |
2406static int 2407zone_alloc_bucket(uma_zone_t zone, int flags) 2408{ 2409 uma_bucket_t bucket; 2410 int bflags; 2411 int max; 2412 |
2413#ifdef SMP 2414 /* 2415 * This code is here to limit the number of simultaneous bucket fills 2416 * for any given zone to the number of per cpu caches in this zone. This 2417 * is done so that we don't allocate more memory than we really need. 2418 */ 2419 if (zone->uz_fills >= mp_ncpus) |
2420 return (0); |
2421 2422#endif 2423 zone->uz_fills++; |
2424 max = zone->uz_count; |
2425 |
2426 /* 2427 * Try this zone's free list first so we don't allocate extra buckets. 2428 */ 2429 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2430 KASSERT(bucket->ub_cnt == 0, 2431 ("zone_alloc_bucket: Bucket on free list is not empty.")); 2432 LIST_REMOVE(bucket, ub_link); 2433 ZONE_UNLOCK(zone); 2434 } else { 2435 bflags = (flags & ~M_ZERO); 2436 if (zone->uz_flags & UMA_ZFLAG_CACHEONLY) 2437 bflags |= M_NOVM; 2438 ZONE_UNLOCK(zone); 2439 bucket = bucket_alloc(zone->uz_count, bflags); 2440 if (bucket == NULL) 2441 goto out; |
2442 } |
2443 |
2444 max = MIN(bucket->ub_entries, max); 2445 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, 2446 max, flags); 2447 |
2448 /* |
2449 * Initialize the memory if necessary. |
2450 */ |
2451 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { |
2452 int i; 2453 |
2454 for (i = 0; i < bucket->ub_cnt; i++) |
2455 if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size, |
2456 flags) != 0) |
2457 break; 2458 /* 2459 * If we couldn't initialize the whole bucket, put the 2460 * rest back onto the freelist. 2461 */ 2462 if (i != bucket->ub_cnt) { |
2463 zone->uz_release(zone->uz_arg, bucket->ub_bucket[i], 2464 bucket->ub_cnt - i); |
2465#ifdef INVARIANTS |
2466 bzero(&bucket->ub_bucket[i], 2467 sizeof(void *) * (bucket->ub_cnt - i)); |
2468#endif |
2469 bucket->ub_cnt = i; 2470 } |
2471 } 2472 |
2473out: 2474 ZONE_LOCK(zone); |
2475 zone->uz_fills--; |
2476 if (bucket != NULL && bucket->ub_cnt != 0) { |
2477 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2478 bucket, ub_link); 2479 return (1); 2480 } |
2481 atomic_add_long(&zone->uz_fails, 1); 2482 if (bucket != NULL) 2483 bucket_free(bucket); |
2484 2485 return (0); 2486} 2487/* |
2488 * Allocates a single item from a zone. |
2489 * 2490 * Arguments 2491 * zone The zone to alloc for. 2492 * udata The data to be passed to the constructor. 2493 * flags M_WAITOK, M_NOWAIT, M_ZERO. 2494 * 2495 * Returns 2496 * NULL if there is no memory and M_NOWAIT is set 2497 * An item if successful 2498 */ 2499 2500static void * 2501zone_alloc_item(uma_zone_t zone, void *udata, int flags) 2502{ |
2503 void *item; 2504 2505 item = NULL; 2506 2507#ifdef UMA_DEBUG_ALLOC 2508 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 2509#endif |
2510 if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1) 2511 goto fail; 2512 atomic_add_long(&zone->uz_allocs, 1); |
2513 |
2514 /* 2515 * We have to call both the zone's init (not the keg's init) 2516 * and the zone's ctor. This is because the item is going from 2517 * a keg slab directly to the user, and the user is expecting it 2518 * to be both zone-init'd as well as zone-ctor'd. 2519 */ 2520 if (zone->uz_init != NULL) { 2521 if (zone->uz_init(item, zone->uz_size, flags) != 0) { |
2522 zone_free_item(zone, item, udata, SKIP_FINI); 2523 goto fail; |
2524 } 2525 } 2526 if (zone->uz_ctor != NULL) { 2527 if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) { |
2528 zone_free_item(zone, item, udata, SKIP_DTOR); 2529 goto fail; |
2530 } 2531 } 2532#ifdef INVARIANTS |
2533 uma_dbg_alloc(zone, NULL, item); |
2534#endif 2535 if (flags & M_ZERO) 2536 bzero(item, zone->uz_size); 2537 2538 return (item); |
2539 2540fail: 2541 atomic_add_long(&zone->uz_fails, 1); 2542 return (NULL); |
2543} 2544 2545/* See uma.h */ 2546void 2547uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 2548{ 2549 uma_cache_t cache; 2550 uma_bucket_t bucket; --- 113 unchanged lines hidden (view full) --- 2664 (cache->uc_allocbucket->ub_cnt < 2665 cache->uc_freebucket->ub_cnt)) { 2666 ZONE_UNLOCK(zone); 2667 goto zfree_start; 2668 } 2669 } 2670 2671 /* Since we have locked the zone we may as well send back our stats */ |
2672 atomic_add_long(&zone->uz_allocs, cache->uc_allocs); 2673 atomic_add_long(&zone->uz_frees, cache->uc_frees); |
2674 cache->uc_allocs = 0; |
2675 cache->uc_frees = 0; 2676 2677 bucket = cache->uc_freebucket; 2678 cache->uc_freebucket = NULL; 2679 2680 /* Can we throw this on the zone full list? */ 2681 if (bucket != NULL) { 2682#ifdef UMA_DEBUG_ALLOC --- 32 unchanged lines hidden (view full) --- 2715 ZONE_UNLOCK(zone); 2716 goto zfree_restart; 2717 } 2718 2719 /* 2720 * If nothing else caught this, we'll just do an internal free. 2721 */ 2722zfree_internal: |
2723 zone_free_item(zone, item, udata, SKIP_DTOR); |
2724 2725 return; 2726} 2727 |
2728static void |
2729slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item) |
2730{ |
2731 uint8_t freei; |
2732 |
2733 mtx_assert(&keg->uk_lock, MA_OWNED); |
2734 MPASS(keg == slab->us_keg); 2735 2736 /* Do we need to remove from any lists? */ 2737 if (slab->us_freecount+1 == keg->uk_ipers) { 2738 LIST_REMOVE(slab, us_link); 2739 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2740 } else if (slab->us_freecount == 0) { 2741 LIST_REMOVE(slab, us_link); 2742 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2743 } 2744 2745 /* Slab management. */ 2746 freei = ((uintptr_t)item - (uintptr_t)slab->us_data) / keg->uk_rsize; 2747 BIT_SET(SLAB_SETSIZE, freei, &slab->us_free); 2748 slab->us_freecount++; 2749 2750 /* Keg statistics. */ 2751 keg->uk_free++; |
2752} |
2753 |
2754static void 2755zone_release(uma_zone_t zone, void **bucket, int cnt) 2756{ 2757 void *item; 2758 uma_slab_t slab; 2759 uma_keg_t keg; 2760 uint8_t *mem; 2761 int clearfull; 2762 int i; 2763 |
2764 clearfull = 0; |
2765 ZONE_LOCK(zone); 2766 keg = zone_first_keg(zone); 2767 for (i = 0; i < cnt; i++) { 2768 item = bucket[i]; 2769 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { 2770 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 2771 if (zone->uz_flags & UMA_ZONE_HASH) { 2772 slab = hash_sfind(&keg->uk_hash, mem); 2773 } else { 2774 mem += keg->uk_pgoff; 2775 slab = (uma_slab_t)mem; 2776 } 2777 } else { 2778 slab = vtoslab((vm_offset_t)item); 2779 if (slab->us_keg != keg) { 2780 KEG_UNLOCK(keg); 2781 keg = slab->us_keg; 2782 KEG_LOCK(keg); 2783 } |
2784 } |
2785 slab_free_item(keg, slab, item); 2786 if (keg->uk_flags & UMA_ZFLAG_FULL) { 2787 if (keg->uk_pages < keg->uk_maxpages) { 2788 keg->uk_flags &= ~UMA_ZFLAG_FULL; 2789 clearfull = 1; 2790 } |
2791 |
2792 /* 2793 * We can handle one more allocation. Since we're 2794 * clearing ZFLAG_FULL, wake up all procs blocked 2795 * on pages. This should be uncommon, so keeping this 2796 * simple for now (rather than adding count of blocked 2797 * threads etc). 2798 */ 2799 wakeup(keg); 2800 } |
2801 } |
2802 zone_relock(zone, keg); |
2803 if (clearfull) { |
2804 zone->uz_flags &= ~UMA_ZFLAG_FULL; 2805 wakeup(zone); |
2806 } 2807 ZONE_UNLOCK(zone); |
2808 2809} 2810 |
2811/* 2812 * Frees a single item to any zone. 2813 * 2814 * Arguments: 2815 * zone The zone to free to 2816 * item The item we're freeing 2817 * udata User supplied data for the dtor 2818 * skip Skip dtors and finis 2819 */ 2820static void 2821zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) 2822{ 2823 2824#ifdef INVARIANTS 2825 if (skip == SKIP_NONE) { 2826 if (zone->uz_flags & UMA_ZONE_MALLOC) 2827 uma_dbg_free(zone, udata, item); 2828 else 2829 uma_dbg_free(zone, NULL, item); 2830 } 2831#endif 2832 if (skip < SKIP_DTOR && zone->uz_dtor) 2833 zone->uz_dtor(item, zone->uz_size, udata); 2834 2835 if (skip < SKIP_FINI && zone->uz_fini) 2836 zone->uz_fini(item, zone->uz_size); 2837 2838 atomic_add_long(&zone->uz_frees, 1); 2839 zone->uz_release(zone->uz_arg, &item, 1); 2840} 2841 |
2842/* See uma.h */ 2843int 2844uma_zone_set_max(uma_zone_t zone, int nitems) 2845{ 2846 uma_keg_t keg; 2847 |
2848 keg = zone_first_keg(zone); |
2849 if (keg == NULL) 2850 return (0); 2851 ZONE_LOCK(zone); |
2852 keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera; 2853 if (keg->uk_maxpages * keg->uk_ipers < nitems) 2854 keg->uk_maxpages += keg->uk_ppera; 2855 nitems = keg->uk_maxpages * keg->uk_ipers; 2856 ZONE_UNLOCK(zone); 2857 2858 return (nitems); 2859} 2860 2861/* See uma.h */ 2862int 2863uma_zone_get_max(uma_zone_t zone) 2864{ 2865 int nitems; 2866 uma_keg_t keg; 2867 |
2868 keg = zone_first_keg(zone); |
2869 if (keg == NULL) 2870 return (0); 2871 ZONE_LOCK(zone); |
2872 nitems = keg->uk_maxpages * keg->uk_ipers; 2873 ZONE_UNLOCK(zone); 2874 2875 return (nitems); 2876} 2877 2878/* See uma.h */ 2879void --- 31 unchanged lines hidden (view full) --- 2911/* See uma.h */ 2912void 2913uma_zone_set_init(uma_zone_t zone, uma_init uminit) 2914{ 2915 uma_keg_t keg; 2916 2917 ZONE_LOCK(zone); 2918 keg = zone_first_keg(zone); |
2919 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); |
2920 KASSERT(keg->uk_pages == 0, 2921 ("uma_zone_set_init on non-empty keg")); 2922 keg->uk_init = uminit; 2923 ZONE_UNLOCK(zone); 2924} 2925 2926/* See uma.h */ 2927void 2928uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 2929{ 2930 uma_keg_t keg; 2931 2932 ZONE_LOCK(zone); 2933 keg = zone_first_keg(zone); |
2934 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); |
2935 KASSERT(keg->uk_pages == 0, 2936 ("uma_zone_set_fini on non-empty keg")); 2937 keg->uk_fini = fini; 2938 ZONE_UNLOCK(zone); 2939} 2940 2941/* See uma.h */ 2942void --- 17 unchanged lines hidden (view full) --- 2960 ZONE_UNLOCK(zone); 2961} 2962 2963/* See uma.h */ 2964/* XXX uk_freef is not actually used with the zone locked */ 2965void 2966uma_zone_set_freef(uma_zone_t zone, uma_free freef) 2967{ |
2968 uma_keg_t keg; |
2969 2970 ZONE_LOCK(zone); |
2971 keg = zone_first_keg(zone); 2972 KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type")); 2973 keg->uk_freef = freef; |
2974 ZONE_UNLOCK(zone); 2975} 2976 2977/* See uma.h */ 2978/* XXX uk_allocf is not actually used with the zone locked */ 2979void 2980uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 2981{ --- 10 unchanged lines hidden (view full) --- 2992int 2993uma_zone_reserve_kva(uma_zone_t zone, int count) 2994{ 2995 uma_keg_t keg; 2996 vm_offset_t kva; 2997 int pages; 2998 2999 keg = zone_first_keg(zone); |
3000 if (keg == NULL) 3001 return (0); |
3002 pages = count / keg->uk_ipers; 3003 3004 if (pages * keg->uk_ipers < count) 3005 pages++; 3006 3007#ifdef UMA_MD_SMALL_ALLOC 3008 if (keg->uk_ppera > 1) { 3009#else --- 22 unchanged lines hidden (view full) --- 3032void 3033uma_prealloc(uma_zone_t zone, int items) 3034{ 3035 int slabs; 3036 uma_slab_t slab; 3037 uma_keg_t keg; 3038 3039 keg = zone_first_keg(zone); |
3040 if (keg == NULL) 3041 return; |
3042 ZONE_LOCK(zone); 3043 slabs = items / keg->uk_ipers; 3044 if (slabs * keg->uk_ipers < items) 3045 slabs++; 3046 while (slabs > 0) { 3047 slab = keg_alloc_slab(keg, zone, M_WAITOK); 3048 if (slab == NULL) 3049 break; --- 73 unchanged lines hidden (view full) --- 3123 return (NULL); 3124 mem = page_alloc(NULL, size, &flags, wait); 3125 if (mem) { 3126 vsetslab((vm_offset_t)mem, slab); 3127 slab->us_data = mem; 3128 slab->us_flags = flags | UMA_SLAB_MALLOC; 3129 slab->us_size = size; 3130 } else { |
3131 zone_free_item(slabzone, slab, NULL, SKIP_NONE); |
3132 } 3133 3134 return (mem); 3135} 3136 3137void 3138uma_large_free(uma_slab_t slab) 3139{ 3140 vsetobj((vm_offset_t)slab->us_data, kmem_object); 3141 page_free(slab->us_data, slab->us_size, slab->us_flags); |
3142 zone_free_item(slabzone, slab, NULL, SKIP_NONE); |
3143} 3144 3145void 3146uma_print_stats(void) 3147{ 3148 zone_foreach(uma_print_zone); 3149} 3150 --- 258 unchanged lines hidden --- |