1/*- 2 * Copyright (c) 2004-2005 Robert N. M. Watson 3 * Copyright (c) 2004, 2005, 4 * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 5 * Copyright (c) 2002, 2003, 2004, 2005, 6 * Jeffrey Roberson <jeff@FreeBSD.org>. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 35 unchanged lines hidden (view full) --- 44 45/* 46 * TODO: 47 * - Improve memory usage for large allocations 48 * - Investigate cache size adjustments 49 */ 50 51#include <sys/cdefs.h> |
52__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 148070 2005-07-15 23:34:39Z rwatson $"); |
53 54/* I should really use ktr.. */ 55/* 56#define UMA_DEBUG 1 57#define UMA_DEBUG_ALLOC 1 58#define UMA_DEBUG_ALLOC_1 1 59*/ 60 --- 126 unchanged lines hidden (view full) --- 187#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) 188 189/* 190 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket 191 * of approximately the right size. 192 */ 193static uint8_t bucket_size[BUCKET_ZONES]; 194 |
195/* 196 * Flags and enumerations to be passed to internal functions. 197 */ |
198enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; 199 |
200#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */ 201 |
202/* Prototypes.. */ 203 204static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 205static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 206static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); 207static void page_free(void *, int, u_int8_t); 208static uma_slab_t slab_zalloc(uma_zone_t, int); 209static void cache_drain(uma_zone_t); --- 9 unchanged lines hidden (view full) --- 219static void zone_foreach(void (*zfunc)(uma_zone_t)); 220static void zone_timeout(uma_zone_t zone); 221static int hash_alloc(struct uma_hash *); 222static int hash_expand(struct uma_hash *, struct uma_hash *); 223static void hash_free(struct uma_hash *hash); 224static void uma_timeout(void *); 225static void uma_startup3(void); 226static void *uma_zalloc_internal(uma_zone_t, void *, int); |
227static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip, 228 int); |
229static void bucket_enable(void); 230static void bucket_init(void); 231static uma_bucket_t bucket_alloc(int, int); 232static void bucket_free(uma_bucket_t); 233static void bucket_zone_drain(void); 234static int uma_zalloc_bucket(uma_zone_t zone, int flags); 235static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); 236static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); --- 108 unchanged lines hidden (view full) --- 345} 346 347static void 348bucket_free(uma_bucket_t bucket) 349{ 350 struct uma_bucket_zone *ubz; 351 352 ubz = bucket_zone_lookup(bucket->ub_entries); |
353 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE, 0); |
354} 355 356static void 357bucket_zone_drain(void) 358{ 359 struct uma_bucket_zone *ubz; 360 361 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) --- 174 unchanged lines hidden (view full) --- 536 */ 537static void 538hash_free(struct uma_hash *hash) 539{ 540 if (hash->uh_slab_hash == NULL) 541 return; 542 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 543 uma_zfree_internal(hashzone, |
544 hash->uh_slab_hash, NULL, SKIP_NONE, 0); |
545 else 546 free(hash->uh_slab_hash, M_UMAHASH); 547} 548 549/* 550 * Frees all outstanding items in a bucket 551 * 552 * Arguments: --- 32 unchanged lines hidden (view full) --- 585 /* 586 * This is extremely inefficient. The slab pointer was passed 587 * to uma_zfree_arg, but we lost it because the buckets don't 588 * hold them. This will go away when free() gets a size passed 589 * to it. 590 */ 591 if (mzone) 592 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); |
593 uma_zfree_internal(zone, item, slab, SKIP_DTOR, 0); |
594 } 595} 596 597/* 598 * Drains the per cpu caches for a zone. 599 * 600 * NOTE: This may only be called while the zone is being turn down, and not 601 * during normal operation. This is necessary in order that we do not have --- 150 unchanged lines hidden (view full) --- 752 else 753 obj = NULL; 754 for (i = 0; i < keg->uk_ppera; i++) 755 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 756 obj); 757 } 758 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 759 uma_zfree_internal(keg->uk_slabzone, slab, NULL, |
760 SKIP_NONE, 0); |
761#ifdef UMA_DEBUG 762 printf("%s: Returning %d bytes.\n", 763 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera); 764#endif 765 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); 766 } 767} 768 --- 46 unchanged lines hidden (view full) --- 815 else 816 wait &= ~M_ZERO; 817 818 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, 819 &flags, wait); 820 if (mem == NULL) { 821 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 822 uma_zfree_internal(keg->uk_slabzone, slab, NULL, |
823 SKIP_NONE, 0); |
824 ZONE_LOCK(zone); 825 return (NULL); 826 } 827 828 /* Point the slab into the allocated memory */ 829 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 830 slab = (uma_slab_t )(mem + keg->uk_pgoff); 831 --- 40 unchanged lines hidden (view full) --- 872 else 873 obj = NULL; 874 for (i = 0; i < keg->uk_ppera; i++) 875 vsetobj((vm_offset_t)mem + 876 (i * PAGE_SIZE), obj); 877 } 878 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 879 uma_zfree_internal(keg->uk_slabzone, slab, |
880 NULL, SKIP_NONE, 0); |
881 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, 882 flags); 883 ZONE_LOCK(zone); 884 return (NULL); 885 } 886 } 887 ZONE_LOCK(zone); 888 --- 447 unchanged lines hidden (view full) --- 1336 bzero(zone, size); 1337 zone->uz_name = arg->name; 1338 zone->uz_ctor = arg->ctor; 1339 zone->uz_dtor = arg->dtor; 1340 zone->uz_init = NULL; 1341 zone->uz_fini = NULL; 1342 zone->uz_allocs = 0; 1343 zone->uz_frees = 0; |
1344 zone->uz_fails = 0; |
1345 zone->uz_fills = zone->uz_count = 0; 1346 1347 if (arg->flags & UMA_ZONE_SECONDARY) { 1348 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 1349 keg = arg->keg; 1350 zone->uz_keg = keg; 1351 zone->uz_init = arg->uminit; 1352 zone->uz_fini = arg->fini; --- 110 unchanged lines hidden (view full) --- 1463 if (LIST_EMPTY(&keg->uk_zones)) 1464 keg->uk_flags &= ~UMA_ZONE_SECONDARY; 1465 ZONE_UNLOCK(zone); 1466 mtx_unlock(&uma_mtx); 1467 } else { 1468 LIST_REMOVE(keg, uk_link); 1469 LIST_REMOVE(zone, uz_link); 1470 mtx_unlock(&uma_mtx); |
1471 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE, 0); |
1472 } 1473 zone->uz_keg = NULL; 1474} 1475 1476/* 1477 * Traverses every zone in the system and calls a callback 1478 * 1479 * Arguments: --- 287 unchanged lines hidden (view full) --- 1767 1768 return (uma_zalloc_internal(zones, &args, M_WAITOK)); 1769} 1770 1771/* See uma.h */ 1772void 1773uma_zdestroy(uma_zone_t zone) 1774{ |
1775 uma_zfree_internal(zones, zone, NULL, SKIP_NONE, 0); |
1776} 1777 1778/* See uma.h */ 1779void * 1780uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1781{ 1782 void *item; 1783 uma_cache_t cache; --- 67 unchanged lines hidden (view full) --- 1851 ZONE_LOCK(zone); 1852 uma_dbg_alloc(zone, NULL, item); 1853 ZONE_UNLOCK(zone); 1854#endif 1855 if (zone->uz_ctor != NULL) { 1856 if (zone->uz_ctor(item, zone->uz_keg->uk_size, 1857 udata, flags) != 0) { 1858 uma_zfree_internal(zone, item, udata, |
1859 SKIP_DTOR, ZFREE_STATFAIL); |
1860 return (NULL); 1861 } 1862 } 1863 if (flags & M_ZERO) 1864 bzero(item, zone->uz_keg->uk_size); 1865 return (item); 1866 } else if (cache->uc_freebucket) { 1867 /* --- 290 unchanged lines hidden (view full) --- 2158 * If we couldn't initialize the whole bucket, put the 2159 * rest back onto the freelist. 2160 */ 2161 if (i != bucket->ub_cnt) { 2162 int j; 2163 2164 for (j = i; j < bucket->ub_cnt; j++) { 2165 uma_zfree_internal(zone, bucket->ub_bucket[j], |
2166 NULL, SKIP_FINI, 0); |
2167#ifdef INVARIANTS 2168 bucket->ub_bucket[j] = NULL; 2169#endif 2170 } 2171 bucket->ub_cnt = i; 2172 } 2173 ZONE_LOCK(zone); 2174 } --- 36 unchanged lines hidden (view full) --- 2211 2212#ifdef UMA_DEBUG_ALLOC 2213 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 2214#endif 2215 ZONE_LOCK(zone); 2216 2217 slab = uma_zone_slab(zone, flags); 2218 if (slab == NULL) { |
2219 zone->uz_fails++; |
2220 ZONE_UNLOCK(zone); 2221 return (NULL); 2222 } 2223 2224 item = uma_slab_alloc(zone, slab); 2225 2226 zone->uz_allocs++; 2227 2228 ZONE_UNLOCK(zone); 2229 2230 /* 2231 * We have to call both the zone's init (not the keg's init) 2232 * and the zone's ctor. This is because the item is going from 2233 * a keg slab directly to the user, and the user is expecting it 2234 * to be both zone-init'd as well as zone-ctor'd. 2235 */ 2236 if (zone->uz_init != NULL) { 2237 if (zone->uz_init(item, keg->uk_size, flags) != 0) { |
2238 uma_zfree_internal(zone, item, udata, SKIP_FINI, 2239 ZFREE_STATFAIL); |
2240 return (NULL); 2241 } 2242 } 2243 if (zone->uz_ctor != NULL) { 2244 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) { |
2245 uma_zfree_internal(zone, item, udata, SKIP_DTOR, 2246 ZFREE_STATFAIL); |
2247 return (NULL); 2248 } 2249 } 2250 if (flags & M_ZERO) 2251 bzero(item, keg->uk_size); 2252 2253 return (item); 2254} --- 156 unchanged lines hidden (view full) --- 2411 ZONE_UNLOCK(zone); 2412 goto zfree_restart; 2413 } 2414 2415 /* 2416 * If nothing else caught this, we'll just do an internal free. 2417 */ 2418zfree_internal: |
2419 uma_zfree_internal(zone, item, udata, SKIP_DTOR, ZFREE_STATFAIL); |
2420 2421 return; 2422} 2423 2424/* 2425 * Frees an item to an INTERNAL zone or allocates a free bucket 2426 * 2427 * Arguments: 2428 * zone The zone to free to 2429 * item The item we're freeing 2430 * udata User supplied data for the dtor 2431 * skip Skip dtors and finis 2432 */ 2433static void 2434uma_zfree_internal(uma_zone_t zone, void *item, void *udata, |
2435 enum zfreeskip skip, int flags) |
2436{ 2437 uma_slab_t slab; 2438 uma_slabrefcnt_t slabref; 2439 uma_keg_t keg; 2440 u_int8_t *mem; 2441 u_int8_t freei; 2442 2443 keg = zone->uz_keg; 2444 2445 if (skip < SKIP_DTOR && zone->uz_dtor) 2446 zone->uz_dtor(item, keg->uk_size, udata); 2447 if (skip < SKIP_FINI && zone->uz_fini) 2448 zone->uz_fini(item, keg->uk_size); 2449 2450 ZONE_LOCK(zone); 2451 |
2452 if (flags & ZFREE_STATFAIL) 2453 zone->uz_fails++; 2454 |
2455 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) { 2456 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 2457 if (keg->uk_flags & UMA_ZONE_HASH) 2458 slab = hash_sfind(&keg->uk_hash, mem); 2459 else { 2460 mem += keg->uk_pgoff; 2461 slab = (uma_slab_t)mem; 2462 } --- 235 unchanged lines hidden (view full) --- 2698 return (NULL); 2699 mem = page_alloc(NULL, size, &flags, wait); 2700 if (mem) { 2701 vsetslab((vm_offset_t)mem, slab); 2702 slab->us_data = mem; 2703 slab->us_flags = flags | UMA_SLAB_MALLOC; 2704 slab->us_size = size; 2705 } else { |
2706 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, 2707 ZFREE_STATFAIL); |
2708 } 2709 2710 return (mem); 2711} 2712 2713void 2714uma_large_free(uma_slab_t slab) 2715{ 2716 vsetobj((vm_offset_t)slab->us_data, kmem_object); 2717 page_free(slab->us_data, slab->us_size, slab->us_flags); |
2718 uma_zfree_internal(slabzone, slab, NULL, SKIP_NONE, 0); |
2719} 2720 2721void 2722uma_print_stats(void) 2723{ 2724 zone_foreach(uma_print_zone); 2725} 2726 --- 253 unchanged lines hidden (view full) --- 2980 kz->uk_ppera; 2981 else 2982 uth.uth_limit = kz->uk_maxpages * 2983 kz->uk_ipers; 2984 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) 2985 uth.uth_zone_free += bucket->ub_cnt; 2986 uth.uth_allocs = z->uz_allocs; 2987 uth.uth_frees = z->uz_frees; |
2988 uth.uth_fails = z->uz_fails; |
2989 ZONE_UNLOCK(z); 2990 if (sbuf_bcat(&sbuf, &uth, sizeof(uth)) < 0) { 2991 mtx_unlock(&uma_mtx); 2992 error = ENOMEM; 2993 goto out; 2994 } 2995 /* 2996 * XXXRW: Should not access bucket fields from --- 33 unchanged lines hidden --- |