1/*- 2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org> 3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions --- 34 unchanged lines hidden (view full) --- 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h> |
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249313 2013-04-09 17:43:48Z glebius $"); |
52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 --- 103 unchanged lines hidden (view full) --- 163 const char *name; 164 size_t size; 165 uma_ctor ctor; 166 uma_dtor dtor; 167 uma_init uminit; 168 uma_fini fini; 169 uma_keg_t keg; 170 int align; |
171 uint32_t flags; |
172}; 173 174struct uma_kctor_args { 175 uma_zone_t zone; 176 size_t size; 177 uma_init uminit; 178 uma_fini fini; 179 int align; |
180 uint32_t flags; |
181}; 182 183struct uma_bucket_zone { 184 uma_zone_t ubz_zone; 185 char *ubz_name; 186 int ubz_entries; 187}; 188 --- 21 unchanged lines hidden (view full) --- 210 */ 211enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; 212 213#define ZFREE_STATFAIL 0x00000001 /* Update zone failure statistic. */ 214#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic. */ 215 216/* Prototypes.. */ 217 |
218static void *noobj_alloc(uma_zone_t, int, uint8_t *, int); 219static void *page_alloc(uma_zone_t, int, uint8_t *, int); 220static void *startup_alloc(uma_zone_t, int, uint8_t *, int); 221static void page_free(void *, int, uint8_t); |
222static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int); 223static void cache_drain(uma_zone_t); 224static void bucket_drain(uma_zone_t, uma_bucket_t); 225static void bucket_cache_drain(uma_zone_t zone); 226static int keg_ctor(void *, int, void *, int); 227static void keg_dtor(void *, int, void *); 228static int zone_ctor(void *, int, void *, int); 229static void zone_dtor(void *, int, void *); --- 15 unchanged lines hidden (view full) --- 245static uma_bucket_t bucket_alloc(int, int); 246static void bucket_free(uma_bucket_t); 247static void bucket_zone_drain(void); 248static int zone_alloc_bucket(uma_zone_t zone, int flags); 249static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags); 250static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int flags); 251static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab); 252static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, |
253 uma_fini fini, int align, uint32_t flags); |
254static inline void zone_relock(uma_zone_t zone, uma_keg_t keg); 255static inline void keg_relock(uma_keg_t keg, uma_zone_t zone); 256 257void uma_print_zone(uma_zone_t); 258void uma_print_stats(void); 259static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 260static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 261 --- 428 unchanged lines hidden (view full) --- 690 * Returns nothing. 691 */ 692static void 693keg_drain(uma_keg_t keg) 694{ 695 struct slabhead freeslabs = { 0 }; 696 uma_slab_t slab; 697 uma_slab_t n; |
698 uint8_t flags; 699 uint8_t *mem; |
700 int i; 701 702 /* 703 * We don't want to take pages from statically allocated kegs at this 704 * time 705 */ 706 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 707 return; --- 115 unchanged lines hidden (view full) --- 823 * caller specified M_NOWAIT. 824 */ 825static uma_slab_t 826keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait) 827{ 828 uma_slabrefcnt_t slabref; 829 uma_alloc allocf; 830 uma_slab_t slab; |
831 uint8_t *mem; 832 uint8_t flags; |
833 int i; 834 835 mtx_assert(&keg->uk_lock, MA_OWNED); 836 slab = NULL; 837 838#ifdef UMA_DEBUG 839 printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name); 840#endif --- 104 unchanged lines hidden (view full) --- 945} 946 947/* 948 * This function is intended to be used early on in place of page_alloc() so 949 * that we may use the boot time page cache to satisfy allocations before 950 * the VM is ready. 951 */ 952static void * |
953startup_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait) |
954{ 955 uma_keg_t keg; 956 uma_slab_t tmps; 957 int pages, check_pages; 958 959 keg = zone_first_keg(zone); 960 pages = howmany(bytes, PAGE_SIZE); 961 check_pages = pages - 1; --- 43 unchanged lines hidden (view full) --- 1005 * bytes The number of bytes requested 1006 * wait Shall we wait? 1007 * 1008 * Returns: 1009 * A pointer to the alloced memory or possibly 1010 * NULL if M_NOWAIT is set. 1011 */ 1012static void * |
1013page_alloc(uma_zone_t zone, int bytes, uint8_t *pflag, int wait) |
1014{ 1015 void *p; /* Returned page */ 1016 1017 *pflag = UMA_SLAB_KMEM; 1018 p = (void *) kmem_malloc(kmem_map, bytes, wait); 1019 1020 return (p); 1021} --- 5 unchanged lines hidden (view full) --- 1027 * bytes The number of bytes requested 1028 * wait Shall we wait? 1029 * 1030 * Returns: 1031 * A pointer to the alloced memory or possibly 1032 * NULL if M_NOWAIT is set. 1033 */ 1034static void * |
1035noobj_alloc(uma_zone_t zone, int bytes, uint8_t *flags, int wait) |
1036{ 1037 TAILQ_HEAD(, vm_page) alloctail; 1038 u_long npages; 1039 vm_offset_t retkva, zkva; 1040 vm_page_t p, p_next; 1041 uma_keg_t keg; 1042 1043 TAILQ_INIT(&alloctail); --- 46 unchanged lines hidden (view full) --- 1090 * mem A pointer to the memory to be freed 1091 * size The size of the memory being freed 1092 * flags The original p->us_flags field 1093 * 1094 * Returns: 1095 * Nothing 1096 */ 1097static void |
1098page_free(void *mem, int size, uint8_t flags) |
1099{ 1100 vm_map_t map; 1101 1102 if (flags & UMA_SLAB_KMEM) 1103 map = kmem_map; 1104 else if (flags & UMA_SLAB_KERNEL) 1105 map = kernel_map; 1106 else --- 640 unchanged lines hidden (view full) --- 1747 args.flags = UMA_ZFLAG_INTERNAL; 1748 /* The initial zone has no Per cpu queues so it's smaller */ 1749 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK); 1750 1751#ifdef UMA_DEBUG 1752 printf("Filling boot free list.\n"); 1753#endif 1754 for (i = 0; i < boot_pages; i++) { |
1755 slab = (uma_slab_t)((uint8_t *)bootmem + (i * UMA_SLAB_SIZE)); 1756 slab->us_data = (uint8_t *)slab; |
1757 slab->us_flags = UMA_SLAB_BOOT; 1758 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 1759 } 1760 mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); 1761 1762#ifdef UMA_DEBUG 1763 printf("Creating uma zone headers zone and keg.\n"); 1764#endif --- 82 unchanged lines hidden (view full) --- 1847 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 1848#ifdef UMA_DEBUG 1849 printf("UMA startup3 complete.\n"); 1850#endif 1851} 1852 1853static uma_keg_t 1854uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, |
1855 int align, uint32_t flags) |
1856{ 1857 struct uma_kctor_args args; 1858 1859 args.size = size; 1860 args.uminit = uminit; 1861 args.fini = fini; 1862 args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; 1863 args.flags = flags; --- 8 unchanged lines hidden (view full) --- 1872 1873 if (align != UMA_ALIGN_CACHE) 1874 uma_align_cache = align; 1875} 1876 1877/* See uma.h */ 1878uma_zone_t 1879uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, |
1880 uma_init uminit, uma_fini fini, int align, uint32_t flags) |
1881 1882{ 1883 struct uma_zctor_args args; 1884 1885 /* This stuff is essential for the zone ctor */ 1886 args.name = name; 1887 args.size = size; 1888 args.ctor = ctor; --- 510 unchanged lines hidden (view full) --- 2399} 2400 2401static void * 2402slab_alloc_item(uma_zone_t zone, uma_slab_t slab) 2403{ 2404 uma_keg_t keg; 2405 uma_slabrefcnt_t slabref; 2406 void *item; |
2407 uint8_t freei; |
2408 2409 keg = slab->us_keg; 2410 mtx_assert(&keg->uk_lock, MA_OWNED); 2411 2412 freei = slab->us_firstfree; 2413 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2414 slabref = (uma_slabrefcnt_t)slab; 2415 slab->us_firstfree = slabref->us_freelist[freei].us_item; --- 387 unchanged lines hidden (view full) --- 2803 */ 2804static void 2805zone_free_item(uma_zone_t zone, void *item, void *udata, 2806 enum zfreeskip skip, int flags) 2807{ 2808 uma_slab_t slab; 2809 uma_slabrefcnt_t slabref; 2810 uma_keg_t keg; |
2811 uint8_t *mem; 2812 uint8_t freei; |
2813 int clearfull; 2814 2815 if (skip < SKIP_DTOR && zone->uz_dtor) 2816 zone->uz_dtor(item, zone->uz_size, udata); 2817 2818 if (skip < SKIP_FINI && zone->uz_fini) 2819 zone->uz_fini(item, zone->uz_size); 2820 2821 ZONE_LOCK(zone); 2822 2823 if (flags & ZFREE_STATFAIL) 2824 zone->uz_fails++; 2825 if (flags & ZFREE_STATFREE) 2826 zone->uz_frees++; 2827 2828 if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) { |
2829 mem = (uint8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); |
2830 keg = zone_first_keg(zone); /* Must only be one. */ 2831 if (zone->uz_flags & UMA_ZONE_HASH) { 2832 slab = hash_sfind(&keg->uk_hash, mem); 2833 } else { 2834 mem += keg->uk_pgoff; 2835 slab = (uma_slab_t)mem; 2836 } 2837 } else { --- 259 unchanged lines hidden (view full) --- 3097 MPASS(slab->us_keg == keg); 3098 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 3099 slabs--; 3100 } 3101 ZONE_UNLOCK(zone); 3102} 3103 3104/* See uma.h */ |
3105uint32_t * |
3106uma_find_refcnt(uma_zone_t zone, void *item) 3107{ 3108 uma_slabrefcnt_t slabref; 3109 uma_keg_t keg; |
3110 uint32_t *refcnt; |
3111 int idx; 3112 3113 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item & 3114 (~UMA_SLAB_MASK)); 3115 keg = slabref->us_keg; 3116 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT, 3117 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); 3118 idx = ((unsigned long)item - (unsigned long)slabref->us_data) --- 39 unchanged lines hidden (view full) --- 3158 return (zone->uz_flags & UMA_ZFLAG_FULL); 3159} 3160 3161void * 3162uma_large_malloc(int size, int wait) 3163{ 3164 void *mem; 3165 uma_slab_t slab; |
3166 uint8_t flags; |
3167 3168 slab = zone_alloc_item(slabzone, NULL, wait); 3169 if (slab == NULL) 3170 return (NULL); 3171 mem = page_alloc(NULL, size, &flags, wait); 3172 if (mem) { 3173 vsetslab((vm_offset_t)mem, slab); 3174 slab->us_data = mem; --- 87 unchanged lines hidden (view full) --- 3262 * Note: does not update the zone statistics, as it can't safely clear the 3263 * per-CPU cache statistic. 3264 * 3265 * XXXRW: Following the uc_allocbucket and uc_freebucket pointers here isn't 3266 * safe from off-CPU; we should modify the caches to track this information 3267 * directly so that we don't have to. 3268 */ 3269static void |
3270uma_zone_sumstat(uma_zone_t z, int *cachefreep, uint64_t *allocsp, 3271 uint64_t *freesp, uint64_t *sleepsp) |
3272{ 3273 uma_cache_t cache; |
3274 uint64_t allocs, frees, sleeps; |
3275 int cachefree, cpu; 3276 3277 allocs = frees = sleeps = 0; 3278 cachefree = 0; 3279 CPU_FOREACH(cpu) { 3280 cache = &z->uz_cpu[cpu]; 3281 if (cache->uc_allocbucket != NULL) 3282 cachefree += cache->uc_allocbucket->ub_cnt; --- 134 unchanged lines hidden (view full) --- 3417 error = sbuf_finish(&sbuf); 3418 sbuf_delete(&sbuf); 3419 return (error); 3420} 3421 3422#ifdef DDB 3423DB_SHOW_COMMAND(uma, db_show_uma) 3424{ |
3425 uint64_t allocs, frees, sleeps; |
3426 uma_bucket_t bucket; 3427 uma_keg_t kz; 3428 uma_zone_t z; 3429 int cachefree; 3430 3431 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", 3432 "Requests", "Sleeps"); 3433 LIST_FOREACH(kz, &uma_kegs, uk_link) { --- 24 unchanged lines hidden --- |