1/*- 2 * Copyright (c) 2002-2005, 2009, 2013 Jeffrey Roberson <jeff@FreeBSD.org> 3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions --- 34 unchanged lines hidden (view full) --- 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h> |
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 296243 2016-03-01 00:33:32Z glebius $"); |
52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 --- 47 unchanged lines hidden (view full) --- 107static struct uma_keg masterkeg; 108static struct uma_zone masterzone_k; 109static struct uma_zone masterzone_z; 110static uma_zone_t kegs = &masterzone_k; 111static uma_zone_t zones = &masterzone_z; 112 113/* This is the zone from which all of uma_slab_t's are allocated. */ 114static uma_zone_t slabzone; |
115 116/* 117 * The initial hash tables come out of this zone so they can be allocated 118 * prior to malloc coming up. 119 */ 120static uma_zone_t hashzone; 121 122/* The boot-time adjusted value for cache line alignment. */ --- 26 unchanged lines hidden (view full) --- 149static struct sx uma_drain_lock; 150 151/* Is the VM done starting up? */ 152static int booted = 0; 153#define UMA_STARTUP 1 154#define UMA_STARTUP2 2 155 156/* |
157 * This is the handle used to schedule events that need to happen 158 * outside of the allocation fast path. 159 */ 160static struct callout uma_callout; 161#define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 162 163/* 164 * This structure is passed as the zone ctor arg so that I don't have to create --- 774 unchanged lines hidden (view full) --- 939 * 940 * Returns: 941 * The slab that was allocated or NULL if there is no memory and the 942 * caller specified M_NOWAIT. 943 */ 944static uma_slab_t 945keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int wait) 946{ |
947 uma_alloc allocf; 948 uma_slab_t slab; 949 uint8_t *mem; 950 uint8_t flags; 951 int i; 952 953 mtx_assert(&keg->uk_lock, MA_OWNED); 954 slab = NULL; --- 46 unchanged lines hidden (view full) --- 1001 slab->us_keg = keg; 1002 slab->us_data = mem; 1003 slab->us_freecount = keg->uk_ipers; 1004 slab->us_flags = flags; 1005 BIT_FILL(SLAB_SETSIZE, &slab->us_free); 1006#ifdef INVARIANTS 1007 BIT_ZERO(SLAB_SETSIZE, &slab->us_debugfree); 1008#endif |
1009 1010 if (keg->uk_init != NULL) { 1011 for (i = 0; i < keg->uk_ipers; i++) 1012 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 1013 keg->uk_size, wait) != 0) 1014 break; 1015 if (i != keg->uk_ipers) { 1016 keg_free_slab(keg, slab, i); --- 231 unchanged lines hidden (view full) --- 1248 if (rsize & keg->uk_align) 1249 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1250 keg->uk_rsize = rsize; 1251 1252 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 1253 keg->uk_rsize < sizeof(struct pcpu), 1254 ("%s: size %u too large", __func__, keg->uk_rsize)); 1255 |
1256 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 1257 shsize = 0; 1258 else 1259 shsize = sizeof(struct uma_slab); 1260 1261 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; 1262 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_SETSIZE, 1263 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); --- 71 unchanged lines hidden (view full) --- 1335 1336 /* We can't do OFFPAGE if we're internal, bail out here. */ 1337 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) 1338 return; 1339 1340 /* Check whether we have enough space to not do OFFPAGE. */ 1341 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) == 0) { 1342 shsize = sizeof(struct uma_slab); |
1343 if (shsize & UMA_ALIGN_PTR) 1344 shsize = (shsize & ~UMA_ALIGN_PTR) + 1345 (UMA_ALIGN_PTR + 1); 1346 1347 if ((PAGE_SIZE * keg->uk_ppera) - keg->uk_rsize < shsize) 1348 keg->uk_flags |= UMA_ZONE_OFFPAGE; 1349 } 1350 --- 72 unchanged lines hidden (view full) --- 1423 keg->uk_name = zone->uz_name; 1424 1425 if (arg->flags & UMA_ZONE_VM) 1426 keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1427 1428 if (arg->flags & UMA_ZONE_ZINIT) 1429 keg->uk_init = zero_init; 1430 |
1431 if (arg->flags & UMA_ZONE_MALLOC) |
1432 keg->uk_flags |= UMA_ZONE_VTOSLAB; 1433 1434 if (arg->flags & UMA_ZONE_PCPU) 1435#ifdef SMP 1436 keg->uk_flags |= UMA_ZONE_OFFPAGE; 1437#else 1438 keg->uk_flags &= ~UMA_ZONE_PCPU; 1439#endif 1440 1441 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) { 1442 keg_cachespread_init(keg); |
1443 } else { 1444 if (keg->uk_size > (UMA_SLAB_SIZE - sizeof(struct uma_slab))) 1445 keg_large_init(keg); 1446 else 1447 keg_small_init(keg); 1448 } 1449 |
1450 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 1451 keg->uk_slabzone = slabzone; |
1452 1453 /* 1454 * If we haven't booted yet we need allocations to go through the 1455 * startup cache until the vm is ready. 1456 */ 1457 if (keg->uk_ppera == 1) { 1458#ifdef UMA_MD_SMALL_ALLOC 1459 keg->uk_allocf = uma_small_alloc; --- 20 unchanged lines hidden (view full) --- 1480 * justified offset into the memory on an ALIGN_PTR boundary. 1481 */ 1482 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 1483 u_int totsize; 1484 1485 /* Size of the slab struct and free list */ 1486 totsize = sizeof(struct uma_slab); 1487 |
1488 if (totsize & UMA_ALIGN_PTR) 1489 totsize = (totsize & ~UMA_ALIGN_PTR) + 1490 (UMA_ALIGN_PTR + 1); 1491 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; 1492 1493 /* 1494 * The only way the following is possible is if with our 1495 * UMA_ALIGN_PTR adjustments we are now bigger than 1496 * UMA_SLAB_SIZE. I haven't checked whether this is 1497 * mathematically possible for all cases, so we make 1498 * sure here anyway. 1499 */ 1500 totsize = keg->uk_pgoff + sizeof(struct uma_slab); |
1501 if (totsize > PAGE_SIZE * keg->uk_ppera) { 1502 printf("zone %s ipers %d rsize %d size %d\n", 1503 zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1504 keg->uk_size); 1505 panic("UMA slab won't fit."); 1506 } 1507 } 1508 --- 245 unchanged lines hidden (view full) --- 1754 1755/* Public functions */ 1756/* See uma.h */ 1757void 1758uma_startup(void *bootmem, int boot_pages) 1759{ 1760 struct uma_zctor_args args; 1761 uma_slab_t slab; |
1762 int i; 1763 1764#ifdef UMA_DEBUG 1765 printf("Creating uma keg headers zone and keg.\n"); 1766#endif 1767 rw_init(&uma_rwlock, "UMA lock"); 1768 1769 /* "manually" create the initial zone */ --- 42 unchanged lines hidden (view full) --- 1812#endif 1813 1814 /* Now make a zone for slab headers */ 1815 slabzone = uma_zcreate("UMA Slabs", 1816 sizeof(struct uma_slab), 1817 NULL, NULL, NULL, NULL, 1818 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1819 |
1820 hashzone = uma_zcreate("UMA Hash", 1821 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 1822 NULL, NULL, NULL, NULL, 1823 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1824 1825 bucket_init(); 1826 1827 booted = UMA_STARTUP; --- 206 unchanged lines hidden (view full) --- 2034 } 2035 /* 2036 * The new master must also use vtoslab(). 2037 */ 2038 if ((zone->uz_flags & UMA_ZONE_VTOSLAB) != UMA_ZONE_VTOSLAB) { 2039 error = EINVAL; 2040 goto out; 2041 } |
2042 |
2043 /* |
2044 * The underlying object must be the same size. rsize 2045 * may be different. 2046 */ 2047 if (master->uz_size != zone->uz_size) { 2048 error = E2BIG; 2049 goto out; 2050 } 2051 /* --- 1105 unchanged lines hidden (view full) --- 3157 MPASS(slab->us_keg == keg); 3158 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 3159 slabs--; 3160 } 3161 KEG_UNLOCK(keg); 3162} 3163 3164/* See uma.h */ |
3165static void 3166uma_reclaim_locked(bool kmem_danger) 3167{ 3168 3169#ifdef UMA_DEBUG 3170 printf("UMA: vm asked us to release pages!\n"); 3171#endif 3172 sx_assert(&uma_drain_lock, SA_XLOCKED); --- 4 unchanged lines hidden (view full) --- 3177 zone_foreach(zone_drain); 3178 } 3179 /* 3180 * Some slabs may have been freed but this zone will be visited early 3181 * we visit again so that we can free pages that are empty once other 3182 * zones are drained. We have to do the same for buckets. 3183 */ 3184 zone_drain(slabzone); |
3185 bucket_zone_drain(); 3186} 3187 3188void 3189uma_reclaim(void) 3190{ 3191 3192 sx_xlock(&uma_drain_lock); --- 498 unchanged lines hidden --- |