1/*- 2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org> 3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 4 * Copyright (c) 2004-2006 Robert N. M. Watson 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions --- 34 unchanged lines hidden (view full) --- 43 44/* 45 * TODO: 46 * - Improve memory usage for large allocations 47 * - Investigate cache size adjustments 48 */ 49 50#include <sys/cdefs.h> |
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249264 2013-04-08 19:10:45Z glebius $"); |
52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 --- 700 unchanged lines hidden (view full) --- 760 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 761 obj); 762 } 763 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 764 zone_free_item(keg->uk_slabzone, slab, NULL, 765 SKIP_NONE, ZFREE_STATFREE); 766#ifdef UMA_DEBUG 767 printf("%s: Returning %d bytes.\n", |
768 keg->uk_name, PAGE_SIZE * keg->uk_ppera); |
769#endif |
770 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags); |
771 } 772} 773 774static void 775zone_drain_wait(uma_zone_t zone, int waitok) 776{ 777 778 /* --- 81 unchanged lines hidden (view full) --- 860 wait |= M_ZERO; 861 else 862 wait &= ~M_ZERO; 863 864 if (keg->uk_flags & UMA_ZONE_NODUMP) 865 wait |= M_NODUMP; 866 867 /* zone is passed for legacy reasons. */ |
868 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait); |
869 if (mem == NULL) { 870 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 871 zone_free_item(keg->uk_slabzone, slab, NULL, 872 SKIP_NONE, ZFREE_STATFREE); 873 KEG_LOCK(keg); 874 return (NULL); 875 } 876 --- 45 unchanged lines hidden (view full) --- 922 obj = NULL; 923 for (i = 0; i < keg->uk_ppera; i++) 924 vsetobj((vm_offset_t)mem + 925 (i * PAGE_SIZE), obj); 926 } 927 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 928 zone_free_item(keg->uk_slabzone, slab, 929 NULL, SKIP_NONE, ZFREE_STATFREE); |
930 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, |
931 flags); 932 KEG_LOCK(keg); 933 return (NULL); 934 } 935 } 936 KEG_LOCK(keg); 937 938 if (keg->uk_flags & UMA_ZONE_HASH) --- 194 unchanged lines hidden (view full) --- 1133static void 1134keg_small_init(uma_keg_t keg) 1135{ 1136 u_int rsize; 1137 u_int memused; 1138 u_int wastedspace; 1139 u_int shsize; 1140 |
1141 if (keg->uk_flags & UMA_ZONE_PCPU) { 1142 keg->uk_slabsize = sizeof(struct pcpu); 1143 keg->uk_ppera = howmany(mp_ncpus * sizeof(struct pcpu), 1144 PAGE_SIZE); 1145 } else { 1146 keg->uk_slabsize = UMA_SLAB_SIZE; 1147 keg->uk_ppera = 1; 1148 } 1149 |
1150 rsize = keg->uk_size; 1151 |
1152 if (rsize & keg->uk_align) 1153 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); |
1154 if (rsize < keg->uk_slabsize / 256) 1155 rsize = keg->uk_slabsize / 256; |
1156 1157 keg->uk_rsize = rsize; |
1158 |
1159 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 1160 keg->uk_rsize < sizeof(struct pcpu), 1161 ("%s: size %u too large", __func__, keg->uk_rsize)); 1162 |
1163 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 1164 shsize = 0; 1165 } else if (keg->uk_flags & UMA_ZONE_REFCNT) { 1166 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */ 1167 shsize = sizeof(struct uma_slab_refcnt); 1168 } else { 1169 rsize += UMA_FRITM_SZ; /* Account for linkage */ 1170 shsize = sizeof(struct uma_slab); 1171 } 1172 |
1173 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize; 1174 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255, 1175 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); 1176 |
1177 memused = keg->uk_ipers * rsize + shsize; |
1178 wastedspace = keg->uk_slabsize - memused; |
1179 1180 /* 1181 * We can't do OFFPAGE if we're internal or if we've been 1182 * asked to not go to the VM for buckets. If we do this we 1183 * may end up going to the VM (kmem_map) for slabs which we 1184 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a 1185 * result of UMA_ZONE_VM, which clearly forbids it. 1186 */ 1187 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1188 (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 1189 return; 1190 |
1191 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) && 1192 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) { 1193 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize; 1194 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255, 1195 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers)); |
1196#ifdef UMA_DEBUG 1197 printf("UMA decided we need offpage slab headers for " 1198 "keg: %s, calculated wastedspace = %d, " 1199 "maximum wasted space allowed = %d, " 1200 "calculated ipers = %d, " 1201 "new wasted space = %d\n", keg->uk_name, wastedspace, |
1202 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers, 1203 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize); |
1204#endif 1205 keg->uk_flags |= UMA_ZONE_OFFPAGE; |
1206 } |
1207 1208 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) && 1209 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0) 1210 keg->uk_flags |= UMA_ZONE_HASH; |
1211} 1212 1213/* 1214 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do 1215 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 1216 * more complicated. 1217 * 1218 * Arguments 1219 * keg The keg we should initialize 1220 * 1221 * Returns 1222 * Nothing 1223 */ 1224static void 1225keg_large_init(uma_keg_t keg) 1226{ |
1227 1228 KASSERT(keg != NULL, ("Keg is null in keg_large_init")); 1229 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0, 1230 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg")); |
1231 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1232 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__)); |
1233 |
1234 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE); 1235 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE; |
1236 keg->uk_ipers = 1; 1237 keg->uk_rsize = keg->uk_size; 1238 1239 /* We can't do OFFPAGE if we're internal, bail out here. */ 1240 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) 1241 return; 1242 1243 keg->uk_flags |= UMA_ZONE_OFFPAGE; --- 4 unchanged lines hidden (view full) --- 1248static void 1249keg_cachespread_init(uma_keg_t keg) 1250{ 1251 int alignsize; 1252 int trailer; 1253 int pages; 1254 int rsize; 1255 |
1256 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0, 1257 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__)); 1258 |
1259 alignsize = keg->uk_align + 1; 1260 rsize = keg->uk_size; 1261 /* 1262 * We want one item to start on every align boundary in a page. To 1263 * do this we will span pages. We will also extend the item by the 1264 * size of align if it is an even multiple of align. Otherwise, it 1265 * would fall on the same boundary every time. 1266 */ 1267 if (rsize & keg->uk_align) 1268 rsize = (rsize & ~keg->uk_align) + alignsize; 1269 if ((rsize & alignsize) == 0) 1270 rsize += alignsize; 1271 trailer = rsize - keg->uk_size; 1272 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE; 1273 pages = MIN(pages, (128 * 1024) / PAGE_SIZE); 1274 keg->uk_rsize = rsize; 1275 keg->uk_ppera = pages; |
1276 keg->uk_slabsize = UMA_SLAB_SIZE; |
1277 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize; 1278 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB; 1279 KASSERT(keg->uk_ipers <= uma_max_ipers, 1280 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__, 1281 keg->uk_ipers)); 1282} 1283 1284/* --- 33 unchanged lines hidden (view full) --- 1318 keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1319 1320 if (arg->flags & UMA_ZONE_ZINIT) 1321 keg->uk_init = zero_init; 1322 1323 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC) 1324 keg->uk_flags |= UMA_ZONE_VTOSLAB; 1325 |
1326 if (arg->flags & UMA_ZONE_PCPU) 1327#ifdef SMP 1328 keg->uk_flags |= UMA_ZONE_OFFPAGE; 1329#else 1330 keg->uk_flags &= ~UMA_ZONE_PCPU; 1331#endif 1332 |
1333 /* 1334 * The +UMA_FRITM_SZ added to uk_size is to account for the 1335 * linkage that is added to the size in keg_small_init(). If 1336 * we don't account for this here then we may end up in 1337 * keg_small_init() with a calculated 'ipers' of 0. 1338 */ 1339 if (keg->uk_flags & UMA_ZONE_REFCNT) { 1340 if (keg->uk_flags & UMA_ZONE_CACHESPREAD) --- 61 unchanged lines hidden (view full) --- 1402 keg->uk_ipers * UMA_FRITMREF_SZ; 1403 else 1404 totsize = sizeof(struct uma_slab) + 1405 keg->uk_ipers * UMA_FRITM_SZ; 1406 1407 if (totsize & UMA_ALIGN_PTR) 1408 totsize = (totsize & ~UMA_ALIGN_PTR) + 1409 (UMA_ALIGN_PTR + 1); |
1410 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize; |
1411 1412 if (keg->uk_flags & UMA_ZONE_REFCNT) 1413 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt) 1414 + keg->uk_ipers * UMA_FRITMREF_SZ; 1415 else 1416 totsize = keg->uk_pgoff + sizeof(struct uma_slab) 1417 + keg->uk_ipers * UMA_FRITM_SZ; 1418 1419 /* 1420 * The only way the following is possible is if with our 1421 * UMA_ALIGN_PTR adjustments we are now bigger than 1422 * UMA_SLAB_SIZE. I haven't checked whether this is 1423 * mathematically possible for all cases, so we make 1424 * sure here anyway. 1425 */ |
1426 if (totsize > PAGE_SIZE * keg->uk_ppera) { |
1427 printf("zone %s ipers %d rsize %d size %d\n", 1428 zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1429 keg->uk_size); 1430 panic("UMA slab won't fit."); 1431 } 1432 } 1433 1434 if (keg->uk_flags & UMA_ZONE_HASH) --- 258 unchanged lines hidden (view full) --- 1693 * object with offpage slab header would have ipers = 1694 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is 1695 * 1 greater than what our byte-integer freelist index can 1696 * accomodate, but we know that this situation never occurs as 1697 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate 1698 * that we need to go to offpage slab headers. Or, if we do, 1699 * then we trap that condition below and panic in the INVARIANTS case. 1700 */ |
1701 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) - 1702 (UMA_SLAB_SIZE / UMA_MAX_WASTE); |
1703 totsize = wsize; 1704 objsize = UMA_SMALLEST_UNIT; 1705 while (totsize >= wsize) { 1706 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / 1707 (objsize + UMA_FRITM_SZ); 1708 totsize *= (UMA_FRITM_SZ + objsize); 1709 objsize++; 1710 } 1711 if (objsize > UMA_SMALLEST_UNIT) 1712 objsize--; 1713 uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64); 1714 |
1715 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - 1716 (UMA_SLAB_SIZE / UMA_MAX_WASTE); |
1717 totsize = wsize; 1718 objsize = UMA_SMALLEST_UNIT; 1719 while (totsize >= wsize) { 1720 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) / 1721 (objsize + UMA_FRITMREF_SZ); 1722 totsize *= (UMA_FRITMREF_SZ + objsize); 1723 objsize++; 1724 } --- 1733 unchanged lines hidden --- |