1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions

--- 34 unchanged lines hidden (view full) ---

43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 248084 2013-03-09 02:32:23Z attilio $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59

--- 700 unchanged lines hidden (view full) ---

760 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
761 obj);
762 }
763 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
764 zone_free_item(keg->uk_slabzone, slab, NULL,
765 SKIP_NONE, ZFREE_STATFREE);
766#ifdef UMA_DEBUG
767 printf("%s: Returning %d bytes.\n",
768 keg->uk_name, UMA_SLAB_SIZE * keg->uk_ppera);
769#endif
770 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags);
771 }
772}
773
774static void
775zone_drain_wait(uma_zone_t zone, int waitok)
776{
777
778 /*

--- 81 unchanged lines hidden (view full) ---

860 wait |= M_ZERO;
861 else
862 wait &= ~M_ZERO;
863
864 if (keg->uk_flags & UMA_ZONE_NODUMP)
865 wait |= M_NODUMP;
866
867 /* zone is passed for legacy reasons. */
868 mem = allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, &flags, wait);
869 if (mem == NULL) {
870 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
871 zone_free_item(keg->uk_slabzone, slab, NULL,
872 SKIP_NONE, ZFREE_STATFREE);
873 KEG_LOCK(keg);
874 return (NULL);
875 }
876

--- 45 unchanged lines hidden (view full) ---

922 obj = NULL;
923 for (i = 0; i < keg->uk_ppera; i++)
924 vsetobj((vm_offset_t)mem +
925 (i * PAGE_SIZE), obj);
926 }
927 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
928 zone_free_item(keg->uk_slabzone, slab,
929 NULL, SKIP_NONE, ZFREE_STATFREE);
930 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera,
931 flags);
932 KEG_LOCK(keg);
933 return (NULL);
934 }
935 }
936 KEG_LOCK(keg);
937
938 if (keg->uk_flags & UMA_ZONE_HASH)

--- 194 unchanged lines hidden (view full) ---

1133static void
1134keg_small_init(uma_keg_t keg)
1135{
1136 u_int rsize;
1137 u_int memused;
1138 u_int wastedspace;
1139 u_int shsize;
1140
1141 KASSERT(keg != NULL, ("Keg is null in keg_small_init"));
1142 rsize = keg->uk_size;
1143
1144 if (rsize < UMA_SMALLEST_UNIT)
1145 rsize = UMA_SMALLEST_UNIT;
1146 if (rsize & keg->uk_align)
1147 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1148
1149 keg->uk_rsize = rsize;
1150 keg->uk_ppera = 1;
1151
1152 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1153 shsize = 0;
1154 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1155 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */
1156 shsize = sizeof(struct uma_slab_refcnt);
1157 } else {
1158 rsize += UMA_FRITM_SZ; /* Account for linkage */
1159 shsize = sizeof(struct uma_slab);
1160 }
1161
1162 keg->uk_ipers = (UMA_SLAB_SIZE - shsize) / rsize;
1163 KASSERT(keg->uk_ipers != 0, ("keg_small_init: ipers is 0"));
1164 memused = keg->uk_ipers * rsize + shsize;
1165 wastedspace = UMA_SLAB_SIZE - memused;
1166
1167 /*
1168 * We can't do OFFPAGE if we're internal or if we've been
1169 * asked to not go to the VM for buckets. If we do this we
1170 * may end up going to the VM (kmem_map) for slabs which we
1171 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
1172 * result of UMA_ZONE_VM, which clearly forbids it.
1173 */
1174 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1175 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1176 return;
1177
1178 if ((wastedspace >= UMA_MAX_WASTE) &&
1179 (keg->uk_ipers < (UMA_SLAB_SIZE / keg->uk_rsize))) {
1180 keg->uk_ipers = UMA_SLAB_SIZE / keg->uk_rsize;
1181 KASSERT(keg->uk_ipers <= 255,
1182 ("keg_small_init: keg->uk_ipers too high!"));
1183#ifdef UMA_DEBUG
1184 printf("UMA decided we need offpage slab headers for "
1185 "keg: %s, calculated wastedspace = %d, "
1186 "maximum wasted space allowed = %d, "
1187 "calculated ipers = %d, "
1188 "new wasted space = %d\n", keg->uk_name, wastedspace,
1189 UMA_MAX_WASTE, keg->uk_ipers,
1190 UMA_SLAB_SIZE - keg->uk_ipers * keg->uk_rsize);
1191#endif
1192 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1193 if ((keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1194 keg->uk_flags |= UMA_ZONE_HASH;
1195 }
1196}
1197
1198/*
1199 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1200 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1201 * more complicated.
1202 *
1203 * Arguments
1204 * keg The keg we should initialize
1205 *
1206 * Returns
1207 * Nothing
1208 */
1209static void
1210keg_large_init(uma_keg_t keg)
1211{
1212 int pages;
1213
1214 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1215 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1216 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1217
1218 pages = keg->uk_size / UMA_SLAB_SIZE;
1219
1220 /* Account for remainder */
1221 if ((pages * UMA_SLAB_SIZE) < keg->uk_size)
1222 pages++;
1223
1224 keg->uk_ppera = pages;
1225 keg->uk_ipers = 1;
1226 keg->uk_rsize = keg->uk_size;
1227
1228 /* We can't do OFFPAGE if we're internal, bail out here. */
1229 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1230 return;
1231
1232 keg->uk_flags |= UMA_ZONE_OFFPAGE;

--- 4 unchanged lines hidden (view full) ---

1237static void
1238keg_cachespread_init(uma_keg_t keg)
1239{
1240 int alignsize;
1241 int trailer;
1242 int pages;
1243 int rsize;
1244
1245 alignsize = keg->uk_align + 1;
1246 rsize = keg->uk_size;
1247 /*
1248 * We want one item to start on every align boundary in a page. To
1249 * do this we will span pages. We will also extend the item by the
1250 * size of align if it is an even multiple of align. Otherwise, it
1251 * would fall on the same boundary every time.
1252 */
1253 if (rsize & keg->uk_align)
1254 rsize = (rsize & ~keg->uk_align) + alignsize;
1255 if ((rsize & alignsize) == 0)
1256 rsize += alignsize;
1257 trailer = rsize - keg->uk_size;
1258 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1259 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1260 keg->uk_rsize = rsize;
1261 keg->uk_ppera = pages;
1262 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1263 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1264 KASSERT(keg->uk_ipers <= uma_max_ipers,
1265 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1266 keg->uk_ipers));
1267}
1268
1269/*

--- 33 unchanged lines hidden (view full) ---

1303 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1304
1305 if (arg->flags & UMA_ZONE_ZINIT)
1306 keg->uk_init = zero_init;
1307
1308 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1309 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1310
1311 /*
1312 * The +UMA_FRITM_SZ added to uk_size is to account for the
1313 * linkage that is added to the size in keg_small_init(). If
1314 * we don't account for this here then we may end up in
1315 * keg_small_init() with a calculated 'ipers' of 0.
1316 */
1317 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1318 if (keg->uk_flags & UMA_ZONE_CACHESPREAD)

--- 61 unchanged lines hidden (view full) ---

1380 keg->uk_ipers * UMA_FRITMREF_SZ;
1381 else
1382 totsize = sizeof(struct uma_slab) +
1383 keg->uk_ipers * UMA_FRITM_SZ;
1384
1385 if (totsize & UMA_ALIGN_PTR)
1386 totsize = (totsize & ~UMA_ALIGN_PTR) +
1387 (UMA_ALIGN_PTR + 1);
1388 keg->uk_pgoff = (UMA_SLAB_SIZE * keg->uk_ppera) - totsize;
1389
1390 if (keg->uk_flags & UMA_ZONE_REFCNT)
1391 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt)
1392 + keg->uk_ipers * UMA_FRITMREF_SZ;
1393 else
1394 totsize = keg->uk_pgoff + sizeof(struct uma_slab)
1395 + keg->uk_ipers * UMA_FRITM_SZ;
1396
1397 /*
1398 * The only way the following is possible is if with our
1399 * UMA_ALIGN_PTR adjustments we are now bigger than
1400 * UMA_SLAB_SIZE. I haven't checked whether this is
1401 * mathematically possible for all cases, so we make
1402 * sure here anyway.
1403 */
1404 if (totsize > UMA_SLAB_SIZE * keg->uk_ppera) {
1405 printf("zone %s ipers %d rsize %d size %d\n",
1406 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1407 keg->uk_size);
1408 panic("UMA slab won't fit.");
1409 }
1410 }
1411
1412 if (keg->uk_flags & UMA_ZONE_HASH)

--- 258 unchanged lines hidden (view full) ---

1671 * object with offpage slab header would have ipers =
1672 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is
1673 * 1 greater than what our byte-integer freelist index can
1674 * accomodate, but we know that this situation never occurs as
1675 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate
1676 * that we need to go to offpage slab headers. Or, if we do,
1677 * then we trap that condition below and panic in the INVARIANTS case.
1678 */
1679 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) - UMA_MAX_WASTE;
1680 totsize = wsize;
1681 objsize = UMA_SMALLEST_UNIT;
1682 while (totsize >= wsize) {
1683 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) /
1684 (objsize + UMA_FRITM_SZ);
1685 totsize *= (UMA_FRITM_SZ + objsize);
1686 objsize++;
1687 }
1688 if (objsize > UMA_SMALLEST_UNIT)
1689 objsize--;
1690 uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64);
1691
1692 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - UMA_MAX_WASTE;
1693 totsize = wsize;
1694 objsize = UMA_SMALLEST_UNIT;
1695 while (totsize >= wsize) {
1696 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) /
1697 (objsize + UMA_FRITMREF_SZ);
1698 totsize *= (UMA_FRITMREF_SZ + objsize);
1699 objsize++;
1700 }

--- 1733 unchanged lines hidden ---