1/*-
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249264 2013-04-08 19:10:45Z glebius $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 700 unchanged lines hidden (view full) ---
760 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
761 obj);
762 }
763 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
764 zone_free_item(keg->uk_slabzone, slab, NULL,
765 SKIP_NONE, ZFREE_STATFREE);
766#ifdef UMA_DEBUG
767 printf("%s: Returning %d bytes.\n",
768 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
769#endif
770 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
771 }
772}
773
774static void
775zone_drain_wait(uma_zone_t zone, int waitok)
776{
777
778 /*
--- 81 unchanged lines hidden (view full) ---
860 wait |= M_ZERO;
861 else
862 wait &= ~M_ZERO;
863
864 if (keg->uk_flags & UMA_ZONE_NODUMP)
865 wait |= M_NODUMP;
866
867 /* zone is passed for legacy reasons. */
868 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
869 if (mem == NULL) {
870 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
871 zone_free_item(keg->uk_slabzone, slab, NULL,
872 SKIP_NONE, ZFREE_STATFREE);
873 KEG_LOCK(keg);
874 return (NULL);
875 }
876
--- 45 unchanged lines hidden (view full) ---
922 obj = NULL;
923 for (i = 0; i < keg->uk_ppera; i++)
924 vsetobj((vm_offset_t)mem +
925 (i * PAGE_SIZE), obj);
926 }
927 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
928 zone_free_item(keg->uk_slabzone, slab,
929 NULL, SKIP_NONE, ZFREE_STATFREE);
930 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
931 flags);
932 KEG_LOCK(keg);
933 return (NULL);
934 }
935 }
936 KEG_LOCK(keg);
937
938 if (keg->uk_flags & UMA_ZONE_HASH)
--- 194 unchanged lines hidden (view full) ---
1133static void
1134keg_small_init(uma_keg_t keg)
1135{
1136 u_int rsize;
1137 u_int memused;
1138 u_int wastedspace;
1139 u_int shsize;
1140
1141 if (keg->uk_flags & UMA_ZONE_PCPU) {
1142 keg->uk_slabsize = sizeof(struct pcpu);
1143 keg->uk_ppera = howmany(mp_ncpus * sizeof(struct pcpu),
1144 PAGE_SIZE);
1145 } else {
1146 keg->uk_slabsize = UMA_SLAB_SIZE;
1147 keg->uk_ppera = 1;
1148 }
1149
1150 rsize = keg->uk_size;
1151
1152 if (rsize & keg->uk_align)
1153 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1154 if (rsize < keg->uk_slabsize / 256)
1155 rsize = keg->uk_slabsize / 256;
1156
1157 keg->uk_rsize = rsize;
1158
1159 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1160 keg->uk_rsize < sizeof(struct pcpu),
1161 ("%s: size %u too large", __func__, keg->uk_rsize));
1162
1163 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1164 shsize = 0;
1165 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1166 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */
1167 shsize = sizeof(struct uma_slab_refcnt);
1168 } else {
1169 rsize += UMA_FRITM_SZ; /* Account for linkage */
1170 shsize = sizeof(struct uma_slab);
1171 }
1172
1173 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1174 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255,
1175 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1176
1177 memused = keg->uk_ipers * rsize + shsize;
1178 wastedspace = keg->uk_slabsize - memused;
1179
1180 /*
1181 * We can't do OFFPAGE if we're internal or if we've been
1182 * asked to not go to the VM for buckets. If we do this we
1183 * may end up going to the VM (kmem_map) for slabs which we
1184 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
1185 * result of UMA_ZONE_VM, which clearly forbids it.
1186 */
1187 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1188 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1189 return;
1190
1191 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1192 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1193 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1194 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255,
1195 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1196#ifdef UMA_DEBUG
1197 printf("UMA decided we need offpage slab headers for "
1198 "keg: %s, calculated wastedspace = %d, "
1199 "maximum wasted space allowed = %d, "
1200 "calculated ipers = %d, "
1201 "new wasted space = %d\n", keg->uk_name, wastedspace,
1202 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1203 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1204#endif
1205 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1206 }
1207
1208 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1209 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1210 keg->uk_flags |= UMA_ZONE_HASH;
1211}
1212
1213/*
1214 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1215 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1216 * more complicated.
1217 *
1218 * Arguments
1219 * keg The keg we should initialize
1220 *
1221 * Returns
1222 * Nothing
1223 */
1224static void
1225keg_large_init(uma_keg_t keg)
1226{
1227
1228 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1229 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1230 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1231 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1232 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
1233
1234 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1235 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1236 keg->uk_ipers = 1;
1237 keg->uk_rsize = keg->uk_size;
1238
1239 /* We can't do OFFPAGE if we're internal, bail out here. */
1240 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1241 return;
1242
1243 keg->uk_flags |= UMA_ZONE_OFFPAGE;
--- 4 unchanged lines hidden (view full) ---
1248static void
1249keg_cachespread_init(uma_keg_t keg)
1250{
1251 int alignsize;
1252 int trailer;
1253 int pages;
1254 int rsize;
1255
1256 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1257 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1258
1259 alignsize = keg->uk_align + 1;
1260 rsize = keg->uk_size;
1261 /*
1262 * We want one item to start on every align boundary in a page. To
1263 * do this we will span pages. We will also extend the item by the
1264 * size of align if it is an even multiple of align. Otherwise, it
1265 * would fall on the same boundary every time.
1266 */
1267 if (rsize & keg->uk_align)
1268 rsize = (rsize & ~keg->uk_align) + alignsize;
1269 if ((rsize & alignsize) == 0)
1270 rsize += alignsize;
1271 trailer = rsize - keg->uk_size;
1272 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1273 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1274 keg->uk_rsize = rsize;
1275 keg->uk_ppera = pages;
1276 keg->uk_slabsize = UMA_SLAB_SIZE;
1277 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1278 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1279 KASSERT(keg->uk_ipers <= uma_max_ipers,
1280 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1281 keg->uk_ipers));
1282}
1283
1284/*
--- 33 unchanged lines hidden (view full) ---
1318 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1319
1320 if (arg->flags & UMA_ZONE_ZINIT)
1321 keg->uk_init = zero_init;
1322
1323 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1324 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1325
1326 if (arg->flags & UMA_ZONE_PCPU)
1327#ifdef SMP
1328 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1329#else
1330 keg->uk_flags &= ~UMA_ZONE_PCPU;
1331#endif
1332
1333 /*
1334 * The +UMA_FRITM_SZ added to uk_size is to account for the
1335 * linkage that is added to the size in keg_small_init(). If
1336 * we don't account for this here then we may end up in
1337 * keg_small_init() with a calculated 'ipers' of 0.
1338 */
1339 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1340 if (keg->uk_flags & UMA_ZONE_CACHESPREAD)
--- 61 unchanged lines hidden (view full) ---
1402 keg->uk_ipers * UMA_FRITMREF_SZ;
1403 else
1404 totsize = sizeof(struct uma_slab) +
1405 keg->uk_ipers * UMA_FRITM_SZ;
1406
1407 if (totsize & UMA_ALIGN_PTR)
1408 totsize = (totsize & ~UMA_ALIGN_PTR) +
1409 (UMA_ALIGN_PTR + 1);
1410 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1411
1412 if (keg->uk_flags & UMA_ZONE_REFCNT)
1413 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt)
1414 + keg->uk_ipers * UMA_FRITMREF_SZ;
1415 else
1416 totsize = keg->uk_pgoff + sizeof(struct uma_slab)
1417 + keg->uk_ipers * UMA_FRITM_SZ;
1418
1419 /*
1420 * The only way the following is possible is if with our
1421 * UMA_ALIGN_PTR adjustments we are now bigger than
1422 * UMA_SLAB_SIZE. I haven't checked whether this is
1423 * mathematically possible for all cases, so we make
1424 * sure here anyway.
1425 */
1426 if (totsize > PAGE_SIZE * keg->uk_ppera) {
1427 printf("zone %s ipers %d rsize %d size %d\n",
1428 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1429 keg->uk_size);
1430 panic("UMA slab won't fit.");
1431 }
1432 }
1433
1434 if (keg->uk_flags & UMA_ZONE_HASH)
--- 258 unchanged lines hidden (view full) ---
1693 * object with offpage slab header would have ipers =
1694 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is
1695 * 1 greater than what our byte-integer freelist index can
1696 * accomodate, but we know that this situation never occurs as
1697 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate
1698 * that we need to go to offpage slab headers. Or, if we do,
1699 * then we trap that condition below and panic in the INVARIANTS case.
1700 */
1701 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) -
1702 (UMA_SLAB_SIZE / UMA_MAX_WASTE);
1703 totsize = wsize;
1704 objsize = UMA_SMALLEST_UNIT;
1705 while (totsize >= wsize) {
1706 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) /
1707 (objsize + UMA_FRITM_SZ);
1708 totsize *= (UMA_FRITM_SZ + objsize);
1709 objsize++;
1710 }
1711 if (objsize > UMA_SMALLEST_UNIT)
1712 objsize--;
1713 uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64);
1714
1715 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1716 (UMA_SLAB_SIZE / UMA_MAX_WASTE);
1717 totsize = wsize;
1718 objsize = UMA_SMALLEST_UNIT;
1719 while (totsize >= wsize) {
1720 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) /
1721 (objsize + UMA_FRITMREF_SZ);
1722 totsize *= (UMA_FRITMREF_SZ + objsize);
1723 objsize++;
1724 }
--- 1733 unchanged lines hidden ---
2 * Copyright (c) 2002-2005, 2009 Jeffrey Roberson <jeff@FreeBSD.org>
3 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org>
4 * Copyright (c) 2004-2006 Robert N. M. Watson
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
--- 34 unchanged lines hidden (view full) ---
43
44/*
45 * TODO:
46 * - Improve memory usage for large allocations
47 * - Investigate cache size adjustments
48 */
49
50#include <sys/cdefs.h>
51__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 249264 2013-04-08 19:10:45Z glebius $");
52
53/* I should really use ktr.. */
54/*
55#define UMA_DEBUG 1
56#define UMA_DEBUG_ALLOC 1
57#define UMA_DEBUG_ALLOC_1 1
58*/
59
--- 700 unchanged lines hidden (view full) ---
760 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE),
761 obj);
762 }
763 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
764 zone_free_item(keg->uk_slabzone, slab, NULL,
765 SKIP_NONE, ZFREE_STATFREE);
766#ifdef UMA_DEBUG
767 printf("%s: Returning %d bytes.\n",
768 keg->uk_name, PAGE_SIZE * keg->uk_ppera);
769#endif
770 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
771 }
772}
773
774static void
775zone_drain_wait(uma_zone_t zone, int waitok)
776{
777
778 /*
--- 81 unchanged lines hidden (view full) ---
860 wait |= M_ZERO;
861 else
862 wait &= ~M_ZERO;
863
864 if (keg->uk_flags & UMA_ZONE_NODUMP)
865 wait |= M_NODUMP;
866
867 /* zone is passed for legacy reasons. */
868 mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
869 if (mem == NULL) {
870 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
871 zone_free_item(keg->uk_slabzone, slab, NULL,
872 SKIP_NONE, ZFREE_STATFREE);
873 KEG_LOCK(keg);
874 return (NULL);
875 }
876
--- 45 unchanged lines hidden (view full) ---
922 obj = NULL;
923 for (i = 0; i < keg->uk_ppera; i++)
924 vsetobj((vm_offset_t)mem +
925 (i * PAGE_SIZE), obj);
926 }
927 if (keg->uk_flags & UMA_ZONE_OFFPAGE)
928 zone_free_item(keg->uk_slabzone, slab,
929 NULL, SKIP_NONE, ZFREE_STATFREE);
930 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
931 flags);
932 KEG_LOCK(keg);
933 return (NULL);
934 }
935 }
936 KEG_LOCK(keg);
937
938 if (keg->uk_flags & UMA_ZONE_HASH)
--- 194 unchanged lines hidden (view full) ---
1133static void
1134keg_small_init(uma_keg_t keg)
1135{
1136 u_int rsize;
1137 u_int memused;
1138 u_int wastedspace;
1139 u_int shsize;
1140
1141 if (keg->uk_flags & UMA_ZONE_PCPU) {
1142 keg->uk_slabsize = sizeof(struct pcpu);
1143 keg->uk_ppera = howmany(mp_ncpus * sizeof(struct pcpu),
1144 PAGE_SIZE);
1145 } else {
1146 keg->uk_slabsize = UMA_SLAB_SIZE;
1147 keg->uk_ppera = 1;
1148 }
1149
1150 rsize = keg->uk_size;
1151
1152 if (rsize & keg->uk_align)
1153 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1);
1154 if (rsize < keg->uk_slabsize / 256)
1155 rsize = keg->uk_slabsize / 256;
1156
1157 keg->uk_rsize = rsize;
1158
1159 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
1160 keg->uk_rsize < sizeof(struct pcpu),
1161 ("%s: size %u too large", __func__, keg->uk_rsize));
1162
1163 if (keg->uk_flags & UMA_ZONE_OFFPAGE) {
1164 shsize = 0;
1165 } else if (keg->uk_flags & UMA_ZONE_REFCNT) {
1166 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */
1167 shsize = sizeof(struct uma_slab_refcnt);
1168 } else {
1169 rsize += UMA_FRITM_SZ; /* Account for linkage */
1170 shsize = sizeof(struct uma_slab);
1171 }
1172
1173 keg->uk_ipers = (keg->uk_slabsize - shsize) / rsize;
1174 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255,
1175 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1176
1177 memused = keg->uk_ipers * rsize + shsize;
1178 wastedspace = keg->uk_slabsize - memused;
1179
1180 /*
1181 * We can't do OFFPAGE if we're internal or if we've been
1182 * asked to not go to the VM for buckets. If we do this we
1183 * may end up going to the VM (kmem_map) for slabs which we
1184 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a
1185 * result of UMA_ZONE_VM, which clearly forbids it.
1186 */
1187 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) ||
1188 (keg->uk_flags & UMA_ZFLAG_CACHEONLY))
1189 return;
1190
1191 if ((wastedspace >= keg->uk_slabsize / UMA_MAX_WASTE) &&
1192 (keg->uk_ipers < (keg->uk_slabsize / keg->uk_rsize))) {
1193 keg->uk_ipers = keg->uk_slabsize / keg->uk_rsize;
1194 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= 255,
1195 ("%s: keg->uk_ipers %u", __func__, keg->uk_ipers));
1196#ifdef UMA_DEBUG
1197 printf("UMA decided we need offpage slab headers for "
1198 "keg: %s, calculated wastedspace = %d, "
1199 "maximum wasted space allowed = %d, "
1200 "calculated ipers = %d, "
1201 "new wasted space = %d\n", keg->uk_name, wastedspace,
1202 keg->uk_slabsize / UMA_MAX_WASTE, keg->uk_ipers,
1203 keg->uk_slabsize - keg->uk_ipers * keg->uk_rsize);
1204#endif
1205 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1206 }
1207
1208 if ((keg->uk_flags & UMA_ZONE_OFFPAGE) &&
1209 (keg->uk_flags & UMA_ZONE_VTOSLAB) == 0)
1210 keg->uk_flags |= UMA_ZONE_HASH;
1211}
1212
1213/*
1214 * Finish creating a large (> UMA_SLAB_SIZE) uma kegs. Just give in and do
1215 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be
1216 * more complicated.
1217 *
1218 * Arguments
1219 * keg The keg we should initialize
1220 *
1221 * Returns
1222 * Nothing
1223 */
1224static void
1225keg_large_init(uma_keg_t keg)
1226{
1227
1228 KASSERT(keg != NULL, ("Keg is null in keg_large_init"));
1229 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0,
1230 ("keg_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY keg"));
1231 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1232 ("%s: Cannot large-init a UMA_ZONE_PCPU keg", __func__));
1233
1234 keg->uk_ppera = howmany(keg->uk_size, PAGE_SIZE);
1235 keg->uk_slabsize = keg->uk_ppera * PAGE_SIZE;
1236 keg->uk_ipers = 1;
1237 keg->uk_rsize = keg->uk_size;
1238
1239 /* We can't do OFFPAGE if we're internal, bail out here. */
1240 if (keg->uk_flags & UMA_ZFLAG_INTERNAL)
1241 return;
1242
1243 keg->uk_flags |= UMA_ZONE_OFFPAGE;
--- 4 unchanged lines hidden (view full) ---
1248static void
1249keg_cachespread_init(uma_keg_t keg)
1250{
1251 int alignsize;
1252 int trailer;
1253 int pages;
1254 int rsize;
1255
1256 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0,
1257 ("%s: Cannot cachespread-init a UMA_ZONE_PCPU keg", __func__));
1258
1259 alignsize = keg->uk_align + 1;
1260 rsize = keg->uk_size;
1261 /*
1262 * We want one item to start on every align boundary in a page. To
1263 * do this we will span pages. We will also extend the item by the
1264 * size of align if it is an even multiple of align. Otherwise, it
1265 * would fall on the same boundary every time.
1266 */
1267 if (rsize & keg->uk_align)
1268 rsize = (rsize & ~keg->uk_align) + alignsize;
1269 if ((rsize & alignsize) == 0)
1270 rsize += alignsize;
1271 trailer = rsize - keg->uk_size;
1272 pages = (rsize * (PAGE_SIZE / alignsize)) / PAGE_SIZE;
1273 pages = MIN(pages, (128 * 1024) / PAGE_SIZE);
1274 keg->uk_rsize = rsize;
1275 keg->uk_ppera = pages;
1276 keg->uk_slabsize = UMA_SLAB_SIZE;
1277 keg->uk_ipers = ((pages * PAGE_SIZE) + trailer) / rsize;
1278 keg->uk_flags |= UMA_ZONE_OFFPAGE | UMA_ZONE_VTOSLAB;
1279 KASSERT(keg->uk_ipers <= uma_max_ipers,
1280 ("%s: keg->uk_ipers too high(%d) increase max_ipers", __func__,
1281 keg->uk_ipers));
1282}
1283
1284/*
--- 33 unchanged lines hidden (view full) ---
1318 keg->uk_flags |= UMA_ZFLAG_CACHEONLY;
1319
1320 if (arg->flags & UMA_ZONE_ZINIT)
1321 keg->uk_init = zero_init;
1322
1323 if (arg->flags & UMA_ZONE_REFCNT || arg->flags & UMA_ZONE_MALLOC)
1324 keg->uk_flags |= UMA_ZONE_VTOSLAB;
1325
1326 if (arg->flags & UMA_ZONE_PCPU)
1327#ifdef SMP
1328 keg->uk_flags |= UMA_ZONE_OFFPAGE;
1329#else
1330 keg->uk_flags &= ~UMA_ZONE_PCPU;
1331#endif
1332
1333 /*
1334 * The +UMA_FRITM_SZ added to uk_size is to account for the
1335 * linkage that is added to the size in keg_small_init(). If
1336 * we don't account for this here then we may end up in
1337 * keg_small_init() with a calculated 'ipers' of 0.
1338 */
1339 if (keg->uk_flags & UMA_ZONE_REFCNT) {
1340 if (keg->uk_flags & UMA_ZONE_CACHESPREAD)
--- 61 unchanged lines hidden (view full) ---
1402 keg->uk_ipers * UMA_FRITMREF_SZ;
1403 else
1404 totsize = sizeof(struct uma_slab) +
1405 keg->uk_ipers * UMA_FRITM_SZ;
1406
1407 if (totsize & UMA_ALIGN_PTR)
1408 totsize = (totsize & ~UMA_ALIGN_PTR) +
1409 (UMA_ALIGN_PTR + 1);
1410 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - totsize;
1411
1412 if (keg->uk_flags & UMA_ZONE_REFCNT)
1413 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt)
1414 + keg->uk_ipers * UMA_FRITMREF_SZ;
1415 else
1416 totsize = keg->uk_pgoff + sizeof(struct uma_slab)
1417 + keg->uk_ipers * UMA_FRITM_SZ;
1418
1419 /*
1420 * The only way the following is possible is if with our
1421 * UMA_ALIGN_PTR adjustments we are now bigger than
1422 * UMA_SLAB_SIZE. I haven't checked whether this is
1423 * mathematically possible for all cases, so we make
1424 * sure here anyway.
1425 */
1426 if (totsize > PAGE_SIZE * keg->uk_ppera) {
1427 printf("zone %s ipers %d rsize %d size %d\n",
1428 zone->uz_name, keg->uk_ipers, keg->uk_rsize,
1429 keg->uk_size);
1430 panic("UMA slab won't fit.");
1431 }
1432 }
1433
1434 if (keg->uk_flags & UMA_ZONE_HASH)
--- 258 unchanged lines hidden (view full) ---
1693 * object with offpage slab header would have ipers =
1694 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is
1695 * 1 greater than what our byte-integer freelist index can
1696 * accomodate, but we know that this situation never occurs as
1697 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate
1698 * that we need to go to offpage slab headers. Or, if we do,
1699 * then we trap that condition below and panic in the INVARIANTS case.
1700 */
1701 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) -
1702 (UMA_SLAB_SIZE / UMA_MAX_WASTE);
1703 totsize = wsize;
1704 objsize = UMA_SMALLEST_UNIT;
1705 while (totsize >= wsize) {
1706 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) /
1707 (objsize + UMA_FRITM_SZ);
1708 totsize *= (UMA_FRITM_SZ + objsize);
1709 objsize++;
1710 }
1711 if (objsize > UMA_SMALLEST_UNIT)
1712 objsize--;
1713 uma_max_ipers = MAX(UMA_SLAB_SIZE / objsize, 64);
1714
1715 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) -
1716 (UMA_SLAB_SIZE / UMA_MAX_WASTE);
1717 totsize = wsize;
1718 objsize = UMA_SMALLEST_UNIT;
1719 while (totsize >= wsize) {
1720 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) /
1721 (objsize + UMA_FRITMREF_SZ);
1722 totsize *= (UMA_FRITMREF_SZ + objsize);
1723 objsize++;
1724 }
--- 1733 unchanged lines hidden ---