Lines Matching refs:keg
115 * This is the zone and keg from which all zones are spawned.
171 * Mutex for global lists: uma_kegs, uma_cachezones, and the per-keg list of
225 uma_keg_t keg;
298 static void keg_drain(uma_keg_t keg, int domain);
324 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
348 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg);
350 static bool uma_dbg_kskip(uma_keg_t keg, void *mem);
1022 uma_keg_t keg;
1028 keg = zone->uz_keg;
1034 KEG_LOCK(keg, 0);
1035 pages = keg->uk_domain[0].ud_pages;
1038 * Expand the keg hash table.
1044 if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) {
1051 * while the keg lock is held will lead to deadlock.
1055 KEG_UNLOCK(keg, 0);
1057 KEG_LOCK(keg, 0);
1059 if (hash_expand(&keg->uk_hash, &newhash)) {
1060 oldhash = keg->uk_hash;
1061 keg->uk_hash = newhash;
1065 KEG_UNLOCK(keg, 0);
1070 KEG_UNLOCK(keg, 0);
1437 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start)
1443 CTR4(KTR_UMA, "keg_free_slab keg %s(%p) slab %p, returning %d bytes",
1444 keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera);
1446 mem = slab_data(slab, keg);
1449 if (keg->uk_fini != NULL) {
1460 if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) ||
1461 keg->uk_fini != trash_fini)
1463 keg->uk_fini(slab_item(slab, keg, i), keg->uk_size);
1465 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE)
1466 zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab),
1468 keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera, flags);
1469 uma_total_dec(PAGE_SIZE * keg->uk_ppera);
1473 keg_drain_domain(uma_keg_t keg, int domain)
1480 dom = &keg->uk_domain[domain];
1484 keg->uk_name, keg, domain, dom->ud_free_items);
1486 KEG_LOCK(keg, domain);
1493 partial = dom->ud_free_items - dom->ud_free_slabs * keg->uk_ipers;
1494 if (partial < keg->uk_reserve) {
1496 howmany(keg->uk_reserve - partial, keg->uk_ipers));
1516 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) {
1518 UMA_HASH_REMOVE(&keg->uk_hash, slab);
1520 dom->ud_free_items -= stofree * keg->uk_ipers;
1522 dom->ud_pages -= stofree * keg->uk_ppera;
1523 KEG_UNLOCK(keg, domain);
1526 keg_free_slab(keg, slab, keg->uk_ipers);
1530 * Frees pages from a keg back to the system. This is done on demand from
1536 keg_drain(uma_keg_t keg, int domain)
1540 if ((keg->uk_flags & UMA_ZONE_NOFREE) != 0)
1543 keg_drain_domain(keg, domain);
1546 keg_drain_domain(keg, i);
1598 * Allocate a new slab for a keg and inserts it into the partial slab list.
1599 * The keg should be unlocked on entry. If the allocation succeeds it will
1611 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags,
1625 allocf = keg->uk_allocf;
1628 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) {
1630 hslab = zone_alloc_item(slabzone(keg->uk_ipers), NULL,
1644 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0)
1649 if (keg->uk_flags & UMA_ZONE_NODUMP)
1653 size = keg->uk_ppera * PAGE_SIZE;
1656 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE)
1657 zone_free_item(slabzone(keg->uk_ipers),
1664 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0)
1668 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE))
1669 slab = (uma_slab_t )(mem + keg->uk_pgoff);
1673 if (keg->uk_flags & UMA_ZFLAG_VTOSLAB)
1674 for (i = 0; i < keg->uk_ppera; i++)
1678 slab->us_freecount = keg->uk_ipers;
1682 BIT_FILL(keg->uk_ipers, &slab->us_free);
1684 BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg));
1687 if (keg->uk_init != NULL) {
1688 for (i = 0; i < keg->uk_ipers; i++)
1689 if (keg->uk_init(slab_item(slab, keg, i),
1690 keg->uk_size, flags) != 0)
1692 if (i != keg->uk_ipers) {
1693 keg_free_slab(keg, slab, i);
1697 KEG_LOCK(keg, domain);
1700 slab, keg->uk_name, keg);
1702 if (keg->uk_flags & UMA_ZFLAG_HASH)
1703 UMA_HASH_INSERT(&keg->uk_hash, slab, mem);
1710 dom = &keg->uk_domain[domain];
1712 dom->ud_pages += keg->uk_ppera;
1713 dom->ud_free_items += keg->uk_ipers;
1885 uma_keg_t keg;
1888 keg = zone->uz_keg;
1916 zkva = keg->uk_kva +
1917 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes));
2016 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg)
2019 return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers)));
2083 keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt,
2097 kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize,
2109 * Determine the format of a uma keg. This determines where the slab header
2113 * keg The zone we should initialize
2119 keg_layout(uma_keg_t keg)
2130 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 ||
2131 (keg->uk_size <= UMA_PCPU_ALLOC_SIZE &&
2132 (keg->uk_flags & UMA_ZONE_CACHESPREAD) == 0),
2133 ("%s: cannot configure for PCPU: keg=%s, size=%u, flags=0x%b",
2134 __func__, keg->uk_name, keg->uk_size, keg->uk_flags,
2136 KASSERT((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) == 0 ||
2137 (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0,
2138 ("%s: incompatible flags 0x%b", __func__, keg->uk_flags,
2141 alignsize = keg->uk_align + 1;
2148 rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT);
2151 if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) {
2170 slabsize = round_page(keg->uk_size);
2173 /* Build a list of all of the available formats for this keg. */
2177 if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0)
2194 if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) != 0)
2207 i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize);
2208 KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u",
2209 keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize,
2213 round_page(rsize * (i - 1) + keg->uk_size);
2221 keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp);
2227 CTR6(KTR_UMA, "keg %s layout: format %#x "
2229 keg->uk_name, kl.format, kl.ipers, rsize,
2239 (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0)
2244 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0)
2247 keg->uk_rsize = rsize;
2248 keg->uk_ipers = kl.ipers;
2249 keg->uk_ppera = pages;
2250 keg->uk_flags |= kl.format;
2257 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 ||
2258 (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) {
2259 if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0)
2260 keg->uk_flags |= UMA_ZFLAG_HASH;
2262 keg->uk_flags |= UMA_ZFLAG_VTOSLAB;
2265 CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u",
2266 __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers,
2268 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE,
2269 ("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__,
2270 keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize,
2271 keg->uk_ipers, pages));
2276 * the keg onto the global keg list.
2285 uma_keg_t keg = mem;
2289 bzero(keg, size);
2290 keg->uk_size = arg->size;
2291 keg->uk_init = arg->uminit;
2292 keg->uk_fini = arg->fini;
2293 keg->uk_align = arg->align;
2294 keg->uk_reserve = 0;
2295 keg->uk_flags = arg->flags;
2302 keg->uk_dr.dr_policy = DOMAINSET_RR();
2303 keg->uk_dr.dr_iter = 0;
2306 * The primary zone is passed to us at keg-creation time.
2309 keg->uk_name = zone->uz_name;
2312 keg->uk_init = zero_init;
2315 keg->uk_flags |= UMA_ZFLAG_VTOSLAB;
2318 keg->uk_flags &= ~UMA_ZONE_PCPU;
2321 keg_layout(keg);
2330 if ((keg->uk_flags &
2332 keg->uk_flags |= UMA_ZONE_FIRSTTOUCH;
2333 else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0)
2334 keg->uk_flags |= UMA_ZONE_ROUNDROBIN;
2342 if (keg->uk_ppera == 1)
2343 keg->uk_allocf = uma_small_alloc;
2347 keg->uk_allocf = startup_alloc;
2348 else if (keg->uk_flags & UMA_ZONE_PCPU)
2349 keg->uk_allocf = pcpu_page_alloc;
2350 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1)
2351 keg->uk_allocf = contig_alloc;
2353 keg->uk_allocf = page_alloc;
2355 if (keg->uk_ppera == 1)
2356 keg->uk_freef = uma_small_free;
2359 if (keg->uk_flags & UMA_ZONE_PCPU)
2360 keg->uk_freef = pcpu_page_free;
2362 keg->uk_freef = page_free;
2365 * Initialize keg's locks.
2368 KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS));
2375 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) {
2378 shsize = slab_sizeof(keg->uk_ipers);
2379 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize;
2387 KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera,
2389 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size));
2392 if (keg->uk_flags & UMA_ZFLAG_HASH)
2393 hash_alloc(&keg->uk_hash, 0);
2395 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone);
2397 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link);
2400 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link);
2408 uma_keg_t keg;
2412 KEG_GET(zone, keg);
2414 if (keg->uk_allocf == startup_alloc) {
2416 if (keg->uk_flags & UMA_ZONE_PCPU)
2417 keg->uk_allocf = pcpu_page_alloc;
2418 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 &&
2419 keg->uk_ppera > 1)
2420 keg->uk_allocf = contig_alloc;
2422 keg->uk_allocf = page_alloc;
2441 uma_keg_t keg;
2486 * keg if present.
2493 "keg", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "");
2494 keg = zone->uz_keg;
2497 "name", CTLFLAG_RD, keg->uk_name, "Keg name");
2499 "rsize", CTLFLAG_RD, &keg->uk_rsize, 0,
2502 "ppera", CTLFLAG_RD, &keg->uk_ppera, 0,
2505 "ipers", CTLFLAG_RD, &keg->uk_ipers, 0,
2508 "align", CTLFLAG_RD, &keg->uk_align, 0,
2511 "reserve", CTLFLAG_RD, &keg->uk_reserve, 0,
2515 keg, 0, sysctl_handle_uma_slab_efficiency, "I",
2520 dom = &keg->uk_domain[i];
2661 uma_keg_t keg;
2725 * Use the regular zone/keg/slab allocator.
2730 keg = arg->keg;
2735 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
2741 LIST_FOREACH(z, &keg->uk_zones, uz_link) {
2749 } else if (keg == NULL) {
2750 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini,
2764 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg,
2770 /* Inherit properties from the keg. */
2771 zone->uz_keg = keg;
2772 zone->uz_size = keg->uk_size;
2773 zone->uz_flags |= (keg->uk_flags &
2812 * table and removes the keg from the global list.
2820 uma_keg_t keg;
2824 keg = (uma_keg_t)arg;
2827 free += keg->uk_domain[i].ud_free_items;
2828 pages += keg->uk_domain[i].ud_pages;
2829 KEG_LOCK_FINI(keg, i);
2832 printf("Freed UMA keg (%s) was not empty (%u items). "
2834 keg->uk_name ? keg->uk_name : "",
2835 pages / keg->uk_ppera * keg->uk_ipers - free, pages);
2837 hash_free(&keg->uk_hash);
2850 uma_keg_t keg;
2864 keg = zone->uz_keg;
2865 keg->uk_reserve = 0;
2873 keg = zone->uz_keg;
2875 LIST_REMOVE(keg, uk_link);
2877 zone_free_item(kegs, keg, NULL, SKIP_NONE);
2892 uma_keg_t keg;
2895 LIST_FOREACH(keg, &uma_kegs, uk_link) {
2896 LIST_FOREACH(zone, &keg->uk_zones, uz_link)
2949 /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */
2971 args.keg = primarykeg;
2982 args.keg = NULL;
3135 args.keg = NULL;
3150 uma_keg_t keg;
3153 keg = primary->uz_keg;
3156 args.size = keg->uk_size;
3161 args.align = keg->uk_align;
3162 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY;
3163 args.keg = keg;
3645 * Try to allocate from the bucket cache before falling back to the keg.
3684 keg_first_slab(uma_keg_t keg, int domain, bool rr)
3692 KEG_LOCK_ASSERT(keg, domain);
3697 dom = &keg->uk_domain[domain];
3715 * keg domain lock held if a slab was found or unlocked if not.
3718 keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags)
3724 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0)
3727 KEG_LOCK(keg, domain);
3728 reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve;
3729 if (keg->uk_domain[domain].ud_free_items <= reserve ||
3730 (slab = keg_first_slab(keg, domain, rr)) == NULL) {
3731 KEG_UNLOCK(keg, domain);
3738 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags)
3747 * Use the keg's policy if upper layers haven't already specified a
3750 * To avoid races we run the iterator with the keg lock held, but that
3757 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
3765 slab = keg_fetch_free_slab(keg, domain, rr, flags);
3775 slab = keg_alloc_slab(keg, zone, domain, flags, aflags);
3782 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0);
3794 if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL)
3801 slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
3807 KEG_LOCK_ASSERT(keg, slab->us_domain);
3809 dom = &keg->uk_domain[slab->us_domain];
3810 freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1;
3811 BIT_CLR(keg->uk_ipers, freei, &slab->us_free);
3812 item = slab_item(slab, keg, freei);
3835 uma_keg_t keg;
3843 keg = zone->uz_keg;
3846 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL)
3851 dom = &keg->uk_domain[slab->us_domain];
3853 bucket[i++] = slab_alloc_item(keg, slab);
3854 if (dom->ud_free_items <= keg->uk_reserve) {
3860 KEG_UNLOCK(keg, slab->us_domain);
3877 KEG_UNLOCK(keg, slab->us_domain);
4143 * We have to call both the zone's init (not the keg's init)
4145 * a keg slab directly to the user, and the user is expecting it
4542 uma_keg_t keg;
4546 keg = zone->uz_keg;
4547 KEG_LOCK_ASSERT(keg, slab->us_domain);
4550 dom = &keg->uk_domain[slab->us_domain];
4551 if (slab->us_freecount + 1 == keg->uk_ipers) {
4561 freei = slab_item_index(slab, keg, item);
4562 BIT_SET(keg->uk_ipers, freei, &slab->us_free);
4575 uma_keg_t keg;
4581 keg = zone->uz_keg;
4584 lock = KEG_LOCK(keg, 0);
4592 slab = hash_sfind(&keg->uk_hash, mem);
4594 slab = (uma_slab_t)(mem + keg->uk_pgoff);
4596 if (lock != KEG_LOCKPTR(keg, slab->us_domain)) {
4599 lock = KEG_LOCK(keg, slab->us_domain);
4788 uma_keg_get_allocs(uma_keg_t keg)
4794 LIST_FOREACH(z, &keg->uk_zones, uz_link)
4805 uma_keg_t keg;
4807 KEG_GET(zone, keg);
4808 KEG_ASSERT_COLD(keg);
4809 keg->uk_init = uminit;
4816 uma_keg_t keg;
4818 KEG_GET(zone, keg);
4819 KEG_ASSERT_COLD(keg);
4820 keg->uk_fini = fini;
4845 uma_keg_t keg;
4847 KEG_GET(zone, keg);
4848 KEG_ASSERT_COLD(keg);
4849 keg->uk_freef = freef;
4856 uma_keg_t keg;
4858 KEG_GET(zone, keg);
4859 KEG_ASSERT_COLD(keg);
4860 keg->uk_allocf = allocf;
4889 uma_keg_t keg;
4891 KEG_GET(zone, keg);
4892 KEG_ASSERT_COLD(keg);
4893 keg->uk_reserve = items;
4900 uma_keg_t keg;
4904 KEG_GET(zone, keg);
4905 KEG_ASSERT_COLD(keg);
4908 pages = howmany(count, keg->uk_ipers) * keg->uk_ppera;
4911 if (keg->uk_ppera > 1) {
4921 MPASS(keg->uk_kva == 0);
4922 keg->uk_kva = kva;
4923 keg->uk_offset = 0;
4924 zone->uz_max_items = pages * keg->uk_ipers;
4926 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc;
4928 keg->uk_allocf = noobj_alloc;
4930 keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE;
4944 uma_keg_t keg;
4947 KEG_GET(zone, keg);
4948 slabs = howmany(items, keg->uk_ipers);
4951 vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain,
4954 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK,
4957 dom = &keg->uk_domain[slab->us_domain];
4966 KEG_UNLOCK(keg, slab->us_domain);
4970 vm_wait_doms(&keg->uk_dr.dr_policy->ds_mask, 0);
5292 * on the keg's zone list.
5396 uma_keg_t keg = arg1;
5399 total = keg->uk_ppera * PAGE_SIZE;
5400 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0)
5401 total += slabzone(keg->uk_ipers)->uz_keg->uk_rsize;
5407 avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1);
5408 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0)
5429 uma_keg_t keg;
5442 keg = zone->uz_keg;
5443 if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0)
5444 return ((uma_slab_t)(mem + keg->uk_pgoff));
5445 KEG_LOCK(keg, 0);
5446 slab = hash_sfind(&keg->uk_hash, mem);
5447 KEG_UNLOCK(keg, 0);
5463 uma_dbg_kskip(uma_keg_t keg, void *mem)
5474 if (keg->uk_ipers > 1) {
5475 idx *= keg->uk_ipers;
5476 idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize;
5495 uma_keg_t keg;
5504 keg = zone->uz_keg;
5505 freei = slab_item_index(slab, keg, item);
5507 if (BIT_TEST_SET_ATOMIC(keg->uk_ipers, freei,
5508 slab_dbg_bits(slab, keg)))
5521 uma_keg_t keg;
5530 keg = zone->uz_keg;
5531 freei = slab_item_index(slab, keg, item);
5533 if (freei >= keg->uk_ipers)
5537 if (slab_item(slab, keg, freei) != item)
5541 if (!BIT_TEST_CLR_ATOMIC(keg->uk_ipers, freei,
5542 slab_dbg_bits(slab, keg)))