52 53/* I should really use ktr.. */ 54/* 55#define UMA_DEBUG 1 56#define UMA_DEBUG_ALLOC 1 57#define UMA_DEBUG_ALLOC_1 1 58*/ 59 60#include "opt_param.h" 61#include <sys/param.h> 62#include <sys/systm.h> 63#include <sys/kernel.h> 64#include <sys/types.h> 65#include <sys/queue.h> 66#include <sys/malloc.h> 67#include <sys/ktr.h> 68#include <sys/lock.h> 69#include <sys/sysctl.h> 70#include <sys/mutex.h> 71#include <sys/proc.h> 72#include <sys/smp.h> 73#include <sys/vmmeter.h> 74 75#include <vm/vm.h> 76#include <vm/vm_object.h> 77#include <vm/vm_page.h> 78#include <vm/vm_param.h> 79#include <vm/vm_map.h> 80#include <vm/vm_kern.h> 81#include <vm/vm_extern.h> 82#include <vm/uma.h> 83#include <vm/uma_int.h> 84#include <vm/uma_dbg.h> 85 86#include <machine/vmparam.h> 87 88/* 89 * This is the zone and keg from which all zones are spawned. The idea is that 90 * even the zone & keg heads are allocated from the allocator, so we use the 91 * bss section to bootstrap us. 92 */ 93static struct uma_keg masterkeg; 94static struct uma_zone masterzone_k; 95static struct uma_zone masterzone_z; 96static uma_zone_t kegs = &masterzone_k; 97static uma_zone_t zones = &masterzone_z; 98 99/* This is the zone from which all of uma_slab_t's are allocated. */ 100static uma_zone_t slabzone; 101static uma_zone_t slabrefzone; /* With refcounters (for UMA_ZONE_REFCNT) */ 102 103/* 104 * The initial hash tables come out of this zone so they can be allocated 105 * prior to malloc coming up. 106 */ 107static uma_zone_t hashzone; 108 109static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 110 111/* 112 * Are we allowed to allocate buckets? 113 */ 114static int bucketdisable = 1; 115 116/* Linked list of all kegs in the system */ 117static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(&uma_kegs); 118 119/* This mutex protects the keg list */ 120static struct mtx uma_mtx; 121 122/* These are the pcpu cache locks */ 123static struct mtx uma_pcpu_mtx[MAXCPU]; 124 125/* Linked list of boot time pages */ 126static LIST_HEAD(,uma_slab) uma_boot_pages = 127 LIST_HEAD_INITIALIZER(&uma_boot_pages); 128 129/* Count of free boottime pages */ 130static int uma_boot_free = 0; 131 132/* Is the VM done starting up? */ 133static int booted = 0; 134 135/* Maximum number of allowed items-per-slab if the slab header is OFFPAGE */ 136static u_int uma_max_ipers; 137static u_int uma_max_ipers_ref; 138 139/* 140 * This is the handle used to schedule events that need to happen 141 * outside of the allocation fast path. 142 */ 143static struct callout uma_callout; 144#define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 145 146/* 147 * This structure is passed as the zone ctor arg so that I don't have to create 148 * a special allocation function just for zones. 149 */ 150struct uma_zctor_args { 151 char *name; 152 size_t size; 153 uma_ctor ctor; 154 uma_dtor dtor; 155 uma_init uminit; 156 uma_fini fini; 157 uma_keg_t keg; 158 int align; 159 u_int16_t flags; 160}; 161 162struct uma_kctor_args { 163 uma_zone_t zone; 164 size_t size; 165 uma_init uminit; 166 uma_fini fini; 167 int align; 168 u_int16_t flags; 169}; 170 171struct uma_bucket_zone { 172 uma_zone_t ubz_zone; 173 char *ubz_name; 174 int ubz_entries; 175}; 176 177#define BUCKET_MAX 128 178 179struct uma_bucket_zone bucket_zones[] = { 180 { NULL, "16 Bucket", 16 }, 181 { NULL, "32 Bucket", 32 }, 182 { NULL, "64 Bucket", 64 }, 183 { NULL, "128 Bucket", 128 }, 184 { NULL, NULL, 0} 185}; 186 187#define BUCKET_SHIFT 4 188#define BUCKET_ZONES ((BUCKET_MAX >> BUCKET_SHIFT) + 1) 189 190/* 191 * bucket_size[] maps requested bucket sizes to zones that allocate a bucket 192 * of approximately the right size. 193 */ 194static uint8_t bucket_size[BUCKET_ZONES]; 195 196enum zfreeskip { SKIP_NONE, SKIP_DTOR, SKIP_FINI }; 197 198/* Prototypes.. */ 199 200static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 201static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 202static void *startup_alloc(uma_zone_t, int, u_int8_t *, int); 203static void page_free(void *, int, u_int8_t); 204static uma_slab_t slab_zalloc(uma_zone_t, int); 205static void cache_drain(uma_zone_t); 206static void bucket_drain(uma_zone_t, uma_bucket_t); 207static void bucket_cache_drain(uma_zone_t zone); 208static int keg_ctor(void *, int, void *, int); 209static void keg_dtor(void *, int, void *); 210static int zone_ctor(void *, int, void *, int); 211static void zone_dtor(void *, int, void *); 212static int zero_init(void *, int, int); 213static void zone_small_init(uma_zone_t zone); 214static void zone_large_init(uma_zone_t zone); 215static void zone_foreach(void (*zfunc)(uma_zone_t)); 216static void zone_timeout(uma_zone_t zone); 217static int hash_alloc(struct uma_hash *); 218static int hash_expand(struct uma_hash *, struct uma_hash *); 219static void hash_free(struct uma_hash *hash); 220static void uma_timeout(void *); 221static void uma_startup3(void); 222static void *uma_zalloc_internal(uma_zone_t, void *, int); 223static void uma_zfree_internal(uma_zone_t, void *, void *, enum zfreeskip); 224static void bucket_enable(void); 225static void bucket_init(void); 226static uma_bucket_t bucket_alloc(int, int); 227static void bucket_free(uma_bucket_t); 228static void bucket_zone_drain(void); 229static int uma_zalloc_bucket(uma_zone_t zone, int flags); 230static uma_slab_t uma_zone_slab(uma_zone_t zone, int flags); 231static void *uma_slab_alloc(uma_zone_t zone, uma_slab_t slab); 232static void zone_drain(uma_zone_t); 233static uma_zone_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 234 uma_fini fini, int align, u_int16_t flags); 235 236void uma_print_zone(uma_zone_t); 237void uma_print_stats(void); 238static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 239 240#ifdef WITNESS 241static int nosleepwithlocks = 1; 242SYSCTL_INT(_debug, OID_AUTO, nosleepwithlocks, CTLFLAG_RW, &nosleepwithlocks, 243 0, "Convert M_WAITOK to M_NOWAIT to avoid lock-held-across-sleep paths"); 244#else 245static int nosleepwithlocks = 0; 246SYSCTL_INT(_debug, OID_AUTO, nosleepwithlocks, CTLFLAG_RW, &nosleepwithlocks, 247 0, "Convert M_WAITOK to M_NOWAIT to avoid lock-held-across-sleep paths"); 248#endif 249SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, 250 NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 251SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 252 253/* 254 * This routine checks to see whether or not it's safe to enable buckets. 255 */ 256 257static void 258bucket_enable(void) 259{ 260 if (cnt.v_free_count < cnt.v_free_min) 261 bucketdisable = 1; 262 else 263 bucketdisable = 0; 264} 265 266/* 267 * Initialize bucket_zones, the array of zones of buckets of various sizes. 268 * 269 * For each zone, calculate the memory required for each bucket, consisting 270 * of the header and an array of pointers. Initialize bucket_size[] to point 271 * the range of appropriate bucket sizes at the zone. 272 */ 273static void 274bucket_init(void) 275{ 276 struct uma_bucket_zone *ubz; 277 int i; 278 int j; 279 280 for (i = 0, j = 0; bucket_zones[j].ubz_entries != 0; j++) { 281 int size; 282 283 ubz = &bucket_zones[j]; 284 size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 285 size += sizeof(void *) * ubz->ubz_entries; 286 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 287 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 288 for (; i <= ubz->ubz_entries; i += (1 << BUCKET_SHIFT)) 289 bucket_size[i >> BUCKET_SHIFT] = j; 290 } 291} 292 293/* 294 * Given a desired number of entries for a bucket, return the zone from which 295 * to allocate the bucket. 296 */ 297static struct uma_bucket_zone * 298bucket_zone_lookup(int entries) 299{ 300 int idx; 301 302 idx = howmany(entries, 1 << BUCKET_SHIFT); 303 return (&bucket_zones[bucket_size[idx]]); 304} 305 306static uma_bucket_t 307bucket_alloc(int entries, int bflags) 308{ 309 struct uma_bucket_zone *ubz; 310 uma_bucket_t bucket; 311 312 /* 313 * This is to stop us from allocating per cpu buckets while we're 314 * running out of UMA_BOOT_PAGES. Otherwise, we would exhaust the 315 * boot pages. This also prevents us from allocating buckets in 316 * low memory situations. 317 */ 318 if (bucketdisable) 319 return (NULL); 320 321 ubz = bucket_zone_lookup(entries); 322 bucket = uma_zalloc_internal(ubz->ubz_zone, NULL, bflags); 323 if (bucket) { 324#ifdef INVARIANTS 325 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 326#endif 327 bucket->ub_cnt = 0; 328 bucket->ub_entries = ubz->ubz_entries; 329 } 330 331 return (bucket); 332} 333 334static void 335bucket_free(uma_bucket_t bucket) 336{ 337 struct uma_bucket_zone *ubz; 338 339 ubz = bucket_zone_lookup(bucket->ub_entries); 340 uma_zfree_internal(ubz->ubz_zone, bucket, NULL, SKIP_NONE); 341} 342 343static void 344bucket_zone_drain(void) 345{ 346 struct uma_bucket_zone *ubz; 347 348 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 349 zone_drain(ubz->ubz_zone); 350} 351 352 353/* 354 * Routine called by timeout which is used to fire off some time interval 355 * based calculations. (stats, hash size, etc.) 356 * 357 * Arguments: 358 * arg Unused 359 * 360 * Returns: 361 * Nothing 362 */ 363static void 364uma_timeout(void *unused) 365{ 366 bucket_enable(); 367 zone_foreach(zone_timeout); 368 369 /* Reschedule this event */ 370 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 371} 372 373/* 374 * Routine to perform timeout driven calculations. This expands the 375 * hashes and does per cpu statistics aggregation. 376 * 377 * Arguments: 378 * zone The zone to operate on 379 * 380 * Returns: 381 * Nothing 382 */ 383static void 384zone_timeout(uma_zone_t zone) 385{ 386 uma_keg_t keg; 387 uma_cache_t cache; 388 u_int64_t alloc; 389 int cpu; 390 391 keg = zone->uz_keg; 392 alloc = 0; 393 394 /* 395 * Aggregate per cpu cache statistics back to the zone. 396 * 397 * XXX This should be done in the sysctl handler. 398 * 399 * I may rewrite this to set a flag in the per cpu cache instead of 400 * locking. If the flag is not cleared on the next round I will have 401 * to lock and do it here instead so that the statistics don't get too 402 * far out of sync. 403 */ 404 if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL)) { 405 for (cpu = 0; cpu <= mp_maxid; cpu++) { 406 if (CPU_ABSENT(cpu)) 407 continue; 408 CPU_LOCK(cpu); 409 cache = &zone->uz_cpu[cpu]; 410 /* Add them up, and reset */ 411 alloc += cache->uc_allocs; 412 cache->uc_allocs = 0; 413 CPU_UNLOCK(cpu); 414 } 415 } 416 417 /* Now push these stats back into the zone.. */ 418 ZONE_LOCK(zone); 419 zone->uz_allocs += alloc; 420 421 /* 422 * Expand the zone hash table. 423 * 424 * This is done if the number of slabs is larger than the hash size. 425 * What I'm trying to do here is completely reduce collisions. This 426 * may be a little aggressive. Should I allow for two collisions max? 427 */ 428 429 if (keg->uk_flags & UMA_ZONE_HASH && 430 keg->uk_pages / keg->uk_ppera >= keg->uk_hash.uh_hashsize) { 431 struct uma_hash newhash; 432 struct uma_hash oldhash; 433 int ret; 434 435 /* 436 * This is so involved because allocating and freeing 437 * while the zone lock is held will lead to deadlock. 438 * I have to do everything in stages and check for 439 * races. 440 */ 441 newhash = keg->uk_hash; 442 ZONE_UNLOCK(zone); 443 ret = hash_alloc(&newhash); 444 ZONE_LOCK(zone); 445 if (ret) { 446 if (hash_expand(&keg->uk_hash, &newhash)) { 447 oldhash = keg->uk_hash; 448 keg->uk_hash = newhash; 449 } else 450 oldhash = newhash; 451 452 ZONE_UNLOCK(zone); 453 hash_free(&oldhash); 454 ZONE_LOCK(zone); 455 } 456 } 457 ZONE_UNLOCK(zone); 458} 459 460/* 461 * Allocate and zero fill the next sized hash table from the appropriate 462 * backing store. 463 * 464 * Arguments: 465 * hash A new hash structure with the old hash size in uh_hashsize 466 * 467 * Returns: 468 * 1 on sucess and 0 on failure. 469 */ 470static int 471hash_alloc(struct uma_hash *hash) 472{ 473 int oldsize; 474 int alloc; 475 476 oldsize = hash->uh_hashsize; 477 478 /* We're just going to go to a power of two greater */ 479 if (oldsize) { 480 hash->uh_hashsize = oldsize * 2; 481 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 482 hash->uh_slab_hash = (struct slabhead *)malloc(alloc, 483 M_UMAHASH, M_NOWAIT); 484 } else { 485 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 486 hash->uh_slab_hash = uma_zalloc_internal(hashzone, NULL, 487 M_WAITOK); 488 hash->uh_hashsize = UMA_HASH_SIZE_INIT; 489 } 490 if (hash->uh_slab_hash) { 491 bzero(hash->uh_slab_hash, alloc); 492 hash->uh_hashmask = hash->uh_hashsize - 1; 493 return (1); 494 } 495 496 return (0); 497} 498 499/* 500 * Expands the hash table for HASH zones. This is done from zone_timeout 501 * to reduce collisions. This must not be done in the regular allocation 502 * path, otherwise, we can recurse on the vm while allocating pages. 503 * 504 * Arguments: 505 * oldhash The hash you want to expand 506 * newhash The hash structure for the new table 507 * 508 * Returns: 509 * Nothing 510 * 511 * Discussion: 512 */ 513static int 514hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 515{ 516 uma_slab_t slab; 517 int hval; 518 int i; 519 520 if (!newhash->uh_slab_hash) 521 return (0); 522 523 if (oldhash->uh_hashsize >= newhash->uh_hashsize) 524 return (0); 525 526 /* 527 * I need to investigate hash algorithms for resizing without a 528 * full rehash. 529 */ 530 531 for (i = 0; i < oldhash->uh_hashsize; i++) 532 while (!SLIST_EMPTY(&oldhash->uh_slab_hash[i])) { 533 slab = SLIST_FIRST(&oldhash->uh_slab_hash[i]); 534 SLIST_REMOVE_HEAD(&oldhash->uh_slab_hash[i], us_hlink); 535 hval = UMA_HASH(newhash, slab->us_data); 536 SLIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 537 slab, us_hlink); 538 } 539 540 return (1); 541} 542 543/* 544 * Free the hash bucket to the appropriate backing store. 545 * 546 * Arguments: 547 * slab_hash The hash bucket we're freeing 548 * hashsize The number of entries in that hash bucket 549 * 550 * Returns: 551 * Nothing 552 */ 553static void 554hash_free(struct uma_hash *hash) 555{ 556 if (hash->uh_slab_hash == NULL) 557 return; 558 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 559 uma_zfree_internal(hashzone, 560 hash->uh_slab_hash, NULL, SKIP_NONE); 561 else 562 free(hash->uh_slab_hash, M_UMAHASH); 563} 564 565/* 566 * Frees all outstanding items in a bucket 567 * 568 * Arguments: 569 * zone The zone to free to, must be unlocked. 570 * bucket The free/alloc bucket with items, cpu queue must be locked. 571 * 572 * Returns: 573 * Nothing 574 */ 575 576static void 577bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 578{ 579 uma_slab_t slab; 580 int mzone; 581 void *item; 582 583 if (bucket == NULL) 584 return; 585 586 slab = NULL; 587 mzone = 0; 588 589 /* We have to lookup the slab again for malloc.. */ 590 if (zone->uz_keg->uk_flags & UMA_ZONE_MALLOC) 591 mzone = 1; 592 593 while (bucket->ub_cnt > 0) { 594 bucket->ub_cnt--; 595 item = bucket->ub_bucket[bucket->ub_cnt]; 596#ifdef INVARIANTS 597 bucket->ub_bucket[bucket->ub_cnt] = NULL; 598 KASSERT(item != NULL, 599 ("bucket_drain: botched ptr, item is NULL")); 600#endif 601 /* 602 * This is extremely inefficient. The slab pointer was passed 603 * to uma_zfree_arg, but we lost it because the buckets don't 604 * hold them. This will go away when free() gets a size passed 605 * to it. 606 */ 607 if (mzone) 608 slab = vtoslab((vm_offset_t)item & (~UMA_SLAB_MASK)); 609 uma_zfree_internal(zone, item, slab, SKIP_DTOR); 610 } 611} 612 613/* 614 * Drains the per cpu caches for a zone. 615 * 616 * Arguments: 617 * zone The zone to drain, must be unlocked. 618 * 619 * Returns: 620 * Nothing 621 */ 622static void 623cache_drain(uma_zone_t zone) 624{ 625 uma_cache_t cache; 626 int cpu; 627 628 /* 629 * We have to lock each cpu cache before locking the zone 630 */ 631 for (cpu = 0; cpu <= mp_maxid; cpu++) { 632 if (CPU_ABSENT(cpu)) 633 continue; 634 CPU_LOCK(cpu); 635 cache = &zone->uz_cpu[cpu]; 636 bucket_drain(zone, cache->uc_allocbucket); 637 bucket_drain(zone, cache->uc_freebucket); 638 if (cache->uc_allocbucket != NULL) 639 bucket_free(cache->uc_allocbucket); 640 if (cache->uc_freebucket != NULL) 641 bucket_free(cache->uc_freebucket); 642 cache->uc_allocbucket = cache->uc_freebucket = NULL; 643 } 644 ZONE_LOCK(zone); 645 bucket_cache_drain(zone); 646 ZONE_UNLOCK(zone); 647 for (cpu = 0; cpu <= mp_maxid; cpu++) { 648 if (CPU_ABSENT(cpu)) 649 continue; 650 CPU_UNLOCK(cpu); 651 } 652} 653 654/* 655 * Drain the cached buckets from a zone. Expects a locked zone on entry. 656 */ 657static void 658bucket_cache_drain(uma_zone_t zone) 659{ 660 uma_bucket_t bucket; 661 662 /* 663 * Drain the bucket queues and free the buckets, we just keep two per 664 * cpu (alloc/free). 665 */ 666 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 667 LIST_REMOVE(bucket, ub_link); 668 ZONE_UNLOCK(zone); 669 bucket_drain(zone, bucket); 670 bucket_free(bucket); 671 ZONE_LOCK(zone); 672 } 673 674 /* Now we do the free queue.. */ 675 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 676 LIST_REMOVE(bucket, ub_link); 677 bucket_free(bucket); 678 } 679} 680 681/* 682 * Frees pages from a zone back to the system. This is done on demand from 683 * the pageout daemon. 684 * 685 * Arguments: 686 * zone The zone to free pages from 687 * all Should we drain all items? 688 * 689 * Returns: 690 * Nothing. 691 */ 692static void 693zone_drain(uma_zone_t zone) 694{ 695 struct slabhead freeslabs = { 0 }; 696 uma_keg_t keg; 697 uma_slab_t slab; 698 uma_slab_t n; 699 u_int8_t flags; 700 u_int8_t *mem; 701 int i; 702 703 keg = zone->uz_keg; 704 705 /* 706 * We don't want to take pages from statically allocated zones at this 707 * time 708 */ 709 if (keg->uk_flags & UMA_ZONE_NOFREE || keg->uk_freef == NULL) 710 return; 711 712 ZONE_LOCK(zone); 713 714#ifdef UMA_DEBUG 715 printf("%s free items: %u\n", zone->uz_name, keg->uk_free); 716#endif 717 bucket_cache_drain(zone); 718 if (keg->uk_free == 0) 719 goto finished; 720 721 slab = LIST_FIRST(&keg->uk_free_slab); 722 while (slab) { 723 n = LIST_NEXT(slab, us_link); 724 725 /* We have no where to free these to */ 726 if (slab->us_flags & UMA_SLAB_BOOT) { 727 slab = n; 728 continue; 729 } 730 731 LIST_REMOVE(slab, us_link); 732 keg->uk_pages -= keg->uk_ppera; 733 keg->uk_free -= keg->uk_ipers; 734 735 if (keg->uk_flags & UMA_ZONE_HASH) 736 UMA_HASH_REMOVE(&keg->uk_hash, slab, slab->us_data); 737 738 SLIST_INSERT_HEAD(&freeslabs, slab, us_hlink); 739 740 slab = n; 741 } 742finished: 743 ZONE_UNLOCK(zone); 744 745 while ((slab = SLIST_FIRST(&freeslabs)) != NULL) { 746 SLIST_REMOVE(&freeslabs, slab, uma_slab, us_hlink); 747 if (keg->uk_fini) 748 for (i = 0; i < keg->uk_ipers; i++) 749 keg->uk_fini( 750 slab->us_data + (keg->uk_rsize * i), 751 keg->uk_size); 752 flags = slab->us_flags; 753 mem = slab->us_data; 754 755 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 756 (keg->uk_flags & UMA_ZONE_REFCNT)) { 757 for (i = 0; i < keg->uk_ppera; i++) 758 vsetobj((vm_offset_t)mem + (i * PAGE_SIZE), 759 kmem_object); 760 } 761 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 762 uma_zfree_internal(keg->uk_slabzone, slab, NULL, 763 SKIP_NONE); 764#ifdef UMA_DEBUG 765 printf("%s: Returning %d bytes.\n", 766 zone->uz_name, UMA_SLAB_SIZE * keg->uk_ppera); 767#endif 768 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, flags); 769 } 770} 771 772/* 773 * Allocate a new slab for a zone. This does not insert the slab onto a list. 774 * 775 * Arguments: 776 * zone The zone to allocate slabs for 777 * wait Shall we wait? 778 * 779 * Returns: 780 * The slab that was allocated or NULL if there is no memory and the 781 * caller specified M_NOWAIT. 782 */ 783static uma_slab_t 784slab_zalloc(uma_zone_t zone, int wait) 785{ 786 uma_slabrefcnt_t slabref; 787 uma_slab_t slab; 788 uma_keg_t keg; 789 u_int8_t *mem; 790 u_int8_t flags; 791 int i; 792 793 slab = NULL; 794 keg = zone->uz_keg; 795 796#ifdef UMA_DEBUG 797 printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); 798#endif 799 ZONE_UNLOCK(zone); 800 801 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 802 slab = uma_zalloc_internal(keg->uk_slabzone, NULL, wait); 803 if (slab == NULL) { 804 ZONE_LOCK(zone); 805 return NULL; 806 } 807 } 808 809 /* 810 * This reproduces the old vm_zone behavior of zero filling pages the 811 * first time they are added to a zone. 812 * 813 * Malloced items are zeroed in uma_zalloc. 814 */ 815 816 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 817 wait |= M_ZERO; 818 else 819 wait &= ~M_ZERO; 820 821 mem = keg->uk_allocf(zone, keg->uk_ppera * UMA_SLAB_SIZE, 822 &flags, wait); 823 if (mem == NULL) { 824 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 825 uma_zfree_internal(keg->uk_slabzone, slab, NULL, 0); 826 ZONE_LOCK(zone); 827 return (NULL); 828 } 829 830 /* Point the slab into the allocated memory */ 831 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) 832 slab = (uma_slab_t )(mem + keg->uk_pgoff); 833 834 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 835 (keg->uk_flags & UMA_ZONE_REFCNT)) 836 for (i = 0; i < keg->uk_ppera; i++) 837 vsetslab((vm_offset_t)mem + (i * PAGE_SIZE), slab); 838 839 slab->us_keg = keg; 840 slab->us_data = mem; 841 slab->us_freecount = keg->uk_ipers; 842 slab->us_firstfree = 0; 843 slab->us_flags = flags; 844 845 if (keg->uk_flags & UMA_ZONE_REFCNT) { 846 slabref = (uma_slabrefcnt_t)slab; 847 for (i = 0; i < keg->uk_ipers; i++) { 848 slabref->us_freelist[i].us_refcnt = 0; 849 slabref->us_freelist[i].us_item = i+1; 850 } 851 } else { 852 for (i = 0; i < keg->uk_ipers; i++) 853 slab->us_freelist[i].us_item = i+1; 854 } 855 856 if (keg->uk_init != NULL) { 857 for (i = 0; i < keg->uk_ipers; i++) 858 if (keg->uk_init(slab->us_data + (keg->uk_rsize * i), 859 keg->uk_size, wait) != 0) 860 break; 861 if (i != keg->uk_ipers) { 862 if (keg->uk_fini != NULL) { 863 for (i--; i > -1; i--) 864 keg->uk_fini(slab->us_data + 865 (keg->uk_rsize * i), 866 keg->uk_size); 867 } 868 if ((keg->uk_flags & UMA_ZONE_MALLOC) || 869 (keg->uk_flags & UMA_ZONE_REFCNT)) 870 for (i = 0; i < keg->uk_ppera; i++) 871 vsetobj((vm_offset_t)mem + 872 (i * PAGE_SIZE), kmem_object); 873 if (keg->uk_flags & UMA_ZONE_OFFPAGE) 874 uma_zfree_internal(keg->uk_slabzone, slab, 875 NULL, SKIP_NONE); 876 keg->uk_freef(mem, UMA_SLAB_SIZE * keg->uk_ppera, 877 flags); 878 ZONE_LOCK(zone); 879 return (NULL); 880 } 881 } 882 ZONE_LOCK(zone); 883 884 if (keg->uk_flags & UMA_ZONE_HASH) 885 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 886 887 keg->uk_pages += keg->uk_ppera; 888 keg->uk_free += keg->uk_ipers; 889 890 return (slab); 891} 892 893/* 894 * This function is intended to be used early on in place of page_alloc() so 895 * that we may use the boot time page cache to satisfy allocations before 896 * the VM is ready. 897 */ 898static void * 899startup_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 900{ 901 uma_keg_t keg; 902 903 keg = zone->uz_keg; 904 905 /* 906 * Check our small startup cache to see if it has pages remaining. 907 */ 908 mtx_lock(&uma_mtx); 909 if (uma_boot_free != 0) { 910 uma_slab_t tmps; 911 912 tmps = LIST_FIRST(&uma_boot_pages); 913 LIST_REMOVE(tmps, us_link); 914 uma_boot_free--; 915 mtx_unlock(&uma_mtx); 916 *pflag = tmps->us_flags; 917 return (tmps->us_data); 918 } 919 mtx_unlock(&uma_mtx); 920 if (booted == 0) 921 panic("UMA: Increase UMA_BOOT_PAGES"); 922 /* 923 * Now that we've booted reset these users to their real allocator. 924 */ 925#ifdef UMA_MD_SMALL_ALLOC 926 keg->uk_allocf = uma_small_alloc; 927#else 928 keg->uk_allocf = page_alloc; 929#endif 930 return keg->uk_allocf(zone, bytes, pflag, wait); 931} 932 933/* 934 * Allocates a number of pages from the system 935 * 936 * Arguments: 937 * zone Unused 938 * bytes The number of bytes requested 939 * wait Shall we wait? 940 * 941 * Returns: 942 * A pointer to the alloced memory or possibly 943 * NULL if M_NOWAIT is set. 944 */ 945static void * 946page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 947{ 948 void *p; /* Returned page */ 949 950 *pflag = UMA_SLAB_KMEM; 951 p = (void *) kmem_malloc(kmem_map, bytes, wait); 952 953 return (p); 954} 955 956/* 957 * Allocates a number of pages from within an object 958 * 959 * Arguments: 960 * zone Unused 961 * bytes The number of bytes requested 962 * wait Shall we wait? 963 * 964 * Returns: 965 * A pointer to the alloced memory or possibly 966 * NULL if M_NOWAIT is set. 967 */ 968static void * 969obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 970{ 971 vm_object_t object; 972 vm_offset_t retkva, zkva; 973 vm_page_t p; 974 int pages, startpages; 975 976 object = zone->uz_keg->uk_obj; 977 retkva = 0; 978 979 /* 980 * This looks a little weird since we're getting one page at a time. 981 */ 982 VM_OBJECT_LOCK(object); 983 p = TAILQ_LAST(&object->memq, pglist); 984 pages = p != NULL ? p->pindex + 1 : 0; 985 startpages = pages; 986 zkva = zone->uz_keg->uk_kva + pages * PAGE_SIZE; 987 for (; bytes > 0; bytes -= PAGE_SIZE) { 988 p = vm_page_alloc(object, pages, 989 VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED); 990 if (p == NULL) { 991 if (pages != startpages) 992 pmap_qremove(retkva, pages - startpages); 993 while (pages != startpages) { 994 pages--; 995 p = TAILQ_LAST(&object->memq, pglist); 996 vm_page_lock_queues(); 997 vm_page_unwire(p, 0); 998 vm_page_free(p); 999 vm_page_unlock_queues(); 1000 } 1001 retkva = 0; 1002 goto done; 1003 } 1004 pmap_qenter(zkva, &p, 1); 1005 if (retkva == 0) 1006 retkva = zkva; 1007 zkva += PAGE_SIZE; 1008 pages += 1; 1009 } 1010done: 1011 VM_OBJECT_UNLOCK(object); 1012 *flags = UMA_SLAB_PRIV; 1013 1014 return ((void *)retkva); 1015} 1016 1017/* 1018 * Frees a number of pages to the system 1019 * 1020 * Arguments: 1021 * mem A pointer to the memory to be freed 1022 * size The size of the memory being freed 1023 * flags The original p->us_flags field 1024 * 1025 * Returns: 1026 * Nothing 1027 */ 1028static void 1029page_free(void *mem, int size, u_int8_t flags) 1030{ 1031 vm_map_t map; 1032 1033 if (flags & UMA_SLAB_KMEM) 1034 map = kmem_map; 1035 else 1036 panic("UMA: page_free used with invalid flags %d\n", flags); 1037 1038 kmem_free(map, (vm_offset_t)mem, size); 1039} 1040 1041/* 1042 * Zero fill initializer 1043 * 1044 * Arguments/Returns follow uma_init specifications 1045 */ 1046static int 1047zero_init(void *mem, int size, int flags) 1048{ 1049 bzero(mem, size); 1050 return (0); 1051} 1052 1053/* 1054 * Finish creating a small uma zone. This calculates ipers, and the zone size. 1055 * 1056 * Arguments 1057 * zone The zone we should initialize 1058 * 1059 * Returns 1060 * Nothing 1061 */ 1062static void 1063zone_small_init(uma_zone_t zone) 1064{ 1065 uma_keg_t keg; 1066 u_int rsize; 1067 u_int memused; 1068 u_int wastedspace; 1069 u_int shsize; 1070 1071 keg = zone->uz_keg; 1072 KASSERT(keg != NULL, ("Keg is null in zone_small_init")); 1073 rsize = keg->uk_size; 1074 1075 if (rsize < UMA_SMALLEST_UNIT) 1076 rsize = UMA_SMALLEST_UNIT; 1077 if (rsize & keg->uk_align) 1078 rsize = (rsize & ~keg->uk_align) + (keg->uk_align + 1); 1079 1080 keg->uk_rsize = rsize; 1081 keg->uk_ppera = 1; 1082 1083 if (keg->uk_flags & UMA_ZONE_REFCNT) { 1084 rsize += UMA_FRITMREF_SZ; /* linkage & refcnt */ 1085 shsize = sizeof(struct uma_slab_refcnt); 1086 } else { 1087 rsize += UMA_FRITM_SZ; /* Account for linkage */ 1088 shsize = sizeof(struct uma_slab); 1089 } 1090 1091 keg->uk_ipers = (UMA_SLAB_SIZE - shsize) / rsize; 1092 KASSERT(keg->uk_ipers != 0, ("zone_small_init: ipers is 0")); 1093 memused = keg->uk_ipers * rsize + shsize; 1094 wastedspace = UMA_SLAB_SIZE - memused; 1095 1096 /* 1097 * We can't do OFFPAGE if we're internal or if we've been 1098 * asked to not go to the VM for buckets. If we do this we 1099 * may end up going to the VM (kmem_map) for slabs which we 1100 * do not want to do if we're UMA_ZFLAG_CACHEONLY as a 1101 * result of UMA_ZONE_VM, which clearly forbids it. 1102 */ 1103 if ((keg->uk_flags & UMA_ZFLAG_INTERNAL) || 1104 (keg->uk_flags & UMA_ZFLAG_CACHEONLY)) 1105 return; 1106 1107 if ((wastedspace >= UMA_MAX_WASTE) && 1108 (keg->uk_ipers < (UMA_SLAB_SIZE / keg->uk_rsize))) { 1109 keg->uk_ipers = UMA_SLAB_SIZE / keg->uk_rsize; 1110 KASSERT(keg->uk_ipers <= 255, 1111 ("zone_small_init: keg->uk_ipers too high!")); 1112#ifdef UMA_DEBUG 1113 printf("UMA decided we need offpage slab headers for " 1114 "zone: %s, calculated wastedspace = %d, " 1115 "maximum wasted space allowed = %d, " 1116 "calculated ipers = %d, " 1117 "new wasted space = %d\n", zone->uz_name, wastedspace, 1118 UMA_MAX_WASTE, keg->uk_ipers, 1119 UMA_SLAB_SIZE - keg->uk_ipers * keg->uk_rsize); 1120#endif 1121 keg->uk_flags |= UMA_ZONE_OFFPAGE; 1122 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 1123 keg->uk_flags |= UMA_ZONE_HASH; 1124 } 1125} 1126 1127/* 1128 * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do 1129 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 1130 * more complicated. 1131 * 1132 * Arguments 1133 * zone The zone we should initialize 1134 * 1135 * Returns 1136 * Nothing 1137 */ 1138static void 1139zone_large_init(uma_zone_t zone) 1140{ 1141 uma_keg_t keg; 1142 int pages; 1143 1144 keg = zone->uz_keg; 1145 1146 KASSERT(keg != NULL, ("Keg is null in zone_large_init")); 1147 KASSERT((keg->uk_flags & UMA_ZFLAG_CACHEONLY) == 0, 1148 ("zone_large_init: Cannot large-init a UMA_ZFLAG_CACHEONLY zone")); 1149 1150 pages = keg->uk_size / UMA_SLAB_SIZE; 1151 1152 /* Account for remainder */ 1153 if ((pages * UMA_SLAB_SIZE) < keg->uk_size) 1154 pages++; 1155 1156 keg->uk_ppera = pages; 1157 keg->uk_ipers = 1; 1158 1159 keg->uk_flags |= UMA_ZONE_OFFPAGE; 1160 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 1161 keg->uk_flags |= UMA_ZONE_HASH; 1162 1163 keg->uk_rsize = keg->uk_size; 1164} 1165 1166/* 1167 * Keg header ctor. This initializes all fields, locks, etc. And inserts 1168 * the keg onto the global keg list. 1169 * 1170 * Arguments/Returns follow uma_ctor specifications 1171 * udata Actually uma_kctor_args 1172 */ 1173static int 1174keg_ctor(void *mem, int size, void *udata, int flags) 1175{ 1176 struct uma_kctor_args *arg = udata; 1177 uma_keg_t keg = mem; 1178 uma_zone_t zone; 1179 1180 bzero(keg, size); 1181 keg->uk_size = arg->size; 1182 keg->uk_init = arg->uminit; 1183 keg->uk_fini = arg->fini; 1184 keg->uk_align = arg->align; 1185 keg->uk_free = 0; 1186 keg->uk_pages = 0; 1187 keg->uk_flags = arg->flags; 1188 keg->uk_allocf = page_alloc; 1189 keg->uk_freef = page_free; 1190 keg->uk_recurse = 0; 1191 keg->uk_slabzone = NULL; 1192 1193 /* 1194 * The master zone is passed to us at keg-creation time. 1195 */ 1196 zone = arg->zone; 1197 zone->uz_keg = keg; 1198 1199 if (arg->flags & UMA_ZONE_VM) 1200 keg->uk_flags |= UMA_ZFLAG_CACHEONLY; 1201 1202 if (arg->flags & UMA_ZONE_ZINIT) 1203 keg->uk_init = zero_init; 1204 1205 /* 1206 * The +UMA_FRITM_SZ added to uk_size is to account for the 1207 * linkage that is added to the size in zone_small_init(). If 1208 * we don't account for this here then we may end up in 1209 * zone_small_init() with a calculated 'ipers' of 0. 1210 */ 1211 if (keg->uk_flags & UMA_ZONE_REFCNT) { 1212 if ((keg->uk_size+UMA_FRITMREF_SZ) > 1213 (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt))) 1214 zone_large_init(zone); 1215 else 1216 zone_small_init(zone); 1217 } else { 1218 if ((keg->uk_size+UMA_FRITM_SZ) > 1219 (UMA_SLAB_SIZE - sizeof(struct uma_slab))) 1220 zone_large_init(zone); 1221 else 1222 zone_small_init(zone); 1223 } 1224 1225 if (keg->uk_flags & UMA_ZONE_OFFPAGE) { 1226 if (keg->uk_flags & UMA_ZONE_REFCNT) 1227 keg->uk_slabzone = slabrefzone; 1228 else 1229 keg->uk_slabzone = slabzone; 1230 } 1231 1232 /* 1233 * If we haven't booted yet we need allocations to go through the 1234 * startup cache until the vm is ready. 1235 */ 1236 if (keg->uk_ppera == 1) { 1237#ifdef UMA_MD_SMALL_ALLOC 1238 keg->uk_allocf = uma_small_alloc; 1239 keg->uk_freef = uma_small_free; 1240#endif 1241 if (booted == 0) 1242 keg->uk_allocf = startup_alloc; 1243 } 1244 1245 /* 1246 * Initialize keg's lock (shared among zones) through 1247 * Master zone 1248 */ 1249 zone->uz_lock = &keg->uk_lock; 1250 if (arg->flags & UMA_ZONE_MTXCLASS) 1251 ZONE_LOCK_INIT(zone, 1); 1252 else 1253 ZONE_LOCK_INIT(zone, 0); 1254 1255 /* 1256 * If we're putting the slab header in the actual page we need to 1257 * figure out where in each page it goes. This calculates a right 1258 * justified offset into the memory on an ALIGN_PTR boundary. 1259 */ 1260 if (!(keg->uk_flags & UMA_ZONE_OFFPAGE)) { 1261 u_int totsize; 1262 1263 /* Size of the slab struct and free list */ 1264 if (keg->uk_flags & UMA_ZONE_REFCNT) 1265 totsize = sizeof(struct uma_slab_refcnt) + 1266 keg->uk_ipers * UMA_FRITMREF_SZ; 1267 else 1268 totsize = sizeof(struct uma_slab) + 1269 keg->uk_ipers * UMA_FRITM_SZ; 1270 1271 if (totsize & UMA_ALIGN_PTR) 1272 totsize = (totsize & ~UMA_ALIGN_PTR) + 1273 (UMA_ALIGN_PTR + 1); 1274 keg->uk_pgoff = UMA_SLAB_SIZE - totsize; 1275 1276 if (keg->uk_flags & UMA_ZONE_REFCNT) 1277 totsize = keg->uk_pgoff + sizeof(struct uma_slab_refcnt) 1278 + keg->uk_ipers * UMA_FRITMREF_SZ; 1279 else 1280 totsize = keg->uk_pgoff + sizeof(struct uma_slab) 1281 + keg->uk_ipers * UMA_FRITM_SZ; 1282 1283 /* 1284 * The only way the following is possible is if with our 1285 * UMA_ALIGN_PTR adjustments we are now bigger than 1286 * UMA_SLAB_SIZE. I haven't checked whether this is 1287 * mathematically possible for all cases, so we make 1288 * sure here anyway. 1289 */ 1290 if (totsize > UMA_SLAB_SIZE) { 1291 printf("zone %s ipers %d rsize %d size %d\n", 1292 zone->uz_name, keg->uk_ipers, keg->uk_rsize, 1293 keg->uk_size); 1294 panic("UMA slab won't fit.\n"); 1295 } 1296 } 1297 1298 if (keg->uk_flags & UMA_ZONE_HASH) 1299 hash_alloc(&keg->uk_hash); 1300 1301#ifdef UMA_DEBUG 1302 printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 1303 zone->uz_name, zone, 1304 keg->uk_size, keg->uk_ipers, 1305 keg->uk_ppera, keg->uk_pgoff); 1306#endif 1307 1308 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 1309 1310 mtx_lock(&uma_mtx); 1311 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 1312 mtx_unlock(&uma_mtx); 1313 return (0); 1314} 1315 1316/* 1317 * Zone header ctor. This initializes all fields, locks, etc. 1318 * 1319 * Arguments/Returns follow uma_ctor specifications 1320 * udata Actually uma_zctor_args 1321 */ 1322 1323static int 1324zone_ctor(void *mem, int size, void *udata, int flags) 1325{ 1326 struct uma_zctor_args *arg = udata; 1327 uma_zone_t zone = mem; 1328 uma_zone_t z; 1329 uma_keg_t keg; 1330 1331 bzero(zone, size); 1332 zone->uz_name = arg->name; 1333 zone->uz_ctor = arg->ctor; 1334 zone->uz_dtor = arg->dtor; 1335 zone->uz_init = NULL; 1336 zone->uz_fini = NULL; 1337 zone->uz_allocs = 0; 1338 zone->uz_fills = zone->uz_count = 0; 1339 1340 if (arg->flags & UMA_ZONE_SECONDARY) { 1341 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 1342 keg = arg->keg; 1343 zone->uz_keg = keg; 1344 zone->uz_init = arg->uminit; 1345 zone->uz_fini = arg->fini; 1346 zone->uz_lock = &keg->uk_lock; 1347 mtx_lock(&uma_mtx); 1348 ZONE_LOCK(zone); 1349 keg->uk_flags |= UMA_ZONE_SECONDARY; 1350 LIST_FOREACH(z, &keg->uk_zones, uz_link) { 1351 if (LIST_NEXT(z, uz_link) == NULL) { 1352 LIST_INSERT_AFTER(z, zone, uz_link); 1353 break; 1354 } 1355 } 1356 ZONE_UNLOCK(zone); 1357 mtx_unlock(&uma_mtx); 1358 } else if (arg->keg == NULL) { 1359 if (uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 1360 arg->align, arg->flags) == NULL) 1361 return (ENOMEM); 1362 } else { 1363 struct uma_kctor_args karg; 1364 int error; 1365 1366 /* We should only be here from uma_startup() */ 1367 karg.size = arg->size; 1368 karg.uminit = arg->uminit; 1369 karg.fini = arg->fini; 1370 karg.align = arg->align; 1371 karg.flags = arg->flags; 1372 karg.zone = zone; 1373 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 1374 flags); 1375 if (error) 1376 return (error); 1377 } 1378 keg = zone->uz_keg; 1379 zone->uz_lock = &keg->uk_lock; 1380 1381 /* 1382 * Some internal zones don't have room allocated for the per cpu 1383 * caches. If we're internal, bail out here. 1384 */ 1385 if (keg->uk_flags & UMA_ZFLAG_INTERNAL) { 1386 KASSERT((keg->uk_flags & UMA_ZONE_SECONDARY) == 0, 1387 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 1388 return (0); 1389 } 1390 1391 if (keg->uk_flags & UMA_ZONE_MAXBUCKET) 1392 zone->uz_count = BUCKET_MAX; 1393 else if (keg->uk_ipers <= BUCKET_MAX) 1394 zone->uz_count = keg->uk_ipers; 1395 else 1396 zone->uz_count = BUCKET_MAX; 1397 return (0); 1398} 1399 1400/* 1401 * Keg header dtor. This frees all data, destroys locks, frees the hash 1402 * table and removes the keg from the global list. 1403 * 1404 * Arguments/Returns follow uma_dtor specifications 1405 * udata unused 1406 */ 1407static void 1408keg_dtor(void *arg, int size, void *udata) 1409{ 1410 uma_keg_t keg; 1411 1412 keg = (uma_keg_t)arg; 1413 mtx_lock(&keg->uk_lock); 1414 if (keg->uk_free != 0) { 1415 printf("Freed UMA keg was not empty (%d items). " 1416 " Lost %d pages of memory.\n", 1417 keg->uk_free, keg->uk_pages); 1418 } 1419 mtx_unlock(&keg->uk_lock); 1420 1421 if (keg->uk_flags & UMA_ZONE_HASH) 1422 hash_free(&keg->uk_hash); 1423 1424 mtx_destroy(&keg->uk_lock); 1425} 1426 1427/* 1428 * Zone header dtor. 1429 * 1430 * Arguments/Returns follow uma_dtor specifications 1431 * udata unused 1432 */ 1433static void 1434zone_dtor(void *arg, int size, void *udata) 1435{ 1436 uma_zone_t zone; 1437 uma_keg_t keg; 1438 1439 zone = (uma_zone_t)arg; 1440 keg = zone->uz_keg; 1441 1442 if (!(keg->uk_flags & UMA_ZFLAG_INTERNAL)) 1443 cache_drain(zone); 1444 1445 mtx_lock(&uma_mtx); 1446 zone_drain(zone); 1447 if (keg->uk_flags & UMA_ZONE_SECONDARY) { 1448 LIST_REMOVE(zone, uz_link); 1449 /* 1450 * XXX there are some races here where 1451 * the zone can be drained but zone lock 1452 * released and then refilled before we 1453 * remove it... we dont care for now 1454 */ 1455 ZONE_LOCK(zone); 1456 if (LIST_EMPTY(&keg->uk_zones)) 1457 keg->uk_flags &= ~UMA_ZONE_SECONDARY; 1458 ZONE_UNLOCK(zone); 1459 mtx_unlock(&uma_mtx); 1460 } else { 1461 LIST_REMOVE(keg, uk_link); 1462 LIST_REMOVE(zone, uz_link); 1463 mtx_unlock(&uma_mtx); 1464 uma_zfree_internal(kegs, keg, NULL, SKIP_NONE); 1465 } 1466 zone->uz_keg = NULL; 1467} 1468 1469/* 1470 * Traverses every zone in the system and calls a callback 1471 * 1472 * Arguments: 1473 * zfunc A pointer to a function which accepts a zone 1474 * as an argument. 1475 * 1476 * Returns: 1477 * Nothing 1478 */ 1479static void 1480zone_foreach(void (*zfunc)(uma_zone_t)) 1481{ 1482 uma_keg_t keg; 1483 uma_zone_t zone; 1484 1485 mtx_lock(&uma_mtx); 1486 LIST_FOREACH(keg, &uma_kegs, uk_link) { 1487 LIST_FOREACH(zone, &keg->uk_zones, uz_link) 1488 zfunc(zone); 1489 } 1490 mtx_unlock(&uma_mtx); 1491} 1492 1493/* Public functions */ 1494/* See uma.h */ 1495void 1496uma_startup(void *bootmem) 1497{ 1498 struct uma_zctor_args args; 1499 uma_slab_t slab; 1500 u_int slabsize; 1501 u_int objsize, totsize, wsize; 1502 int i; 1503 1504#ifdef UMA_DEBUG 1505 printf("Creating uma keg headers zone and keg.\n"); 1506#endif 1507 /* 1508 * The general UMA lock is a recursion-allowed lock because 1509 * there is a code path where, while we're still configured 1510 * to use startup_alloc() for backend page allocations, we 1511 * may end up in uma_reclaim() which calls zone_foreach(zone_drain), 1512 * which grabs uma_mtx, only to later call into startup_alloc() 1513 * because while freeing we needed to allocate a bucket. Since 1514 * startup_alloc() also takes uma_mtx, we need to be able to 1515 * recurse on it. 1516 */ 1517 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF | MTX_RECURSE); 1518 1519 /* 1520 * Figure out the maximum number of items-per-slab we'll have if 1521 * we're using the OFFPAGE slab header to track free items, given 1522 * all possible object sizes and the maximum desired wastage 1523 * (UMA_MAX_WASTE). 1524 * 1525 * We iterate until we find an object size for 1526 * which the calculated wastage in zone_small_init() will be 1527 * enough to warrant OFFPAGE. Since wastedspace versus objsize 1528 * is an overall increasing see-saw function, we find the smallest 1529 * objsize such that the wastage is always acceptable for objects 1530 * with that objsize or smaller. Since a smaller objsize always 1531 * generates a larger possible uma_max_ipers, we use this computed 1532 * objsize to calculate the largest ipers possible. Since the 1533 * ipers calculated for OFFPAGE slab headers is always larger than 1534 * the ipers initially calculated in zone_small_init(), we use 1535 * the former's equation (UMA_SLAB_SIZE / keg->uk_rsize) to 1536 * obtain the maximum ipers possible for offpage slab headers. 1537 * 1538 * It should be noted that ipers versus objsize is an inversly 1539 * proportional function which drops off rather quickly so as 1540 * long as our UMA_MAX_WASTE is such that the objsize we calculate 1541 * falls into the portion of the inverse relation AFTER the steep 1542 * falloff, then uma_max_ipers shouldn't be too high (~10 on i386). 1543 * 1544 * Note that we have 8-bits (1 byte) to use as a freelist index 1545 * inside the actual slab header itself and this is enough to 1546 * accomodate us. In the worst case, a UMA_SMALLEST_UNIT sized 1547 * object with offpage slab header would have ipers = 1548 * UMA_SLAB_SIZE / UMA_SMALLEST_UNIT (currently = 256), which is 1549 * 1 greater than what our byte-integer freelist index can 1550 * accomodate, but we know that this situation never occurs as 1551 * for UMA_SMALLEST_UNIT-sized objects, we will never calculate 1552 * that we need to go to offpage slab headers. Or, if we do, 1553 * then we trap that condition below and panic in the INVARIANTS case. 1554 */ 1555 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab) - UMA_MAX_WASTE; 1556 totsize = wsize; 1557 objsize = UMA_SMALLEST_UNIT; 1558 while (totsize >= wsize) { 1559 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / 1560 (objsize + UMA_FRITM_SZ); 1561 totsize *= (UMA_FRITM_SZ + objsize); 1562 objsize++; 1563 } 1564 if (objsize > UMA_SMALLEST_UNIT) 1565 objsize--; 1566 uma_max_ipers = UMA_SLAB_SIZE / objsize; 1567 1568 wsize = UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt) - UMA_MAX_WASTE; 1569 totsize = wsize; 1570 objsize = UMA_SMALLEST_UNIT; 1571 while (totsize >= wsize) { 1572 totsize = (UMA_SLAB_SIZE - sizeof(struct uma_slab_refcnt)) / 1573 (objsize + UMA_FRITMREF_SZ); 1574 totsize *= (UMA_FRITMREF_SZ + objsize); 1575 objsize++; 1576 } 1577 if (objsize > UMA_SMALLEST_UNIT) 1578 objsize--; 1579 uma_max_ipers_ref = UMA_SLAB_SIZE / objsize; 1580 1581 KASSERT((uma_max_ipers_ref <= 255) && (uma_max_ipers <= 255), 1582 ("uma_startup: calculated uma_max_ipers values too large!")); 1583 1584#ifdef UMA_DEBUG 1585 printf("Calculated uma_max_ipers (for OFFPAGE) is %d\n", uma_max_ipers); 1586 printf("Calculated uma_max_ipers_slab (for OFFPAGE) is %d\n", 1587 uma_max_ipers_ref); 1588#endif 1589 1590 /* "manually" create the initial zone */ 1591 args.name = "UMA Kegs"; 1592 args.size = sizeof(struct uma_keg); 1593 args.ctor = keg_ctor; 1594 args.dtor = keg_dtor; 1595 args.uminit = zero_init; 1596 args.fini = NULL; 1597 args.keg = &masterkeg; 1598 args.align = 32 - 1; 1599 args.flags = UMA_ZFLAG_INTERNAL; 1600 /* The initial zone has no Per cpu queues so it's smaller */ 1601 zone_ctor(kegs, sizeof(struct uma_zone), &args, M_WAITOK); 1602 1603#ifdef UMA_DEBUG 1604 printf("Filling boot free list.\n"); 1605#endif 1606 for (i = 0; i < UMA_BOOT_PAGES; i++) { 1607 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 1608 slab->us_data = (u_int8_t *)slab; 1609 slab->us_flags = UMA_SLAB_BOOT; 1610 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 1611 uma_boot_free++; 1612 } 1613 1614#ifdef UMA_DEBUG 1615 printf("Creating uma zone headers zone and keg.\n"); 1616#endif 1617 args.name = "UMA Zones"; 1618 args.size = sizeof(struct uma_zone) + 1619 (sizeof(struct uma_cache) * (mp_maxid + 1)); 1620 args.ctor = zone_ctor; 1621 args.dtor = zone_dtor; 1622 args.uminit = zero_init; 1623 args.fini = NULL; 1624 args.keg = NULL; 1625 args.align = 32 - 1; 1626 args.flags = UMA_ZFLAG_INTERNAL; 1627 /* The initial zone has no Per cpu queues so it's smaller */ 1628 zone_ctor(zones, sizeof(struct uma_zone), &args, M_WAITOK); 1629 1630#ifdef UMA_DEBUG 1631 printf("Initializing pcpu cache locks.\n"); 1632#endif 1633 /* Initialize the pcpu cache lock set once and for all */ 1634 for (i = 0; i <= mp_maxid; i++) 1635 CPU_LOCK_INIT(i); 1636 1637#ifdef UMA_DEBUG 1638 printf("Creating slab and hash zones.\n"); 1639#endif 1640 1641 /* 1642 * This is the max number of free list items we'll have with 1643 * offpage slabs. 1644 */ 1645 slabsize = uma_max_ipers * UMA_FRITM_SZ; 1646 slabsize += sizeof(struct uma_slab); 1647 1648 /* Now make a zone for slab headers */ 1649 slabzone = uma_zcreate("UMA Slabs", 1650 slabsize, 1651 NULL, NULL, NULL, NULL, 1652 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1653 1654 /* 1655 * We also create a zone for the bigger slabs with reference 1656 * counts in them, to accomodate UMA_ZONE_REFCNT zones. 1657 */ 1658 slabsize = uma_max_ipers_ref * UMA_FRITMREF_SZ; 1659 slabsize += sizeof(struct uma_slab_refcnt); 1660 slabrefzone = uma_zcreate("UMA RCntSlabs", 1661 slabsize, 1662 NULL, NULL, NULL, NULL, 1663 UMA_ALIGN_PTR, 1664 UMA_ZFLAG_INTERNAL); 1665 1666 hashzone = uma_zcreate("UMA Hash", 1667 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 1668 NULL, NULL, NULL, NULL, 1669 UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 1670 1671 bucket_init(); 1672 1673#ifdef UMA_MD_SMALL_ALLOC 1674 booted = 1; 1675#endif 1676 1677#ifdef UMA_DEBUG 1678 printf("UMA startup complete.\n"); 1679#endif 1680} 1681 1682/* see uma.h */ 1683void 1684uma_startup2(void) 1685{ 1686 booted = 1; 1687 bucket_enable(); 1688#ifdef UMA_DEBUG 1689 printf("UMA startup2 complete.\n"); 1690#endif 1691} 1692 1693/* 1694 * Initialize our callout handle 1695 * 1696 */ 1697 1698static void 1699uma_startup3(void) 1700{ 1701#ifdef UMA_DEBUG 1702 printf("Starting callout.\n"); 1703#endif 1704 callout_init(&uma_callout, CALLOUT_MPSAFE); 1705 callout_reset(&uma_callout, UMA_TIMEOUT * hz, uma_timeout, NULL); 1706#ifdef UMA_DEBUG 1707 printf("UMA startup3 complete.\n"); 1708#endif 1709} 1710 1711static uma_zone_t 1712uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 1713 int align, u_int16_t flags) 1714{ 1715 struct uma_kctor_args args; 1716 1717 args.size = size; 1718 args.uminit = uminit; 1719 args.fini = fini; 1720 args.align = align; 1721 args.flags = flags; 1722 args.zone = zone; 1723 return (uma_zalloc_internal(kegs, &args, M_WAITOK)); 1724} 1725 1726/* See uma.h */ 1727uma_zone_t 1728uma_zcreate(char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 1729 uma_init uminit, uma_fini fini, int align, u_int16_t flags) 1730 1731{ 1732 struct uma_zctor_args args; 1733 1734 /* This stuff is essential for the zone ctor */ 1735 args.name = name; 1736 args.size = size; 1737 args.ctor = ctor; 1738 args.dtor = dtor; 1739 args.uminit = uminit; 1740 args.fini = fini; 1741 args.align = align; 1742 args.flags = flags; 1743 args.keg = NULL; 1744 1745 return (uma_zalloc_internal(zones, &args, M_WAITOK)); 1746} 1747 1748/* See uma.h */ 1749uma_zone_t 1750uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor dtor, 1751 uma_init zinit, uma_fini zfini, uma_zone_t master) 1752{ 1753 struct uma_zctor_args args; 1754 1755 args.name = name; 1756 args.size = master->uz_keg->uk_size; 1757 args.ctor = ctor; 1758 args.dtor = dtor; 1759 args.uminit = zinit; 1760 args.fini = zfini; 1761 args.align = master->uz_keg->uk_align; 1762 args.flags = master->uz_keg->uk_flags | UMA_ZONE_SECONDARY; 1763 args.keg = master->uz_keg; 1764 1765 return (uma_zalloc_internal(zones, &args, M_WAITOK)); 1766} 1767 1768/* See uma.h */ 1769void 1770uma_zdestroy(uma_zone_t zone) 1771{ 1772 uma_zfree_internal(zones, zone, NULL, SKIP_NONE); 1773} 1774 1775/* See uma.h */ 1776void * 1777uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 1778{ 1779 void *item; 1780 uma_cache_t cache; 1781 uma_bucket_t bucket; 1782 int cpu; 1783 int badness; 1784 1785 /* This is the fast path allocation */ 1786#ifdef UMA_DEBUG_ALLOC_1 1787 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 1788#endif 1789 CTR3(KTR_UMA, "uma_zalloc_arg thread %x zone %s flags %d", curthread, 1790 zone->uz_name, flags); 1791 1792 if (!(flags & M_NOWAIT)) { 1793 KASSERT(curthread->td_intr_nesting_level == 0, 1794 ("malloc(M_WAITOK) in interrupt context")); 1795 if (nosleepwithlocks) { 1796#ifdef WITNESS 1797 badness = WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, 1798 NULL, 1799 "malloc(M_WAITOK) of \"%s\", forcing M_NOWAIT", 1800 zone->uz_name); 1801#else 1802 badness = 1; 1803#endif 1804 } else { 1805 badness = 0; 1806#ifdef WITNESS 1807 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 1808 "malloc(M_WAITOK) of \"%s\"", zone->uz_name); 1809#endif 1810 } 1811 if (badness) { 1812 flags &= ~M_WAITOK; 1813 flags |= M_NOWAIT; 1814 } 1815 } 1816 1817zalloc_restart: 1818 cpu = PCPU_GET(cpuid); 1819 CPU_LOCK(cpu); 1820 cache = &zone->uz_cpu[cpu]; 1821 1822zalloc_start: 1823 bucket = cache->uc_allocbucket; 1824 1825 if (bucket) { 1826 if (bucket->ub_cnt > 0) { 1827 bucket->ub_cnt--; 1828 item = bucket->ub_bucket[bucket->ub_cnt]; 1829#ifdef INVARIANTS 1830 bucket->ub_bucket[bucket->ub_cnt] = NULL; 1831#endif 1832 KASSERT(item != NULL, 1833 ("uma_zalloc: Bucket pointer mangled.")); 1834 cache->uc_allocs++; 1835#ifdef INVARIANTS 1836 ZONE_LOCK(zone); 1837 uma_dbg_alloc(zone, NULL, item); 1838 ZONE_UNLOCK(zone); 1839#endif 1840 CPU_UNLOCK(cpu); 1841 if (zone->uz_ctor != NULL) { 1842 if (zone->uz_ctor(item, zone->uz_keg->uk_size, 1843 udata, flags) != 0) { 1844 uma_zfree_internal(zone, item, udata, 1845 SKIP_DTOR); 1846 return (NULL); 1847 } 1848 } 1849 if (flags & M_ZERO) 1850 bzero(item, zone->uz_keg->uk_size); 1851 return (item); 1852 } else if (cache->uc_freebucket) { 1853 /* 1854 * We have run out of items in our allocbucket. 1855 * See if we can switch with our free bucket. 1856 */ 1857 if (cache->uc_freebucket->ub_cnt > 0) { 1858#ifdef UMA_DEBUG_ALLOC 1859 printf("uma_zalloc: Swapping empty with" 1860 " alloc.\n"); 1861#endif 1862 bucket = cache->uc_freebucket; 1863 cache->uc_freebucket = cache->uc_allocbucket; 1864 cache->uc_allocbucket = bucket; 1865 1866 goto zalloc_start; 1867 } 1868 } 1869 } 1870 ZONE_LOCK(zone); 1871 /* Since we have locked the zone we may as well send back our stats */ 1872 zone->uz_allocs += cache->uc_allocs; 1873 cache->uc_allocs = 0; 1874 1875 /* Our old one is now a free bucket */ 1876 if (cache->uc_allocbucket) { 1877 KASSERT(cache->uc_allocbucket->ub_cnt == 0, 1878 ("uma_zalloc_arg: Freeing a non free bucket.")); 1879 LIST_INSERT_HEAD(&zone->uz_free_bucket, 1880 cache->uc_allocbucket, ub_link); 1881 cache->uc_allocbucket = NULL; 1882 } 1883 1884 /* Check the free list for a new alloc bucket */ 1885 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 1886 KASSERT(bucket->ub_cnt != 0, 1887 ("uma_zalloc_arg: Returning an empty bucket.")); 1888 1889 LIST_REMOVE(bucket, ub_link); 1890 cache->uc_allocbucket = bucket; 1891 ZONE_UNLOCK(zone); 1892 goto zalloc_start; 1893 } 1894 /* We are no longer associated with this cpu!!! */ 1895 CPU_UNLOCK(cpu); 1896 1897 /* Bump up our uz_count so we get here less */ 1898 if (zone->uz_count < BUCKET_MAX) 1899 zone->uz_count++; 1900 1901 /* 1902 * Now lets just fill a bucket and put it on the free list. If that 1903 * works we'll restart the allocation from the begining. 1904 */ 1905 if (uma_zalloc_bucket(zone, flags)) { 1906 ZONE_UNLOCK(zone); 1907 goto zalloc_restart; 1908 } 1909 ZONE_UNLOCK(zone); 1910 /* 1911 * We may not be able to get a bucket so return an actual item. 1912 */ 1913#ifdef UMA_DEBUG 1914 printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 1915#endif 1916 1917 return (uma_zalloc_internal(zone, udata, flags)); 1918} 1919 1920static uma_slab_t 1921uma_zone_slab(uma_zone_t zone, int flags) 1922{ 1923 uma_slab_t slab; 1924 uma_keg_t keg; 1925 1926 keg = zone->uz_keg; 1927 1928 /* 1929 * This is to prevent us from recursively trying to allocate 1930 * buckets. The problem is that if an allocation forces us to 1931 * grab a new bucket we will call page_alloc, which will go off 1932 * and cause the vm to allocate vm_map_entries. If we need new 1933 * buckets there too we will recurse in kmem_alloc and bad 1934 * things happen. So instead we return a NULL bucket, and make 1935 * the code that allocates buckets smart enough to deal with it 1936 * 1937 * XXX: While we want this protection for the bucket zones so that 1938 * recursion from the VM is handled (and the calling code that 1939 * allocates buckets knows how to deal with it), we do not want 1940 * to prevent allocation from the slab header zones (slabzone 1941 * and slabrefzone) if uk_recurse is not zero for them. The 1942 * reason is that it could lead to NULL being returned for 1943 * slab header allocations even in the M_WAITOK case, and the 1944 * caller can't handle that. 1945 */ 1946 if (keg->uk_flags & UMA_ZFLAG_INTERNAL && keg->uk_recurse != 0) 1947 if ((zone != slabzone) && (zone != slabrefzone)) 1948 return (NULL); 1949 1950 slab = NULL; 1951 1952 for (;;) { 1953 /* 1954 * Find a slab with some space. Prefer slabs that are partially 1955 * used over those that are totally full. This helps to reduce 1956 * fragmentation. 1957 */ 1958 if (keg->uk_free != 0) { 1959 if (!LIST_EMPTY(&keg->uk_part_slab)) { 1960 slab = LIST_FIRST(&keg->uk_part_slab); 1961 } else { 1962 slab = LIST_FIRST(&keg->uk_free_slab); 1963 LIST_REMOVE(slab, us_link); 1964 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, 1965 us_link); 1966 } 1967 return (slab); 1968 } 1969 1970 /* 1971 * M_NOVM means don't ask at all! 1972 */ 1973 if (flags & M_NOVM) 1974 break; 1975 1976 if (keg->uk_maxpages && 1977 keg->uk_pages >= keg->uk_maxpages) { 1978 keg->uk_flags |= UMA_ZFLAG_FULL; 1979 1980 if (flags & M_NOWAIT) 1981 break; 1982 else 1983 msleep(keg, &keg->uk_lock, PVM, 1984 "zonelimit", 0); 1985 continue; 1986 } 1987 keg->uk_recurse++; 1988 slab = slab_zalloc(zone, flags); 1989 keg->uk_recurse--; 1990 1991 /* 1992 * If we got a slab here it's safe to mark it partially used 1993 * and return. We assume that the caller is going to remove 1994 * at least one item. 1995 */ 1996 if (slab) { 1997 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 1998 return (slab); 1999 } 2000 /* 2001 * We might not have been able to get a slab but another cpu 2002 * could have while we were unlocked. Check again before we 2003 * fail. 2004 */ 2005 if (flags & M_NOWAIT) 2006 flags |= M_NOVM; 2007 } 2008 return (slab); 2009} 2010 2011static void * 2012uma_slab_alloc(uma_zone_t zone, uma_slab_t slab) 2013{ 2014 uma_keg_t keg; 2015 uma_slabrefcnt_t slabref; 2016 void *item; 2017 u_int8_t freei; 2018 2019 keg = zone->uz_keg; 2020 2021 freei = slab->us_firstfree; 2022 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2023 slabref = (uma_slabrefcnt_t)slab; 2024 slab->us_firstfree = slabref->us_freelist[freei].us_item; 2025 } else { 2026 slab->us_firstfree = slab->us_freelist[freei].us_item; 2027 } 2028 item = slab->us_data + (keg->uk_rsize * freei); 2029 2030 slab->us_freecount--; 2031 keg->uk_free--; 2032#ifdef INVARIANTS 2033 uma_dbg_alloc(zone, slab, item); 2034#endif 2035 /* Move this slab to the full list */ 2036 if (slab->us_freecount == 0) { 2037 LIST_REMOVE(slab, us_link); 2038 LIST_INSERT_HEAD(&keg->uk_full_slab, slab, us_link); 2039 } 2040 2041 return (item); 2042} 2043 2044static int 2045uma_zalloc_bucket(uma_zone_t zone, int flags) 2046{ 2047 uma_bucket_t bucket; 2048 uma_slab_t slab; 2049 int16_t saved; 2050 int max, origflags = flags; 2051 2052 /* 2053 * Try this zone's free list first so we don't allocate extra buckets. 2054 */ 2055 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2056 KASSERT(bucket->ub_cnt == 0, 2057 ("uma_zalloc_bucket: Bucket on free list is not empty.")); 2058 LIST_REMOVE(bucket, ub_link); 2059 } else { 2060 int bflags; 2061 2062 bflags = (flags & ~M_ZERO); 2063 if (zone->uz_keg->uk_flags & UMA_ZFLAG_CACHEONLY) 2064 bflags |= M_NOVM; 2065 2066 ZONE_UNLOCK(zone); 2067 bucket = bucket_alloc(zone->uz_count, bflags); 2068 ZONE_LOCK(zone); 2069 } 2070 2071 if (bucket == NULL) 2072 return (0); 2073 2074#ifdef SMP 2075 /* 2076 * This code is here to limit the number of simultaneous bucket fills 2077 * for any given zone to the number of per cpu caches in this zone. This 2078 * is done so that we don't allocate more memory than we really need. 2079 */ 2080 if (zone->uz_fills >= mp_ncpus) 2081 goto done; 2082 2083#endif 2084 zone->uz_fills++; 2085 2086 max = MIN(bucket->ub_entries, zone->uz_count); 2087 /* Try to keep the buckets totally full */ 2088 saved = bucket->ub_cnt; 2089 while (bucket->ub_cnt < max && 2090 (slab = uma_zone_slab(zone, flags)) != NULL) { 2091 while (slab->us_freecount && bucket->ub_cnt < max) { 2092 bucket->ub_bucket[bucket->ub_cnt++] = 2093 uma_slab_alloc(zone, slab); 2094 } 2095 2096 /* Don't block on the next fill */ 2097 flags |= M_NOWAIT; 2098 } 2099 2100 /* 2101 * We unlock here because we need to call the zone's init. 2102 * It should be safe to unlock because the slab dealt with 2103 * above is already on the appropriate list within the keg 2104 * and the bucket we filled is not yet on any list, so we 2105 * own it. 2106 */ 2107 if (zone->uz_init != NULL) { 2108 int i; 2109 2110 ZONE_UNLOCK(zone); 2111 for (i = saved; i < bucket->ub_cnt; i++) 2112 if (zone->uz_init(bucket->ub_bucket[i], 2113 zone->uz_keg->uk_size, origflags) != 0) 2114 break; 2115 /* 2116 * If we couldn't initialize the whole bucket, put the 2117 * rest back onto the freelist. 2118 */ 2119 if (i != bucket->ub_cnt) { 2120 int j; 2121 2122 for (j = i; j < bucket->ub_cnt; j++) { 2123 uma_zfree_internal(zone, bucket->ub_bucket[j], 2124 NULL, SKIP_FINI); 2125#ifdef INVARIANTS 2126 bucket->ub_bucket[j] = NULL; 2127#endif 2128 } 2129 bucket->ub_cnt = i; 2130 } 2131 ZONE_LOCK(zone); 2132 } 2133 2134 zone->uz_fills--; 2135 if (bucket->ub_cnt != 0) { 2136 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2137 bucket, ub_link); 2138 return (1); 2139 } 2140#ifdef SMP 2141done: 2142#endif 2143 bucket_free(bucket); 2144 2145 return (0); 2146} 2147/* 2148 * Allocates an item for an internal zone 2149 * 2150 * Arguments 2151 * zone The zone to alloc for. 2152 * udata The data to be passed to the constructor. 2153 * flags M_WAITOK, M_NOWAIT, M_ZERO. 2154 * 2155 * Returns 2156 * NULL if there is no memory and M_NOWAIT is set 2157 * An item if successful 2158 */ 2159 2160static void * 2161uma_zalloc_internal(uma_zone_t zone, void *udata, int flags) 2162{ 2163 uma_keg_t keg; 2164 uma_slab_t slab; 2165 void *item; 2166 2167 item = NULL; 2168 keg = zone->uz_keg; 2169 2170#ifdef UMA_DEBUG_ALLOC 2171 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 2172#endif 2173 ZONE_LOCK(zone); 2174 2175 slab = uma_zone_slab(zone, flags); 2176 if (slab == NULL) { 2177 ZONE_UNLOCK(zone); 2178 return (NULL); 2179 } 2180 2181 item = uma_slab_alloc(zone, slab); 2182 2183 ZONE_UNLOCK(zone); 2184 2185 /* 2186 * We have to call both the zone's init (not the keg's init) 2187 * and the zone's ctor. This is because the item is going from 2188 * a keg slab directly to the user, and the user is expecting it 2189 * to be both zone-init'd as well as zone-ctor'd. 2190 */ 2191 if (zone->uz_init != NULL) { 2192 if (zone->uz_init(item, keg->uk_size, flags) != 0) { 2193 uma_zfree_internal(zone, item, udata, SKIP_FINI); 2194 return (NULL); 2195 } 2196 } 2197 if (zone->uz_ctor != NULL) { 2198 if (zone->uz_ctor(item, keg->uk_size, udata, flags) != 0) { 2199 uma_zfree_internal(zone, item, udata, SKIP_DTOR); 2200 return (NULL); 2201 } 2202 } 2203 if (flags & M_ZERO) 2204 bzero(item, keg->uk_size); 2205 2206 return (item); 2207} 2208 2209/* See uma.h */ 2210void 2211uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 2212{ 2213 uma_keg_t keg; 2214 uma_cache_t cache; 2215 uma_bucket_t bucket; 2216 int bflags; 2217 int cpu; 2218 enum zfreeskip skip; 2219 2220 /* This is the fast path free */ 2221 skip = SKIP_NONE; 2222 keg = zone->uz_keg; 2223 2224#ifdef UMA_DEBUG_ALLOC_1 2225 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 2226#endif 2227 CTR2(KTR_UMA, "uma_zfree_arg thread %x zone %s", curthread, 2228 zone->uz_name); 2229 2230 /* 2231 * The race here is acceptable. If we miss it we'll just have to wait 2232 * a little longer for the limits to be reset. 2233 */ 2234 2235 if (keg->uk_flags & UMA_ZFLAG_FULL) 2236 goto zfree_internal; 2237 2238 if (zone->uz_dtor) { 2239 zone->uz_dtor(item, keg->uk_size, udata); 2240 skip = SKIP_DTOR; 2241 } 2242 2243zfree_restart: 2244 cpu = PCPU_GET(cpuid); 2245 CPU_LOCK(cpu); 2246 cache = &zone->uz_cpu[cpu]; 2247 2248zfree_start: 2249 bucket = cache->uc_freebucket; 2250 2251 if (bucket) { 2252 /* 2253 * Do we have room in our bucket? It is OK for this uz count 2254 * check to be slightly out of sync. 2255 */ 2256 2257 if (bucket->ub_cnt < bucket->ub_entries) { 2258 KASSERT(bucket->ub_bucket[bucket->ub_cnt] == NULL, 2259 ("uma_zfree: Freeing to non free bucket index.")); 2260 bucket->ub_bucket[bucket->ub_cnt] = item; 2261 bucket->ub_cnt++; 2262#ifdef INVARIANTS 2263 ZONE_LOCK(zone); 2264 if (keg->uk_flags & UMA_ZONE_MALLOC) 2265 uma_dbg_free(zone, udata, item); 2266 else 2267 uma_dbg_free(zone, NULL, item); 2268 ZONE_UNLOCK(zone); 2269#endif 2270 CPU_UNLOCK(cpu); 2271 return; 2272 } else if (cache->uc_allocbucket) { 2273#ifdef UMA_DEBUG_ALLOC 2274 printf("uma_zfree: Swapping buckets.\n"); 2275#endif 2276 /* 2277 * We have run out of space in our freebucket. 2278 * See if we can switch with our alloc bucket. 2279 */ 2280 if (cache->uc_allocbucket->ub_cnt < 2281 cache->uc_freebucket->ub_cnt) { 2282 bucket = cache->uc_freebucket; 2283 cache->uc_freebucket = cache->uc_allocbucket; 2284 cache->uc_allocbucket = bucket; 2285 goto zfree_start; 2286 } 2287 } 2288 } 2289 /* 2290 * We can get here for two reasons: 2291 * 2292 * 1) The buckets are NULL 2293 * 2) The alloc and free buckets are both somewhat full. 2294 */ 2295 2296 ZONE_LOCK(zone); 2297 2298 bucket = cache->uc_freebucket; 2299 cache->uc_freebucket = NULL; 2300 2301 /* Can we throw this on the zone full list? */ 2302 if (bucket != NULL) { 2303#ifdef UMA_DEBUG_ALLOC 2304 printf("uma_zfree: Putting old bucket on the free list.\n"); 2305#endif 2306 /* ub_cnt is pointing to the last free item */ 2307 KASSERT(bucket->ub_cnt != 0, 2308 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 2309 LIST_INSERT_HEAD(&zone->uz_full_bucket, 2310 bucket, ub_link); 2311 } 2312 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 2313 LIST_REMOVE(bucket, ub_link); 2314 ZONE_UNLOCK(zone); 2315 cache->uc_freebucket = bucket; 2316 goto zfree_start; 2317 } 2318 /* We're done with this CPU now */ 2319 CPU_UNLOCK(cpu); 2320 2321 /* And the zone.. */ 2322 ZONE_UNLOCK(zone); 2323 2324#ifdef UMA_DEBUG_ALLOC 2325 printf("uma_zfree: Allocating new free bucket.\n"); 2326#endif 2327 bflags = M_NOWAIT; 2328 2329 if (keg->uk_flags & UMA_ZFLAG_CACHEONLY) 2330 bflags |= M_NOVM; 2331 bucket = bucket_alloc(zone->uz_count, bflags); 2332 if (bucket) { 2333 ZONE_LOCK(zone); 2334 LIST_INSERT_HEAD(&zone->uz_free_bucket, 2335 bucket, ub_link); 2336 ZONE_UNLOCK(zone); 2337 goto zfree_restart; 2338 } 2339 2340 /* 2341 * If nothing else caught this, we'll just do an internal free. 2342 */ 2343 2344zfree_internal: 2345 2346#ifdef INVARIANTS 2347 /* 2348 * If we need to skip the dtor and the uma_dbg_free in 2349 * uma_zfree_internal because we've already called the dtor 2350 * above, but we ended up here, then we need to make sure 2351 * that we take care of the uma_dbg_free immediately. 2352 */ 2353 if (skip) { 2354 ZONE_LOCK(zone); 2355 if (keg->uk_flags & UMA_ZONE_MALLOC) 2356 uma_dbg_free(zone, udata, item); 2357 else 2358 uma_dbg_free(zone, NULL, item); 2359 ZONE_UNLOCK(zone); 2360 } 2361#endif 2362 uma_zfree_internal(zone, item, udata, skip); 2363 2364 return; 2365} 2366 2367/* 2368 * Frees an item to an INTERNAL zone or allocates a free bucket 2369 * 2370 * Arguments: 2371 * zone The zone to free to 2372 * item The item we're freeing 2373 * udata User supplied data for the dtor 2374 * skip Skip dtors and finis 2375 */ 2376static void 2377uma_zfree_internal(uma_zone_t zone, void *item, void *udata, 2378 enum zfreeskip skip) 2379{ 2380 uma_slab_t slab; 2381 uma_slabrefcnt_t slabref; 2382 uma_keg_t keg; 2383 u_int8_t *mem; 2384 u_int8_t freei; 2385 2386 keg = zone->uz_keg; 2387 2388 if (skip < SKIP_DTOR && zone->uz_dtor) 2389 zone->uz_dtor(item, keg->uk_size, udata); 2390 if (skip < SKIP_FINI && zone->uz_fini) 2391 zone->uz_fini(item, keg->uk_size); 2392 2393 ZONE_LOCK(zone); 2394 2395 if (!(keg->uk_flags & UMA_ZONE_MALLOC)) { 2396 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 2397 if (keg->uk_flags & UMA_ZONE_HASH) 2398 slab = hash_sfind(&keg->uk_hash, mem); 2399 else { 2400 mem += keg->uk_pgoff; 2401 slab = (uma_slab_t)mem; 2402 } 2403 } else { 2404 slab = (uma_slab_t)udata; 2405 } 2406 2407 /* Do we need to remove from any lists? */ 2408 if (slab->us_freecount+1 == keg->uk_ipers) { 2409 LIST_REMOVE(slab, us_link); 2410 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2411 } else if (slab->us_freecount == 0) { 2412 LIST_REMOVE(slab, us_link); 2413 LIST_INSERT_HEAD(&keg->uk_part_slab, slab, us_link); 2414 } 2415 2416 /* Slab management stuff */ 2417 freei = ((unsigned long)item - (unsigned long)slab->us_data) 2418 / keg->uk_rsize; 2419 2420#ifdef INVARIANTS 2421 if (!skip) 2422 uma_dbg_free(zone, slab, item); 2423#endif 2424 2425 if (keg->uk_flags & UMA_ZONE_REFCNT) { 2426 slabref = (uma_slabrefcnt_t)slab; 2427 slabref->us_freelist[freei].us_item = slab->us_firstfree; 2428 } else { 2429 slab->us_freelist[freei].us_item = slab->us_firstfree; 2430 } 2431 slab->us_firstfree = freei; 2432 slab->us_freecount++; 2433 2434 /* Zone statistics */ 2435 keg->uk_free++; 2436 2437 if (keg->uk_flags & UMA_ZFLAG_FULL) { 2438 if (keg->uk_pages < keg->uk_maxpages) 2439 keg->uk_flags &= ~UMA_ZFLAG_FULL; 2440 2441 /* We can handle one more allocation */ 2442 wakeup_one(keg); 2443 } 2444 2445 ZONE_UNLOCK(zone); 2446} 2447 2448/* See uma.h */ 2449void 2450uma_zone_set_max(uma_zone_t zone, int nitems) 2451{ 2452 uma_keg_t keg; 2453 2454 keg = zone->uz_keg; 2455 ZONE_LOCK(zone); 2456 if (keg->uk_ppera > 1) 2457 keg->uk_maxpages = nitems * keg->uk_ppera; 2458 else 2459 keg->uk_maxpages = nitems / keg->uk_ipers; 2460 2461 if (keg->uk_maxpages * keg->uk_ipers < nitems) 2462 keg->uk_maxpages++; 2463 2464 ZONE_UNLOCK(zone); 2465} 2466 2467/* See uma.h */ 2468void 2469uma_zone_set_init(uma_zone_t zone, uma_init uminit) 2470{ 2471 ZONE_LOCK(zone); 2472 KASSERT(zone->uz_keg->uk_pages == 0, 2473 ("uma_zone_set_init on non-empty keg")); 2474 zone->uz_keg->uk_init = uminit; 2475 ZONE_UNLOCK(zone); 2476} 2477 2478/* See uma.h */ 2479void 2480uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 2481{ 2482 ZONE_LOCK(zone); 2483 KASSERT(zone->uz_keg->uk_pages == 0, 2484 ("uma_zone_set_fini on non-empty keg")); 2485 zone->uz_keg->uk_fini = fini; 2486 ZONE_UNLOCK(zone); 2487} 2488 2489/* See uma.h */ 2490void 2491uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 2492{ 2493 ZONE_LOCK(zone); 2494 KASSERT(zone->uz_keg->uk_pages == 0, 2495 ("uma_zone_set_zinit on non-empty keg")); 2496 zone->uz_init = zinit; 2497 ZONE_UNLOCK(zone); 2498} 2499 2500/* See uma.h */ 2501void 2502uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 2503{ 2504 ZONE_LOCK(zone); 2505 KASSERT(zone->uz_keg->uk_pages == 0, 2506 ("uma_zone_set_zfini on non-empty keg")); 2507 zone->uz_fini = zfini; 2508 ZONE_UNLOCK(zone); 2509} 2510 2511/* See uma.h */ 2512/* XXX uk_freef is not actually used with the zone locked */ 2513void 2514uma_zone_set_freef(uma_zone_t zone, uma_free freef) 2515{ 2516 ZONE_LOCK(zone); 2517 zone->uz_keg->uk_freef = freef; 2518 ZONE_UNLOCK(zone); 2519} 2520 2521/* See uma.h */ 2522/* XXX uk_allocf is not actually used with the zone locked */ 2523void 2524uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 2525{ 2526 ZONE_LOCK(zone); 2527 zone->uz_keg->uk_flags |= UMA_ZFLAG_PRIVALLOC; 2528 zone->uz_keg->uk_allocf = allocf; 2529 ZONE_UNLOCK(zone); 2530} 2531 2532/* See uma.h */ 2533int 2534uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 2535{ 2536 uma_keg_t keg; 2537 vm_offset_t kva; 2538 int pages; 2539 2540 keg = zone->uz_keg; 2541 pages = count / keg->uk_ipers; 2542 2543 if (pages * keg->uk_ipers < count) 2544 pages++; 2545 2546 kva = kmem_alloc_nofault(kernel_map, pages * UMA_SLAB_SIZE); 2547 2548 if (kva == 0) 2549 return (0); 2550 if (obj == NULL) { 2551 obj = vm_object_allocate(OBJT_DEFAULT, 2552 pages); 2553 } else { 2554 VM_OBJECT_LOCK_INIT(obj, "uma object"); 2555 _vm_object_allocate(OBJT_DEFAULT, 2556 pages, obj); 2557 } 2558 ZONE_LOCK(zone); 2559 keg->uk_kva = kva; 2560 keg->uk_obj = obj; 2561 keg->uk_maxpages = pages; 2562 keg->uk_allocf = obj_alloc; 2563 keg->uk_flags |= UMA_ZONE_NOFREE | UMA_ZFLAG_PRIVALLOC; 2564 ZONE_UNLOCK(zone); 2565 return (1); 2566} 2567 2568/* See uma.h */ 2569void 2570uma_prealloc(uma_zone_t zone, int items) 2571{ 2572 int slabs; 2573 uma_slab_t slab; 2574 uma_keg_t keg; 2575 2576 keg = zone->uz_keg; 2577 ZONE_LOCK(zone); 2578 slabs = items / keg->uk_ipers; 2579 if (slabs * keg->uk_ipers < items) 2580 slabs++; 2581 while (slabs > 0) { 2582 slab = slab_zalloc(zone, M_WAITOK); 2583 LIST_INSERT_HEAD(&keg->uk_free_slab, slab, us_link); 2584 slabs--; 2585 } 2586 ZONE_UNLOCK(zone); 2587} 2588 2589/* See uma.h */ 2590u_int32_t * 2591uma_find_refcnt(uma_zone_t zone, void *item) 2592{ 2593 uma_slabrefcnt_t slabref; 2594 uma_keg_t keg; 2595 u_int32_t *refcnt; 2596 int idx; 2597 2598 keg = zone->uz_keg; 2599 slabref = (uma_slabrefcnt_t)vtoslab((vm_offset_t)item & 2600 (~UMA_SLAB_MASK)); 2601 KASSERT(slabref != NULL && slabref->us_keg->uk_flags & UMA_ZONE_REFCNT, 2602 ("uma_find_refcnt(): zone possibly not UMA_ZONE_REFCNT")); 2603 idx = ((unsigned long)item - (unsigned long)slabref->us_data) 2604 / keg->uk_rsize; 2605 refcnt = &slabref->us_freelist[idx].us_refcnt; 2606 return refcnt; 2607} 2608 2609/* See uma.h */ 2610void 2611uma_reclaim(void) 2612{ 2613#ifdef UMA_DEBUG 2614 printf("UMA: vm asked us to release pages!\n"); 2615#endif 2616 bucket_enable(); 2617 zone_foreach(zone_drain); 2618 /* 2619 * Some slabs may have been freed but this zone will be visited early 2620 * we visit again so that we can free pages that are empty once other 2621 * zones are drained. We have to do the same for buckets. 2622 */ 2623 zone_drain(slabzone); 2624 zone_drain(slabrefzone); 2625 bucket_zone_drain(); 2626} 2627 2628void * 2629uma_large_malloc(int size, int wait) 2630{ 2631 void *mem; 2632 uma_slab_t slab; 2633 u_int8_t flags; 2634 2635 slab = uma_zalloc_internal(slabzone, NULL, wait); 2636 if (slab == NULL) 2637 return (NULL); 2638 mem = page_alloc(NULL, size, &flags, wait); 2639 if (mem) { 2640 vsetslab((vm_offset_t)mem, slab); 2641 slab->us_data = mem; 2642 slab->us_flags = flags | UMA_SLAB_MALLOC; 2643 slab->us_size = size; 2644 } else { 2645 uma_zfree_internal(slabzone, slab, NULL, 0); 2646 } 2647 2648 return (mem); 2649} 2650 2651void 2652uma_large_free(uma_slab_t slab) 2653{ 2654 vsetobj((vm_offset_t)slab->us_data, kmem_object); 2655 page_free(slab->us_data, slab->us_size, slab->us_flags); 2656 uma_zfree_internal(slabzone, slab, NULL, 0); 2657} 2658 2659void 2660uma_print_stats(void) 2661{ 2662 zone_foreach(uma_print_zone); 2663} 2664 2665static void 2666slab_print(uma_slab_t slab) 2667{ 2668 printf("slab: keg %p, data %p, freecount %d, firstfree %d\n", 2669 slab->us_keg, slab->us_data, slab->us_freecount, 2670 slab->us_firstfree); 2671} 2672 2673static void 2674cache_print(uma_cache_t cache) 2675{ 2676 printf("alloc: %p(%d), free: %p(%d)\n", 2677 cache->uc_allocbucket, 2678 cache->uc_allocbucket?cache->uc_allocbucket->ub_cnt:0, 2679 cache->uc_freebucket, 2680 cache->uc_freebucket?cache->uc_freebucket->ub_cnt:0); 2681} 2682 2683void 2684uma_print_zone(uma_zone_t zone) 2685{ 2686 uma_cache_t cache; 2687 uma_keg_t keg; 2688 uma_slab_t slab; 2689 int i; 2690 2691 keg = zone->uz_keg; 2692 printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 2693 zone->uz_name, zone, keg->uk_size, keg->uk_rsize, keg->uk_flags, 2694 keg->uk_ipers, keg->uk_ppera, 2695 (keg->uk_ipers * keg->uk_pages) - keg->uk_free, keg->uk_free); 2696 printf("Part slabs:\n"); 2697 LIST_FOREACH(slab, &keg->uk_part_slab, us_link) 2698 slab_print(slab); 2699 printf("Free slabs:\n"); 2700 LIST_FOREACH(slab, &keg->uk_free_slab, us_link) 2701 slab_print(slab); 2702 printf("Full slabs:\n"); 2703 LIST_FOREACH(slab, &keg->uk_full_slab, us_link) 2704 slab_print(slab); 2705 for (i = 0; i <= mp_maxid; i++) { 2706 if (CPU_ABSENT(i)) 2707 continue; 2708 cache = &zone->uz_cpu[i]; 2709 printf("CPU %d Cache:\n", i); 2710 cache_print(cache); 2711 } 2712} 2713 2714/* 2715 * Sysctl handler for vm.zone 2716 * 2717 * stolen from vm_zone.c 2718 */ 2719static int 2720sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 2721{ 2722 int error, len, cnt; 2723 const int linesize = 128; /* conservative */ 2724 int totalfree; 2725 char *tmpbuf, *offset; 2726 uma_zone_t z; 2727 uma_keg_t zk; 2728 char *p; 2729 int cpu; 2730 int cachefree; 2731 uma_bucket_t bucket; 2732 uma_cache_t cache; 2733 2734 cnt = 0; 2735 mtx_lock(&uma_mtx); 2736 LIST_FOREACH(zk, &uma_kegs, uk_link) { 2737 LIST_FOREACH(z, &zk->uk_zones, uz_link) 2738 cnt++; 2739 } 2740 mtx_unlock(&uma_mtx); 2741 MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, 2742 M_TEMP, M_WAITOK); 2743 len = snprintf(tmpbuf, linesize, 2744 "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); 2745 if (cnt == 0) 2746 tmpbuf[len - 1] = '\0'; 2747 error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); 2748 if (error || cnt == 0) 2749 goto out; 2750 offset = tmpbuf; 2751 mtx_lock(&uma_mtx); 2752 LIST_FOREACH(zk, &uma_kegs, uk_link) { 2753 LIST_FOREACH(z, &zk->uk_zones, uz_link) { 2754 if (cnt == 0) /* list may have changed size */ 2755 break; 2756 if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) { 2757 for (cpu = 0; cpu <= mp_maxid; cpu++) { 2758 if (CPU_ABSENT(cpu)) 2759 continue; 2760 CPU_LOCK(cpu); 2761 } 2762 } 2763 ZONE_LOCK(z); 2764 cachefree = 0; 2765 if (!(zk->uk_flags & UMA_ZFLAG_INTERNAL)) { 2766 for (cpu = 0; cpu <= mp_maxid; cpu++) { 2767 if (CPU_ABSENT(cpu)) 2768 continue; 2769 cache = &z->uz_cpu[cpu]; 2770 if (cache->uc_allocbucket != NULL) 2771 cachefree += cache->uc_allocbucket->ub_cnt; 2772 if (cache->uc_freebucket != NULL) 2773 cachefree += cache->uc_freebucket->ub_cnt; 2774 CPU_UNLOCK(cpu); 2775 } 2776 } 2777 LIST_FOREACH(bucket, &z->uz_full_bucket, ub_link) { 2778 cachefree += bucket->ub_cnt; 2779 } 2780 totalfree = zk->uk_free + cachefree; 2781 len = snprintf(offset, linesize, 2782 "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", 2783 z->uz_name, zk->uk_size, 2784 zk->uk_maxpages * zk->uk_ipers, 2785 (zk->uk_ipers * (zk->uk_pages / zk->uk_ppera)) - totalfree, 2786 totalfree, 2787 (unsigned long long)z->uz_allocs); 2788 ZONE_UNLOCK(z); 2789 for (p = offset + 12; p > offset && *p == ' '; --p) 2790 /* nothing */ ; 2791 p[1] = ':'; 2792 cnt--; 2793 offset += len; 2794 } 2795 } 2796 mtx_unlock(&uma_mtx); 2797 *offset++ = '\0'; 2798 error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); 2799out: 2800 FREE(tmpbuf, M_TEMP); 2801 return (error); 2802}
|