1/* 2 * Copyright (c) 2002, Jeffrey Roberson <jroberson@chesapeake.net> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 *
|
26 * $FreeBSD: head/sys/vm/uma_core.c 94159 2002-04-08 02:42:55Z jeff $
|
26 * $FreeBSD: head/sys/vm/uma_core.c 94161 2002-04-08 04:48:58Z jeff $ |
27 * 28 */ 29 30/* 31 * uma_core.c Implementation of the Universal Memory allocator 32 * 33 * This allocator is intended to replace the multitude of similar object caches 34 * in the standard FreeBSD kernel. The intent is to be flexible as well as 35 * effecient. A primary design goal is to return unused memory to the rest of 36 * the system. This will make the system as a whole more flexible due to the 37 * ability to move memory to subsystems which most need it instead of leaving 38 * pools of reserved memory unused. 39 * 40 * The basic ideas stem from similar slab/zone based allocators whose algorithms 41 * are well known. 42 * 43 */ 44 45/* 46 * TODO: 47 * - Improve memory usage for large allocations 48 * - Improve INVARIANTS (0xdeadc0de write out) 49 * - Investigate cache size adjustments 50 */ 51 52/* I should really use ktr.. */ 53/* 54#define UMA_DEBUG 1 55#define UMA_DEBUG_ALLOC 1 56#define UMA_DEBUG_ALLOC_1 1 57*/ 58 59 60#include "opt_param.h" 61#include <sys/param.h> 62#include <sys/systm.h> 63#include <sys/kernel.h> 64#include <sys/types.h> 65#include <sys/queue.h> 66#include <sys/malloc.h> 67#include <sys/lock.h> 68#include <sys/sysctl.h> 69#include <machine/types.h> 70#include <sys/mutex.h> 71#include <sys/smp.h> 72 73#include <vm/vm.h> 74#include <vm/vm_object.h> 75#include <vm/vm_page.h> 76#include <vm/vm_param.h> 77#include <vm/vm_map.h> 78#include <vm/vm_kern.h> 79#include <vm/vm_extern.h> 80#include <vm/uma.h> 81#include <vm/uma_int.h> 82 83/* 84 * This is the zone from which all zones are spawned. The idea is that even 85 * the zone heads are allocated from the allocator, so we use the bss section 86 * to bootstrap us. 87 */ 88static struct uma_zone master_zone; 89static uma_zone_t zones = &master_zone; 90 91/* This is the zone from which all of uma_slab_t's are allocated. */ 92static uma_zone_t slabzone; 93 94/* 95 * The initial hash tables come out of this zone so they can be allocated 96 * prior to malloc coming up. 97 */ 98static uma_zone_t hashzone; 99 100/* 101 * Zone that buckets come from. 102 */ 103static uma_zone_t bucketzone; 104 105/* Linked list of all zones in the system */ 106static LIST_HEAD(,uma_zone) uma_zones = LIST_HEAD_INITIALIZER(&uma_zones); 107 108/* This mutex protects the zone list */ 109static struct mtx uma_mtx; 110 111/* Linked list of boot time pages */ 112static LIST_HEAD(,uma_slab) uma_boot_pages = 113 LIST_HEAD_INITIALIZER(&uma_boot_pages); 114 115/* Count of free boottime pages */ 116static int uma_boot_free = 0; 117 118/* Is the VM done starting up? */ 119static int booted = 0; 120 121/* This is the handle used to schedule our working set calculator */ 122static struct callout uma_callout; 123 124/* This is mp_maxid + 1, for use while looping over each cpu */ 125static int maxcpu; 126 127/* 128 * This structure is passed as the zone ctor arg so that I don't have to create 129 * a special allocation function just for zones. 130 */ 131struct uma_zctor_args { 132 char *name; 133 int size; 134 uma_ctor ctor; 135 uma_dtor dtor; 136 uma_init uminit; 137 uma_fini fini; 138 int align; 139 u_int16_t flags; 140}; 141 142/* 143 * This is the malloc hash table which is used to find the zone that a 144 * malloc allocation came from. It is not currently resizeable. The 145 * memory for the actual hash bucket is allocated in kmeminit. 146 */ 147struct uma_hash mhash; 148struct uma_hash *mallochash = &mhash; 149 150/* Prototypes.. */ 151 152static void *obj_alloc(uma_zone_t, int, u_int8_t *, int); 153static void *page_alloc(uma_zone_t, int, u_int8_t *, int); 154static void page_free(void *, int, u_int8_t); 155static uma_slab_t slab_zalloc(uma_zone_t, int); 156static void cache_drain(uma_zone_t); 157static void bucket_drain(uma_zone_t, uma_bucket_t); 158static void zone_drain(uma_zone_t); 159static void zone_ctor(void *, int, void *);
|
160static void zone_dtor(void *, int, void *); |
161static void zero_init(void *, int); 162static void zone_small_init(uma_zone_t zone); 163static void zone_large_init(uma_zone_t zone); 164static void zone_foreach(void (*zfunc)(uma_zone_t)); 165static void zone_timeout(uma_zone_t zone); 166static void hash_expand(struct uma_hash *);
|
167static void hash_free(struct uma_hash *hash); |
168static void uma_timeout(void *); 169static void uma_startup3(void); 170static void *uma_zalloc_internal(uma_zone_t, void *, int, uma_bucket_t); 171static void uma_zfree_internal(uma_zone_t, 172 void *, void *, int); 173void uma_print_zone(uma_zone_t); 174void uma_print_stats(void); 175static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS); 176 177SYSCTL_OID(_vm, OID_AUTO, zone, CTLTYPE_STRING|CTLFLAG_RD, 178 NULL, 0, sysctl_vm_zone, "A", "Zone Info"); 179SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 180 181 182/* 183 * Routine called by timeout which is used to fire off some time interval 184 * based calculations. (working set, stats, etc.) 185 * 186 * Arguments: 187 * arg Unused 188 * 189 * Returns: 190 * Nothing 191 */ 192static void 193uma_timeout(void *unused) 194{ 195 zone_foreach(zone_timeout); 196 197 /* Reschedule this event */ 198 callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 199} 200 201/* 202 * Routine to perform timeout driven calculations. This does the working set 203 * as well as hash expanding, and per cpu statistics aggregation. 204 * 205 * Arguments: 206 * zone The zone to operate on 207 * 208 * Returns: 209 * Nothing 210 */ 211static void 212zone_timeout(uma_zone_t zone) 213{ 214 uma_cache_t cache; 215 u_int64_t alloc; 216 int free; 217 int cpu; 218 219 alloc = 0; 220 free = 0; 221 222 /* 223 * Aggregate per cpu cache statistics back to the zone. 224 * 225 * I may rewrite this to set a flag in the per cpu cache instead of 226 * locking. If the flag is not cleared on the next round I will have 227 * to lock and do it here instead so that the statistics don't get too 228 * far out of sync. 229 */ 230 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) { 231 for (cpu = 0; cpu < maxcpu; cpu++) { 232 if (CPU_ABSENT(cpu)) 233 continue; 234 CPU_LOCK(zone, cpu); 235 cache = &zone->uz_cpu[cpu]; 236 /* Add them up, and reset */ 237 alloc += cache->uc_allocs; 238 cache->uc_allocs = 0; 239 if (cache->uc_allocbucket) 240 free += cache->uc_allocbucket->ub_ptr + 1; 241 if (cache->uc_freebucket) 242 free += cache->uc_freebucket->ub_ptr + 1; 243 CPU_UNLOCK(zone, cpu); 244 } 245 } 246 247 /* Now push these stats back into the zone.. */ 248 ZONE_LOCK(zone); 249 zone->uz_allocs += alloc; 250 251 /* 252 * cachefree is an instantanious snapshot of what is in the per cpu 253 * caches, not an accurate counter 254 */ 255 zone->uz_cachefree = free; 256 257 /* 258 * Expand the zone hash table. 259 * 260 * This is done if the number of slabs is larger than the hash size. 261 * What I'm trying to do here is completely reduce collisions. This 262 * may be a little aggressive. Should I allow for two collisions max? 263 */ 264 265 if ((zone->uz_flags & UMA_ZFLAG_OFFPAGE) && 266 !(zone->uz_flags & UMA_ZFLAG_MALLOC)) { 267 if (zone->uz_pages / zone->uz_ppera 268 >= zone->uz_hash.uh_hashsize) 269 hash_expand(&zone->uz_hash); 270 } 271 272 /* 273 * Here we compute the working set size as the total number of items 274 * left outstanding since the last time interval. This is slightly 275 * suboptimal. What we really want is the highest number of outstanding 276 * items during the last time quantum. This should be close enough. 277 * 278 * The working set size is used to throttle the zone_drain function. 279 * We don't want to return memory that we may need again immediately. 280 */ 281 alloc = zone->uz_allocs - zone->uz_oallocs; 282 zone->uz_oallocs = zone->uz_allocs; 283 zone->uz_wssize = alloc; 284 285 ZONE_UNLOCK(zone); 286} 287 288/* 289 * Expands the hash table for OFFPAGE zones. This is done from zone_timeout 290 * to reduce collisions. This must not be done in the regular allocation path, 291 * otherwise, we can recurse on the vm while allocating pages. 292 * 293 * Arguments: 294 * hash The hash you want to expand by a factor of two. 295 * 296 * Returns: 297 * Nothing 298 * 299 * Discussion: 300 */ 301static void 302hash_expand(struct uma_hash *hash) 303{ 304 struct slabhead *newhash; 305 struct slabhead *oldhash; 306 uma_slab_t slab;
|
305 int hzonefree;
306 int hashsize;
|
307 int oldsize; 308 int newsize; |
309 int alloc; 310 int hval; 311 int i; 312 313 314 /* 315 * Remember the old hash size and see if it has to go back to the 316 * hash zone, or malloc. The hash zone is used for the initial hash 317 */ 318
|
317 hashsize = hash->uh_hashsize;
|
319 oldsize = hash->uh_hashsize; |
320 oldhash = hash->uh_slab_hash; 321
|
320 if (hashsize == UMA_HASH_SIZE_INIT)
321 hzonefree = 1;
322 else
323 hzonefree = 0;
324
325
|
322 /* We're just going to go to a power of two greater */ 323 if (hash->uh_hashsize) {
|
328 alloc = sizeof(hash->uh_slab_hash[0]) * (hash->uh_hashsize * 2);
|
324 newsize = oldsize * 2; 325 alloc = sizeof(hash->uh_slab_hash[0]) * newsize; |
326 /* XXX Shouldn't be abusing DEVBUF here */ 327 newhash = (struct slabhead *)malloc(alloc, M_DEVBUF, M_NOWAIT); 328 if (newhash == NULL) { 329 return; 330 }
|
334 hash->uh_hashsize *= 2;
|
331 } else { 332 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 333 newhash = uma_zalloc_internal(hashzone, NULL, M_WAITOK, NULL);
|
338 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
|
334 newsize = UMA_HASH_SIZE_INIT; |
335 } 336 337 bzero(newhash, alloc); 338
|
343 hash->uh_hashmask = hash->uh_hashsize - 1;
|
339 hash->uh_hashmask = newsize - 1; |
340 341 /* 342 * I need to investigate hash algorithms for resizing without a 343 * full rehash. 344 */ 345
|
350 for (i = 0; i < hashsize; i++)
|
346 for (i = 0; i < oldsize; i++) |
347 while (!SLIST_EMPTY(&hash->uh_slab_hash[i])) { 348 slab = SLIST_FIRST(&hash->uh_slab_hash[i]); 349 SLIST_REMOVE_HEAD(&hash->uh_slab_hash[i], us_hlink); 350 hval = UMA_HASH(hash, slab->us_data); 351 SLIST_INSERT_HEAD(&newhash[hval], slab, us_hlink); 352 } 353
|
358 if (hash->uh_slab_hash) {
359 if (hzonefree)
360 uma_zfree_internal(hashzone,
361 hash->uh_slab_hash, NULL, 0);
362 else
363 free(hash->uh_slab_hash, M_DEVBUF);
364 }
|
354 if (oldhash) 355 hash_free(hash); 356 |
357 hash->uh_slab_hash = newhash;
|
358 hash->uh_hashsize = newsize; |
359 360 return; 361} 362
|
363static void 364hash_free(struct uma_hash *hash) 365{ 366 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 367 uma_zfree_internal(hashzone, 368 hash->uh_slab_hash, NULL, 0); 369 else 370 free(hash->uh_slab_hash, M_DEVBUF); 371 372 hash->uh_slab_hash = NULL; 373} 374 |
375/* 376 * Frees all outstanding items in a bucket 377 * 378 * Arguments: 379 * zone The zone to free to, must be unlocked. 380 * bucket The free/alloc bucket with items, cpu queue must be locked. 381 * 382 * Returns: 383 * Nothing 384 */ 385 386static void 387bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 388{ 389 uma_slab_t slab; 390 int mzone; 391 void *item; 392 393 if (bucket == NULL) 394 return; 395 396 slab = NULL; 397 mzone = 0; 398 399 /* We have to lookup the slab again for malloc.. */ 400 if (zone->uz_flags & UMA_ZFLAG_MALLOC) 401 mzone = 1; 402 403 while (bucket->ub_ptr > -1) { 404 item = bucket->ub_bucket[bucket->ub_ptr]; 405#ifdef INVARIANTS 406 bucket->ub_bucket[bucket->ub_ptr] = NULL; 407 KASSERT(item != NULL, 408 ("bucket_drain: botched ptr, item is NULL")); 409#endif 410 bucket->ub_ptr--; 411 /* 412 * This is extremely inefficient. The slab pointer was passed 413 * to uma_zfree_arg, but we lost it because the buckets don't 414 * hold them. This will go away when free() gets a size passed 415 * to it. 416 */ 417 if (mzone) 418 slab = hash_sfind(mallochash, 419 (u_int8_t *)((unsigned long)item & 420 (~UMA_SLAB_MASK))); 421 uma_zfree_internal(zone, item, slab, 1); 422 } 423} 424 425/* 426 * Drains the per cpu caches for a zone. 427 * 428 * Arguments: 429 * zone The zone to drain, must be unlocked. 430 * 431 * Returns: 432 * Nothing 433 * 434 * This function returns with the zone locked so that the per cpu queues can 435 * not be filled until zone_drain is finished. 436 * 437 */ 438static void 439cache_drain(uma_zone_t zone) 440{ 441 uma_bucket_t bucket; 442 uma_cache_t cache; 443 int cpu; 444 445 /* 446 * Flush out the per cpu queues. 447 * 448 * XXX This causes unnecessary thrashing due to immediately having 449 * empty per cpu queues. I need to improve this. 450 */ 451 452 /* 453 * We have to lock each cpu cache before locking the zone 454 */ 455 ZONE_UNLOCK(zone); 456 457 for (cpu = 0; cpu < maxcpu; cpu++) { 458 if (CPU_ABSENT(cpu)) 459 continue; 460 CPU_LOCK(zone, cpu); 461 cache = &zone->uz_cpu[cpu]; 462 bucket_drain(zone, cache->uc_allocbucket); 463 bucket_drain(zone, cache->uc_freebucket); 464 } 465 466 /* 467 * Drain the bucket queues and free the buckets, we just keep two per 468 * cpu (alloc/free). 469 */ 470 ZONE_LOCK(zone); 471 while ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 472 LIST_REMOVE(bucket, ub_link); 473 ZONE_UNLOCK(zone); 474 bucket_drain(zone, bucket); 475 uma_zfree_internal(bucketzone, bucket, NULL, 0); 476 ZONE_LOCK(zone); 477 } 478 479 /* Now we do the free queue.. */ 480 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 481 LIST_REMOVE(bucket, ub_link); 482 uma_zfree_internal(bucketzone, bucket, NULL, 0); 483 } 484 485 /* We unlock here, but they will all block until the zone is unlocked */ 486 for (cpu = 0; cpu < maxcpu; cpu++) { 487 if (CPU_ABSENT(cpu)) 488 continue; 489 CPU_UNLOCK(zone, cpu); 490 } 491 492 zone->uz_cachefree = 0; 493} 494 495/* 496 * Frees pages from a zone back to the system. This is done on demand from 497 * the pageout daemon. 498 * 499 * Arguments: 500 * zone The zone to free pages from
|
501 * all Should we drain all items? |
502 * 503 * Returns: 504 * Nothing. 505 */ 506static void 507zone_drain(uma_zone_t zone) 508{ 509 uma_slab_t slab; 510 uma_slab_t n; 511 u_int64_t extra; 512 u_int8_t flags; 513 u_int8_t *mem; 514 int i; 515 516 /* 517 * We don't want to take pages from staticly allocated zones at this 518 * time 519 */ 520 if (zone->uz_flags & UMA_ZFLAG_NOFREE || zone->uz_freef == NULL) 521 return; 522 523 ZONE_LOCK(zone); 524 525 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 526 cache_drain(zone); 527 528 if (zone->uz_free < zone->uz_wssize) 529 goto finished; 530#ifdef UMA_DEBUG 531 printf("%s working set size: %llu free items: %u\n", 532 zone->uz_name, (unsigned long long)zone->uz_wssize, zone->uz_free); 533#endif
|
528 extra = zone->uz_wssize - zone->uz_free;
|
534 extra = zone->uz_free - zone->uz_wssize; |
535 extra /= zone->uz_ipers; 536 537 /* extra is now the number of extra slabs that we can free */ 538 539 if (extra == 0) 540 goto finished; 541 542 slab = LIST_FIRST(&zone->uz_free_slab); 543 while (slab && extra) { 544 n = LIST_NEXT(slab, us_link); 545 546 /* We have no where to free these to */ 547 if (slab->us_flags & UMA_SLAB_BOOT) { 548 slab = n; 549 continue; 550 } 551 552 LIST_REMOVE(slab, us_link); 553 zone->uz_pages -= zone->uz_ppera; 554 zone->uz_free -= zone->uz_ipers; 555 if (zone->uz_fini) 556 for (i = 0; i < zone->uz_ipers; i++) 557 zone->uz_fini( 558 slab->us_data + (zone->uz_rsize * i), 559 zone->uz_size); 560 flags = slab->us_flags; 561 mem = slab->us_data; 562 if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) { 563 if (zone->uz_flags & UMA_ZFLAG_MALLOC) { 564 UMA_HASH_REMOVE(mallochash, 565 slab, slab->us_data); 566 } else { 567 UMA_HASH_REMOVE(&zone->uz_hash, 568 slab, slab->us_data); 569 } 570 uma_zfree_internal(slabzone, slab, NULL, 0); 571 } else if (zone->uz_flags & UMA_ZFLAG_MALLOC) 572 UMA_HASH_REMOVE(mallochash, slab, slab->us_data); 573#ifdef UMA_DEBUG 574 printf("%s: Returning %d bytes.\n", 575 zone->uz_name, UMA_SLAB_SIZE * zone->uz_ppera); 576#endif 577 zone->uz_freef(mem, UMA_SLAB_SIZE * zone->uz_ppera, flags); 578 579 slab = n; 580 extra--; 581 } 582 583finished: 584 ZONE_UNLOCK(zone); 585} 586 587/* 588 * Allocate a new slab for a zone. This does not insert the slab onto a list. 589 * 590 * Arguments: 591 * zone The zone to allocate slabs for 592 * wait Shall we wait? 593 * 594 * Returns: 595 * The slab that was allocated or NULL if there is no memory and the 596 * caller specified M_NOWAIT. 597 * 598 */ 599static uma_slab_t 600slab_zalloc(uma_zone_t zone, int wait) 601{ 602 uma_slab_t slab; /* Starting slab */ 603 u_int8_t *mem; 604 u_int8_t flags; 605 int i; 606 607 slab = NULL; 608 609#ifdef UMA_DEBUG 610 printf("slab_zalloc: Allocating a new slab for %s\n", zone->uz_name); 611#endif 612 if (zone->uz_maxpages && 613 zone->uz_pages + zone->uz_ppera > zone->uz_maxpages) 614 return (NULL); 615 616 ZONE_UNLOCK(zone); 617 618 if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) { 619 slab = uma_zalloc_internal(slabzone, NULL, wait, NULL); 620 if (slab == NULL) { 621 ZONE_LOCK(zone); 622 return NULL; 623 } 624 } 625 626 if (booted || (zone->uz_flags & UMA_ZFLAG_PRIVALLOC)) { 627 mtx_lock(&Giant); 628 mem = zone->uz_allocf(zone, 629 zone->uz_ppera * UMA_SLAB_SIZE, &flags, wait); 630 mtx_unlock(&Giant); 631 if (mem == NULL) { 632 ZONE_LOCK(zone); 633 return (NULL); 634 } 635 } else { 636 uma_slab_t tmps; 637 638 if (zone->uz_ppera > 1) 639 panic("UMA: Attemping to allocate multiple pages before vm has started.\n"); 640 if (zone->uz_flags & UMA_ZFLAG_MALLOC) 641 panic("Mallocing before uma_startup2 has been called.\n"); 642 if (uma_boot_free == 0) 643 panic("UMA: Ran out of pre init pages, increase UMA_BOOT_PAGES\n"); 644 tmps = LIST_FIRST(&uma_boot_pages); 645 LIST_REMOVE(tmps, us_link); 646 uma_boot_free--; 647 mem = tmps->us_data; 648 } 649 650 ZONE_LOCK(zone); 651 652 /* Alloc slab structure for offpage, otherwise adjust it's position */ 653 if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) { 654 slab = (uma_slab_t )(mem + zone->uz_pgoff); 655 } else { 656 if (!(zone->uz_flags & UMA_ZFLAG_MALLOC)) 657 UMA_HASH_INSERT(&zone->uz_hash, slab, mem); 658 } 659 if (zone->uz_flags & UMA_ZFLAG_MALLOC) { 660#ifdef UMA_DEBUG 661 printf("Inserting %p into malloc hash from slab %p\n", 662 mem, slab); 663#endif 664 /* XXX Yikes! No lock on the malloc hash! */ 665 UMA_HASH_INSERT(mallochash, slab, mem); 666 } 667 668 slab->us_zone = zone; 669 slab->us_data = mem; 670 671 /* 672 * This is intended to spread data out across cache lines. 673 * 674 * This code doesn't seem to work properly on x86, and on alpha 675 * it makes absolutely no performance difference. I'm sure it could 676 * use some tuning, but sun makes outrageous claims about it's 677 * performance. 678 */ 679#if 0 680 if (zone->uz_cachemax) { 681 slab->us_data += zone->uz_cacheoff; 682 zone->uz_cacheoff += UMA_CACHE_INC; 683 if (zone->uz_cacheoff > zone->uz_cachemax) 684 zone->uz_cacheoff = 0; 685 } 686#endif 687 688 slab->us_freecount = zone->uz_ipers; 689 slab->us_firstfree = 0; 690 slab->us_flags = flags; 691 for (i = 0; i < zone->uz_ipers; i++) 692 slab->us_freelist[i] = i+1; 693 694 if (zone->uz_init) 695 for (i = 0; i < zone->uz_ipers; i++) 696 zone->uz_init(slab->us_data + (zone->uz_rsize * i), 697 zone->uz_size); 698 699 zone->uz_pages += zone->uz_ppera; 700 zone->uz_free += zone->uz_ipers; 701 702 return (slab); 703} 704 705/* 706 * Allocates a number of pages from the system 707 * 708 * Arguments: 709 * zone Unused 710 * bytes The number of bytes requested 711 * wait Shall we wait? 712 * 713 * Returns: 714 * A pointer to the alloced memory or possibly 715 * NULL if M_NOWAIT is set. 716 */ 717static void * 718page_alloc(uma_zone_t zone, int bytes, u_int8_t *pflag, int wait) 719{ 720 void *p; /* Returned page */ 721 722 /* 723 * XXX The original zone allocator did this, but I don't think it's 724 * necessary in current. 725 */ 726 727 if (lockstatus(&kernel_map->lock, NULL)) { 728 *pflag = UMA_SLAB_KMEM; 729 p = (void *) kmem_malloc(kmem_map, bytes, wait); 730 } else { 731 *pflag = UMA_SLAB_KMAP; 732 p = (void *) kmem_alloc(kernel_map, bytes); 733 } 734 735 return (p); 736} 737 738/* 739 * Allocates a number of pages from within an object 740 * 741 * Arguments: 742 * zone Unused 743 * bytes The number of bytes requested 744 * wait Shall we wait? 745 * 746 * Returns: 747 * A pointer to the alloced memory or possibly 748 * NULL if M_NOWAIT is set. 749 */ 750static void * 751obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) 752{ 753 vm_offset_t zkva; 754 vm_offset_t retkva; 755 vm_page_t p; 756 int pages; 757 758 retkva = NULL; 759 pages = zone->uz_pages; 760 761 /* 762 * This looks a little weird since we're getting one page at a time 763 */ 764 while (bytes > 0) { 765 p = vm_page_alloc(zone->uz_obj, pages, 766 VM_ALLOC_INTERRUPT); 767 if (p == NULL) 768 return (NULL); 769 770 zkva = zone->uz_kva + pages * PAGE_SIZE; 771 if (retkva == NULL) 772 retkva = zkva; 773 pmap_qenter(zkva, &p, 1); 774 bytes -= PAGE_SIZE; 775 pages += 1; 776 } 777 778 *flags = UMA_SLAB_PRIV; 779 780 return ((void *)retkva); 781} 782 783/* 784 * Frees a number of pages to the system 785 * 786 * Arguments: 787 * mem A pointer to the memory to be freed 788 * size The size of the memory being freed 789 * flags The original p->us_flags field 790 * 791 * Returns: 792 * Nothing 793 * 794 */ 795static void 796page_free(void *mem, int size, u_int8_t flags) 797{ 798 vm_map_t map; 799 if (flags & UMA_SLAB_KMEM) 800 map = kmem_map; 801 else if (flags & UMA_SLAB_KMAP) 802 map = kernel_map; 803 else 804 panic("UMA: page_free used with invalid flags %d\n", flags); 805 806 kmem_free(map, (vm_offset_t)mem, size); 807} 808 809/* 810 * Zero fill initializer 811 * 812 * Arguments/Returns follow uma_init specifications 813 * 814 */ 815static void 816zero_init(void *mem, int size) 817{ 818 bzero(mem, size); 819} 820 821/* 822 * Finish creating a small uma zone. This calculates ipers, and the zone size. 823 * 824 * Arguments 825 * zone The zone we should initialize 826 * 827 * Returns 828 * Nothing 829 */ 830static void 831zone_small_init(uma_zone_t zone) 832{ 833 int rsize; 834 int memused; 835 int ipers; 836 837 rsize = zone->uz_size; 838 839 if (rsize < UMA_SMALLEST_UNIT) 840 rsize = UMA_SMALLEST_UNIT; 841 842 if (rsize & zone->uz_align) 843 rsize = (rsize & ~zone->uz_align) + (zone->uz_align + 1); 844 845 zone->uz_rsize = rsize; 846 847 rsize += 1; /* Account for the byte of linkage */ 848 zone->uz_ipers = (UMA_SLAB_SIZE - sizeof(struct uma_slab)) / rsize; 849 zone->uz_ppera = 1; 850 851 memused = zone->uz_ipers * zone->uz_rsize; 852 853 /* Can we do any better? */ 854 if ((UMA_SLAB_SIZE - memused) >= UMA_MAX_WASTE) { 855 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 856 return; 857 ipers = UMA_SLAB_SIZE / zone->uz_rsize; 858 if (ipers > zone->uz_ipers) { 859 zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 860 zone->uz_ipers = ipers; 861 } 862 } 863 864} 865 866/* 867 * Finish creating a large (> UMA_SLAB_SIZE) uma zone. Just give in and do 868 * OFFPAGE for now. When I can allow for more dynamic slab sizes this will be 869 * more complicated. 870 * 871 * Arguments 872 * zone The zone we should initialize 873 * 874 * Returns 875 * Nothing 876 */ 877static void 878zone_large_init(uma_zone_t zone) 879{ 880 int pages; 881 882 pages = zone->uz_size / UMA_SLAB_SIZE; 883 884 /* Account for remainder */ 885 if ((pages * UMA_SLAB_SIZE) < zone->uz_size) 886 pages++; 887 888 zone->uz_ppera = pages; 889 zone->uz_ipers = 1; 890 891 zone->uz_flags |= UMA_ZFLAG_OFFPAGE; 892 zone->uz_rsize = zone->uz_size; 893} 894 895/* 896 * Zone header ctor. This initializes all fields, locks, etc. And inserts 897 * the zone onto the global zone list. 898 * 899 * Arguments/Returns follow uma_ctor specifications 900 * udata Actually uma_zcreat_args 901 * 902 */ 903 904static void 905zone_ctor(void *mem, int size, void *udata) 906{ 907 struct uma_zctor_args *arg = udata; 908 uma_zone_t zone = mem; 909 int cplen; 910 int cpu; 911 912 bzero(zone, size); 913 zone->uz_name = arg->name; 914 zone->uz_size = arg->size; 915 zone->uz_ctor = arg->ctor; 916 zone->uz_dtor = arg->dtor; 917 zone->uz_init = arg->uminit; 918 zone->uz_align = arg->align; 919 zone->uz_free = 0; 920 zone->uz_pages = 0; 921 zone->uz_flags = 0; 922 zone->uz_allocf = page_alloc; 923 zone->uz_freef = page_free; 924 925 if (arg->flags & UMA_ZONE_ZINIT) 926 zone->uz_init = zero_init; 927 928 if (arg->flags & UMA_ZONE_INTERNAL) 929 zone->uz_flags |= UMA_ZFLAG_INTERNAL; 930 931 if (arg->flags & UMA_ZONE_MALLOC) 932 zone->uz_flags |= UMA_ZFLAG_MALLOC; 933 934 if (arg->flags & UMA_ZONE_NOFREE) 935 zone->uz_flags |= UMA_ZFLAG_NOFREE; 936 937 if (zone->uz_size > UMA_SLAB_SIZE) 938 zone_large_init(zone); 939 else 940 zone_small_init(zone); 941 942 /* We do this so that the per cpu lock name is unique for each zone */ 943 memcpy(zone->uz_lname, "PCPU ", 5); 944 cplen = min(strlen(zone->uz_name) + 1, LOCKNAME_LEN - 6); 945 memcpy(zone->uz_lname+5, zone->uz_name, cplen); 946 zone->uz_lname[LOCKNAME_LEN - 1] = '\0'; 947 948 /* 949 * If we're putting the slab header in the actual page we need to 950 * figure out where in each page it goes. This calculates a right 951 * justified offset into the memory on a ALIGN_PTR boundary. 952 */ 953 if (!(zone->uz_flags & UMA_ZFLAG_OFFPAGE)) { 954 int totsize; 955 int waste; 956 957 /* Size of the slab struct and free list */ 958 totsize = sizeof(struct uma_slab) + zone->uz_ipers; 959 if (totsize & UMA_ALIGN_PTR) 960 totsize = (totsize & ~UMA_ALIGN_PTR) + 961 (UMA_ALIGN_PTR + 1); 962 zone->uz_pgoff = UMA_SLAB_SIZE - totsize; 963 964 waste = zone->uz_pgoff; 965 waste -= (zone->uz_ipers * zone->uz_rsize); 966 967 /* 968 * This calculates how much space we have for cache line size 969 * optimizations. It works by offseting each slab slightly. 970 * Currently it breaks on x86, and so it is disabled. 971 */ 972 973 if (zone->uz_align < UMA_CACHE_INC && waste > UMA_CACHE_INC) { 974 zone->uz_cachemax = waste - UMA_CACHE_INC; 975 zone->uz_cacheoff = 0; 976 } 977 978 totsize = zone->uz_pgoff + sizeof(struct uma_slab) 979 + zone->uz_ipers; 980 /* I don't think it's possible, but I'll make sure anyway */ 981 if (totsize > UMA_SLAB_SIZE) { 982 printf("zone %s ipers %d rsize %d size %d\n", 983 zone->uz_name, zone->uz_ipers, zone->uz_rsize, 984 zone->uz_size); 985 panic("UMA slab won't fit.\n"); 986 } 987 } else { 988 /* hash_expand here to allocate the initial hash table */ 989 hash_expand(&zone->uz_hash); 990 zone->uz_pgoff = 0; 991 } 992 993#ifdef UMA_DEBUG 994 printf("%s(%p) size = %d ipers = %d ppera = %d pgoff = %d\n", 995 zone->uz_name, zone, 996 zone->uz_size, zone->uz_ipers, 997 zone->uz_ppera, zone->uz_pgoff); 998#endif 999 ZONE_LOCK_INIT(zone); 1000 1001 mtx_lock(&uma_mtx); 1002 LIST_INSERT_HEAD(&uma_zones, zone, uz_link); 1003 mtx_unlock(&uma_mtx); 1004 1005 /* 1006 * Some internal zones don't have room allocated for the per cpu 1007 * caches. If we're internal, bail out here. 1008 */ 1009 1010 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 1011 return; 1012 1013 if (zone->uz_ipers < UMA_BUCKET_SIZE) 1014 zone->uz_count = zone->uz_ipers - 1; 1015 else 1016 zone->uz_count = UMA_BUCKET_SIZE - 1; 1017 1018 for (cpu = 0; cpu < maxcpu; cpu++) 1019 CPU_LOCK_INIT(zone, cpu); 1020} 1021
|
1022/* 1023 * Zone header dtor. This frees all data, destroys locks, frees the hash table 1024 * and removes the zone from the global list. 1025 * 1026 * Arguments/Returns follow uma_dtor specifications 1027 * udata unused 1028 */ 1029 1030static void 1031zone_dtor(void *arg, int size, void *udata) 1032{ 1033 uma_zone_t zone; 1034 int cpu; 1035 1036 zone = (uma_zone_t)arg; 1037 1038 mtx_lock(&uma_mtx); 1039 LIST_REMOVE(zone, uz_link); 1040 mtx_unlock(&uma_mtx); 1041 1042 ZONE_LOCK(zone); 1043 zone->uz_wssize = 0; 1044 ZONE_UNLOCK(zone); 1045 1046 zone_drain(zone); 1047 ZONE_LOCK(zone); 1048 if (zone->uz_free != 0) 1049 printf("Zone %s was not empty. Lost %d pages of memory.\n", 1050 zone->uz_name, zone->uz_pages); 1051 1052 if ((zone->uz_flags & UMA_ZFLAG_INTERNAL) != 0) 1053 for (cpu = 0; cpu < maxcpu; cpu++) 1054 CPU_LOCK_FINI(zone, cpu); 1055 1056 if ((zone->uz_flags & UMA_ZFLAG_OFFPAGE) != 0) 1057 hash_free(&zone->uz_hash); 1058 1059 ZONE_UNLOCK(zone); 1060 ZONE_LOCK_FINI(zone); 1061} |
1062/* 1063 * Traverses every zone in the system and calls a callback 1064 * 1065 * Arguments: 1066 * zfunc A pointer to a function which accepts a zone 1067 * as an argument. 1068 * 1069 * Returns: 1070 * Nothing 1071 */ 1072static void 1073zone_foreach(void (*zfunc)(uma_zone_t)) 1074{ 1075 uma_zone_t zone; 1076 1077 mtx_lock(&uma_mtx); 1078 LIST_FOREACH(zone, &uma_zones, uz_link) { 1079 zfunc(zone); 1080 } 1081 mtx_unlock(&uma_mtx); 1082} 1083 1084/* Public functions */ 1085/* See uma.h */ 1086void 1087uma_startup(void *bootmem) 1088{ 1089 struct uma_zctor_args args; 1090 uma_slab_t slab; 1091 int slabsize; 1092 int i; 1093 1094#ifdef UMA_DEBUG 1095 printf("Creating uma zone headers zone.\n"); 1096#endif 1097#ifdef SMP 1098 maxcpu = mp_maxid + 1; 1099#else 1100 maxcpu = 1; 1101#endif 1102#ifdef UMA_DEBUG 1103 printf("Max cpu = %d, mp_maxid = %d\n", maxcpu, mp_maxid); 1104 Debugger("stop"); 1105#endif 1106 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF); 1107 /* "manually" Create the initial zone */ 1108 args.name = "UMA Zones"; 1109 args.size = sizeof(struct uma_zone) + 1110 (sizeof(struct uma_cache) * (maxcpu - 1)); 1111 args.ctor = zone_ctor;
|
1066 args.dtor = NULL;
|
1112 args.dtor = zone_dtor; |
1113 args.uminit = zero_init; 1114 args.fini = NULL; 1115 args.align = 32 - 1; 1116 args.flags = UMA_ZONE_INTERNAL; 1117 /* The initial zone has no Per cpu queues so it's smaller */ 1118 zone_ctor(zones, sizeof(struct uma_zone), &args); 1119 1120#ifdef UMA_DEBUG 1121 printf("Filling boot free list.\n"); 1122#endif 1123 for (i = 0; i < UMA_BOOT_PAGES; i++) { 1124 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE)); 1125 slab->us_data = (u_int8_t *)slab; 1126 slab->us_flags = UMA_SLAB_BOOT; 1127 LIST_INSERT_HEAD(&uma_boot_pages, slab, us_link); 1128 uma_boot_free++; 1129 } 1130 1131#ifdef UMA_DEBUG 1132 printf("Creating slab zone.\n"); 1133#endif 1134 1135 /* 1136 * This is the max number of free list items we'll have with 1137 * offpage slabs. 1138 */ 1139 1140 slabsize = UMA_SLAB_SIZE - sizeof(struct uma_slab); 1141 slabsize /= UMA_MAX_WASTE; 1142 slabsize++; /* In case there it's rounded */ 1143 slabsize += sizeof(struct uma_slab); 1144 1145 /* Now make a zone for slab headers */ 1146 slabzone = uma_zcreate("UMA Slabs", 1147 slabsize, 1148 NULL, NULL, NULL, NULL, 1149 UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 1150 1151 hashzone = uma_zcreate("UMA Hash", 1152 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 1153 NULL, NULL, NULL, NULL, 1154 UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 1155 1156 bucketzone = uma_zcreate("UMA Buckets", sizeof(struct uma_bucket), 1157 NULL, NULL, NULL, NULL, 1158 UMA_ALIGN_PTR, UMA_ZONE_INTERNAL); 1159 1160 1161#ifdef UMA_DEBUG 1162 printf("UMA startup complete.\n"); 1163#endif 1164} 1165 1166/* see uma.h */ 1167void 1168uma_startup2(void *hashmem, u_long elems) 1169{ 1170 bzero(hashmem, elems * sizeof(void *)); 1171 mallochash->uh_slab_hash = hashmem; 1172 mallochash->uh_hashsize = elems; 1173 mallochash->uh_hashmask = elems - 1; 1174 booted = 1; 1175#ifdef UMA_DEBUG 1176 printf("UMA startup2 complete.\n"); 1177#endif 1178} 1179 1180/* 1181 * Initialize our callout handle 1182 * 1183 */ 1184 1185static void 1186uma_startup3(void) 1187{ 1188#ifdef UMA_DEBUG 1189 printf("Starting callout.\n"); 1190#endif 1191 /* We'll be mpsafe once the vm is locked. */ 1192 callout_init(&uma_callout, 0); 1193 callout_reset(&uma_callout, UMA_WORKING_TIME * hz, uma_timeout, NULL); 1194#ifdef UMA_DEBUG 1195 printf("UMA startup3 complete.\n"); 1196#endif 1197} 1198 1199/* See uma.h */ 1200uma_zone_t 1201uma_zcreate(char *name, int size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, 1202 uma_fini fini, int align, u_int16_t flags) 1203 1204{ 1205 struct uma_zctor_args args; 1206 1207 /* This stuff is essential for the zone ctor */ 1208 args.name = name; 1209 args.size = size; 1210 args.ctor = ctor; 1211 args.dtor = dtor; 1212 args.uminit = uminit; 1213 args.fini = fini; 1214 args.align = align; 1215 args.flags = flags; 1216 1217 return (uma_zalloc_internal(zones, &args, M_WAITOK, NULL)); 1218} 1219 1220/* See uma.h */
|
1221void 1222uma_zdestroy(uma_zone_t zone) 1223{ 1224 uma_zfree_internal(zones, zone, NULL, 0); 1225} 1226 1227/* See uma.h */ |
1228void * 1229uma_zalloc_arg(uma_zone_t zone, void *udata, int wait) 1230{ 1231 void *item; 1232 uma_cache_t cache; 1233 uma_bucket_t bucket; 1234 int cpu; 1235 1236 /* This is the fast path allocation */ 1237#ifdef UMA_DEBUG_ALLOC_1 1238 printf("Allocating one item from %s(%p)\n", zone->uz_name, zone); 1239#endif 1240 1241zalloc_restart: 1242 cpu = PCPU_GET(cpuid); 1243 CPU_LOCK(zone, cpu); 1244 cache = &zone->uz_cpu[cpu]; 1245 1246zalloc_start: 1247 bucket = cache->uc_allocbucket; 1248 1249 if (bucket) { 1250 if (bucket->ub_ptr > -1) { 1251 item = bucket->ub_bucket[bucket->ub_ptr]; 1252#ifdef INVARIANTS 1253 bucket->ub_bucket[bucket->ub_ptr] = NULL; 1254#endif 1255 bucket->ub_ptr--; 1256 KASSERT(item != NULL, 1257 ("uma_zalloc: Bucket pointer mangled.")); 1258 cache->uc_allocs++; 1259 CPU_UNLOCK(zone, cpu); 1260 if (zone->uz_ctor) 1261 zone->uz_ctor(item, zone->uz_size, udata); 1262 return (item); 1263 } else if (cache->uc_freebucket) { 1264 /* 1265 * We have run out of items in our allocbucket. 1266 * See if we can switch with our free bucket. 1267 */ 1268 if (cache->uc_freebucket->ub_ptr > -1) { 1269 uma_bucket_t swap; 1270 1271#ifdef UMA_DEBUG_ALLOC 1272 printf("uma_zalloc: Swapping empty with alloc.\n"); 1273#endif 1274 swap = cache->uc_freebucket; 1275 cache->uc_freebucket = cache->uc_allocbucket; 1276 cache->uc_allocbucket = swap; 1277 1278 goto zalloc_start; 1279 } 1280 } 1281 } 1282 ZONE_LOCK(zone); 1283 /* Since we have locked the zone we may as well send back our stats */ 1284 zone->uz_allocs += cache->uc_allocs; 1285 cache->uc_allocs = 0; 1286 1287 /* Our old one is now a free bucket */ 1288 if (cache->uc_allocbucket) { 1289 KASSERT(cache->uc_allocbucket->ub_ptr == -1, 1290 ("uma_zalloc_arg: Freeing a non free bucket.")); 1291 LIST_INSERT_HEAD(&zone->uz_free_bucket, 1292 cache->uc_allocbucket, ub_link); 1293 cache->uc_allocbucket = NULL; 1294 } 1295 1296 /* Check the free list for a new alloc bucket */ 1297 if ((bucket = LIST_FIRST(&zone->uz_full_bucket)) != NULL) { 1298 KASSERT(bucket->ub_ptr != -1, 1299 ("uma_zalloc_arg: Returning an empty bucket.")); 1300 1301 LIST_REMOVE(bucket, ub_link); 1302 cache->uc_allocbucket = bucket; 1303 ZONE_UNLOCK(zone); 1304 goto zalloc_start; 1305 } 1306 /* Bump up our uz_count so we get here less */ 1307 if (zone->uz_count < UMA_BUCKET_SIZE - 1) 1308 zone->uz_count++; 1309 1310 /* We are no longer associated with this cpu!!! */ 1311 CPU_UNLOCK(zone, cpu); 1312 1313 /* 1314 * Now lets just fill a bucket and put it on the free list. If that 1315 * works we'll restart the allocation from the begining. 1316 * 1317 * Try this zone's free list first so we don't allocate extra buckets. 1318 */ 1319 1320 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) 1321 LIST_REMOVE(bucket, ub_link); 1322 1323 /* Now we no longer need the zone lock. */ 1324 ZONE_UNLOCK(zone); 1325 1326 if (bucket == NULL) 1327 bucket = uma_zalloc_internal(bucketzone, 1328 NULL, wait, NULL); 1329 1330 if (bucket != NULL) { 1331#ifdef INVARIANTS 1332 bzero(bucket, bucketzone->uz_size); 1333#endif 1334 bucket->ub_ptr = -1; 1335 1336 if (uma_zalloc_internal(zone, udata, wait, bucket)) 1337 goto zalloc_restart; 1338 else 1339 uma_zfree_internal(bucketzone, bucket, NULL, 0); 1340 } 1341 /* 1342 * We may not get a bucket if we recurse, so 1343 * return an actual item. 1344 */ 1345#ifdef UMA_DEBUG 1346 printf("uma_zalloc_arg: Bucketzone returned NULL\n"); 1347#endif 1348 1349 return (uma_zalloc_internal(zone, udata, wait, NULL)); 1350} 1351 1352/* 1353 * Allocates an item for an internal zone OR fills a bucket 1354 * 1355 * Arguments 1356 * zone The zone to alloc for. 1357 * udata The data to be passed to the constructor. 1358 * wait M_WAITOK or M_NOWAIT. 1359 * bucket The bucket to fill or NULL 1360 * 1361 * Returns 1362 * NULL if there is no memory and M_NOWAIT is set 1363 * An item if called on an interal zone 1364 * Non NULL if called to fill a bucket and it was successful. 1365 * 1366 * Discussion: 1367 * This was much cleaner before it had to do per cpu caches. It is 1368 * complicated now because it has to handle the simple internal case, and 1369 * the more involved bucket filling and allocation. 1370 */ 1371 1372static void * 1373uma_zalloc_internal(uma_zone_t zone, void *udata, int wait, uma_bucket_t bucket) 1374{ 1375 uma_slab_t slab; 1376 u_int8_t freei; 1377 void *item; 1378 1379 item = NULL; 1380 1381 /* 1382 * This is to stop us from allocating per cpu buckets while we're 1383 * running out of UMA_BOOT_PAGES. Otherwise, we would exhaust the 1384 * boot pages. 1385 */ 1386 1387 if (!booted && zone == bucketzone) 1388 return (NULL); 1389 1390#ifdef UMA_DEBUG_ALLOC 1391 printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name, zone); 1392#endif 1393 ZONE_LOCK(zone); 1394 1395 /* 1396 * This code is here to limit the number of simultaneous bucket fills 1397 * for any given zone to the number of per cpu caches in this zone. This 1398 * is done so that we don't allocate more memory than we really need. 1399 */ 1400 1401 if (bucket) { 1402#ifdef SMP 1403 if (zone->uz_fills >= mp_ncpus) 1404#else 1405 if (zone->uz_fills > 1) 1406#endif 1407 return (NULL); 1408 1409 zone->uz_fills++; 1410 } 1411 1412new_slab: 1413 1414 /* Find a slab with some space */ 1415 if (zone->uz_free) { 1416 if (!LIST_EMPTY(&zone->uz_part_slab)) { 1417 slab = LIST_FIRST(&zone->uz_part_slab); 1418 } else { 1419 slab = LIST_FIRST(&zone->uz_free_slab); 1420 LIST_REMOVE(slab, us_link); 1421 LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 1422 } 1423 } else { 1424 /* 1425 * This is to prevent us from recursively trying to allocate 1426 * buckets. The problem is that if an allocation forces us to 1427 * grab a new bucket we will call page_alloc, which will go off 1428 * and cause the vm to allocate vm_map_entries. If we need new 1429 * buckets there too we will recurse in kmem_alloc and bad 1430 * things happen. So instead we return a NULL bucket, and make 1431 * the code that allocates buckets smart enough to deal with it */ 1432 if (zone == bucketzone && zone->uz_recurse != 0) { 1433 ZONE_UNLOCK(zone); 1434 return (NULL); 1435 } 1436 zone->uz_recurse++; 1437 slab = slab_zalloc(zone, wait); 1438 zone->uz_recurse--; 1439 if (slab) { 1440 LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 1441 /* 1442 * We might not have been able to get a page, but another cpu 1443 * could have while we were unlocked. 1444 */ 1445 } else if (zone->uz_free == 0) { 1446 /* If we're filling a bucket return what we have */ 1447 if (bucket != NULL) 1448 zone->uz_fills--; 1449 ZONE_UNLOCK(zone); 1450 1451 if (bucket != NULL && bucket->ub_ptr != -1) 1452 return (bucket); 1453 else 1454 return (NULL); 1455 } else { 1456 /* Another cpu must have succeeded */ 1457 if ((slab = LIST_FIRST(&zone->uz_part_slab)) == NULL) { 1458 slab = LIST_FIRST(&zone->uz_free_slab); 1459 LIST_REMOVE(slab, us_link); 1460 LIST_INSERT_HEAD(&zone->uz_part_slab, 1461 slab, us_link); 1462 } 1463 } 1464 } 1465 /* 1466 * If this is our first time though put this guy on the list. 1467 */ 1468 if (bucket != NULL && bucket->ub_ptr == -1) 1469 LIST_INSERT_HEAD(&zone->uz_full_bucket, 1470 bucket, ub_link); 1471 1472 1473 while (slab->us_freecount) { 1474 freei = slab->us_firstfree; 1475 slab->us_firstfree = slab->us_freelist[freei]; 1476#ifdef INVARIANTS 1477 slab->us_freelist[freei] = 255; 1478#endif 1479 slab->us_freecount--; 1480 zone->uz_free--; 1481 item = slab->us_data + (zone->uz_rsize * freei); 1482 1483 if (bucket == NULL) { 1484 zone->uz_allocs++; 1485 break; 1486 } 1487 bucket->ub_bucket[++bucket->ub_ptr] = item; 1488 1489 /* Don't overfill the bucket! */ 1490 if (bucket->ub_ptr == zone->uz_count) 1491 break; 1492 } 1493 1494 /* Move this slab to the full list */ 1495 if (slab->us_freecount == 0) { 1496 LIST_REMOVE(slab, us_link); 1497 LIST_INSERT_HEAD(&zone->uz_full_slab, slab, us_link); 1498 } 1499 1500 if (bucket != NULL) { 1501 /* Try to keep the buckets totally full, but don't block */ 1502 if (bucket->ub_ptr < zone->uz_count) { 1503 wait = M_NOWAIT; 1504 goto new_slab; 1505 } else 1506 zone->uz_fills--; 1507 } 1508 1509 ZONE_UNLOCK(zone); 1510 1511 /* Only construct at this time if we're not filling a bucket */ 1512 if (bucket == NULL && zone->uz_ctor != NULL) 1513 zone->uz_ctor(item, zone->uz_size, udata); 1514 1515 return (item); 1516} 1517 1518/* See uma.h */ 1519void 1520uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 1521{ 1522 uma_cache_t cache; 1523 uma_bucket_t bucket; 1524 int cpu; 1525 1526 /* This is the fast path free */ 1527#ifdef UMA_DEBUG_ALLOC_1 1528 printf("Freeing item %p to %s(%p)\n", item, zone->uz_name, zone); 1529#endif 1530zfree_restart: 1531 cpu = PCPU_GET(cpuid); 1532 CPU_LOCK(zone, cpu); 1533 cache = &zone->uz_cpu[cpu]; 1534 1535zfree_start: 1536 bucket = cache->uc_freebucket; 1537 1538 if (bucket) { 1539 /* 1540 * Do we have room in our bucket? It is OK for this uz count 1541 * check to be slightly out of sync. 1542 */ 1543 1544 if (bucket->ub_ptr < zone->uz_count) { 1545 bucket->ub_ptr++; 1546 KASSERT(bucket->ub_bucket[bucket->ub_ptr] == NULL, 1547 ("uma_zfree: Freeing to non free bucket index.")); 1548 bucket->ub_bucket[bucket->ub_ptr] = item; 1549 CPU_UNLOCK(zone, cpu); 1550 if (zone->uz_dtor) 1551 zone->uz_dtor(item, zone->uz_size, udata); 1552 return; 1553 } else if (cache->uc_allocbucket) { 1554#ifdef UMA_DEBUG_ALLOC 1555 printf("uma_zfree: Swapping buckets.\n"); 1556#endif 1557 /* 1558 * We have run out of space in our freebucket. 1559 * See if we can switch with our alloc bucket. 1560 */ 1561 if (cache->uc_allocbucket->ub_ptr < 1562 cache->uc_freebucket->ub_ptr) { 1563 uma_bucket_t swap; 1564 1565 swap = cache->uc_freebucket; 1566 cache->uc_freebucket = cache->uc_allocbucket; 1567 cache->uc_allocbucket = swap; 1568 1569 goto zfree_start; 1570 } 1571 } 1572 } 1573 1574 /* 1575 * We can get here for two reasons: 1576 * 1577 * 1) The buckets are NULL 1578 * 2) The alloc and free buckets are both somewhat full. 1579 * 1580 */ 1581 1582 ZONE_LOCK(zone); 1583 1584 bucket = cache->uc_freebucket; 1585 cache->uc_freebucket = NULL; 1586 1587 /* Can we throw this on the zone full list? */ 1588 if (bucket != NULL) { 1589#ifdef UMA_DEBUG_ALLOC 1590 printf("uma_zfree: Putting old bucket on the free list.\n"); 1591#endif 1592 /* ub_ptr is pointing to the last free item */ 1593 KASSERT(bucket->ub_ptr != -1, 1594 ("uma_zfree: Attempting to insert an empty bucket onto the full list.\n")); 1595 LIST_INSERT_HEAD(&zone->uz_full_bucket, 1596 bucket, ub_link); 1597 } 1598 if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) { 1599 LIST_REMOVE(bucket, ub_link); 1600 ZONE_UNLOCK(zone); 1601 cache->uc_freebucket = bucket; 1602 goto zfree_start; 1603 } 1604 /* We're done with this CPU now */ 1605 CPU_UNLOCK(zone, cpu); 1606 1607 /* And the zone.. */ 1608 ZONE_UNLOCK(zone); 1609 1610#ifdef UMA_DEBUG_ALLOC 1611 printf("uma_zfree: Allocating new free bucket.\n"); 1612#endif 1613 bucket = uma_zalloc_internal(bucketzone, 1614 NULL, M_NOWAIT, NULL); 1615 if (bucket) { 1616#ifdef INVARIANTS 1617 bzero(bucket, bucketzone->uz_size); 1618#endif 1619 bucket->ub_ptr = -1; 1620 ZONE_LOCK(zone); 1621 LIST_INSERT_HEAD(&zone->uz_free_bucket, 1622 bucket, ub_link); 1623 ZONE_UNLOCK(zone); 1624 goto zfree_restart; 1625 } 1626 1627 /* 1628 * If nothing else caught this, we'll just do an internal free. 1629 */ 1630 1631 uma_zfree_internal(zone, item, udata, 0); 1632 1633 return; 1634 1635} 1636 1637/* 1638 * Frees an item to an INTERNAL zone or allocates a free bucket 1639 * 1640 * Arguments: 1641 * zone The zone to free to 1642 * item The item we're freeing 1643 * udata User supplied data for the dtor 1644 * skip Skip the dtor, it was done in uma_zfree_arg 1645 */ 1646 1647static void 1648uma_zfree_internal(uma_zone_t zone, void *item, void *udata, int skip) 1649{ 1650 uma_slab_t slab; 1651 u_int8_t *mem; 1652 u_int8_t freei; 1653 1654 ZONE_LOCK(zone); 1655 1656 if (!(zone->uz_flags & UMA_ZFLAG_MALLOC)) { 1657 mem = (u_int8_t *)((unsigned long)item & (~UMA_SLAB_MASK)); 1658 if (zone->uz_flags & UMA_ZFLAG_OFFPAGE) 1659 slab = hash_sfind(&zone->uz_hash, mem); 1660 else { 1661 mem += zone->uz_pgoff; 1662 slab = (uma_slab_t)mem; 1663 } 1664 } else { 1665 slab = (uma_slab_t)udata; 1666 } 1667 1668 /* Do we need to remove from any lists? */ 1669 if (slab->us_freecount+1 == zone->uz_ipers) { 1670 LIST_REMOVE(slab, us_link); 1671 LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 1672 } else if (slab->us_freecount == 0) { 1673 LIST_REMOVE(slab, us_link); 1674 LIST_INSERT_HEAD(&zone->uz_part_slab, slab, us_link); 1675 } 1676 1677 /* Slab management stuff */ 1678 freei = ((unsigned long)item - (unsigned long)slab->us_data) 1679 / zone->uz_rsize; 1680#ifdef INVARIANTS 1681 if (((freei * zone->uz_rsize) + slab->us_data) != item) 1682 panic("zone: %s(%p) slab %p freed address %p unaligned.\n", 1683 zone->uz_name, zone, slab, item); 1684 if (freei >= zone->uz_ipers) 1685 panic("zone: %s(%p) slab %p freelist %i out of range 0-%d\n", 1686 zone->uz_name, zone, slab, freei, zone->uz_ipers-1); 1687 1688 if (slab->us_freelist[freei] != 255) { 1689 printf("Slab at %p, freei %d = %d.\n", 1690 slab, freei, slab->us_freelist[freei]); 1691 panic("Duplicate free of item %p from zone %p(%s)\n", 1692 item, zone, zone->uz_name); 1693 } 1694#endif 1695 slab->us_freelist[freei] = slab->us_firstfree; 1696 slab->us_firstfree = freei; 1697 slab->us_freecount++; 1698 1699 /* Zone statistics */ 1700 zone->uz_free++; 1701 1702 ZONE_UNLOCK(zone); 1703 1704 if (!skip && zone->uz_dtor) 1705 zone->uz_dtor(item, zone->uz_size, udata); 1706} 1707 1708/* See uma.h */ 1709void 1710uma_zone_set_max(uma_zone_t zone, int nitems) 1711{ 1712 ZONE_LOCK(zone); 1713 if (zone->uz_ppera > 1) 1714 zone->uz_maxpages = nitems / zone->uz_ppera; 1715 else 1716 zone->uz_maxpages = nitems / zone->uz_ipers; 1717 ZONE_UNLOCK(zone); 1718} 1719 1720/* See uma.h */ 1721void 1722uma_zone_set_freef(uma_zone_t zone, uma_free freef) 1723{ 1724 ZONE_LOCK(zone); 1725 1726 zone->uz_freef = freef; 1727 1728 ZONE_UNLOCK(zone); 1729} 1730 1731/* See uma.h */ 1732void 1733uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 1734{ 1735 ZONE_LOCK(zone); 1736 1737 zone->uz_flags |= UMA_ZFLAG_PRIVALLOC; 1738 zone->uz_allocf = allocf; 1739 1740 ZONE_UNLOCK(zone); 1741} 1742 1743/* See uma.h */ 1744int 1745uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) 1746{ 1747 int pages; 1748 vm_offset_t kva; 1749 1750 mtx_lock(&Giant); 1751 1752 pages = count / zone->uz_ipers; 1753 1754 if (pages * zone->uz_ipers < count) 1755 pages++; 1756 1757 kva = kmem_alloc_pageable(kernel_map, pages * UMA_SLAB_SIZE); 1758 1759 if (kva == 0) { 1760 mtx_unlock(&Giant); 1761 return (0); 1762 } 1763 1764 1765 if (obj == NULL) 1766 obj = vm_object_allocate(OBJT_DEFAULT, 1767 zone->uz_maxpages); 1768 else 1769 _vm_object_allocate(OBJT_DEFAULT, 1770 zone->uz_maxpages, obj); 1771 1772 ZONE_LOCK(zone); 1773 zone->uz_kva = kva; 1774 zone->uz_obj = obj; 1775 zone->uz_maxpages = pages; 1776 1777 zone->uz_allocf = obj_alloc; 1778 zone->uz_flags |= UMA_ZFLAG_NOFREE | UMA_ZFLAG_PRIVALLOC; 1779 1780 ZONE_UNLOCK(zone); 1781 mtx_unlock(&Giant); 1782 1783 return (1); 1784} 1785 1786/* See uma.h */ 1787void 1788uma_prealloc(uma_zone_t zone, int items) 1789{ 1790 int slabs; 1791 uma_slab_t slab; 1792 1793 ZONE_LOCK(zone); 1794 slabs = items / zone->uz_ipers; 1795 if (slabs * zone->uz_ipers < items) 1796 slabs++; 1797 1798 while (slabs > 0) { 1799 slab = slab_zalloc(zone, M_WAITOK); 1800 LIST_INSERT_HEAD(&zone->uz_free_slab, slab, us_link); 1801 slabs--; 1802 } 1803 ZONE_UNLOCK(zone); 1804} 1805 1806/* See uma.h */ 1807void 1808uma_reclaim(void) 1809{ 1810 /* 1811 * You might think that the delay below would improve performance since 1812 * the allocator will give away memory that it may ask for immediately. 1813 * Really, it makes things worse, since cpu cycles are so much cheaper 1814 * than disk activity. 1815 */ 1816#if 0 1817 static struct timeval tv = {0}; 1818 struct timeval now; 1819 getmicrouptime(&now); 1820 if (now.tv_sec > tv.tv_sec + 30) 1821 tv = now; 1822 else 1823 return; 1824#endif 1825#ifdef UMA_DEBUG 1826 printf("UMA: vm asked us to release pages!\n"); 1827#endif 1828 zone_foreach(zone_drain); 1829 1830 /* 1831 * Some slabs may have been freed but this zone will be visited early 1832 * we visit again so that we can free pages that are empty once other 1833 * zones are drained. We have to do the same for buckets. 1834 */ 1835 zone_drain(slabzone); 1836 zone_drain(bucketzone); 1837} 1838 1839void * 1840uma_large_malloc(int size, int wait) 1841{ 1842 void *mem; 1843 uma_slab_t slab; 1844 u_int8_t flags; 1845 1846 slab = uma_zalloc_internal(slabzone, NULL, wait, NULL); 1847 if (slab == NULL) 1848 return (NULL); 1849 1850 mem = page_alloc(NULL, size, &flags, wait); 1851 if (mem) { 1852 slab->us_data = mem; 1853 slab->us_flags = flags | UMA_SLAB_MALLOC; 1854 slab->us_size = size; 1855 UMA_HASH_INSERT(mallochash, slab, mem); 1856 } else { 1857 uma_zfree_internal(slabzone, slab, NULL, 0); 1858 } 1859 1860 1861 return (mem); 1862} 1863 1864void 1865uma_large_free(uma_slab_t slab) 1866{ 1867 UMA_HASH_REMOVE(mallochash, slab, slab->us_data); 1868 page_free(slab->us_data, slab->us_size, slab->us_flags); 1869 uma_zfree_internal(slabzone, slab, NULL, 0); 1870} 1871 1872void 1873uma_print_stats(void) 1874{ 1875 zone_foreach(uma_print_zone); 1876} 1877 1878void 1879uma_print_zone(uma_zone_t zone) 1880{ 1881 printf("%s(%p) size %d(%d) flags %d ipers %d ppera %d out %d free %d\n", 1882 zone->uz_name, zone, zone->uz_size, zone->uz_rsize, zone->uz_flags, 1883 zone->uz_ipers, zone->uz_ppera, 1884 (zone->uz_ipers * zone->uz_pages) - zone->uz_free, zone->uz_free); 1885} 1886 1887/* 1888 * Sysctl handler for vm.zone 1889 * 1890 * stolen from vm_zone.c 1891 */ 1892static int 1893sysctl_vm_zone(SYSCTL_HANDLER_ARGS) 1894{ 1895 int error, len, cnt; 1896 const int linesize = 128; /* conservative */ 1897 int totalfree; 1898 char *tmpbuf, *offset; 1899 uma_zone_t z; 1900 char *p; 1901 1902 cnt = 0; 1903 LIST_FOREACH(z, &uma_zones, uz_link) 1904 cnt++; 1905 MALLOC(tmpbuf, char *, (cnt == 0 ? 1 : cnt) * linesize, 1906 M_TEMP, M_WAITOK); 1907 len = snprintf(tmpbuf, linesize, 1908 "\nITEM SIZE LIMIT USED FREE REQUESTS\n\n"); 1909 if (cnt == 0) 1910 tmpbuf[len - 1] = '\0'; 1911 error = SYSCTL_OUT(req, tmpbuf, cnt == 0 ? len-1 : len); 1912 if (error || cnt == 0) 1913 goto out; 1914 offset = tmpbuf; 1915 mtx_lock(&uma_mtx); 1916 LIST_FOREACH(z, &uma_zones, uz_link) { 1917 if (cnt == 0) /* list may have changed size */ 1918 break; 1919 ZONE_LOCK(z); 1920 totalfree = z->uz_free + z->uz_cachefree; 1921 len = snprintf(offset, linesize, 1922 "%-12.12s %6.6u, %8.8u, %6.6u, %6.6u, %8.8llu\n", 1923 z->uz_name, z->uz_size, 1924 z->uz_maxpages * z->uz_ipers, 1925 (z->uz_ipers * (z->uz_pages / z->uz_ppera)) - totalfree, 1926 totalfree, 1927 (unsigned long long)z->uz_allocs); 1928 ZONE_UNLOCK(z); 1929 for (p = offset + 12; p > offset && *p == ' '; --p) 1930 /* nothing */ ; 1931 p[1] = ':'; 1932 cnt--; 1933 offset += len; 1934 } 1935 mtx_unlock(&uma_mtx); 1936 *offset++ = '\0'; 1937 error = SYSCTL_OUT(req, tmpbuf, offset - tmpbuf); 1938out: 1939 FREE(tmpbuf, M_TEMP); 1940 return (error); 1941}
|