1/*
2 * Copyright (c) 2002, Jeffrey Roberson <jroberson@chesapeake.net>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 9 unchanged lines hidden (view full) ---

18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 *
26 * $FreeBSD: head/sys/vm/uma_core.c 94159 2002-04-08 02:42:55Z jeff $
27 *
28 */
29
30/*
31 * uma_core.c Implementation of the Universal Memory allocator
32 *
33 * This allocator is intended to replace the multitude of similar object caches
34 * in the standard FreeBSD kernel. The intent is to be flexible as well as

--- 117 unchanged lines hidden (view full) ---

152static void *obj_alloc(uma_zone_t, int, u_int8_t *, int);
153static void *page_alloc(uma_zone_t, int, u_int8_t *, int);
154static void page_free(void *, int, u_int8_t);
155static uma_slab_t slab_zalloc(uma_zone_t, int);
156static void cache_drain(uma_zone_t);
157static void bucket_drain(uma_zone_t, uma_bucket_t);
158static void zone_drain(uma_zone_t);
159static void zone_ctor(void *, int, void *);
160static void zero_init(void *, int);
161static void zone_small_init(uma_zone_t zone);
162static void zone_large_init(uma_zone_t zone);
163static void zone_foreach(void (*zfunc)(uma_zone_t));
164static void zone_timeout(uma_zone_t zone);
165static void hash_expand(struct uma_hash *);
166static void uma_timeout(void *);
167static void uma_startup3(void);
168static void *uma_zalloc_internal(uma_zone_t, void *, int, uma_bucket_t);
169static void uma_zfree_internal(uma_zone_t,
170 void *, void *, int);
171void uma_print_zone(uma_zone_t);
172void uma_print_stats(void);
173static int sysctl_vm_zone(SYSCTL_HANDLER_ARGS);

--- 123 unchanged lines hidden (view full) ---

297 * Discussion:
298 */
299static void
300hash_expand(struct uma_hash *hash)
301{
302 struct slabhead *newhash;
303 struct slabhead *oldhash;
304 uma_slab_t slab;
305 int hzonefree;
306 int hashsize;
307 int alloc;
308 int hval;
309 int i;
310
311
312 /*
313 * Remember the old hash size and see if it has to go back to the
314 * hash zone, or malloc. The hash zone is used for the initial hash
315 */
316
317 hashsize = hash->uh_hashsize;
318 oldhash = hash->uh_slab_hash;
319
320 if (hashsize == UMA_HASH_SIZE_INIT)
321 hzonefree = 1;
322 else
323 hzonefree = 0;
324
325
326 /* We're just going to go to a power of two greater */
327 if (hash->uh_hashsize) {
328 alloc = sizeof(hash->uh_slab_hash[0]) * (hash->uh_hashsize * 2);
329 /* XXX Shouldn't be abusing DEVBUF here */
330 newhash = (struct slabhead *)malloc(alloc, M_DEVBUF, M_NOWAIT);
331 if (newhash == NULL) {
332 return;
333 }
334 hash->uh_hashsize *= 2;
335 } else {
336 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT;
337 newhash = uma_zalloc_internal(hashzone, NULL, M_WAITOK, NULL);
338 hash->uh_hashsize = UMA_HASH_SIZE_INIT;
339 }
340
341 bzero(newhash, alloc);
342
343 hash->uh_hashmask = hash->uh_hashsize - 1;
344
345 /*
346 * I need to investigate hash algorithms for resizing without a
347 * full rehash.
348 */
349
350 for (i = 0; i < hashsize; i++)
351 while (!SLIST_EMPTY(&hash->uh_slab_hash[i])) {
352 slab = SLIST_FIRST(&hash->uh_slab_hash[i]);
353 SLIST_REMOVE_HEAD(&hash->uh_slab_hash[i], us_hlink);
354 hval = UMA_HASH(hash, slab->us_data);
355 SLIST_INSERT_HEAD(&newhash[hval], slab, us_hlink);
356 }
357
358 if (hash->uh_slab_hash) {
359 if (hzonefree)
360 uma_zfree_internal(hashzone,
361 hash->uh_slab_hash, NULL, 0);
362 else
363 free(hash->uh_slab_hash, M_DEVBUF);
364 }
365 hash->uh_slab_hash = newhash;
366
367 return;
368}
369
370/*
371 * Frees all outstanding items in a bucket
372 *
373 * Arguments:
374 * zone The zone to free to, must be unlocked.
375 * bucket The free/alloc bucket with items, cpu queue must be locked.
376 *
377 * Returns:

--- 110 unchanged lines hidden (view full) ---

488}
489
490/*
491 * Frees pages from a zone back to the system. This is done on demand from
492 * the pageout daemon.
493 *
494 * Arguments:
495 * zone The zone to free pages from
496 *
497 * Returns:
498 * Nothing.
499 */
500static void
501zone_drain(uma_zone_t zone)
502{
503 uma_slab_t slab;

--- 16 unchanged lines hidden (view full) ---

520 cache_drain(zone);
521
522 if (zone->uz_free < zone->uz_wssize)
523 goto finished;
524#ifdef UMA_DEBUG
525 printf("%s working set size: %llu free items: %u\n",
526 zone->uz_name, (unsigned long long)zone->uz_wssize, zone->uz_free);
527#endif
528 extra = zone->uz_wssize - zone->uz_free;
529 extra /= zone->uz_ipers;
530
531 /* extra is now the number of extra slabs that we can free */
532
533 if (extra == 0)
534 goto finished;
535
536 slab = LIST_FIRST(&zone->uz_free_slab);

--- 471 unchanged lines hidden (view full) ---

1008 zone->uz_count = zone->uz_ipers - 1;
1009 else
1010 zone->uz_count = UMA_BUCKET_SIZE - 1;
1011
1012 for (cpu = 0; cpu < maxcpu; cpu++)
1013 CPU_LOCK_INIT(zone, cpu);
1014}
1015
1016/*
1017 * Traverses every zone in the system and calls a callback
1018 *
1019 * Arguments:
1020 * zfunc A pointer to a function which accepts a zone
1021 * as an argument.
1022 *
1023 * Returns:

--- 34 unchanged lines hidden (view full) ---

1058 Debugger("stop");
1059#endif
1060 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1061 /* "manually" Create the initial zone */
1062 args.name = "UMA Zones";
1063 args.size = sizeof(struct uma_zone) +
1064 (sizeof(struct uma_cache) * (maxcpu - 1));
1065 args.ctor = zone_ctor;
1066 args.dtor = NULL;
1067 args.uminit = zero_init;
1068 args.fini = NULL;
1069 args.align = 32 - 1;
1070 args.flags = UMA_ZONE_INTERNAL;
1071 /* The initial zone has no Per cpu queues so it's smaller */
1072 zone_ctor(zones, sizeof(struct uma_zone), &args);
1073
1074#ifdef UMA_DEBUG

--- 92 unchanged lines hidden (view full) ---

1167 args.fini = fini;
1168 args.align = align;
1169 args.flags = flags;
1170
1171 return (uma_zalloc_internal(zones, &args, M_WAITOK, NULL));
1172}
1173
1174/* See uma.h */
1175void *
1176uma_zalloc_arg(uma_zone_t zone, void *udata, int wait)
1177{
1178 void *item;
1179 uma_cache_t cache;
1180 uma_bucket_t bucket;
1181 int cpu;
1182

--- 706 unchanged lines hidden ---