Deleted Added
sdiff udiff text old ( 123073 ) new ( 123126 )
full compact
uma_core.c (123073) uma_core.c (123126)
1/*
2 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 32 unchanged lines hidden (view full) ---

41
42/*
43 * TODO:
44 * - Improve memory usage for large allocations
45 * - Investigate cache size adjustments
46 */
47
48#include <sys/cdefs.h>
1/*
2 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 32 unchanged lines hidden (view full) ---

41
42/*
43 * TODO:
44 * - Improve memory usage for large allocations
45 * - Investigate cache size adjustments
46 */
47
48#include <sys/cdefs.h>
49__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 123073 2003-11-30 22:18:14Z jeff $");
49__FBSDID("$FreeBSD: head/sys/vm/uma_core.c 123126 2003-12-03 14:57:26Z jhb $");
50
51/* I should really use ktr.. */
52/*
53#define UMA_DEBUG 1
54#define UMA_DEBUG_ALLOC 1
55#define UMA_DEBUG_ALLOC_1 1
56*/
57

--- 64 unchanged lines hidden (view full) ---

122
123/* Count of free boottime pages */
124static int uma_boot_free = 0;
125
126/* Is the VM done starting up? */
127static int booted = 0;
128
129/*
50
51/* I should really use ktr.. */
52/*
53#define UMA_DEBUG 1
54#define UMA_DEBUG_ALLOC 1
55#define UMA_DEBUG_ALLOC_1 1
56*/
57

--- 64 unchanged lines hidden (view full) ---

122
123/* Count of free boottime pages */
124static int uma_boot_free = 0;
125
126/* Is the VM done starting up? */
127static int booted = 0;
128
129/*
130 * Rather than #ifdef SMP all over, just give us a bogus definition for
131 * this on UP.
132 */
133#ifndef SMP
134static int mp_maxid = 1;
135#endif
136
137/*
138 * This is the handle used to schedule events that need to happen
139 * outside of the allocation fast path.
140 */
141static struct callout uma_callout;
142#define UMA_TIMEOUT 20 /* Seconds for callout interval. */
143
144/*
145 * This structure is passed as the zone ctor arg so that I don't have to create

--- 199 unchanged lines hidden (view full) ---

345 * Aggregate per cpu cache statistics back to the zone.
346 *
347 * I may rewrite this to set a flag in the per cpu cache instead of
348 * locking. If the flag is not cleared on the next round I will have
349 * to lock and do it here instead so that the statistics don't get too
350 * far out of sync.
351 */
352 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) {
130 * This is the handle used to schedule events that need to happen
131 * outside of the allocation fast path.
132 */
133static struct callout uma_callout;
134#define UMA_TIMEOUT 20 /* Seconds for callout interval. */
135
136/*
137 * This structure is passed as the zone ctor arg so that I don't have to create

--- 199 unchanged lines hidden (view full) ---

337 * Aggregate per cpu cache statistics back to the zone.
338 *
339 * I may rewrite this to set a flag in the per cpu cache instead of
340 * locking. If the flag is not cleared on the next round I will have
341 * to lock and do it here instead so that the statistics don't get too
342 * far out of sync.
343 */
344 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) {
353 for (cpu = 0; cpu < mp_maxid; cpu++) {
345 for (cpu = 0; cpu <= mp_maxid; cpu++) {
354 if (CPU_ABSENT(cpu))
355 continue;
356 CPU_LOCK(cpu);
357 cache = &zone->uz_cpu[cpu];
358 /* Add them up, and reset */
359 alloc += cache->uc_allocs;
360 cache->uc_allocs = 0;
361 CPU_UNLOCK(cpu);

--- 210 unchanged lines hidden (view full) ---

572{
573 uma_bucket_t bucket;
574 uma_cache_t cache;
575 int cpu;
576
577 /*
578 * We have to lock each cpu cache before locking the zone
579 */
346 if (CPU_ABSENT(cpu))
347 continue;
348 CPU_LOCK(cpu);
349 cache = &zone->uz_cpu[cpu];
350 /* Add them up, and reset */
351 alloc += cache->uc_allocs;
352 cache->uc_allocs = 0;
353 CPU_UNLOCK(cpu);

--- 210 unchanged lines hidden (view full) ---

564{
565 uma_bucket_t bucket;
566 uma_cache_t cache;
567 int cpu;
568
569 /*
570 * We have to lock each cpu cache before locking the zone
571 */
580 for (cpu = 0; cpu < mp_maxid; cpu++) {
572 for (cpu = 0; cpu <= mp_maxid; cpu++) {
581 if (CPU_ABSENT(cpu))
582 continue;
583 CPU_LOCK(cpu);
584 cache = &zone->uz_cpu[cpu];
585 bucket_drain(zone, cache->uc_allocbucket);
586 bucket_drain(zone, cache->uc_freebucket);
587 if (cache->uc_allocbucket != NULL)
588 bucket_free(cache->uc_allocbucket);

--- 15 unchanged lines hidden (view full) ---

604 ZONE_LOCK(zone);
605 }
606
607 /* Now we do the free queue.. */
608 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
609 LIST_REMOVE(bucket, ub_link);
610 bucket_free(bucket);
611 }
573 if (CPU_ABSENT(cpu))
574 continue;
575 CPU_LOCK(cpu);
576 cache = &zone->uz_cpu[cpu];
577 bucket_drain(zone, cache->uc_allocbucket);
578 bucket_drain(zone, cache->uc_freebucket);
579 if (cache->uc_allocbucket != NULL)
580 bucket_free(cache->uc_allocbucket);

--- 15 unchanged lines hidden (view full) ---

596 ZONE_LOCK(zone);
597 }
598
599 /* Now we do the free queue.. */
600 while ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
601 LIST_REMOVE(bucket, ub_link);
602 bucket_free(bucket);
603 }
612 for (cpu = 0; cpu < mp_maxid; cpu++) {
604 for (cpu = 0; cpu <= mp_maxid; cpu++) {
613 if (CPU_ABSENT(cpu))
614 continue;
615 CPU_UNLOCK(cpu);
616 }
617 ZONE_UNLOCK(zone);
618}
619
620/*

--- 602 unchanged lines hidden (view full) ---

1223
1224#ifdef UMA_DEBUG
1225 printf("Creating uma zone headers zone.\n");
1226#endif
1227 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1228 /* "manually" Create the initial zone */
1229 args.name = "UMA Zones";
1230 args.size = sizeof(struct uma_zone) +
605 if (CPU_ABSENT(cpu))
606 continue;
607 CPU_UNLOCK(cpu);
608 }
609 ZONE_UNLOCK(zone);
610}
611
612/*

--- 602 unchanged lines hidden (view full) ---

1215
1216#ifdef UMA_DEBUG
1217 printf("Creating uma zone headers zone.\n");
1218#endif
1219 mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
1220 /* "manually" Create the initial zone */
1221 args.name = "UMA Zones";
1222 args.size = sizeof(struct uma_zone) +
1231 (sizeof(struct uma_cache) * mp_maxid);
1223 (sizeof(struct uma_cache) * (mp_maxid + 1));
1232 args.ctor = zone_ctor;
1233 args.dtor = zone_dtor;
1234 args.uminit = zero_init;
1235 args.fini = NULL;
1236 args.align = 32 - 1;
1237 args.flags = UMA_ZFLAG_INTERNAL;
1238 /* The initial zone has no Per cpu queues so it's smaller */
1239 zone_ctor(zones, sizeof(struct uma_zone), &args);
1240
1241 /* Initialize the pcpu cache lock set once and for all */
1224 args.ctor = zone_ctor;
1225 args.dtor = zone_dtor;
1226 args.uminit = zero_init;
1227 args.fini = NULL;
1228 args.align = 32 - 1;
1229 args.flags = UMA_ZFLAG_INTERNAL;
1230 /* The initial zone has no Per cpu queues so it's smaller */
1231 zone_ctor(zones, sizeof(struct uma_zone), &args);
1232
1233 /* Initialize the pcpu cache lock set once and for all */
1242 for (i = 0; i < mp_maxid; i++)
1234 for (i = 0; i <= mp_maxid; i++)
1243 CPU_LOCK_INIT(i);
1244#ifdef UMA_DEBUG
1245 printf("Filling boot free list.\n");
1246#endif
1247 for (i = 0; i < UMA_BOOT_PAGES; i++) {
1248 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE));
1249 slab->us_data = (u_int8_t *)slab;
1250 slab->us_flags = UMA_SLAB_BOOT;

--- 849 unchanged lines hidden (view full) ---

2100 LIST_FOREACH(slab, &zone->uz_part_slab, us_link)
2101 slab_print(slab);
2102 printf("Free slabs:\n");
2103 LIST_FOREACH(slab, &zone->uz_free_slab, us_link)
2104 slab_print(slab);
2105 printf("Full slabs:\n");
2106 LIST_FOREACH(slab, &zone->uz_full_slab, us_link)
2107 slab_print(slab);
1235 CPU_LOCK_INIT(i);
1236#ifdef UMA_DEBUG
1237 printf("Filling boot free list.\n");
1238#endif
1239 for (i = 0; i < UMA_BOOT_PAGES; i++) {
1240 slab = (uma_slab_t)((u_int8_t *)bootmem + (i * UMA_SLAB_SIZE));
1241 slab->us_data = (u_int8_t *)slab;
1242 slab->us_flags = UMA_SLAB_BOOT;

--- 849 unchanged lines hidden (view full) ---

2092 LIST_FOREACH(slab, &zone->uz_part_slab, us_link)
2093 slab_print(slab);
2094 printf("Free slabs:\n");
2095 LIST_FOREACH(slab, &zone->uz_free_slab, us_link)
2096 slab_print(slab);
2097 printf("Full slabs:\n");
2098 LIST_FOREACH(slab, &zone->uz_full_slab, us_link)
2099 slab_print(slab);
2108 for (i = 0; i < mp_maxid; i++) {
2100 for (i = 0; i <= mp_maxid; i++) {
2109 if (CPU_ABSENT(i))
2110 continue;
2111 cache = &zone->uz_cpu[i];
2112 printf("CPU %d Cache:\n", i);
2113 cache_print(cache);
2114 }
2115}
2116

--- 31 unchanged lines hidden (view full) ---

2148 if (error || cnt == 0)
2149 goto out;
2150 offset = tmpbuf;
2151 mtx_lock(&uma_mtx);
2152 LIST_FOREACH(z, &uma_zones, uz_link) {
2153 if (cnt == 0) /* list may have changed size */
2154 break;
2155 if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) {
2101 if (CPU_ABSENT(i))
2102 continue;
2103 cache = &zone->uz_cpu[i];
2104 printf("CPU %d Cache:\n", i);
2105 cache_print(cache);
2106 }
2107}
2108

--- 31 unchanged lines hidden (view full) ---

2140 if (error || cnt == 0)
2141 goto out;
2142 offset = tmpbuf;
2143 mtx_lock(&uma_mtx);
2144 LIST_FOREACH(z, &uma_zones, uz_link) {
2145 if (cnt == 0) /* list may have changed size */
2146 break;
2147 if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) {
2156 for (cpu = 0; cpu < mp_maxid; cpu++) {
2148 for (cpu = 0; cpu <= mp_maxid; cpu++) {
2157 if (CPU_ABSENT(cpu))
2158 continue;
2159 CPU_LOCK(cpu);
2160 }
2161 }
2162 ZONE_LOCK(z);
2163 cachefree = 0;
2164 if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) {
2149 if (CPU_ABSENT(cpu))
2150 continue;
2151 CPU_LOCK(cpu);
2152 }
2153 }
2154 ZONE_LOCK(z);
2155 cachefree = 0;
2156 if (!(z->uz_flags & UMA_ZFLAG_INTERNAL)) {
2165 for (cpu = 0; cpu < mp_maxid; cpu++) {
2157 for (cpu = 0; cpu <= mp_maxid; cpu++) {
2166 if (CPU_ABSENT(cpu))
2167 continue;
2168 cache = &z->uz_cpu[cpu];
2169 if (cache->uc_allocbucket != NULL)
2170 cachefree += cache->uc_allocbucket->ub_cnt;
2171 if (cache->uc_freebucket != NULL)
2172 cachefree += cache->uc_freebucket->ub_cnt;
2173 CPU_UNLOCK(cpu);

--- 27 unchanged lines hidden --- 		2158 if (CPU_ABSENT(cpu))
2159 continue;
2160 cache = &z->uz_cpu[cpu];
2161 if (cache->uc_allocbucket != NULL)
2162 cachefree += cache->uc_allocbucket->ub_cnt;
2163 if (cache->uc_freebucket != NULL)
2164 cachefree += cache->uc_freebucket->ub_cnt;
2165 CPU_UNLOCK(cpu);

--- 27 unchanged lines hidden ---