1/*
2 * Copyright 2008-2010, Axel D��rfler. All Rights Reserved.
3 * Copyright 2007, Hugo Santos. All Rights Reserved.
4 *
5 * Distributed under the terms of the MIT License.
6 */
7
8
9#include "ObjectCache.h"
10
11#include <string.h>
12
13#include <smp.h>
14#include <util/AutoLock.h>
15#include <vm/vm.h>
16#include <vm/VMAddressSpace.h>
17
18#include "MemoryManager.h"
19#include "slab_private.h"
20
21
22RANGE_MARKER_FUNCTION_BEGIN(SlabObjectCache)
23
24
25static void
26object_cache_return_object_wrapper(object_depot* depot, void* cookie,
27	void* object, uint32 flags)
28{
29	ObjectCache* cache = (ObjectCache*)cookie;
30
31	MutexLocker _(cache->lock);
32	cache->ReturnObjectToSlab(cache->ObjectSlab(object), object, flags);
33}
34
35
36// #pragma mark -
37
38
39ObjectCache::~ObjectCache()
40{
41}
42
43
44status_t
45ObjectCache::Init(const char* name, size_t objectSize, size_t alignment,
46	size_t maximum, size_t magazineCapacity, size_t maxMagazineCount,
47	uint32 flags, void* cookie, object_cache_constructor constructor,
48	object_cache_destructor destructor, object_cache_reclaimer reclaimer)
49{
50	strlcpy(this->name, name, sizeof(this->name));
51
52	mutex_init(&lock, this->name);
53
54	if (objectSize < sizeof(object_link))
55		objectSize = sizeof(object_link);
56
57	if (alignment < kMinObjectAlignment)
58		alignment = kMinObjectAlignment;
59
60	if (alignment > 0 && (objectSize & (alignment - 1)))
61		object_size = objectSize + alignment - (objectSize & (alignment - 1));
62	else
63		object_size = objectSize;
64
65	TRACE_CACHE(this, "init %lu, %lu -> %lu", objectSize, alignment,
66		object_size);
67
68	this->alignment = alignment;
69	cache_color_cycle = 0;
70	total_objects = 0;
71	used_count = 0;
72	empty_count = 0;
73	pressure = 0;
74	min_object_reserve = 0;
75
76	maintenance_pending = false;
77	maintenance_in_progress = false;
78	maintenance_resize = false;
79	maintenance_delete = false;
80
81	usage = 0;
82	this->maximum = maximum;
83
84	this->flags = flags;
85
86	resize_request = NULL;
87	resize_entry_can_wait = NULL;
88	resize_entry_dont_wait = NULL;
89
90	// no gain in using the depot in single cpu setups
91	if (smp_get_num_cpus() == 1)
92		this->flags |= CACHE_NO_DEPOT;
93
94	if (!(this->flags & CACHE_NO_DEPOT)) {
95		// Determine usable magazine configuration values if none had been given
96		if (magazineCapacity == 0) {
97			magazineCapacity = objectSize < 256
98				? 32 : (objectSize < 512 ? 16 : 8);
99		}
100		if (maxMagazineCount == 0)
101			maxMagazineCount = magazineCapacity / 2;
102
103		status_t status = object_depot_init(&depot, magazineCapacity,
104			maxMagazineCount, flags, this, object_cache_return_object_wrapper);
105		if (status != B_OK) {
106			mutex_destroy(&lock);
107			return status;
108		}
109	}
110
111	this->cookie = cookie;
112	this->constructor = constructor;
113	this->destructor = destructor;
114	this->reclaimer = reclaimer;
115
116	return B_OK;
117}
118
119
120slab*
121ObjectCache::InitSlab(slab* slab, void* pages, size_t byteCount, uint32 flags)
122{
123	TRACE_CACHE(this, "construct (%p, %p .. %p, %lu)", slab, pages,
124		((uint8*)pages) + byteCount, byteCount);
125
126	slab->pages = pages;
127	slab->count = slab->size = byteCount / object_size;
128	slab->free = NULL;
129
130	size_t spareBytes = byteCount - (slab->size * object_size);
131
132	slab->offset = cache_color_cycle;
133
134	cache_color_cycle += alignment;
135	if (cache_color_cycle > spareBytes)
136		cache_color_cycle = 0;
137
138	TRACE_CACHE(this, "  %lu objects, %lu spare bytes, offset %lu",
139		slab->size, spareBytes, slab->offset);
140
141	uint8* data = ((uint8*)pages) + slab->offset;
142
143	CREATE_PARANOIA_CHECK_SET(slab, "slab");
144
145
146	for (size_t i = 0; i < slab->size; i++) {
147		status_t status = B_OK;
148		if (constructor)
149			status = constructor(cookie, data);
150
151		if (status != B_OK) {
152			data = ((uint8*)pages) + slab->offset;
153			for (size_t j = 0; j < i; j++) {
154				if (destructor)
155					destructor(cookie, data);
156				data += object_size;
157			}
158
159			DELETE_PARANOIA_CHECK_SET(slab);
160
161			return NULL;
162		}
163
164		_push(slab->free, object_to_link(data, object_size));
165
166		ADD_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, slab,
167			&object_to_link(data, object_size)->next, sizeof(void*));
168
169		data += object_size;
170	}
171
172	return slab;
173}
174
175
176void
177ObjectCache::UninitSlab(slab* slab)
178{
179	TRACE_CACHE(this, "destruct %p", slab);
180
181	if (slab->count != slab->size)
182		panic("cache: destroying a slab which isn't empty.");
183
184	usage -= slab_size;
185	total_objects -= slab->size;
186
187	DELETE_PARANOIA_CHECK_SET(slab);
188
189	uint8* data = ((uint8*)slab->pages) + slab->offset;
190
191	for (size_t i = 0; i < slab->size; i++) {
192		if (destructor)
193			destructor(cookie, data);
194		data += object_size;
195	}
196}
197
198
199void
200ObjectCache::ReturnObjectToSlab(slab* source, void* object, uint32 flags)
201{
202	if (source == NULL) {
203		panic("object_cache: free'd object %p has no slab", object);
204		return;
205	}
206
207	ParanoiaChecker _(source);
208
209#if KDEBUG >= 1
210	uint8* objectsStart = (uint8*)source->pages + source->offset;
211	if (object < objectsStart
212		|| object >= objectsStart + source->size * object_size
213		|| ((uint8*)object - objectsStart) % object_size != 0) {
214		panic("object_cache: tried to free invalid object pointer %p", object);
215		return;
216	}
217#endif // KDEBUG
218
219	object_link* link = object_to_link(object, object_size);
220
221	TRACE_CACHE(this, "returning %p (%p) to %p, %lu used (%lu empty slabs).",
222		object, link, source, source->size - source->count,
223		empty_count);
224
225	_push(source->free, link);
226	source->count++;
227	used_count--;
228
229	ADD_PARANOIA_CHECK(PARANOIA_SUSPICIOUS, source, &link->next, sizeof(void*));
230
231	if (source->count == source->size) {
232		partial.Remove(source);
233
234		if (empty_count < pressure
235			&& total_objects - used_count - source->size
236				>= min_object_reserve) {
237			empty_count++;
238			empty.Add(source);
239		} else {
240			ReturnSlab(source, flags);
241		}
242	} else if (source->count == 1) {
243		full.Remove(source);
244		partial.Add(source);
245	}
246}
247
248
249void*
250ObjectCache::ObjectAtIndex(slab* source, int32 index) const
251{
252	return (uint8*)source->pages + source->offset + index * object_size;
253}
254
255
256#if PARANOID_KERNEL_FREE
257
258bool
259ObjectCache::AssertObjectNotFreed(void* object)
260{
261	MutexLocker locker(lock);
262
263	slab* source = ObjectSlab(object);
264	if (!partial.Contains(source) && !full.Contains(source)) {
265		panic("object_cache: to be freed object %p: slab not part of cache!",
266			object);
267		return false;
268	}
269
270	object_link* link = object_to_link(object, object_size);
271	for (object_link* freeLink = source->free; freeLink != NULL;
272			freeLink = freeLink->next) {
273		if (freeLink == link) {
274			panic("object_cache: double free of %p (slab %p, cache %p)",
275				object, source, this);
276			return false;
277		}
278	}
279
280	return true;
281}
282
283#endif // PARANOID_KERNEL_FREE
284
285
286#if SLAB_OBJECT_CACHE_ALLOCATION_TRACKING
287
288status_t
289ObjectCache::AllocateTrackingInfos(slab* slab, size_t byteCount, uint32 flags)
290{
291	void* pages;
292	size_t objectCount = byteCount / object_size;
293	status_t result = MemoryManager::AllocateRaw(
294		objectCount * sizeof(AllocationTrackingInfo), flags, pages);
295	if (result == B_OK) {
296		slab->tracking = (AllocationTrackingInfo*)pages;
297		for (size_t i = 0; i < objectCount; i++)
298			slab->tracking[i].Clear();
299	}
300
301	return result;
302}
303
304
305void
306ObjectCache::FreeTrackingInfos(slab* slab, uint32 flags)
307{
308	MemoryManager::FreeRawOrReturnCache(slab->tracking, flags);
309}
310
311
312AllocationTrackingInfo*
313ObjectCache::TrackingInfoFor(void* object) const
314{
315	slab* objectSlab = ObjectSlab(object);
316	return &objectSlab->tracking[((addr_t)object - objectSlab->offset
317		- (addr_t)objectSlab->pages) / object_size];
318}
319
320#endif // SLAB_OBJECT_CACHE_ALLOCATION_TRACKING
321
322
323RANGE_MARKER_FUNCTION_END(SlabObjectCache)
324