1#define	JEMALLOC_TCACHE_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4/******************************************************************************/
5/* Data. */
6
7bool	opt_tcache = true;
8ssize_t	opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
9
10tcache_bin_info_t	*tcache_bin_info;
11static unsigned		stack_nelms; /* Total stack elms per tcache. */
12
13unsigned		nhbins;
14size_t			tcache_maxclass;
15
16tcaches_t		*tcaches;
17
18/* Index of first element within tcaches that has never been used. */
19static unsigned		tcaches_past;
20
21/* Head of singly linked list tracking available tcaches elements. */
22static tcaches_t	*tcaches_avail;
23
24/******************************************************************************/
25
26size_t
27tcache_salloc(tsdn_t *tsdn, const void *ptr)
28{
29
30	return (arena_salloc(tsdn, ptr, false));
31}
32
33void
34tcache_event_hard(tsd_t *tsd, tcache_t *tcache)
35{
36	szind_t binind = tcache->next_gc_bin;
37	tcache_bin_t *tbin = &tcache->tbins[binind];
38	tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
39
40	if (tbin->low_water > 0) {
41		/*
42		 * Flush (ceiling) 3/4 of the objects below the low water mark.
43		 */
44		if (binind < NBINS) {
45			tcache_bin_flush_small(tsd, tcache, tbin, binind,
46			    tbin->ncached - tbin->low_water + (tbin->low_water
47			    >> 2));
48		} else {
49			tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached
50			    - tbin->low_water + (tbin->low_water >> 2), tcache);
51		}
52		/*
53		 * Reduce fill count by 2X.  Limit lg_fill_div such that the
54		 * fill count is always at least 1.
55		 */
56		if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1)
57			tbin->lg_fill_div++;
58	} else if (tbin->low_water < 0) {
59		/*
60		 * Increase fill count by 2X.  Make sure lg_fill_div stays
61		 * greater than 0.
62		 */
63		if (tbin->lg_fill_div > 1)
64			tbin->lg_fill_div--;
65	}
66	tbin->low_water = tbin->ncached;
67
68	tcache->next_gc_bin++;
69	if (tcache->next_gc_bin == nhbins)
70		tcache->next_gc_bin = 0;
71}
72
73void *
74tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
75    tcache_bin_t *tbin, szind_t binind, bool *tcache_success)
76{
77	void *ret;
78
79	arena_tcache_fill_small(tsdn, arena, tbin, binind, config_prof ?
80	    tcache->prof_accumbytes : 0);
81	if (config_prof)
82		tcache->prof_accumbytes = 0;
83	ret = tcache_alloc_easy(tbin, tcache_success);
84
85	return (ret);
86}
87
88void
89tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
90    szind_t binind, unsigned rem)
91{
92	arena_t *arena;
93	void *ptr;
94	unsigned i, nflush, ndeferred;
95	bool merged_stats = false;
96
97	assert(binind < NBINS);
98	assert(rem <= tbin->ncached);
99
100	arena = arena_choose(tsd, NULL);
101	assert(arena != NULL);
102	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
103		/* Lock the arena bin associated with the first object. */
104		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
105		    *(tbin->avail - 1));
106		arena_t *bin_arena = extent_node_arena_get(&chunk->node);
107		arena_bin_t *bin = &bin_arena->bins[binind];
108
109		if (config_prof && bin_arena == arena) {
110			if (arena_prof_accum(tsd_tsdn(tsd), arena,
111			    tcache->prof_accumbytes))
112				prof_idump(tsd_tsdn(tsd));
113			tcache->prof_accumbytes = 0;
114		}
115
116		malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
117		if (config_stats && bin_arena == arena) {
118			assert(!merged_stats);
119			merged_stats = true;
120			bin->stats.nflushes++;
121			bin->stats.nrequests += tbin->tstats.nrequests;
122			tbin->tstats.nrequests = 0;
123		}
124		ndeferred = 0;
125		for (i = 0; i < nflush; i++) {
126			ptr = *(tbin->avail - 1 - i);
127			assert(ptr != NULL);
128			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
129			if (extent_node_arena_get(&chunk->node) == bin_arena) {
130				size_t pageind = ((uintptr_t)ptr -
131				    (uintptr_t)chunk) >> LG_PAGE;
132				arena_chunk_map_bits_t *bitselm =
133				    arena_bitselm_get_mutable(chunk, pageind);
134				arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
135				    bin_arena, chunk, ptr, bitselm);
136			} else {
137				/*
138				 * This object was allocated via a different
139				 * arena bin than the one that is currently
140				 * locked.  Stash the object, so that it can be
141				 * handled in a future pass.
142				 */
143				*(tbin->avail - 1 - ndeferred) = ptr;
144				ndeferred++;
145			}
146		}
147		malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
148		arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred);
149	}
150	if (config_stats && !merged_stats) {
151		/*
152		 * The flush loop didn't happen to flush to this thread's
153		 * arena, so the stats didn't get merged.  Manually do so now.
154		 */
155		arena_bin_t *bin = &arena->bins[binind];
156		malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
157		bin->stats.nflushes++;
158		bin->stats.nrequests += tbin->tstats.nrequests;
159		tbin->tstats.nrequests = 0;
160		malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
161	}
162
163	memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
164	    sizeof(void *));
165	tbin->ncached = rem;
166	if ((int)tbin->ncached < tbin->low_water)
167		tbin->low_water = tbin->ncached;
168}
169
170void
171tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
172    unsigned rem, tcache_t *tcache)
173{
174	arena_t *arena;
175	void *ptr;
176	unsigned i, nflush, ndeferred;
177	bool merged_stats = false;
178
179	assert(binind < nhbins);
180	assert(rem <= tbin->ncached);
181
182	arena = arena_choose(tsd, NULL);
183	assert(arena != NULL);
184	for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
185		/* Lock the arena associated with the first object. */
186		arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
187		    *(tbin->avail - 1));
188		arena_t *locked_arena = extent_node_arena_get(&chunk->node);
189		UNUSED bool idump;
190
191		if (config_prof)
192			idump = false;
193		malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->lock);
194		if ((config_prof || config_stats) && locked_arena == arena) {
195			if (config_prof) {
196				idump = arena_prof_accum_locked(arena,
197				    tcache->prof_accumbytes);
198				tcache->prof_accumbytes = 0;
199			}
200			if (config_stats) {
201				merged_stats = true;
202				arena->stats.nrequests_large +=
203				    tbin->tstats.nrequests;
204				arena->stats.lstats[binind - NBINS].nrequests +=
205				    tbin->tstats.nrequests;
206				tbin->tstats.nrequests = 0;
207			}
208		}
209		ndeferred = 0;
210		for (i = 0; i < nflush; i++) {
211			ptr = *(tbin->avail - 1 - i);
212			assert(ptr != NULL);
213			chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
214			if (extent_node_arena_get(&chunk->node) ==
215			    locked_arena) {
216				arena_dalloc_large_junked_locked(tsd_tsdn(tsd),
217				    locked_arena, chunk, ptr);
218			} else {
219				/*
220				 * This object was allocated via a different
221				 * arena than the one that is currently locked.
222				 * Stash the object, so that it can be handled
223				 * in a future pass.
224				 */
225				*(tbin->avail - 1 - ndeferred) = ptr;
226				ndeferred++;
227			}
228		}
229		malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->lock);
230		if (config_prof && idump)
231			prof_idump(tsd_tsdn(tsd));
232		arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush -
233		    ndeferred);
234	}
235	if (config_stats && !merged_stats) {
236		/*
237		 * The flush loop didn't happen to flush to this thread's
238		 * arena, so the stats didn't get merged.  Manually do so now.
239		 */
240		malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
241		arena->stats.nrequests_large += tbin->tstats.nrequests;
242		arena->stats.lstats[binind - NBINS].nrequests +=
243		    tbin->tstats.nrequests;
244		tbin->tstats.nrequests = 0;
245		malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
246	}
247
248	memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
249	    sizeof(void *));
250	tbin->ncached = rem;
251	if ((int)tbin->ncached < tbin->low_water)
252		tbin->low_water = tbin->ncached;
253}
254
255static void
256tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
257{
258
259	if (config_stats) {
260		/* Link into list of extant tcaches. */
261		malloc_mutex_lock(tsdn, &arena->lock);
262		ql_elm_new(tcache, link);
263		ql_tail_insert(&arena->tcache_ql, tcache, link);
264		malloc_mutex_unlock(tsdn, &arena->lock);
265	}
266}
267
268static void
269tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
270{
271
272	if (config_stats) {
273		/* Unlink from list of extant tcaches. */
274		malloc_mutex_lock(tsdn, &arena->lock);
275		if (config_debug) {
276			bool in_ql = false;
277			tcache_t *iter;
278			ql_foreach(iter, &arena->tcache_ql, link) {
279				if (iter == tcache) {
280					in_ql = true;
281					break;
282				}
283			}
284			assert(in_ql);
285		}
286		ql_remove(&arena->tcache_ql, tcache, link);
287		tcache_stats_merge(tsdn, tcache, arena);
288		malloc_mutex_unlock(tsdn, &arena->lock);
289	}
290}
291
292void
293tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *oldarena,
294    arena_t *newarena)
295{
296
297	tcache_arena_dissociate(tsdn, tcache, oldarena);
298	tcache_arena_associate(tsdn, tcache, newarena);
299}
300
301tcache_t *
302tcache_get_hard(tsd_t *tsd)
303{
304	arena_t *arena;
305
306	if (!tcache_enabled_get()) {
307		if (tsd_nominal(tsd))
308			tcache_enabled_set(false); /* Memoize. */
309		return (NULL);
310	}
311	arena = arena_choose(tsd, NULL);
312	if (unlikely(arena == NULL))
313		return (NULL);
314	return (tcache_create(tsd_tsdn(tsd), arena));
315}
316
317tcache_t *
318tcache_create(tsdn_t *tsdn, arena_t *arena)
319{
320	tcache_t *tcache;
321	size_t size, stack_offset;
322	unsigned i;
323
324	size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins);
325	/* Naturally align the pointer stacks. */
326	size = PTR_CEILING(size);
327	stack_offset = size;
328	size += stack_nelms * sizeof(void *);
329	/* Avoid false cacheline sharing. */
330	size = sa2u(size, CACHELINE);
331
332	tcache = ipallocztm(tsdn, size, CACHELINE, true, NULL, true,
333	    arena_get(TSDN_NULL, 0, true));
334	if (tcache == NULL)
335		return (NULL);
336
337	tcache_arena_associate(tsdn, tcache, arena);
338
339	ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
340
341	assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
342	for (i = 0; i < nhbins; i++) {
343		tcache->tbins[i].lg_fill_div = 1;
344		stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
345		/*
346		 * avail points past the available space.  Allocations will
347		 * access the slots toward higher addresses (for the benefit of
348		 * prefetch).
349		 */
350		tcache->tbins[i].avail = (void **)((uintptr_t)tcache +
351		    (uintptr_t)stack_offset);
352	}
353
354	return (tcache);
355}
356
357static void
358tcache_destroy(tsd_t *tsd, tcache_t *tcache)
359{
360	arena_t *arena;
361	unsigned i;
362
363	arena = arena_choose(tsd, NULL);
364	tcache_arena_dissociate(tsd_tsdn(tsd), tcache, arena);
365
366	for (i = 0; i < NBINS; i++) {
367		tcache_bin_t *tbin = &tcache->tbins[i];
368		tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
369
370		if (config_stats && tbin->tstats.nrequests != 0) {
371			arena_bin_t *bin = &arena->bins[i];
372			malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
373			bin->stats.nrequests += tbin->tstats.nrequests;
374			malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
375		}
376	}
377
378	for (; i < nhbins; i++) {
379		tcache_bin_t *tbin = &tcache->tbins[i];
380		tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
381
382		if (config_stats && tbin->tstats.nrequests != 0) {
383			malloc_mutex_lock(tsd_tsdn(tsd), &arena->lock);
384			arena->stats.nrequests_large += tbin->tstats.nrequests;
385			arena->stats.lstats[i - NBINS].nrequests +=
386			    tbin->tstats.nrequests;
387			malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
388		}
389	}
390
391	if (config_prof && tcache->prof_accumbytes > 0 &&
392	    arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes))
393		prof_idump(tsd_tsdn(tsd));
394
395	idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true);
396}
397
398void
399tcache_cleanup(tsd_t *tsd)
400{
401	tcache_t *tcache;
402
403	if (!config_tcache)
404		return;
405
406	if ((tcache = tsd_tcache_get(tsd)) != NULL) {
407		tcache_destroy(tsd, tcache);
408		tsd_tcache_set(tsd, NULL);
409	}
410}
411
412void
413tcache_enabled_cleanup(tsd_t *tsd)
414{
415
416	/* Do nothing. */
417}
418
419void
420tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena)
421{
422	unsigned i;
423
424	cassert(config_stats);
425
426	malloc_mutex_assert_owner(tsdn, &arena->lock);
427
428	/* Merge and reset tcache stats. */
429	for (i = 0; i < NBINS; i++) {
430		arena_bin_t *bin = &arena->bins[i];
431		tcache_bin_t *tbin = &tcache->tbins[i];
432		malloc_mutex_lock(tsdn, &bin->lock);
433		bin->stats.nrequests += tbin->tstats.nrequests;
434		malloc_mutex_unlock(tsdn, &bin->lock);
435		tbin->tstats.nrequests = 0;
436	}
437
438	for (; i < nhbins; i++) {
439		malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS];
440		tcache_bin_t *tbin = &tcache->tbins[i];
441		arena->stats.nrequests_large += tbin->tstats.nrequests;
442		lstats->nrequests += tbin->tstats.nrequests;
443		tbin->tstats.nrequests = 0;
444	}
445}
446
447bool
448tcaches_create(tsdn_t *tsdn, unsigned *r_ind)
449{
450	arena_t *arena;
451	tcache_t *tcache;
452	tcaches_t *elm;
453
454	if (tcaches == NULL) {
455		tcaches = base_alloc(tsdn, sizeof(tcache_t *) *
456		    (MALLOCX_TCACHE_MAX+1));
457		if (tcaches == NULL)
458			return (true);
459	}
460
461	if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX)
462		return (true);
463	arena = arena_ichoose(tsdn, NULL);
464	if (unlikely(arena == NULL))
465		return (true);
466	tcache = tcache_create(tsdn, arena);
467	if (tcache == NULL)
468		return (true);
469
470	if (tcaches_avail != NULL) {
471		elm = tcaches_avail;
472		tcaches_avail = tcaches_avail->next;
473		elm->tcache = tcache;
474		*r_ind = (unsigned)(elm - tcaches);
475	} else {
476		elm = &tcaches[tcaches_past];
477		elm->tcache = tcache;
478		*r_ind = tcaches_past;
479		tcaches_past++;
480	}
481
482	return (false);
483}
484
485static void
486tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm)
487{
488
489	if (elm->tcache == NULL)
490		return;
491	tcache_destroy(tsd, elm->tcache);
492	elm->tcache = NULL;
493}
494
495void
496tcaches_flush(tsd_t *tsd, unsigned ind)
497{
498
499	tcaches_elm_flush(tsd, &tcaches[ind]);
500}
501
502void
503tcaches_destroy(tsd_t *tsd, unsigned ind)
504{
505	tcaches_t *elm = &tcaches[ind];
506	tcaches_elm_flush(tsd, elm);
507	elm->next = tcaches_avail;
508	tcaches_avail = elm;
509}
510
511bool
512tcache_boot(tsdn_t *tsdn)
513{
514	unsigned i;
515
516	/*
517	 * If necessary, clamp opt_lg_tcache_max, now that large_maxclass is
518	 * known.
519	 */
520	if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS)
521		tcache_maxclass = SMALL_MAXCLASS;
522	else if ((1U << opt_lg_tcache_max) > large_maxclass)
523		tcache_maxclass = large_maxclass;
524	else
525		tcache_maxclass = (1U << opt_lg_tcache_max);
526
527	nhbins = size2index(tcache_maxclass) + 1;
528
529	/* Initialize tcache_bin_info. */
530	tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, nhbins *
531	    sizeof(tcache_bin_info_t));
532	if (tcache_bin_info == NULL)
533		return (true);
534	stack_nelms = 0;
535	for (i = 0; i < NBINS; i++) {
536		if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
537			tcache_bin_info[i].ncached_max =
538			    TCACHE_NSLOTS_SMALL_MIN;
539		} else if ((arena_bin_info[i].nregs << 1) <=
540		    TCACHE_NSLOTS_SMALL_MAX) {
541			tcache_bin_info[i].ncached_max =
542			    (arena_bin_info[i].nregs << 1);
543		} else {
544			tcache_bin_info[i].ncached_max =
545			    TCACHE_NSLOTS_SMALL_MAX;
546		}
547		stack_nelms += tcache_bin_info[i].ncached_max;
548	}
549	for (; i < nhbins; i++) {
550		tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
551		stack_nelms += tcache_bin_info[i].ncached_max;
552	}
553
554	return (false);
555}
556