Lines Matching refs:tcache
17 static unsigned stack_nelms; /* Total stack elms per tcache. */
41 tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
42 szind_t binind = tcache->next_gc_bin;
46 tbin = tcache_small_bin_get(tcache, binind);
48 tbin = tcache_large_bin_get(tcache, binind);
55 tcache_bin_flush_small(tsd, tcache, tbin, binind,
64 (tcache->lg_fill_div[binind] + 1)) >= 1) {
65 tcache->lg_fill_div[binind]++;
69 - tbin->low_water + (tbin->low_water >> 2), tcache);
76 if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) {
77 tcache->lg_fill_div[binind]--;
82 tcache->next_gc_bin++;
83 if (tcache->next_gc_bin == nhbins) {
84 tcache->next_gc_bin = 0;
89 tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
93 assert(tcache->arena != NULL);
94 arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind,
95 config_prof ? tcache->prof_accumbytes : 0);
97 tcache->prof_accumbytes = 0;
112 * Verify that the items in the tcache all have the correct size; this
134 tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
141 arena_t *arena = tcache->arena;
168 tcache->prof_accumbytes)) {
171 tcache->prof_accumbytes = 0;
231 unsigned rem, tcache_t *tcache) {
237 arena_t *tcache_arena = tcache->arena;
280 tcache_arena, tcache->prof_accumbytes);
281 tcache->prof_accumbytes = 0;
341 tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
342 assert(tcache->arena == NULL);
343 tcache->arena = arena;
349 ql_elm_new(tcache, link);
350 ql_tail_insert(&arena->tcache_ql, tcache, link);
352 &tcache->cache_bin_array_descriptor, tcache->bins_small,
353 tcache->bins_large);
355 &tcache->cache_bin_array_descriptor, link);
362 tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
363 arena_t *arena = tcache->arena;
372 if (iter == tcache) {
379 ql_remove(&arena->tcache_ql, tcache, link);
381 &tcache->cache_bin_array_descriptor, link);
382 tcache_stats_merge(tsdn, tcache, arena);
385 tcache->arena = NULL;
389 tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
390 tcache_arena_dissociate(tsdn, tcache);
391 tcache_arena_associate(tsdn, tcache, arena);
401 /* Trigger tcache init. */
408 /* Initialize auto tcache (embedded in TSD). */
410 tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
411 memset(&tcache->link, 0, sizeof(ql_elm(tcache_t)));
412 tcache->prof_accumbytes = 0;
413 tcache->next_gc_bin = 0;
414 tcache->arena = NULL;
416 ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
420 memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS);
421 memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS));
424 tcache->lg_fill_div[i] = 1;
431 tcache_small_bin_get(tcache, i)->avail =
436 tcache_large_bin_get(tcache, i)->avail =
442 /* Initialize auto tcache (embedded in TSD). */
445 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
446 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
457 tcache_init(tsd, tcache, avail_array);
460 * threads can rely on arena_choose and associate tcache accordingly.
463 * associate its tcache to a0 temporarily, and later on
466 tcache->arena = NULL;
471 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
474 /* This may happen if thread.tcache.enabled is used. */
475 if (tcache->arena == NULL) {
476 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
479 assert(arena == tcache->arena);
484 /* Created manual tcache for tcache.create mallctl. */
487 tcache_t *tcache;
498 tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true,
500 if (tcache == NULL) {
504 tcache_init(tsd, tcache,
505 (void *)((uintptr_t)tcache + (uintptr_t)stack_offset));
506 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL));
508 return tcache;
512 tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
513 assert(tcache->arena != NULL);
516 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
517 tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
524 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
525 tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
532 if (config_prof && tcache->prof_accumbytes > 0 &&
533 arena_prof_accum(tsd_tsdn(tsd), tcache->arena,
534 tcache->prof_accumbytes)) {
546 tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
547 tcache_flush_cache(tsd, tcache);
548 arena_t *arena = tcache->arena;
549 tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
552 /* Release the avail array for the TSD embedded auto tcache. */
554 (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail -
558 /* Release both the tcache struct and avail array. */
559 idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true);
563 * The deallocation and tcache flush above may not trigger decay since
564 * we are on the tcache shutdown path (potentially with non-nominal
566 * include arena 0 because the tcache array is allocated from it.
580 /* For auto tcache (embedded in TSD) only. */
583 tcache_t *tcache = tsd_tcachep_get(tsd);
587 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
592 assert(tcache_small_bin_get(tcache, 0)->avail != NULL);
594 tcache_destroy(tsd, tcache, true);
596 tcache_small_bin_get(tcache, 0)->avail = NULL;
601 tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
606 /* Merge and reset tcache stats. */
608 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
617 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
661 tcache_t *tcache = tcache_create_explicit(tsd);
662 if (tcache == NULL) {
672 elm->tcache = tcache;
676 elm->tcache = tcache;
692 if (elm->tcache == NULL) {
695 tcache_t *tcache = elm->tcache;
697 elm->tcache = TCACHES_ELM_NEED_REINIT;
699 elm->tcache = NULL;
702 if (tcache == TCACHES_ELM_NEED_REINIT) {
705 return tcache;
711 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true);
713 if (tcache != NULL) {
714 /* Destroy the tcache; recreate in tcaches_get() if needed. */
715 tcache_destroy(tsd, tcache, false);
723 tcache_t *tcache = tcaches_elm_remove(tsd, elm, false);
727 if (tcache != NULL) {
728 tcache_destroy(tsd, tcache, false);