jemalloc/src/arena.c

23 #define	SIZE_CLASS(bin, delta, size)					\
24 	S2B_##delta(bin)
66 static arena_run_t	*arena_bin_runs_first(arena_bin_t *bin);
67 static void	arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run);
68 static void	arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run);
69 static arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin);
70 static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin);
71 static void	*arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin);
73     arena_bin_t *bin);
75     arena_run_t *run, arena_bin_t *bin);
77     arena_run_t *run, arena_bin_t *bin);
794 				binind = arena_bin_index(arena, run->bin);
984 		size_t binind = arena_bin_index(arena, run->bin);
1170 arena_bin_runs_first(arena_bin_t *bin)
1172 	arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs);
1191 arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run)
1197 	assert(arena_run_tree_search(&bin->runs, mapelm) == NULL);
1199 	arena_run_tree_insert(&bin->runs, mapelm);
1203 arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run)
1209 	assert(arena_run_tree_search(&bin->runs, mapelm) != NULL);
1211 	arena_run_tree_remove(&bin->runs, mapelm);
1215 arena_bin_nonfull_run_tryget(arena_bin_t *bin)
1217 	arena_run_t *run = arena_bin_runs_first(bin);
1219 		arena_bin_runs_remove(bin, run);
1221 			bin->stats.reruns++;
1227 arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
1234 	run = arena_bin_nonfull_run_tryget(bin);
1239 	binind = arena_bin_index(arena, bin);
1243 	malloc_mutex_unlock(&bin->lock);
1252 		run->bin = bin;
1259 	malloc_mutex_lock(&bin->lock);
1262 			bin->stats.nruns++;
1263 			bin->stats.curruns++;
1270 	 * sufficient memory available while this one dropped bin->lock above,
1273 	run = arena_bin_nonfull_run_tryget(bin);
1280 /* Re-fill bin->runcur, then call arena_run_reg_alloc(). */
1282 arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin)
1289 	binind = arena_bin_index(arena, bin);
1291 	bin->runcur = NULL;
1292 	run = arena_bin_nonfull_run_get(arena, bin);
1293 	if (bin->runcur != NULL && bin->runcur->nfree > 0) {
1296 		 * bin lock in arena_bin_nonfull_run_get().
1298 		assert(bin->runcur->nfree > 0);
1299 		ret = arena_run_reg_alloc(bin->runcur, bin_info);
1305 			 * have pulled run from the bin's run tree.  Therefore
1313 				arena_dalloc_bin_run(arena, chunk, run, bin);
1315 				arena_bin_lower_run(arena, chunk, run, bin);
1323 	bin->runcur = run;
1325 	assert(bin->runcur->nfree > 0);
1327 	return (arena_run_reg_alloc(bin->runcur, bin_info));
1335 	arena_bin_t *bin;
1343 	bin = &arena->bins[binind];
1344 	malloc_mutex_lock(&bin->lock);
1347 		if ((run = bin->runcur) != NULL && run->nfree > 0)
1350 			ptr = arena_bin_malloc_hard(arena, bin);
1361 		bin->stats.allocated += i * arena_bin_info[binind].reg_size;
1362 		bin->stats.nmalloc += i;
1363 		bin->stats.nrequests += tbin->tstats.nrequests;
1364 		bin->stats.nfills++;
1367 	malloc_mutex_unlock(&bin->lock);
1424 	arena_bin_t *bin;
1430 	bin = &arena->bins[binind];
1433 	malloc_mutex_lock(&bin->lock);
1434 	if ((run = bin->runcur) != NULL && run->nfree > 0)
1437 		ret = arena_bin_malloc_hard(arena, bin);
1440 		malloc_mutex_unlock(&bin->lock);
1445 		bin->stats.allocated += size;
1446 		bin->stats.nmalloc++;
1447 		bin->stats.nrequests++;
1449 	malloc_mutex_unlock(&bin->lock);
1594     arena_bin_t *bin)
1597 	/* Dissociate run from bin. */
1598 	if (run == bin->runcur)
1599 		bin->runcur = NULL;
1601 		size_t binind = arena_bin_index(chunk->arena, bin);
1610 			arena_bin_runs_remove(bin, run);
1617     arena_bin_t *bin)
1623 	assert(run != bin->runcur);
1624 	assert(arena_run_tree_search(&bin->runs,
1628 	binind = arena_bin_index(chunk->arena, run->bin);
1631 	malloc_mutex_unlock(&bin->lock);
1661 	malloc_mutex_lock(&bin->lock);
1663 		bin->stats.curruns--;
1668     arena_bin_t *bin)
1672 	 * Make sure that if bin->runcur is non-NULL, it refers to the lowest
1676 	if ((uintptr_t)run < (uintptr_t)bin->runcur) {
1678 		if (bin->runcur->nfree > 0)
1679 			arena_bin_runs_insert(bin, bin->runcur);
1680 		bin->runcur = run;
1682 			bin->stats.reruns++;
1684 		arena_bin_runs_insert(bin, run);
1693 	arena_bin_t *bin;
1700 	bin = run->bin;
1711 		arena_dissociate_bin_run(chunk, run, bin);
1712 		arena_dalloc_bin_run(arena, chunk, run, bin);
1713 	} else if (run->nfree == 1 && run != bin->runcur)
1714 		arena_bin_lower_run(arena, chunk, run, bin);
1717 		bin->stats.allocated -= size;
1718 		bin->stats.ndalloc++;
1727 	arena_bin_t *bin;
1731 	bin = run->bin;
1732 	malloc_mutex_lock(&bin->lock);
1734 	malloc_mutex_unlock(&bin->lock);
2068 		arena_bin_t *bin = &arena->bins[i];
2070 		malloc_mutex_lock(&bin->lock);
2071 		bstats[i].allocated += bin->stats.allocated;
2072 		bstats[i].nmalloc += bin->stats.nmalloc;
2073 		bstats[i].ndalloc += bin->stats.ndalloc;
2074 		bstats[i].nrequests += bin->stats.nrequests;
2076 			bstats[i].nfills += bin->stats.nfills;
2077 			bstats[i].nflushes += bin->stats.nflushes;
2079 		bstats[i].nruns += bin->stats.nruns;
2080 		bstats[i].reruns += bin->stats.reruns;
2081 		bstats[i].curruns += bin->stats.curruns;
2082 		malloc_mutex_unlock(&bin->lock);
2090 	arena_bin_t *bin;
2128 		bin = &arena->bins[i];
2129 		if (malloc_mutex_init(&bin->lock))
2131 		bin->runcur = NULL;
2132 		arena_run_tree_new(&bin->runs);
2134 			memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));
2296 #define	SIZE_CLASS(bin, delta, size)					\
2297 	bin_info = &arena_bin_info[bin];				\