jemalloc/src/arena.c

14 #define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs,		\
16 	BIN_INFO_bin_##bin((1U<<lg_grp) + (ndelta<<lg_delta),		\
34     extent_t *slab, arena_bin_t *bin);
36     extent_t *slab, arena_bin_t *bin);
160 #define	SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs,		\
162 	REGIND_bin_##bin(index, (1U<<lg_grp) + (ndelta<<lg_delta))
767 arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab)
770 	extent_heap_insert(&bin->slabs_nonfull, slab);
774 arena_bin_slabs_nonfull_remove(arena_bin_t *bin, extent_t *slab)
776 	extent_heap_remove(&bin->slabs_nonfull, slab);
780 arena_bin_slabs_nonfull_tryget(arena_bin_t *bin)
782 	extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull);
786 		bin->stats.reslabs++;
791 arena_bin_slabs_full_insert(arena_bin_t *bin, extent_t *slab)
794 	extent_ring_insert(&bin->slabs_full, slab);
849 		arena_bin_t *bin = &arena->bins[i];
850 		malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
851 		if (bin->slabcur != NULL) {
852 			slab = bin->slabcur;
853 			bin->slabcur = NULL;
854 			malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
856 			malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
858 		while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) !=
860 			malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
862 			malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
864 		for (slab = qr_next(&bin->slabs_full, qr_link); slab !=
865 		    &bin->slabs_full; slab = qr_next(&bin->slabs_full,
868 			malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
870 			malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
873 			bin->stats.curregs = 0;
874 			bin->stats.curslabs = 0;
876 		malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
1001 arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
1008 	slab = arena_bin_slabs_nonfull_tryget(bin);
1016 	malloc_mutex_unlock(tsdn, &bin->lock);
1022 	malloc_mutex_lock(tsdn, &bin->lock);
1025 			bin->stats.nslabs++;
1026 			bin->stats.curslabs++;
1033 	 * sufficient memory available while this one dropped bin->lock above,
1036 	slab = arena_bin_slabs_nonfull_tryget(bin);
1043 /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */
1045 arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin,
1052 	if (bin->slabcur != NULL) {
1053 		arena_bin_slabs_full_insert(bin, bin->slabcur);
1054 		bin->slabcur = NULL;
1056 	slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind);
1057 	if (bin->slabcur != NULL) {
1060 		 * bin lock in arena_bin_nonfull_slab_get().
1062 		if (extent_slab_data_get(bin->slabcur)->nfree > 0) {
1063 			void *ret = arena_slab_reg_alloc(tsdn, bin->slabcur,
1078 					    bin);
1081 					    bin);
1087 		arena_bin_slabs_full_insert(bin, bin->slabcur);
1088 		bin->slabcur = NULL;
1093 	bin->slabcur = slab;
1095 	assert(extent_slab_data_get(bin->slabcur)->nfree > 0);
1105 	arena_bin_t *bin;
1111 	bin = &arena->bins[binind];
1112 	malloc_mutex_lock(tsdn, &bin->lock);
1117 		if ((slab = bin->slabcur) != NULL &&
1122 			ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind);
1143 		bin->stats.nmalloc += i;
1144 		bin->stats.nrequests += tbin->tstats.nrequests;
1145 		bin->stats.curregs += i;
1146 		bin->stats.nfills++;
1149 	malloc_mutex_unlock(tsdn, &bin->lock);
1181 	arena_bin_t *bin;
1186 	bin = &arena->bins[binind];
1189 	malloc_mutex_lock(tsdn, &bin->lock);
1190 	if ((slab = bin->slabcur) != NULL && extent_slab_data_get(slab)->nfree >
1194 		ret = arena_bin_malloc_hard(tsdn, arena, bin, binind);
1197 		malloc_mutex_unlock(tsdn, &bin->lock);
1202 		bin->stats.nmalloc++;
1203 		bin->stats.nrequests++;
1204 		bin->stats.curregs++;
1206 	malloc_mutex_unlock(tsdn, &bin->lock);
1326 arena_dissociate_bin_slab(extent_t *slab, arena_bin_t *bin)
1328 	/* Dissociate slab from bin. */
1329 	if (slab == bin->slabcur)
1330 		bin->slabcur = NULL;
1343 			arena_bin_slabs_nonfull_remove(bin, slab);
1349     arena_bin_t *bin)
1351 	assert(slab != bin->slabcur);
1353 	malloc_mutex_unlock(tsdn, &bin->lock);
1359 	malloc_mutex_lock(tsdn, &bin->lock);
1361 		bin->stats.curslabs--;
1366     arena_bin_t *bin)
1371 	 * Make sure that if bin->slabcur is non-NULL, it refers to the
1376 	if (bin->slabcur != NULL && extent_snad_comp(bin->slabcur, slab) > 0) {
1378 		if (extent_slab_data_get(bin->slabcur)->nfree > 0)
1379 			arena_bin_slabs_nonfull_insert(bin, bin->slabcur);
1381 			arena_bin_slabs_full_insert(bin, bin->slabcur);
1382 		bin->slabcur = slab;
1384 			bin->stats.reslabs++;
1386 		arena_bin_slabs_nonfull_insert(bin, slab);
1395 	arena_bin_t *bin = &arena->bins[binind];
1403 		arena_dissociate_bin_slab(slab, bin);
1404 		arena_dalloc_bin_slab(tsdn, arena, slab, bin);
1405 	} else if (slab_data->nfree == 1 && slab != bin->slabcur) {
1407 		arena_bin_lower_slab(tsdn, arena, slab, bin);
1411 		bin->stats.ndalloc++;
1412 		bin->stats.curregs--;
1426 	arena_bin_t *bin = &arena->bins[extent_slab_data_get(extent)->binind];
1428 	malloc_mutex_lock(tsdn, &bin->lock);
1430 	malloc_mutex_unlock(tsdn, &bin->lock);
1646 		arena_bin_t *bin = &arena->bins[i];
1648 		malloc_mutex_lock(tsdn, &bin->lock);
1649 		bstats[i].nmalloc += bin->stats.nmalloc;
1650 		bstats[i].ndalloc += bin->stats.ndalloc;
1651 		bstats[i].nrequests += bin->stats.nrequests;
1652 		bstats[i].curregs += bin->stats.curregs;
1654 			bstats[i].nfills += bin->stats.nfills;
1655 			bstats[i].nflushes += bin->stats.nflushes;
1657 		bstats[i].nslabs += bin->stats.nslabs;
1658 		bstats[i].reslabs += bin->stats.reslabs;
1659 		bstats[i].curslabs += bin->stats.curslabs;
1660 		malloc_mutex_unlock(tsdn, &bin->lock);
1766 		arena_bin_t *bin = &arena->bins[i];
1767 		if (malloc_mutex_init(&bin->lock, "arena_bin",
1770 		bin->slabcur = NULL;
1771 		extent_heap_new(&bin->slabs_nonfull);
1772 		extent_init(&bin->slabs_full, arena, NULL, 0, 0, 0, false,
1775 			memset(&bin->stats, 0, sizeof(malloc_bin_stats_t));