jemalloc/src/arena.c

44     size_t pageind, size_t npages, bool maybe_adjac_pred,
47     size_t pageind, size_t npages, bool maybe_adjac_pred,
209 arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind)
213 	if (pageind-1 < map_bias)
216 		ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0);
218 		    pageind-1) != arena_mapbits_dirty_get(chunk, pageind));
224 arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages)
228 	if (pageind+npages == chunk_npages)
231 		assert(pageind+npages < chunk_npages);
232 		ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0);
233 		assert(ret == false || arena_mapbits_dirty_get(chunk, pageind)
234 		    != arena_mapbits_dirty_get(chunk, pageind+npages));
240 arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages)
243 	return (arena_avail_adjac_pred(chunk, pageind) ||
244 	    arena_avail_adjac_succ(chunk, pageind, npages));
248 arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
252 	assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
262 	if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind))
264 	if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages))
269 	if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
277 	    pageind));
281 arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
285 	assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
295 	if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind))
297 	if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages))
303 	if (arena_mapbits_dirty_get(chunk, pageind) != 0) {
311 	    pageind));
339 	size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
340 	size_t mapbits = arena_mapbits_get(chunk, pageind);
639 		size_t pageind = (((uintptr_t)mapelm -
643 		run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
716 	size_t pageind, npages;
758 	for (pageind = map_bias; pageind < chunk_npages; pageind += npages) {
759 		mapelm = arena_mapp_get(chunk, pageind);
760 		if (arena_mapbits_allocated_get(chunk, pageind) == 0) {
762 			    arena_mapbits_unallocated_size_get(chunk, pageind);
765 			assert(pageind + npages <= chunk_npages);
766 			assert(arena_mapbits_dirty_get(chunk, pageind) ==
767 			    arena_mapbits_dirty_get(chunk, pageind+npages-1));
769 			if (arena_mapbits_dirty_get(chunk, pageind) != 0 &&
770 			    (all || arena_avail_adjac(chunk, pageind,
773 				    chunk + (uintptr_t)(pageind << LG_PAGE));
783 			if (arena_mapbits_large_get(chunk, pageind) != 0) {
785 				    pageind) >> LG_PAGE;
790 				    chunk + (uintptr_t)(pageind << LG_PAGE));
793 				    pageind) == 0);
800 	assert(pageind == chunk_npages);
812 		pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
814 		npages = arena_mapbits_large_size_get(chunk, pageind) >>
816 		assert(pageind + npages <= chunk_npages);
817 		unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind <<
831 			arena_mapbits_unzeroed_set(chunk, pageind+i,
847 		pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
849 		run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind <<
1108 	size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
1110 	size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1119 	assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1120 	arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty);
1121 	arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty);
1126 		    pageind+head_npages+tail_npages-1) == 0);
1128 		    pageind+head_npages+tail_npages-1) == flag_dirty);
1130 	arena_mapbits_large_set(chunk, pageind+head_npages, newsize,
1140 	size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
1142 	size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
1151 	assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
1152 	arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty);
1153 	arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty);
1158 		    pageind+head_npages+tail_npages-1) == 0);
1160 		    pageind+head_npages+tail_npages-1) == flag_dirty);
1162 	arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize,
1175 		size_t pageind;
1179 		pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) /
1181 		run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
1182 		    arena_mapbits_small_runind_get(chunk, pageind)) <<
1194 	size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
1195 	arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
1206 	size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
1207 	arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
1573 	size_t pageind, binind;
1583 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
1586 	arena_mapbits_large_binind_set(chunk, pageind, binind);
1691 	size_t pageind;
1697 	pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
1698 	run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
1699 	    arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE));
1724     size_t pageind, arena_chunk_map_t *mapelm)
1729 	run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
1730 	    arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE));
1739     size_t pageind)
1746 		    pageind)) != BININD_INVALID);
1748 	mapelm = arena_mapp_get(chunk, pageind);
1749 	arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm);
1757 		size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
1758 		size_t size = arena_mapbits_large_size_get(chunk, pageind);
1816 	size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
1820 	assert(oldsize == arena_mapbits_large_size_get(chunk, pageind));
1825 	if (pageind + npages < chunk_npages &&
1826 	    arena_mapbits_allocated_get(chunk, pageind+npages) == 0 &&
1828 	    pageind+npages)) >= size - oldsize) {
1838 		    ((pageind+npages) << LG_PAGE)), splitsize, true,
1852 		flag_dirty = arena_mapbits_dirty_get(chunk, pageind) |
1853 		    arena_mapbits_dirty_get(chunk, pageind+npages-1);
1854 		arena_mapbits_large_set(chunk, pageind, size, flag_dirty);
1855 		arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty);