jemalloc/src/arena.c

362 arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages)
365 	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
367 	memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0,
372 arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
375 	UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE));
377 	VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind <<
388 	size_t run_ind, total_pages, need_pages, rem_pages, i;
395 	run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
396 	flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
397 	total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
399 	assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==
406 	arena_avail_remove(arena, chunk, run_ind, total_pages, true, true);
423 			arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
426 			    run_ind+total_pages-1, (rem_pages << LG_PAGE),
429 			arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
432 			    run_ind+need_pages));
434 			    run_ind+total_pages-1, (rem_pages << LG_PAGE),
436 			    run_ind+total_pages-1));
438 		arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages,
455 					    run_ind+i) != 0) {
456 						arena_run_zero(chunk, run_ind+i,
460 						    chunk, run_ind+i);
468 				arena_run_zero(chunk, run_ind, need_pages);
476 		arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0,
478 		arena_mapbits_large_set(chunk, run_ind, size, flag_dirty);
486 		arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty);
493 		    arena_mapbits_unzeroed_get(chunk, run_ind) == 0)
494 			arena_run_page_validate_zeroed(chunk, run_ind);
496 			arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0);
498 			    arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) {
500 				    run_ind+i);
503 		arena_mapbits_small_set(chunk, run_ind+need_pages-1,
506 		    arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) ==
509 			    run_ind+need_pages-1);
512 	VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind <<
972 	size_t size, run_ind, run_pages, flag_dirty;
975 	run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
976 	assert(run_ind >= map_bias);
977 	assert(run_ind < chunk_npages);
978 	if (arena_mapbits_large_get(chunk, run_ind) != 0) {
979 		size = arena_mapbits_large_size_get(chunk, run_ind);
982 		    run_ind+(size>>LG_PAGE)-1) == 0);
1006 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1007 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1008 	if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0)
1014 		arena_mapbits_unallocated_set(chunk, run_ind, size,
1016 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1019 		arena_mapbits_unallocated_set(chunk, run_ind, size,
1020 		    arena_mapbits_unzeroed_get(chunk, run_ind));
1021 		arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
1022 		    arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));
1026 	if (run_ind + run_pages < chunk_npages &&
1027 	    arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&
1028 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) {
1030 		    run_ind+run_pages);
1038 		    run_ind+run_pages+nrun_pages-1) == nrun_size);
1040 		    run_ind+run_pages+nrun_pages-1) == flag_dirty);
1041 		arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages,
1047 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1048 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1053 	if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1)
1054 	    == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) {
1056 		    run_ind-1);
1059 		run_ind -= prun_pages;
1065 		assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1067 		assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);
1068 		arena_avail_remove(arena, chunk, run_ind, prun_pages, true,
1074 		arena_mapbits_unallocated_size_set(chunk, run_ind, size);
1075 		arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
1080 	assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
1081 	    arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));
1082 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1083 	    arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
1084 	arena_avail_insert(arena, chunk, run_ind, run_pages, true, true);
1088 		assert(run_ind == map_bias);
1621 	size_t npages, run_ind, past;
1634 	run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
1646 	assert(arena_mapbits_dirty_get(chunk, run_ind) ==
1647 	    arena_mapbits_dirty_get(chunk, run_ind+npages-1));
1648 	if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind <
1652 		arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0);
1653 		arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0);
1655 		    ((past - run_ind) << LG_PAGE), false);
1656 		/* npages = past - run_ind; */