jemalloc/src/jemalloc.c

1450 ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
1457 	if (usize <= SMALL_MAXCLASS) {
1462 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1464 		p = ialloc(tsd, usize, ind, zero, slow_path);
1470 ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
1475 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1477 		p = ialloc_prof_sample(tsd, usize, ind, zero, tctx, slow_path);
1479 		p = ialloc(tsd, usize, ind, zero, slow_path);
1484 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1496 ialloc_body(size_t size, bool zero, tsdn_t **tsdn, size_t *usize,
1517 		*usize = index2size(ind);
1518 		assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
1522 		return (ialloc_prof(tsd, *usize, ind, zero, slow_path));
1528 ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
1544 		assert(usize == isalloc(tsdn, ret, config_prof));
1545 		*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
1557 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1563 		ret = ialloc_body(size, false, &tsdn, &usize, false);
1564 		ialloc_post_check(ret, tsdn, usize, "malloc", true, false);
1566 		ret = ialloc_body(size, false, &tsdn, &usize, true);
1567 		ialloc_post_check(ret, tsdn, usize, "malloc", true, true);
1569 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, false);
1576 imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
1583 	if (usize <= SMALL_MAXCLASS) {
1588 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1590 		p = ipalloc(tsd, usize, alignment, false);
1596 imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
1601 	tctx = prof_alloc_prep(tsd, usize, prof_active_get_unlocked(), true);
1603 		p = imemalign_prof_sample(tsd, alignment, usize, tctx);
1605 		p = ipalloc(tsd, usize, alignment, false);
1610 	prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
1621 	size_t usize;
1649 	usize = sa2u(size, alignment);
1650 	if (unlikely(usize == 0 || usize > HUGE_MAXCLASS)) {
1656 		result = imemalign_prof(tsd, alignment, usize);
1658 		result = ipalloc(tsd, usize, alignment, false);
1667 		assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
1668 		*tsd_thread_allocatedp_get(tsd) += usize;
1671 	JEMALLOC_VALGRIND_MALLOC(result != NULL, tsd_tsdn(tsd), result, usize,
1722 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1740 		ret = ialloc_body(num_size, true, &tsdn, &usize, false);
1741 		ialloc_post_check(ret, tsdn, usize, "calloc", true, false);
1743 		ret = ialloc_body(num_size, true, &tsdn, &usize, true);
1744 		ialloc_post_check(ret, tsdn, usize, "calloc", true, true);
1746 		JEMALLOC_VALGRIND_MALLOC(ret != NULL, tsdn, ret, usize, true);
1753 irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
1760 	if (usize <= SMALL_MAXCLASS) {
1764 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
1766 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1772 irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
1780 	tctx = prof_alloc_prep(tsd, usize, prof_active, true);
1782 		p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
1784 		p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
1789 	prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
1798 	size_t usize;
1807 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1808 		prof_free(tsd, ptr, usize);
1810 		usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
1812 		*tsd_thread_deallocatedp_get(tsd) += usize;
1825 isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
1835 		prof_free(tsd, ptr, usize);
1837 		*tsd_thread_deallocatedp_get(tsd) += usize;
1840 	isqalloc(tsd, ptr, usize, tcache, slow_path);
1851 	size_t usize JEMALLOC_CC_SILENCE_INIT(0);
1875 			usize = s2u(size);
1876 			ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
1877 			    NULL : irealloc_prof(tsd, ptr, old_usize, usize);
1881 				usize = s2u(size);
1888 			ret = ialloc_body(size, false, &tsdn, &usize, false);
1890 			ret = ialloc_body(size, false, &tsdn, &usize, true);
1905 		assert(usize == isalloc(tsdn, ret, config_prof));
1907 		*tsd_thread_allocatedp_get(tsd) += usize;
1911 	JEMALLOC_VALGRIND_REALLOC(true, tsdn, ret, usize, true, ptr, old_usize,
2003 imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize,
2009 		*usize = s2u(size);
2012 		*usize = sa2u(size, *alignment);
2014 	if (unlikely(*usize == 0 || *usize > HUGE_MAXCLASS))
2035 imallocx_flags(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2041 		return (ipalloct(tsdn, usize, alignment, zero, tcache, arena));
2042 	ind = size2index(usize);
2044 	return (iallocztm(tsdn, usize, ind, zero, tcache, false, arena,
2049 imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
2054 	if (usize <= SMALL_MAXCLASS) {
2061 		arena_prof_promoted(tsdn, p, usize);
2063 		p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
2071 imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
2080 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2083 	tctx = prof_alloc_prep(tsd, *usize, prof_active_get_unlocked(), true);
2085 		p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero,
2088 		p = imallocx_prof_sample(tsd_tsdn(tsd), *usize, alignment, zero,
2096 	prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
2103 imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize,
2112 	if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment,
2115 	p = imallocx_flags(tsd_tsdn(tsd), *usize, alignment, zero, tcache,
2123 imallocx_body(size_t size, int flags, tsdn_t **tsdn, size_t *usize,
2143 			*usize = index2size(ind);
2144 			assert(*usize > 0 && *usize <= HUGE_MAXCLASS);
2148 			return (ialloc_prof(tsd, *usize, ind, false,
2156 		return (imallocx_prof(tsd, size, flags, usize, slow_path));
2158 	return (imallocx_no_prof(tsd, size, flags, usize, slow_path));
2168 	size_t usize;
2173 		p = imallocx_body(size, flags, &tsdn, &usize, false);
2174 		ialloc_post_check(p, tsdn, usize, "mallocx", false, false);
2176 		p = imallocx_body(size, flags, &tsdn, &usize, true);
2177 		ialloc_post_check(p, tsdn, usize, "mallocx", false, true);
2179 		JEMALLOC_VALGRIND_MALLOC(p != NULL, tsdn, p, usize,
2188     size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
2195 	if (usize <= SMALL_MAXCLASS) {
2200 		arena_prof_promoted(tsd_tsdn(tsd), p, usize);
2202 		p = iralloct(tsd, old_ptr, old_usize, usize, alignment, zero,
2211     size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
2220 	tctx = prof_alloc_prep(tsd, *usize, prof_active, false);
2222 		p = irallocx_prof_sample(tsd, old_ptr, old_usize, *usize,
2239 		 * be the same as the current usize because of in-place large
2240 		 * reallocation.  Therefore, query the actual value of usize.
2242 		*usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2244 	prof_realloc(tsd, p, *usize, tctx, prof_active, false, old_ptr,
2257 	size_t usize;
2293 		usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
2294 		if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
2296 		p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
2306 			usize = isalloc(tsd_tsdn(tsd), p, config_prof);
2311 		*tsd_thread_allocatedp_get(tsd) += usize;
2315 	JEMALLOC_VALGRIND_REALLOC(true, tsd_tsdn(tsd), p, usize, false, ptr,
2333 	size_t usize;
2337 	usize = isalloc(tsdn, ptr, config_prof);
2339 	return (usize);
2346 	size_t usize;
2350 	usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
2353 	return (usize);
2360 	size_t usize_max, usize;
2367 	 * usize isn't knowable before ixalloc() returns when extra is non-zero.
2370 	 * prof_realloc() will use the actual usize to decide whether to sample.
2390 		usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
2393 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2396 	if (usize == old_usize) {
2398 		return (usize);
2400 	prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
2403 	return (usize);
2410 	size_t usize, old_usize;
2435 		usize = old_usize;
2445 		usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
2448 		usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
2451 	if (unlikely(usize == old_usize))
2455 		*tsd_thread_allocatedp_get(tsd) += usize;
2458 	JEMALLOC_VALGRIND_REALLOC(false, tsd_tsdn(tsd), ptr, usize, false, ptr,
2463 	return (usize);
2470 	size_t usize;
2480 		usize = ivsalloc(tsdn, ptr, config_prof);
2482 		usize = isalloc(tsdn, ptr, config_prof);
2485 	return (usize);
2518 	size_t usize;
2523 		usize = s2u(size);
2525 		usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags));
2527 	return (usize);
2535 	size_t usize;
2540 	usize = inallocx(tsd_tsdn(tsd), size, flags);
2541 	assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
2554 		isfree(tsd, ptr, usize, tcache, false);
2556 		isfree(tsd, ptr, usize, tcache, true);
2564 	size_t usize;
2575 	usize = inallocx(tsdn, size, flags);
2576 	if (unlikely(usize > HUGE_MAXCLASS))
2580 	return (usize);
2711 		size_t usize = je_xallocx(*ptr, size, extra, flags);
2712 		ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
2714 			*rsize = usize;
2748 	size_t usize;
2750 	usize = je_nallocx(size, flags);
2751 	if (usize == 0)
2754 		*rsize = usize;