huge.c revision 288090
1#define JEMALLOC_HUGE_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3 4/******************************************************************************/ 5 6static extent_node_t * 7huge_node_get(const void *ptr) 8{ 9 extent_node_t *node; 10 11 node = chunk_lookup(ptr, true); 12 assert(!extent_node_achunk_get(node)); 13 14 return (node); 15} 16 17static bool 18huge_node_set(const void *ptr, extent_node_t *node) 19{ 20 21 assert(extent_node_addr_get(node) == ptr); 22 assert(!extent_node_achunk_get(node)); 23 return (chunk_register(ptr, node)); 24} 25 26static void 27huge_node_unset(const void *ptr, const extent_node_t *node) 28{ 29 30 chunk_deregister(ptr, node); 31} 32 33void * 34huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, 35 tcache_t *tcache) 36{ 37 size_t usize; 38 39 usize = s2u(size); 40 if (usize == 0) { 41 /* size_t overflow. */ 42 return (NULL); 43 } 44 45 return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache)); 46} 47 48void * 49huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, 50 bool zero, tcache_t *tcache) 51{ 52 void *ret; 53 size_t usize; 54 extent_node_t *node; 55 bool is_zeroed; 56 57 /* Allocate one or more contiguous chunks for this request. */ 58 59 usize = sa2u(size, alignment); 60 if (unlikely(usize == 0)) 61 return (NULL); 62 assert(usize >= chunksize); 63 64 /* Allocate an extent node with which to track the chunk. */ 65 node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), 66 CACHELINE, false, tcache, true, arena); 67 if (node == NULL) 68 return (NULL); 69 70 /* 71 * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that 72 * it is possible to make correct junk/zero fill decisions below. 73 */ 74 is_zeroed = zero; 75 arena = arena_choose(tsd, arena); 76 if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, 77 size, alignment, &is_zeroed)) == NULL) { 78 idalloctm(tsd, node, tcache, true); 79 return (NULL); 80 } 81 82 extent_node_init(node, arena, ret, size, is_zeroed, true); 83 84 if (huge_node_set(ret, node)) { 85 arena_chunk_dalloc_huge(arena, ret, size); 86 idalloctm(tsd, node, tcache, true); 87 return (NULL); 88 } 89 90 /* Insert node into huge. */ 91 malloc_mutex_lock(&arena->huge_mtx); 92 ql_elm_new(node, ql_link); 93 ql_tail_insert(&arena->huge, node, ql_link); 94 malloc_mutex_unlock(&arena->huge_mtx); 95 96 if (zero || (config_fill && unlikely(opt_zero))) { 97 if (!is_zeroed) 98 memset(ret, 0, size); 99 } else if (config_fill && unlikely(opt_junk_alloc)) 100 memset(ret, 0xa5, size); 101 102 return (ret); 103} 104 105#ifdef JEMALLOC_JET 106#undef huge_dalloc_junk 107#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) 108#endif 109static void 110huge_dalloc_junk(void *ptr, size_t usize) 111{ 112 113 if (config_fill && have_dss && unlikely(opt_junk_free)) { 114 /* 115 * Only bother junk filling if the chunk isn't about to be 116 * unmapped. 117 */ 118 if (!config_munmap || (have_dss && chunk_in_dss(ptr))) 119 memset(ptr, 0x5a, usize); 120 } 121} 122#ifdef JEMALLOC_JET 123#undef huge_dalloc_junk 124#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) 125huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); 126#endif 127 128static void 129huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize_min, 130 size_t usize_max, bool zero) 131{ 132 size_t usize, usize_next; 133 extent_node_t *node; 134 arena_t *arena; 135 chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER; 136 bool zeroed; 137 138 /* Increase usize to incorporate extra. */ 139 for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1)) 140 <= oldsize; usize = usize_next) 141 ; /* Do nothing. */ 142 143 if (oldsize == usize) 144 return; 145 146 node = huge_node_get(ptr); 147 arena = extent_node_arena_get(node); 148 149 /* Fill if necessary (shrinking). */ 150 if (oldsize > usize) { 151 size_t sdiff = oldsize - usize; 152 if (config_fill && unlikely(opt_junk_free)) { 153 memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); 154 zeroed = false; 155 } else { 156 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, ptr, 157 CHUNK_CEILING(oldsize), usize, sdiff); 158 } 159 } else 160 zeroed = true; 161 162 malloc_mutex_lock(&arena->huge_mtx); 163 /* Update the size of the huge allocation. */ 164 assert(extent_node_size_get(node) != usize); 165 extent_node_size_set(node, usize); 166 /* Clear node's zeroed field if zeroing failed above. */ 167 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed); 168 malloc_mutex_unlock(&arena->huge_mtx); 169 170 arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); 171 172 /* Fill if necessary (growing). */ 173 if (oldsize < usize) { 174 if (zero || (config_fill && unlikely(opt_zero))) { 175 if (!zeroed) { 176 memset((void *)((uintptr_t)ptr + oldsize), 0, 177 usize - oldsize); 178 } 179 } else if (config_fill && unlikely(opt_junk_alloc)) { 180 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - 181 oldsize); 182 } 183 } 184} 185 186static bool 187huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) 188{ 189 extent_node_t *node; 190 arena_t *arena; 191 chunk_hooks_t chunk_hooks; 192 size_t cdiff; 193 bool zeroed; 194 195 node = huge_node_get(ptr); 196 arena = extent_node_arena_get(node); 197 chunk_hooks = chunk_hooks_get(arena); 198 199 assert(oldsize > usize); 200 201 /* Split excess chunks. */ 202 cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); 203 if (cdiff != 0 && chunk_hooks.split(ptr, CHUNK_CEILING(oldsize), 204 CHUNK_CEILING(usize), cdiff, true, arena->ind)) 205 return (true); 206 207 if (oldsize > usize) { 208 size_t sdiff = oldsize - usize; 209 if (config_fill && unlikely(opt_junk_free)) { 210 huge_dalloc_junk((void *)((uintptr_t)ptr + usize), 211 sdiff); 212 zeroed = false; 213 } else { 214 zeroed = !chunk_purge_wrapper(arena, &chunk_hooks, 215 CHUNK_ADDR2BASE((uintptr_t)ptr + usize), 216 CHUNK_CEILING(oldsize), 217 CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); 218 } 219 } else 220 zeroed = true; 221 222 malloc_mutex_lock(&arena->huge_mtx); 223 /* Update the size of the huge allocation. */ 224 extent_node_size_set(node, usize); 225 /* Clear node's zeroed field if zeroing failed above. */ 226 extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed); 227 malloc_mutex_unlock(&arena->huge_mtx); 228 229 /* Zap the excess chunks. */ 230 arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); 231 232 return (false); 233} 234 235static bool 236huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t usize, bool zero) { 237 extent_node_t *node; 238 arena_t *arena; 239 bool is_zeroed_subchunk, is_zeroed_chunk; 240 241 node = huge_node_get(ptr); 242 arena = extent_node_arena_get(node); 243 malloc_mutex_lock(&arena->huge_mtx); 244 is_zeroed_subchunk = extent_node_zeroed_get(node); 245 malloc_mutex_unlock(&arena->huge_mtx); 246 247 /* 248 * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so 249 * that it is possible to make correct junk/zero fill decisions below. 250 */ 251 is_zeroed_chunk = zero; 252 253 if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize, 254 &is_zeroed_chunk)) 255 return (true); 256 257 malloc_mutex_lock(&arena->huge_mtx); 258 /* Update the size of the huge allocation. */ 259 extent_node_size_set(node, usize); 260 malloc_mutex_unlock(&arena->huge_mtx); 261 262 if (zero || (config_fill && unlikely(opt_zero))) { 263 if (!is_zeroed_subchunk) { 264 memset((void *)((uintptr_t)ptr + oldsize), 0, 265 CHUNK_CEILING(oldsize) - oldsize); 266 } 267 if (!is_zeroed_chunk) { 268 memset((void *)((uintptr_t)ptr + 269 CHUNK_CEILING(oldsize)), 0, usize - 270 CHUNK_CEILING(oldsize)); 271 } 272 } else if (config_fill && unlikely(opt_junk_alloc)) { 273 memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - 274 oldsize); 275 } 276 277 return (false); 278} 279 280bool 281huge_ralloc_no_move(void *ptr, size_t oldsize, size_t usize_min, 282 size_t usize_max, bool zero) 283{ 284 285 assert(s2u(oldsize) == oldsize); 286 287 /* Both allocations must be huge to avoid a move. */ 288 if (oldsize < chunksize || usize_max < chunksize) 289 return (true); 290 291 if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) { 292 /* Attempt to expand the allocation in-place. */ 293 if (!huge_ralloc_no_move_expand(ptr, oldsize, usize_max, zero)) 294 return (false); 295 /* Try again, this time with usize_min. */ 296 if (usize_min < usize_max && CHUNK_CEILING(usize_min) > 297 CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(ptr, 298 oldsize, usize_min, zero)) 299 return (false); 300 } 301 302 /* 303 * Avoid moving the allocation if the existing chunk size accommodates 304 * the new size. 305 */ 306 if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min) 307 && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) { 308 huge_ralloc_no_move_similar(ptr, oldsize, usize_min, usize_max, 309 zero); 310 return (false); 311 } 312 313 /* Attempt to shrink the allocation in-place. */ 314 if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) 315 return (huge_ralloc_no_move_shrink(ptr, oldsize, usize_max)); 316 return (true); 317} 318 319static void * 320huge_ralloc_move_helper(tsd_t *tsd, arena_t *arena, size_t usize, 321 size_t alignment, bool zero, tcache_t *tcache) 322{ 323 324 if (alignment <= chunksize) 325 return (huge_malloc(tsd, arena, usize, zero, tcache)); 326 return (huge_palloc(tsd, arena, usize, alignment, zero, tcache)); 327} 328 329void * 330huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t usize, 331 size_t alignment, bool zero, tcache_t *tcache) 332{ 333 void *ret; 334 size_t copysize; 335 336 /* Try to avoid moving the allocation. */ 337 if (!huge_ralloc_no_move(ptr, oldsize, usize, usize, zero)) 338 return (ptr); 339 340 /* 341 * usize and oldsize are different enough that we need to use a 342 * different size class. In that case, fall back to allocating new 343 * space and copying. 344 */ 345 ret = huge_ralloc_move_helper(tsd, arena, usize, alignment, zero, 346 tcache); 347 if (ret == NULL) 348 return (NULL); 349 350 copysize = (usize < oldsize) ? usize : oldsize; 351 memcpy(ret, ptr, copysize); 352 isqalloc(tsd, ptr, oldsize, tcache); 353 return (ret); 354} 355 356void 357huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) 358{ 359 extent_node_t *node; 360 arena_t *arena; 361 362 node = huge_node_get(ptr); 363 arena = extent_node_arena_get(node); 364 huge_node_unset(ptr, node); 365 malloc_mutex_lock(&arena->huge_mtx); 366 ql_remove(&arena->huge, node, ql_link); 367 malloc_mutex_unlock(&arena->huge_mtx); 368 369 huge_dalloc_junk(extent_node_addr_get(node), 370 extent_node_size_get(node)); 371 arena_chunk_dalloc_huge(extent_node_arena_get(node), 372 extent_node_addr_get(node), extent_node_size_get(node)); 373 idalloctm(tsd, node, tcache, true); 374} 375 376arena_t * 377huge_aalloc(const void *ptr) 378{ 379 380 return (extent_node_arena_get(huge_node_get(ptr))); 381} 382 383size_t 384huge_salloc(const void *ptr) 385{ 386 size_t size; 387 extent_node_t *node; 388 arena_t *arena; 389 390 node = huge_node_get(ptr); 391 arena = extent_node_arena_get(node); 392 malloc_mutex_lock(&arena->huge_mtx); 393 size = extent_node_size_get(node); 394 malloc_mutex_unlock(&arena->huge_mtx); 395 396 return (size); 397} 398 399prof_tctx_t * 400huge_prof_tctx_get(const void *ptr) 401{ 402 prof_tctx_t *tctx; 403 extent_node_t *node; 404 arena_t *arena; 405 406 node = huge_node_get(ptr); 407 arena = extent_node_arena_get(node); 408 malloc_mutex_lock(&arena->huge_mtx); 409 tctx = extent_node_prof_tctx_get(node); 410 malloc_mutex_unlock(&arena->huge_mtx); 411 412 return (tctx); 413} 414 415void 416huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) 417{ 418 extent_node_t *node; 419 arena_t *arena; 420 421 node = huge_node_get(ptr); 422 arena = extent_node_arena_get(node); 423 malloc_mutex_lock(&arena->huge_mtx); 424 extent_node_prof_tctx_set(node, tctx); 425 malloc_mutex_unlock(&arena->huge_mtx); 426} 427 428void 429huge_prof_tctx_reset(const void *ptr) 430{ 431 432 huge_prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); 433} 434