Lines Matching defs:tsdn
71 static void extent_record(tsdn_t *tsdn, arena_t *arena,
78 extent_alloc(tsdn_t *tsdn, arena_t *arena)
82 malloc_mutex_lock(tsdn, &arena->extent_cache_mtx);
85 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
86 return (base_alloc(tsdn, arena->base, sizeof(extent_t),
90 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
95 extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
97 malloc_mutex_lock(tsdn, &arena->extent_cache_mtx);
100 malloc_mutex_unlock(tsdn, &arena->extent_cache_mtx);
197 extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
203 malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx);
209 extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1],
215 malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx);
221 extent_rtree_acquire(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
225 *r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx,
232 *r_elm_b = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx,
236 rtree_elm_release(tsdn, &extents_rtree, *r_elm_a);
247 extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a,
250 rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent);
252 rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent);
256 extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b)
258 rtree_elm_release(tsdn, &extents_rtree, elm_a);
260 rtree_elm_release(tsdn, &extents_rtree, elm_b);
264 extent_interior_register(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
272 rtree_write(tsdn, &extents_rtree, rtree_ctx,
279 extent_gprof_add(tsdn_t *tsdn, const extent_t *extent)
295 prof_gdump(tsdn);
300 extent_gprof_sub(tsdn_t *tsdn, const extent_t *extent)
312 extent_register(tsdn_t *tsdn, const extent_t *extent)
315 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
318 if (extent_rtree_acquire(tsdn, rtree_ctx, extent, false, true, &elm_a,
321 extent_rtree_write_acquired(tsdn, elm_a, elm_b, extent);
323 extent_interior_register(tsdn, rtree_ctx, extent);
324 extent_rtree_release(tsdn, elm_a, elm_b);
327 extent_gprof_add(tsdn, extent);
333 extent_reregister(tsdn_t *tsdn, const extent_t *extent)
335 bool err = extent_register(tsdn, extent);
340 extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx,
348 rtree_clear(tsdn, &extents_rtree, rtree_ctx,
355 extent_deregister(tsdn_t *tsdn, extent_t *extent)
358 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
361 extent_rtree_acquire(tsdn, rtree_ctx, extent, true, false, &elm_a,
363 extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL);
365 extent_interior_deregister(tsdn, rtree_ctx, extent);
368 extent_rtree_release(tsdn, elm_a, elm_b);
371 extent_gprof_sub(tsdn, extent);
379 extent_first_best_fit(tsdn_t *tsdn, arena_t *arena,
384 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
397 extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
405 if (extent_purge_lazy_wrapper(tsdn, arena, r_extent_hooks,
407 extent_purge_forced_wrapper(tsdn, arena, r_extent_hooks,
411 extent_dalloc(tsdn, arena, extent);
415 extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
422 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
426 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
447 prev = extent_lookup(tsdn, (void *)((uintptr_t)new_addr - PAGE),
458 malloc_mutex_lock(tsdn, &arena->extents_mtx);
463 elm = rtree_elm_acquire(tsdn, &extents_rtree, rtree_ctx,
466 extent = rtree_elm_read_acquired(tsdn, &extents_rtree,
476 rtree_elm_release(tsdn, &extents_rtree, elm);
480 extent = extent_first_best_fit(tsdn, arena, extent_heaps,
485 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
488 extent_heaps_remove(tsdn, extent_heaps, extent);
489 arena_extent_cache_maybe_remove(tsdn, arena, extent, cache);
504 extent = extent_split_wrapper(tsdn, arena, r_extent_hooks,
508 extent_deregister(tsdn, lead);
509 extent_leak(tsdn, arena, r_extent_hooks, cache, lead);
511 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
514 extent_heaps_insert(tsdn, extent_heaps, lead);
515 arena_extent_cache_maybe_insert(tsdn, arena, lead, cache);
520 extent_t *trail = extent_split_wrapper(tsdn, arena,
523 extent_deregister(tsdn, extent);
524 extent_leak(tsdn, arena, r_extent_hooks, cache,
527 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
530 extent_heaps_insert(tsdn, extent_heaps, trail);
531 arena_extent_cache_maybe_insert(tsdn, arena, trail, cache);
541 if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
544 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
545 extent_record(tsdn, arena, r_extent_hooks, extent_heaps,
553 extent_addr_randomize(tsdn, extent, alignment);
557 extent_interior_register(tsdn, rtree_ctx, extent);
561 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
586 extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
596 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
605 extent_alloc_dss(tsdn, arena, new_addr, size, alignment, zero,
614 extent_alloc_cache_impl(tsdn_t *tsdn, arena_t *arena,
623 extent = extent_recycle(tsdn, arena, r_extent_hooks,
630 extent_alloc_cache_locked(tsdn_t *tsdn, arena_t *arena,
634 malloc_mutex_assert_owner(tsdn, &arena->extents_mtx);
636 return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, true,
641 extent_alloc_cache(tsdn_t *tsdn, arena_t *arena,
645 return (extent_alloc_cache_impl(tsdn, arena, r_extent_hooks, false,
650 extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr,
655 ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero,
664 tsdn_t *tsdn;
669 tsdn = tsdn_fetch();
670 arena = arena_get(tsdn, arena_ind, false);
677 return (extent_alloc_default_impl(tsdn, arena, new_addr, size,
682 extent_retain(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
687 extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
697 extent_grow_retained(tsdn_t *tsdn, arena_t *arena,
720 extent = extent_alloc(tsdn, arena);
725 ptr = extent_alloc_core(tsdn, arena, new_addr, alloc_size, PAGE,
729 if (ptr == NULL || extent_register(tsdn, extent)) {
730 extent_dalloc(tsdn, arena, extent);
752 extent = extent_split_wrapper(tsdn, arena, r_extent_hooks, lead,
755 extent_deregister(tsdn, lead);
756 extent_leak(tsdn, arena, r_extent_hooks, false, lead);
759 extent_retain(tsdn, arena, r_extent_hooks, lead);
764 extent_t *trail = extent_split_wrapper(tsdn, arena,
767 extent_deregister(tsdn, extent);
768 extent_leak(tsdn, arena, r_extent_hooks, false, extent);
771 extent_retain(tsdn, arena, r_extent_hooks, trail);
781 if (extent_commit_wrapper(tsdn, arena, r_extent_hooks, extent,
783 extent_retain(tsdn, arena, r_extent_hooks, extent);
791 extent_gprof_add(tsdn, extent);
794 extent_addr_randomize(tsdn, extent, alignment);
797 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn,
801 extent_interior_register(tsdn, rtree_ctx, extent);
811 extent_alloc_retained(tsdn_t *tsdn, arena_t *arena,
820 extent = extent_recycle(tsdn, arena, r_extent_hooks,
829 extent_gprof_add(tsdn, extent);
832 extent = extent_grow_retained(tsdn, arena, r_extent_hooks,
840 extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena,
849 extent = extent_alloc(tsdn, arena);
853 /* Call directly to propagate tsdn. */
854 addr = extent_alloc_default_impl(tsdn, arena, new_addr, size,
861 extent_dalloc(tsdn, arena, extent);
867 extent_addr_randomize(tsdn, extent, alignment);
868 if (extent_register(tsdn, extent)) {
869 extent_leak(tsdn, arena, r_extent_hooks, false, extent);
877 extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
885 extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr,
888 extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks,
911 extent_try_coalesce(tsdn_t *tsdn, arena_t *arena,
918 extent_heaps_remove(tsdn, extent_heaps, a);
919 extent_heaps_remove(tsdn, extent_heaps, b);
921 arena_extent_cache_maybe_remove(tsdn, extent_arena_get(a), a, cache);
922 arena_extent_cache_maybe_remove(tsdn, extent_arena_get(b), b, cache);
924 if (extent_merge_wrapper(tsdn, arena, r_extent_hooks, a, b)) {
925 extent_heaps_insert(tsdn, extent_heaps, a);
926 extent_heaps_insert(tsdn, extent_heaps, b);
927 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a,
929 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(b), b,
934 extent_heaps_insert(tsdn, extent_heaps, a);
935 arena_extent_cache_maybe_insert(tsdn, extent_arena_get(a), a, cache);
939 extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks,
944 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
948 malloc_mutex_lock(tsdn, &arena->extents_mtx);
955 extent_interior_deregister(tsdn, rtree_ctx, extent);
959 assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent);
960 extent_heaps_insert(tsdn, extent_heaps, extent);
961 arena_extent_cache_maybe_insert(tsdn, arena, extent, cache);
964 next = rtree_read(tsdn, &extents_rtree, rtree_ctx,
967 extent_try_coalesce(tsdn, arena, r_extent_hooks, extent, next,
972 prev = rtree_read(tsdn, &extents_rtree, rtree_ctx,
975 extent_try_coalesce(tsdn, arena, r_extent_hooks, prev, extent,
979 malloc_mutex_unlock(tsdn, &arena->extents_mtx);
983 extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent)
987 if (extent_register(tsdn, extent)) {
988 extent_leak(tsdn, arena, &extent_hooks, false, extent);
991 extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent);
995 extent_dalloc_cache(tsdn_t *tsdn, arena_t *arena,
1004 extent_record(tsdn, arena, r_extent_hooks, arena->extents_cached, true,
1026 extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena,
1041 extent_deregister(tsdn, extent);
1043 /* Call directly to propagate tsdn. */
1054 extent_dalloc(tsdn, arena, extent);
1060 extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena,
1065 if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent))
1068 extent_reregister(tsdn, extent);
1072 else if (!extent_decommit_wrapper(tsdn, arena, r_extent_hooks, extent,
1092 extent_gprof_sub(tsdn, extent);
1094 extent_record(tsdn, arena, r_extent_hooks, arena->extents_retained,
1109 extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena,
1134 extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena,
1167 extent_purge_lazy_wrapper(tsdn_t *tsdn, arena_t *arena,
1195 extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena,
1220 extent_split_wrapper(tsdn_t *tsdn, arena_t *arena,
1226 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
1236 trail = extent_alloc(tsdn, arena);
1248 if (extent_rtree_acquire(tsdn, rtree_ctx, &lead, false, true,
1257 if (extent_rtree_acquire(tsdn, rtree_ctx, trail, false, true,
1269 extent_rtree_write_acquired(tsdn, lead_elm_a, lead_elm_b, extent);
1270 extent_rtree_write_acquired(tsdn, trail_elm_a, trail_elm_b, trail);
1272 extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
1273 extent_rtree_release(tsdn, trail_elm_a, trail_elm_b);
1277 extent_rtree_release(tsdn, trail_elm_a, trail_elm_b);
1279 extent_rtree_release(tsdn, lead_elm_a, lead_elm_b);
1281 extent_dalloc(tsdn, arena, trail);
1309 extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena,
1314 rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback);
1323 /* Call directly to propagate tsdn. */
1341 extent_rtree_acquire(tsdn, rtree_ctx, a, true, false, &a_elm_a,
1343 extent_rtree_acquire(tsdn, rtree_ctx, b, true, false, &b_elm_a,
1347 rtree_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, NULL);
1348 rtree_elm_release(tsdn, &extents_rtree, a_elm_b);
1351 rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL);
1352 rtree_elm_release(tsdn, &extents_rtree, b_elm_a);
1362 extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a);
1363 extent_rtree_release(tsdn, a_elm_a, b_elm_b);
1365 extent_dalloc(tsdn, extent_arena_get(b), b);