Lines Matching defs:msp
283 static void metaslab_passivate(metaslab_t *msp, uint64_t weight);
284 static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp);
588 metaslab_allocated_space(metaslab_t *msp)
590 return (msp->ms_allocated_space);
597 metaslab_verify_space(metaslab_t *msp, uint64_t txg)
599 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
603 ASSERT(MUTEX_HELD(&msp->ms_lock));
604 ASSERT(!msp->ms_condensing);
616 if (txg != spa_syncing_txg(spa) || msp->ms_sm == NULL ||
617 !msp->ms_loaded)
625 ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0);
627 sm_free_space = msp->ms_size - metaslab_allocated_space(msp);
635 range_tree_space(msp->ms_allocating[(txg + t) & TXG_MASK]);
638 ASSERT3U(msp->ms_deferspace, ==,
639 range_tree_space(msp->ms_defer[0]) +
640 range_tree_space(msp->ms_defer[1]));
642 msp_free_space = range_tree_space(msp->ms_allocatable) + allocating +
643 msp->ms_deferspace + range_tree_space(msp->ms_freed);
883 metaslab_t *msp = mg->mg_primaries[i];
884 if (msp != NULL) {
885 mutex_enter(&msp->ms_lock);
886 metaslab_passivate(msp,
887 metaslab_weight_from_range_tree(msp));
888 mutex_exit(&msp->ms_lock);
890 msp = mg->mg_secondaries[i];
891 if (msp != NULL) {
892 mutex_enter(&msp->ms_lock);
893 metaslab_passivate(msp,
894 metaslab_weight_from_range_tree(msp));
895 mutex_exit(&msp->ms_lock);
948 metaslab_t *msp = vd->vdev_ms[m];
949 ASSERT(msp != NULL);
952 if (msp->ms_sm == NULL || msp->ms_group != mg)
957 msp->ms_sm->sm_phys->smp_histogram[i];
967 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp)
972 ASSERT(MUTEX_HELD(&msp->ms_lock));
973 if (msp->ms_sm == NULL)
979 msp->ms_sm->sm_phys->smp_histogram[i];
981 msp->ms_sm->sm_phys->smp_histogram[i];
987 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp)
992 ASSERT(MUTEX_HELD(&msp->ms_lock));
993 if (msp->ms_sm == NULL)
999 msp->ms_sm->sm_phys->smp_histogram[i]);
1001 msp->ms_sm->sm_phys->smp_histogram[i]);
1004 msp->ms_sm->sm_phys->smp_histogram[i];
1006 msp->ms_sm->sm_phys->smp_histogram[i];
1012 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp)
1014 ASSERT(msp->ms_group == NULL);
1016 msp->ms_group = mg;
1017 msp->ms_weight = 0;
1018 avl_add(&mg->mg_metaslab_tree, msp);
1021 mutex_enter(&msp->ms_lock);
1022 metaslab_group_histogram_add(mg, msp);
1023 mutex_exit(&msp->ms_lock);
1027 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp)
1029 mutex_enter(&msp->ms_lock);
1030 metaslab_group_histogram_remove(mg, msp);
1031 mutex_exit(&msp->ms_lock);
1034 ASSERT(msp->ms_group == mg);
1035 avl_remove(&mg->mg_metaslab_tree, msp);
1036 msp->ms_group = NULL;
1041 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
1044 ASSERT(msp->ms_group == mg);
1045 avl_remove(&mg->mg_metaslab_tree, msp);
1046 msp->ms_weight = weight;
1047 avl_add(&mg->mg_metaslab_tree, msp);
1052 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
1059 ASSERT(MUTEX_HELD(&msp->ms_lock));
1062 metaslab_group_sort_impl(mg, msp, weight);
1081 metaslab_t *msp = vd->vdev_ms[m];
1083 if (msp->ms_fragmentation == ZFS_FRAG_INVALID)
1085 if (msp->ms_group != mg)
1089 fragmentation += msp->ms_fragmentation;
1245 metaslab_block_maxsize(metaslab_t *msp)
1247 avl_tree_t *t = &msp->ms_allocatable_by_size;
1311 metaslab_ff_alloc(metaslab_t *msp, uint64_t size)
1321 uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1];
1322 avl_tree_t *t = &msp->ms_allocatable->rt_root;
1340 metaslab_df_alloc(metaslab_t *msp, uint64_t size)
1350 uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1];
1351 range_tree_t *rt = msp->ms_allocatable;
1353 uint64_t max_size = metaslab_block_maxsize(msp);
1354 int free_pct = range_tree_space(rt) * 100 / msp->ms_size;
1356 ASSERT(MUTEX_HELD(&msp->ms_lock));
1358 avl_numnodes(&msp->ms_allocatable_by_size));
1369 t = &msp->ms_allocatable_by_size;
1390 metaslab_cf_alloc(metaslab_t *msp, uint64_t size)
1392 range_tree_t *rt = msp->ms_allocatable;
1393 avl_tree_t *t = &msp->ms_allocatable_by_size;
1394 uint64_t *cursor = &msp->ms_lbas[0];
1395 uint64_t *cursor_end = &msp->ms_lbas[1];
1398 ASSERT(MUTEX_HELD(&msp->ms_lock));
1406 rs = avl_last(&msp->ms_allocatable_by_size);
1440 metaslab_ndf_alloc(metaslab_t *msp, uint64_t size)
1442 avl_tree_t *t = &msp->ms_allocatable->rt_root;
1446 uint64_t *cursor = &msp->ms_lbas[hbit - 1];
1447 uint64_t max_size = metaslab_block_maxsize(msp);
1449 ASSERT(MUTEX_HELD(&msp->ms_lock));
1451 avl_numnodes(&msp->ms_allocatable_by_size));
1461 t = &msp->ms_allocatable_by_size;
1492 metaslab_aux_histograms_clear(metaslab_t *msp)
1498 ASSERT(msp->ms_loaded);
1500 bzero(msp->ms_synchist, sizeof (msp->ms_synchist));
1502 bzero(msp->ms_deferhist[t], sizeof (msp->ms_deferhist[t]));
1539 metaslab_aux_histograms_update(metaslab_t *msp)
1541 space_map_t *sm = msp->ms_sm;
1550 if (msp->ms_loaded) {
1551 metaslab_aux_histograms_clear(msp);
1553 metaslab_aux_histogram_add(msp->ms_synchist,
1554 sm->sm_shift, msp->ms_freed);
1557 metaslab_aux_histogram_add(msp->ms_deferhist[t],
1558 sm->sm_shift, msp->ms_defer[t]);
1562 metaslab_aux_histogram_add(msp->ms_synchist,
1563 sm->sm_shift, msp->ms_freeing);
1572 metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed)
1574 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
1575 space_map_t *sm = msp->ms_sm;
1592 bcopy(msp->ms_synchist, msp->ms_deferhist[hist_index],
1593 sizeof (msp->ms_synchist));
1595 bzero(msp->ms_deferhist[hist_index],
1596 sizeof (msp->ms_deferhist[hist_index]));
1598 bzero(msp->ms_synchist, sizeof (msp->ms_synchist));
1607 metaslab_verify_weight_and_frag(metaslab_t *msp)
1609 ASSERT(MUTEX_HELD(&msp->ms_lock));
1615 if (msp->ms_group == NULL)
1623 vdev_t *vd = msp->ms_group->mg_vd;
1633 if (txg_list_member(&vd->vdev_ms_list, msp, t))
1642 if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa))
1646 if (msp->ms_loaded) {
1647 range_tree_stat_verify(msp->ms_allocatable);
1648 VERIFY(space_map_histogram_verify(msp->ms_sm,
1649 msp->ms_allocatable));
1652 uint64_t weight = msp->ms_weight;
1653 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
1654 boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight);
1655 uint64_t frag = msp->ms_fragmentation;
1656 uint64_t max_segsize = msp->ms_max_size;
1658 msp->ms_weight = 0;
1659 msp->ms_fragmentation = 0;
1660 msp->ms_max_size = 0;
1669 msp->ms_weight = metaslab_weight(msp) | was_active;
1671 VERIFY3U(max_segsize, ==, msp->ms_max_size);
1677 if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) ||
1678 (!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) {
1679 msp->ms_fragmentation = frag;
1680 msp->ms_weight = weight;
1684 VERIFY3U(msp->ms_fragmentation, ==, frag);
1685 VERIFY3U(msp->ms_weight, ==, weight);
1692 metaslab_load_wait(metaslab_t *msp)
1694 ASSERT(MUTEX_HELD(&msp->ms_lock));
1696 while (msp->ms_loading) {
1697 ASSERT(!msp->ms_loaded);
1698 cv_wait(&msp->ms_load_cv, &msp->ms_lock);
1703 metaslab_load_impl(metaslab_t *msp)
1707 ASSERT(MUTEX_HELD(&msp->ms_lock));
1708 ASSERT(msp->ms_loading);
1709 ASSERT(!msp->ms_condensing);
1729 uint64_t length = msp->ms_synced_length;
1730 mutex_exit(&msp->ms_lock);
1732 if (msp->ms_sm != NULL) {
1733 error = space_map_load_length(msp->ms_sm, msp->ms_allocatable,
1741 range_tree_add(msp->ms_allocatable,
1742 msp->ms_start, msp->ms_size);
1752 mutex_enter(&msp->ms_sync_lock);
1753 mutex_enter(&msp->ms_lock);
1754 ASSERT(!msp->ms_condensing);
1757 mutex_exit(&msp->ms_sync_lock);
1761 ASSERT3P(msp->ms_group, !=, NULL);
1762 msp->ms_loaded = B_TRUE;
1771 range_tree_walk(msp->ms_defer[t],
1772 range_tree_remove, msp->ms_allocatable);
1787 uint64_t weight = msp->ms_weight;
1788 metaslab_recalculate_weight_and_sort(msp);
1790 ASSERT3U(weight, <=, msp->ms_weight);
1791 msp->ms_max_size = metaslab_block_maxsize(msp);
1793 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
1794 metaslab_verify_space(msp, spa_syncing_txg(spa));
1795 mutex_exit(&msp->ms_sync_lock);
1801 metaslab_load(metaslab_t *msp)
1803 ASSERT(MUTEX_HELD(&msp->ms_lock));
1809 metaslab_load_wait(msp);
1810 if (msp->ms_loaded)
1812 VERIFY(!msp->ms_loading);
1813 ASSERT(!msp->ms_condensing);
1815 msp->ms_loading = B_TRUE;
1816 int error = metaslab_load_impl(msp);
1817 msp->ms_loading = B_FALSE;
1818 cv_broadcast(&msp->ms_load_cv);
1824 metaslab_unload(metaslab_t *msp)
1826 ASSERT(MUTEX_HELD(&msp->ms_lock));
1828 metaslab_verify_weight_and_frag(msp);
1830 range_tree_vacate(msp->ms_allocatable, NULL, NULL);
1831 msp->ms_loaded = B_FALSE;
1833 msp->ms_weight &= ~METASLAB_ACTIVE_MASK;
1834 msp->ms_max_size = 0;
1848 if (msp->ms_group != NULL)
1849 metaslab_recalculate_weight_and_sort(msp);
1867 metaslab_t **msp)
1956 *msp = ms;
1962 metaslab_fini(metaslab_t *msp)
1964 metaslab_group_t *mg = msp->ms_group;
1967 metaslab_group_remove(mg, msp);
1969 mutex_enter(&msp->ms_lock);
1970 VERIFY(msp->ms_group == NULL);
1972 -metaslab_allocated_space(msp), 0, -msp->ms_size);
1974 space_map_close(msp->ms_sm);
1976 metaslab_unload(msp);
1978 range_tree_destroy(msp->ms_allocatable);
1979 range_tree_destroy(msp->ms_freeing);
1980 range_tree_destroy(msp->ms_freed);
1983 range_tree_destroy(msp->ms_allocating[t]);
1987 range_tree_destroy(msp->ms_defer[t]);
1989 ASSERT0(msp->ms_deferspace);
1991 range_tree_destroy(msp->ms_checkpointing);
1994 ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t));
1996 mutex_exit(&msp->ms_lock);
1997 cv_destroy(&msp->ms_load_cv);
1998 mutex_destroy(&msp->ms_lock);
1999 mutex_destroy(&msp->ms_sync_lock);
2000 ASSERT3U(msp->ms_allocator, ==, -1);
2002 kmem_free(msp, sizeof (metaslab_t));
2052 metaslab_set_fragmentation(metaslab_t *msp)
2054 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
2061 msp->ms_fragmentation = ZFS_FRAG_INVALID;
2069 if (msp->ms_sm == NULL) {
2070 msp->ms_fragmentation = 0;
2078 if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) {
2080 vdev_t *vd = msp->ms_group->mg_vd;
2090 msp->ms_condense_wanted = B_TRUE;
2091 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
2093 "ms_id %llu, vdev_id %llu", txg, msp->ms_id,
2096 msp->ms_fragmentation = ZFS_FRAG_INVALID;
2102 uint8_t shift = msp->ms_sm->sm_shift;
2107 if (msp->ms_sm->sm_phys->smp_histogram[i] == 0)
2110 space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift);
2121 msp->ms_fragmentation = fragmentation;
2130 metaslab_space_weight(metaslab_t *msp)
2132 metaslab_group_t *mg = msp->ms_group;
2136 ASSERT(MUTEX_HELD(&msp->ms_lock));
2142 space = msp->ms_size - metaslab_allocated_space(msp);
2145 msp->ms_fragmentation != ZFS_FRAG_INVALID) {
2153 space = (space * (100 - (msp->ms_fragmentation - 1))) / 100;
2177 weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count;
2187 if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID &&
2188 msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) {
2189 weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK);
2203 metaslab_weight_from_range_tree(metaslab_t *msp)
2208 ASSERT(msp->ms_loaded);
2212 uint8_t shift = msp->ms_group->mg_vd->vdev_ashift;
2216 segments += msp->ms_allocatable->rt_histogram[i];
2244 metaslab_weight_from_spacemap(metaslab_t *msp)
2246 space_map_t *sm = msp->ms_sm;
2247 ASSERT(!msp->ms_loaded);
2262 deferspace_histogram[i] += msp->ms_synchist[i];
2265 deferspace_histogram[i] += msp->ms_deferhist[t][i];
2291 metaslab_segment_weight(metaslab_t *msp)
2293 metaslab_group_t *mg = msp->ms_group;
2297 ASSERT(MUTEX_HELD(&msp->ms_lock));
2302 if (metaslab_allocated_space(msp) == 0) {
2303 int idx = highbit64(msp->ms_size) - 1;
2319 ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t));
2324 if (metaslab_allocated_space(msp) == msp->ms_size)
2331 if (msp->ms_loaded) {
2332 weight = metaslab_weight_from_range_tree(msp);
2334 weight = metaslab_weight_from_spacemap(msp);
2342 if (msp->ms_activation_weight != 0 && weight != 0)
2343 WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight));
2357 metaslab_should_allocate(metaslab_t *msp, uint64_t asize)
2361 if (msp->ms_max_size != 0)
2362 return (msp->ms_max_size >= asize);
2364 if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
2372 1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1));
2375 (msp->ms_weight & ~METASLAB_WEIGHT_TYPE));
2381 metaslab_weight(metaslab_t *msp)
2383 vdev_t *vd = msp->ms_group->mg_vd;
2387 ASSERT(MUTEX_HELD(&msp->ms_lock));
2396 metaslab_set_fragmentation(msp);
2403 if (msp->ms_loaded)
2404 msp->ms_max_size = metaslab_block_maxsize(msp);
2406 ASSERT0(msp->ms_max_size);
2413 (msp->ms_sm == NULL || msp->ms_sm->sm_dbuf->db_size ==
2415 weight = metaslab_segment_weight(msp);
2417 weight = metaslab_space_weight(msp);
2423 metaslab_recalculate_weight_and_sort(metaslab_t *msp)
2426 uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
2427 metaslab_group_sort(msp->ms_group, msp,
2428 metaslab_weight(msp) | was_active);
2432 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp,
2444 ASSERT(MUTEX_HELD(&msp->ms_lock));
2451 arr[allocator] = msp;
2452 ASSERT3S(msp->ms_allocator, ==, -1);
2453 msp->ms_allocator = allocator;
2454 msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY);
2461 metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight)
2463 ASSERT(MUTEX_HELD(&msp->ms_lock));
2465 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0) {
2466 int error = metaslab_load(msp);
2468 metaslab_group_sort(msp->ms_group, msp, 0);
2471 if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) {
2478 if ((error = metaslab_activate_allocator(msp->ms_group, msp,
2483 msp->ms_activation_weight = msp->ms_weight;
2484 metaslab_group_sort(msp->ms_group, msp,
2485 msp->ms_weight | activation_weight);
2487 ASSERT(msp->ms_loaded);
2488 ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
2494 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp,
2497 ASSERT(MUTEX_HELD(&msp->ms_lock));
2498 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) {
2499 metaslab_group_sort(mg, msp, weight);
2504 ASSERT3P(msp->ms_group, ==, mg);
2505 if (msp->ms_primary) {
2506 ASSERT3U(0, <=, msp->ms_allocator);
2507 ASSERT3U(msp->ms_allocator, <, mg->mg_allocators);
2508 ASSERT3P(mg->mg_primaries[msp->ms_allocator], ==, msp);
2509 ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
2510 mg->mg_primaries[msp->ms_allocator] = NULL;
2512 ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
2513 ASSERT3P(mg->mg_secondaries[msp->ms_allocator], ==, msp);
2514 mg->mg_secondaries[msp->ms_allocator] = NULL;
2516 msp->ms_allocator = -1;
2517 metaslab_group_sort_impl(mg, msp, weight);
2522 metaslab_passivate(metaslab_t *msp, uint64_t weight)
2532 range_tree_is_empty(msp->ms_allocatable));
2535 msp->ms_activation_weight = 0;
2536 metaslab_passivate_allocator(msp->ms_group, msp, weight);
2537 ASSERT((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0);
2552 metaslab_segment_may_passivate(metaslab_t *msp)
2554 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
2556 if (WEIGHT_IS_SPACEBASED(msp->ms_weight) || spa_sync_pass(spa) > 1)
2564 uint64_t weight = metaslab_weight_from_range_tree(msp);
2565 int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight);
2569 metaslab_passivate(msp, weight);
2575 metaslab_t *msp = arg;
2576 spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
2578 ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock));
2580 mutex_enter(&msp->ms_lock);
2581 (void) metaslab_load(msp);
2582 msp->ms_selected_txg = spa_syncing_txg(spa);
2583 mutex_exit(&msp->ms_lock);
2590 metaslab_t *msp;
2604 for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) {
2605 ASSERT3P(msp->ms_group, ==, mg);
2613 if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) {
2618 msp, TQ_SLEEP) != 0);
2644 metaslab_should_condense(metaslab_t *msp)
2646 space_map_t *sm = msp->ms_sm;
2647 vdev_t *vd = msp->ms_group->mg_vd;
2651 ASSERT(MUTEX_HELD(&msp->ms_lock));
2652 ASSERT(msp->ms_loaded);
2667 if (msp->ms_condense_checked_txg == current_txg)
2669 msp->ms_condense_checked_txg = current_txg;
2675 if (avl_is_empty(&msp->ms_allocatable_by_size) ||
2676 msp->ms_condense_wanted)
2679 uint64_t object_size = space_map_length(msp->ms_sm);
2681 msp->ms_allocatable, SM_NO_VDEVID);
2697 metaslab_condense(metaslab_t *msp, uint64_t txg, dmu_tx_t *tx)
2700 space_map_t *sm = msp->ms_sm;
2702 ASSERT(MUTEX_HELD(&msp->ms_lock));
2703 ASSERT(msp->ms_loaded);
2705 zfs_dbgmsg("condensing: txg %llu, msp[%llu] %p, vdev id %llu, "
2707 msp->ms_id, msp, msp->ms_group->mg_vd->vdev_id,
2708 msp->ms_group->mg_vd->vdev_spa->spa_name,
2709 space_map_length(msp->ms_sm),
2710 avl_numnodes(&msp->ms_allocatable->rt_root),
2711 msp->ms_condense_wanted ? "TRUE" : "FALSE");
2713 msp->ms_condense_wanted = B_FALSE;
2723 range_tree_add(condense_tree, msp->ms_start, msp->ms_size);
2725 range_tree_walk(msp->ms_freeing, range_tree_remove, condense_tree);
2726 range_tree_walk(msp->ms_freed, range_tree_remove, condense_tree);
2729 range_tree_walk(msp->ms_defer[t],
2734 range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK],
2747 msp->ms_condensing = B_TRUE;
2749 mutex_exit(&msp->ms_lock);
2766 space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx);
2767 mutex_enter(&msp->ms_lock);
2768 msp->ms_condensing = B_FALSE;
2775 metaslab_sync(metaslab_t *msp, uint64_t txg)
2777 metaslab_group_t *mg = msp->ms_group;
2781 range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK];
2783 uint64_t object = space_map_object(msp->ms_sm);
2790 if (msp->ms_freeing == NULL) {
2796 ASSERT3P(msp->ms_freeing, !=, NULL);
2797 ASSERT3P(msp->ms_freed, !=, NULL);
2798 ASSERT3P(msp->ms_checkpointing, !=, NULL);
2806 range_tree_is_empty(msp->ms_freeing) &&
2807 range_tree_is_empty(msp->ms_checkpointing) &&
2808 !(msp->ms_loaded && msp->ms_condense_wanted))
2829 if (msp->ms_sm == NULL) {
2835 VERIFY0(space_map_open(&msp->ms_sm, mos, new_object,
2836 msp->ms_start, msp->ms_size, vd->vdev_ashift));
2838 ASSERT(msp->ms_sm != NULL);
2839 ASSERT0(metaslab_allocated_space(msp));
2842 if (!range_tree_is_empty(msp->ms_checkpointing) &&
2864 mutex_enter(&msp->ms_sync_lock);
2865 mutex_enter(&msp->ms_lock);
2874 metaslab_group_histogram_remove(mg, msp);
2876 if (msp->ms_loaded && metaslab_should_condense(msp)) {
2877 metaslab_condense(msp, txg, tx);
2879 mutex_exit(&msp->ms_lock);
2880 space_map_write(msp->ms_sm, alloctree, SM_ALLOC,
2882 space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE,
2884 mutex_enter(&msp->ms_lock);
2887 msp->ms_allocated_space += range_tree_space(alloctree);
2888 ASSERT3U(msp->ms_allocated_space, >=,
2889 range_tree_space(msp->ms_freeing));
2890 msp->ms_allocated_space -= range_tree_space(msp->ms_freeing);
2892 if (!range_tree_is_empty(msp->ms_checkpointing)) {
2901 mutex_exit(&msp->ms_lock);
2903 msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx);
2904 mutex_enter(&msp->ms_lock);
2907 range_tree_space(msp->ms_checkpointing);
2909 range_tree_space(msp->ms_checkpointing);
2913 range_tree_vacate(msp->ms_checkpointing, NULL, NULL);
2916 if (msp->ms_loaded) {
2923 space_map_histogram_clear(msp->ms_sm);
2924 space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx);
2933 space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx);
2943 space_map_histogram_add(msp->ms_sm,
2944 msp->ms_defer[t], tx);
2955 space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx);
2956 metaslab_aux_histograms_update(msp);
2958 metaslab_group_histogram_add(mg, msp);
2969 range_tree_swap(&msp->ms_freeing, &msp->ms_freed);
2970 ASSERT0(msp->ms_allocated_this_txg);
2972 range_tree_vacate(msp->ms_freeing,
2973 range_tree_add, msp->ms_freed);
2975 msp->ms_allocated_this_txg += range_tree_space(alloctree);
2978 ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
2979 ASSERT0(range_tree_space(msp->ms_allocating[TXG_CLEAN(txg)
2981 ASSERT0(range_tree_space(msp->ms_freeing));
2982 ASSERT0(range_tree_space(msp->ms_checkpointing));
2984 mutex_exit(&msp->ms_lock);
2986 if (object != space_map_object(msp->ms_sm)) {
2987 object = space_map_object(msp->ms_sm);
2989 msp->ms_id, sizeof (uint64_t), &object, tx);
2991 mutex_exit(&msp->ms_sync_lock);
3000 metaslab_sync_done(metaslab_t *msp, uint64_t txg)
3002 metaslab_group_t *mg = msp->ms_group;
3011 mutex_enter(&msp->ms_lock);
3017 if (msp->ms_freed == NULL) {
3019 ASSERT(msp->ms_allocating[t] == NULL);
3021 msp->ms_allocating[t] = range_tree_create(NULL, NULL);
3024 ASSERT3P(msp->ms_freeing, ==, NULL);
3025 msp->ms_freeing = range_tree_create(NULL, NULL);
3027 ASSERT3P(msp->ms_freed, ==, NULL);
3028 msp->ms_freed = range_tree_create(NULL, NULL);
3031 ASSERT(msp->ms_defer[t] == NULL);
3033 msp->ms_defer[t] = range_tree_create(NULL, NULL);
3036 ASSERT3P(msp->ms_checkpointing, ==, NULL);
3037 msp->ms_checkpointing = range_tree_create(NULL, NULL);
3039 metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size);
3041 ASSERT0(range_tree_space(msp->ms_freeing));
3042 ASSERT0(range_tree_space(msp->ms_checkpointing));
3044 defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE];
3053 alloc_delta = msp->ms_allocated_this_txg -
3054 range_tree_space(msp->ms_freed);
3056 defer_delta = range_tree_space(msp->ms_freed) -
3069 metaslab_load_wait(msp);
3078 msp->ms_loaded ? range_tree_add : NULL, msp->ms_allocatable);
3080 range_tree_swap(&msp->ms_freed, defer_tree);
3082 range_tree_vacate(msp->ms_freed,
3083 msp->ms_loaded ? range_tree_add : NULL,
3084 msp->ms_allocatable);
3087 msp->ms_synced_length = space_map_length(msp->ms_sm);
3089 msp->ms_deferspace += defer_delta;
3090 ASSERT3S(msp->ms_deferspace, >=, 0);
3091 ASSERT3S(msp->ms_deferspace, <=, msp->ms_size);
3092 if (msp->ms_deferspace != 0) {
3097 vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
3099 metaslab_aux_histograms_update_done(msp, defer_allowed);
3101 if (msp->ms_new) {
3102 msp->ms_new = B_FALSE;
3112 metaslab_recalculate_weight_and_sort(msp);
3118 if (msp->ms_loaded &&
3119 msp->ms_initializing == 0 &&
3120 msp->ms_selected_txg + metaslab_unload_delay < txg) {
3123 msp->ms_allocating[(txg + t) & TXG_MASK]));
3125 if (msp->ms_allocator != -1) {
3126 metaslab_passivate(msp, msp->ms_weight &
3131 metaslab_unload(msp);
3134 ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
3135 ASSERT0(range_tree_space(msp->ms_freeing));
3136 ASSERT0(range_tree_space(msp->ms_freed));
3137 ASSERT0(range_tree_space(msp->ms_checkpointing));
3139 msp->ms_allocated_this_txg = 0;
3140 mutex_exit(&msp->ms_lock);
3171 metaslab_is_unique(metaslab_t *msp, dva_t *dva)
3178 if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva))
3181 dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift;
3183 return (msp->ms_id != dva_ms_id);
3228 metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset,
3257 mat->mat_msp = msp;
3264 if (msp != NULL)
3265 mat->mat_weight = msp->ms_weight;
3298 #define metaslab_trace_add(zal, mg, msp, psize, id, off, alloc)
3394 metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t txg)
3397 range_tree_t *rt = msp->ms_allocatable;
3398 metaslab_class_t *mc = msp->ms_group->mg_class;
3400 VERIFY(!msp->ms_condensing);
3401 VERIFY0(msp->ms_initializing);
3403 start = mc->mc_ops->msop_alloc(msp, size);
3405 metaslab_group_t *mg = msp->ms_group;
3410 VERIFY3U(range_tree_space(rt) - size, <=, msp->ms_size);
3413 if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
3414 vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg);
3416 range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, size);
3419 msp->ms_alloc_txg = txg;
3420 metaslab_verify_space(msp, txg);
3427 msp->ms_max_size = metaslab_block_maxsize(msp);
3449 metaslab_t *msp = avl_find(t, search, &idx);
3450 if (msp == NULL)
3451 msp = avl_nearest(t, idx, AVL_AFTER);
3453 for (; msp != NULL; msp = AVL_NEXT(t, msp)) {
3455 if (!metaslab_should_allocate(msp, asize)) {
3456 metaslab_trace_add(zal, mg, msp, asize, d,
3465 if (msp->ms_condensing || msp->ms_initializing > 0)
3468 *was_active = msp->ms_allocator != -1;
3481 !metaslab_is_unique(msp, &dva[i]))
3488 if (msp != NULL) {
3489 search->ms_weight = msp->ms_weight;
3490 search->ms_start = msp->ms_start + 1;
3491 search->ms_allocator = msp->ms_allocator;
3492 search->ms_primary = msp->ms_primary;
3494 return (msp);
3503 metaslab_t *msp = NULL;
3547 msp = mg->mg_primaries[allocator];
3551 msp = mg->mg_secondaries[allocator];
3554 msp = find_valid_metaslab(mg, activation_weight, dva, d,
3560 if (msp == NULL) {
3565 mutex_enter(&msp->ms_lock);
3574 if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) {
3575 mutex_exit(&msp->ms_lock);
3585 if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) &&
3586 (msp->ms_allocator != -1) &&
3587 (msp->ms_allocator != allocator || ((activation_weight ==
3588 METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) {
3589 mutex_exit(&msp->ms_lock);
3593 if (msp->ms_weight & METASLAB_WEIGHT_CLAIM &&
3595 metaslab_passivate(msp, msp->ms_weight &
3597 mutex_exit(&msp->ms_lock);
3601 if (metaslab_activate(msp, allocator, activation_weight) != 0) {
3602 mutex_exit(&msp->ms_lock);
3606 msp->ms_selected_txg = txg;
3615 if (!metaslab_should_allocate(msp, asize)) {
3617 metaslab_trace_add(zal, mg, msp, asize, d,
3629 if (msp->ms_condensing) {
3630 metaslab_trace_add(zal, mg, msp, asize, d,
3632 metaslab_passivate(msp, msp->ms_weight &
3634 mutex_exit(&msp->ms_lock);
3636 } else if (msp->ms_initializing > 0) {
3637 metaslab_trace_add(zal, mg, msp, asize, d,
3639 metaslab_passivate(msp, msp->ms_weight &
3641 mutex_exit(&msp->ms_lock);
3645 offset = metaslab_block_alloc(msp, asize, txg);
3646 metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator);
3650 metaslab_segment_may_passivate(msp);
3654 ASSERT(msp->ms_loaded);
3677 if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
3678 uint64_t weight = metaslab_block_maxsize(msp);
3680 metaslab_passivate(msp, weight);
3682 metaslab_passivate(msp,
3683 metaslab_weight_from_range_tree(msp));
3692 ASSERT(!metaslab_should_allocate(msp, asize));
3694 mutex_exit(&msp->ms_lock);
3696 mutex_exit(&msp->ms_lock);
3951 metaslab_t *msp;
3958 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
3960 VERIFY(!msp->ms_condensing);
3961 VERIFY3U(offset, >=, msp->ms_start);
3962 VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size);
3968 mutex_enter(&msp->ms_lock);
3969 if (range_tree_is_empty(msp->ms_freeing) &&
3970 range_tree_is_empty(msp->ms_checkpointing)) {
3971 vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa));
3976 range_tree_add(msp->ms_checkpointing, offset, asize);
3978 range_tree_add(msp->ms_freeing, offset, asize);
3980 mutex_exit(&msp->ms_lock);
4170 metaslab_t *msp;
4198 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
4200 mutex_enter(&msp->ms_lock);
4201 range_tree_remove(msp->ms_allocating[txg & TXG_MASK],
4204 VERIFY(!msp->ms_condensing);
4205 VERIFY3U(offset, >=, msp->ms_start);
4206 VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size);
4207 VERIFY3U(range_tree_space(msp->ms_allocatable) + size, <=,
4208 msp->ms_size);
4211 range_tree_add(msp->ms_allocatable, offset, size);
4212 mutex_exit(&msp->ms_lock);
4295 metaslab_t *msp;
4303 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
4305 mutex_enter(&msp->ms_lock);
4307 if ((txg != 0 && spa_writeable(spa)) || !msp->ms_loaded)
4308 error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM);
4317 !range_tree_contains(msp->ms_allocatable, offset, size))
4321 mutex_exit(&msp->ms_lock);
4325 VERIFY(!msp->ms_condensing);
4328 VERIFY3U(range_tree_space(msp->ms_allocatable) - size, <=,
4329 msp->ms_size);
4330 range_tree_remove(msp->ms_allocatable, offset, size);
4333 if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
4334 vdev_dirty(vd, VDD_METASLAB, msp, txg);
4335 range_tree_add(msp->ms_allocating[txg & TXG_MASK],
4339 mutex_exit(&msp->ms_lock);
4570 metaslab_t *msp;
4586 msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
4588 mutex_enter(&msp->ms_lock);
4589 if (msp->ms_loaded) {
4590 range_tree_verify_not_present(msp->ms_allocatable,
4594 range_tree_verify_not_present(msp->ms_freeing, offset, size);
4595 range_tree_verify_not_present(msp->ms_checkpointing, offset, size);
4596 range_tree_verify_not_present(msp->ms_freed, offset, size);
4598 range_tree_verify_not_present(msp->ms_defer[j], offset, size);
4599 mutex_exit(&msp->ms_lock);