1168404Spjd/* 2168404Spjd * CDDL HEADER START 3168404Spjd * 4168404Spjd * The contents of this file are subject to the terms of the 5168404Spjd * Common Development and Distribution License (the "License"). 6168404Spjd * You may not use this file except in compliance with the License. 7168404Spjd * 8168404Spjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9168404Spjd * or http://www.opensolaris.org/os/licensing. 10168404Spjd * See the License for the specific language governing permissions 11168404Spjd * and limitations under the License. 12168404Spjd * 13168404Spjd * When distributing Covered Code, include this CDDL HEADER in each 14168404Spjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15168404Spjd * If applicable, add the following below this CDDL HEADER, with the 16168404Spjd * fields enclosed by brackets "[]" replaced with your own identifying 17168404Spjd * information: Portions Copyright [yyyy] [name of copyright owner] 18168404Spjd * 19168404Spjd * CDDL HEADER END 20168404Spjd */ 21168404Spjd/* 22209962Smm * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23168404Spjd * Use is subject to license terms. 24168404Spjd */ 25240415Smm/* 26332547Smav * Copyright (c) 2012, 2017 by Delphix. All rights reserved. 27240415Smm */ 28168404Spjd 29168404Spjd#include <sys/zfs_context.h> 30168404Spjd#include <sys/spa.h> 31168404Spjd#include <sys/dmu.h> 32258717Savg#include <sys/dmu_tx.h> 33258717Savg#include <sys/dnode.h> 34258717Savg#include <sys/dsl_pool.h> 35168404Spjd#include <sys/zio.h> 36168404Spjd#include <sys/space_map.h> 37258717Savg#include <sys/refcount.h> 38258717Savg#include <sys/zfeature.h> 39168404Spjd 40273267SdelphijSYSCTL_DECL(_vfs_zfs); 41273267Sdelphij 42168404Spjd/* 43332547Smav * Note on space map block size: 44332547Smav * 45272504Sdelphij * The data for a given space map can be kept on blocks of any size. 46339104Smav * Larger blocks entail fewer I/O operations, but they also cause the 47339104Smav * DMU to keep more data in-core, and also to waste more I/O bandwidth 48272504Sdelphij * when only a few blocks have changed since the last transaction group. 49168404Spjd */ 50168404Spjd 51168404Spjd/* 52339104Smav * Enabled whenever we want to stress test the use of double-word 53339104Smav * space map entries. 54339104Smav */ 55339104Smavboolean_t zfs_force_some_double_word_sm_entries = B_FALSE; 56339104Smav 57339120Smav/* 58339120Smav * Override the default indirect block size of 128K, instead using 16K for 59339120Smav * spacemaps (2^14 bytes). This dramatically reduces write inflation since 60339120Smav * appending to a spacemap typically has to write one data block (4KB) and one 61339120Smav * or two indirect blocks (16K-32K, rather than 128K). 62339120Smav */ 63339120Smavint space_map_ibs = 14; 64339120Smav 65339120SmavSYSCTL_INT(_vfs_zfs, OID_AUTO, space_map_ibs, CTLFLAG_RWTUN, 66339120Smav &space_map_ibs, 0, "Space map indirect block shift"); 67339120Smav 68339104Smavboolean_t 69339104Smavsm_entry_is_debug(uint64_t e) 70339104Smav{ 71339104Smav return (SM_PREFIX_DECODE(e) == SM_DEBUG_PREFIX); 72339104Smav} 73339104Smav 74339104Smavboolean_t 75339104Smavsm_entry_is_single_word(uint64_t e) 76339104Smav{ 77339104Smav uint8_t prefix = SM_PREFIX_DECODE(e); 78339104Smav return (prefix != SM_DEBUG_PREFIX && prefix != SM2_PREFIX); 79339104Smav} 80339104Smav 81339104Smavboolean_t 82339104Smavsm_entry_is_double_word(uint64_t e) 83339104Smav{ 84339104Smav return (SM_PREFIX_DECODE(e) == SM2_PREFIX); 85339104Smav} 86339104Smav 87339104Smav/* 88332525Smav * Iterate through the space map, invoking the callback on each (non-debug) 89332525Smav * space map entry. 90168404Spjd */ 91168404Spjdint 92332525Smavspace_map_iterate(space_map_t *sm, sm_cb_t callback, void *arg) 93168404Spjd{ 94339104Smav uint64_t sm_len = space_map_length(sm); 95339104Smav ASSERT3U(sm->sm_blksz, !=, 0); 96339104Smav 97339104Smav dmu_prefetch(sm->sm_os, space_map_object(sm), 0, 0, sm_len, 98339104Smav ZIO_PRIORITY_SYNC_READ); 99339104Smav 100339104Smav uint64_t blksz = sm->sm_blksz; 101185029Spjd int error = 0; 102339104Smav for (uint64_t block_base = 0; block_base < sm_len && error == 0; 103339104Smav block_base += blksz) { 104339104Smav dmu_buf_t *db; 105339104Smav error = dmu_buf_hold(sm->sm_os, space_map_object(sm), 106339104Smav block_base, FTAG, &db, DMU_READ_PREFETCH); 107339104Smav if (error != 0) 108339104Smav return (error); 109168404Spjd 110339104Smav uint64_t *block_start = db->db_data; 111339104Smav uint64_t block_length = MIN(sm_len - block_base, blksz); 112339104Smav uint64_t *block_end = block_start + 113339104Smav (block_length / sizeof (uint64_t)); 114168404Spjd 115339104Smav VERIFY0(P2PHASE(block_length, sizeof (uint64_t))); 116339104Smav VERIFY3U(block_length, !=, 0); 117339104Smav ASSERT3U(blksz, ==, db->db_size); 118168404Spjd 119339104Smav for (uint64_t *block_cursor = block_start; 120339104Smav block_cursor < block_end && error == 0; block_cursor++) { 121339104Smav uint64_t e = *block_cursor; 122339104Smav 123339104Smav if (sm_entry_is_debug(e)) /* Skip debug entries */ 124339104Smav continue; 125339104Smav 126339104Smav uint64_t raw_offset, raw_run, vdev_id; 127339104Smav maptype_t type; 128339104Smav if (sm_entry_is_single_word(e)) { 129339104Smav type = SM_TYPE_DECODE(e); 130339104Smav vdev_id = SM_NO_VDEVID; 131339104Smav raw_offset = SM_OFFSET_DECODE(e); 132339104Smav raw_run = SM_RUN_DECODE(e); 133339104Smav } else { 134339104Smav /* it is a two-word entry */ 135339104Smav ASSERT(sm_entry_is_double_word(e)); 136339104Smav raw_run = SM2_RUN_DECODE(e); 137339104Smav vdev_id = SM2_VDEV_DECODE(e); 138339104Smav 139339104Smav /* move on to the second word */ 140339104Smav block_cursor++; 141339104Smav e = *block_cursor; 142339104Smav VERIFY3P(block_cursor, <=, block_end); 143339104Smav 144339104Smav type = SM2_TYPE_DECODE(e); 145339104Smav raw_offset = SM2_OFFSET_DECODE(e); 146339104Smav } 147339104Smav 148339104Smav uint64_t entry_offset = (raw_offset << sm->sm_shift) + 149339104Smav sm->sm_start; 150339104Smav uint64_t entry_run = raw_run << sm->sm_shift; 151339104Smav 152339104Smav VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift)); 153339104Smav VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift)); 154339104Smav ASSERT3U(entry_offset, >=, sm->sm_start); 155339104Smav ASSERT3U(entry_offset, <, sm->sm_start + sm->sm_size); 156339104Smav ASSERT3U(entry_run, <=, sm->sm_size); 157339104Smav ASSERT3U(entry_offset + entry_run, <=, 158339104Smav sm->sm_start + sm->sm_size); 159339104Smav 160339104Smav space_map_entry_t sme = { 161339104Smav .sme_type = type, 162339104Smav .sme_vdev = vdev_id, 163339104Smav .sme_offset = entry_offset, 164339104Smav .sme_run = entry_run 165339104Smav }; 166339104Smav error = callback(&sme, arg); 167339104Smav } 168339104Smav dmu_buf_rele(db, FTAG); 169258717Savg } 170339104Smav return (error); 171339104Smav} 172168404Spjd 173339104Smav/* 174339104Smav * Reads the entries from the last block of the space map into 175339104Smav * buf in reverse order. Populates nwords with number of words 176339104Smav * in the last block. 177339104Smav * 178339104Smav * Refer to block comment within space_map_incremental_destroy() 179339104Smav * to understand why this function is needed. 180339104Smav */ 181339104Smavstatic int 182339104Smavspace_map_reversed_last_block_entries(space_map_t *sm, uint64_t *buf, 183339104Smav uint64_t bufsz, uint64_t *nwords) 184339104Smav{ 185339104Smav int error = 0; 186339104Smav dmu_buf_t *db; 187168404Spjd 188339104Smav /* 189339104Smav * Find the offset of the last word in the space map and use 190339104Smav * that to read the last block of the space map with 191339104Smav * dmu_buf_hold(). 192339104Smav */ 193339104Smav uint64_t last_word_offset = 194339104Smav sm->sm_phys->smp_objsize - sizeof (uint64_t); 195339104Smav error = dmu_buf_hold(sm->sm_os, space_map_object(sm), last_word_offset, 196339104Smav FTAG, &db, DMU_READ_NO_PREFETCH); 197339104Smav if (error != 0) 198339104Smav return (error); 199168404Spjd 200339104Smav ASSERT3U(sm->sm_object, ==, db->db_object); 201339104Smav ASSERT3U(sm->sm_blksz, ==, db->db_size); 202339104Smav ASSERT3U(bufsz, >=, db->db_size); 203339104Smav ASSERT(nwords != NULL); 204168404Spjd 205339104Smav uint64_t *words = db->db_data; 206339104Smav *nwords = 207339104Smav (sm->sm_phys->smp_objsize - db->db_offset) / sizeof (uint64_t); 208168404Spjd 209339104Smav ASSERT3U(*nwords, <=, bufsz / sizeof (uint64_t)); 210168404Spjd 211339104Smav uint64_t n = *nwords; 212339104Smav uint64_t j = n - 1; 213339104Smav for (uint64_t i = 0; i < n; i++) { 214339104Smav uint64_t entry = words[i]; 215339104Smav if (sm_entry_is_double_word(entry)) { 216339104Smav /* 217339104Smav * Since we are populating the buffer backwards 218339104Smav * we have to be extra careful and add the two 219339104Smav * words of the double-word entry in the right 220339104Smav * order. 221339104Smav */ 222339104Smav ASSERT3U(j, >, 0); 223339104Smav buf[j - 1] = entry; 224258717Savg 225339104Smav i++; 226339104Smav ASSERT3U(i, <, n); 227339104Smav entry = words[i]; 228339104Smav buf[j] = entry; 229339104Smav j -= 2; 230339104Smav } else { 231339104Smav ASSERT(sm_entry_is_debug(entry) || 232339104Smav sm_entry_is_single_word(entry)); 233339104Smav buf[j] = entry; 234339104Smav j--; 235168404Spjd } 236168404Spjd } 237168404Spjd 238339104Smav /* 239339104Smav * Assert that we wrote backwards all the 240339104Smav * way to the beginning of the buffer. 241339104Smav */ 242339104Smav ASSERT3S(j, ==, -1); 243339104Smav 244339104Smav dmu_buf_rele(db, FTAG); 245332525Smav return (error); 246332525Smav} 247332525Smav 248332547Smav/* 249332547Smav * Note: This function performs destructive actions - specifically 250332547Smav * it deletes entries from the end of the space map. Thus, callers 251332547Smav * should ensure that they are holding the appropriate locks for 252332547Smav * the space map that they provide. 253332547Smav */ 254332547Smavint 255332547Smavspace_map_incremental_destroy(space_map_t *sm, sm_cb_t callback, void *arg, 256332547Smav dmu_tx_t *tx) 257332547Smav{ 258339104Smav uint64_t bufsz = MAX(sm->sm_blksz, SPA_MINBLOCKSIZE); 259339104Smav uint64_t *buf = zio_buf_alloc(bufsz); 260332547Smav 261332547Smav dmu_buf_will_dirty(sm->sm_dbuf, tx); 262332547Smav 263332547Smav /* 264339104Smav * Ideally we would want to iterate from the beginning of the 265339104Smav * space map to the end in incremental steps. The issue with this 266339104Smav * approach is that we don't have any field on-disk that points 267339104Smav * us where to start between each step. We could try zeroing out 268339104Smav * entries that we've destroyed, but this doesn't work either as 269339104Smav * an entry that is 0 is a valid one (ALLOC for range [0x0:0x200]). 270332547Smav * 271339104Smav * As a result, we destroy its entries incrementally starting from 272339104Smav * the end after applying the callback to each of them. 273332547Smav * 274339104Smav * The problem with this approach is that we cannot literally 275339104Smav * iterate through the words in the space map backwards as we 276339104Smav * can't distinguish two-word space map entries from their second 277339104Smav * word. Thus we do the following: 278339104Smav * 279339104Smav * 1] We get all the entries from the last block of the space map 280339104Smav * and put them into a buffer in reverse order. This way the 281339104Smav * last entry comes first in the buffer, the second to last is 282339104Smav * second, etc. 283339104Smav * 2] We iterate through the entries in the buffer and we apply 284339104Smav * the callback to each one. As we move from entry to entry we 285339104Smav * we decrease the size of the space map, deleting effectively 286339104Smav * each entry. 287339104Smav * 3] If there are no more entries in the space map or the callback 288339104Smav * returns a value other than 0, we stop iterating over the 289339104Smav * space map. If there are entries remaining and the callback 290339104Smav * returned 0, we go back to step [1]. 291332547Smav */ 292339104Smav int error = 0; 293339104Smav while (space_map_length(sm) > 0 && error == 0) { 294339104Smav uint64_t nwords = 0; 295339104Smav error = space_map_reversed_last_block_entries(sm, buf, bufsz, 296339104Smav &nwords); 297332547Smav if (error != 0) 298332547Smav break; 299332547Smav 300339104Smav ASSERT3U(nwords, <=, bufsz / sizeof (uint64_t)); 301332547Smav 302339104Smav for (uint64_t i = 0; i < nwords; i++) { 303339104Smav uint64_t e = buf[i]; 304332547Smav 305339104Smav if (sm_entry_is_debug(e)) { 306339104Smav sm->sm_phys->smp_objsize -= sizeof (uint64_t); 307339104Smav space_map_update(sm); 308339104Smav continue; 309339104Smav } 310339104Smav 311339104Smav int words = 1; 312339104Smav uint64_t raw_offset, raw_run, vdev_id; 313332547Smav maptype_t type; 314339104Smav if (sm_entry_is_single_word(e)) { 315339104Smav type = SM_TYPE_DECODE(e); 316339104Smav vdev_id = SM_NO_VDEVID; 317339104Smav raw_offset = SM_OFFSET_DECODE(e); 318339104Smav raw_run = SM_RUN_DECODE(e); 319339104Smav } else { 320339104Smav ASSERT(sm_entry_is_double_word(e)); 321339104Smav words = 2; 322332547Smav 323339104Smav raw_run = SM2_RUN_DECODE(e); 324339104Smav vdev_id = SM2_VDEV_DECODE(e); 325332547Smav 326339104Smav /* move to the second word */ 327339104Smav i++; 328339104Smav e = buf[i]; 329332547Smav 330339104Smav ASSERT3P(i, <=, nwords); 331339104Smav 332339104Smav type = SM2_TYPE_DECODE(e); 333339104Smav raw_offset = SM2_OFFSET_DECODE(e); 334332547Smav } 335332547Smav 336339104Smav uint64_t entry_offset = 337339104Smav (raw_offset << sm->sm_shift) + sm->sm_start; 338339104Smav uint64_t entry_run = raw_run << sm->sm_shift; 339332547Smav 340332547Smav VERIFY0(P2PHASE(entry_offset, 1ULL << sm->sm_shift)); 341339104Smav VERIFY0(P2PHASE(entry_run, 1ULL << sm->sm_shift)); 342332547Smav VERIFY3U(entry_offset, >=, sm->sm_start); 343339104Smav VERIFY3U(entry_offset, <, sm->sm_start + sm->sm_size); 344339104Smav VERIFY3U(entry_run, <=, sm->sm_size); 345339104Smav VERIFY3U(entry_offset + entry_run, <=, 346332547Smav sm->sm_start + sm->sm_size); 347332547Smav 348339104Smav space_map_entry_t sme = { 349339104Smav .sme_type = type, 350339104Smav .sme_vdev = vdev_id, 351339104Smav .sme_offset = entry_offset, 352339104Smav .sme_run = entry_run 353339104Smav }; 354339104Smav error = callback(&sme, arg); 355332547Smav if (error != 0) 356332547Smav break; 357332547Smav 358332547Smav if (type == SM_ALLOC) 359339104Smav sm->sm_phys->smp_alloc -= entry_run; 360332547Smav else 361339104Smav sm->sm_phys->smp_alloc += entry_run; 362339104Smav sm->sm_phys->smp_objsize -= words * sizeof (uint64_t); 363332547Smav space_map_update(sm); 364332547Smav } 365332547Smav } 366332547Smav 367339104Smav if (space_map_length(sm) == 0) { 368332547Smav ASSERT0(error); 369332547Smav ASSERT0(sm->sm_phys->smp_objsize); 370332547Smav ASSERT0(sm->sm_alloc); 371332547Smav } 372332547Smav 373339104Smav zio_buf_free(buf, bufsz); 374332547Smav return (error); 375332547Smav} 376332547Smav 377332525Smavtypedef struct space_map_load_arg { 378332525Smav space_map_t *smla_sm; 379332525Smav range_tree_t *smla_rt; 380332525Smav maptype_t smla_type; 381332525Smav} space_map_load_arg_t; 382332525Smav 383332525Smavstatic int 384339104Smavspace_map_load_callback(space_map_entry_t *sme, void *arg) 385332525Smav{ 386332525Smav space_map_load_arg_t *smla = arg; 387339104Smav if (sme->sme_type == smla->smla_type) { 388339104Smav VERIFY3U(range_tree_space(smla->smla_rt) + sme->sme_run, <=, 389332525Smav smla->smla_sm->sm_size); 390339104Smav range_tree_add(smla->smla_rt, sme->sme_offset, sme->sme_run); 391332525Smav } else { 392339104Smav range_tree_remove(smla->smla_rt, sme->sme_offset, sme->sme_run); 393332525Smav } 394332525Smav 395332525Smav return (0); 396332525Smav} 397332525Smav 398332525Smav/* 399332525Smav * Load the space map disk into the specified range tree. Segments of maptype 400332525Smav * are added to the range tree, other segment types are removed. 401332525Smav */ 402332525Smavint 403332525Smavspace_map_load(space_map_t *sm, range_tree_t *rt, maptype_t maptype) 404332525Smav{ 405332525Smav uint64_t space; 406332525Smav int err; 407332525Smav space_map_load_arg_t smla; 408332525Smav 409332525Smav VERIFY0(range_tree_space(rt)); 410332525Smav space = space_map_allocated(sm); 411332525Smav 412332525Smav if (maptype == SM_FREE) { 413332525Smav range_tree_add(rt, sm->sm_start, sm->sm_size); 414332525Smav space = sm->sm_size - space; 415332525Smav } 416332525Smav 417332525Smav smla.smla_rt = rt; 418332525Smav smla.smla_sm = sm; 419332525Smav smla.smla_type = maptype; 420332525Smav err = space_map_iterate(sm, space_map_load_callback, &smla); 421332525Smav 422332525Smav if (err == 0) { 423258717Savg VERIFY3U(range_tree_space(rt), ==, space); 424332525Smav } else { 425258717Savg range_tree_vacate(rt, NULL, NULL); 426332525Smav } 427185029Spjd 428332525Smav return (err); 429258717Savg} 430168404Spjd 431258717Savgvoid 432258717Savgspace_map_histogram_clear(space_map_t *sm) 433258717Savg{ 434258717Savg if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) 435258717Savg return; 436168404Spjd 437258717Savg bzero(sm->sm_phys->smp_histogram, sizeof (sm->sm_phys->smp_histogram)); 438258717Savg} 439168404Spjd 440258717Savgboolean_t 441258717Savgspace_map_histogram_verify(space_map_t *sm, range_tree_t *rt) 442258717Savg{ 443258717Savg /* 444258717Savg * Verify that the in-core range tree does not have any 445258717Savg * ranges smaller than our sm_shift size. 446258717Savg */ 447258717Savg for (int i = 0; i < sm->sm_shift; i++) { 448258717Savg if (rt->rt_histogram[i] != 0) 449258717Savg return (B_FALSE); 450258717Savg } 451258717Savg return (B_TRUE); 452168404Spjd} 453168404Spjd 454168404Spjdvoid 455258717Savgspace_map_histogram_add(space_map_t *sm, range_tree_t *rt, dmu_tx_t *tx) 456168404Spjd{ 457258717Savg int idx = 0; 458168404Spjd 459258717Savg ASSERT(dmu_tx_is_syncing(tx)); 460258717Savg VERIFY3U(space_map_object(sm), !=, 0); 461168404Spjd 462258717Savg if (sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) 463258717Savg return; 464168404Spjd 465258717Savg dmu_buf_will_dirty(sm->sm_dbuf, tx); 466168404Spjd 467258717Savg ASSERT(space_map_histogram_verify(sm, rt)); 468258717Savg /* 469258717Savg * Transfer the content of the range tree histogram to the space 470258717Savg * map histogram. The space map histogram contains 32 buckets ranging 471258717Savg * between 2^sm_shift to 2^(32+sm_shift-1). The range tree, 472258717Savg * however, can represent ranges from 2^0 to 2^63. Since the space 473258717Savg * map only cares about allocatable blocks (minimum of sm_shift) we 474258717Savg * can safely ignore all ranges in the range tree smaller than sm_shift. 475258717Savg */ 476258717Savg for (int i = sm->sm_shift; i < RANGE_TREE_HISTOGRAM_SIZE; i++) { 477258717Savg 478258717Savg /* 479258717Savg * Since the largest histogram bucket in the space map is 480258717Savg * 2^(32+sm_shift-1), we need to normalize the values in 481258717Savg * the range tree for any bucket larger than that size. For 482258717Savg * example given an sm_shift of 9, ranges larger than 2^40 483258717Savg * would get normalized as if they were 1TB ranges. Assume 484258717Savg * the range tree had a count of 5 in the 2^44 (16TB) bucket, 485258717Savg * the calculation below would normalize this to 5 * 2^4 (16). 486258717Savg */ 487258717Savg ASSERT3U(i, >=, idx + sm->sm_shift); 488258717Savg sm->sm_phys->smp_histogram[idx] += 489258717Savg rt->rt_histogram[i] << (i - idx - sm->sm_shift); 490258717Savg 491258717Savg /* 492258717Savg * Increment the space map's index as long as we haven't 493258717Savg * reached the maximum bucket size. Accumulate all ranges 494258717Savg * larger than the max bucket size into the last bucket. 495258717Savg */ 496269118Sdelphij if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) { 497258717Savg ASSERT3U(idx + sm->sm_shift, ==, i); 498258717Savg idx++; 499269118Sdelphij ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE); 500258717Savg } 501258717Savg } 502209962Smm} 503209962Smm 504339104Smavstatic void 505339104Smavspace_map_write_intro_debug(space_map_t *sm, maptype_t maptype, dmu_tx_t *tx) 506168404Spjd{ 507339104Smav dmu_buf_will_dirty(sm->sm_dbuf, tx); 508168404Spjd 509339104Smav uint64_t dentry = SM_PREFIX_ENCODE(SM_DEBUG_PREFIX) | 510339104Smav SM_DEBUG_ACTION_ENCODE(maptype) | 511339104Smav SM_DEBUG_SYNCPASS_ENCODE(spa_sync_pass(tx->tx_pool->dp_spa)) | 512339104Smav SM_DEBUG_TXG_ENCODE(dmu_tx_get_txg(tx)); 513339104Smav 514339104Smav dmu_write(sm->sm_os, space_map_object(sm), sm->sm_phys->smp_objsize, 515339104Smav sizeof (dentry), &dentry, tx); 516339104Smav 517339104Smav sm->sm_phys->smp_objsize += sizeof (dentry); 518339104Smav} 519339104Smav 520339104Smav/* 521339104Smav * Writes one or more entries given a segment. 522339104Smav * 523339104Smav * Note: The function may release the dbuf from the pointer initially 524339104Smav * passed to it, and return a different dbuf. Also, the space map's 525339104Smav * dbuf must be dirty for the changes in sm_phys to take effect. 526339104Smav */ 527339104Smavstatic void 528339104Smavspace_map_write_seg(space_map_t *sm, range_seg_t *rs, maptype_t maptype, 529339104Smav uint64_t vdev_id, uint8_t words, dmu_buf_t **dbp, void *tag, dmu_tx_t *tx) 530339104Smav{ 531339104Smav ASSERT3U(words, !=, 0); 532339104Smav ASSERT3U(words, <=, 2); 533339104Smav 534339104Smav /* ensure the vdev_id can be represented by the space map */ 535339104Smav ASSERT3U(vdev_id, <=, SM_NO_VDEVID); 536339104Smav 537258717Savg /* 538339104Smav * if this is a single word entry, ensure that no vdev was 539339104Smav * specified. 540258717Savg */ 541339104Smav IMPLY(words == 1, vdev_id == SM_NO_VDEVID); 542258717Savg 543339104Smav dmu_buf_t *db = *dbp; 544339104Smav ASSERT3U(db->db_size, ==, sm->sm_blksz); 545339104Smav 546339104Smav uint64_t *block_base = db->db_data; 547339104Smav uint64_t *block_end = block_base + (sm->sm_blksz / sizeof (uint64_t)); 548339104Smav uint64_t *block_cursor = block_base + 549339104Smav (sm->sm_phys->smp_objsize - db->db_offset) / sizeof (uint64_t); 550339104Smav 551339104Smav ASSERT3P(block_cursor, <=, block_end); 552339104Smav 553339104Smav uint64_t size = (rs->rs_end - rs->rs_start) >> sm->sm_shift; 554339104Smav uint64_t start = (rs->rs_start - sm->sm_start) >> sm->sm_shift; 555339104Smav uint64_t run_max = (words == 2) ? SM2_RUN_MAX : SM_RUN_MAX; 556339104Smav 557339104Smav ASSERT3U(rs->rs_start, >=, sm->sm_start); 558339104Smav ASSERT3U(rs->rs_start, <, sm->sm_start + sm->sm_size); 559339104Smav ASSERT3U(rs->rs_end - rs->rs_start, <=, sm->sm_size); 560339104Smav ASSERT3U(rs->rs_end, <=, sm->sm_start + sm->sm_size); 561339104Smav 562339104Smav while (size != 0) { 563339104Smav ASSERT3P(block_cursor, <=, block_end); 564339104Smav 565339104Smav /* 566339104Smav * If we are at the end of this block, flush it and start 567339104Smav * writing again from the beginning. 568339104Smav */ 569339104Smav if (block_cursor == block_end) { 570339104Smav dmu_buf_rele(db, tag); 571339104Smav 572339104Smav uint64_t next_word_offset = sm->sm_phys->smp_objsize; 573339104Smav VERIFY0(dmu_buf_hold(sm->sm_os, 574339104Smav space_map_object(sm), next_word_offset, 575339104Smav tag, &db, DMU_READ_PREFETCH)); 576339104Smav dmu_buf_will_dirty(db, tx); 577339104Smav 578339104Smav /* update caller's dbuf */ 579339104Smav *dbp = db; 580339104Smav 581339104Smav ASSERT3U(db->db_size, ==, sm->sm_blksz); 582339104Smav 583339104Smav block_base = db->db_data; 584339104Smav block_cursor = block_base; 585339104Smav block_end = block_base + 586339104Smav (db->db_size / sizeof (uint64_t)); 587339104Smav } 588339104Smav 589339104Smav /* 590339104Smav * If we are writing a two-word entry and we only have one 591339104Smav * word left on this block, just pad it with an empty debug 592339104Smav * entry and write the two-word entry in the next block. 593339104Smav */ 594339104Smav uint64_t *next_entry = block_cursor + 1; 595339104Smav if (next_entry == block_end && words > 1) { 596339104Smav ASSERT3U(words, ==, 2); 597339104Smav *block_cursor = SM_PREFIX_ENCODE(SM_DEBUG_PREFIX) | 598339104Smav SM_DEBUG_ACTION_ENCODE(0) | 599339104Smav SM_DEBUG_SYNCPASS_ENCODE(0) | 600339104Smav SM_DEBUG_TXG_ENCODE(0); 601339104Smav block_cursor++; 602339104Smav sm->sm_phys->smp_objsize += sizeof (uint64_t); 603339104Smav ASSERT3P(block_cursor, ==, block_end); 604339104Smav continue; 605339104Smav } 606339104Smav 607339104Smav uint64_t run_len = MIN(size, run_max); 608339104Smav switch (words) { 609339104Smav case 1: 610339104Smav *block_cursor = SM_OFFSET_ENCODE(start) | 611339104Smav SM_TYPE_ENCODE(maptype) | 612339104Smav SM_RUN_ENCODE(run_len); 613339104Smav block_cursor++; 614339104Smav break; 615339104Smav case 2: 616339104Smav /* write the first word of the entry */ 617339104Smav *block_cursor = SM_PREFIX_ENCODE(SM2_PREFIX) | 618339104Smav SM2_RUN_ENCODE(run_len) | 619339104Smav SM2_VDEV_ENCODE(vdev_id); 620339104Smav block_cursor++; 621339104Smav 622339104Smav /* move on to the second word of the entry */ 623339104Smav ASSERT3P(block_cursor, <, block_end); 624339104Smav *block_cursor = SM2_TYPE_ENCODE(maptype) | 625339104Smav SM2_OFFSET_ENCODE(start); 626339104Smav block_cursor++; 627339104Smav break; 628339104Smav default: 629339104Smav panic("%d-word space map entries are not supported", 630339104Smav words); 631339104Smav break; 632339104Smav } 633339104Smav sm->sm_phys->smp_objsize += words * sizeof (uint64_t); 634339104Smav 635339104Smav start += run_len; 636339104Smav size -= run_len; 637339104Smav } 638339104Smav ASSERT0(size); 639339104Smav 640339104Smav} 641339104Smav 642339104Smav/* 643339104Smav * Note: The space map's dbuf must be dirty for the changes in sm_phys to 644339104Smav * take effect. 645339104Smav */ 646339104Smavstatic void 647339104Smavspace_map_write_impl(space_map_t *sm, range_tree_t *rt, maptype_t maptype, 648339104Smav uint64_t vdev_id, dmu_tx_t *tx) 649339104Smav{ 650339104Smav spa_t *spa = tx->tx_pool->dp_spa; 651339104Smav dmu_buf_t *db; 652339104Smav 653339104Smav space_map_write_intro_debug(sm, maptype, tx); 654339104Smav 655339104Smav#ifdef DEBUG 656258717Savg /* 657339104Smav * We do this right after we write the intro debug entry 658339104Smav * because the estimate does not take it into account. 659258717Savg */ 660339104Smav uint64_t initial_objsize = sm->sm_phys->smp_objsize; 661339104Smav uint64_t estimated_growth = 662339104Smav space_map_estimate_optimal_size(sm, rt, SM_NO_VDEVID); 663339104Smav uint64_t estimated_final_objsize = initial_objsize + estimated_growth; 664339104Smav#endif 665339104Smav 666339104Smav /* 667339104Smav * Find the offset right after the last word in the space map 668339104Smav * and use that to get a hold of the last block, so we can 669339104Smav * start appending to it. 670339104Smav */ 671339104Smav uint64_t next_word_offset = sm->sm_phys->smp_objsize; 672339104Smav VERIFY0(dmu_buf_hold(sm->sm_os, space_map_object(sm), 673339104Smav next_word_offset, FTAG, &db, DMU_READ_PREFETCH)); 674339104Smav ASSERT3U(db->db_size, ==, sm->sm_blksz); 675339104Smav 676339104Smav dmu_buf_will_dirty(db, tx); 677339104Smav 678339104Smav avl_tree_t *t = &rt->rt_root; 679339104Smav for (range_seg_t *rs = avl_first(t); rs != NULL; rs = AVL_NEXT(t, rs)) { 680339104Smav uint64_t offset = (rs->rs_start - sm->sm_start) >> sm->sm_shift; 681339104Smav uint64_t length = (rs->rs_end - rs->rs_start) >> sm->sm_shift; 682339104Smav uint8_t words = 1; 683339104Smav 684339104Smav /* 685339104Smav * We only write two-word entries when both of the following 686339104Smav * are true: 687339104Smav * 688339104Smav * [1] The feature is enabled. 689339104Smav * [2] The offset or run is too big for a single-word entry, 690339120Smav * or the vdev_id is set (meaning not equal to 691339120Smav * SM_NO_VDEVID). 692339104Smav * 693339104Smav * Note that for purposes of testing we've added the case that 694339104Smav * we write two-word entries occasionally when the feature is 695339104Smav * enabled and zfs_force_some_double_word_sm_entries has been 696339104Smav * set. 697339104Smav */ 698339104Smav if (spa_feature_is_active(spa, SPA_FEATURE_SPACEMAP_V2) && 699339104Smav (offset >= (1ULL << SM_OFFSET_BITS) || 700339104Smav length > SM_RUN_MAX || 701339104Smav vdev_id != SM_NO_VDEVID || 702339104Smav (zfs_force_some_double_word_sm_entries && 703339104Smav spa_get_random(100) == 0))) 704339104Smav words = 2; 705339104Smav 706339104Smav space_map_write_seg(sm, rs, maptype, vdev_id, words, 707339104Smav &db, FTAG, tx); 708258717Savg } 709339104Smav 710339104Smav dmu_buf_rele(db, FTAG); 711339104Smav 712339104Smav#ifdef DEBUG 713339104Smav /* 714339104Smav * We expect our estimation to be based on the worst case 715339104Smav * scenario [see comment in space_map_estimate_optimal_size()]. 716339104Smav * Therefore we expect the actual objsize to be equal or less 717339104Smav * than whatever we estimated it to be. 718339104Smav */ 719339104Smav ASSERT3U(estimated_final_objsize, >=, sm->sm_phys->smp_objsize); 720339104Smav#endif 721168404Spjd} 722168404Spjd 723339104Smav/* 724339104Smav * Note: This function manipulates the state of the given space map but 725339104Smav * does not hold any locks implicitly. Thus the caller is responsible 726339104Smav * for synchronizing writes to the space map. 727339104Smav */ 728168404Spjdvoid 729258717Savgspace_map_write(space_map_t *sm, range_tree_t *rt, maptype_t maptype, 730339104Smav uint64_t vdev_id, dmu_tx_t *tx) 731168404Spjd{ 732258717Savg objset_t *os = sm->sm_os; 733168404Spjd 734258717Savg ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 735258717Savg VERIFY3U(space_map_object(sm), !=, 0); 736339104Smav 737258717Savg dmu_buf_will_dirty(sm->sm_dbuf, tx); 738168404Spjd 739258717Savg /* 740258717Savg * This field is no longer necessary since the in-core space map 741258717Savg * now contains the object number but is maintained for backwards 742258717Savg * compatibility. 743258717Savg */ 744258717Savg sm->sm_phys->smp_object = sm->sm_object; 745258717Savg 746332547Smav if (range_tree_is_empty(rt)) { 747258717Savg VERIFY3U(sm->sm_object, ==, sm->sm_phys->smp_object); 748168404Spjd return; 749258717Savg } 750168404Spjd 751168404Spjd if (maptype == SM_ALLOC) 752258717Savg sm->sm_phys->smp_alloc += range_tree_space(rt); 753168404Spjd else 754258717Savg sm->sm_phys->smp_alloc -= range_tree_space(rt); 755168404Spjd 756339104Smav uint64_t nodes = avl_numnodes(&rt->rt_root); 757339104Smav uint64_t rt_space = range_tree_space(rt); 758258717Savg 759339104Smav space_map_write_impl(sm, rt, maptype, vdev_id, tx); 760168404Spjd 761243503Smm /* 762243503Smm * Ensure that the space_map's accounting wasn't changed 763243503Smm * while we were in the middle of writing it out. 764243503Smm */ 765258717Savg VERIFY3U(nodes, ==, avl_numnodes(&rt->rt_root)); 766258717Savg VERIFY3U(range_tree_space(rt), ==, rt_space); 767168404Spjd} 768168404Spjd 769258717Savgstatic int 770258717Savgspace_map_open_impl(space_map_t *sm) 771168404Spjd{ 772258717Savg int error; 773258717Savg u_longlong_t blocks; 774168404Spjd 775258717Savg error = dmu_bonus_hold(sm->sm_os, sm->sm_object, sm, &sm->sm_dbuf); 776258717Savg if (error) 777258717Savg return (error); 778258717Savg 779258717Savg dmu_object_size_from_db(sm->sm_dbuf, &sm->sm_blksz, &blocks); 780258717Savg sm->sm_phys = sm->sm_dbuf->db_data; 781258717Savg return (0); 782168404Spjd} 783209962Smm 784258717Savgint 785258717Savgspace_map_open(space_map_t **smp, objset_t *os, uint64_t object, 786332525Smav uint64_t start, uint64_t size, uint8_t shift) 787209962Smm{ 788258717Savg space_map_t *sm; 789258717Savg int error; 790209962Smm 791258717Savg ASSERT(*smp == NULL); 792258717Savg ASSERT(os != NULL); 793258717Savg ASSERT(object != 0); 794209962Smm 795258717Savg sm = kmem_zalloc(sizeof (space_map_t), KM_SLEEP); 796209962Smm 797258717Savg sm->sm_start = start; 798258717Savg sm->sm_size = size; 799258717Savg sm->sm_shift = shift; 800258717Savg sm->sm_os = os; 801258717Savg sm->sm_object = object; 802258717Savg 803258717Savg error = space_map_open_impl(sm); 804258717Savg if (error != 0) { 805258717Savg space_map_close(sm); 806258717Savg return (error); 807258717Savg } 808258717Savg *smp = sm; 809258717Savg 810209962Smm return (0); 811209962Smm} 812209962Smm 813209962Smmvoid 814258717Savgspace_map_close(space_map_t *sm) 815209962Smm{ 816258717Savg if (sm == NULL) 817258717Savg return; 818209962Smm 819258717Savg if (sm->sm_dbuf != NULL) 820258717Savg dmu_buf_rele(sm->sm_dbuf, sm); 821258717Savg sm->sm_dbuf = NULL; 822258717Savg sm->sm_phys = NULL; 823209962Smm 824258717Savg kmem_free(sm, sizeof (*sm)); 825209962Smm} 826209962Smm 827209962Smmvoid 828332547Smavspace_map_truncate(space_map_t *sm, int blocksize, dmu_tx_t *tx) 829209962Smm{ 830258717Savg objset_t *os = sm->sm_os; 831258717Savg spa_t *spa = dmu_objset_spa(os); 832258717Savg dmu_object_info_t doi; 833258717Savg 834258717Savg ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 835258717Savg ASSERT(dmu_tx_is_syncing(tx)); 836321554Smav VERIFY3U(dmu_tx_get_txg(tx), <=, spa_final_dirty_txg(spa)); 837258717Savg 838258717Savg dmu_object_info_from_db(sm->sm_dbuf, &doi); 839258717Savg 840272504Sdelphij /* 841272504Sdelphij * If the space map has the wrong bonus size (because 842272504Sdelphij * SPA_FEATURE_SPACEMAP_HISTOGRAM has recently been enabled), or 843272504Sdelphij * the wrong block size (because space_map_blksz has changed), 844272504Sdelphij * free and re-allocate its object with the updated sizes. 845272504Sdelphij * 846272504Sdelphij * Otherwise, just truncate the current object. 847272504Sdelphij */ 848272504Sdelphij if ((spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) && 849272504Sdelphij doi.doi_bonus_size != sizeof (space_map_phys_t)) || 850339120Smav doi.doi_data_block_size != blocksize || 851339120Smav doi.doi_metadata_block_size != 1 << space_map_ibs) { 852321554Smav zfs_dbgmsg("txg %llu, spa %s, sm %p, reallocating " 853321554Smav "object[%llu]: old bonus %u, old blocksz %u", 854321554Smav dmu_tx_get_txg(tx), spa_name(spa), sm, sm->sm_object, 855321554Smav doi.doi_bonus_size, doi.doi_data_block_size); 856272504Sdelphij 857272504Sdelphij space_map_free(sm, tx); 858272504Sdelphij dmu_buf_rele(sm->sm_dbuf, sm); 859272504Sdelphij 860332547Smav sm->sm_object = space_map_alloc(sm->sm_os, blocksize, tx); 861272504Sdelphij VERIFY0(space_map_open_impl(sm)); 862272504Sdelphij } else { 863272504Sdelphij VERIFY0(dmu_free_range(os, space_map_object(sm), 0, -1ULL, tx)); 864272504Sdelphij 865272504Sdelphij /* 866272504Sdelphij * If the spacemap is reallocated, its histogram 867272504Sdelphij * will be reset. Do the same in the common case so that 868272504Sdelphij * bugs related to the uncommon case do not go unnoticed. 869272504Sdelphij */ 870272504Sdelphij bzero(sm->sm_phys->smp_histogram, 871272504Sdelphij sizeof (sm->sm_phys->smp_histogram)); 872258717Savg } 873258717Savg 874258717Savg dmu_buf_will_dirty(sm->sm_dbuf, tx); 875258717Savg sm->sm_phys->smp_objsize = 0; 876258717Savg sm->sm_phys->smp_alloc = 0; 877209962Smm} 878209962Smm 879209962Smm/* 880258717Savg * Update the in-core space_map allocation and length values. 881209962Smm */ 882209962Smmvoid 883258717Savgspace_map_update(space_map_t *sm) 884209962Smm{ 885258717Savg if (sm == NULL) 886258717Savg return; 887209962Smm 888258717Savg sm->sm_alloc = sm->sm_phys->smp_alloc; 889258717Savg sm->sm_length = sm->sm_phys->smp_objsize; 890209962Smm} 891209962Smm 892258717Savguint64_t 893332547Smavspace_map_alloc(objset_t *os, int blocksize, dmu_tx_t *tx) 894258717Savg{ 895258717Savg spa_t *spa = dmu_objset_spa(os); 896258717Savg uint64_t object; 897258717Savg int bonuslen; 898258717Savg 899259813Sdelphij if (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) { 900259813Sdelphij spa_feature_incr(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM, tx); 901258717Savg bonuslen = sizeof (space_map_phys_t); 902258717Savg ASSERT3U(bonuslen, <=, dmu_bonus_max()); 903258717Savg } else { 904258717Savg bonuslen = SPACE_MAP_SIZE_V0; 905258717Savg } 906258717Savg 907339120Smav object = dmu_object_alloc_ibs(os, DMU_OT_SPACE_MAP, blocksize, 908339120Smav space_map_ibs, DMU_OT_SPACE_MAP_HEADER, bonuslen, tx); 909258717Savg 910258717Savg return (object); 911258717Savg} 912258717Savg 913209962Smmvoid 914332525Smavspace_map_free_obj(objset_t *os, uint64_t smobj, dmu_tx_t *tx) 915209962Smm{ 916332525Smav spa_t *spa = dmu_objset_spa(os); 917259813Sdelphij if (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM)) { 918258717Savg dmu_object_info_t doi; 919209962Smm 920332525Smav VERIFY0(dmu_object_info(os, smobj, &doi)); 921258717Savg if (doi.doi_bonus_size != SPACE_MAP_SIZE_V0) { 922259813Sdelphij spa_feature_decr(spa, 923259813Sdelphij SPA_FEATURE_SPACEMAP_HISTOGRAM, tx); 924209962Smm } 925209962Smm } 926258717Savg 927332525Smav VERIFY0(dmu_object_free(os, smobj, tx)); 928332525Smav} 929332525Smav 930332525Smavvoid 931332525Smavspace_map_free(space_map_t *sm, dmu_tx_t *tx) 932332525Smav{ 933332525Smav if (sm == NULL) 934332525Smav return; 935332525Smav 936332525Smav space_map_free_obj(sm->sm_os, space_map_object(sm), tx); 937258717Savg sm->sm_object = 0; 938209962Smm} 939258717Savg 940339104Smav/* 941339104Smav * Given a range tree, it makes a worst-case estimate of how much 942339104Smav * space would the tree's segments take if they were written to 943339104Smav * the given space map. 944339104Smav */ 945258717Savguint64_t 946339104Smavspace_map_estimate_optimal_size(space_map_t *sm, range_tree_t *rt, 947339104Smav uint64_t vdev_id) 948339104Smav{ 949339104Smav spa_t *spa = dmu_objset_spa(sm->sm_os); 950339104Smav uint64_t shift = sm->sm_shift; 951339104Smav uint64_t *histogram = rt->rt_histogram; 952339104Smav uint64_t entries_for_seg = 0; 953339104Smav 954339104Smav /* 955339104Smav * In order to get a quick estimate of the optimal size that this 956339104Smav * range tree would have on-disk as a space map, we iterate through 957339104Smav * its histogram buckets instead of iterating through its nodes. 958339104Smav * 959339104Smav * Note that this is a highest-bound/worst-case estimate for the 960339104Smav * following reasons: 961339104Smav * 962339104Smav * 1] We assume that we always add a debug padding for each block 963339104Smav * we write and we also assume that we start at the last word 964339104Smav * of a block attempting to write a two-word entry. 965339104Smav * 2] Rounding up errors due to the way segments are distributed 966339104Smav * in the buckets of the range tree's histogram. 967339104Smav * 3] The activation of zfs_force_some_double_word_sm_entries 968339104Smav * (tunable) when testing. 969339104Smav * 970339104Smav * = Math and Rounding Errors = 971339104Smav * 972339104Smav * rt_histogram[i] bucket of a range tree represents the number 973339104Smav * of entries in [2^i, (2^(i+1))-1] of that range_tree. Given 974339104Smav * that, we want to divide the buckets into groups: Buckets that 975339104Smav * can be represented using a single-word entry, ones that can 976339104Smav * be represented with a double-word entry, and ones that can 977339104Smav * only be represented with multiple two-word entries. 978339104Smav * 979339104Smav * [Note that if the new encoding feature is not enabled there 980339104Smav * are only two groups: single-word entry buckets and multiple 981339104Smav * single-word entry buckets. The information below assumes 982339104Smav * two-word entries enabled, but it can easily applied when 983339104Smav * the feature is not enabled] 984339104Smav * 985339104Smav * To find the highest bucket that can be represented with a 986339104Smav * single-word entry we look at the maximum run that such entry 987339104Smav * can have, which is 2^(SM_RUN_BITS + sm_shift) [remember that 988339104Smav * the run of a space map entry is shifted by sm_shift, thus we 989339104Smav * add it to the exponent]. This way, excluding the value of the 990339104Smav * maximum run that can be represented by a single-word entry, 991339104Smav * all runs that are smaller exist in buckets 0 to 992339104Smav * SM_RUN_BITS + shift - 1. 993339104Smav * 994339104Smav * To find the highest bucket that can be represented with a 995339104Smav * double-word entry, we follow the same approach. Finally, any 996339104Smav * bucket higher than that are represented with multiple two-word 997339104Smav * entries. To be more specific, if the highest bucket whose 998339104Smav * segments can be represented with a single two-word entry is X, 999339104Smav * then bucket X+1 will need 2 two-word entries for each of its 1000339104Smav * segments, X+2 will need 4, X+3 will need 8, ...etc. 1001339104Smav * 1002339104Smav * With all of the above we make our estimation based on bucket 1003339104Smav * groups. There is a rounding error though. As we mentioned in 1004339104Smav * the example with the one-word entry, the maximum run that can 1005339104Smav * be represented in a one-word entry 2^(SM_RUN_BITS + shift) is 1006339104Smav * not part of bucket SM_RUN_BITS + shift - 1. Thus, segments of 1007339104Smav * that length fall into the next bucket (and bucket group) where 1008339104Smav * we start counting two-word entries and this is one more reason 1009339104Smav * why the estimated size may end up being bigger than the actual 1010339104Smav * size written. 1011339104Smav */ 1012339104Smav uint64_t size = 0; 1013339104Smav uint64_t idx = 0; 1014339104Smav 1015339104Smav if (!spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2) || 1016339104Smav (vdev_id == SM_NO_VDEVID && sm->sm_size < SM_OFFSET_MAX)) { 1017339104Smav 1018339104Smav /* 1019339104Smav * If we are trying to force some double word entries just 1020339104Smav * assume the worst-case of every single word entry being 1021339104Smav * written as a double word entry. 1022339104Smav */ 1023339104Smav uint64_t entry_size = 1024339104Smav (spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2) && 1025339104Smav zfs_force_some_double_word_sm_entries) ? 1026339104Smav (2 * sizeof (uint64_t)) : sizeof (uint64_t); 1027339104Smav 1028339104Smav uint64_t single_entry_max_bucket = SM_RUN_BITS + shift - 1; 1029339104Smav for (; idx <= single_entry_max_bucket; idx++) 1030339104Smav size += histogram[idx] * entry_size; 1031339104Smav 1032339104Smav if (!spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2)) { 1033339104Smav for (; idx < RANGE_TREE_HISTOGRAM_SIZE; idx++) { 1034339104Smav ASSERT3U(idx, >=, single_entry_max_bucket); 1035339104Smav entries_for_seg = 1036339104Smav 1ULL << (idx - single_entry_max_bucket); 1037339104Smav size += histogram[idx] * 1038339104Smav entries_for_seg * entry_size; 1039339104Smav } 1040339104Smav return (size); 1041339104Smav } 1042339104Smav } 1043339104Smav 1044339104Smav ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_V2)); 1045339104Smav 1046339104Smav uint64_t double_entry_max_bucket = SM2_RUN_BITS + shift - 1; 1047339104Smav for (; idx <= double_entry_max_bucket; idx++) 1048339104Smav size += histogram[idx] * 2 * sizeof (uint64_t); 1049339104Smav 1050339104Smav for (; idx < RANGE_TREE_HISTOGRAM_SIZE; idx++) { 1051339104Smav ASSERT3U(idx, >=, double_entry_max_bucket); 1052339104Smav entries_for_seg = 1ULL << (idx - double_entry_max_bucket); 1053339104Smav size += histogram[idx] * 1054339104Smav entries_for_seg * 2 * sizeof (uint64_t); 1055339104Smav } 1056339104Smav 1057339104Smav /* 1058339104Smav * Assume the worst case where we start with the padding at the end 1059339104Smav * of the current block and we add an extra padding entry at the end 1060339104Smav * of all subsequent blocks. 1061339104Smav */ 1062339104Smav size += ((size / sm->sm_blksz) + 1) * sizeof (uint64_t); 1063339104Smav 1064339104Smav return (size); 1065339104Smav} 1066339104Smav 1067339104Smavuint64_t 1068258717Savgspace_map_object(space_map_t *sm) 1069258717Savg{ 1070258717Savg return (sm != NULL ? sm->sm_object : 0); 1071258717Savg} 1072258717Savg 1073258717Savg/* 1074258717Savg * Returns the already synced, on-disk allocated space. 1075258717Savg */ 1076258717Savguint64_t 1077258717Savgspace_map_allocated(space_map_t *sm) 1078258717Savg{ 1079258717Savg return (sm != NULL ? sm->sm_alloc : 0); 1080258717Savg} 1081258717Savg 1082258717Savg/* 1083258717Savg * Returns the already synced, on-disk length; 1084258717Savg */ 1085258717Savguint64_t 1086258717Savgspace_map_length(space_map_t *sm) 1087258717Savg{ 1088258717Savg return (sm != NULL ? sm->sm_length : 0); 1089258717Savg} 1090258717Savg 1091258717Savg/* 1092258717Savg * Returns the allocated space that is currently syncing. 1093258717Savg */ 1094258717Savgint64_t 1095258717Savgspace_map_alloc_delta(space_map_t *sm) 1096258717Savg{ 1097258717Savg if (sm == NULL) 1098258717Savg return (0); 1099258717Savg ASSERT(sm->sm_dbuf != NULL); 1100258717Savg return (sm->sm_phys->smp_alloc - space_map_allocated(sm)); 1101258717Savg} 1102