1219089Spjd/* 2219089Spjd * CDDL HEADER START 3219089Spjd * 4219089Spjd * The contents of this file are subject to the terms of the 5219089Spjd * Common Development and Distribution License (the "License"). 6219089Spjd * You may not use this file except in compliance with the License. 7219089Spjd * 8219089Spjd * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9219089Spjd * or http://www.opensolaris.org/os/licensing. 10219089Spjd * See the License for the specific language governing permissions 11219089Spjd * and limitations under the License. 12219089Spjd * 13219089Spjd * When distributing Covered Code, include this CDDL HEADER in each 14219089Spjd * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15219089Spjd * If applicable, add the following below this CDDL HEADER, with the 16219089Spjd * fields enclosed by brackets "[]" replaced with your own identifying 17219089Spjd * information: Portions Copyright [yyyy] [name of copyright owner] 18219089Spjd * 19219089Spjd * CDDL HEADER END 20219089Spjd */ 21219089Spjd/* 22219089Spjd * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23268657Sdelphij * Copyright (c) 2011, 2014 by Delphix. All rights reserved. 24219089Spjd */ 25219089Spjd 26219089Spjd#include <sys/dsl_scan.h> 27219089Spjd#include <sys/dsl_pool.h> 28219089Spjd#include <sys/dsl_dataset.h> 29219089Spjd#include <sys/dsl_prop.h> 30219089Spjd#include <sys/dsl_dir.h> 31219089Spjd#include <sys/dsl_synctask.h> 32219089Spjd#include <sys/dnode.h> 33219089Spjd#include <sys/dmu_tx.h> 34219089Spjd#include <sys/dmu_objset.h> 35219089Spjd#include <sys/arc.h> 36219089Spjd#include <sys/zap.h> 37219089Spjd#include <sys/zio.h> 38219089Spjd#include <sys/zfs_context.h> 39219089Spjd#include <sys/fs/zfs.h> 40219089Spjd#include <sys/zfs_znode.h> 41219089Spjd#include <sys/spa_impl.h> 42219089Spjd#include <sys/vdev_impl.h> 43219089Spjd#include <sys/zil_impl.h> 44219089Spjd#include <sys/zio_checksum.h> 45219089Spjd#include <sys/ddt.h> 46219089Spjd#include <sys/sa.h> 47219089Spjd#include <sys/sa_impl.h> 48236884Smm#include <sys/zfeature.h> 49219089Spjd#ifdef _KERNEL 50219089Spjd#include <sys/zfs_vfsops.h> 51219089Spjd#endif 52219089Spjd 53268657Sdelphijtypedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, 54268657Sdelphij const zbookmark_phys_t *); 55219089Spjd 56219089Spjdstatic scan_cb_t dsl_scan_scrub_cb; 57248571Smmstatic void dsl_scan_cancel_sync(void *, dmu_tx_t *); 58219089Spjdstatic void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *tx); 59219089Spjd 60237972Smmunsigned int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ 61237972Smmunsigned int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ 62237972Smmunsigned int zfs_scrub_delay = 4; /* number of ticks to delay scrub */ 63237972Smmunsigned int zfs_scan_idle = 50; /* idle window in clock ticks */ 64219089Spjd 65237972Smmunsigned int zfs_scan_min_time_ms = 1000; /* min millisecs to scrub per txg */ 66237972Smmunsigned int zfs_free_min_time_ms = 1000; /* min millisecs to free per txg */ 67237972Smmunsigned int zfs_resilver_min_time_ms = 3000; /* min millisecs to resilver 68237972Smm per txg */ 69219089Spjdboolean_t zfs_no_scrub_io = B_FALSE; /* set to disable scrub i/o */ 70268650Sdelphijboolean_t zfs_no_scrub_prefetch = B_FALSE; /* set to disable scrub prefetch */ 71237972Smm 72237972SmmSYSCTL_DECL(_vfs_zfs); 73237972SmmTUNABLE_INT("vfs.zfs.top_maxinflight", &zfs_top_maxinflight); 74237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, top_maxinflight, CTLFLAG_RW, 75237972Smm &zfs_top_maxinflight, 0, "Maximum I/Os per top-level vdev"); 76237972SmmTUNABLE_INT("vfs.zfs.resilver_delay", &zfs_resilver_delay); 77237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, resilver_delay, CTLFLAG_RW, 78237972Smm &zfs_resilver_delay, 0, "Number of ticks to delay resilver"); 79237972SmmTUNABLE_INT("vfs.zfs.scrub_delay", &zfs_scrub_delay); 80237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, scrub_delay, CTLFLAG_RW, 81237972Smm &zfs_scrub_delay, 0, "Number of ticks to delay scrub"); 82237972SmmTUNABLE_INT("vfs.zfs.scan_idle", &zfs_scan_idle); 83237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, scan_idle, CTLFLAG_RW, 84237972Smm &zfs_scan_idle, 0, "Idle scan window in clock ticks"); 85237972SmmTUNABLE_INT("vfs.zfs.scan_min_time_ms", &zfs_scan_min_time_ms); 86237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, scan_min_time_ms, CTLFLAG_RW, 87237972Smm &zfs_scan_min_time_ms, 0, "Min millisecs to scrub per txg"); 88237972SmmTUNABLE_INT("vfs.zfs.free_min_time_ms", &zfs_free_min_time_ms); 89237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, free_min_time_ms, CTLFLAG_RW, 90237972Smm &zfs_free_min_time_ms, 0, "Min millisecs to free per txg"); 91237972SmmTUNABLE_INT("vfs.zfs.resilver_min_time_ms", &zfs_resilver_min_time_ms); 92237972SmmSYSCTL_UINT(_vfs_zfs, OID_AUTO, resilver_min_time_ms, CTLFLAG_RW, 93237972Smm &zfs_resilver_min_time_ms, 0, "Min millisecs to resilver per txg"); 94237972SmmTUNABLE_INT("vfs.zfs.no_scrub_io", &zfs_no_scrub_io); 95237972SmmSYSCTL_INT(_vfs_zfs, OID_AUTO, no_scrub_io, CTLFLAG_RW, 96237972Smm &zfs_no_scrub_io, 0, "Disable scrub I/O"); 97237972SmmTUNABLE_INT("vfs.zfs.no_scrub_prefetch", &zfs_no_scrub_prefetch); 98237972SmmSYSCTL_INT(_vfs_zfs, OID_AUTO, no_scrub_prefetch, CTLFLAG_RW, 99237972Smm &zfs_no_scrub_prefetch, 0, "Disable scrub prefetching"); 100237972Smm 101219089Spjdenum ddt_class zfs_scrub_ddt_class_max = DDT_CLASS_DUPLICATE; 102219089Spjd 103219089Spjd#define DSL_SCAN_IS_SCRUB_RESILVER(scn) \ 104219089Spjd ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \ 105219089Spjd (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER) 106219089Spjd 107219089Spjdextern int zfs_txg_timeout; 108219089Spjd 109219089Spjd/* the order has to match pool_scan_type */ 110219089Spjdstatic scan_cb_t *scan_funcs[POOL_SCAN_FUNCS] = { 111219089Spjd NULL, 112219089Spjd dsl_scan_scrub_cb, /* POOL_SCAN_SCRUB */ 113219089Spjd dsl_scan_scrub_cb, /* POOL_SCAN_RESILVER */ 114219089Spjd}; 115219089Spjd 116219089Spjdint 117219089Spjddsl_scan_init(dsl_pool_t *dp, uint64_t txg) 118219089Spjd{ 119219089Spjd int err; 120219089Spjd dsl_scan_t *scn; 121219089Spjd spa_t *spa = dp->dp_spa; 122219089Spjd uint64_t f; 123219089Spjd 124219089Spjd scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP); 125219089Spjd scn->scn_dp = dp; 126219089Spjd 127249858Smm /* 128249858Smm * It's possible that we're resuming a scan after a reboot so 129249858Smm * make sure that the scan_async_destroying flag is initialized 130249858Smm * appropriately. 131249858Smm */ 132249858Smm ASSERT(!scn->scn_async_destroying); 133249858Smm scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa, 134263390Sdelphij SPA_FEATURE_ASYNC_DESTROY); 135249858Smm 136219089Spjd err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 137219089Spjd "scrub_func", sizeof (uint64_t), 1, &f); 138219089Spjd if (err == 0) { 139219089Spjd /* 140219089Spjd * There was an old-style scrub in progress. Restart a 141219089Spjd * new-style scrub from the beginning. 142219089Spjd */ 143219089Spjd scn->scn_restart_txg = txg; 144219089Spjd zfs_dbgmsg("old-style scrub was in progress; " 145219089Spjd "restarting new-style scrub in txg %llu", 146219089Spjd scn->scn_restart_txg); 147219089Spjd 148219089Spjd /* 149219089Spjd * Load the queue obj from the old location so that it 150219089Spjd * can be freed by dsl_scan_done(). 151219089Spjd */ 152219089Spjd (void) zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 153219089Spjd "scrub_queue", sizeof (uint64_t), 1, 154219089Spjd &scn->scn_phys.scn_queue_obj); 155219089Spjd } else { 156219089Spjd err = zap_lookup(dp->dp_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 157219089Spjd DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 158219089Spjd &scn->scn_phys); 159219089Spjd if (err == ENOENT) 160219089Spjd return (0); 161219089Spjd else if (err) 162219089Spjd return (err); 163219089Spjd 164219089Spjd if (scn->scn_phys.scn_state == DSS_SCANNING && 165219089Spjd spa_prev_software_version(dp->dp_spa) < SPA_VERSION_SCAN) { 166219089Spjd /* 167219089Spjd * A new-type scrub was in progress on an old 168219089Spjd * pool, and the pool was accessed by old 169219089Spjd * software. Restart from the beginning, since 170219089Spjd * the old software may have changed the pool in 171219089Spjd * the meantime. 172219089Spjd */ 173219089Spjd scn->scn_restart_txg = txg; 174219089Spjd zfs_dbgmsg("new-style scrub was modified " 175219089Spjd "by old software; restarting in txg %llu", 176219089Spjd scn->scn_restart_txg); 177219089Spjd } 178219089Spjd } 179219089Spjd 180219089Spjd spa_scan_stat_init(spa); 181219089Spjd return (0); 182219089Spjd} 183219089Spjd 184219089Spjdvoid 185219089Spjddsl_scan_fini(dsl_pool_t *dp) 186219089Spjd{ 187219089Spjd if (dp->dp_scan) { 188219089Spjd kmem_free(dp->dp_scan, sizeof (dsl_scan_t)); 189219089Spjd dp->dp_scan = NULL; 190219089Spjd } 191219089Spjd} 192219089Spjd 193219089Spjd/* ARGSUSED */ 194219089Spjdstatic int 195248571Smmdsl_scan_setup_check(void *arg, dmu_tx_t *tx) 196219089Spjd{ 197248571Smm dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 198219089Spjd 199219089Spjd if (scn->scn_phys.scn_state == DSS_SCANNING) 200249195Smm return (SET_ERROR(EBUSY)); 201219089Spjd 202219089Spjd return (0); 203219089Spjd} 204219089Spjd 205219089Spjdstatic void 206248571Smmdsl_scan_setup_sync(void *arg, dmu_tx_t *tx) 207219089Spjd{ 208248571Smm dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 209248571Smm pool_scan_func_t *funcp = arg; 210219089Spjd dmu_object_type_t ot = 0; 211219089Spjd dsl_pool_t *dp = scn->scn_dp; 212219089Spjd spa_t *spa = dp->dp_spa; 213219089Spjd 214219089Spjd ASSERT(scn->scn_phys.scn_state != DSS_SCANNING); 215219089Spjd ASSERT(*funcp > POOL_SCAN_NONE && *funcp < POOL_SCAN_FUNCS); 216219089Spjd bzero(&scn->scn_phys, sizeof (scn->scn_phys)); 217219089Spjd scn->scn_phys.scn_func = *funcp; 218219089Spjd scn->scn_phys.scn_state = DSS_SCANNING; 219219089Spjd scn->scn_phys.scn_min_txg = 0; 220219089Spjd scn->scn_phys.scn_max_txg = tx->tx_txg; 221219089Spjd scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */ 222219089Spjd scn->scn_phys.scn_start_time = gethrestime_sec(); 223219089Spjd scn->scn_phys.scn_errors = 0; 224219089Spjd scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc; 225219089Spjd scn->scn_restart_txg = 0; 226254112Sdelphij scn->scn_done_txg = 0; 227219089Spjd spa_scan_stat_init(spa); 228219089Spjd 229219089Spjd if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 230219089Spjd scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max; 231219089Spjd 232219089Spjd /* rewrite all disk labels */ 233219089Spjd vdev_config_dirty(spa->spa_root_vdev); 234219089Spjd 235219089Spjd if (vdev_resilver_needed(spa->spa_root_vdev, 236219089Spjd &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) { 237219089Spjd spa_event_notify(spa, NULL, ESC_ZFS_RESILVER_START); 238219089Spjd } else { 239219089Spjd spa_event_notify(spa, NULL, ESC_ZFS_SCRUB_START); 240219089Spjd } 241219089Spjd 242219089Spjd spa->spa_scrub_started = B_TRUE; 243219089Spjd /* 244219089Spjd * If this is an incremental scrub, limit the DDT scrub phase 245219089Spjd * to just the auto-ditto class (for correctness); the rest 246219089Spjd * of the scrub should go faster using top-down pruning. 247219089Spjd */ 248219089Spjd if (scn->scn_phys.scn_min_txg > TXG_INITIAL) 249219089Spjd scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO; 250219089Spjd 251219089Spjd } 252219089Spjd 253219089Spjd /* back to the generic stuff */ 254219089Spjd 255219089Spjd if (dp->dp_blkstats == NULL) { 256219089Spjd dp->dp_blkstats = 257219089Spjd kmem_alloc(sizeof (zfs_all_blkstats_t), KM_SLEEP); 258219089Spjd } 259219089Spjd bzero(dp->dp_blkstats, sizeof (zfs_all_blkstats_t)); 260219089Spjd 261219089Spjd if (spa_version(spa) < SPA_VERSION_DSL_SCRUB) 262219089Spjd ot = DMU_OT_ZAP_OTHER; 263219089Spjd 264219089Spjd scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, 265219089Spjd ot ? ot : DMU_OT_SCAN_QUEUE, DMU_OT_NONE, 0, tx); 266219089Spjd 267219089Spjd dsl_scan_sync_state(scn, tx); 268219089Spjd 269248571Smm spa_history_log_internal(spa, "scan setup", tx, 270219089Spjd "func=%u mintxg=%llu maxtxg=%llu", 271219089Spjd *funcp, scn->scn_phys.scn_min_txg, scn->scn_phys.scn_max_txg); 272219089Spjd} 273219089Spjd 274219089Spjd/* ARGSUSED */ 275219089Spjdstatic void 276219089Spjddsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) 277219089Spjd{ 278219089Spjd static const char *old_names[] = { 279219089Spjd "scrub_bookmark", 280219089Spjd "scrub_ddt_bookmark", 281219089Spjd "scrub_ddt_class_max", 282219089Spjd "scrub_queue", 283219089Spjd "scrub_min_txg", 284219089Spjd "scrub_max_txg", 285219089Spjd "scrub_func", 286219089Spjd "scrub_errors", 287219089Spjd NULL 288219089Spjd }; 289219089Spjd 290219089Spjd dsl_pool_t *dp = scn->scn_dp; 291219089Spjd spa_t *spa = dp->dp_spa; 292219089Spjd int i; 293219089Spjd 294219089Spjd /* Remove any remnants of an old-style scrub. */ 295219089Spjd for (i = 0; old_names[i]; i++) { 296219089Spjd (void) zap_remove(dp->dp_meta_objset, 297219089Spjd DMU_POOL_DIRECTORY_OBJECT, old_names[i], tx); 298219089Spjd } 299219089Spjd 300219089Spjd if (scn->scn_phys.scn_queue_obj != 0) { 301219089Spjd VERIFY(0 == dmu_object_free(dp->dp_meta_objset, 302219089Spjd scn->scn_phys.scn_queue_obj, tx)); 303219089Spjd scn->scn_phys.scn_queue_obj = 0; 304219089Spjd } 305219089Spjd 306219089Spjd /* 307219089Spjd * If we were "restarted" from a stopped state, don't bother 308219089Spjd * with anything else. 309219089Spjd */ 310219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 311219089Spjd return; 312219089Spjd 313219089Spjd if (complete) 314219089Spjd scn->scn_phys.scn_state = DSS_FINISHED; 315219089Spjd else 316219089Spjd scn->scn_phys.scn_state = DSS_CANCELED; 317219089Spjd 318248571Smm spa_history_log_internal(spa, "scan done", tx, 319219089Spjd "complete=%u", complete); 320219089Spjd 321219089Spjd if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 322219089Spjd mutex_enter(&spa->spa_scrub_lock); 323219089Spjd while (spa->spa_scrub_inflight > 0) { 324219089Spjd cv_wait(&spa->spa_scrub_io_cv, 325219089Spjd &spa->spa_scrub_lock); 326219089Spjd } 327219089Spjd mutex_exit(&spa->spa_scrub_lock); 328219089Spjd spa->spa_scrub_started = B_FALSE; 329219089Spjd spa->spa_scrub_active = B_FALSE; 330219089Spjd 331219089Spjd /* 332219089Spjd * If the scrub/resilver completed, update all DTLs to 333219089Spjd * reflect this. Whether it succeeded or not, vacate 334219089Spjd * all temporary scrub DTLs. 335219089Spjd */ 336219089Spjd vdev_dtl_reassess(spa->spa_root_vdev, tx->tx_txg, 337219089Spjd complete ? scn->scn_phys.scn_max_txg : 0, B_TRUE); 338219089Spjd if (complete) { 339219089Spjd spa_event_notify(spa, NULL, scn->scn_phys.scn_min_txg ? 340219089Spjd ESC_ZFS_RESILVER_FINISH : ESC_ZFS_SCRUB_FINISH); 341219089Spjd } 342219089Spjd spa_errlog_rotate(spa); 343219089Spjd 344219089Spjd /* 345219089Spjd * We may have finished replacing a device. 346219089Spjd * Let the async thread assess this and handle the detach. 347219089Spjd */ 348219089Spjd spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 349219089Spjd } 350219089Spjd 351219089Spjd scn->scn_phys.scn_end_time = gethrestime_sec(); 352219089Spjd} 353219089Spjd 354219089Spjd/* ARGSUSED */ 355219089Spjdstatic int 356248571Smmdsl_scan_cancel_check(void *arg, dmu_tx_t *tx) 357219089Spjd{ 358248571Smm dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 359219089Spjd 360219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 361249195Smm return (SET_ERROR(ENOENT)); 362219089Spjd return (0); 363219089Spjd} 364219089Spjd 365219089Spjd/* ARGSUSED */ 366219089Spjdstatic void 367248571Smmdsl_scan_cancel_sync(void *arg, dmu_tx_t *tx) 368219089Spjd{ 369248571Smm dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan; 370219089Spjd 371219089Spjd dsl_scan_done(scn, B_FALSE, tx); 372219089Spjd dsl_scan_sync_state(scn, tx); 373219089Spjd} 374219089Spjd 375219089Spjdint 376219089Spjddsl_scan_cancel(dsl_pool_t *dp) 377219089Spjd{ 378248571Smm return (dsl_sync_task(spa_name(dp->dp_spa), dsl_scan_cancel_check, 379269006Sdelphij dsl_scan_cancel_sync, NULL, 3, ZFS_SPACE_CHECK_RESERVED)); 380219089Spjd} 381219089Spjd 382219089Spjdstatic void dsl_scan_visitbp(blkptr_t *bp, 383268657Sdelphij const zbookmark_phys_t *zb, dnode_phys_t *dnp, arc_buf_t *pbuf, 384219089Spjd dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype, 385219089Spjd dmu_tx_t *tx); 386219089Spjdstatic void dsl_scan_visitdnode(dsl_scan_t *, dsl_dataset_t *ds, 387219089Spjd dmu_objset_type_t ostype, 388219089Spjd dnode_phys_t *dnp, arc_buf_t *buf, uint64_t object, dmu_tx_t *tx); 389219089Spjd 390219089Spjdvoid 391219089Spjddsl_free(dsl_pool_t *dp, uint64_t txg, const blkptr_t *bp) 392219089Spjd{ 393219089Spjd zio_free(dp->dp_spa, txg, bp); 394219089Spjd} 395219089Spjd 396219089Spjdvoid 397219089Spjddsl_free_sync(zio_t *pio, dsl_pool_t *dp, uint64_t txg, const blkptr_t *bpp) 398219089Spjd{ 399219089Spjd ASSERT(dsl_pool_sync_context(dp)); 400240868Spjd zio_nowait(zio_free_sync(pio, dp->dp_spa, txg, bpp, BP_GET_PSIZE(bpp), 401240868Spjd pio->io_flags)); 402219089Spjd} 403219089Spjd 404219089Spjdstatic uint64_t 405219089Spjddsl_scan_ds_maxtxg(dsl_dataset_t *ds) 406219089Spjd{ 407219089Spjd uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg; 408219089Spjd if (dsl_dataset_is_snapshot(ds)) 409219089Spjd return (MIN(smt, ds->ds_phys->ds_creation_txg)); 410219089Spjd return (smt); 411219089Spjd} 412219089Spjd 413219089Spjdstatic void 414219089Spjddsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx) 415219089Spjd{ 416248571Smm VERIFY0(zap_update(scn->scn_dp->dp_meta_objset, 417219089Spjd DMU_POOL_DIRECTORY_OBJECT, 418219089Spjd DMU_POOL_SCAN, sizeof (uint64_t), SCAN_PHYS_NUMINTS, 419219089Spjd &scn->scn_phys, tx)); 420219089Spjd} 421219089Spjd 422219089Spjdstatic boolean_t 423268657Sdelphijdsl_scan_check_pause(dsl_scan_t *scn, const zbookmark_phys_t *zb) 424219089Spjd{ 425219089Spjd uint64_t elapsed_nanosecs; 426237972Smm unsigned int mintime; 427219089Spjd 428219089Spjd /* we never skip user/group accounting objects */ 429219089Spjd if (zb && (int64_t)zb->zb_object < 0) 430219089Spjd return (B_FALSE); 431219089Spjd 432219089Spjd if (scn->scn_pausing) 433219089Spjd return (B_TRUE); /* we're already pausing */ 434219089Spjd 435236884Smm if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) 436219089Spjd return (B_FALSE); /* we're resuming */ 437219089Spjd 438219089Spjd /* We only know how to resume from level-0 blocks. */ 439219089Spjd if (zb && zb->zb_level != 0) 440219089Spjd return (B_FALSE); 441219089Spjd 442219089Spjd mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ? 443219089Spjd zfs_resilver_min_time_ms : zfs_scan_min_time_ms; 444219089Spjd elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 445219089Spjd if (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 446255437Sdelphij (NSEC2MSEC(elapsed_nanosecs) > mintime && 447219089Spjd txg_sync_waiting(scn->scn_dp)) || 448219089Spjd spa_shutting_down(scn->scn_dp->dp_spa)) { 449219089Spjd if (zb) { 450219089Spjd dprintf("pausing at bookmark %llx/%llx/%llx/%llx\n", 451219089Spjd (longlong_t)zb->zb_objset, 452219089Spjd (longlong_t)zb->zb_object, 453219089Spjd (longlong_t)zb->zb_level, 454219089Spjd (longlong_t)zb->zb_blkid); 455219089Spjd scn->scn_phys.scn_bookmark = *zb; 456219089Spjd } 457219089Spjd dprintf("pausing at DDT bookmark %llx/%llx/%llx/%llx\n", 458219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 459219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 460219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 461219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 462219089Spjd scn->scn_pausing = B_TRUE; 463219089Spjd return (B_TRUE); 464219089Spjd } 465219089Spjd return (B_FALSE); 466219089Spjd} 467219089Spjd 468219089Spjdtypedef struct zil_scan_arg { 469219089Spjd dsl_pool_t *zsa_dp; 470219089Spjd zil_header_t *zsa_zh; 471219089Spjd} zil_scan_arg_t; 472219089Spjd 473219089Spjd/* ARGSUSED */ 474219089Spjdstatic int 475219089Spjddsl_scan_zil_block(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 476219089Spjd{ 477219089Spjd zil_scan_arg_t *zsa = arg; 478219089Spjd dsl_pool_t *dp = zsa->zsa_dp; 479219089Spjd dsl_scan_t *scn = dp->dp_scan; 480219089Spjd zil_header_t *zh = zsa->zsa_zh; 481268657Sdelphij zbookmark_phys_t zb; 482219089Spjd 483263397Sdelphij if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 484219089Spjd return (0); 485219089Spjd 486219089Spjd /* 487219089Spjd * One block ("stubby") can be allocated a long time ago; we 488219089Spjd * want to visit that one because it has been allocated 489219089Spjd * (on-disk) even if it hasn't been claimed (even though for 490219089Spjd * scrub there's nothing to do to it). 491219089Spjd */ 492219089Spjd if (claim_txg == 0 && bp->blk_birth >= spa_first_txg(dp->dp_spa)) 493219089Spjd return (0); 494219089Spjd 495219089Spjd SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 496219089Spjd ZB_ZIL_OBJECT, ZB_ZIL_LEVEL, bp->blk_cksum.zc_word[ZIL_ZC_SEQ]); 497219089Spjd 498219089Spjd VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 499219089Spjd return (0); 500219089Spjd} 501219089Spjd 502219089Spjd/* ARGSUSED */ 503219089Spjdstatic int 504219089Spjddsl_scan_zil_record(zilog_t *zilog, lr_t *lrc, void *arg, uint64_t claim_txg) 505219089Spjd{ 506219089Spjd if (lrc->lrc_txtype == TX_WRITE) { 507219089Spjd zil_scan_arg_t *zsa = arg; 508219089Spjd dsl_pool_t *dp = zsa->zsa_dp; 509219089Spjd dsl_scan_t *scn = dp->dp_scan; 510219089Spjd zil_header_t *zh = zsa->zsa_zh; 511219089Spjd lr_write_t *lr = (lr_write_t *)lrc; 512219089Spjd blkptr_t *bp = &lr->lr_blkptr; 513268657Sdelphij zbookmark_phys_t zb; 514219089Spjd 515263397Sdelphij if (BP_IS_HOLE(bp) || 516263397Sdelphij bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 517219089Spjd return (0); 518219089Spjd 519219089Spjd /* 520219089Spjd * birth can be < claim_txg if this record's txg is 521219089Spjd * already txg sync'ed (but this log block contains 522219089Spjd * other records that are not synced) 523219089Spjd */ 524219089Spjd if (claim_txg == 0 || bp->blk_birth < claim_txg) 525219089Spjd return (0); 526219089Spjd 527219089Spjd SET_BOOKMARK(&zb, zh->zh_log.blk_cksum.zc_word[ZIL_ZC_OBJSET], 528219089Spjd lr->lr_foid, ZB_ZIL_LEVEL, 529219089Spjd lr->lr_offset / BP_GET_LSIZE(bp)); 530219089Spjd 531219089Spjd VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb)); 532219089Spjd } 533219089Spjd return (0); 534219089Spjd} 535219089Spjd 536219089Spjdstatic void 537219089Spjddsl_scan_zil(dsl_pool_t *dp, zil_header_t *zh) 538219089Spjd{ 539219089Spjd uint64_t claim_txg = zh->zh_claim_txg; 540219089Spjd zil_scan_arg_t zsa = { dp, zh }; 541219089Spjd zilog_t *zilog; 542219089Spjd 543219089Spjd /* 544219089Spjd * We only want to visit blocks that have been claimed but not yet 545219089Spjd * replayed (or, in read-only mode, blocks that *would* be claimed). 546219089Spjd */ 547219089Spjd if (claim_txg == 0 && spa_writeable(dp->dp_spa)) 548219089Spjd return; 549219089Spjd 550219089Spjd zilog = zil_alloc(dp->dp_meta_objset, zh); 551219089Spjd 552219089Spjd (void) zil_parse(zilog, dsl_scan_zil_block, dsl_scan_zil_record, &zsa, 553219089Spjd claim_txg); 554219089Spjd 555219089Spjd zil_free(zilog); 556219089Spjd} 557219089Spjd 558219089Spjd/* ARGSUSED */ 559219089Spjdstatic void 560219089Spjddsl_scan_prefetch(dsl_scan_t *scn, arc_buf_t *buf, blkptr_t *bp, 561219089Spjd uint64_t objset, uint64_t object, uint64_t blkid) 562219089Spjd{ 563268657Sdelphij zbookmark_phys_t czb; 564219089Spjd uint32_t flags = ARC_NOWAIT | ARC_PREFETCH; 565219089Spjd 566219089Spjd if (zfs_no_scrub_prefetch) 567219089Spjd return; 568219089Spjd 569219089Spjd if (BP_IS_HOLE(bp) || bp->blk_birth <= scn->scn_phys.scn_min_txg || 570219089Spjd (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_DNODE)) 571219089Spjd return; 572219089Spjd 573219089Spjd SET_BOOKMARK(&czb, objset, object, BP_GET_LEVEL(bp), blkid); 574219089Spjd 575219089Spjd (void) arc_read(scn->scn_zio_root, scn->scn_dp->dp_spa, bp, 576246666Smm NULL, NULL, ZIO_PRIORITY_ASYNC_READ, 577219089Spjd ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD, &flags, &czb); 578219089Spjd} 579219089Spjd 580219089Spjdstatic boolean_t 581219089Spjddsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp, 582268657Sdelphij const zbookmark_phys_t *zb) 583219089Spjd{ 584219089Spjd /* 585219089Spjd * We never skip over user/group accounting objects (obj<0) 586219089Spjd */ 587236884Smm if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) && 588219089Spjd (int64_t)zb->zb_object >= 0) { 589219089Spjd /* 590219089Spjd * If we already visited this bp & everything below (in 591219089Spjd * a prior txg sync), don't bother doing it again. 592219089Spjd */ 593236884Smm if (zbookmark_is_before(dnp, zb, &scn->scn_phys.scn_bookmark)) 594219089Spjd return (B_TRUE); 595219089Spjd 596219089Spjd /* 597219089Spjd * If we found the block we're trying to resume from, or 598219089Spjd * we went past it to a different object, zero it out to 599219089Spjd * indicate that it's OK to start checking for pausing 600219089Spjd * again. 601219089Spjd */ 602219089Spjd if (bcmp(zb, &scn->scn_phys.scn_bookmark, sizeof (*zb)) == 0 || 603219089Spjd zb->zb_object > scn->scn_phys.scn_bookmark.zb_object) { 604219089Spjd dprintf("resuming at %llx/%llx/%llx/%llx\n", 605219089Spjd (longlong_t)zb->zb_objset, 606219089Spjd (longlong_t)zb->zb_object, 607219089Spjd (longlong_t)zb->zb_level, 608219089Spjd (longlong_t)zb->zb_blkid); 609219089Spjd bzero(&scn->scn_phys.scn_bookmark, sizeof (*zb)); 610219089Spjd } 611219089Spjd } 612219089Spjd return (B_FALSE); 613219089Spjd} 614219089Spjd 615219089Spjd/* 616219089Spjd * Return nonzero on i/o error. 617219089Spjd * Return new buf to write out in *bufp. 618219089Spjd */ 619219089Spjdstatic int 620219089Spjddsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, 621219089Spjd dnode_phys_t *dnp, const blkptr_t *bp, 622268657Sdelphij const zbookmark_phys_t *zb, dmu_tx_t *tx, arc_buf_t **bufp) 623219089Spjd{ 624219089Spjd dsl_pool_t *dp = scn->scn_dp; 625219089Spjd int zio_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_SCAN_THREAD; 626219089Spjd int err; 627219089Spjd 628219089Spjd if (BP_GET_LEVEL(bp) > 0) { 629219089Spjd uint32_t flags = ARC_WAIT; 630219089Spjd int i; 631219089Spjd blkptr_t *cbp; 632219089Spjd int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT; 633219089Spjd 634246666Smm err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 635219089Spjd ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 636219089Spjd if (err) { 637219089Spjd scn->scn_phys.scn_errors++; 638219089Spjd return (err); 639219089Spjd } 640219089Spjd for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) { 641219089Spjd dsl_scan_prefetch(scn, *bufp, cbp, zb->zb_objset, 642219089Spjd zb->zb_object, zb->zb_blkid * epb + i); 643219089Spjd } 644219089Spjd for (i = 0, cbp = (*bufp)->b_data; i < epb; i++, cbp++) { 645268657Sdelphij zbookmark_phys_t czb; 646219089Spjd 647219089Spjd SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object, 648219089Spjd zb->zb_level - 1, 649219089Spjd zb->zb_blkid * epb + i); 650219089Spjd dsl_scan_visitbp(cbp, &czb, dnp, 651219089Spjd *bufp, ds, scn, ostype, tx); 652219089Spjd } 653219089Spjd } else if (BP_GET_TYPE(bp) == DMU_OT_USERGROUP_USED) { 654219089Spjd uint32_t flags = ARC_WAIT; 655219089Spjd 656246666Smm err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 657219089Spjd ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 658219089Spjd if (err) { 659219089Spjd scn->scn_phys.scn_errors++; 660219089Spjd return (err); 661219089Spjd } 662219089Spjd } else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) { 663219089Spjd uint32_t flags = ARC_WAIT; 664219089Spjd dnode_phys_t *cdnp; 665219089Spjd int i, j; 666219089Spjd int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; 667219089Spjd 668246666Smm err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 669219089Spjd ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 670219089Spjd if (err) { 671219089Spjd scn->scn_phys.scn_errors++; 672219089Spjd return (err); 673219089Spjd } 674219089Spjd for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) { 675219089Spjd for (j = 0; j < cdnp->dn_nblkptr; j++) { 676219089Spjd blkptr_t *cbp = &cdnp->dn_blkptr[j]; 677219089Spjd dsl_scan_prefetch(scn, *bufp, cbp, 678219089Spjd zb->zb_objset, zb->zb_blkid * epb + i, j); 679219089Spjd } 680219089Spjd } 681219089Spjd for (i = 0, cdnp = (*bufp)->b_data; i < epb; i++, cdnp++) { 682219089Spjd dsl_scan_visitdnode(scn, ds, ostype, 683219089Spjd cdnp, *bufp, zb->zb_blkid * epb + i, tx); 684219089Spjd } 685219089Spjd 686219089Spjd } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { 687219089Spjd uint32_t flags = ARC_WAIT; 688219089Spjd objset_phys_t *osp; 689219089Spjd 690246666Smm err = arc_read(NULL, dp->dp_spa, bp, arc_getbuf_func, bufp, 691219089Spjd ZIO_PRIORITY_ASYNC_READ, zio_flags, &flags, zb); 692219089Spjd if (err) { 693219089Spjd scn->scn_phys.scn_errors++; 694219089Spjd return (err); 695219089Spjd } 696219089Spjd 697219089Spjd osp = (*bufp)->b_data; 698219089Spjd 699219089Spjd dsl_scan_visitdnode(scn, ds, osp->os_type, 700219089Spjd &osp->os_meta_dnode, *bufp, DMU_META_DNODE_OBJECT, tx); 701219089Spjd 702219089Spjd if (OBJSET_BUF_HAS_USERUSED(*bufp)) { 703219089Spjd /* 704219089Spjd * We also always visit user/group accounting 705219089Spjd * objects, and never skip them, even if we are 706219089Spjd * pausing. This is necessary so that the space 707219089Spjd * deltas from this txg get integrated. 708219089Spjd */ 709219089Spjd dsl_scan_visitdnode(scn, ds, osp->os_type, 710219089Spjd &osp->os_groupused_dnode, *bufp, 711219089Spjd DMU_GROUPUSED_OBJECT, tx); 712219089Spjd dsl_scan_visitdnode(scn, ds, osp->os_type, 713219089Spjd &osp->os_userused_dnode, *bufp, 714219089Spjd DMU_USERUSED_OBJECT, tx); 715219089Spjd } 716219089Spjd } 717219089Spjd 718219089Spjd return (0); 719219089Spjd} 720219089Spjd 721219089Spjdstatic void 722219089Spjddsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, 723219089Spjd dmu_objset_type_t ostype, dnode_phys_t *dnp, arc_buf_t *buf, 724219089Spjd uint64_t object, dmu_tx_t *tx) 725219089Spjd{ 726219089Spjd int j; 727219089Spjd 728219089Spjd for (j = 0; j < dnp->dn_nblkptr; j++) { 729268657Sdelphij zbookmark_phys_t czb; 730219089Spjd 731219089Spjd SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 732219089Spjd dnp->dn_nlevels - 1, j); 733219089Spjd dsl_scan_visitbp(&dnp->dn_blkptr[j], 734219089Spjd &czb, dnp, buf, ds, scn, ostype, tx); 735219089Spjd } 736219089Spjd 737219089Spjd if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { 738268657Sdelphij zbookmark_phys_t czb; 739219089Spjd SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 740219089Spjd 0, DMU_SPILL_BLKID); 741219089Spjd dsl_scan_visitbp(&dnp->dn_spill, 742219089Spjd &czb, dnp, buf, ds, scn, ostype, tx); 743219089Spjd } 744219089Spjd} 745219089Spjd 746219089Spjd/* 747219089Spjd * The arguments are in this order because mdb can only print the 748219089Spjd * first 5; we want them to be useful. 749219089Spjd */ 750219089Spjdstatic void 751268657Sdelphijdsl_scan_visitbp(blkptr_t *bp, const zbookmark_phys_t *zb, 752219089Spjd dnode_phys_t *dnp, arc_buf_t *pbuf, 753219089Spjd dsl_dataset_t *ds, dsl_scan_t *scn, dmu_objset_type_t ostype, 754219089Spjd dmu_tx_t *tx) 755219089Spjd{ 756219089Spjd dsl_pool_t *dp = scn->scn_dp; 757219089Spjd arc_buf_t *buf = NULL; 758219089Spjd blkptr_t bp_toread = *bp; 759219089Spjd 760219089Spjd /* ASSERT(pbuf == NULL || arc_released(pbuf)); */ 761219089Spjd 762219089Spjd if (dsl_scan_check_pause(scn, zb)) 763219089Spjd return; 764219089Spjd 765219089Spjd if (dsl_scan_check_resume(scn, dnp, zb)) 766219089Spjd return; 767219089Spjd 768263397Sdelphij if (BP_IS_HOLE(bp)) 769219089Spjd return; 770219089Spjd 771219089Spjd scn->scn_visited_this_txg++; 772219089Spjd 773219089Spjd dprintf_bp(bp, 774219089Spjd "visiting ds=%p/%llu zb=%llx/%llx/%llx/%llx buf=%p bp=%p", 775219089Spjd ds, ds ? ds->ds_object : 0, 776219089Spjd zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid, 777219089Spjd pbuf, bp); 778219089Spjd 779219089Spjd if (bp->blk_birth <= scn->scn_phys.scn_cur_min_txg) 780219089Spjd return; 781219089Spjd 782219089Spjd if (dsl_scan_recurse(scn, ds, ostype, dnp, &bp_toread, zb, tx, 783219089Spjd &buf) != 0) 784219089Spjd return; 785219089Spjd 786219089Spjd /* 787219089Spjd * If dsl_scan_ddt() has aready visited this block, it will have 788219089Spjd * already done any translations or scrubbing, so don't call the 789219089Spjd * callback again. 790219089Spjd */ 791219089Spjd if (ddt_class_contains(dp->dp_spa, 792219089Spjd scn->scn_phys.scn_ddt_class_max, bp)) { 793219089Spjd ASSERT(buf == NULL); 794219089Spjd return; 795219089Spjd } 796219089Spjd 797219089Spjd /* 798219089Spjd * If this block is from the future (after cur_max_txg), then we 799219089Spjd * are doing this on behalf of a deleted snapshot, and we will 800219089Spjd * revisit the future block on the next pass of this dataset. 801219089Spjd * Don't scan it now unless we need to because something 802219089Spjd * under it was modified. 803219089Spjd */ 804254112Sdelphij if (BP_PHYSICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_max_txg) { 805219089Spjd scan_funcs[scn->scn_phys.scn_func](dp, bp, zb); 806219089Spjd } 807219089Spjd if (buf) 808219089Spjd (void) arc_buf_remove_ref(buf, &buf); 809219089Spjd} 810219089Spjd 811219089Spjdstatic void 812219089Spjddsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp, 813219089Spjd dmu_tx_t *tx) 814219089Spjd{ 815268657Sdelphij zbookmark_phys_t zb; 816219089Spjd 817219089Spjd SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 818219089Spjd ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); 819219089Spjd dsl_scan_visitbp(bp, &zb, NULL, NULL, 820219089Spjd ds, scn, DMU_OST_NONE, tx); 821219089Spjd 822219089Spjd dprintf_ds(ds, "finished scan%s", ""); 823219089Spjd} 824219089Spjd 825219089Spjdvoid 826219089Spjddsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx) 827219089Spjd{ 828219089Spjd dsl_pool_t *dp = ds->ds_dir->dd_pool; 829219089Spjd dsl_scan_t *scn = dp->dp_scan; 830219089Spjd uint64_t mintxg; 831219089Spjd 832219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 833219089Spjd return; 834219089Spjd 835219089Spjd if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 836219089Spjd if (dsl_dataset_is_snapshot(ds)) { 837219089Spjd /* Note, scn_cur_{min,max}_txg stays the same. */ 838219089Spjd scn->scn_phys.scn_bookmark.zb_objset = 839219089Spjd ds->ds_phys->ds_next_snap_obj; 840219089Spjd zfs_dbgmsg("destroying ds %llu; currently traversing; " 841219089Spjd "reset zb_objset to %llu", 842219089Spjd (u_longlong_t)ds->ds_object, 843219089Spjd (u_longlong_t)ds->ds_phys->ds_next_snap_obj); 844219089Spjd scn->scn_phys.scn_flags |= DSF_VISIT_DS_AGAIN; 845219089Spjd } else { 846219089Spjd SET_BOOKMARK(&scn->scn_phys.scn_bookmark, 847219089Spjd ZB_DESTROYED_OBJSET, 0, 0, 0); 848219089Spjd zfs_dbgmsg("destroying ds %llu; currently traversing; " 849219089Spjd "reset bookmark to -1,0,0,0", 850219089Spjd (u_longlong_t)ds->ds_object); 851219089Spjd } 852219089Spjd } else if (zap_lookup_int_key(dp->dp_meta_objset, 853219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 854219089Spjd ASSERT3U(ds->ds_phys->ds_num_children, <=, 1); 855219089Spjd VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 856219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 857219089Spjd if (dsl_dataset_is_snapshot(ds)) { 858219089Spjd /* 859219089Spjd * We keep the same mintxg; it could be > 860219089Spjd * ds_creation_txg if the previous snapshot was 861219089Spjd * deleted too. 862219089Spjd */ 863219089Spjd VERIFY(zap_add_int_key(dp->dp_meta_objset, 864219089Spjd scn->scn_phys.scn_queue_obj, 865219089Spjd ds->ds_phys->ds_next_snap_obj, mintxg, tx) == 0); 866219089Spjd zfs_dbgmsg("destroying ds %llu; in queue; " 867219089Spjd "replacing with %llu", 868219089Spjd (u_longlong_t)ds->ds_object, 869219089Spjd (u_longlong_t)ds->ds_phys->ds_next_snap_obj); 870219089Spjd } else { 871219089Spjd zfs_dbgmsg("destroying ds %llu; in queue; removing", 872219089Spjd (u_longlong_t)ds->ds_object); 873219089Spjd } 874219089Spjd } else { 875219089Spjd zfs_dbgmsg("destroying ds %llu; ignoring", 876219089Spjd (u_longlong_t)ds->ds_object); 877219089Spjd } 878219089Spjd 879219089Spjd /* 880219089Spjd * dsl_scan_sync() should be called after this, and should sync 881219089Spjd * out our changed state, but just to be safe, do it here. 882219089Spjd */ 883219089Spjd dsl_scan_sync_state(scn, tx); 884219089Spjd} 885219089Spjd 886219089Spjdvoid 887219089Spjddsl_scan_ds_snapshotted(dsl_dataset_t *ds, dmu_tx_t *tx) 888219089Spjd{ 889219089Spjd dsl_pool_t *dp = ds->ds_dir->dd_pool; 890219089Spjd dsl_scan_t *scn = dp->dp_scan; 891219089Spjd uint64_t mintxg; 892219089Spjd 893219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 894219089Spjd return; 895219089Spjd 896219089Spjd ASSERT(ds->ds_phys->ds_prev_snap_obj != 0); 897219089Spjd 898219089Spjd if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) { 899219089Spjd scn->scn_phys.scn_bookmark.zb_objset = 900219089Spjd ds->ds_phys->ds_prev_snap_obj; 901219089Spjd zfs_dbgmsg("snapshotting ds %llu; currently traversing; " 902219089Spjd "reset zb_objset to %llu", 903219089Spjd (u_longlong_t)ds->ds_object, 904219089Spjd (u_longlong_t)ds->ds_phys->ds_prev_snap_obj); 905219089Spjd } else if (zap_lookup_int_key(dp->dp_meta_objset, 906219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_object, &mintxg) == 0) { 907219089Spjd VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 908219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_object, tx)); 909219089Spjd VERIFY(zap_add_int_key(dp->dp_meta_objset, 910219089Spjd scn->scn_phys.scn_queue_obj, 911219089Spjd ds->ds_phys->ds_prev_snap_obj, mintxg, tx) == 0); 912219089Spjd zfs_dbgmsg("snapshotting ds %llu; in queue; " 913219089Spjd "replacing with %llu", 914219089Spjd (u_longlong_t)ds->ds_object, 915219089Spjd (u_longlong_t)ds->ds_phys->ds_prev_snap_obj); 916219089Spjd } 917219089Spjd dsl_scan_sync_state(scn, tx); 918219089Spjd} 919219089Spjd 920219089Spjdvoid 921219089Spjddsl_scan_ds_clone_swapped(dsl_dataset_t *ds1, dsl_dataset_t *ds2, dmu_tx_t *tx) 922219089Spjd{ 923219089Spjd dsl_pool_t *dp = ds1->ds_dir->dd_pool; 924219089Spjd dsl_scan_t *scn = dp->dp_scan; 925219089Spjd uint64_t mintxg; 926219089Spjd 927219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 928219089Spjd return; 929219089Spjd 930219089Spjd if (scn->scn_phys.scn_bookmark.zb_objset == ds1->ds_object) { 931219089Spjd scn->scn_phys.scn_bookmark.zb_objset = ds2->ds_object; 932219089Spjd zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 933219089Spjd "reset zb_objset to %llu", 934219089Spjd (u_longlong_t)ds1->ds_object, 935219089Spjd (u_longlong_t)ds2->ds_object); 936219089Spjd } else if (scn->scn_phys.scn_bookmark.zb_objset == ds2->ds_object) { 937219089Spjd scn->scn_phys.scn_bookmark.zb_objset = ds1->ds_object; 938219089Spjd zfs_dbgmsg("clone_swap ds %llu; currently traversing; " 939219089Spjd "reset zb_objset to %llu", 940219089Spjd (u_longlong_t)ds2->ds_object, 941219089Spjd (u_longlong_t)ds1->ds_object); 942219089Spjd } 943219089Spjd 944219089Spjd if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 945219089Spjd ds1->ds_object, &mintxg) == 0) { 946219089Spjd int err; 947219089Spjd 948219089Spjd ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg); 949219089Spjd ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg); 950219089Spjd VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 951219089Spjd scn->scn_phys.scn_queue_obj, ds1->ds_object, tx)); 952219089Spjd err = zap_add_int_key(dp->dp_meta_objset, 953219089Spjd scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg, tx); 954219089Spjd VERIFY(err == 0 || err == EEXIST); 955219089Spjd if (err == EEXIST) { 956219089Spjd /* Both were there to begin with */ 957219089Spjd VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 958219089Spjd scn->scn_phys.scn_queue_obj, 959219089Spjd ds1->ds_object, mintxg, tx)); 960219089Spjd } 961219089Spjd zfs_dbgmsg("clone_swap ds %llu; in queue; " 962219089Spjd "replacing with %llu", 963219089Spjd (u_longlong_t)ds1->ds_object, 964219089Spjd (u_longlong_t)ds2->ds_object); 965219089Spjd } else if (zap_lookup_int_key(dp->dp_meta_objset, 966219089Spjd scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg) == 0) { 967219089Spjd ASSERT3U(mintxg, ==, ds1->ds_phys->ds_prev_snap_txg); 968219089Spjd ASSERT3U(mintxg, ==, ds2->ds_phys->ds_prev_snap_txg); 969219089Spjd VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 970219089Spjd scn->scn_phys.scn_queue_obj, ds2->ds_object, tx)); 971219089Spjd VERIFY(0 == zap_add_int_key(dp->dp_meta_objset, 972219089Spjd scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg, tx)); 973219089Spjd zfs_dbgmsg("clone_swap ds %llu; in queue; " 974219089Spjd "replacing with %llu", 975219089Spjd (u_longlong_t)ds2->ds_object, 976219089Spjd (u_longlong_t)ds1->ds_object); 977219089Spjd } 978219089Spjd 979219089Spjd dsl_scan_sync_state(scn, tx); 980219089Spjd} 981219089Spjd 982219089Spjdstruct enqueue_clones_arg { 983219089Spjd dmu_tx_t *tx; 984219089Spjd uint64_t originobj; 985219089Spjd}; 986219089Spjd 987219089Spjd/* ARGSUSED */ 988219089Spjdstatic int 989248571Smmenqueue_clones_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 990219089Spjd{ 991219089Spjd struct enqueue_clones_arg *eca = arg; 992219089Spjd dsl_dataset_t *ds; 993219089Spjd int err; 994219089Spjd dsl_scan_t *scn = dp->dp_scan; 995219089Spjd 996248571Smm if (hds->ds_dir->dd_phys->dd_origin_obj != eca->originobj) 997248571Smm return (0); 998248571Smm 999248571Smm err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 1000219089Spjd if (err) 1001219089Spjd return (err); 1002219089Spjd 1003248571Smm while (ds->ds_phys->ds_prev_snap_obj != eca->originobj) { 1004248571Smm dsl_dataset_t *prev; 1005248571Smm err = dsl_dataset_hold_obj(dp, 1006248571Smm ds->ds_phys->ds_prev_snap_obj, FTAG, &prev); 1007219089Spjd 1008248571Smm dsl_dataset_rele(ds, FTAG); 1009248571Smm if (err) 1010248571Smm return (err); 1011248571Smm ds = prev; 1012219089Spjd } 1013248571Smm VERIFY(zap_add_int_key(dp->dp_meta_objset, 1014248571Smm scn->scn_phys.scn_queue_obj, ds->ds_object, 1015248571Smm ds->ds_phys->ds_prev_snap_txg, eca->tx) == 0); 1016219089Spjd dsl_dataset_rele(ds, FTAG); 1017219089Spjd return (0); 1018219089Spjd} 1019219089Spjd 1020219089Spjdstatic void 1021219089Spjddsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) 1022219089Spjd{ 1023219089Spjd dsl_pool_t *dp = scn->scn_dp; 1024219089Spjd dsl_dataset_t *ds; 1025219089Spjd objset_t *os; 1026219089Spjd 1027219089Spjd VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 1028219089Spjd 1029219089Spjd if (dmu_objset_from_ds(ds, &os)) 1030219089Spjd goto out; 1031219089Spjd 1032219089Spjd /* 1033219089Spjd * Only the ZIL in the head (non-snapshot) is valid. Even though 1034219089Spjd * snapshots can have ZIL block pointers (which may be the same 1035219089Spjd * BP as in the head), they must be ignored. So we traverse the 1036219089Spjd * ZIL here, rather than in scan_recurse(), because the regular 1037219089Spjd * snapshot block-sharing rules don't apply to it. 1038219089Spjd */ 1039219089Spjd if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds)) 1040219089Spjd dsl_scan_zil(dp, &os->os_zil_header); 1041219089Spjd 1042219089Spjd /* 1043219089Spjd * Iterate over the bps in this ds. 1044219089Spjd */ 1045219089Spjd dmu_buf_will_dirty(ds->ds_dbuf, tx); 1046219089Spjd dsl_scan_visit_rootbp(scn, ds, &ds->ds_phys->ds_bp, tx); 1047219089Spjd 1048219089Spjd char *dsname = kmem_alloc(ZFS_MAXNAMELEN, KM_SLEEP); 1049219089Spjd dsl_dataset_name(ds, dsname); 1050219089Spjd zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " 1051219089Spjd "pausing=%u", 1052219089Spjd (longlong_t)dsobj, dsname, 1053219089Spjd (longlong_t)scn->scn_phys.scn_cur_min_txg, 1054219089Spjd (longlong_t)scn->scn_phys.scn_cur_max_txg, 1055219089Spjd (int)scn->scn_pausing); 1056219089Spjd kmem_free(dsname, ZFS_MAXNAMELEN); 1057219089Spjd 1058219089Spjd if (scn->scn_pausing) 1059219089Spjd goto out; 1060219089Spjd 1061219089Spjd /* 1062219089Spjd * We've finished this pass over this dataset. 1063219089Spjd */ 1064219089Spjd 1065219089Spjd /* 1066219089Spjd * If we did not completely visit this dataset, do another pass. 1067219089Spjd */ 1068219089Spjd if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) { 1069219089Spjd zfs_dbgmsg("incomplete pass; visiting again"); 1070219089Spjd scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN; 1071219089Spjd VERIFY(zap_add_int_key(dp->dp_meta_objset, 1072219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_object, 1073219089Spjd scn->scn_phys.scn_cur_max_txg, tx) == 0); 1074219089Spjd goto out; 1075219089Spjd } 1076219089Spjd 1077219089Spjd /* 1078219089Spjd * Add descendent datasets to work queue. 1079219089Spjd */ 1080219089Spjd if (ds->ds_phys->ds_next_snap_obj != 0) { 1081219089Spjd VERIFY(zap_add_int_key(dp->dp_meta_objset, 1082219089Spjd scn->scn_phys.scn_queue_obj, ds->ds_phys->ds_next_snap_obj, 1083219089Spjd ds->ds_phys->ds_creation_txg, tx) == 0); 1084219089Spjd } 1085219089Spjd if (ds->ds_phys->ds_num_children > 1) { 1086219089Spjd boolean_t usenext = B_FALSE; 1087219089Spjd if (ds->ds_phys->ds_next_clones_obj != 0) { 1088219089Spjd uint64_t count; 1089219089Spjd /* 1090219089Spjd * A bug in a previous version of the code could 1091219089Spjd * cause upgrade_clones_cb() to not set 1092219089Spjd * ds_next_snap_obj when it should, leading to a 1093219089Spjd * missing entry. Therefore we can only use the 1094219089Spjd * next_clones_obj when its count is correct. 1095219089Spjd */ 1096219089Spjd int err = zap_count(dp->dp_meta_objset, 1097219089Spjd ds->ds_phys->ds_next_clones_obj, &count); 1098219089Spjd if (err == 0 && 1099219089Spjd count == ds->ds_phys->ds_num_children - 1) 1100219089Spjd usenext = B_TRUE; 1101219089Spjd } 1102219089Spjd 1103219089Spjd if (usenext) { 1104248571Smm VERIFY0(zap_join_key(dp->dp_meta_objset, 1105219089Spjd ds->ds_phys->ds_next_clones_obj, 1106219089Spjd scn->scn_phys.scn_queue_obj, 1107248571Smm ds->ds_phys->ds_creation_txg, tx)); 1108219089Spjd } else { 1109219089Spjd struct enqueue_clones_arg eca; 1110219089Spjd eca.tx = tx; 1111219089Spjd eca.originobj = ds->ds_object; 1112219089Spjd 1113248571Smm VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 1114248571Smm enqueue_clones_cb, &eca, DS_FIND_CHILDREN)); 1115219089Spjd } 1116219089Spjd } 1117219089Spjd 1118219089Spjdout: 1119219089Spjd dsl_dataset_rele(ds, FTAG); 1120219089Spjd} 1121219089Spjd 1122219089Spjd/* ARGSUSED */ 1123219089Spjdstatic int 1124248571Smmenqueue_cb(dsl_pool_t *dp, dsl_dataset_t *hds, void *arg) 1125219089Spjd{ 1126219089Spjd dmu_tx_t *tx = arg; 1127219089Spjd dsl_dataset_t *ds; 1128219089Spjd int err; 1129219089Spjd dsl_scan_t *scn = dp->dp_scan; 1130219089Spjd 1131248571Smm err = dsl_dataset_hold_obj(dp, hds->ds_object, FTAG, &ds); 1132219089Spjd if (err) 1133219089Spjd return (err); 1134219089Spjd 1135219089Spjd while (ds->ds_phys->ds_prev_snap_obj != 0) { 1136219089Spjd dsl_dataset_t *prev; 1137219089Spjd err = dsl_dataset_hold_obj(dp, ds->ds_phys->ds_prev_snap_obj, 1138219089Spjd FTAG, &prev); 1139219089Spjd if (err) { 1140219089Spjd dsl_dataset_rele(ds, FTAG); 1141219089Spjd return (err); 1142219089Spjd } 1143219089Spjd 1144219089Spjd /* 1145219089Spjd * If this is a clone, we don't need to worry about it for now. 1146219089Spjd */ 1147219089Spjd if (prev->ds_phys->ds_next_snap_obj != ds->ds_object) { 1148219089Spjd dsl_dataset_rele(ds, FTAG); 1149219089Spjd dsl_dataset_rele(prev, FTAG); 1150219089Spjd return (0); 1151219089Spjd } 1152219089Spjd dsl_dataset_rele(ds, FTAG); 1153219089Spjd ds = prev; 1154219089Spjd } 1155219089Spjd 1156219089Spjd VERIFY(zap_add_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj, 1157219089Spjd ds->ds_object, ds->ds_phys->ds_prev_snap_txg, tx) == 0); 1158219089Spjd dsl_dataset_rele(ds, FTAG); 1159219089Spjd return (0); 1160219089Spjd} 1161219089Spjd 1162219089Spjd/* 1163219089Spjd * Scrub/dedup interaction. 1164219089Spjd * 1165219089Spjd * If there are N references to a deduped block, we don't want to scrub it 1166219089Spjd * N times -- ideally, we should scrub it exactly once. 1167219089Spjd * 1168219089Spjd * We leverage the fact that the dde's replication class (enum ddt_class) 1169219089Spjd * is ordered from highest replication class (DDT_CLASS_DITTO) to lowest 1170219089Spjd * (DDT_CLASS_UNIQUE) so that we may walk the DDT in that order. 1171219089Spjd * 1172219089Spjd * To prevent excess scrubbing, the scrub begins by walking the DDT 1173219089Spjd * to find all blocks with refcnt > 1, and scrubs each of these once. 1174219089Spjd * Since there are two replication classes which contain blocks with 1175219089Spjd * refcnt > 1, we scrub the highest replication class (DDT_CLASS_DITTO) first. 1176219089Spjd * Finally the top-down scrub begins, only visiting blocks with refcnt == 1. 1177219089Spjd * 1178219089Spjd * There would be nothing more to say if a block's refcnt couldn't change 1179219089Spjd * during a scrub, but of course it can so we must account for changes 1180219089Spjd * in a block's replication class. 1181219089Spjd * 1182219089Spjd * Here's an example of what can occur: 1183219089Spjd * 1184219089Spjd * If a block has refcnt > 1 during the DDT scrub phase, but has refcnt == 1 1185219089Spjd * when visited during the top-down scrub phase, it will be scrubbed twice. 1186219089Spjd * This negates our scrub optimization, but is otherwise harmless. 1187219089Spjd * 1188219089Spjd * If a block has refcnt == 1 during the DDT scrub phase, but has refcnt > 1 1189219089Spjd * on each visit during the top-down scrub phase, it will never be scrubbed. 1190219089Spjd * To catch this, ddt_sync_entry() notifies the scrub code whenever a block's 1191219089Spjd * reference class transitions to a higher level (i.e DDT_CLASS_UNIQUE to 1192219089Spjd * DDT_CLASS_DUPLICATE); if it transitions from refcnt == 1 to refcnt > 1 1193219089Spjd * while a scrub is in progress, it scrubs the block right then. 1194219089Spjd */ 1195219089Spjdstatic void 1196219089Spjddsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx) 1197219089Spjd{ 1198219089Spjd ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark; 1199219089Spjd ddt_entry_t dde = { 0 }; 1200219089Spjd int error; 1201219089Spjd uint64_t n = 0; 1202219089Spjd 1203219089Spjd while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &dde)) == 0) { 1204219089Spjd ddt_t *ddt; 1205219089Spjd 1206219089Spjd if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max) 1207219089Spjd break; 1208219089Spjd dprintf("visiting ddb=%llu/%llu/%llu/%llx\n", 1209219089Spjd (longlong_t)ddb->ddb_class, 1210219089Spjd (longlong_t)ddb->ddb_type, 1211219089Spjd (longlong_t)ddb->ddb_checksum, 1212219089Spjd (longlong_t)ddb->ddb_cursor); 1213219089Spjd 1214219089Spjd /* There should be no pending changes to the dedup table */ 1215219089Spjd ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum]; 1216219089Spjd ASSERT(avl_first(&ddt->ddt_tree) == NULL); 1217219089Spjd 1218219089Spjd dsl_scan_ddt_entry(scn, ddb->ddb_checksum, &dde, tx); 1219219089Spjd n++; 1220219089Spjd 1221219089Spjd if (dsl_scan_check_pause(scn, NULL)) 1222219089Spjd break; 1223219089Spjd } 1224219089Spjd 1225219089Spjd zfs_dbgmsg("scanned %llu ddt entries with class_max = %u; pausing=%u", 1226219089Spjd (longlong_t)n, (int)scn->scn_phys.scn_ddt_class_max, 1227219089Spjd (int)scn->scn_pausing); 1228219089Spjd 1229219089Spjd ASSERT(error == 0 || error == ENOENT); 1230219089Spjd ASSERT(error != ENOENT || 1231219089Spjd ddb->ddb_class > scn->scn_phys.scn_ddt_class_max); 1232219089Spjd} 1233219089Spjd 1234219089Spjd/* ARGSUSED */ 1235219089Spjdvoid 1236219089Spjddsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum, 1237219089Spjd ddt_entry_t *dde, dmu_tx_t *tx) 1238219089Spjd{ 1239219089Spjd const ddt_key_t *ddk = &dde->dde_key; 1240219089Spjd ddt_phys_t *ddp = dde->dde_phys; 1241219089Spjd blkptr_t bp; 1242268657Sdelphij zbookmark_phys_t zb = { 0 }; 1243219089Spjd 1244219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 1245219089Spjd return; 1246219089Spjd 1247219089Spjd for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 1248219089Spjd if (ddp->ddp_phys_birth == 0 || 1249254112Sdelphij ddp->ddp_phys_birth > scn->scn_phys.scn_max_txg) 1250219089Spjd continue; 1251219089Spjd ddt_bp_create(checksum, ddk, ddp, &bp); 1252219089Spjd 1253219089Spjd scn->scn_visited_this_txg++; 1254219089Spjd scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb); 1255219089Spjd } 1256219089Spjd} 1257219089Spjd 1258219089Spjdstatic void 1259219089Spjddsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx) 1260219089Spjd{ 1261219089Spjd dsl_pool_t *dp = scn->scn_dp; 1262219089Spjd zap_cursor_t zc; 1263219089Spjd zap_attribute_t za; 1264219089Spjd 1265219089Spjd if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 1266219089Spjd scn->scn_phys.scn_ddt_class_max) { 1267219089Spjd scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 1268219089Spjd scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 1269219089Spjd dsl_scan_ddt(scn, tx); 1270219089Spjd if (scn->scn_pausing) 1271219089Spjd return; 1272219089Spjd } 1273219089Spjd 1274219089Spjd if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) { 1275219089Spjd /* First do the MOS & ORIGIN */ 1276219089Spjd 1277219089Spjd scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg; 1278219089Spjd scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg; 1279219089Spjd dsl_scan_visit_rootbp(scn, NULL, 1280219089Spjd &dp->dp_meta_rootbp, tx); 1281219089Spjd spa_set_rootblkptr(dp->dp_spa, &dp->dp_meta_rootbp); 1282219089Spjd if (scn->scn_pausing) 1283219089Spjd return; 1284219089Spjd 1285219089Spjd if (spa_version(dp->dp_spa) < SPA_VERSION_DSL_SCRUB) { 1286248571Smm VERIFY0(dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 1287248571Smm enqueue_cb, tx, DS_FIND_CHILDREN)); 1288219089Spjd } else { 1289219089Spjd dsl_scan_visitds(scn, 1290219089Spjd dp->dp_origin_snap->ds_object, tx); 1291219089Spjd } 1292219089Spjd ASSERT(!scn->scn_pausing); 1293219089Spjd } else if (scn->scn_phys.scn_bookmark.zb_objset != 1294219089Spjd ZB_DESTROYED_OBJSET) { 1295219089Spjd /* 1296219089Spjd * If we were paused, continue from here. Note if the 1297219089Spjd * ds we were paused on was deleted, the zb_objset may 1298219089Spjd * be -1, so we will skip this and find a new objset 1299219089Spjd * below. 1300219089Spjd */ 1301219089Spjd dsl_scan_visitds(scn, scn->scn_phys.scn_bookmark.zb_objset, tx); 1302219089Spjd if (scn->scn_pausing) 1303219089Spjd return; 1304219089Spjd } 1305219089Spjd 1306219089Spjd /* 1307219089Spjd * In case we were paused right at the end of the ds, zero the 1308219089Spjd * bookmark so we don't think that we're still trying to resume. 1309219089Spjd */ 1310268657Sdelphij bzero(&scn->scn_phys.scn_bookmark, sizeof (zbookmark_phys_t)); 1311219089Spjd 1312219089Spjd /* keep pulling things out of the zap-object-as-queue */ 1313219089Spjd while (zap_cursor_init(&zc, dp->dp_meta_objset, 1314219089Spjd scn->scn_phys.scn_queue_obj), 1315219089Spjd zap_cursor_retrieve(&zc, &za) == 0) { 1316219089Spjd dsl_dataset_t *ds; 1317219089Spjd uint64_t dsobj; 1318219089Spjd 1319219089Spjd dsobj = strtonum(za.za_name, NULL); 1320219089Spjd VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset, 1321219089Spjd scn->scn_phys.scn_queue_obj, dsobj, tx)); 1322219089Spjd 1323219089Spjd /* Set up min/max txg */ 1324219089Spjd VERIFY3U(0, ==, dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds)); 1325219089Spjd if (za.za_first_integer != 0) { 1326219089Spjd scn->scn_phys.scn_cur_min_txg = 1327219089Spjd MAX(scn->scn_phys.scn_min_txg, 1328219089Spjd za.za_first_integer); 1329219089Spjd } else { 1330219089Spjd scn->scn_phys.scn_cur_min_txg = 1331219089Spjd MAX(scn->scn_phys.scn_min_txg, 1332219089Spjd ds->ds_phys->ds_prev_snap_txg); 1333219089Spjd } 1334219089Spjd scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds); 1335219089Spjd dsl_dataset_rele(ds, FTAG); 1336219089Spjd 1337219089Spjd dsl_scan_visitds(scn, dsobj, tx); 1338219089Spjd zap_cursor_fini(&zc); 1339219089Spjd if (scn->scn_pausing) 1340219089Spjd return; 1341219089Spjd } 1342219089Spjd zap_cursor_fini(&zc); 1343219089Spjd} 1344219089Spjd 1345236884Smmstatic boolean_t 1346236884Smmdsl_scan_free_should_pause(dsl_scan_t *scn) 1347219089Spjd{ 1348219089Spjd uint64_t elapsed_nanosecs; 1349219089Spjd 1350262120Savg if (zfs_recover) 1351262120Savg return (B_FALSE); 1352262120Savg 1353219089Spjd elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time; 1354236884Smm return (elapsed_nanosecs / NANOSEC > zfs_txg_timeout || 1355255437Sdelphij (NSEC2MSEC(elapsed_nanosecs) > zfs_free_min_time_ms && 1356219089Spjd txg_sync_waiting(scn->scn_dp)) || 1357236884Smm spa_shutting_down(scn->scn_dp->dp_spa)); 1358236884Smm} 1359219089Spjd 1360236884Smmstatic int 1361236884Smmdsl_scan_free_block_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 1362236884Smm{ 1363236884Smm dsl_scan_t *scn = arg; 1364236884Smm 1365236884Smm if (!scn->scn_is_bptree || 1366236884Smm (BP_GET_LEVEL(bp) == 0 && BP_GET_TYPE(bp) != DMU_OT_OBJSET)) { 1367236884Smm if (dsl_scan_free_should_pause(scn)) 1368249195Smm return (SET_ERROR(ERESTART)); 1369236884Smm } 1370236884Smm 1371219089Spjd zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa, 1372240868Spjd dmu_tx_get_txg(tx), bp, BP_GET_PSIZE(bp), 0)); 1373219089Spjd dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 1374219089Spjd -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp), 1375219089Spjd -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 1376219089Spjd scn->scn_visited_this_txg++; 1377219089Spjd return (0); 1378219089Spjd} 1379219089Spjd 1380219089Spjdboolean_t 1381219089Spjddsl_scan_active(dsl_scan_t *scn) 1382219089Spjd{ 1383219089Spjd spa_t *spa = scn->scn_dp->dp_spa; 1384219089Spjd uint64_t used = 0, comp, uncomp; 1385219089Spjd 1386219089Spjd if (spa->spa_load_state != SPA_LOAD_NONE) 1387219089Spjd return (B_FALSE); 1388219089Spjd if (spa_shutting_down(spa)) 1389219089Spjd return (B_FALSE); 1390249858Smm if (scn->scn_phys.scn_state == DSS_SCANNING || 1391268650Sdelphij (scn->scn_async_destroying && !scn->scn_async_stalled)) 1392219089Spjd return (B_TRUE); 1393219089Spjd 1394219089Spjd if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1395219089Spjd (void) bpobj_space(&scn->scn_dp->dp_free_bpobj, 1396219089Spjd &used, &comp, &uncomp); 1397219089Spjd } 1398219089Spjd return (used != 0); 1399219089Spjd} 1400219089Spjd 1401219089Spjdvoid 1402219089Spjddsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) 1403219089Spjd{ 1404219089Spjd dsl_scan_t *scn = dp->dp_scan; 1405219089Spjd spa_t *spa = dp->dp_spa; 1406268650Sdelphij int err = 0; 1407219089Spjd 1408219089Spjd /* 1409219089Spjd * Check for scn_restart_txg before checking spa_load_state, so 1410219089Spjd * that we can restart an old-style scan while the pool is being 1411219089Spjd * imported (see dsl_scan_init). 1412219089Spjd */ 1413219089Spjd if (scn->scn_restart_txg != 0 && 1414219089Spjd scn->scn_restart_txg <= tx->tx_txg) { 1415219089Spjd pool_scan_func_t func = POOL_SCAN_SCRUB; 1416219089Spjd dsl_scan_done(scn, B_FALSE, tx); 1417219089Spjd if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) 1418219089Spjd func = POOL_SCAN_RESILVER; 1419219089Spjd zfs_dbgmsg("restarting scan func=%u txg=%llu", 1420219089Spjd func, tx->tx_txg); 1421248571Smm dsl_scan_setup_sync(&func, tx); 1422219089Spjd } 1423219089Spjd 1424268650Sdelphij /* 1425268650Sdelphij * If the scan is inactive due to a stalled async destroy, try again. 1426268650Sdelphij */ 1427268650Sdelphij if ((!scn->scn_async_stalled && !dsl_scan_active(scn)) || 1428219089Spjd spa_sync_pass(dp->dp_spa) > 1) 1429219089Spjd return; 1430219089Spjd 1431219089Spjd scn->scn_visited_this_txg = 0; 1432219089Spjd scn->scn_pausing = B_FALSE; 1433219089Spjd scn->scn_sync_start_time = gethrtime(); 1434219089Spjd spa->spa_scrub_active = B_TRUE; 1435219089Spjd 1436219089Spjd /* 1437268650Sdelphij * First process the async destroys. If we pause, don't do 1438268650Sdelphij * any scrubbing or resilvering. This ensures that there are no 1439268650Sdelphij * async destroys while we are scanning, so the scan code doesn't 1440268650Sdelphij * have to worry about traversing it. It is also faster to free the 1441268650Sdelphij * blocks than to scrub them. 1442219089Spjd */ 1443219089Spjd if (spa_version(dp->dp_spa) >= SPA_VERSION_DEADLISTS) { 1444236884Smm scn->scn_is_bptree = B_FALSE; 1445219089Spjd scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1446219089Spjd NULL, ZIO_FLAG_MUSTSUCCEED); 1447219089Spjd err = bpobj_iterate(&dp->dp_free_bpobj, 1448236884Smm dsl_scan_free_block_cb, scn, tx); 1449219089Spjd VERIFY3U(0, ==, zio_wait(scn->scn_zio_root)); 1450236884Smm 1451268650Sdelphij if (err != 0 && err != ERESTART) 1452268650Sdelphij zfs_panic_recover("error %u from bpobj_iterate()", err); 1453268650Sdelphij } 1454236884Smm 1455268650Sdelphij if (err == 0 && spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY)) { 1456268650Sdelphij ASSERT(scn->scn_async_destroying); 1457268650Sdelphij scn->scn_is_bptree = B_TRUE; 1458268650Sdelphij scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1459268650Sdelphij NULL, ZIO_FLAG_MUSTSUCCEED); 1460268650Sdelphij err = bptree_iterate(dp->dp_meta_objset, 1461268650Sdelphij dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx); 1462268650Sdelphij VERIFY0(zio_wait(scn->scn_zio_root)); 1463268650Sdelphij 1464268650Sdelphij if (err == EIO || err == ECKSUM) { 1465268650Sdelphij err = 0; 1466268650Sdelphij } else if (err != 0 && err != ERESTART) { 1467268650Sdelphij zfs_panic_recover("error %u from " 1468268650Sdelphij "traverse_dataset_destroyed()", err); 1469236884Smm } 1470268650Sdelphij 1471268650Sdelphij if (bptree_is_empty(dp->dp_meta_objset, dp->dp_bptree_obj)) { 1472268650Sdelphij /* finished; deactivate async destroy feature */ 1473268650Sdelphij spa_feature_decr(spa, SPA_FEATURE_ASYNC_DESTROY, tx); 1474268650Sdelphij ASSERT(!spa_feature_is_active(spa, 1475268650Sdelphij SPA_FEATURE_ASYNC_DESTROY)); 1476268650Sdelphij VERIFY0(zap_remove(dp->dp_meta_objset, 1477268650Sdelphij DMU_POOL_DIRECTORY_OBJECT, 1478268650Sdelphij DMU_POOL_BPTREE_OBJ, tx)); 1479268650Sdelphij VERIFY0(bptree_free(dp->dp_meta_objset, 1480268650Sdelphij dp->dp_bptree_obj, tx)); 1481268650Sdelphij dp->dp_bptree_obj = 0; 1482268650Sdelphij scn->scn_async_destroying = B_FALSE; 1483273215Ssmh } else { 1484273215Ssmh /* 1485273215Ssmh * If we didn't make progress, mark the async destroy as 1486273215Ssmh * stalled, so that we will not initiate a spa_sync() on 1487273215Ssmh * its behalf. 1488273215Ssmh */ 1489273215Ssmh scn->scn_async_stalled = 1490273215Ssmh (scn->scn_visited_this_txg == 0); 1491219089Spjd } 1492268650Sdelphij } 1493268650Sdelphij if (scn->scn_visited_this_txg) { 1494268650Sdelphij zfs_dbgmsg("freed %llu blocks in %llums from " 1495268650Sdelphij "free_bpobj/bptree txg %llu; err=%u", 1496268650Sdelphij (longlong_t)scn->scn_visited_this_txg, 1497268650Sdelphij (longlong_t) 1498268650Sdelphij NSEC2MSEC(gethrtime() - scn->scn_sync_start_time), 1499268650Sdelphij (longlong_t)tx->tx_txg, err); 1500268650Sdelphij scn->scn_visited_this_txg = 0; 1501268650Sdelphij 1502268650Sdelphij /* 1503268650Sdelphij * Write out changes to the DDT that may be required as a 1504268650Sdelphij * result of the blocks freed. This ensures that the DDT 1505268650Sdelphij * is clean when a scrub/resilver runs. 1506268650Sdelphij */ 1507268650Sdelphij ddt_sync(spa, tx->tx_txg); 1508268650Sdelphij } 1509268650Sdelphij if (err != 0) 1510268650Sdelphij return; 1511268650Sdelphij if (!scn->scn_async_destroying && zfs_free_leak_on_eio && 1512268650Sdelphij (dp->dp_free_dir->dd_phys->dd_used_bytes != 0 || 1513268650Sdelphij dp->dp_free_dir->dd_phys->dd_compressed_bytes != 0 || 1514268650Sdelphij dp->dp_free_dir->dd_phys->dd_uncompressed_bytes != 0)) { 1515268650Sdelphij /* 1516268650Sdelphij * We have finished background destroying, but there is still 1517268650Sdelphij * some space left in the dp_free_dir. Transfer this leaked 1518268650Sdelphij * space to the dp_leak_dir. 1519268650Sdelphij */ 1520268650Sdelphij if (dp->dp_leak_dir == NULL) { 1521268650Sdelphij rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 1522268650Sdelphij (void) dsl_dir_create_sync(dp, dp->dp_root_dir, 1523268650Sdelphij LEAK_DIR_NAME, tx); 1524268650Sdelphij VERIFY0(dsl_pool_open_special_dir(dp, 1525268650Sdelphij LEAK_DIR_NAME, &dp->dp_leak_dir)); 1526268650Sdelphij rrw_exit(&dp->dp_config_rwlock, FTAG); 1527268650Sdelphij } 1528268650Sdelphij dsl_dir_diduse_space(dp->dp_leak_dir, DD_USED_HEAD, 1529268650Sdelphij dp->dp_free_dir->dd_phys->dd_used_bytes, 1530268650Sdelphij dp->dp_free_dir->dd_phys->dd_compressed_bytes, 1531268650Sdelphij dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx); 1532268650Sdelphij dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD, 1533268650Sdelphij -dp->dp_free_dir->dd_phys->dd_used_bytes, 1534268650Sdelphij -dp->dp_free_dir->dd_phys->dd_compressed_bytes, 1535268650Sdelphij -dp->dp_free_dir->dd_phys->dd_uncompressed_bytes, tx); 1536268650Sdelphij } 1537268650Sdelphij if (!scn->scn_async_destroying) { 1538268649Sdelphij /* finished; verify that space accounting went to zero */ 1539268649Sdelphij ASSERT0(dp->dp_free_dir->dd_phys->dd_used_bytes); 1540268649Sdelphij ASSERT0(dp->dp_free_dir->dd_phys->dd_compressed_bytes); 1541268649Sdelphij ASSERT0(dp->dp_free_dir->dd_phys->dd_uncompressed_bytes); 1542219089Spjd } 1543219089Spjd 1544219089Spjd if (scn->scn_phys.scn_state != DSS_SCANNING) 1545219089Spjd return; 1546219089Spjd 1547254112Sdelphij if (scn->scn_done_txg == tx->tx_txg) { 1548254112Sdelphij ASSERT(!scn->scn_pausing); 1549254112Sdelphij /* finished with scan. */ 1550254112Sdelphij zfs_dbgmsg("txg %llu scan complete", tx->tx_txg); 1551254112Sdelphij dsl_scan_done(scn, B_TRUE, tx); 1552254112Sdelphij ASSERT3U(spa->spa_scrub_inflight, ==, 0); 1553254112Sdelphij dsl_scan_sync_state(scn, tx); 1554254112Sdelphij return; 1555254112Sdelphij } 1556254112Sdelphij 1557219089Spjd if (scn->scn_phys.scn_ddt_bookmark.ddb_class <= 1558219089Spjd scn->scn_phys.scn_ddt_class_max) { 1559219089Spjd zfs_dbgmsg("doing scan sync txg %llu; " 1560219089Spjd "ddt bm=%llu/%llu/%llu/%llx", 1561219089Spjd (longlong_t)tx->tx_txg, 1562219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_class, 1563219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_type, 1564219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_checksum, 1565219089Spjd (longlong_t)scn->scn_phys.scn_ddt_bookmark.ddb_cursor); 1566219089Spjd ASSERT(scn->scn_phys.scn_bookmark.zb_objset == 0); 1567219089Spjd ASSERT(scn->scn_phys.scn_bookmark.zb_object == 0); 1568219089Spjd ASSERT(scn->scn_phys.scn_bookmark.zb_level == 0); 1569219089Spjd ASSERT(scn->scn_phys.scn_bookmark.zb_blkid == 0); 1570219089Spjd } else { 1571219089Spjd zfs_dbgmsg("doing scan sync txg %llu; bm=%llu/%llu/%llu/%llu", 1572219089Spjd (longlong_t)tx->tx_txg, 1573219089Spjd (longlong_t)scn->scn_phys.scn_bookmark.zb_objset, 1574219089Spjd (longlong_t)scn->scn_phys.scn_bookmark.zb_object, 1575219089Spjd (longlong_t)scn->scn_phys.scn_bookmark.zb_level, 1576219089Spjd (longlong_t)scn->scn_phys.scn_bookmark.zb_blkid); 1577219089Spjd } 1578219089Spjd 1579219089Spjd scn->scn_zio_root = zio_root(dp->dp_spa, NULL, 1580219089Spjd NULL, ZIO_FLAG_CANFAIL); 1581248571Smm dsl_pool_config_enter(dp, FTAG); 1582219089Spjd dsl_scan_visit(scn, tx); 1583248571Smm dsl_pool_config_exit(dp, FTAG); 1584219089Spjd (void) zio_wait(scn->scn_zio_root); 1585219089Spjd scn->scn_zio_root = NULL; 1586219089Spjd 1587219089Spjd zfs_dbgmsg("visited %llu blocks in %llums", 1588219089Spjd (longlong_t)scn->scn_visited_this_txg, 1589255437Sdelphij (longlong_t)NSEC2MSEC(gethrtime() - scn->scn_sync_start_time)); 1590219089Spjd 1591219089Spjd if (!scn->scn_pausing) { 1592254112Sdelphij scn->scn_done_txg = tx->tx_txg + 1; 1593254112Sdelphij zfs_dbgmsg("txg %llu traversal complete, waiting till txg %llu", 1594254112Sdelphij tx->tx_txg, scn->scn_done_txg); 1595219089Spjd } 1596219089Spjd 1597219089Spjd if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { 1598219089Spjd mutex_enter(&spa->spa_scrub_lock); 1599219089Spjd while (spa->spa_scrub_inflight > 0) { 1600219089Spjd cv_wait(&spa->spa_scrub_io_cv, 1601219089Spjd &spa->spa_scrub_lock); 1602219089Spjd } 1603219089Spjd mutex_exit(&spa->spa_scrub_lock); 1604219089Spjd } 1605219089Spjd 1606219089Spjd dsl_scan_sync_state(scn, tx); 1607219089Spjd} 1608219089Spjd 1609219089Spjd/* 1610219089Spjd * This will start a new scan, or restart an existing one. 1611219089Spjd */ 1612219089Spjdvoid 1613219089Spjddsl_resilver_restart(dsl_pool_t *dp, uint64_t txg) 1614219089Spjd{ 1615219089Spjd if (txg == 0) { 1616219089Spjd dmu_tx_t *tx; 1617219089Spjd tx = dmu_tx_create_dd(dp->dp_mos_dir); 1618219089Spjd VERIFY(0 == dmu_tx_assign(tx, TXG_WAIT)); 1619219089Spjd 1620219089Spjd txg = dmu_tx_get_txg(tx); 1621219089Spjd dp->dp_scan->scn_restart_txg = txg; 1622219089Spjd dmu_tx_commit(tx); 1623219089Spjd } else { 1624219089Spjd dp->dp_scan->scn_restart_txg = txg; 1625219089Spjd } 1626219089Spjd zfs_dbgmsg("restarting resilver txg=%llu", txg); 1627219089Spjd} 1628219089Spjd 1629219089Spjdboolean_t 1630219089Spjddsl_scan_resilvering(dsl_pool_t *dp) 1631219089Spjd{ 1632219089Spjd return (dp->dp_scan->scn_phys.scn_state == DSS_SCANNING && 1633219089Spjd dp->dp_scan->scn_phys.scn_func == POOL_SCAN_RESILVER); 1634219089Spjd} 1635219089Spjd 1636219089Spjd/* 1637219089Spjd * scrub consumers 1638219089Spjd */ 1639219089Spjd 1640219089Spjdstatic void 1641219089Spjdcount_block(zfs_all_blkstats_t *zab, const blkptr_t *bp) 1642219089Spjd{ 1643219089Spjd int i; 1644219089Spjd 1645219089Spjd /* 1646219089Spjd * If we resume after a reboot, zab will be NULL; don't record 1647219089Spjd * incomplete stats in that case. 1648219089Spjd */ 1649219089Spjd if (zab == NULL) 1650219089Spjd return; 1651219089Spjd 1652219089Spjd for (i = 0; i < 4; i++) { 1653219089Spjd int l = (i < 2) ? BP_GET_LEVEL(bp) : DN_MAX_LEVELS; 1654219089Spjd int t = (i & 1) ? BP_GET_TYPE(bp) : DMU_OT_TOTAL; 1655236884Smm if (t & DMU_OT_NEWTYPE) 1656236884Smm t = DMU_OT_OTHER; 1657219089Spjd zfs_blkstat_t *zb = &zab->zab_type[l][t]; 1658219089Spjd int equal; 1659219089Spjd 1660219089Spjd zb->zb_count++; 1661219089Spjd zb->zb_asize += BP_GET_ASIZE(bp); 1662219089Spjd zb->zb_lsize += BP_GET_LSIZE(bp); 1663219089Spjd zb->zb_psize += BP_GET_PSIZE(bp); 1664219089Spjd zb->zb_gangs += BP_COUNT_GANG(bp); 1665219089Spjd 1666219089Spjd switch (BP_GET_NDVAS(bp)) { 1667219089Spjd case 2: 1668219089Spjd if (DVA_GET_VDEV(&bp->blk_dva[0]) == 1669219089Spjd DVA_GET_VDEV(&bp->blk_dva[1])) 1670219089Spjd zb->zb_ditto_2_of_2_samevdev++; 1671219089Spjd break; 1672219089Spjd case 3: 1673219089Spjd equal = (DVA_GET_VDEV(&bp->blk_dva[0]) == 1674219089Spjd DVA_GET_VDEV(&bp->blk_dva[1])) + 1675219089Spjd (DVA_GET_VDEV(&bp->blk_dva[0]) == 1676219089Spjd DVA_GET_VDEV(&bp->blk_dva[2])) + 1677219089Spjd (DVA_GET_VDEV(&bp->blk_dva[1]) == 1678219089Spjd DVA_GET_VDEV(&bp->blk_dva[2])); 1679219089Spjd if (equal == 1) 1680219089Spjd zb->zb_ditto_2_of_3_samevdev++; 1681219089Spjd else if (equal == 3) 1682219089Spjd zb->zb_ditto_3_of_3_samevdev++; 1683219089Spjd break; 1684219089Spjd } 1685219089Spjd } 1686219089Spjd} 1687219089Spjd 1688219089Spjdstatic void 1689219089Spjddsl_scan_scrub_done(zio_t *zio) 1690219089Spjd{ 1691219089Spjd spa_t *spa = zio->io_spa; 1692219089Spjd 1693219089Spjd zio_data_buf_free(zio->io_data, zio->io_size); 1694219089Spjd 1695219089Spjd mutex_enter(&spa->spa_scrub_lock); 1696219089Spjd spa->spa_scrub_inflight--; 1697219089Spjd cv_broadcast(&spa->spa_scrub_io_cv); 1698219089Spjd 1699219089Spjd if (zio->io_error && (zio->io_error != ECKSUM || 1700219089Spjd !(zio->io_flags & ZIO_FLAG_SPECULATIVE))) { 1701219089Spjd spa->spa_dsl_pool->dp_scan->scn_phys.scn_errors++; 1702219089Spjd } 1703219089Spjd mutex_exit(&spa->spa_scrub_lock); 1704219089Spjd} 1705219089Spjd 1706219089Spjdstatic int 1707219089Spjddsl_scan_scrub_cb(dsl_pool_t *dp, 1708268657Sdelphij const blkptr_t *bp, const zbookmark_phys_t *zb) 1709219089Spjd{ 1710219089Spjd dsl_scan_t *scn = dp->dp_scan; 1711219089Spjd size_t size = BP_GET_PSIZE(bp); 1712219089Spjd spa_t *spa = dp->dp_spa; 1713219089Spjd uint64_t phys_birth = BP_PHYSICAL_BIRTH(bp); 1714219089Spjd boolean_t needs_io; 1715219089Spjd int zio_flags = ZIO_FLAG_SCAN_THREAD | ZIO_FLAG_RAW | ZIO_FLAG_CANFAIL; 1716237972Smm unsigned int scan_delay = 0; 1717219089Spjd 1718219089Spjd if (phys_birth <= scn->scn_phys.scn_min_txg || 1719219089Spjd phys_birth >= scn->scn_phys.scn_max_txg) 1720219089Spjd return (0); 1721219089Spjd 1722219089Spjd count_block(dp->dp_blkstats, bp); 1723219089Spjd 1724268649Sdelphij if (BP_IS_EMBEDDED(bp)) 1725268649Sdelphij return (0); 1726268649Sdelphij 1727219089Spjd ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn)); 1728219089Spjd if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) { 1729219089Spjd zio_flags |= ZIO_FLAG_SCRUB; 1730219089Spjd needs_io = B_TRUE; 1731219089Spjd scan_delay = zfs_scrub_delay; 1732247187Smm } else { 1733247187Smm ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER); 1734219089Spjd zio_flags |= ZIO_FLAG_RESILVER; 1735219089Spjd needs_io = B_FALSE; 1736219089Spjd scan_delay = zfs_resilver_delay; 1737219089Spjd } 1738219089Spjd 1739219089Spjd /* If it's an intent log block, failure is expected. */ 1740219089Spjd if (zb->zb_level == ZB_ZIL_LEVEL) 1741219089Spjd zio_flags |= ZIO_FLAG_SPECULATIVE; 1742219089Spjd 1743219089Spjd for (int d = 0; d < BP_GET_NDVAS(bp); d++) { 1744219089Spjd vdev_t *vd = vdev_lookup_top(spa, 1745219089Spjd DVA_GET_VDEV(&bp->blk_dva[d])); 1746219089Spjd 1747219089Spjd /* 1748219089Spjd * Keep track of how much data we've examined so that 1749219089Spjd * zpool(1M) status can make useful progress reports. 1750219089Spjd */ 1751219089Spjd scn->scn_phys.scn_examined += DVA_GET_ASIZE(&bp->blk_dva[d]); 1752219089Spjd spa->spa_scan_pass_exam += DVA_GET_ASIZE(&bp->blk_dva[d]); 1753219089Spjd 1754219089Spjd /* if it's a resilver, this may not be in the target range */ 1755219089Spjd if (!needs_io) { 1756219089Spjd if (DVA_GET_GANG(&bp->blk_dva[d])) { 1757219089Spjd /* 1758219089Spjd * Gang members may be spread across multiple 1759219089Spjd * vdevs, so the best estimate we have is the 1760219089Spjd * scrub range, which has already been checked. 1761219089Spjd * XXX -- it would be better to change our 1762219089Spjd * allocation policy to ensure that all 1763219089Spjd * gang members reside on the same vdev. 1764219089Spjd */ 1765219089Spjd needs_io = B_TRUE; 1766219089Spjd } else { 1767219089Spjd needs_io = vdev_dtl_contains(vd, DTL_PARTIAL, 1768219089Spjd phys_birth, 1); 1769219089Spjd } 1770219089Spjd } 1771219089Spjd } 1772219089Spjd 1773219089Spjd if (needs_io && !zfs_no_scrub_io) { 1774219089Spjd vdev_t *rvd = spa->spa_root_vdev; 1775237972Smm uint64_t maxinflight = rvd->vdev_children * 1776237972Smm MAX(zfs_top_maxinflight, 1); 1777219089Spjd void *data = zio_data_buf_alloc(size); 1778219089Spjd 1779219089Spjd mutex_enter(&spa->spa_scrub_lock); 1780219089Spjd while (spa->spa_scrub_inflight >= maxinflight) 1781219089Spjd cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 1782219089Spjd spa->spa_scrub_inflight++; 1783219089Spjd mutex_exit(&spa->spa_scrub_lock); 1784219089Spjd 1785219089Spjd /* 1786219089Spjd * If we're seeing recent (zfs_scan_idle) "important" I/Os 1787219089Spjd * then throttle our workload to limit the impact of a scan. 1788219089Spjd */ 1789219089Spjd if (ddi_get_lbolt64() - spa->spa_last_io <= zfs_scan_idle) 1790237972Smm delay(MAX((int)scan_delay, 0)); 1791219089Spjd 1792219089Spjd zio_nowait(zio_read(NULL, spa, bp, data, size, 1793260763Savg dsl_scan_scrub_done, NULL, ZIO_PRIORITY_SCRUB, 1794219089Spjd zio_flags, zb)); 1795219089Spjd } 1796219089Spjd 1797219089Spjd /* do not relocate this block */ 1798219089Spjd return (0); 1799219089Spjd} 1800219089Spjd 1801219089Spjdint 1802219089Spjddsl_scan(dsl_pool_t *dp, pool_scan_func_t func) 1803219089Spjd{ 1804219089Spjd spa_t *spa = dp->dp_spa; 1805219089Spjd 1806219089Spjd /* 1807219089Spjd * Purge all vdev caches and probe all devices. We do this here 1808219089Spjd * rather than in sync context because this requires a writer lock 1809219089Spjd * on the spa_config lock, which we can't do from sync context. The 1810219089Spjd * spa_scrub_reopen flag indicates that vdev_open() should not 1811219089Spjd * attempt to start another scrub. 1812219089Spjd */ 1813219089Spjd spa_vdev_state_enter(spa, SCL_NONE); 1814219089Spjd spa->spa_scrub_reopen = B_TRUE; 1815219089Spjd vdev_reopen(spa->spa_root_vdev); 1816219089Spjd spa->spa_scrub_reopen = B_FALSE; 1817219089Spjd (void) spa_vdev_state_exit(spa, NULL, 0); 1818219089Spjd 1819248571Smm return (dsl_sync_task(spa_name(spa), dsl_scan_setup_check, 1820269006Sdelphij dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_NONE)); 1821219089Spjd} 1822