1/* 2 * CDDL HEADER START 3 * 4 * This file and its contents are supplied under the terms of the 5 * Common Development and Distribution License ("CDDL"), version 1.0. 6 * You may only use this file in accordance with the terms of version 7 * 1.0 of the CDDL. 8 * 9 * A full copy of the text of the CDDL should have accompanied this 10 * source. A copy of the CDDL is also available via the Internet at 11 * http://www.illumos.org/license/CDDL. 12 * 13 * CDDL HEADER END 14 */ 15 16/* 17 * Copyright (c) 2014, 2017 by Delphix. All rights reserved. 18 */ 19 20#include <sys/zfs_context.h> 21#include <sys/spa.h> 22#include <sys/spa_impl.h> 23#include <sys/vdev_impl.h> 24#include <sys/fs/zfs.h> 25#include <sys/zio.h> 26#include <sys/zio_checksum.h> 27#include <sys/metaslab.h> 28#include <sys/refcount.h> 29#include <sys/dmu.h> 30#include <sys/vdev_indirect_mapping.h> 31#include <sys/dmu_tx.h> 32#include <sys/dsl_synctask.h> 33#include <sys/zap.h> 34#include <sys/abd.h> 35#include <sys/zthr.h> 36 37/* 38 * An indirect vdev corresponds to a vdev that has been removed. Since 39 * we cannot rewrite block pointers of snapshots, etc., we keep a 40 * mapping from old location on the removed device to the new location 41 * on another device in the pool and use this mapping whenever we need 42 * to access the DVA. Unfortunately, this mapping did not respect 43 * logical block boundaries when it was first created, and so a DVA on 44 * this indirect vdev may be "split" into multiple sections that each 45 * map to a different location. As a consequence, not all DVAs can be 46 * translated to an equivalent new DVA. Instead we must provide a 47 * "vdev_remap" operation that executes a callback on each contiguous 48 * segment of the new location. This function is used in multiple ways: 49 * 50 * - i/os to this vdev use the callback to determine where the 51 * data is now located, and issue child i/os for each segment's new 52 * location. 53 * 54 * - frees and claims to this vdev use the callback to free or claim 55 * each mapped segment. (Note that we don't actually need to claim 56 * log blocks on indirect vdevs, because we don't allocate to 57 * removing vdevs. However, zdb uses zio_claim() for its leak 58 * detection.) 59 */ 60 61/* 62 * "Big theory statement" for how we mark blocks obsolete. 63 * 64 * When a block on an indirect vdev is freed or remapped, a section of 65 * that vdev's mapping may no longer be referenced (aka "obsolete"). We 66 * keep track of how much of each mapping entry is obsolete. When 67 * an entry becomes completely obsolete, we can remove it, thus reducing 68 * the memory used by the mapping. The complete picture of obsolescence 69 * is given by the following data structures, described below: 70 * - the entry-specific obsolete count 71 * - the vdev-specific obsolete spacemap 72 * - the pool-specific obsolete bpobj 73 * 74 * == On disk data structures used == 75 * 76 * We track the obsolete space for the pool using several objects. Each 77 * of these objects is created on demand and freed when no longer 78 * needed, and is assumed to be empty if it does not exist. 79 * SPA_FEATURE_OBSOLETE_COUNTS includes the count of these objects. 80 * 81 * - Each vic_mapping_object (associated with an indirect vdev) can 82 * have a vimp_counts_object. This is an array of uint32_t's 83 * with the same number of entries as the vic_mapping_object. When 84 * the mapping is condensed, entries from the vic_obsolete_sm_object 85 * (see below) are folded into the counts. Therefore, each 86 * obsolete_counts entry tells us the number of bytes in the 87 * corresponding mapping entry that were not referenced when the 88 * mapping was last condensed. 89 * 90 * - Each indirect or removing vdev can have a vic_obsolete_sm_object. 91 * This is a space map containing an alloc entry for every DVA that 92 * has been obsoleted since the last time this indirect vdev was 93 * condensed. We use this object in order to improve performance 94 * when marking a DVA as obsolete. Instead of modifying an arbitrary 95 * offset of the vimp_counts_object, we only need to append an entry 96 * to the end of this object. When a DVA becomes obsolete, it is 97 * added to the obsolete space map. This happens when the DVA is 98 * freed, remapped and not referenced by a snapshot, or the last 99 * snapshot referencing it is destroyed. 100 * 101 * - Each dataset can have a ds_remap_deadlist object. This is a 102 * deadlist object containing all blocks that were remapped in this 103 * dataset but referenced in a previous snapshot. Blocks can *only* 104 * appear on this list if they were remapped (dsl_dataset_block_remapped); 105 * blocks that were killed in a head dataset are put on the normal 106 * ds_deadlist and marked obsolete when they are freed. 107 * 108 * - The pool can have a dp_obsolete_bpobj. This is a list of blocks 109 * in the pool that need to be marked obsolete. When a snapshot is 110 * destroyed, we move some of the ds_remap_deadlist to the obsolete 111 * bpobj (see dsl_destroy_snapshot_handle_remaps()). We then 112 * asynchronously process the obsolete bpobj, moving its entries to 113 * the specific vdevs' obsolete space maps. 114 * 115 * == Summary of how we mark blocks as obsolete == 116 * 117 * - When freeing a block: if any DVA is on an indirect vdev, append to 118 * vic_obsolete_sm_object. 119 * - When remapping a block, add dva to ds_remap_deadlist (if prev snap 120 * references; otherwise append to vic_obsolete_sm_object). 121 * - When freeing a snapshot: move parts of ds_remap_deadlist to 122 * dp_obsolete_bpobj (same algorithm as ds_deadlist). 123 * - When syncing the spa: process dp_obsolete_bpobj, moving ranges to 124 * individual vdev's vic_obsolete_sm_object. 125 */ 126 127/* 128 * "Big theory statement" for how we condense indirect vdevs. 129 * 130 * Condensing an indirect vdev's mapping is the process of determining 131 * the precise counts of obsolete space for each mapping entry (by 132 * integrating the obsolete spacemap into the obsolete counts) and 133 * writing out a new mapping that contains only referenced entries. 134 * 135 * We condense a vdev when we expect the mapping to shrink (see 136 * vdev_indirect_should_condense()), but only perform one condense at a 137 * time to limit the memory usage. In addition, we use a separate 138 * open-context thread (spa_condense_indirect_thread) to incrementally 139 * create the new mapping object in a way that minimizes the impact on 140 * the rest of the system. 141 * 142 * == Generating a new mapping == 143 * 144 * To generate a new mapping, we follow these steps: 145 * 146 * 1. Save the old obsolete space map and create a new mapping object 147 * (see spa_condense_indirect_start_sync()). This initializes the 148 * spa_condensing_indirect_phys with the "previous obsolete space map", 149 * which is now read only. Newly obsolete DVAs will be added to a 150 * new (initially empty) obsolete space map, and will not be 151 * considered as part of this condense operation. 152 * 153 * 2. Construct in memory the precise counts of obsolete space for each 154 * mapping entry, by incorporating the obsolete space map into the 155 * counts. (See vdev_indirect_mapping_load_obsolete_{counts,spacemap}().) 156 * 157 * 3. Iterate through each mapping entry, writing to the new mapping any 158 * entries that are not completely obsolete (i.e. which don't have 159 * obsolete count == mapping length). (See 160 * spa_condense_indirect_generate_new_mapping().) 161 * 162 * 4. Destroy the old mapping object and switch over to the new one 163 * (spa_condense_indirect_complete_sync). 164 * 165 * == Restarting from failure == 166 * 167 * To restart the condense when we import/open the pool, we must start 168 * at the 2nd step above: reconstruct the precise counts in memory, 169 * based on the space map + counts. Then in the 3rd step, we start 170 * iterating where we left off: at vimp_max_offset of the new mapping 171 * object. 172 */ 173 174boolean_t zfs_condense_indirect_vdevs_enable = B_TRUE; 175 176/* 177 * Condense if at least this percent of the bytes in the mapping is 178 * obsolete. With the default of 25%, the amount of space mapped 179 * will be reduced to 1% of its original size after at most 16 180 * condenses. Higher values will condense less often (causing less 181 * i/o); lower values will reduce the mapping size more quickly. 182 */ 183int zfs_indirect_condense_obsolete_pct = 25; 184 185/* 186 * Condense if the obsolete space map takes up more than this amount of 187 * space on disk (logically). This limits the amount of disk space 188 * consumed by the obsolete space map; the default of 1GB is small enough 189 * that we typically don't mind "wasting" it. 190 */ 191uint64_t zfs_condense_max_obsolete_bytes = 1024 * 1024 * 1024; 192 193/* 194 * Don't bother condensing if the mapping uses less than this amount of 195 * memory. The default of 128KB is considered a "trivial" amount of 196 * memory and not worth reducing. 197 */ 198uint64_t zfs_condense_min_mapping_bytes = 128 * 1024; 199 200/* 201 * This is used by the test suite so that it can ensure that certain 202 * actions happen while in the middle of a condense (which might otherwise 203 * complete too quickly). If used to reduce the performance impact of 204 * condensing in production, a maximum value of 1 should be sufficient. 205 */ 206int zfs_condense_indirect_commit_entry_delay_ticks = 0; 207 208/* 209 * If an indirect split block contains more than this many possible unique 210 * combinations when being reconstructed, consider it too computationally 211 * expensive to check them all. Instead, try at most 100 randomly-selected 212 * combinations each time the block is accessed. This allows all segment 213 * copies to participate fairly in the reconstruction when all combinations 214 * cannot be checked and prevents repeated use of one bad copy. 215 */ 216int zfs_reconstruct_indirect_combinations_max = 256; 217 218 219/* 220 * Enable to simulate damaged segments and validate reconstruction. 221 * Used by ztest 222 */ 223unsigned long zfs_reconstruct_indirect_damage_fraction = 0; 224 225/* 226 * The indirect_child_t represents the vdev that we will read from, when we 227 * need to read all copies of the data (e.g. for scrub or reconstruction). 228 * For plain (non-mirror) top-level vdevs (i.e. is_vdev is not a mirror), 229 * ic_vdev is the same as is_vdev. However, for mirror top-level vdevs, 230 * ic_vdev is a child of the mirror. 231 */ 232typedef struct indirect_child { 233 abd_t *ic_data; 234 vdev_t *ic_vdev; 235 236 /* 237 * ic_duplicate is NULL when the ic_data contents are unique, when it 238 * is determined to be a duplicate it references the primary child. 239 */ 240 struct indirect_child *ic_duplicate; 241 list_node_t ic_node; /* node on is_unique_child */ 242} indirect_child_t; 243 244/* 245 * The indirect_split_t represents one mapped segment of an i/o to the 246 * indirect vdev. For non-split (contiguously-mapped) blocks, there will be 247 * only one indirect_split_t, with is_split_offset==0 and is_size==io_size. 248 * For split blocks, there will be several of these. 249 */ 250typedef struct indirect_split { 251 list_node_t is_node; /* link on iv_splits */ 252 253 /* 254 * is_split_offset is the offset into the i/o. 255 * This is the sum of the previous splits' is_size's. 256 */ 257 uint64_t is_split_offset; 258 259 vdev_t *is_vdev; /* top-level vdev */ 260 uint64_t is_target_offset; /* offset on is_vdev */ 261 uint64_t is_size; 262 int is_children; /* number of entries in is_child[] */ 263 int is_unique_children; /* number of entries in is_unique_child */ 264 list_t is_unique_child; 265 266 /* 267 * is_good_child is the child that we are currently using to 268 * attempt reconstruction. 269 */ 270 indirect_child_t *is_good_child; 271 272 indirect_child_t is_child[1]; /* variable-length */ 273} indirect_split_t; 274 275/* 276 * The indirect_vsd_t is associated with each i/o to the indirect vdev. 277 * It is the "Vdev-Specific Data" in the zio_t's io_vsd. 278 */ 279typedef struct indirect_vsd { 280 boolean_t iv_split_block; 281 boolean_t iv_reconstruct; 282 uint64_t iv_unique_combinations; 283 uint64_t iv_attempts; 284 uint64_t iv_attempts_max; 285 286 list_t iv_splits; /* list of indirect_split_t's */ 287} indirect_vsd_t; 288 289static void 290vdev_indirect_map_free(zio_t *zio) 291{ 292 indirect_vsd_t *iv = zio->io_vsd; 293 294 indirect_split_t *is; 295 while ((is = list_head(&iv->iv_splits)) != NULL) { 296 for (int c = 0; c < is->is_children; c++) { 297 indirect_child_t *ic = &is->is_child[c]; 298 if (ic->ic_data != NULL) 299 abd_free(ic->ic_data); 300 } 301 list_remove(&iv->iv_splits, is); 302 303 indirect_child_t *ic; 304 while ((ic = list_head(&is->is_unique_child)) != NULL) 305 list_remove(&is->is_unique_child, ic); 306 307 list_destroy(&is->is_unique_child); 308 309 kmem_free(is, 310 offsetof(indirect_split_t, is_child[is->is_children])); 311 } 312 kmem_free(iv, sizeof (*iv)); 313} 314 315static const zio_vsd_ops_t vdev_indirect_vsd_ops = { 316 vdev_indirect_map_free, 317 zio_vsd_default_cksum_report 318}; 319/* 320 * Mark the given offset and size as being obsolete. 321 */ 322void 323vdev_indirect_mark_obsolete(vdev_t *vd, uint64_t offset, uint64_t size) 324{ 325 spa_t *spa = vd->vdev_spa; 326 327 ASSERT3U(vd->vdev_indirect_config.vic_mapping_object, !=, 0); 328 ASSERT(vd->vdev_removing || vd->vdev_ops == &vdev_indirect_ops); 329 ASSERT(size > 0); 330 VERIFY(vdev_indirect_mapping_entry_for_offset( 331 vd->vdev_indirect_mapping, offset) != NULL); 332 333 if (spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)) { 334 mutex_enter(&vd->vdev_obsolete_lock); 335 range_tree_add(vd->vdev_obsolete_segments, offset, size); 336 mutex_exit(&vd->vdev_obsolete_lock); 337 vdev_dirty(vd, 0, NULL, spa_syncing_txg(spa)); 338 } 339} 340 341/* 342 * Mark the DVA vdev_id:offset:size as being obsolete in the given tx. This 343 * wrapper is provided because the DMU does not know about vdev_t's and 344 * cannot directly call vdev_indirect_mark_obsolete. 345 */ 346void 347spa_vdev_indirect_mark_obsolete(spa_t *spa, uint64_t vdev_id, uint64_t offset, 348 uint64_t size, dmu_tx_t *tx) 349{ 350 vdev_t *vd = vdev_lookup_top(spa, vdev_id); 351 ASSERT(dmu_tx_is_syncing(tx)); 352 353 /* The DMU can only remap indirect vdevs. */ 354 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops); 355 vdev_indirect_mark_obsolete(vd, offset, size); 356} 357 358static spa_condensing_indirect_t * 359spa_condensing_indirect_create(spa_t *spa) 360{ 361 spa_condensing_indirect_phys_t *scip = 362 &spa->spa_condensing_indirect_phys; 363 spa_condensing_indirect_t *sci = kmem_zalloc(sizeof (*sci), KM_SLEEP); 364 objset_t *mos = spa->spa_meta_objset; 365 366 for (int i = 0; i < TXG_SIZE; i++) { 367 list_create(&sci->sci_new_mapping_entries[i], 368 sizeof (vdev_indirect_mapping_entry_t), 369 offsetof(vdev_indirect_mapping_entry_t, vime_node)); 370 } 371 372 sci->sci_new_mapping = 373 vdev_indirect_mapping_open(mos, scip->scip_next_mapping_object); 374 375 return (sci); 376} 377 378static void 379spa_condensing_indirect_destroy(spa_condensing_indirect_t *sci) 380{ 381 for (int i = 0; i < TXG_SIZE; i++) 382 list_destroy(&sci->sci_new_mapping_entries[i]); 383 384 if (sci->sci_new_mapping != NULL) 385 vdev_indirect_mapping_close(sci->sci_new_mapping); 386 387 kmem_free(sci, sizeof (*sci)); 388} 389 390boolean_t 391vdev_indirect_should_condense(vdev_t *vd) 392{ 393 vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping; 394 spa_t *spa = vd->vdev_spa; 395 396 ASSERT(dsl_pool_sync_context(spa->spa_dsl_pool)); 397 398 if (!zfs_condense_indirect_vdevs_enable) 399 return (B_FALSE); 400 401 /* 402 * We can only condense one indirect vdev at a time. 403 */ 404 if (spa->spa_condensing_indirect != NULL) 405 return (B_FALSE); 406 407 if (spa_shutting_down(spa)) 408 return (B_FALSE); 409 410 /* 411 * The mapping object size must not change while we are 412 * condensing, so we can only condense indirect vdevs 413 * (not vdevs that are still in the middle of being removed). 414 */ 415 if (vd->vdev_ops != &vdev_indirect_ops) 416 return (B_FALSE); 417 418 /* 419 * If nothing new has been marked obsolete, there is no 420 * point in condensing. 421 */ 422 if (vd->vdev_obsolete_sm == NULL) { 423 ASSERT0(vdev_obsolete_sm_object(vd)); 424 return (B_FALSE); 425 } 426 427 ASSERT(vd->vdev_obsolete_sm != NULL); 428 429 ASSERT3U(vdev_obsolete_sm_object(vd), ==, 430 space_map_object(vd->vdev_obsolete_sm)); 431 432 uint64_t bytes_mapped = vdev_indirect_mapping_bytes_mapped(vim); 433 uint64_t bytes_obsolete = space_map_allocated(vd->vdev_obsolete_sm); 434 uint64_t mapping_size = vdev_indirect_mapping_size(vim); 435 uint64_t obsolete_sm_size = space_map_length(vd->vdev_obsolete_sm); 436 437 ASSERT3U(bytes_obsolete, <=, bytes_mapped); 438 439 /* 440 * If a high percentage of the bytes that are mapped have become 441 * obsolete, condense (unless the mapping is already small enough). 442 * This has a good chance of reducing the amount of memory used 443 * by the mapping. 444 */ 445 if (bytes_obsolete * 100 / bytes_mapped >= 446 zfs_indirect_condense_obsolete_pct && 447 mapping_size > zfs_condense_min_mapping_bytes) { 448 zfs_dbgmsg("should condense vdev %llu because obsolete " 449 "spacemap covers %d%% of %lluMB mapping", 450 (u_longlong_t)vd->vdev_id, 451 (int)(bytes_obsolete * 100 / bytes_mapped), 452 (u_longlong_t)bytes_mapped / 1024 / 1024); 453 return (B_TRUE); 454 } 455 456 /* 457 * If the obsolete space map takes up too much space on disk, 458 * condense in order to free up this disk space. 459 */ 460 if (obsolete_sm_size >= zfs_condense_max_obsolete_bytes) { 461 zfs_dbgmsg("should condense vdev %llu because obsolete sm " 462 "length %lluMB >= max size %lluMB", 463 (u_longlong_t)vd->vdev_id, 464 (u_longlong_t)obsolete_sm_size / 1024 / 1024, 465 (u_longlong_t)zfs_condense_max_obsolete_bytes / 466 1024 / 1024); 467 return (B_TRUE); 468 } 469 470 return (B_FALSE); 471} 472 473/* 474 * This sync task completes (finishes) a condense, deleting the old 475 * mapping and replacing it with the new one. 476 */ 477static void 478spa_condense_indirect_complete_sync(void *arg, dmu_tx_t *tx) 479{ 480 spa_condensing_indirect_t *sci = arg; 481 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 482 spa_condensing_indirect_phys_t *scip = 483 &spa->spa_condensing_indirect_phys; 484 vdev_t *vd = vdev_lookup_top(spa, scip->scip_vdev); 485 vdev_indirect_config_t *vic = &vd->vdev_indirect_config; 486 objset_t *mos = spa->spa_meta_objset; 487 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping; 488 uint64_t old_count = vdev_indirect_mapping_num_entries(old_mapping); 489 uint64_t new_count = 490 vdev_indirect_mapping_num_entries(sci->sci_new_mapping); 491 492 ASSERT(dmu_tx_is_syncing(tx)); 493 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops); 494 ASSERT3P(sci, ==, spa->spa_condensing_indirect); 495 for (int i = 0; i < TXG_SIZE; i++) { 496 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[i])); 497 } 498 ASSERT(vic->vic_mapping_object != 0); 499 ASSERT3U(vd->vdev_id, ==, scip->scip_vdev); 500 ASSERT(scip->scip_next_mapping_object != 0); 501 ASSERT(scip->scip_prev_obsolete_sm_object != 0); 502 503 /* 504 * Reset vdev_indirect_mapping to refer to the new object. 505 */ 506 rw_enter(&vd->vdev_indirect_rwlock, RW_WRITER); 507 vdev_indirect_mapping_close(vd->vdev_indirect_mapping); 508 vd->vdev_indirect_mapping = sci->sci_new_mapping; 509 rw_exit(&vd->vdev_indirect_rwlock); 510 511 sci->sci_new_mapping = NULL; 512 vdev_indirect_mapping_free(mos, vic->vic_mapping_object, tx); 513 vic->vic_mapping_object = scip->scip_next_mapping_object; 514 scip->scip_next_mapping_object = 0; 515 516 space_map_free_obj(mos, scip->scip_prev_obsolete_sm_object, tx); 517 spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx); 518 scip->scip_prev_obsolete_sm_object = 0; 519 520 scip->scip_vdev = 0; 521 522 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 523 DMU_POOL_CONDENSING_INDIRECT, tx)); 524 spa_condensing_indirect_destroy(spa->spa_condensing_indirect); 525 spa->spa_condensing_indirect = NULL; 526 527 zfs_dbgmsg("finished condense of vdev %llu in txg %llu: " 528 "new mapping object %llu has %llu entries " 529 "(was %llu entries)", 530 vd->vdev_id, dmu_tx_get_txg(tx), vic->vic_mapping_object, 531 new_count, old_count); 532 533 vdev_config_dirty(spa->spa_root_vdev); 534} 535 536/* 537 * This sync task appends entries to the new mapping object. 538 */ 539static void 540spa_condense_indirect_commit_sync(void *arg, dmu_tx_t *tx) 541{ 542 spa_condensing_indirect_t *sci = arg; 543 uint64_t txg = dmu_tx_get_txg(tx); 544 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 545 546 ASSERT(dmu_tx_is_syncing(tx)); 547 ASSERT3P(sci, ==, spa->spa_condensing_indirect); 548 549 vdev_indirect_mapping_add_entries(sci->sci_new_mapping, 550 &sci->sci_new_mapping_entries[txg & TXG_MASK], tx); 551 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[txg & TXG_MASK])); 552} 553 554/* 555 * Open-context function to add one entry to the new mapping. The new 556 * entry will be remembered and written from syncing context. 557 */ 558static void 559spa_condense_indirect_commit_entry(spa_t *spa, 560 vdev_indirect_mapping_entry_phys_t *vimep, uint32_t count) 561{ 562 spa_condensing_indirect_t *sci = spa->spa_condensing_indirect; 563 564 ASSERT3U(count, <, DVA_GET_ASIZE(&vimep->vimep_dst)); 565 566 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 567 dmu_tx_hold_space(tx, sizeof (*vimep) + sizeof (count)); 568 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 569 int txgoff = dmu_tx_get_txg(tx) & TXG_MASK; 570 571 /* 572 * If we are the first entry committed this txg, kick off the sync 573 * task to write to the MOS on our behalf. 574 */ 575 if (list_is_empty(&sci->sci_new_mapping_entries[txgoff])) { 576 dsl_sync_task_nowait(dmu_tx_pool(tx), 577 spa_condense_indirect_commit_sync, sci, 578 0, ZFS_SPACE_CHECK_NONE, tx); 579 } 580 581 vdev_indirect_mapping_entry_t *vime = 582 kmem_alloc(sizeof (*vime), KM_SLEEP); 583 vime->vime_mapping = *vimep; 584 vime->vime_obsolete_count = count; 585 list_insert_tail(&sci->sci_new_mapping_entries[txgoff], vime); 586 587 dmu_tx_commit(tx); 588} 589 590static void 591spa_condense_indirect_generate_new_mapping(vdev_t *vd, 592 uint32_t *obsolete_counts, uint64_t start_index, zthr_t *zthr) 593{ 594 spa_t *spa = vd->vdev_spa; 595 uint64_t mapi = start_index; 596 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping; 597 uint64_t old_num_entries = 598 vdev_indirect_mapping_num_entries(old_mapping); 599 600 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops); 601 ASSERT3U(vd->vdev_id, ==, spa->spa_condensing_indirect_phys.scip_vdev); 602 603 zfs_dbgmsg("starting condense of vdev %llu from index %llu", 604 (u_longlong_t)vd->vdev_id, 605 (u_longlong_t)mapi); 606 607 while (mapi < old_num_entries) { 608 609 if (zthr_iscancelled(zthr)) { 610 zfs_dbgmsg("pausing condense of vdev %llu " 611 "at index %llu", (u_longlong_t)vd->vdev_id, 612 (u_longlong_t)mapi); 613 break; 614 } 615 616 vdev_indirect_mapping_entry_phys_t *entry = 617 &old_mapping->vim_entries[mapi]; 618 uint64_t entry_size = DVA_GET_ASIZE(&entry->vimep_dst); 619 ASSERT3U(obsolete_counts[mapi], <=, entry_size); 620 if (obsolete_counts[mapi] < entry_size) { 621 spa_condense_indirect_commit_entry(spa, entry, 622 obsolete_counts[mapi]); 623 624 /* 625 * This delay may be requested for testing, debugging, 626 * or performance reasons. 627 */ 628 delay(zfs_condense_indirect_commit_entry_delay_ticks); 629 } 630 631 mapi++; 632 } 633} 634 635/* ARGSUSED */ 636static boolean_t 637spa_condense_indirect_thread_check(void *arg, zthr_t *zthr) 638{ 639 spa_t *spa = arg; 640 641 return (spa->spa_condensing_indirect != NULL); 642} 643 644/* ARGSUSED */ 645static void 646spa_condense_indirect_thread(void *arg, zthr_t *zthr) 647{ 648 spa_t *spa = arg; 649 vdev_t *vd; 650 651 ASSERT3P(spa->spa_condensing_indirect, !=, NULL); 652 spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER); 653 vd = vdev_lookup_top(spa, spa->spa_condensing_indirect_phys.scip_vdev); 654 ASSERT3P(vd, !=, NULL); 655 spa_config_exit(spa, SCL_VDEV, FTAG); 656 657 spa_condensing_indirect_t *sci = spa->spa_condensing_indirect; 658 spa_condensing_indirect_phys_t *scip = 659 &spa->spa_condensing_indirect_phys; 660 uint32_t *counts; 661 uint64_t start_index; 662 vdev_indirect_mapping_t *old_mapping = vd->vdev_indirect_mapping; 663 space_map_t *prev_obsolete_sm = NULL; 664 665 ASSERT3U(vd->vdev_id, ==, scip->scip_vdev); 666 ASSERT(scip->scip_next_mapping_object != 0); 667 ASSERT(scip->scip_prev_obsolete_sm_object != 0); 668 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops); 669 670 for (int i = 0; i < TXG_SIZE; i++) { 671 /* 672 * The list must start out empty in order for the 673 * _commit_sync() sync task to be properly registered 674 * on the first call to _commit_entry(); so it's wise 675 * to double check and ensure we actually are starting 676 * with empty lists. 677 */ 678 ASSERT(list_is_empty(&sci->sci_new_mapping_entries[i])); 679 } 680 681 VERIFY0(space_map_open(&prev_obsolete_sm, spa->spa_meta_objset, 682 scip->scip_prev_obsolete_sm_object, 0, vd->vdev_asize, 0)); 683 counts = vdev_indirect_mapping_load_obsolete_counts(old_mapping); 684 if (prev_obsolete_sm != NULL) { 685 vdev_indirect_mapping_load_obsolete_spacemap(old_mapping, 686 counts, prev_obsolete_sm); 687 } 688 space_map_close(prev_obsolete_sm); 689 690 /* 691 * Generate new mapping. Determine what index to continue from 692 * based on the max offset that we've already written in the 693 * new mapping. 694 */ 695 uint64_t max_offset = 696 vdev_indirect_mapping_max_offset(sci->sci_new_mapping); 697 if (max_offset == 0) { 698 /* We haven't written anything to the new mapping yet. */ 699 start_index = 0; 700 } else { 701 /* 702 * Pick up from where we left off. _entry_for_offset() 703 * returns a pointer into the vim_entries array. If 704 * max_offset is greater than any of the mappings 705 * contained in the table NULL will be returned and 706 * that indicates we've exhausted our iteration of the 707 * old_mapping. 708 */ 709 710 vdev_indirect_mapping_entry_phys_t *entry = 711 vdev_indirect_mapping_entry_for_offset_or_next(old_mapping, 712 max_offset); 713 714 if (entry == NULL) { 715 /* 716 * We've already written the whole new mapping. 717 * This special value will cause us to skip the 718 * generate_new_mapping step and just do the sync 719 * task to complete the condense. 720 */ 721 start_index = UINT64_MAX; 722 } else { 723 start_index = entry - old_mapping->vim_entries; 724 ASSERT3U(start_index, <, 725 vdev_indirect_mapping_num_entries(old_mapping)); 726 } 727 } 728 729 spa_condense_indirect_generate_new_mapping(vd, counts, 730 start_index, zthr); 731 732 vdev_indirect_mapping_free_obsolete_counts(old_mapping, counts); 733 734 /* 735 * If the zthr has received a cancellation signal while running 736 * in generate_new_mapping() or at any point after that, then bail 737 * early. We don't want to complete the condense if the spa is 738 * shutting down. 739 */ 740 if (zthr_iscancelled(zthr)) 741 return; 742 743 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 744 spa_condense_indirect_complete_sync, sci, 0, 745 ZFS_SPACE_CHECK_EXTRA_RESERVED)); 746} 747 748/* 749 * Sync task to begin the condensing process. 750 */ 751void 752spa_condense_indirect_start_sync(vdev_t *vd, dmu_tx_t *tx) 753{ 754 spa_t *spa = vd->vdev_spa; 755 spa_condensing_indirect_phys_t *scip = 756 &spa->spa_condensing_indirect_phys; 757 758 ASSERT0(scip->scip_next_mapping_object); 759 ASSERT0(scip->scip_prev_obsolete_sm_object); 760 ASSERT0(scip->scip_vdev); 761 ASSERT(dmu_tx_is_syncing(tx)); 762 ASSERT3P(vd->vdev_ops, ==, &vdev_indirect_ops); 763 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_OBSOLETE_COUNTS)); 764 ASSERT(vdev_indirect_mapping_num_entries(vd->vdev_indirect_mapping)); 765 766 uint64_t obsolete_sm_obj = vdev_obsolete_sm_object(vd); 767 ASSERT(obsolete_sm_obj != 0); 768 769 scip->scip_vdev = vd->vdev_id; 770 scip->scip_next_mapping_object = 771 vdev_indirect_mapping_alloc(spa->spa_meta_objset, tx); 772 773 scip->scip_prev_obsolete_sm_object = obsolete_sm_obj; 774 775 /* 776 * We don't need to allocate a new space map object, since 777 * vdev_indirect_sync_obsolete will allocate one when needed. 778 */ 779 space_map_close(vd->vdev_obsolete_sm); 780 vd->vdev_obsolete_sm = NULL; 781 VERIFY0(zap_remove(spa->spa_meta_objset, vd->vdev_top_zap, 782 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, tx)); 783 784 VERIFY0(zap_add(spa->spa_dsl_pool->dp_meta_objset, 785 DMU_POOL_DIRECTORY_OBJECT, 786 DMU_POOL_CONDENSING_INDIRECT, sizeof (uint64_t), 787 sizeof (*scip) / sizeof (uint64_t), scip, tx)); 788 789 ASSERT3P(spa->spa_condensing_indirect, ==, NULL); 790 spa->spa_condensing_indirect = spa_condensing_indirect_create(spa); 791 792 zfs_dbgmsg("starting condense of vdev %llu in txg %llu: " 793 "posm=%llu nm=%llu", 794 vd->vdev_id, dmu_tx_get_txg(tx), 795 (u_longlong_t)scip->scip_prev_obsolete_sm_object, 796 (u_longlong_t)scip->scip_next_mapping_object); 797 798 zthr_wakeup(spa->spa_condense_zthr); 799} 800 801/* 802 * Sync to the given vdev's obsolete space map any segments that are no longer 803 * referenced as of the given txg. 804 * 805 * If the obsolete space map doesn't exist yet, create and open it. 806 */ 807void 808vdev_indirect_sync_obsolete(vdev_t *vd, dmu_tx_t *tx) 809{ 810 spa_t *spa = vd->vdev_spa; 811 vdev_indirect_config_t *vic = &vd->vdev_indirect_config; 812 813 ASSERT3U(vic->vic_mapping_object, !=, 0); 814 ASSERT(range_tree_space(vd->vdev_obsolete_segments) > 0); 815 ASSERT(vd->vdev_removing || vd->vdev_ops == &vdev_indirect_ops); 816 ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS)); 817 818 if (vdev_obsolete_sm_object(vd) == 0) { 819 uint64_t obsolete_sm_object = 820 space_map_alloc(spa->spa_meta_objset, 821 vdev_standard_sm_blksz, tx); 822 823 ASSERT(vd->vdev_top_zap != 0); 824 VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap, 825 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, 826 sizeof (obsolete_sm_object), 1, &obsolete_sm_object, tx)); 827 ASSERT3U(vdev_obsolete_sm_object(vd), !=, 0); 828 829 spa_feature_incr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx); 830 VERIFY0(space_map_open(&vd->vdev_obsolete_sm, 831 spa->spa_meta_objset, obsolete_sm_object, 832 0, vd->vdev_asize, 0)); 833 } 834 835 ASSERT(vd->vdev_obsolete_sm != NULL); 836 ASSERT3U(vdev_obsolete_sm_object(vd), ==, 837 space_map_object(vd->vdev_obsolete_sm)); 838 839 space_map_write(vd->vdev_obsolete_sm, 840 vd->vdev_obsolete_segments, SM_ALLOC, SM_NO_VDEVID, tx); 841 range_tree_vacate(vd->vdev_obsolete_segments, NULL, NULL); 842} 843 844int 845spa_condense_init(spa_t *spa) 846{ 847 int error = zap_lookup(spa->spa_meta_objset, 848 DMU_POOL_DIRECTORY_OBJECT, 849 DMU_POOL_CONDENSING_INDIRECT, sizeof (uint64_t), 850 sizeof (spa->spa_condensing_indirect_phys) / sizeof (uint64_t), 851 &spa->spa_condensing_indirect_phys); 852 if (error == 0) { 853 if (spa_writeable(spa)) { 854 spa->spa_condensing_indirect = 855 spa_condensing_indirect_create(spa); 856 } 857 return (0); 858 } else if (error == ENOENT) { 859 return (0); 860 } else { 861 return (error); 862 } 863} 864 865void 866spa_condense_fini(spa_t *spa) 867{ 868 if (spa->spa_condensing_indirect != NULL) { 869 spa_condensing_indirect_destroy(spa->spa_condensing_indirect); 870 spa->spa_condensing_indirect = NULL; 871 } 872} 873 874void 875spa_start_indirect_condensing_thread(spa_t *spa) 876{ 877 ASSERT3P(spa->spa_condense_zthr, ==, NULL); 878 spa->spa_condense_zthr = zthr_create(spa_condense_indirect_thread_check, 879 spa_condense_indirect_thread, spa); 880} 881 882/* 883 * Gets the obsolete spacemap object from the vdev's ZAP. 884 * Returns the spacemap object, or 0 if it wasn't in the ZAP or the ZAP doesn't 885 * exist yet. 886 */ 887int 888vdev_obsolete_sm_object(vdev_t *vd) 889{ 890 ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 891 if (vd->vdev_top_zap == 0) { 892 return (0); 893 } 894 895 uint64_t sm_obj = 0; 896 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap, 897 VDEV_TOP_ZAP_INDIRECT_OBSOLETE_SM, sizeof (sm_obj), 1, &sm_obj); 898 899 ASSERT(err == 0 || err == ENOENT); 900 901 return (sm_obj); 902} 903 904boolean_t 905vdev_obsolete_counts_are_precise(vdev_t *vd) 906{ 907 ASSERT0(spa_config_held(vd->vdev_spa, SCL_ALL, RW_WRITER)); 908 if (vd->vdev_top_zap == 0) { 909 return (B_FALSE); 910 } 911 912 uint64_t val = 0; 913 int err = zap_lookup(vd->vdev_spa->spa_meta_objset, vd->vdev_top_zap, 914 VDEV_TOP_ZAP_OBSOLETE_COUNTS_ARE_PRECISE, sizeof (val), 1, &val); 915 916 ASSERT(err == 0 || err == ENOENT); 917 918 return (val != 0); 919} 920 921/* ARGSUSED */ 922static void 923vdev_indirect_close(vdev_t *vd) 924{ 925} 926 927/* ARGSUSED */ 928static int 929vdev_indirect_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize, 930 uint64_t *logical_ashift, uint64_t *physical_ashift) 931{ 932 *psize = *max_psize = vd->vdev_asize + 933 VDEV_LABEL_START_SIZE + VDEV_LABEL_END_SIZE; 934 *logical_ashift = vd->vdev_ashift; 935 *physical_ashift = vd->vdev_physical_ashift; 936 return (0); 937} 938 939typedef struct remap_segment { 940 vdev_t *rs_vd; 941 uint64_t rs_offset; 942 uint64_t rs_asize; 943 uint64_t rs_split_offset; 944 list_node_t rs_node; 945} remap_segment_t; 946 947remap_segment_t * 948rs_alloc(vdev_t *vd, uint64_t offset, uint64_t asize, uint64_t split_offset) 949{ 950 remap_segment_t *rs = kmem_alloc(sizeof (remap_segment_t), KM_SLEEP); 951 rs->rs_vd = vd; 952 rs->rs_offset = offset; 953 rs->rs_asize = asize; 954 rs->rs_split_offset = split_offset; 955 return (rs); 956} 957 958/* 959 * Given an indirect vdev and an extent on that vdev, it duplicates the 960 * physical entries of the indirect mapping that correspond to the extent 961 * to a new array and returns a pointer to it. In addition, copied_entries 962 * is populated with the number of mapping entries that were duplicated. 963 * 964 * Note that the function assumes that the caller holds vdev_indirect_rwlock. 965 * This ensures that the mapping won't change due to condensing as we 966 * copy over its contents. 967 * 968 * Finally, since we are doing an allocation, it is up to the caller to 969 * free the array allocated in this function. 970 */ 971vdev_indirect_mapping_entry_phys_t * 972vdev_indirect_mapping_duplicate_adjacent_entries(vdev_t *vd, uint64_t offset, 973 uint64_t asize, uint64_t *copied_entries) 974{ 975 vdev_indirect_mapping_entry_phys_t *duplicate_mappings = NULL; 976 vdev_indirect_mapping_t *vim = vd->vdev_indirect_mapping; 977 uint64_t entries = 0; 978 979 ASSERT(RW_READ_HELD(&vd->vdev_indirect_rwlock)); 980 981 vdev_indirect_mapping_entry_phys_t *first_mapping = 982 vdev_indirect_mapping_entry_for_offset(vim, offset); 983 ASSERT3P(first_mapping, !=, NULL); 984 985 vdev_indirect_mapping_entry_phys_t *m = first_mapping; 986 while (asize > 0) { 987 uint64_t size = DVA_GET_ASIZE(&m->vimep_dst); 988 989 ASSERT3U(offset, >=, DVA_MAPPING_GET_SRC_OFFSET(m)); 990 ASSERT3U(offset, <, DVA_MAPPING_GET_SRC_OFFSET(m) + size); 991 992 uint64_t inner_offset = offset - DVA_MAPPING_GET_SRC_OFFSET(m); 993 uint64_t inner_size = MIN(asize, size - inner_offset); 994 995 offset += inner_size; 996 asize -= inner_size; 997 entries++; 998 m++; 999 } 1000 1001 size_t copy_length = entries * sizeof (*first_mapping); 1002 duplicate_mappings = kmem_alloc(copy_length, KM_SLEEP); 1003 bcopy(first_mapping, duplicate_mappings, copy_length); 1004 *copied_entries = entries; 1005 1006 return (duplicate_mappings); 1007} 1008 1009/* 1010 * Goes through the relevant indirect mappings until it hits a concrete vdev 1011 * and issues the callback. On the way to the concrete vdev, if any other 1012 * indirect vdevs are encountered, then the callback will also be called on 1013 * each of those indirect vdevs. For example, if the segment is mapped to 1014 * segment A on indirect vdev 1, and then segment A on indirect vdev 1 is 1015 * mapped to segment B on concrete vdev 2, then the callback will be called on 1016 * both vdev 1 and vdev 2. 1017 * 1018 * While the callback passed to vdev_indirect_remap() is called on every vdev 1019 * the function encounters, certain callbacks only care about concrete vdevs. 1020 * These types of callbacks should return immediately and explicitly when they 1021 * are called on an indirect vdev. 1022 * 1023 * Because there is a possibility that a DVA section in the indirect device 1024 * has been split into multiple sections in our mapping, we keep track 1025 * of the relevant contiguous segments of the new location (remap_segment_t) 1026 * in a stack. This way we can call the callback for each of the new sections 1027 * created by a single section of the indirect device. Note though, that in 1028 * this scenario the callbacks in each split block won't occur in-order in 1029 * terms of offset, so callers should not make any assumptions about that. 1030 * 1031 * For callbacks that don't handle split blocks and immediately return when 1032 * they encounter them (as is the case for remap_blkptr_cb), the caller can 1033 * assume that its callback will be applied from the first indirect vdev 1034 * encountered to the last one and then the concrete vdev, in that order. 1035 */ 1036static void 1037vdev_indirect_remap(vdev_t *vd, uint64_t offset, uint64_t asize, 1038 void (*func)(uint64_t, vdev_t *, uint64_t, uint64_t, void *), void *arg) 1039{ 1040 list_t stack; 1041 spa_t *spa = vd->vdev_spa; 1042 1043 list_create(&stack, sizeof (remap_segment_t), 1044 offsetof(remap_segment_t, rs_node)); 1045 1046 for (remap_segment_t *rs = rs_alloc(vd, offset, asize, 0); 1047 rs != NULL; rs = list_remove_head(&stack)) { 1048 vdev_t *v = rs->rs_vd; 1049 uint64_t num_entries = 0; 1050 1051 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0); 1052 ASSERT(rs->rs_asize > 0); 1053 1054 /* 1055 * Note: As this function can be called from open context 1056 * (e.g. zio_read()), we need the following rwlock to 1057 * prevent the mapping from being changed by condensing. 1058 * 1059 * So we grab the lock and we make a copy of the entries 1060 * that are relevant to the extent that we are working on. 1061 * Once that is done, we drop the lock and iterate over 1062 * our copy of the mapping. Once we are done with the with 1063 * the remap segment and we free it, we also free our copy 1064 * of the indirect mapping entries that are relevant to it. 1065 * 1066 * This way we don't need to wait until the function is 1067 * finished with a segment, to condense it. In addition, we 1068 * don't need a recursive rwlock for the case that a call to 1069 * vdev_indirect_remap() needs to call itself (through the 1070 * codepath of its callback) for the same vdev in the middle 1071 * of its execution. 1072 */ 1073 rw_enter(&v->vdev_indirect_rwlock, RW_READER); 1074 vdev_indirect_mapping_t *vim = v->vdev_indirect_mapping; 1075 ASSERT3P(vim, !=, NULL); 1076 1077 vdev_indirect_mapping_entry_phys_t *mapping = 1078 vdev_indirect_mapping_duplicate_adjacent_entries(v, 1079 rs->rs_offset, rs->rs_asize, &num_entries); 1080 ASSERT3P(mapping, !=, NULL); 1081 ASSERT3U(num_entries, >, 0); 1082 rw_exit(&v->vdev_indirect_rwlock); 1083 1084 for (uint64_t i = 0; i < num_entries; i++) { 1085 /* 1086 * Note: the vdev_indirect_mapping can not change 1087 * while we are running. It only changes while the 1088 * removal is in progress, and then only from syncing 1089 * context. While a removal is in progress, this 1090 * function is only called for frees, which also only 1091 * happen from syncing context. 1092 */ 1093 vdev_indirect_mapping_entry_phys_t *m = &mapping[i]; 1094 1095 ASSERT3P(m, !=, NULL); 1096 ASSERT3U(rs->rs_asize, >, 0); 1097 1098 uint64_t size = DVA_GET_ASIZE(&m->vimep_dst); 1099 uint64_t dst_offset = DVA_GET_OFFSET(&m->vimep_dst); 1100 uint64_t dst_vdev = DVA_GET_VDEV(&m->vimep_dst); 1101 1102 ASSERT3U(rs->rs_offset, >=, 1103 DVA_MAPPING_GET_SRC_OFFSET(m)); 1104 ASSERT3U(rs->rs_offset, <, 1105 DVA_MAPPING_GET_SRC_OFFSET(m) + size); 1106 ASSERT3U(dst_vdev, !=, v->vdev_id); 1107 1108 uint64_t inner_offset = rs->rs_offset - 1109 DVA_MAPPING_GET_SRC_OFFSET(m); 1110 uint64_t inner_size = 1111 MIN(rs->rs_asize, size - inner_offset); 1112 1113 vdev_t *dst_v = vdev_lookup_top(spa, dst_vdev); 1114 ASSERT3P(dst_v, !=, NULL); 1115 1116 if (dst_v->vdev_ops == &vdev_indirect_ops) { 1117 list_insert_head(&stack, 1118 rs_alloc(dst_v, dst_offset + inner_offset, 1119 inner_size, rs->rs_split_offset)); 1120 1121 } 1122 1123 if ((zfs_flags & ZFS_DEBUG_INDIRECT_REMAP) && 1124 IS_P2ALIGNED(inner_size, 2 * SPA_MINBLOCKSIZE)) { 1125 /* 1126 * Note: This clause exists only solely for 1127 * testing purposes. We use it to ensure that 1128 * split blocks work and that the callbacks 1129 * using them yield the same result if issued 1130 * in reverse order. 1131 */ 1132 uint64_t inner_half = inner_size / 2; 1133 1134 func(rs->rs_split_offset + inner_half, dst_v, 1135 dst_offset + inner_offset + inner_half, 1136 inner_half, arg); 1137 1138 func(rs->rs_split_offset, dst_v, 1139 dst_offset + inner_offset, 1140 inner_half, arg); 1141 } else { 1142 func(rs->rs_split_offset, dst_v, 1143 dst_offset + inner_offset, 1144 inner_size, arg); 1145 } 1146 1147 rs->rs_offset += inner_size; 1148 rs->rs_asize -= inner_size; 1149 rs->rs_split_offset += inner_size; 1150 } 1151 VERIFY0(rs->rs_asize); 1152 1153 kmem_free(mapping, num_entries * sizeof (*mapping)); 1154 kmem_free(rs, sizeof (remap_segment_t)); 1155 } 1156 list_destroy(&stack); 1157} 1158 1159static void 1160vdev_indirect_child_io_done(zio_t *zio) 1161{ 1162 zio_t *pio = zio->io_private; 1163 1164 mutex_enter(&pio->io_lock); 1165 pio->io_error = zio_worst_error(pio->io_error, zio->io_error); 1166 mutex_exit(&pio->io_lock); 1167 1168#ifdef __FreeBSD__ 1169 if (zio->io_abd != NULL) 1170#endif 1171 abd_put(zio->io_abd); 1172} 1173 1174/* 1175 * This is a callback for vdev_indirect_remap() which allocates an 1176 * indirect_split_t for each split segment and adds it to iv_splits. 1177 */ 1178static void 1179vdev_indirect_gather_splits(uint64_t split_offset, vdev_t *vd, uint64_t offset, 1180 uint64_t size, void *arg) 1181{ 1182 zio_t *zio = arg; 1183 indirect_vsd_t *iv = zio->io_vsd; 1184 1185 ASSERT3P(vd, !=, NULL); 1186 1187 if (vd->vdev_ops == &vdev_indirect_ops) 1188 return; 1189 1190 int n = 1; 1191 if (vd->vdev_ops == &vdev_mirror_ops) 1192 n = vd->vdev_children; 1193 1194 indirect_split_t *is = 1195 kmem_zalloc(offsetof(indirect_split_t, is_child[n]), KM_SLEEP); 1196 1197 is->is_children = n; 1198 is->is_size = size; 1199 is->is_split_offset = split_offset; 1200 is->is_target_offset = offset; 1201 is->is_vdev = vd; 1202 list_create(&is->is_unique_child, sizeof (indirect_child_t), 1203 offsetof(indirect_child_t, ic_node)); 1204 1205 /* 1206 * Note that we only consider multiple copies of the data for 1207 * *mirror* vdevs. We don't for "replacing" or "spare" vdevs, even 1208 * though they use the same ops as mirror, because there's only one 1209 * "good" copy under the replacing/spare. 1210 */ 1211 if (vd->vdev_ops == &vdev_mirror_ops) { 1212 for (int i = 0; i < n; i++) { 1213 is->is_child[i].ic_vdev = vd->vdev_child[i]; 1214 list_link_init(&is->is_child[i].ic_node); 1215 } 1216 } else { 1217 is->is_child[0].ic_vdev = vd; 1218 } 1219 1220 list_insert_tail(&iv->iv_splits, is); 1221} 1222 1223static void 1224vdev_indirect_read_split_done(zio_t *zio) 1225{ 1226 indirect_child_t *ic = zio->io_private; 1227 1228 if (zio->io_error != 0) { 1229 /* 1230 * Clear ic_data to indicate that we do not have data for this 1231 * child. 1232 */ 1233 abd_free(ic->ic_data); 1234 ic->ic_data = NULL; 1235 } 1236} 1237 1238/* 1239 * Issue reads for all copies (mirror children) of all splits. 1240 */ 1241static void 1242vdev_indirect_read_all(zio_t *zio) 1243{ 1244 indirect_vsd_t *iv = zio->io_vsd; 1245 1246 ASSERT3U(zio->io_type, ==, ZIO_TYPE_READ); 1247 1248 for (indirect_split_t *is = list_head(&iv->iv_splits); 1249 is != NULL; is = list_next(&iv->iv_splits, is)) { 1250 for (int i = 0; i < is->is_children; i++) { 1251 indirect_child_t *ic = &is->is_child[i]; 1252 1253 if (!vdev_readable(ic->ic_vdev)) 1254 continue; 1255 1256 /* 1257 * Note, we may read from a child whose DTL 1258 * indicates that the data may not be present here. 1259 * While this might result in a few i/os that will 1260 * likely return incorrect data, it simplifies the 1261 * code since we can treat scrub and resilver 1262 * identically. (The incorrect data will be 1263 * detected and ignored when we verify the 1264 * checksum.) 1265 */ 1266 1267 ic->ic_data = abd_alloc_sametype(zio->io_abd, 1268 is->is_size); 1269 ic->ic_duplicate = NULL; 1270 1271 zio_nowait(zio_vdev_child_io(zio, NULL, 1272 ic->ic_vdev, is->is_target_offset, ic->ic_data, 1273 is->is_size, zio->io_type, zio->io_priority, 0, 1274 vdev_indirect_read_split_done, ic)); 1275 } 1276 } 1277 iv->iv_reconstruct = B_TRUE; 1278} 1279 1280static void 1281vdev_indirect_io_start(zio_t *zio) 1282{ 1283 spa_t *spa = zio->io_spa; 1284 indirect_vsd_t *iv = kmem_zalloc(sizeof (*iv), KM_SLEEP); 1285 list_create(&iv->iv_splits, 1286 sizeof (indirect_split_t), offsetof(indirect_split_t, is_node)); 1287 1288 zio->io_vsd = iv; 1289 zio->io_vsd_ops = &vdev_indirect_vsd_ops; 1290 1291 ASSERT(spa_config_held(spa, SCL_ALL, RW_READER) != 0); 1292#ifdef __FreeBSD__ 1293 if (zio->io_type == ZIO_TYPE_WRITE) { 1294#else 1295 if (zio->io_type != ZIO_TYPE_READ) { 1296 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 1297#endif 1298 /* 1299 * Note: this code can handle other kinds of writes, 1300 * but we don't expect them. 1301 */ 1302 ASSERT((zio->io_flags & (ZIO_FLAG_SELF_HEAL | 1303 ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE)) != 0); 1304 } 1305 1306 vdev_indirect_remap(zio->io_vd, zio->io_offset, zio->io_size, 1307 vdev_indirect_gather_splits, zio); 1308 1309 indirect_split_t *first = list_head(&iv->iv_splits); 1310 if (first->is_size == zio->io_size) { 1311 /* 1312 * This is not a split block; we are pointing to the entire 1313 * data, which will checksum the same as the original data. 1314 * Pass the BP down so that the child i/o can verify the 1315 * checksum, and try a different location if available 1316 * (e.g. on a mirror). 1317 * 1318 * While this special case could be handled the same as the 1319 * general (split block) case, doing it this way ensures 1320 * that the vast majority of blocks on indirect vdevs 1321 * (which are not split) are handled identically to blocks 1322 * on non-indirect vdevs. This allows us to be less strict 1323 * about performance in the general (but rare) case. 1324 */ 1325 ASSERT0(first->is_split_offset); 1326 ASSERT3P(list_next(&iv->iv_splits, first), ==, NULL); 1327 zio_nowait(zio_vdev_child_io(zio, zio->io_bp, 1328 first->is_vdev, first->is_target_offset, 1329#ifdef __FreeBSD__ 1330 zio->io_abd == NULL ? NULL : 1331#endif 1332 abd_get_offset(zio->io_abd, 0), 1333 zio->io_size, zio->io_type, zio->io_priority, 0, 1334 vdev_indirect_child_io_done, zio)); 1335 } else { 1336 iv->iv_split_block = B_TRUE; 1337 if (zio->io_type == ZIO_TYPE_READ && 1338 zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER)) { 1339 /* 1340 * Read all copies. Note that for simplicity, 1341 * we don't bother consulting the DTL in the 1342 * resilver case. 1343 */ 1344 vdev_indirect_read_all(zio); 1345 } else { 1346 /* 1347 * If this is a read zio, we read one copy of each 1348 * split segment, from the top-level vdev. Since 1349 * we don't know the checksum of each split 1350 * individually, the child zio can't ensure that 1351 * we get the right data. E.g. if it's a mirror, 1352 * it will just read from a random (healthy) leaf 1353 * vdev. We have to verify the checksum in 1354 * vdev_indirect_io_done(). 1355 * 1356 * For write zios, the vdev code will ensure we write 1357 * to all children. 1358 */ 1359 for (indirect_split_t *is = list_head(&iv->iv_splits); 1360 is != NULL; is = list_next(&iv->iv_splits, is)) { 1361 zio_nowait(zio_vdev_child_io(zio, NULL, 1362 is->is_vdev, is->is_target_offset, 1363#ifdef __FreeBSD__ 1364 zio->io_abd == NULL ? NULL : 1365#endif 1366 abd_get_offset(zio->io_abd, 1367 is->is_split_offset), 1368 is->is_size, zio->io_type, 1369 zio->io_priority, 0, 1370 vdev_indirect_child_io_done, zio)); 1371 } 1372 } 1373 } 1374 1375 zio_execute(zio); 1376} 1377 1378/* 1379 * Report a checksum error for a child. 1380 */ 1381static void 1382vdev_indirect_checksum_error(zio_t *zio, 1383 indirect_split_t *is, indirect_child_t *ic) 1384{ 1385 vdev_t *vd = ic->ic_vdev; 1386 1387 if (zio->io_flags & ZIO_FLAG_SPECULATIVE) 1388 return; 1389 1390 mutex_enter(&vd->vdev_stat_lock); 1391 vd->vdev_stat.vs_checksum_errors++; 1392 mutex_exit(&vd->vdev_stat_lock); 1393 1394 zio_bad_cksum_t zbc = { 0 }; 1395 void *bad_buf = abd_borrow_buf_copy(ic->ic_data, is->is_size); 1396 abd_t *good_abd = is->is_good_child->ic_data; 1397 void *good_buf = abd_borrow_buf_copy(good_abd, is->is_size); 1398 zfs_ereport_post_checksum(zio->io_spa, vd, zio, 1399 is->is_target_offset, is->is_size, good_buf, bad_buf, &zbc); 1400 abd_return_buf(ic->ic_data, bad_buf, is->is_size); 1401 abd_return_buf(good_abd, good_buf, is->is_size); 1402} 1403 1404/* 1405 * Issue repair i/os for any incorrect copies. We do this by comparing 1406 * each split segment's correct data (is_good_child's ic_data) with each 1407 * other copy of the data. If they differ, then we overwrite the bad data 1408 * with the good copy. Note that we do this without regard for the DTL's, 1409 * which simplifies this code and also issues the optimal number of writes 1410 * (based on which copies actually read bad data, as opposed to which we 1411 * think might be wrong). For the same reason, we always use 1412 * ZIO_FLAG_SELF_HEAL, to bypass the DTL check in zio_vdev_io_start(). 1413 */ 1414static void 1415vdev_indirect_repair(zio_t *zio) 1416{ 1417 indirect_vsd_t *iv = zio->io_vsd; 1418 1419 enum zio_flag flags = ZIO_FLAG_IO_REPAIR; 1420 1421 if (!(zio->io_flags & (ZIO_FLAG_SCRUB | ZIO_FLAG_RESILVER))) 1422 flags |= ZIO_FLAG_SELF_HEAL; 1423 1424 if (!spa_writeable(zio->io_spa)) 1425 return; 1426 1427 for (indirect_split_t *is = list_head(&iv->iv_splits); 1428 is != NULL; is = list_next(&iv->iv_splits, is)) { 1429 for (int c = 0; c < is->is_children; c++) { 1430 indirect_child_t *ic = &is->is_child[c]; 1431 if (ic == is->is_good_child) 1432 continue; 1433 if (ic->ic_data == NULL) 1434 continue; 1435 if (ic->ic_duplicate == is->is_good_child) 1436 continue; 1437 1438 zio_nowait(zio_vdev_child_io(zio, NULL, 1439 ic->ic_vdev, is->is_target_offset, 1440 is->is_good_child->ic_data, is->is_size, 1441 ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE, 1442 ZIO_FLAG_IO_REPAIR | ZIO_FLAG_SELF_HEAL, 1443 NULL, NULL)); 1444 1445 vdev_indirect_checksum_error(zio, is, ic); 1446 } 1447 } 1448} 1449 1450/* 1451 * Report checksum errors on all children that we read from. 1452 */ 1453static void 1454vdev_indirect_all_checksum_errors(zio_t *zio) 1455{ 1456 indirect_vsd_t *iv = zio->io_vsd; 1457 1458 if (zio->io_flags & ZIO_FLAG_SPECULATIVE) 1459 return; 1460 1461 for (indirect_split_t *is = list_head(&iv->iv_splits); 1462 is != NULL; is = list_next(&iv->iv_splits, is)) { 1463 for (int c = 0; c < is->is_children; c++) { 1464 indirect_child_t *ic = &is->is_child[c]; 1465 1466 if (ic->ic_data == NULL) 1467 continue; 1468 1469 vdev_t *vd = ic->ic_vdev; 1470 1471 mutex_enter(&vd->vdev_stat_lock); 1472 vd->vdev_stat.vs_checksum_errors++; 1473 mutex_exit(&vd->vdev_stat_lock); 1474 1475 zfs_ereport_post_checksum(zio->io_spa, vd, zio, 1476 is->is_target_offset, is->is_size, 1477 NULL, NULL, NULL); 1478 } 1479 } 1480} 1481 1482/* 1483 * Copy data from all the splits to a main zio then validate the checksum. 1484 * If then checksum is successfully validated return success. 1485 */ 1486static int 1487vdev_indirect_splits_checksum_validate(indirect_vsd_t *iv, zio_t *zio) 1488{ 1489 zio_bad_cksum_t zbc; 1490 1491 for (indirect_split_t *is = list_head(&iv->iv_splits); 1492 is != NULL; is = list_next(&iv->iv_splits, is)) { 1493 1494 ASSERT3P(is->is_good_child->ic_data, !=, NULL); 1495 ASSERT3P(is->is_good_child->ic_duplicate, ==, NULL); 1496 1497 abd_copy_off(zio->io_abd, is->is_good_child->ic_data, 1498 is->is_split_offset, 0, is->is_size); 1499 } 1500 1501 return (zio_checksum_error(zio, &zbc)); 1502} 1503 1504/* 1505 * There are relatively few possible combinations making it feasible to 1506 * deterministically check them all. We do this by setting the good_child 1507 * to the next unique split version. If we reach the end of the list then 1508 * "carry over" to the next unique split version (like counting in base 1509 * is_unique_children, but each digit can have a different base). 1510 */ 1511static int 1512vdev_indirect_splits_enumerate_all(indirect_vsd_t *iv, zio_t *zio) 1513{ 1514 boolean_t more = B_TRUE; 1515 1516 iv->iv_attempts = 0; 1517 1518 for (indirect_split_t *is = list_head(&iv->iv_splits); 1519 is != NULL; is = list_next(&iv->iv_splits, is)) 1520 is->is_good_child = list_head(&is->is_unique_child); 1521 1522 while (more == B_TRUE) { 1523 iv->iv_attempts++; 1524 more = B_FALSE; 1525 1526 if (vdev_indirect_splits_checksum_validate(iv, zio) == 0) 1527 return (0); 1528 1529 for (indirect_split_t *is = list_head(&iv->iv_splits); 1530 is != NULL; is = list_next(&iv->iv_splits, is)) { 1531 is->is_good_child = list_next(&is->is_unique_child, 1532 is->is_good_child); 1533 if (is->is_good_child != NULL) { 1534 more = B_TRUE; 1535 break; 1536 } 1537 1538 is->is_good_child = list_head(&is->is_unique_child); 1539 } 1540 } 1541 1542 ASSERT3S(iv->iv_attempts, <=, iv->iv_unique_combinations); 1543 1544 return (SET_ERROR(ECKSUM)); 1545} 1546 1547/* 1548 * There are too many combinations to try all of them in a reasonable amount 1549 * of time. So try a fixed number of random combinations from the unique 1550 * split versions, after which we'll consider the block unrecoverable. 1551 */ 1552static int 1553vdev_indirect_splits_enumerate_randomly(indirect_vsd_t *iv, zio_t *zio) 1554{ 1555 iv->iv_attempts = 0; 1556 1557 while (iv->iv_attempts < iv->iv_attempts_max) { 1558 iv->iv_attempts++; 1559 1560 for (indirect_split_t *is = list_head(&iv->iv_splits); 1561 is != NULL; is = list_next(&iv->iv_splits, is)) { 1562 indirect_child_t *ic = list_head(&is->is_unique_child); 1563 int children = is->is_unique_children; 1564 1565 for (int i = spa_get_random(children); i > 0; i--) 1566 ic = list_next(&is->is_unique_child, ic); 1567 1568 ASSERT3P(ic, !=, NULL); 1569 is->is_good_child = ic; 1570 } 1571 1572 if (vdev_indirect_splits_checksum_validate(iv, zio) == 0) 1573 return (0); 1574 } 1575 1576 return (SET_ERROR(ECKSUM)); 1577} 1578 1579/* 1580 * This is a validation function for reconstruction. It randomly selects 1581 * a good combination, if one can be found, and then it intentionally 1582 * damages all other segment copes by zeroing them. This forces the 1583 * reconstruction algorithm to locate the one remaining known good copy. 1584 */ 1585static int 1586vdev_indirect_splits_damage(indirect_vsd_t *iv, zio_t *zio) 1587{ 1588 /* Presume all the copies are unique for initial selection. */ 1589 for (indirect_split_t *is = list_head(&iv->iv_splits); 1590 is != NULL; is = list_next(&iv->iv_splits, is)) { 1591 is->is_unique_children = 0; 1592 1593 for (int i = 0; i < is->is_children; i++) { 1594 indirect_child_t *ic = &is->is_child[i]; 1595 if (ic->ic_data != NULL) { 1596 is->is_unique_children++; 1597 list_insert_tail(&is->is_unique_child, ic); 1598 } 1599 } 1600 } 1601 1602 /* 1603 * Set each is_good_child to a randomly-selected child which 1604 * is known to contain validated data. 1605 */ 1606 int error = vdev_indirect_splits_enumerate_randomly(iv, zio); 1607 if (error) 1608 goto out; 1609 1610 /* 1611 * Damage all but the known good copy by zeroing it. This will 1612 * result in two or less unique copies per indirect_child_t. 1613 * Both may need to be checked in order to reconstruct the block. 1614 * Set iv->iv_attempts_max such that all unique combinations will 1615 * enumerated, but limit the damage to at most 16 indirect splits. 1616 */ 1617 iv->iv_attempts_max = 1; 1618 1619 for (indirect_split_t *is = list_head(&iv->iv_splits); 1620 is != NULL; is = list_next(&iv->iv_splits, is)) { 1621 for (int c = 0; c < is->is_children; c++) { 1622 indirect_child_t *ic = &is->is_child[c]; 1623 1624 if (ic == is->is_good_child) 1625 continue; 1626 if (ic->ic_data == NULL) 1627 continue; 1628 1629 abd_zero(ic->ic_data, ic->ic_data->abd_size); 1630 } 1631 1632 iv->iv_attempts_max *= 2; 1633 if (iv->iv_attempts_max > (1ULL << 16)) { 1634 iv->iv_attempts_max = UINT64_MAX; 1635 break; 1636 } 1637 } 1638 1639out: 1640 /* Empty the unique children lists so they can be reconstructed. */ 1641 for (indirect_split_t *is = list_head(&iv->iv_splits); 1642 is != NULL; is = list_next(&iv->iv_splits, is)) { 1643 indirect_child_t *ic; 1644 while ((ic = list_head(&is->is_unique_child)) != NULL) 1645 list_remove(&is->is_unique_child, ic); 1646 1647 is->is_unique_children = 0; 1648 } 1649 1650 return (error); 1651} 1652 1653/* 1654 * This function is called when we have read all copies of the data and need 1655 * to try to find a combination of copies that gives us the right checksum. 1656 * 1657 * If we pointed to any mirror vdevs, this effectively does the job of the 1658 * mirror. The mirror vdev code can't do its own job because we don't know 1659 * the checksum of each split segment individually. 1660 * 1661 * We have to try every unique combination of copies of split segments, until 1662 * we find one that checksums correctly. Duplicate segment copies are first 1663 * identified and latter skipped during reconstruction. This optimization 1664 * reduces the search space and ensures that of the remaining combinations 1665 * at most one is correct. 1666 * 1667 * When the total number of combinations is small they can all be checked. 1668 * For example, if we have 3 segments in the split, and each points to a 1669 * 2-way mirror with unique copies, we will have the following pieces of data: 1670 * 1671 * | mirror child 1672 * split | [0] [1] 1673 * ======|===================== 1674 * A | data_A_0 data_A_1 1675 * B | data_B_0 data_B_1 1676 * C | data_C_0 data_C_1 1677 * 1678 * We will try the following (mirror children)^(number of splits) (2^3=8) 1679 * combinations, which is similar to bitwise-little-endian counting in 1680 * binary. In general each "digit" corresponds to a split segment, and the 1681 * base of each digit is is_children, which can be different for each 1682 * digit. 1683 * 1684 * "low bit" "high bit" 1685 * v v 1686 * data_A_0 data_B_0 data_C_0 1687 * data_A_1 data_B_0 data_C_0 1688 * data_A_0 data_B_1 data_C_0 1689 * data_A_1 data_B_1 data_C_0 1690 * data_A_0 data_B_0 data_C_1 1691 * data_A_1 data_B_0 data_C_1 1692 * data_A_0 data_B_1 data_C_1 1693 * data_A_1 data_B_1 data_C_1 1694 * 1695 * Note that the split segments may be on the same or different top-level 1696 * vdevs. In either case, we may need to try lots of combinations (see 1697 * zfs_reconstruct_indirect_combinations_max). This ensures that if a mirror 1698 * has small silent errors on all of its children, we can still reconstruct 1699 * the correct data, as long as those errors are at sufficiently-separated 1700 * offsets (specifically, separated by the largest block size - default of 1701 * 128KB, but up to 16MB). 1702 */ 1703static void 1704vdev_indirect_reconstruct_io_done(zio_t *zio) 1705{ 1706 indirect_vsd_t *iv = zio->io_vsd; 1707 boolean_t known_good = B_FALSE; 1708 int error; 1709 1710 iv->iv_unique_combinations = 1; 1711 iv->iv_attempts_max = UINT64_MAX; 1712 1713 if (zfs_reconstruct_indirect_combinations_max > 0) 1714 iv->iv_attempts_max = zfs_reconstruct_indirect_combinations_max; 1715 1716 /* 1717 * If nonzero, every 1/x blocks will be damaged, in order to validate 1718 * reconstruction when there are split segments with damaged copies. 1719 * Known_good will TRUE when reconstruction is known to be possible. 1720 */ 1721 if (zfs_reconstruct_indirect_damage_fraction != 0 && 1722 spa_get_random(zfs_reconstruct_indirect_damage_fraction) == 0) 1723 known_good = (vdev_indirect_splits_damage(iv, zio) == 0); 1724 1725 /* 1726 * Determine the unique children for a split segment and add them 1727 * to the is_unique_child list. By restricting reconstruction 1728 * to these children, only unique combinations will be considered. 1729 * This can vastly reduce the search space when there are a large 1730 * number of indirect splits. 1731 */ 1732 for (indirect_split_t *is = list_head(&iv->iv_splits); 1733 is != NULL; is = list_next(&iv->iv_splits, is)) { 1734 is->is_unique_children = 0; 1735 1736 for (int i = 0; i < is->is_children; i++) { 1737 indirect_child_t *ic_i = &is->is_child[i]; 1738 1739 if (ic_i->ic_data == NULL || 1740 ic_i->ic_duplicate != NULL) 1741 continue; 1742 1743 for (int j = i + 1; j < is->is_children; j++) { 1744 indirect_child_t *ic_j = &is->is_child[j]; 1745 1746 if (ic_j->ic_data == NULL || 1747 ic_j->ic_duplicate != NULL) 1748 continue; 1749 1750 if (abd_cmp(ic_i->ic_data, ic_j->ic_data, 1751 is->is_size) == 0) { 1752 ic_j->ic_duplicate = ic_i; 1753 } 1754 } 1755 1756 is->is_unique_children++; 1757 list_insert_tail(&is->is_unique_child, ic_i); 1758 } 1759 1760 /* Reconstruction is impossible, no valid children */ 1761 EQUIV(list_is_empty(&is->is_unique_child), 1762 is->is_unique_children == 0); 1763 if (list_is_empty(&is->is_unique_child)) { 1764 zio->io_error = EIO; 1765 vdev_indirect_all_checksum_errors(zio); 1766 zio_checksum_verified(zio); 1767 return; 1768 } 1769 1770 iv->iv_unique_combinations *= is->is_unique_children; 1771 } 1772 1773 if (iv->iv_unique_combinations <= iv->iv_attempts_max) 1774 error = vdev_indirect_splits_enumerate_all(iv, zio); 1775 else 1776 error = vdev_indirect_splits_enumerate_randomly(iv, zio); 1777 1778 if (error != 0) { 1779 /* All attempted combinations failed. */ 1780 ASSERT3B(known_good, ==, B_FALSE); 1781 zio->io_error = error; 1782 vdev_indirect_all_checksum_errors(zio); 1783 } else { 1784 /* 1785 * The checksum has been successfully validated. Issue 1786 * repair I/Os to any copies of splits which don't match 1787 * the validated version. 1788 */ 1789 ASSERT0(vdev_indirect_splits_checksum_validate(iv, zio)); 1790 vdev_indirect_repair(zio); 1791 zio_checksum_verified(zio); 1792 } 1793} 1794 1795static void 1796vdev_indirect_io_done(zio_t *zio) 1797{ 1798 indirect_vsd_t *iv = zio->io_vsd; 1799 1800 if (iv->iv_reconstruct) { 1801 /* 1802 * We have read all copies of the data (e.g. from mirrors), 1803 * either because this was a scrub/resilver, or because the 1804 * one-copy read didn't checksum correctly. 1805 */ 1806 vdev_indirect_reconstruct_io_done(zio); 1807 return; 1808 } 1809 1810 if (!iv->iv_split_block) { 1811 /* 1812 * This was not a split block, so we passed the BP down, 1813 * and the checksum was handled by the (one) child zio. 1814 */ 1815 return; 1816 } 1817 1818 zio_bad_cksum_t zbc; 1819 int ret = zio_checksum_error(zio, &zbc); 1820 if (ret == 0) { 1821 zio_checksum_verified(zio); 1822 return; 1823 } 1824 1825 /* 1826 * The checksum didn't match. Read all copies of all splits, and 1827 * then we will try to reconstruct. The next time 1828 * vdev_indirect_io_done() is called, iv_reconstruct will be set. 1829 */ 1830 vdev_indirect_read_all(zio); 1831 1832 zio_vdev_io_redone(zio); 1833} 1834 1835vdev_ops_t vdev_indirect_ops = { 1836 vdev_indirect_open, 1837 vdev_indirect_close, 1838 vdev_default_asize, 1839 vdev_indirect_io_start, 1840 vdev_indirect_io_done, 1841 NULL, 1842 NULL, 1843 NULL, 1844 NULL, 1845 vdev_indirect_remap, 1846 NULL, 1847 VDEV_TYPE_INDIRECT, /* name of this vdev type */ 1848 B_FALSE /* leaf vdev */ 1849}; 1850