1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 25 * Copyright (c) 2018, Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. 27 * Copyright 2013 Saso Kiselkov. All rights reserved. 28 * Copyright (c) 2014 Integros [integros.com] 29 * Copyright 2016 Toomas Soome <tsoome@me.com> 30 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 31 * Copyright 2018 Joyent, Inc. 32 * Copyright (c) 2017, 2019, Datto Inc. All rights reserved. 33 * Copyright 2017 Joyent, Inc. 34 * Copyright (c) 2017, Intel Corporation. 35 * Copyright (c) 2021, Colm Buckley <colm@tuatha.org> 36 */ 37 38/* 39 * SPA: Storage Pool Allocator 40 * 41 * This file contains all the routines used when modifying on-disk SPA state. 42 * This includes opening, importing, destroying, exporting a pool, and syncing a 43 * pool. 44 */ 45 46#include <sys/zfs_context.h> 47#include <sys/fm/fs/zfs.h> 48#include <sys/spa_impl.h> 49#include <sys/zio.h> 50#include <sys/zio_checksum.h> 51#include <sys/dmu.h> 52#include <sys/dmu_tx.h> 53#include <sys/zap.h> 54#include <sys/zil.h> 55#include <sys/ddt.h> 56#include <sys/vdev_impl.h> 57#include <sys/vdev_removal.h> 58#include <sys/vdev_indirect_mapping.h> 59#include <sys/vdev_indirect_births.h> 60#include <sys/vdev_initialize.h> 61#include <sys/vdev_rebuild.h> 62#include <sys/vdev_trim.h> 63#include <sys/vdev_disk.h> 64#include <sys/vdev_draid.h> 65#include <sys/metaslab.h> 66#include <sys/metaslab_impl.h> 67#include <sys/mmp.h> 68#include <sys/uberblock_impl.h> 69#include <sys/txg.h> 70#include <sys/avl.h> 71#include <sys/bpobj.h> 72#include <sys/dmu_traverse.h> 73#include <sys/dmu_objset.h> 74#include <sys/unique.h> 75#include <sys/dsl_pool.h> 76#include <sys/dsl_dataset.h> 77#include <sys/dsl_dir.h> 78#include <sys/dsl_prop.h> 79#include <sys/dsl_synctask.h> 80#include <sys/fs/zfs.h> 81#include <sys/arc.h> 82#include <sys/callb.h> 83#include <sys/systeminfo.h> 84#include <sys/spa_boot.h> 85#include <sys/zfs_ioctl.h> 86#include <sys/dsl_scan.h> 87#include <sys/zfeature.h> 88#include <sys/dsl_destroy.h> 89#include <sys/zvol.h> 90 91#ifdef _KERNEL 92#include <sys/fm/protocol.h> 93#include <sys/fm/util.h> 94#include <sys/callb.h> 95#include <sys/zone.h> 96#include <sys/vmsystm.h> 97#endif /* _KERNEL */ 98 99#include "zfs_prop.h" 100#include "zfs_comutil.h" 101 102/* 103 * The interval, in seconds, at which failed configuration cache file writes 104 * should be retried. 105 */ 106int zfs_ccw_retry_interval = 300; 107 108typedef enum zti_modes { 109 ZTI_MODE_FIXED, /* value is # of threads (min 1) */ 110 ZTI_MODE_BATCH, /* cpu-intensive; value is ignored */ 111 ZTI_MODE_SCALE, /* Taskqs scale with CPUs. */ 112 ZTI_MODE_NULL, /* don't create a taskq */ 113 ZTI_NMODES 114} zti_modes_t; 115 116#define ZTI_P(n, q) { ZTI_MODE_FIXED, (n), (q) } 117#define ZTI_PCT(n) { ZTI_MODE_ONLINE_PERCENT, (n), 1 } 118#define ZTI_BATCH { ZTI_MODE_BATCH, 0, 1 } 119#define ZTI_SCALE { ZTI_MODE_SCALE, 0, 1 } 120#define ZTI_NULL { ZTI_MODE_NULL, 0, 0 } 121 122#define ZTI_N(n) ZTI_P(n, 1) 123#define ZTI_ONE ZTI_N(1) 124 125typedef struct zio_taskq_info { 126 zti_modes_t zti_mode; 127 uint_t zti_value; 128 uint_t zti_count; 129} zio_taskq_info_t; 130 131static const char *const zio_taskq_types[ZIO_TASKQ_TYPES] = { 132 "iss", "iss_h", "int", "int_h" 133}; 134 135/* 136 * This table defines the taskq settings for each ZFS I/O type. When 137 * initializing a pool, we use this table to create an appropriately sized 138 * taskq. Some operations are low volume and therefore have a small, static 139 * number of threads assigned to their taskqs using the ZTI_N(#) or ZTI_ONE 140 * macros. Other operations process a large amount of data; the ZTI_BATCH 141 * macro causes us to create a taskq oriented for throughput. Some operations 142 * are so high frequency and short-lived that the taskq itself can become a 143 * point of lock contention. The ZTI_P(#, #) macro indicates that we need an 144 * additional degree of parallelism specified by the number of threads per- 145 * taskq and the number of taskqs; when dispatching an event in this case, the 146 * particular taskq is chosen at random. ZTI_SCALE is similar to ZTI_BATCH, 147 * but with number of taskqs also scaling with number of CPUs. 148 * 149 * The different taskq priorities are to handle the different contexts (issue 150 * and interrupt) and then to reserve threads for ZIO_PRIORITY_NOW I/Os that 151 * need to be handled with minimum delay. 152 */ 153const zio_taskq_info_t zio_taskqs[ZIO_TYPES][ZIO_TASKQ_TYPES] = { 154 /* ISSUE ISSUE_HIGH INTR INTR_HIGH */ 155 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* NULL */ 156 { ZTI_N(8), ZTI_NULL, ZTI_SCALE, ZTI_NULL }, /* READ */ 157 { ZTI_BATCH, ZTI_N(5), ZTI_SCALE, ZTI_N(5) }, /* WRITE */ 158 { ZTI_SCALE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* FREE */ 159 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* CLAIM */ 160 { ZTI_ONE, ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* IOCTL */ 161 { ZTI_N(4), ZTI_NULL, ZTI_ONE, ZTI_NULL }, /* TRIM */ 162}; 163 164static void spa_sync_version(void *arg, dmu_tx_t *tx); 165static void spa_sync_props(void *arg, dmu_tx_t *tx); 166static boolean_t spa_has_active_shared_spare(spa_t *spa); 167static int spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport); 168static void spa_vdev_resilver_done(spa_t *spa); 169 170uint_t zio_taskq_batch_pct = 80; /* 1 thread per cpu in pset */ 171uint_t zio_taskq_batch_tpq; /* threads per taskq */ 172boolean_t zio_taskq_sysdc = B_TRUE; /* use SDC scheduling class */ 173uint_t zio_taskq_basedc = 80; /* base duty cycle */ 174 175boolean_t spa_create_process = B_TRUE; /* no process ==> no sysdc */ 176 177/* 178 * Report any spa_load_verify errors found, but do not fail spa_load. 179 * This is used by zdb to analyze non-idle pools. 180 */ 181boolean_t spa_load_verify_dryrun = B_FALSE; 182 183/* 184 * This (illegal) pool name is used when temporarily importing a spa_t in order 185 * to get the vdev stats associated with the imported devices. 186 */ 187#define TRYIMPORT_NAME "$import" 188 189/* 190 * For debugging purposes: print out vdev tree during pool import. 191 */ 192int spa_load_print_vdev_tree = B_FALSE; 193 194/* 195 * A non-zero value for zfs_max_missing_tvds means that we allow importing 196 * pools with missing top-level vdevs. This is strictly intended for advanced 197 * pool recovery cases since missing data is almost inevitable. Pools with 198 * missing devices can only be imported read-only for safety reasons, and their 199 * fail-mode will be automatically set to "continue". 200 * 201 * With 1 missing vdev we should be able to import the pool and mount all 202 * datasets. User data that was not modified after the missing device has been 203 * added should be recoverable. This means that snapshots created prior to the 204 * addition of that device should be completely intact. 205 * 206 * With 2 missing vdevs, some datasets may fail to mount since there are 207 * dataset statistics that are stored as regular metadata. Some data might be 208 * recoverable if those vdevs were added recently. 209 * 210 * With 3 or more missing vdevs, the pool is severely damaged and MOS entries 211 * may be missing entirely. Chances of data recovery are very low. Note that 212 * there are also risks of performing an inadvertent rewind as we might be 213 * missing all the vdevs with the latest uberblocks. 214 */ 215unsigned long zfs_max_missing_tvds = 0; 216 217/* 218 * The parameters below are similar to zfs_max_missing_tvds but are only 219 * intended for a preliminary open of the pool with an untrusted config which 220 * might be incomplete or out-dated. 221 * 222 * We are more tolerant for pools opened from a cachefile since we could have 223 * an out-dated cachefile where a device removal was not registered. 224 * We could have set the limit arbitrarily high but in the case where devices 225 * are really missing we would want to return the proper error codes; we chose 226 * SPA_DVAS_PER_BP - 1 so that some copies of the MOS would still be available 227 * and we get a chance to retrieve the trusted config. 228 */ 229uint64_t zfs_max_missing_tvds_cachefile = SPA_DVAS_PER_BP - 1; 230 231/* 232 * In the case where config was assembled by scanning device paths (/dev/dsks 233 * by default) we are less tolerant since all the existing devices should have 234 * been detected and we want spa_load to return the right error codes. 235 */ 236uint64_t zfs_max_missing_tvds_scan = 0; 237 238/* 239 * Debugging aid that pauses spa_sync() towards the end. 240 */ 241boolean_t zfs_pause_spa_sync = B_FALSE; 242 243/* 244 * Variables to indicate the livelist condense zthr func should wait at certain 245 * points for the livelist to be removed - used to test condense/destroy races 246 */ 247int zfs_livelist_condense_zthr_pause = 0; 248int zfs_livelist_condense_sync_pause = 0; 249 250/* 251 * Variables to track whether or not condense cancellation has been 252 * triggered in testing. 253 */ 254int zfs_livelist_condense_sync_cancel = 0; 255int zfs_livelist_condense_zthr_cancel = 0; 256 257/* 258 * Variable to track whether or not extra ALLOC blkptrs were added to a 259 * livelist entry while it was being condensed (caused by the way we track 260 * remapped blkptrs in dbuf_remap_impl) 261 */ 262int zfs_livelist_condense_new_alloc = 0; 263 264/* 265 * ========================================================================== 266 * SPA properties routines 267 * ========================================================================== 268 */ 269 270/* 271 * Add a (source=src, propname=propval) list to an nvlist. 272 */ 273static void 274spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, 275 uint64_t intval, zprop_source_t src) 276{ 277 const char *propname = zpool_prop_to_name(prop); 278 nvlist_t *propval; 279 280 VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); 281 VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); 282 283 if (strval != NULL) 284 VERIFY(nvlist_add_string(propval, ZPROP_VALUE, strval) == 0); 285 else 286 VERIFY(nvlist_add_uint64(propval, ZPROP_VALUE, intval) == 0); 287 288 VERIFY(nvlist_add_nvlist(nvl, propname, propval) == 0); 289 nvlist_free(propval); 290} 291 292/* 293 * Get property values from the spa configuration. 294 */ 295static void 296spa_prop_get_config(spa_t *spa, nvlist_t **nvp) 297{ 298 vdev_t *rvd = spa->spa_root_vdev; 299 dsl_pool_t *pool = spa->spa_dsl_pool; 300 uint64_t size, alloc, cap, version; 301 const zprop_source_t src = ZPROP_SRC_NONE; 302 spa_config_dirent_t *dp; 303 metaslab_class_t *mc = spa_normal_class(spa); 304 305 ASSERT(MUTEX_HELD(&spa->spa_props_lock)); 306 307 if (rvd != NULL) { 308 alloc = metaslab_class_get_alloc(mc); 309 alloc += metaslab_class_get_alloc(spa_special_class(spa)); 310 alloc += metaslab_class_get_alloc(spa_dedup_class(spa)); 311 alloc += metaslab_class_get_alloc(spa_embedded_log_class(spa)); 312 313 size = metaslab_class_get_space(mc); 314 size += metaslab_class_get_space(spa_special_class(spa)); 315 size += metaslab_class_get_space(spa_dedup_class(spa)); 316 size += metaslab_class_get_space(spa_embedded_log_class(spa)); 317 318 spa_prop_add_list(*nvp, ZPOOL_PROP_NAME, spa_name(spa), 0, src); 319 spa_prop_add_list(*nvp, ZPOOL_PROP_SIZE, NULL, size, src); 320 spa_prop_add_list(*nvp, ZPOOL_PROP_ALLOCATED, NULL, alloc, src); 321 spa_prop_add_list(*nvp, ZPOOL_PROP_FREE, NULL, 322 size - alloc, src); 323 spa_prop_add_list(*nvp, ZPOOL_PROP_CHECKPOINT, NULL, 324 spa->spa_checkpoint_info.sci_dspace, src); 325 326 spa_prop_add_list(*nvp, ZPOOL_PROP_FRAGMENTATION, NULL, 327 metaslab_class_fragmentation(mc), src); 328 spa_prop_add_list(*nvp, ZPOOL_PROP_EXPANDSZ, NULL, 329 metaslab_class_expandable_space(mc), src); 330 spa_prop_add_list(*nvp, ZPOOL_PROP_READONLY, NULL, 331 (spa_mode(spa) == SPA_MODE_READ), src); 332 333 cap = (size == 0) ? 0 : (alloc * 100 / size); 334 spa_prop_add_list(*nvp, ZPOOL_PROP_CAPACITY, NULL, cap, src); 335 336 spa_prop_add_list(*nvp, ZPOOL_PROP_DEDUPRATIO, NULL, 337 ddt_get_pool_dedup_ratio(spa), src); 338 339 spa_prop_add_list(*nvp, ZPOOL_PROP_HEALTH, NULL, 340 rvd->vdev_state, src); 341 342 version = spa_version(spa); 343 if (version == zpool_prop_default_numeric(ZPOOL_PROP_VERSION)) { 344 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 345 version, ZPROP_SRC_DEFAULT); 346 } else { 347 spa_prop_add_list(*nvp, ZPOOL_PROP_VERSION, NULL, 348 version, ZPROP_SRC_LOCAL); 349 } 350 spa_prop_add_list(*nvp, ZPOOL_PROP_LOAD_GUID, 351 NULL, spa_load_guid(spa), src); 352 } 353 354 if (pool != NULL) { 355 /* 356 * The $FREE directory was introduced in SPA_VERSION_DEADLISTS, 357 * when opening pools before this version freedir will be NULL. 358 */ 359 if (pool->dp_free_dir != NULL) { 360 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, NULL, 361 dsl_dir_phys(pool->dp_free_dir)->dd_used_bytes, 362 src); 363 } else { 364 spa_prop_add_list(*nvp, ZPOOL_PROP_FREEING, 365 NULL, 0, src); 366 } 367 368 if (pool->dp_leak_dir != NULL) { 369 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, NULL, 370 dsl_dir_phys(pool->dp_leak_dir)->dd_used_bytes, 371 src); 372 } else { 373 spa_prop_add_list(*nvp, ZPOOL_PROP_LEAKED, 374 NULL, 0, src); 375 } 376 } 377 378 spa_prop_add_list(*nvp, ZPOOL_PROP_GUID, NULL, spa_guid(spa), src); 379 380 if (spa->spa_comment != NULL) { 381 spa_prop_add_list(*nvp, ZPOOL_PROP_COMMENT, spa->spa_comment, 382 0, ZPROP_SRC_LOCAL); 383 } 384 385 if (spa->spa_compatibility != NULL) { 386 spa_prop_add_list(*nvp, ZPOOL_PROP_COMPATIBILITY, 387 spa->spa_compatibility, 0, ZPROP_SRC_LOCAL); 388 } 389 390 if (spa->spa_root != NULL) 391 spa_prop_add_list(*nvp, ZPOOL_PROP_ALTROOT, spa->spa_root, 392 0, ZPROP_SRC_LOCAL); 393 394 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 395 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 396 MIN(zfs_max_recordsize, SPA_MAXBLOCKSIZE), ZPROP_SRC_NONE); 397 } else { 398 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXBLOCKSIZE, NULL, 399 SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); 400 } 401 402 if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { 403 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 404 DNODE_MAX_SIZE, ZPROP_SRC_NONE); 405 } else { 406 spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, 407 DNODE_MIN_SIZE, ZPROP_SRC_NONE); 408 } 409 410 if ((dp = list_head(&spa->spa_config_list)) != NULL) { 411 if (dp->scd_path == NULL) { 412 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 413 "none", 0, ZPROP_SRC_LOCAL); 414 } else if (strcmp(dp->scd_path, spa_config_path) != 0) { 415 spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, 416 dp->scd_path, 0, ZPROP_SRC_LOCAL); 417 } 418 } 419} 420 421/* 422 * Get zpool property values. 423 */ 424int 425spa_prop_get(spa_t *spa, nvlist_t **nvp) 426{ 427 objset_t *mos = spa->spa_meta_objset; 428 zap_cursor_t zc; 429 zap_attribute_t za; 430 dsl_pool_t *dp; 431 int err; 432 433 err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); 434 if (err) 435 return (err); 436 437 dp = spa_get_dsl(spa); 438 dsl_pool_config_enter(dp, FTAG); 439 mutex_enter(&spa->spa_props_lock); 440 441 /* 442 * Get properties from the spa config. 443 */ 444 spa_prop_get_config(spa, nvp); 445 446 /* If no pool property object, no more prop to get. */ 447 if (mos == NULL || spa->spa_pool_props_object == 0) 448 goto out; 449 450 /* 451 * Get properties from the MOS pool property object. 452 */ 453 for (zap_cursor_init(&zc, mos, spa->spa_pool_props_object); 454 (err = zap_cursor_retrieve(&zc, &za)) == 0; 455 zap_cursor_advance(&zc)) { 456 uint64_t intval = 0; 457 char *strval = NULL; 458 zprop_source_t src = ZPROP_SRC_DEFAULT; 459 zpool_prop_t prop; 460 461 if ((prop = zpool_name_to_prop(za.za_name)) == ZPOOL_PROP_INVAL) 462 continue; 463 464 switch (za.za_integer_length) { 465 case 8: 466 /* integer property */ 467 if (za.za_first_integer != 468 zpool_prop_default_numeric(prop)) 469 src = ZPROP_SRC_LOCAL; 470 471 if (prop == ZPOOL_PROP_BOOTFS) { 472 dsl_dataset_t *ds = NULL; 473 474 err = dsl_dataset_hold_obj(dp, 475 za.za_first_integer, FTAG, &ds); 476 if (err != 0) 477 break; 478 479 strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, 480 KM_SLEEP); 481 dsl_dataset_name(ds, strval); 482 dsl_dataset_rele(ds, FTAG); 483 } else { 484 strval = NULL; 485 intval = za.za_first_integer; 486 } 487 488 spa_prop_add_list(*nvp, prop, strval, intval, src); 489 490 if (strval != NULL) 491 kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); 492 493 break; 494 495 case 1: 496 /* string property */ 497 strval = kmem_alloc(za.za_num_integers, KM_SLEEP); 498 err = zap_lookup(mos, spa->spa_pool_props_object, 499 za.za_name, 1, za.za_num_integers, strval); 500 if (err) { 501 kmem_free(strval, za.za_num_integers); 502 break; 503 } 504 spa_prop_add_list(*nvp, prop, strval, 0, src); 505 kmem_free(strval, za.za_num_integers); 506 break; 507 508 default: 509 break; 510 } 511 } 512 zap_cursor_fini(&zc); 513out: 514 mutex_exit(&spa->spa_props_lock); 515 dsl_pool_config_exit(dp, FTAG); 516 if (err && err != ENOENT) { 517 nvlist_free(*nvp); 518 *nvp = NULL; 519 return (err); 520 } 521 522 return (0); 523} 524 525/* 526 * Validate the given pool properties nvlist and modify the list 527 * for the property values to be set. 528 */ 529static int 530spa_prop_validate(spa_t *spa, nvlist_t *props) 531{ 532 nvpair_t *elem; 533 int error = 0, reset_bootfs = 0; 534 uint64_t objnum = 0; 535 boolean_t has_feature = B_FALSE; 536 537 elem = NULL; 538 while ((elem = nvlist_next_nvpair(props, elem)) != NULL) { 539 uint64_t intval; 540 char *strval, *slash, *check, *fname; 541 const char *propname = nvpair_name(elem); 542 zpool_prop_t prop = zpool_name_to_prop(propname); 543 544 switch (prop) { 545 case ZPOOL_PROP_INVAL: 546 if (!zpool_prop_feature(propname)) { 547 error = SET_ERROR(EINVAL); 548 break; 549 } 550 551 /* 552 * Sanitize the input. 553 */ 554 if (nvpair_type(elem) != DATA_TYPE_UINT64) { 555 error = SET_ERROR(EINVAL); 556 break; 557 } 558 559 if (nvpair_value_uint64(elem, &intval) != 0) { 560 error = SET_ERROR(EINVAL); 561 break; 562 } 563 564 if (intval != 0) { 565 error = SET_ERROR(EINVAL); 566 break; 567 } 568 569 fname = strchr(propname, '@') + 1; 570 if (zfeature_lookup_name(fname, NULL) != 0) { 571 error = SET_ERROR(EINVAL); 572 break; 573 } 574 575 has_feature = B_TRUE; 576 break; 577 578 case ZPOOL_PROP_VERSION: 579 error = nvpair_value_uint64(elem, &intval); 580 if (!error && 581 (intval < spa_version(spa) || 582 intval > SPA_VERSION_BEFORE_FEATURES || 583 has_feature)) 584 error = SET_ERROR(EINVAL); 585 break; 586 587 case ZPOOL_PROP_DELEGATION: 588 case ZPOOL_PROP_AUTOREPLACE: 589 case ZPOOL_PROP_LISTSNAPS: 590 case ZPOOL_PROP_AUTOEXPAND: 591 case ZPOOL_PROP_AUTOTRIM: 592 error = nvpair_value_uint64(elem, &intval); 593 if (!error && intval > 1) 594 error = SET_ERROR(EINVAL); 595 break; 596 597 case ZPOOL_PROP_MULTIHOST: 598 error = nvpair_value_uint64(elem, &intval); 599 if (!error && intval > 1) 600 error = SET_ERROR(EINVAL); 601 602 if (!error) { 603 uint32_t hostid = zone_get_hostid(NULL); 604 if (hostid) 605 spa->spa_hostid = hostid; 606 else 607 error = SET_ERROR(ENOTSUP); 608 } 609 610 break; 611 612 case ZPOOL_PROP_BOOTFS: 613 /* 614 * If the pool version is less than SPA_VERSION_BOOTFS, 615 * or the pool is still being created (version == 0), 616 * the bootfs property cannot be set. 617 */ 618 if (spa_version(spa) < SPA_VERSION_BOOTFS) { 619 error = SET_ERROR(ENOTSUP); 620 break; 621 } 622 623 /* 624 * Make sure the vdev config is bootable 625 */ 626 if (!vdev_is_bootable(spa->spa_root_vdev)) { 627 error = SET_ERROR(ENOTSUP); 628 break; 629 } 630 631 reset_bootfs = 1; 632 633 error = nvpair_value_string(elem, &strval); 634 635 if (!error) { 636 objset_t *os; 637 638 if (strval == NULL || strval[0] == '\0') { 639 objnum = zpool_prop_default_numeric( 640 ZPOOL_PROP_BOOTFS); 641 break; 642 } 643 644 error = dmu_objset_hold(strval, FTAG, &os); 645 if (error != 0) 646 break; 647 648 /* Must be ZPL. */ 649 if (dmu_objset_type(os) != DMU_OST_ZFS) { 650 error = SET_ERROR(ENOTSUP); 651 } else { 652 objnum = dmu_objset_id(os); 653 } 654 dmu_objset_rele(os, FTAG); 655 } 656 break; 657 658 case ZPOOL_PROP_FAILUREMODE: 659 error = nvpair_value_uint64(elem, &intval); 660 if (!error && intval > ZIO_FAILURE_MODE_PANIC) 661 error = SET_ERROR(EINVAL); 662 663 /* 664 * This is a special case which only occurs when 665 * the pool has completely failed. This allows 666 * the user to change the in-core failmode property 667 * without syncing it out to disk (I/Os might 668 * currently be blocked). We do this by returning 669 * EIO to the caller (spa_prop_set) to trick it 670 * into thinking we encountered a property validation 671 * error. 672 */ 673 if (!error && spa_suspended(spa)) { 674 spa->spa_failmode = intval; 675 error = SET_ERROR(EIO); 676 } 677 break; 678 679 case ZPOOL_PROP_CACHEFILE: 680 if ((error = nvpair_value_string(elem, &strval)) != 0) 681 break; 682 683 if (strval[0] == '\0') 684 break; 685 686 if (strcmp(strval, "none") == 0) 687 break; 688 689 if (strval[0] != '/') { 690 error = SET_ERROR(EINVAL); 691 break; 692 } 693 694 slash = strrchr(strval, '/'); 695 ASSERT(slash != NULL); 696 697 if (slash[1] == '\0' || strcmp(slash, "/.") == 0 || 698 strcmp(slash, "/..") == 0) 699 error = SET_ERROR(EINVAL); 700 break; 701 702 case ZPOOL_PROP_COMMENT: 703 if ((error = nvpair_value_string(elem, &strval)) != 0) 704 break; 705 for (check = strval; *check != '\0'; check++) { 706 if (!isprint(*check)) { 707 error = SET_ERROR(EINVAL); 708 break; 709 } 710 } 711 if (strlen(strval) > ZPROP_MAX_COMMENT) 712 error = SET_ERROR(E2BIG); 713 break; 714 715 default: 716 break; 717 } 718 719 if (error) 720 break; 721 } 722 723 (void) nvlist_remove_all(props, 724 zpool_prop_to_name(ZPOOL_PROP_DEDUPDITTO)); 725 726 if (!error && reset_bootfs) { 727 error = nvlist_remove(props, 728 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), DATA_TYPE_STRING); 729 730 if (!error) { 731 error = nvlist_add_uint64(props, 732 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), objnum); 733 } 734 } 735 736 return (error); 737} 738 739void 740spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) 741{ 742 char *cachefile; 743 spa_config_dirent_t *dp; 744 745 if (nvlist_lookup_string(nvp, zpool_prop_to_name(ZPOOL_PROP_CACHEFILE), 746 &cachefile) != 0) 747 return; 748 749 dp = kmem_alloc(sizeof (spa_config_dirent_t), 750 KM_SLEEP); 751 752 if (cachefile[0] == '\0') 753 dp->scd_path = spa_strdup(spa_config_path); 754 else if (strcmp(cachefile, "none") == 0) 755 dp->scd_path = NULL; 756 else 757 dp->scd_path = spa_strdup(cachefile); 758 759 list_insert_head(&spa->spa_config_list, dp); 760 if (need_sync) 761 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 762} 763 764int 765spa_prop_set(spa_t *spa, nvlist_t *nvp) 766{ 767 int error; 768 nvpair_t *elem = NULL; 769 boolean_t need_sync = B_FALSE; 770 771 if ((error = spa_prop_validate(spa, nvp)) != 0) 772 return (error); 773 774 while ((elem = nvlist_next_nvpair(nvp, elem)) != NULL) { 775 zpool_prop_t prop = zpool_name_to_prop(nvpair_name(elem)); 776 777 if (prop == ZPOOL_PROP_CACHEFILE || 778 prop == ZPOOL_PROP_ALTROOT || 779 prop == ZPOOL_PROP_READONLY) 780 continue; 781 782 if (prop == ZPOOL_PROP_VERSION || prop == ZPOOL_PROP_INVAL) { 783 uint64_t ver; 784 785 if (prop == ZPOOL_PROP_VERSION) { 786 VERIFY(nvpair_value_uint64(elem, &ver) == 0); 787 } else { 788 ASSERT(zpool_prop_feature(nvpair_name(elem))); 789 ver = SPA_VERSION_FEATURES; 790 need_sync = B_TRUE; 791 } 792 793 /* Save time if the version is already set. */ 794 if (ver == spa_version(spa)) 795 continue; 796 797 /* 798 * In addition to the pool directory object, we might 799 * create the pool properties object, the features for 800 * read object, the features for write object, or the 801 * feature descriptions object. 802 */ 803 error = dsl_sync_task(spa->spa_name, NULL, 804 spa_sync_version, &ver, 805 6, ZFS_SPACE_CHECK_RESERVED); 806 if (error) 807 return (error); 808 continue; 809 } 810 811 need_sync = B_TRUE; 812 break; 813 } 814 815 if (need_sync) { 816 return (dsl_sync_task(spa->spa_name, NULL, spa_sync_props, 817 nvp, 6, ZFS_SPACE_CHECK_RESERVED)); 818 } 819 820 return (0); 821} 822 823/* 824 * If the bootfs property value is dsobj, clear it. 825 */ 826void 827spa_prop_clear_bootfs(spa_t *spa, uint64_t dsobj, dmu_tx_t *tx) 828{ 829 if (spa->spa_bootfs == dsobj && spa->spa_pool_props_object != 0) { 830 VERIFY(zap_remove(spa->spa_meta_objset, 831 spa->spa_pool_props_object, 832 zpool_prop_to_name(ZPOOL_PROP_BOOTFS), tx) == 0); 833 spa->spa_bootfs = 0; 834 } 835} 836 837/*ARGSUSED*/ 838static int 839spa_change_guid_check(void *arg, dmu_tx_t *tx) 840{ 841 uint64_t *newguid __maybe_unused = arg; 842 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 843 vdev_t *rvd = spa->spa_root_vdev; 844 uint64_t vdev_state; 845 846 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 847 int error = (spa_has_checkpoint(spa)) ? 848 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 849 return (SET_ERROR(error)); 850 } 851 852 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 853 vdev_state = rvd->vdev_state; 854 spa_config_exit(spa, SCL_STATE, FTAG); 855 856 if (vdev_state != VDEV_STATE_HEALTHY) 857 return (SET_ERROR(ENXIO)); 858 859 ASSERT3U(spa_guid(spa), !=, *newguid); 860 861 return (0); 862} 863 864static void 865spa_change_guid_sync(void *arg, dmu_tx_t *tx) 866{ 867 uint64_t *newguid = arg; 868 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 869 uint64_t oldguid; 870 vdev_t *rvd = spa->spa_root_vdev; 871 872 oldguid = spa_guid(spa); 873 874 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 875 rvd->vdev_guid = *newguid; 876 rvd->vdev_guid_sum += (*newguid - oldguid); 877 vdev_config_dirty(rvd); 878 spa_config_exit(spa, SCL_STATE, FTAG); 879 880 spa_history_log_internal(spa, "guid change", tx, "old=%llu new=%llu", 881 (u_longlong_t)oldguid, (u_longlong_t)*newguid); 882} 883 884/* 885 * Change the GUID for the pool. This is done so that we can later 886 * re-import a pool built from a clone of our own vdevs. We will modify 887 * the root vdev's guid, our own pool guid, and then mark all of our 888 * vdevs dirty. Note that we must make sure that all our vdevs are 889 * online when we do this, or else any vdevs that weren't present 890 * would be orphaned from our pool. We are also going to issue a 891 * sysevent to update any watchers. 892 */ 893int 894spa_change_guid(spa_t *spa) 895{ 896 int error; 897 uint64_t guid; 898 899 mutex_enter(&spa->spa_vdev_top_lock); 900 mutex_enter(&spa_namespace_lock); 901 guid = spa_generate_guid(NULL); 902 903 error = dsl_sync_task(spa->spa_name, spa_change_guid_check, 904 spa_change_guid_sync, &guid, 5, ZFS_SPACE_CHECK_RESERVED); 905 906 if (error == 0) { 907 spa_write_cachefile(spa, B_FALSE, B_TRUE); 908 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_REGUID); 909 } 910 911 mutex_exit(&spa_namespace_lock); 912 mutex_exit(&spa->spa_vdev_top_lock); 913 914 return (error); 915} 916 917/* 918 * ========================================================================== 919 * SPA state manipulation (open/create/destroy/import/export) 920 * ========================================================================== 921 */ 922 923static int 924spa_error_entry_compare(const void *a, const void *b) 925{ 926 const spa_error_entry_t *sa = (const spa_error_entry_t *)a; 927 const spa_error_entry_t *sb = (const spa_error_entry_t *)b; 928 int ret; 929 930 ret = memcmp(&sa->se_bookmark, &sb->se_bookmark, 931 sizeof (zbookmark_phys_t)); 932 933 return (TREE_ISIGN(ret)); 934} 935 936/* 937 * Utility function which retrieves copies of the current logs and 938 * re-initializes them in the process. 939 */ 940void 941spa_get_errlists(spa_t *spa, avl_tree_t *last, avl_tree_t *scrub) 942{ 943 ASSERT(MUTEX_HELD(&spa->spa_errlist_lock)); 944 945 bcopy(&spa->spa_errlist_last, last, sizeof (avl_tree_t)); 946 bcopy(&spa->spa_errlist_scrub, scrub, sizeof (avl_tree_t)); 947 948 avl_create(&spa->spa_errlist_scrub, 949 spa_error_entry_compare, sizeof (spa_error_entry_t), 950 offsetof(spa_error_entry_t, se_avl)); 951 avl_create(&spa->spa_errlist_last, 952 spa_error_entry_compare, sizeof (spa_error_entry_t), 953 offsetof(spa_error_entry_t, se_avl)); 954} 955 956static void 957spa_taskqs_init(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 958{ 959 const zio_taskq_info_t *ztip = &zio_taskqs[t][q]; 960 enum zti_modes mode = ztip->zti_mode; 961 uint_t value = ztip->zti_value; 962 uint_t count = ztip->zti_count; 963 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 964 uint_t cpus, flags = TASKQ_DYNAMIC; 965 boolean_t batch = B_FALSE; 966 967 switch (mode) { 968 case ZTI_MODE_FIXED: 969 ASSERT3U(value, >, 0); 970 break; 971 972 case ZTI_MODE_BATCH: 973 batch = B_TRUE; 974 flags |= TASKQ_THREADS_CPU_PCT; 975 value = MIN(zio_taskq_batch_pct, 100); 976 break; 977 978 case ZTI_MODE_SCALE: 979 flags |= TASKQ_THREADS_CPU_PCT; 980 /* 981 * We want more taskqs to reduce lock contention, but we want 982 * less for better request ordering and CPU utilization. 983 */ 984 cpus = MAX(1, boot_ncpus * zio_taskq_batch_pct / 100); 985 if (zio_taskq_batch_tpq > 0) { 986 count = MAX(1, (cpus + zio_taskq_batch_tpq / 2) / 987 zio_taskq_batch_tpq); 988 } else { 989 /* 990 * Prefer 6 threads per taskq, but no more taskqs 991 * than threads in them on large systems. For 80%: 992 * 993 * taskq taskq total 994 * cpus taskqs percent threads threads 995 * ------- ------- ------- ------- ------- 996 * 1 1 80% 1 1 997 * 2 1 80% 1 1 998 * 4 1 80% 3 3 999 * 8 2 40% 3 6 1000 * 16 3 27% 4 12 1001 * 32 5 16% 5 25 1002 * 64 7 11% 7 49 1003 * 128 10 8% 10 100 1004 * 256 14 6% 15 210 1005 */ 1006 count = 1 + cpus / 6; 1007 while (count * count > cpus) 1008 count--; 1009 } 1010 /* Limit each taskq within 100% to not trigger assertion. */ 1011 count = MAX(count, (zio_taskq_batch_pct + 99) / 100); 1012 value = (zio_taskq_batch_pct + count / 2) / count; 1013 break; 1014 1015 case ZTI_MODE_NULL: 1016 tqs->stqs_count = 0; 1017 tqs->stqs_taskq = NULL; 1018 return; 1019 1020 default: 1021 panic("unrecognized mode for %s_%s taskq (%u:%u) in " 1022 "spa_activate()", 1023 zio_type_name[t], zio_taskq_types[q], mode, value); 1024 break; 1025 } 1026 1027 ASSERT3U(count, >, 0); 1028 tqs->stqs_count = count; 1029 tqs->stqs_taskq = kmem_alloc(count * sizeof (taskq_t *), KM_SLEEP); 1030 1031 for (uint_t i = 0; i < count; i++) { 1032 taskq_t *tq; 1033 char name[32]; 1034 1035 if (count > 1) 1036 (void) snprintf(name, sizeof (name), "%s_%s_%u", 1037 zio_type_name[t], zio_taskq_types[q], i); 1038 else 1039 (void) snprintf(name, sizeof (name), "%s_%s", 1040 zio_type_name[t], zio_taskq_types[q]); 1041 1042 if (zio_taskq_sysdc && spa->spa_proc != &p0) { 1043 if (batch) 1044 flags |= TASKQ_DC_BATCH; 1045 1046 tq = taskq_create_sysdc(name, value, 50, INT_MAX, 1047 spa->spa_proc, zio_taskq_basedc, flags); 1048 } else { 1049 pri_t pri = maxclsyspri; 1050 /* 1051 * The write issue taskq can be extremely CPU 1052 * intensive. Run it at slightly less important 1053 * priority than the other taskqs. 1054 * 1055 * Under Linux and FreeBSD this means incrementing 1056 * the priority value as opposed to platforms like 1057 * illumos where it should be decremented. 1058 * 1059 * On FreeBSD, if priorities divided by four (RQ_PPQ) 1060 * are equal then a difference between them is 1061 * insignificant. 1062 */ 1063 if (t == ZIO_TYPE_WRITE && q == ZIO_TASKQ_ISSUE) { 1064#if defined(__linux__) 1065 pri++; 1066#elif defined(__FreeBSD__) 1067 pri += 4; 1068#else 1069#error "unknown OS" 1070#endif 1071 } 1072 tq = taskq_create_proc(name, value, pri, 50, 1073 INT_MAX, spa->spa_proc, flags); 1074 } 1075 1076 tqs->stqs_taskq[i] = tq; 1077 } 1078} 1079 1080static void 1081spa_taskqs_fini(spa_t *spa, zio_type_t t, zio_taskq_type_t q) 1082{ 1083 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1084 1085 if (tqs->stqs_taskq == NULL) { 1086 ASSERT3U(tqs->stqs_count, ==, 0); 1087 return; 1088 } 1089 1090 for (uint_t i = 0; i < tqs->stqs_count; i++) { 1091 ASSERT3P(tqs->stqs_taskq[i], !=, NULL); 1092 taskq_destroy(tqs->stqs_taskq[i]); 1093 } 1094 1095 kmem_free(tqs->stqs_taskq, tqs->stqs_count * sizeof (taskq_t *)); 1096 tqs->stqs_taskq = NULL; 1097} 1098 1099/* 1100 * Dispatch a task to the appropriate taskq for the ZFS I/O type and priority. 1101 * Note that a type may have multiple discrete taskqs to avoid lock contention 1102 * on the taskq itself. In that case we choose which taskq at random by using 1103 * the low bits of gethrtime(). 1104 */ 1105void 1106spa_taskq_dispatch_ent(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1107 task_func_t *func, void *arg, uint_t flags, taskq_ent_t *ent) 1108{ 1109 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1110 taskq_t *tq; 1111 1112 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1113 ASSERT3U(tqs->stqs_count, !=, 0); 1114 1115 if (tqs->stqs_count == 1) { 1116 tq = tqs->stqs_taskq[0]; 1117 } else { 1118 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1119 } 1120 1121 taskq_dispatch_ent(tq, func, arg, flags, ent); 1122} 1123 1124/* 1125 * Same as spa_taskq_dispatch_ent() but block on the task until completion. 1126 */ 1127void 1128spa_taskq_dispatch_sync(spa_t *spa, zio_type_t t, zio_taskq_type_t q, 1129 task_func_t *func, void *arg, uint_t flags) 1130{ 1131 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 1132 taskq_t *tq; 1133 taskqid_t id; 1134 1135 ASSERT3P(tqs->stqs_taskq, !=, NULL); 1136 ASSERT3U(tqs->stqs_count, !=, 0); 1137 1138 if (tqs->stqs_count == 1) { 1139 tq = tqs->stqs_taskq[0]; 1140 } else { 1141 tq = tqs->stqs_taskq[((uint64_t)gethrtime()) % tqs->stqs_count]; 1142 } 1143 1144 id = taskq_dispatch(tq, func, arg, flags); 1145 if (id) 1146 taskq_wait_id(tq, id); 1147} 1148 1149static void 1150spa_create_zio_taskqs(spa_t *spa) 1151{ 1152 for (int t = 0; t < ZIO_TYPES; t++) { 1153 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1154 spa_taskqs_init(spa, t, q); 1155 } 1156 } 1157} 1158 1159/* 1160 * Disabled until spa_thread() can be adapted for Linux. 1161 */ 1162#undef HAVE_SPA_THREAD 1163 1164#if defined(_KERNEL) && defined(HAVE_SPA_THREAD) 1165static void 1166spa_thread(void *arg) 1167{ 1168 psetid_t zio_taskq_psrset_bind = PS_NONE; 1169 callb_cpr_t cprinfo; 1170 1171 spa_t *spa = arg; 1172 user_t *pu = PTOU(curproc); 1173 1174 CALLB_CPR_INIT(&cprinfo, &spa->spa_proc_lock, callb_generic_cpr, 1175 spa->spa_name); 1176 1177 ASSERT(curproc != &p0); 1178 (void) snprintf(pu->u_psargs, sizeof (pu->u_psargs), 1179 "zpool-%s", spa->spa_name); 1180 (void) strlcpy(pu->u_comm, pu->u_psargs, sizeof (pu->u_comm)); 1181 1182 /* bind this thread to the requested psrset */ 1183 if (zio_taskq_psrset_bind != PS_NONE) { 1184 pool_lock(); 1185 mutex_enter(&cpu_lock); 1186 mutex_enter(&pidlock); 1187 mutex_enter(&curproc->p_lock); 1188 1189 if (cpupart_bind_thread(curthread, zio_taskq_psrset_bind, 1190 0, NULL, NULL) == 0) { 1191 curthread->t_bind_pset = zio_taskq_psrset_bind; 1192 } else { 1193 cmn_err(CE_WARN, 1194 "Couldn't bind process for zfs pool \"%s\" to " 1195 "pset %d\n", spa->spa_name, zio_taskq_psrset_bind); 1196 } 1197 1198 mutex_exit(&curproc->p_lock); 1199 mutex_exit(&pidlock); 1200 mutex_exit(&cpu_lock); 1201 pool_unlock(); 1202 } 1203 1204 if (zio_taskq_sysdc) { 1205 sysdc_thread_enter(curthread, 100, 0); 1206 } 1207 1208 spa->spa_proc = curproc; 1209 spa->spa_did = curthread->t_did; 1210 1211 spa_create_zio_taskqs(spa); 1212 1213 mutex_enter(&spa->spa_proc_lock); 1214 ASSERT(spa->spa_proc_state == SPA_PROC_CREATED); 1215 1216 spa->spa_proc_state = SPA_PROC_ACTIVE; 1217 cv_broadcast(&spa->spa_proc_cv); 1218 1219 CALLB_CPR_SAFE_BEGIN(&cprinfo); 1220 while (spa->spa_proc_state == SPA_PROC_ACTIVE) 1221 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1222 CALLB_CPR_SAFE_END(&cprinfo, &spa->spa_proc_lock); 1223 1224 ASSERT(spa->spa_proc_state == SPA_PROC_DEACTIVATE); 1225 spa->spa_proc_state = SPA_PROC_GONE; 1226 spa->spa_proc = &p0; 1227 cv_broadcast(&spa->spa_proc_cv); 1228 CALLB_CPR_EXIT(&cprinfo); /* drops spa_proc_lock */ 1229 1230 mutex_enter(&curproc->p_lock); 1231 lwp_exit(); 1232} 1233#endif 1234 1235/* 1236 * Activate an uninitialized pool. 1237 */ 1238static void 1239spa_activate(spa_t *spa, spa_mode_t mode) 1240{ 1241 ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED); 1242 1243 spa->spa_state = POOL_STATE_ACTIVE; 1244 spa->spa_mode = mode; 1245 1246 spa->spa_normal_class = metaslab_class_create(spa, zfs_metaslab_ops); 1247 spa->spa_log_class = metaslab_class_create(spa, zfs_metaslab_ops); 1248 spa->spa_embedded_log_class = 1249 metaslab_class_create(spa, zfs_metaslab_ops); 1250 spa->spa_special_class = metaslab_class_create(spa, zfs_metaslab_ops); 1251 spa->spa_dedup_class = metaslab_class_create(spa, zfs_metaslab_ops); 1252 1253 /* Try to create a covering process */ 1254 mutex_enter(&spa->spa_proc_lock); 1255 ASSERT(spa->spa_proc_state == SPA_PROC_NONE); 1256 ASSERT(spa->spa_proc == &p0); 1257 spa->spa_did = 0; 1258 1259#ifdef HAVE_SPA_THREAD 1260 /* Only create a process if we're going to be around a while. */ 1261 if (spa_create_process && strcmp(spa->spa_name, TRYIMPORT_NAME) != 0) { 1262 if (newproc(spa_thread, (caddr_t)spa, syscid, maxclsyspri, 1263 NULL, 0) == 0) { 1264 spa->spa_proc_state = SPA_PROC_CREATED; 1265 while (spa->spa_proc_state == SPA_PROC_CREATED) { 1266 cv_wait(&spa->spa_proc_cv, 1267 &spa->spa_proc_lock); 1268 } 1269 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1270 ASSERT(spa->spa_proc != &p0); 1271 ASSERT(spa->spa_did != 0); 1272 } else { 1273#ifdef _KERNEL 1274 cmn_err(CE_WARN, 1275 "Couldn't create process for zfs pool \"%s\"\n", 1276 spa->spa_name); 1277#endif 1278 } 1279 } 1280#endif /* HAVE_SPA_THREAD */ 1281 mutex_exit(&spa->spa_proc_lock); 1282 1283 /* If we didn't create a process, we need to create our taskqs. */ 1284 if (spa->spa_proc == &p0) { 1285 spa_create_zio_taskqs(spa); 1286 } 1287 1288 for (size_t i = 0; i < TXG_SIZE; i++) { 1289 spa->spa_txg_zio[i] = zio_root(spa, NULL, NULL, 1290 ZIO_FLAG_CANFAIL); 1291 } 1292 1293 list_create(&spa->spa_config_dirty_list, sizeof (vdev_t), 1294 offsetof(vdev_t, vdev_config_dirty_node)); 1295 list_create(&spa->spa_evicting_os_list, sizeof (objset_t), 1296 offsetof(objset_t, os_evicting_node)); 1297 list_create(&spa->spa_state_dirty_list, sizeof (vdev_t), 1298 offsetof(vdev_t, vdev_state_dirty_node)); 1299 1300 txg_list_create(&spa->spa_vdev_txg_list, spa, 1301 offsetof(struct vdev, vdev_txg_node)); 1302 1303 avl_create(&spa->spa_errlist_scrub, 1304 spa_error_entry_compare, sizeof (spa_error_entry_t), 1305 offsetof(spa_error_entry_t, se_avl)); 1306 avl_create(&spa->spa_errlist_last, 1307 spa_error_entry_compare, sizeof (spa_error_entry_t), 1308 offsetof(spa_error_entry_t, se_avl)); 1309 1310 spa_keystore_init(&spa->spa_keystore); 1311 1312 /* 1313 * This taskq is used to perform zvol-minor-related tasks 1314 * asynchronously. This has several advantages, including easy 1315 * resolution of various deadlocks. 1316 * 1317 * The taskq must be single threaded to ensure tasks are always 1318 * processed in the order in which they were dispatched. 1319 * 1320 * A taskq per pool allows one to keep the pools independent. 1321 * This way if one pool is suspended, it will not impact another. 1322 * 1323 * The preferred location to dispatch a zvol minor task is a sync 1324 * task. In this context, there is easy access to the spa_t and minimal 1325 * error handling is required because the sync task must succeed. 1326 */ 1327 spa->spa_zvol_taskq = taskq_create("z_zvol", 1, defclsyspri, 1328 1, INT_MAX, 0); 1329 1330 /* 1331 * Taskq dedicated to prefetcher threads: this is used to prevent the 1332 * pool traverse code from monopolizing the global (and limited) 1333 * system_taskq by inappropriately scheduling long running tasks on it. 1334 */ 1335 spa->spa_prefetch_taskq = taskq_create("z_prefetch", 100, 1336 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1337 1338 /* 1339 * The taskq to upgrade datasets in this pool. Currently used by 1340 * feature SPA_FEATURE_USEROBJ_ACCOUNTING/SPA_FEATURE_PROJECT_QUOTA. 1341 */ 1342 spa->spa_upgrade_taskq = taskq_create("z_upgrade", 100, 1343 defclsyspri, 1, INT_MAX, TASKQ_DYNAMIC | TASKQ_THREADS_CPU_PCT); 1344} 1345 1346/* 1347 * Opposite of spa_activate(). 1348 */ 1349static void 1350spa_deactivate(spa_t *spa) 1351{ 1352 ASSERT(spa->spa_sync_on == B_FALSE); 1353 ASSERT(spa->spa_dsl_pool == NULL); 1354 ASSERT(spa->spa_root_vdev == NULL); 1355 ASSERT(spa->spa_async_zio_root == NULL); 1356 ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED); 1357 1358 spa_evicting_os_wait(spa); 1359 1360 if (spa->spa_zvol_taskq) { 1361 taskq_destroy(spa->spa_zvol_taskq); 1362 spa->spa_zvol_taskq = NULL; 1363 } 1364 1365 if (spa->spa_prefetch_taskq) { 1366 taskq_destroy(spa->spa_prefetch_taskq); 1367 spa->spa_prefetch_taskq = NULL; 1368 } 1369 1370 if (spa->spa_upgrade_taskq) { 1371 taskq_destroy(spa->spa_upgrade_taskq); 1372 spa->spa_upgrade_taskq = NULL; 1373 } 1374 1375 txg_list_destroy(&spa->spa_vdev_txg_list); 1376 1377 list_destroy(&spa->spa_config_dirty_list); 1378 list_destroy(&spa->spa_evicting_os_list); 1379 list_destroy(&spa->spa_state_dirty_list); 1380 1381 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 1382 1383 for (int t = 0; t < ZIO_TYPES; t++) { 1384 for (int q = 0; q < ZIO_TASKQ_TYPES; q++) { 1385 spa_taskqs_fini(spa, t, q); 1386 } 1387 } 1388 1389 for (size_t i = 0; i < TXG_SIZE; i++) { 1390 ASSERT3P(spa->spa_txg_zio[i], !=, NULL); 1391 VERIFY0(zio_wait(spa->spa_txg_zio[i])); 1392 spa->spa_txg_zio[i] = NULL; 1393 } 1394 1395 metaslab_class_destroy(spa->spa_normal_class); 1396 spa->spa_normal_class = NULL; 1397 1398 metaslab_class_destroy(spa->spa_log_class); 1399 spa->spa_log_class = NULL; 1400 1401 metaslab_class_destroy(spa->spa_embedded_log_class); 1402 spa->spa_embedded_log_class = NULL; 1403 1404 metaslab_class_destroy(spa->spa_special_class); 1405 spa->spa_special_class = NULL; 1406 1407 metaslab_class_destroy(spa->spa_dedup_class); 1408 spa->spa_dedup_class = NULL; 1409 1410 /* 1411 * If this was part of an import or the open otherwise failed, we may 1412 * still have errors left in the queues. Empty them just in case. 1413 */ 1414 spa_errlog_drain(spa); 1415 avl_destroy(&spa->spa_errlist_scrub); 1416 avl_destroy(&spa->spa_errlist_last); 1417 1418 spa_keystore_fini(&spa->spa_keystore); 1419 1420 spa->spa_state = POOL_STATE_UNINITIALIZED; 1421 1422 mutex_enter(&spa->spa_proc_lock); 1423 if (spa->spa_proc_state != SPA_PROC_NONE) { 1424 ASSERT(spa->spa_proc_state == SPA_PROC_ACTIVE); 1425 spa->spa_proc_state = SPA_PROC_DEACTIVATE; 1426 cv_broadcast(&spa->spa_proc_cv); 1427 while (spa->spa_proc_state == SPA_PROC_DEACTIVATE) { 1428 ASSERT(spa->spa_proc != &p0); 1429 cv_wait(&spa->spa_proc_cv, &spa->spa_proc_lock); 1430 } 1431 ASSERT(spa->spa_proc_state == SPA_PROC_GONE); 1432 spa->spa_proc_state = SPA_PROC_NONE; 1433 } 1434 ASSERT(spa->spa_proc == &p0); 1435 mutex_exit(&spa->spa_proc_lock); 1436 1437 /* 1438 * We want to make sure spa_thread() has actually exited the ZFS 1439 * module, so that the module can't be unloaded out from underneath 1440 * it. 1441 */ 1442 if (spa->spa_did != 0) { 1443 thread_join(spa->spa_did); 1444 spa->spa_did = 0; 1445 } 1446} 1447 1448/* 1449 * Verify a pool configuration, and construct the vdev tree appropriately. This 1450 * will create all the necessary vdevs in the appropriate layout, with each vdev 1451 * in the CLOSED state. This will prep the pool before open/creation/import. 1452 * All vdev validation is done by the vdev_alloc() routine. 1453 */ 1454int 1455spa_config_parse(spa_t *spa, vdev_t **vdp, nvlist_t *nv, vdev_t *parent, 1456 uint_t id, int atype) 1457{ 1458 nvlist_t **child; 1459 uint_t children; 1460 int error; 1461 1462 if ((error = vdev_alloc(spa, vdp, nv, parent, id, atype)) != 0) 1463 return (error); 1464 1465 if ((*vdp)->vdev_ops->vdev_op_leaf) 1466 return (0); 1467 1468 error = nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN, 1469 &child, &children); 1470 1471 if (error == ENOENT) 1472 return (0); 1473 1474 if (error) { 1475 vdev_free(*vdp); 1476 *vdp = NULL; 1477 return (SET_ERROR(EINVAL)); 1478 } 1479 1480 for (int c = 0; c < children; c++) { 1481 vdev_t *vd; 1482 if ((error = spa_config_parse(spa, &vd, child[c], *vdp, c, 1483 atype)) != 0) { 1484 vdev_free(*vdp); 1485 *vdp = NULL; 1486 return (error); 1487 } 1488 } 1489 1490 ASSERT(*vdp != NULL); 1491 1492 return (0); 1493} 1494 1495static boolean_t 1496spa_should_flush_logs_on_unload(spa_t *spa) 1497{ 1498 if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) 1499 return (B_FALSE); 1500 1501 if (!spa_writeable(spa)) 1502 return (B_FALSE); 1503 1504 if (!spa->spa_sync_on) 1505 return (B_FALSE); 1506 1507 if (spa_state(spa) != POOL_STATE_EXPORTED) 1508 return (B_FALSE); 1509 1510 if (zfs_keep_log_spacemaps_at_export) 1511 return (B_FALSE); 1512 1513 return (B_TRUE); 1514} 1515 1516/* 1517 * Opens a transaction that will set the flag that will instruct 1518 * spa_sync to attempt to flush all the metaslabs for that txg. 1519 */ 1520static void 1521spa_unload_log_sm_flush_all(spa_t *spa) 1522{ 1523 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 1524 VERIFY0(dmu_tx_assign(tx, TXG_WAIT)); 1525 1526 ASSERT3U(spa->spa_log_flushall_txg, ==, 0); 1527 spa->spa_log_flushall_txg = dmu_tx_get_txg(tx); 1528 1529 dmu_tx_commit(tx); 1530 txg_wait_synced(spa_get_dsl(spa), spa->spa_log_flushall_txg); 1531} 1532 1533static void 1534spa_unload_log_sm_metadata(spa_t *spa) 1535{ 1536 void *cookie = NULL; 1537 spa_log_sm_t *sls; 1538 while ((sls = avl_destroy_nodes(&spa->spa_sm_logs_by_txg, 1539 &cookie)) != NULL) { 1540 VERIFY0(sls->sls_mscount); 1541 kmem_free(sls, sizeof (spa_log_sm_t)); 1542 } 1543 1544 for (log_summary_entry_t *e = list_head(&spa->spa_log_summary); 1545 e != NULL; e = list_head(&spa->spa_log_summary)) { 1546 VERIFY0(e->lse_mscount); 1547 list_remove(&spa->spa_log_summary, e); 1548 kmem_free(e, sizeof (log_summary_entry_t)); 1549 } 1550 1551 spa->spa_unflushed_stats.sus_nblocks = 0; 1552 spa->spa_unflushed_stats.sus_memused = 0; 1553 spa->spa_unflushed_stats.sus_blocklimit = 0; 1554} 1555 1556static void 1557spa_destroy_aux_threads(spa_t *spa) 1558{ 1559 if (spa->spa_condense_zthr != NULL) { 1560 zthr_destroy(spa->spa_condense_zthr); 1561 spa->spa_condense_zthr = NULL; 1562 } 1563 if (spa->spa_checkpoint_discard_zthr != NULL) { 1564 zthr_destroy(spa->spa_checkpoint_discard_zthr); 1565 spa->spa_checkpoint_discard_zthr = NULL; 1566 } 1567 if (spa->spa_livelist_delete_zthr != NULL) { 1568 zthr_destroy(spa->spa_livelist_delete_zthr); 1569 spa->spa_livelist_delete_zthr = NULL; 1570 } 1571 if (spa->spa_livelist_condense_zthr != NULL) { 1572 zthr_destroy(spa->spa_livelist_condense_zthr); 1573 spa->spa_livelist_condense_zthr = NULL; 1574 } 1575} 1576 1577/* 1578 * Opposite of spa_load(). 1579 */ 1580static void 1581spa_unload(spa_t *spa) 1582{ 1583 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 1584 ASSERT(spa_state(spa) != POOL_STATE_UNINITIALIZED); 1585 1586 spa_import_progress_remove(spa_guid(spa)); 1587 spa_load_note(spa, "UNLOADING"); 1588 1589 spa_wake_waiters(spa); 1590 1591 /* 1592 * If the log space map feature is enabled and the pool is getting 1593 * exported (but not destroyed), we want to spend some time flushing 1594 * as many metaslabs as we can in an attempt to destroy log space 1595 * maps and save import time. 1596 */ 1597 if (spa_should_flush_logs_on_unload(spa)) 1598 spa_unload_log_sm_flush_all(spa); 1599 1600 /* 1601 * Stop async tasks. 1602 */ 1603 spa_async_suspend(spa); 1604 1605 if (spa->spa_root_vdev) { 1606 vdev_t *root_vdev = spa->spa_root_vdev; 1607 vdev_initialize_stop_all(root_vdev, VDEV_INITIALIZE_ACTIVE); 1608 vdev_trim_stop_all(root_vdev, VDEV_TRIM_ACTIVE); 1609 vdev_autotrim_stop_all(spa); 1610 vdev_rebuild_stop_all(spa); 1611 } 1612 1613 /* 1614 * Stop syncing. 1615 */ 1616 if (spa->spa_sync_on) { 1617 txg_sync_stop(spa->spa_dsl_pool); 1618 spa->spa_sync_on = B_FALSE; 1619 } 1620 1621 /* 1622 * This ensures that there is no async metaslab prefetching 1623 * while we attempt to unload the spa. 1624 */ 1625 if (spa->spa_root_vdev != NULL) { 1626 for (int c = 0; c < spa->spa_root_vdev->vdev_children; c++) { 1627 vdev_t *vc = spa->spa_root_vdev->vdev_child[c]; 1628 if (vc->vdev_mg != NULL) 1629 taskq_wait(vc->vdev_mg->mg_taskq); 1630 } 1631 } 1632 1633 if (spa->spa_mmp.mmp_thread) 1634 mmp_thread_stop(spa); 1635 1636 /* 1637 * Wait for any outstanding async I/O to complete. 1638 */ 1639 if (spa->spa_async_zio_root != NULL) { 1640 for (int i = 0; i < max_ncpus; i++) 1641 (void) zio_wait(spa->spa_async_zio_root[i]); 1642 kmem_free(spa->spa_async_zio_root, max_ncpus * sizeof (void *)); 1643 spa->spa_async_zio_root = NULL; 1644 } 1645 1646 if (spa->spa_vdev_removal != NULL) { 1647 spa_vdev_removal_destroy(spa->spa_vdev_removal); 1648 spa->spa_vdev_removal = NULL; 1649 } 1650 1651 spa_destroy_aux_threads(spa); 1652 1653 spa_condense_fini(spa); 1654 1655 bpobj_close(&spa->spa_deferred_bpobj); 1656 1657 spa_config_enter(spa, SCL_ALL, spa, RW_WRITER); 1658 1659 /* 1660 * Close all vdevs. 1661 */ 1662 if (spa->spa_root_vdev) 1663 vdev_free(spa->spa_root_vdev); 1664 ASSERT(spa->spa_root_vdev == NULL); 1665 1666 /* 1667 * Close the dsl pool. 1668 */ 1669 if (spa->spa_dsl_pool) { 1670 dsl_pool_close(spa->spa_dsl_pool); 1671 spa->spa_dsl_pool = NULL; 1672 spa->spa_meta_objset = NULL; 1673 } 1674 1675 ddt_unload(spa); 1676 spa_unload_log_sm_metadata(spa); 1677 1678 /* 1679 * Drop and purge level 2 cache 1680 */ 1681 spa_l2cache_drop(spa); 1682 1683 for (int i = 0; i < spa->spa_spares.sav_count; i++) 1684 vdev_free(spa->spa_spares.sav_vdevs[i]); 1685 if (spa->spa_spares.sav_vdevs) { 1686 kmem_free(spa->spa_spares.sav_vdevs, 1687 spa->spa_spares.sav_count * sizeof (void *)); 1688 spa->spa_spares.sav_vdevs = NULL; 1689 } 1690 if (spa->spa_spares.sav_config) { 1691 nvlist_free(spa->spa_spares.sav_config); 1692 spa->spa_spares.sav_config = NULL; 1693 } 1694 spa->spa_spares.sav_count = 0; 1695 1696 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) { 1697 vdev_clear_stats(spa->spa_l2cache.sav_vdevs[i]); 1698 vdev_free(spa->spa_l2cache.sav_vdevs[i]); 1699 } 1700 if (spa->spa_l2cache.sav_vdevs) { 1701 kmem_free(spa->spa_l2cache.sav_vdevs, 1702 spa->spa_l2cache.sav_count * sizeof (void *)); 1703 spa->spa_l2cache.sav_vdevs = NULL; 1704 } 1705 if (spa->spa_l2cache.sav_config) { 1706 nvlist_free(spa->spa_l2cache.sav_config); 1707 spa->spa_l2cache.sav_config = NULL; 1708 } 1709 spa->spa_l2cache.sav_count = 0; 1710 1711 spa->spa_async_suspended = 0; 1712 1713 spa->spa_indirect_vdevs_loaded = B_FALSE; 1714 1715 if (spa->spa_comment != NULL) { 1716 spa_strfree(spa->spa_comment); 1717 spa->spa_comment = NULL; 1718 } 1719 if (spa->spa_compatibility != NULL) { 1720 spa_strfree(spa->spa_compatibility); 1721 spa->spa_compatibility = NULL; 1722 } 1723 1724 spa_config_exit(spa, SCL_ALL, spa); 1725} 1726 1727/* 1728 * Load (or re-load) the current list of vdevs describing the active spares for 1729 * this pool. When this is called, we have some form of basic information in 1730 * 'spa_spares.sav_config'. We parse this into vdevs, try to open them, and 1731 * then re-generate a more complete list including status information. 1732 */ 1733void 1734spa_load_spares(spa_t *spa) 1735{ 1736 nvlist_t **spares; 1737 uint_t nspares; 1738 int i; 1739 vdev_t *vd, *tvd; 1740 1741#ifndef _KERNEL 1742 /* 1743 * zdb opens both the current state of the pool and the 1744 * checkpointed state (if present), with a different spa_t. 1745 * 1746 * As spare vdevs are shared among open pools, we skip loading 1747 * them when we load the checkpointed state of the pool. 1748 */ 1749 if (!spa_writeable(spa)) 1750 return; 1751#endif 1752 1753 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1754 1755 /* 1756 * First, close and free any existing spare vdevs. 1757 */ 1758 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1759 vd = spa->spa_spares.sav_vdevs[i]; 1760 1761 /* Undo the call to spa_activate() below */ 1762 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1763 B_FALSE)) != NULL && tvd->vdev_isspare) 1764 spa_spare_remove(tvd); 1765 vdev_close(vd); 1766 vdev_free(vd); 1767 } 1768 1769 if (spa->spa_spares.sav_vdevs) 1770 kmem_free(spa->spa_spares.sav_vdevs, 1771 spa->spa_spares.sav_count * sizeof (void *)); 1772 1773 if (spa->spa_spares.sav_config == NULL) 1774 nspares = 0; 1775 else 1776 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 1777 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 1778 1779 spa->spa_spares.sav_count = (int)nspares; 1780 spa->spa_spares.sav_vdevs = NULL; 1781 1782 if (nspares == 0) 1783 return; 1784 1785 /* 1786 * Construct the array of vdevs, opening them to get status in the 1787 * process. For each spare, there is potentially two different vdev_t 1788 * structures associated with it: one in the list of spares (used only 1789 * for basic validation purposes) and one in the active vdev 1790 * configuration (if it's spared in). During this phase we open and 1791 * validate each vdev on the spare list. If the vdev also exists in the 1792 * active configuration, then we also mark this vdev as an active spare. 1793 */ 1794 spa->spa_spares.sav_vdevs = kmem_zalloc(nspares * sizeof (void *), 1795 KM_SLEEP); 1796 for (i = 0; i < spa->spa_spares.sav_count; i++) { 1797 VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, 1798 VDEV_ALLOC_SPARE) == 0); 1799 ASSERT(vd != NULL); 1800 1801 spa->spa_spares.sav_vdevs[i] = vd; 1802 1803 if ((tvd = spa_lookup_by_guid(spa, vd->vdev_guid, 1804 B_FALSE)) != NULL) { 1805 if (!tvd->vdev_isspare) 1806 spa_spare_add(tvd); 1807 1808 /* 1809 * We only mark the spare active if we were successfully 1810 * able to load the vdev. Otherwise, importing a pool 1811 * with a bad active spare would result in strange 1812 * behavior, because multiple pool would think the spare 1813 * is actively in use. 1814 * 1815 * There is a vulnerability here to an equally bizarre 1816 * circumstance, where a dead active spare is later 1817 * brought back to life (onlined or otherwise). Given 1818 * the rarity of this scenario, and the extra complexity 1819 * it adds, we ignore the possibility. 1820 */ 1821 if (!vdev_is_dead(tvd)) 1822 spa_spare_activate(tvd); 1823 } 1824 1825 vd->vdev_top = vd; 1826 vd->vdev_aux = &spa->spa_spares; 1827 1828 if (vdev_open(vd) != 0) 1829 continue; 1830 1831 if (vdev_validate_aux(vd) == 0) 1832 spa_spare_add(vd); 1833 } 1834 1835 /* 1836 * Recompute the stashed list of spares, with status information 1837 * this time. 1838 */ 1839 VERIFY(nvlist_remove(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, 1840 DATA_TYPE_NVLIST_ARRAY) == 0); 1841 1842 spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), 1843 KM_SLEEP); 1844 for (i = 0; i < spa->spa_spares.sav_count; i++) 1845 spares[i] = vdev_config_generate(spa, 1846 spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); 1847 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 1848 ZPOOL_CONFIG_SPARES, spares, spa->spa_spares.sav_count) == 0); 1849 for (i = 0; i < spa->spa_spares.sav_count; i++) 1850 nvlist_free(spares[i]); 1851 kmem_free(spares, spa->spa_spares.sav_count * sizeof (void *)); 1852} 1853 1854/* 1855 * Load (or re-load) the current list of vdevs describing the active l2cache for 1856 * this pool. When this is called, we have some form of basic information in 1857 * 'spa_l2cache.sav_config'. We parse this into vdevs, try to open them, and 1858 * then re-generate a more complete list including status information. 1859 * Devices which are already active have their details maintained, and are 1860 * not re-opened. 1861 */ 1862void 1863spa_load_l2cache(spa_t *spa) 1864{ 1865 nvlist_t **l2cache = NULL; 1866 uint_t nl2cache; 1867 int i, j, oldnvdevs; 1868 uint64_t guid; 1869 vdev_t *vd, **oldvdevs, **newvdevs; 1870 spa_aux_vdev_t *sav = &spa->spa_l2cache; 1871 1872#ifndef _KERNEL 1873 /* 1874 * zdb opens both the current state of the pool and the 1875 * checkpointed state (if present), with a different spa_t. 1876 * 1877 * As L2 caches are part of the ARC which is shared among open 1878 * pools, we skip loading them when we load the checkpointed 1879 * state of the pool. 1880 */ 1881 if (!spa_writeable(spa)) 1882 return; 1883#endif 1884 1885 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 1886 1887 oldvdevs = sav->sav_vdevs; 1888 oldnvdevs = sav->sav_count; 1889 sav->sav_vdevs = NULL; 1890 sav->sav_count = 0; 1891 1892 if (sav->sav_config == NULL) { 1893 nl2cache = 0; 1894 newvdevs = NULL; 1895 goto out; 1896 } 1897 1898 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, 1899 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 1900 newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); 1901 1902 /* 1903 * Process new nvlist of vdevs. 1904 */ 1905 for (i = 0; i < nl2cache; i++) { 1906 VERIFY(nvlist_lookup_uint64(l2cache[i], ZPOOL_CONFIG_GUID, 1907 &guid) == 0); 1908 1909 newvdevs[i] = NULL; 1910 for (j = 0; j < oldnvdevs; j++) { 1911 vd = oldvdevs[j]; 1912 if (vd != NULL && guid == vd->vdev_guid) { 1913 /* 1914 * Retain previous vdev for add/remove ops. 1915 */ 1916 newvdevs[i] = vd; 1917 oldvdevs[j] = NULL; 1918 break; 1919 } 1920 } 1921 1922 if (newvdevs[i] == NULL) { 1923 /* 1924 * Create new vdev 1925 */ 1926 VERIFY(spa_config_parse(spa, &vd, l2cache[i], NULL, 0, 1927 VDEV_ALLOC_L2CACHE) == 0); 1928 ASSERT(vd != NULL); 1929 newvdevs[i] = vd; 1930 1931 /* 1932 * Commit this vdev as an l2cache device, 1933 * even if it fails to open. 1934 */ 1935 spa_l2cache_add(vd); 1936 1937 vd->vdev_top = vd; 1938 vd->vdev_aux = sav; 1939 1940 spa_l2cache_activate(vd); 1941 1942 if (vdev_open(vd) != 0) 1943 continue; 1944 1945 (void) vdev_validate_aux(vd); 1946 1947 if (!vdev_is_dead(vd)) 1948 l2arc_add_vdev(spa, vd); 1949 1950 /* 1951 * Upon cache device addition to a pool or pool 1952 * creation with a cache device or if the header 1953 * of the device is invalid we issue an async 1954 * TRIM command for the whole device which will 1955 * execute if l2arc_trim_ahead > 0. 1956 */ 1957 spa_async_request(spa, SPA_ASYNC_L2CACHE_TRIM); 1958 } 1959 } 1960 1961 sav->sav_vdevs = newvdevs; 1962 sav->sav_count = (int)nl2cache; 1963 1964 /* 1965 * Recompute the stashed list of l2cache devices, with status 1966 * information this time. 1967 */ 1968 VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, 1969 DATA_TYPE_NVLIST_ARRAY) == 0); 1970 1971 if (sav->sav_count > 0) 1972 l2cache = kmem_alloc(sav->sav_count * sizeof (void *), 1973 KM_SLEEP); 1974 for (i = 0; i < sav->sav_count; i++) 1975 l2cache[i] = vdev_config_generate(spa, 1976 sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); 1977 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 1978 ZPOOL_CONFIG_L2CACHE, l2cache, sav->sav_count) == 0); 1979 1980out: 1981 /* 1982 * Purge vdevs that were dropped 1983 */ 1984 for (i = 0; i < oldnvdevs; i++) { 1985 uint64_t pool; 1986 1987 vd = oldvdevs[i]; 1988 if (vd != NULL) { 1989 ASSERT(vd->vdev_isl2cache); 1990 1991 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 1992 pool != 0ULL && l2arc_vdev_present(vd)) 1993 l2arc_remove_vdev(vd); 1994 vdev_clear_stats(vd); 1995 vdev_free(vd); 1996 } 1997 } 1998 1999 if (oldvdevs) 2000 kmem_free(oldvdevs, oldnvdevs * sizeof (void *)); 2001 2002 for (i = 0; i < sav->sav_count; i++) 2003 nvlist_free(l2cache[i]); 2004 if (sav->sav_count) 2005 kmem_free(l2cache, sav->sav_count * sizeof (void *)); 2006} 2007 2008static int 2009load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) 2010{ 2011 dmu_buf_t *db; 2012 char *packed = NULL; 2013 size_t nvsize = 0; 2014 int error; 2015 *value = NULL; 2016 2017 error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db); 2018 if (error) 2019 return (error); 2020 2021 nvsize = *(uint64_t *)db->db_data; 2022 dmu_buf_rele(db, FTAG); 2023 2024 packed = vmem_alloc(nvsize, KM_SLEEP); 2025 error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, 2026 DMU_READ_PREFETCH); 2027 if (error == 0) 2028 error = nvlist_unpack(packed, nvsize, value, 0); 2029 vmem_free(packed, nvsize); 2030 2031 return (error); 2032} 2033 2034/* 2035 * Concrete top-level vdevs that are not missing and are not logs. At every 2036 * spa_sync we write new uberblocks to at least SPA_SYNC_MIN_VDEVS core tvds. 2037 */ 2038static uint64_t 2039spa_healthy_core_tvds(spa_t *spa) 2040{ 2041 vdev_t *rvd = spa->spa_root_vdev; 2042 uint64_t tvds = 0; 2043 2044 for (uint64_t i = 0; i < rvd->vdev_children; i++) { 2045 vdev_t *vd = rvd->vdev_child[i]; 2046 if (vd->vdev_islog) 2047 continue; 2048 if (vdev_is_concrete(vd) && !vdev_is_dead(vd)) 2049 tvds++; 2050 } 2051 2052 return (tvds); 2053} 2054 2055/* 2056 * Checks to see if the given vdev could not be opened, in which case we post a 2057 * sysevent to notify the autoreplace code that the device has been removed. 2058 */ 2059static void 2060spa_check_removed(vdev_t *vd) 2061{ 2062 for (uint64_t c = 0; c < vd->vdev_children; c++) 2063 spa_check_removed(vd->vdev_child[c]); 2064 2065 if (vd->vdev_ops->vdev_op_leaf && vdev_is_dead(vd) && 2066 vdev_is_concrete(vd)) { 2067 zfs_post_autoreplace(vd->vdev_spa, vd); 2068 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_CHECK); 2069 } 2070} 2071 2072static int 2073spa_check_for_missing_logs(spa_t *spa) 2074{ 2075 vdev_t *rvd = spa->spa_root_vdev; 2076 2077 /* 2078 * If we're doing a normal import, then build up any additional 2079 * diagnostic information about missing log devices. 2080 * We'll pass this up to the user for further processing. 2081 */ 2082 if (!(spa->spa_import_flags & ZFS_IMPORT_MISSING_LOG)) { 2083 nvlist_t **child, *nv; 2084 uint64_t idx = 0; 2085 2086 child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t *), 2087 KM_SLEEP); 2088 VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); 2089 2090 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2091 vdev_t *tvd = rvd->vdev_child[c]; 2092 2093 /* 2094 * We consider a device as missing only if it failed 2095 * to open (i.e. offline or faulted is not considered 2096 * as missing). 2097 */ 2098 if (tvd->vdev_islog && 2099 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2100 child[idx++] = vdev_config_generate(spa, tvd, 2101 B_FALSE, VDEV_CONFIG_MISSING); 2102 } 2103 } 2104 2105 if (idx > 0) { 2106 fnvlist_add_nvlist_array(nv, 2107 ZPOOL_CONFIG_CHILDREN, child, idx); 2108 fnvlist_add_nvlist(spa->spa_load_info, 2109 ZPOOL_CONFIG_MISSING_DEVICES, nv); 2110 2111 for (uint64_t i = 0; i < idx; i++) 2112 nvlist_free(child[i]); 2113 } 2114 nvlist_free(nv); 2115 kmem_free(child, rvd->vdev_children * sizeof (char **)); 2116 2117 if (idx > 0) { 2118 spa_load_failed(spa, "some log devices are missing"); 2119 vdev_dbgmsg_print_tree(rvd, 2); 2120 return (SET_ERROR(ENXIO)); 2121 } 2122 } else { 2123 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 2124 vdev_t *tvd = rvd->vdev_child[c]; 2125 2126 if (tvd->vdev_islog && 2127 tvd->vdev_state == VDEV_STATE_CANT_OPEN) { 2128 spa_set_log_state(spa, SPA_LOG_CLEAR); 2129 spa_load_note(spa, "some log devices are " 2130 "missing, ZIL is dropped."); 2131 vdev_dbgmsg_print_tree(rvd, 2); 2132 break; 2133 } 2134 } 2135 } 2136 2137 return (0); 2138} 2139 2140/* 2141 * Check for missing log devices 2142 */ 2143static boolean_t 2144spa_check_logs(spa_t *spa) 2145{ 2146 boolean_t rv = B_FALSE; 2147 dsl_pool_t *dp = spa_get_dsl(spa); 2148 2149 switch (spa->spa_log_state) { 2150 default: 2151 break; 2152 case SPA_LOG_MISSING: 2153 /* need to recheck in case slog has been restored */ 2154 case SPA_LOG_UNKNOWN: 2155 rv = (dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 2156 zil_check_log_chain, NULL, DS_FIND_CHILDREN) != 0); 2157 if (rv) 2158 spa_set_log_state(spa, SPA_LOG_MISSING); 2159 break; 2160 } 2161 return (rv); 2162} 2163 2164/* 2165 * Passivate any log vdevs (note, does not apply to embedded log metaslabs). 2166 */ 2167static boolean_t 2168spa_passivate_log(spa_t *spa) 2169{ 2170 vdev_t *rvd = spa->spa_root_vdev; 2171 boolean_t slog_found = B_FALSE; 2172 2173 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2174 2175 for (int c = 0; c < rvd->vdev_children; c++) { 2176 vdev_t *tvd = rvd->vdev_child[c]; 2177 2178 if (tvd->vdev_islog) { 2179 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2180 metaslab_group_passivate(tvd->vdev_mg); 2181 slog_found = B_TRUE; 2182 } 2183 } 2184 2185 return (slog_found); 2186} 2187 2188/* 2189 * Activate any log vdevs (note, does not apply to embedded log metaslabs). 2190 */ 2191static void 2192spa_activate_log(spa_t *spa) 2193{ 2194 vdev_t *rvd = spa->spa_root_vdev; 2195 2196 ASSERT(spa_config_held(spa, SCL_ALLOC, RW_WRITER)); 2197 2198 for (int c = 0; c < rvd->vdev_children; c++) { 2199 vdev_t *tvd = rvd->vdev_child[c]; 2200 2201 if (tvd->vdev_islog) { 2202 ASSERT3P(tvd->vdev_log_mg, ==, NULL); 2203 metaslab_group_activate(tvd->vdev_mg); 2204 } 2205 } 2206} 2207 2208int 2209spa_reset_logs(spa_t *spa) 2210{ 2211 int error; 2212 2213 error = dmu_objset_find(spa_name(spa), zil_reset, 2214 NULL, DS_FIND_CHILDREN); 2215 if (error == 0) { 2216 /* 2217 * We successfully offlined the log device, sync out the 2218 * current txg so that the "stubby" block can be removed 2219 * by zil_sync(). 2220 */ 2221 txg_wait_synced(spa->spa_dsl_pool, 0); 2222 } 2223 return (error); 2224} 2225 2226static void 2227spa_aux_check_removed(spa_aux_vdev_t *sav) 2228{ 2229 for (int i = 0; i < sav->sav_count; i++) 2230 spa_check_removed(sav->sav_vdevs[i]); 2231} 2232 2233void 2234spa_claim_notify(zio_t *zio) 2235{ 2236 spa_t *spa = zio->io_spa; 2237 2238 if (zio->io_error) 2239 return; 2240 2241 mutex_enter(&spa->spa_props_lock); /* any mutex will do */ 2242 if (spa->spa_claim_max_txg < zio->io_bp->blk_birth) 2243 spa->spa_claim_max_txg = zio->io_bp->blk_birth; 2244 mutex_exit(&spa->spa_props_lock); 2245} 2246 2247typedef struct spa_load_error { 2248 uint64_t sle_meta_count; 2249 uint64_t sle_data_count; 2250} spa_load_error_t; 2251 2252static void 2253spa_load_verify_done(zio_t *zio) 2254{ 2255 blkptr_t *bp = zio->io_bp; 2256 spa_load_error_t *sle = zio->io_private; 2257 dmu_object_type_t type = BP_GET_TYPE(bp); 2258 int error = zio->io_error; 2259 spa_t *spa = zio->io_spa; 2260 2261 abd_free(zio->io_abd); 2262 if (error) { 2263 if ((BP_GET_LEVEL(bp) != 0 || DMU_OT_IS_METADATA(type)) && 2264 type != DMU_OT_INTENT_LOG) 2265 atomic_inc_64(&sle->sle_meta_count); 2266 else 2267 atomic_inc_64(&sle->sle_data_count); 2268 } 2269 2270 mutex_enter(&spa->spa_scrub_lock); 2271 spa->spa_load_verify_bytes -= BP_GET_PSIZE(bp); 2272 cv_broadcast(&spa->spa_scrub_io_cv); 2273 mutex_exit(&spa->spa_scrub_lock); 2274} 2275 2276/* 2277 * Maximum number of inflight bytes is the log2 fraction of the arc size. 2278 * By default, we set it to 1/16th of the arc. 2279 */ 2280int spa_load_verify_shift = 4; 2281int spa_load_verify_metadata = B_TRUE; 2282int spa_load_verify_data = B_TRUE; 2283 2284/*ARGSUSED*/ 2285static int 2286spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, 2287 const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg) 2288{ 2289 if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) || 2290 BP_IS_EMBEDDED(bp) || BP_IS_REDACTED(bp)) 2291 return (0); 2292 /* 2293 * Note: normally this routine will not be called if 2294 * spa_load_verify_metadata is not set. However, it may be useful 2295 * to manually set the flag after the traversal has begun. 2296 */ 2297 if (!spa_load_verify_metadata) 2298 return (0); 2299 if (!BP_IS_METADATA(bp) && !spa_load_verify_data) 2300 return (0); 2301 2302 uint64_t maxinflight_bytes = 2303 arc_target_bytes() >> spa_load_verify_shift; 2304 zio_t *rio = arg; 2305 size_t size = BP_GET_PSIZE(bp); 2306 2307 mutex_enter(&spa->spa_scrub_lock); 2308 while (spa->spa_load_verify_bytes >= maxinflight_bytes) 2309 cv_wait(&spa->spa_scrub_io_cv, &spa->spa_scrub_lock); 2310 spa->spa_load_verify_bytes += size; 2311 mutex_exit(&spa->spa_scrub_lock); 2312 2313 zio_nowait(zio_read(rio, spa, bp, abd_alloc_for_io(size, B_FALSE), size, 2314 spa_load_verify_done, rio->io_private, ZIO_PRIORITY_SCRUB, 2315 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_CANFAIL | 2316 ZIO_FLAG_SCRUB | ZIO_FLAG_RAW, zb)); 2317 return (0); 2318} 2319 2320/* ARGSUSED */ 2321static int 2322verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 2323{ 2324 if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) 2325 return (SET_ERROR(ENAMETOOLONG)); 2326 2327 return (0); 2328} 2329 2330static int 2331spa_load_verify(spa_t *spa) 2332{ 2333 zio_t *rio; 2334 spa_load_error_t sle = { 0 }; 2335 zpool_load_policy_t policy; 2336 boolean_t verify_ok = B_FALSE; 2337 int error = 0; 2338 2339 zpool_get_load_policy(spa->spa_config, &policy); 2340 2341 if (policy.zlp_rewind & ZPOOL_NEVER_REWIND) 2342 return (0); 2343 2344 dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); 2345 error = dmu_objset_find_dp(spa->spa_dsl_pool, 2346 spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, 2347 DS_FIND_CHILDREN); 2348 dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); 2349 if (error != 0) 2350 return (error); 2351 2352 rio = zio_root(spa, NULL, &sle, 2353 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); 2354 2355 if (spa_load_verify_metadata) { 2356 if (spa->spa_extreme_rewind) { 2357 spa_load_note(spa, "performing a complete scan of the " 2358 "pool since extreme rewind is on. This may take " 2359 "a very long time.\n (spa_load_verify_data=%u, " 2360 "spa_load_verify_metadata=%u)", 2361 spa_load_verify_data, spa_load_verify_metadata); 2362 } 2363 2364 error = traverse_pool(spa, spa->spa_verify_min_txg, 2365 TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA | 2366 TRAVERSE_NO_DECRYPT, spa_load_verify_cb, rio); 2367 } 2368 2369 (void) zio_wait(rio); 2370 ASSERT0(spa->spa_load_verify_bytes); 2371 2372 spa->spa_load_meta_errors = sle.sle_meta_count; 2373 spa->spa_load_data_errors = sle.sle_data_count; 2374 2375 if (sle.sle_meta_count != 0 || sle.sle_data_count != 0) { 2376 spa_load_note(spa, "spa_load_verify found %llu metadata errors " 2377 "and %llu data errors", (u_longlong_t)sle.sle_meta_count, 2378 (u_longlong_t)sle.sle_data_count); 2379 } 2380 2381 if (spa_load_verify_dryrun || 2382 (!error && sle.sle_meta_count <= policy.zlp_maxmeta && 2383 sle.sle_data_count <= policy.zlp_maxdata)) { 2384 int64_t loss = 0; 2385 2386 verify_ok = B_TRUE; 2387 spa->spa_load_txg = spa->spa_uberblock.ub_txg; 2388 spa->spa_load_txg_ts = spa->spa_uberblock.ub_timestamp; 2389 2390 loss = spa->spa_last_ubsync_txg_ts - spa->spa_load_txg_ts; 2391 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2392 ZPOOL_CONFIG_LOAD_TIME, spa->spa_load_txg_ts) == 0); 2393 VERIFY(nvlist_add_int64(spa->spa_load_info, 2394 ZPOOL_CONFIG_REWIND_TIME, loss) == 0); 2395 VERIFY(nvlist_add_uint64(spa->spa_load_info, 2396 ZPOOL_CONFIG_LOAD_DATA_ERRORS, sle.sle_data_count) == 0); 2397 } else { 2398 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg; 2399 } 2400 2401 if (spa_load_verify_dryrun) 2402 return (0); 2403 2404 if (error) { 2405 if (error != ENXIO && error != EIO) 2406 error = SET_ERROR(EIO); 2407 return (error); 2408 } 2409 2410 return (verify_ok ? 0 : EIO); 2411} 2412 2413/* 2414 * Find a value in the pool props object. 2415 */ 2416static void 2417spa_prop_find(spa_t *spa, zpool_prop_t prop, uint64_t *val) 2418{ 2419 (void) zap_lookup(spa->spa_meta_objset, spa->spa_pool_props_object, 2420 zpool_prop_to_name(prop), sizeof (uint64_t), 1, val); 2421} 2422 2423/* 2424 * Find a value in the pool directory object. 2425 */ 2426static int 2427spa_dir_prop(spa_t *spa, const char *name, uint64_t *val, boolean_t log_enoent) 2428{ 2429 int error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 2430 name, sizeof (uint64_t), 1, val); 2431 2432 if (error != 0 && (error != ENOENT || log_enoent)) { 2433 spa_load_failed(spa, "couldn't get '%s' value in MOS directory " 2434 "[error=%d]", name, error); 2435 } 2436 2437 return (error); 2438} 2439 2440static int 2441spa_vdev_err(vdev_t *vdev, vdev_aux_t aux, int err) 2442{ 2443 vdev_set_state(vdev, B_TRUE, VDEV_STATE_CANT_OPEN, aux); 2444 return (SET_ERROR(err)); 2445} 2446 2447boolean_t 2448spa_livelist_delete_check(spa_t *spa) 2449{ 2450 return (spa->spa_livelists_to_delete != 0); 2451} 2452 2453/* ARGSUSED */ 2454static boolean_t 2455spa_livelist_delete_cb_check(void *arg, zthr_t *z) 2456{ 2457 spa_t *spa = arg; 2458 return (spa_livelist_delete_check(spa)); 2459} 2460 2461static int 2462delete_blkptr_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 2463{ 2464 spa_t *spa = arg; 2465 zio_free(spa, tx->tx_txg, bp); 2466 dsl_dir_diduse_space(tx->tx_pool->dp_free_dir, DD_USED_HEAD, 2467 -bp_get_dsize_sync(spa, bp), 2468 -BP_GET_PSIZE(bp), -BP_GET_UCSIZE(bp), tx); 2469 return (0); 2470} 2471 2472static int 2473dsl_get_next_livelist_obj(objset_t *os, uint64_t zap_obj, uint64_t *llp) 2474{ 2475 int err; 2476 zap_cursor_t zc; 2477 zap_attribute_t za; 2478 zap_cursor_init(&zc, os, zap_obj); 2479 err = zap_cursor_retrieve(&zc, &za); 2480 zap_cursor_fini(&zc); 2481 if (err == 0) 2482 *llp = za.za_first_integer; 2483 return (err); 2484} 2485 2486/* 2487 * Components of livelist deletion that must be performed in syncing 2488 * context: freeing block pointers and updating the pool-wide data 2489 * structures to indicate how much work is left to do 2490 */ 2491typedef struct sublist_delete_arg { 2492 spa_t *spa; 2493 dsl_deadlist_t *ll; 2494 uint64_t key; 2495 bplist_t *to_free; 2496} sublist_delete_arg_t; 2497 2498static void 2499sublist_delete_sync(void *arg, dmu_tx_t *tx) 2500{ 2501 sublist_delete_arg_t *sda = arg; 2502 spa_t *spa = sda->spa; 2503 dsl_deadlist_t *ll = sda->ll; 2504 uint64_t key = sda->key; 2505 bplist_t *to_free = sda->to_free; 2506 2507 bplist_iterate(to_free, delete_blkptr_cb, spa, tx); 2508 dsl_deadlist_remove_entry(ll, key, tx); 2509} 2510 2511typedef struct livelist_delete_arg { 2512 spa_t *spa; 2513 uint64_t ll_obj; 2514 uint64_t zap_obj; 2515} livelist_delete_arg_t; 2516 2517static void 2518livelist_delete_sync(void *arg, dmu_tx_t *tx) 2519{ 2520 livelist_delete_arg_t *lda = arg; 2521 spa_t *spa = lda->spa; 2522 uint64_t ll_obj = lda->ll_obj; 2523 uint64_t zap_obj = lda->zap_obj; 2524 objset_t *mos = spa->spa_meta_objset; 2525 uint64_t count; 2526 2527 /* free the livelist and decrement the feature count */ 2528 VERIFY0(zap_remove_int(mos, zap_obj, ll_obj, tx)); 2529 dsl_deadlist_free(mos, ll_obj, tx); 2530 spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx); 2531 VERIFY0(zap_count(mos, zap_obj, &count)); 2532 if (count == 0) { 2533 /* no more livelists to delete */ 2534 VERIFY0(zap_remove(mos, DMU_POOL_DIRECTORY_OBJECT, 2535 DMU_POOL_DELETED_CLONES, tx)); 2536 VERIFY0(zap_destroy(mos, zap_obj, tx)); 2537 spa->spa_livelists_to_delete = 0; 2538 spa_notify_waiters(spa); 2539 } 2540} 2541 2542/* 2543 * Load in the value for the livelist to be removed and open it. Then, 2544 * load its first sublist and determine which block pointers should actually 2545 * be freed. Then, call a synctask which performs the actual frees and updates 2546 * the pool-wide livelist data. 2547 */ 2548/* ARGSUSED */ 2549static void 2550spa_livelist_delete_cb(void *arg, zthr_t *z) 2551{ 2552 spa_t *spa = arg; 2553 uint64_t ll_obj = 0, count; 2554 objset_t *mos = spa->spa_meta_objset; 2555 uint64_t zap_obj = spa->spa_livelists_to_delete; 2556 /* 2557 * Determine the next livelist to delete. This function should only 2558 * be called if there is at least one deleted clone. 2559 */ 2560 VERIFY0(dsl_get_next_livelist_obj(mos, zap_obj, &ll_obj)); 2561 VERIFY0(zap_count(mos, ll_obj, &count)); 2562 if (count > 0) { 2563 dsl_deadlist_t *ll; 2564 dsl_deadlist_entry_t *dle; 2565 bplist_t to_free; 2566 ll = kmem_zalloc(sizeof (dsl_deadlist_t), KM_SLEEP); 2567 dsl_deadlist_open(ll, mos, ll_obj); 2568 dle = dsl_deadlist_first(ll); 2569 ASSERT3P(dle, !=, NULL); 2570 bplist_create(&to_free); 2571 int err = dsl_process_sub_livelist(&dle->dle_bpobj, &to_free, 2572 z, NULL); 2573 if (err == 0) { 2574 sublist_delete_arg_t sync_arg = { 2575 .spa = spa, 2576 .ll = ll, 2577 .key = dle->dle_mintxg, 2578 .to_free = &to_free 2579 }; 2580 zfs_dbgmsg("deleting sublist (id %llu) from" 2581 " livelist %llu, %d remaining", 2582 dle->dle_bpobj.bpo_object, ll_obj, count - 1); 2583 VERIFY0(dsl_sync_task(spa_name(spa), NULL, 2584 sublist_delete_sync, &sync_arg, 0, 2585 ZFS_SPACE_CHECK_DESTROY)); 2586 } else { 2587 VERIFY3U(err, ==, EINTR); 2588 } 2589 bplist_clear(&to_free); 2590 bplist_destroy(&to_free); 2591 dsl_deadlist_close(ll); 2592 kmem_free(ll, sizeof (dsl_deadlist_t)); 2593 } else { 2594 livelist_delete_arg_t sync_arg = { 2595 .spa = spa, 2596 .ll_obj = ll_obj, 2597 .zap_obj = zap_obj 2598 }; 2599 zfs_dbgmsg("deletion of livelist %llu completed", ll_obj); 2600 VERIFY0(dsl_sync_task(spa_name(spa), NULL, livelist_delete_sync, 2601 &sync_arg, 0, ZFS_SPACE_CHECK_DESTROY)); 2602 } 2603} 2604 2605static void 2606spa_start_livelist_destroy_thread(spa_t *spa) 2607{ 2608 ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL); 2609 spa->spa_livelist_delete_zthr = 2610 zthr_create("z_livelist_destroy", 2611 spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa); 2612} 2613 2614typedef struct livelist_new_arg { 2615 bplist_t *allocs; 2616 bplist_t *frees; 2617} livelist_new_arg_t; 2618 2619static int 2620livelist_track_new_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 2621 dmu_tx_t *tx) 2622{ 2623 ASSERT(tx == NULL); 2624 livelist_new_arg_t *lna = arg; 2625 if (bp_freed) { 2626 bplist_append(lna->frees, bp); 2627 } else { 2628 bplist_append(lna->allocs, bp); 2629 zfs_livelist_condense_new_alloc++; 2630 } 2631 return (0); 2632} 2633 2634typedef struct livelist_condense_arg { 2635 spa_t *spa; 2636 bplist_t to_keep; 2637 uint64_t first_size; 2638 uint64_t next_size; 2639} livelist_condense_arg_t; 2640 2641static void 2642spa_livelist_condense_sync(void *arg, dmu_tx_t *tx) 2643{ 2644 livelist_condense_arg_t *lca = arg; 2645 spa_t *spa = lca->spa; 2646 bplist_t new_frees; 2647 dsl_dataset_t *ds = spa->spa_to_condense.ds; 2648 2649 /* Have we been cancelled? */ 2650 if (spa->spa_to_condense.cancelled) { 2651 zfs_livelist_condense_sync_cancel++; 2652 goto out; 2653 } 2654 2655 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2656 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2657 dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist; 2658 2659 /* 2660 * It's possible that the livelist was changed while the zthr was 2661 * running. Therefore, we need to check for new blkptrs in the two 2662 * entries being condensed and continue to track them in the livelist. 2663 * Because of the way we handle remapped blkptrs (see dbuf_remap_impl), 2664 * it's possible that the newly added blkptrs are FREEs or ALLOCs so 2665 * we need to sort them into two different bplists. 2666 */ 2667 uint64_t first_obj = first->dle_bpobj.bpo_object; 2668 uint64_t next_obj = next->dle_bpobj.bpo_object; 2669 uint64_t cur_first_size = first->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2670 uint64_t cur_next_size = next->dle_bpobj.bpo_phys->bpo_num_blkptrs; 2671 2672 bplist_create(&new_frees); 2673 livelist_new_arg_t new_bps = { 2674 .allocs = &lca->to_keep, 2675 .frees = &new_frees, 2676 }; 2677 2678 if (cur_first_size > lca->first_size) { 2679 VERIFY0(livelist_bpobj_iterate_from_nofree(&first->dle_bpobj, 2680 livelist_track_new_cb, &new_bps, lca->first_size)); 2681 } 2682 if (cur_next_size > lca->next_size) { 2683 VERIFY0(livelist_bpobj_iterate_from_nofree(&next->dle_bpobj, 2684 livelist_track_new_cb, &new_bps, lca->next_size)); 2685 } 2686 2687 dsl_deadlist_clear_entry(first, ll, tx); 2688 ASSERT(bpobj_is_empty(&first->dle_bpobj)); 2689 dsl_deadlist_remove_entry(ll, next->dle_mintxg, tx); 2690 2691 bplist_iterate(&lca->to_keep, dsl_deadlist_insert_alloc_cb, ll, tx); 2692 bplist_iterate(&new_frees, dsl_deadlist_insert_free_cb, ll, tx); 2693 bplist_destroy(&new_frees); 2694 2695 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 2696 dsl_dataset_name(ds, dsname); 2697 zfs_dbgmsg("txg %llu condensing livelist of %s (id %llu), bpobj %llu " 2698 "(%llu blkptrs) and bpobj %llu (%llu blkptrs) -> bpobj %llu " 2699 "(%llu blkptrs)", tx->tx_txg, dsname, ds->ds_object, first_obj, 2700 cur_first_size, next_obj, cur_next_size, 2701 first->dle_bpobj.bpo_object, 2702 first->dle_bpobj.bpo_phys->bpo_num_blkptrs); 2703out: 2704 dmu_buf_rele(ds->ds_dbuf, spa); 2705 spa->spa_to_condense.ds = NULL; 2706 bplist_clear(&lca->to_keep); 2707 bplist_destroy(&lca->to_keep); 2708 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2709 spa->spa_to_condense.syncing = B_FALSE; 2710} 2711 2712static void 2713spa_livelist_condense_cb(void *arg, zthr_t *t) 2714{ 2715 while (zfs_livelist_condense_zthr_pause && 2716 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2717 delay(1); 2718 2719 spa_t *spa = arg; 2720 dsl_deadlist_entry_t *first = spa->spa_to_condense.first; 2721 dsl_deadlist_entry_t *next = spa->spa_to_condense.next; 2722 uint64_t first_size, next_size; 2723 2724 livelist_condense_arg_t *lca = 2725 kmem_alloc(sizeof (livelist_condense_arg_t), KM_SLEEP); 2726 bplist_create(&lca->to_keep); 2727 2728 /* 2729 * Process the livelists (matching FREEs and ALLOCs) in open context 2730 * so we have minimal work in syncing context to condense. 2731 * 2732 * We save bpobj sizes (first_size and next_size) to use later in 2733 * syncing context to determine if entries were added to these sublists 2734 * while in open context. This is possible because the clone is still 2735 * active and open for normal writes and we want to make sure the new, 2736 * unprocessed blockpointers are inserted into the livelist normally. 2737 * 2738 * Note that dsl_process_sub_livelist() both stores the size number of 2739 * blockpointers and iterates over them while the bpobj's lock held, so 2740 * the sizes returned to us are consistent which what was actually 2741 * processed. 2742 */ 2743 int err = dsl_process_sub_livelist(&first->dle_bpobj, &lca->to_keep, t, 2744 &first_size); 2745 if (err == 0) 2746 err = dsl_process_sub_livelist(&next->dle_bpobj, &lca->to_keep, 2747 t, &next_size); 2748 2749 if (err == 0) { 2750 while (zfs_livelist_condense_sync_pause && 2751 !(zthr_has_waiters(t) || zthr_iscancelled(t))) 2752 delay(1); 2753 2754 dmu_tx_t *tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 2755 dmu_tx_mark_netfree(tx); 2756 dmu_tx_hold_space(tx, 1); 2757 err = dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE); 2758 if (err == 0) { 2759 /* 2760 * Prevent the condense zthr restarting before 2761 * the synctask completes. 2762 */ 2763 spa->spa_to_condense.syncing = B_TRUE; 2764 lca->spa = spa; 2765 lca->first_size = first_size; 2766 lca->next_size = next_size; 2767 dsl_sync_task_nowait(spa_get_dsl(spa), 2768 spa_livelist_condense_sync, lca, tx); 2769 dmu_tx_commit(tx); 2770 return; 2771 } 2772 } 2773 /* 2774 * Condensing can not continue: either it was externally stopped or 2775 * we were unable to assign to a tx because the pool has run out of 2776 * space. In the second case, we'll just end up trying to condense 2777 * again in a later txg. 2778 */ 2779 ASSERT(err != 0); 2780 bplist_clear(&lca->to_keep); 2781 bplist_destroy(&lca->to_keep); 2782 kmem_free(lca, sizeof (livelist_condense_arg_t)); 2783 dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf, spa); 2784 spa->spa_to_condense.ds = NULL; 2785 if (err == EINTR) 2786 zfs_livelist_condense_zthr_cancel++; 2787} 2788 2789/* ARGSUSED */ 2790/* 2791 * Check that there is something to condense but that a condense is not 2792 * already in progress and that condensing has not been cancelled. 2793 */ 2794static boolean_t 2795spa_livelist_condense_cb_check(void *arg, zthr_t *z) 2796{ 2797 spa_t *spa = arg; 2798 if ((spa->spa_to_condense.ds != NULL) && 2799 (spa->spa_to_condense.syncing == B_FALSE) && 2800 (spa->spa_to_condense.cancelled == B_FALSE)) { 2801 return (B_TRUE); 2802 } 2803 return (B_FALSE); 2804} 2805 2806static void 2807spa_start_livelist_condensing_thread(spa_t *spa) 2808{ 2809 spa->spa_to_condense.ds = NULL; 2810 spa->spa_to_condense.first = NULL; 2811 spa->spa_to_condense.next = NULL; 2812 spa->spa_to_condense.syncing = B_FALSE; 2813 spa->spa_to_condense.cancelled = B_FALSE; 2814 2815 ASSERT3P(spa->spa_livelist_condense_zthr, ==, NULL); 2816 spa->spa_livelist_condense_zthr = 2817 zthr_create("z_livelist_condense", 2818 spa_livelist_condense_cb_check, 2819 spa_livelist_condense_cb, spa); 2820} 2821 2822static void 2823spa_spawn_aux_threads(spa_t *spa) 2824{ 2825 ASSERT(spa_writeable(spa)); 2826 2827 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 2828 2829 spa_start_indirect_condensing_thread(spa); 2830 spa_start_livelist_destroy_thread(spa); 2831 spa_start_livelist_condensing_thread(spa); 2832 2833 ASSERT3P(spa->spa_checkpoint_discard_zthr, ==, NULL); 2834 spa->spa_checkpoint_discard_zthr = 2835 zthr_create("z_checkpoint_discard", 2836 spa_checkpoint_discard_thread_check, 2837 spa_checkpoint_discard_thread, spa); 2838} 2839 2840/* 2841 * Fix up config after a partly-completed split. This is done with the 2842 * ZPOOL_CONFIG_SPLIT nvlist. Both the splitting pool and the split-off 2843 * pool have that entry in their config, but only the splitting one contains 2844 * a list of all the guids of the vdevs that are being split off. 2845 * 2846 * This function determines what to do with that list: either rejoin 2847 * all the disks to the pool, or complete the splitting process. To attempt 2848 * the rejoin, each disk that is offlined is marked online again, and 2849 * we do a reopen() call. If the vdev label for every disk that was 2850 * marked online indicates it was successfully split off (VDEV_AUX_SPLIT_POOL) 2851 * then we call vdev_split() on each disk, and complete the split. 2852 * 2853 * Otherwise we leave the config alone, with all the vdevs in place in 2854 * the original pool. 2855 */ 2856static void 2857spa_try_repair(spa_t *spa, nvlist_t *config) 2858{ 2859 uint_t extracted; 2860 uint64_t *glist; 2861 uint_t i, gcount; 2862 nvlist_t *nvl; 2863 vdev_t **vd; 2864 boolean_t attempt_reopen; 2865 2866 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) != 0) 2867 return; 2868 2869 /* check that the config is complete */ 2870 if (nvlist_lookup_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 2871 &glist, &gcount) != 0) 2872 return; 2873 2874 vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); 2875 2876 /* attempt to online all the vdevs & validate */ 2877 attempt_reopen = B_TRUE; 2878 for (i = 0; i < gcount; i++) { 2879 if (glist[i] == 0) /* vdev is hole */ 2880 continue; 2881 2882 vd[i] = spa_lookup_by_guid(spa, glist[i], B_FALSE); 2883 if (vd[i] == NULL) { 2884 /* 2885 * Don't bother attempting to reopen the disks; 2886 * just do the split. 2887 */ 2888 attempt_reopen = B_FALSE; 2889 } else { 2890 /* attempt to re-online it */ 2891 vd[i]->vdev_offline = B_FALSE; 2892 } 2893 } 2894 2895 if (attempt_reopen) { 2896 vdev_reopen(spa->spa_root_vdev); 2897 2898 /* check each device to see what state it's in */ 2899 for (extracted = 0, i = 0; i < gcount; i++) { 2900 if (vd[i] != NULL && 2901 vd[i]->vdev_stat.vs_aux != VDEV_AUX_SPLIT_POOL) 2902 break; 2903 ++extracted; 2904 } 2905 } 2906 2907 /* 2908 * If every disk has been moved to the new pool, or if we never 2909 * even attempted to look at them, then we split them off for 2910 * good. 2911 */ 2912 if (!attempt_reopen || gcount == extracted) { 2913 for (i = 0; i < gcount; i++) 2914 if (vd[i] != NULL) 2915 vdev_split(vd[i]); 2916 vdev_reopen(spa->spa_root_vdev); 2917 } 2918 2919 kmem_free(vd, gcount * sizeof (vdev_t *)); 2920} 2921 2922static int 2923spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type) 2924{ 2925 char *ereport = FM_EREPORT_ZFS_POOL; 2926 int error; 2927 2928 spa->spa_load_state = state; 2929 (void) spa_import_progress_set_state(spa_guid(spa), 2930 spa_load_state(spa)); 2931 2932 gethrestime(&spa->spa_loaded_ts); 2933 error = spa_load_impl(spa, type, &ereport); 2934 2935 /* 2936 * Don't count references from objsets that are already closed 2937 * and are making their way through the eviction process. 2938 */ 2939 spa_evicting_os_wait(spa); 2940 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 2941 if (error) { 2942 if (error != EEXIST) { 2943 spa->spa_loaded_ts.tv_sec = 0; 2944 spa->spa_loaded_ts.tv_nsec = 0; 2945 } 2946 if (error != EBADF) { 2947 (void) zfs_ereport_post(ereport, spa, 2948 NULL, NULL, NULL, 0); 2949 } 2950 } 2951 spa->spa_load_state = error ? SPA_LOAD_ERROR : SPA_LOAD_NONE; 2952 spa->spa_ena = 0; 2953 2954 (void) spa_import_progress_set_state(spa_guid(spa), 2955 spa_load_state(spa)); 2956 2957 return (error); 2958} 2959 2960#ifdef ZFS_DEBUG 2961/* 2962 * Count the number of per-vdev ZAPs associated with all of the vdevs in the 2963 * vdev tree rooted in the given vd, and ensure that each ZAP is present in the 2964 * spa's per-vdev ZAP list. 2965 */ 2966static uint64_t 2967vdev_count_verify_zaps(vdev_t *vd) 2968{ 2969 spa_t *spa = vd->vdev_spa; 2970 uint64_t total = 0; 2971 2972 if (vd->vdev_top_zap != 0) { 2973 total++; 2974 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2975 spa->spa_all_vdev_zaps, vd->vdev_top_zap)); 2976 } 2977 if (vd->vdev_leaf_zap != 0) { 2978 total++; 2979 ASSERT0(zap_lookup_int(spa->spa_meta_objset, 2980 spa->spa_all_vdev_zaps, vd->vdev_leaf_zap)); 2981 } 2982 2983 for (uint64_t i = 0; i < vd->vdev_children; i++) { 2984 total += vdev_count_verify_zaps(vd->vdev_child[i]); 2985 } 2986 2987 return (total); 2988} 2989#endif 2990 2991/* 2992 * Determine whether the activity check is required. 2993 */ 2994static boolean_t 2995spa_activity_check_required(spa_t *spa, uberblock_t *ub, nvlist_t *label, 2996 nvlist_t *config) 2997{ 2998 uint64_t state = 0; 2999 uint64_t hostid = 0; 3000 uint64_t tryconfig_txg = 0; 3001 uint64_t tryconfig_timestamp = 0; 3002 uint16_t tryconfig_mmp_seq = 0; 3003 nvlist_t *nvinfo; 3004 3005 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3006 nvinfo = fnvlist_lookup_nvlist(config, ZPOOL_CONFIG_LOAD_INFO); 3007 (void) nvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG, 3008 &tryconfig_txg); 3009 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 3010 &tryconfig_timestamp); 3011 (void) nvlist_lookup_uint16(nvinfo, ZPOOL_CONFIG_MMP_SEQ, 3012 &tryconfig_mmp_seq); 3013 } 3014 3015 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_STATE, &state); 3016 3017 /* 3018 * Disable the MMP activity check - This is used by zdb which 3019 * is intended to be used on potentially active pools. 3020 */ 3021 if (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) 3022 return (B_FALSE); 3023 3024 /* 3025 * Skip the activity check when the MMP feature is disabled. 3026 */ 3027 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay == 0) 3028 return (B_FALSE); 3029 3030 /* 3031 * If the tryconfig_ values are nonzero, they are the results of an 3032 * earlier tryimport. If they all match the uberblock we just found, 3033 * then the pool has not changed and we return false so we do not test 3034 * a second time. 3035 */ 3036 if (tryconfig_txg && tryconfig_txg == ub->ub_txg && 3037 tryconfig_timestamp && tryconfig_timestamp == ub->ub_timestamp && 3038 tryconfig_mmp_seq && tryconfig_mmp_seq == 3039 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) 3040 return (B_FALSE); 3041 3042 /* 3043 * Allow the activity check to be skipped when importing the pool 3044 * on the same host which last imported it. Since the hostid from 3045 * configuration may be stale use the one read from the label. 3046 */ 3047 if (nvlist_exists(label, ZPOOL_CONFIG_HOSTID)) 3048 hostid = fnvlist_lookup_uint64(label, ZPOOL_CONFIG_HOSTID); 3049 3050 if (hostid == spa_get_hostid(spa)) 3051 return (B_FALSE); 3052 3053 /* 3054 * Skip the activity test when the pool was cleanly exported. 3055 */ 3056 if (state != POOL_STATE_ACTIVE) 3057 return (B_FALSE); 3058 3059 return (B_TRUE); 3060} 3061 3062/* 3063 * Nanoseconds the activity check must watch for changes on-disk. 3064 */ 3065static uint64_t 3066spa_activity_check_duration(spa_t *spa, uberblock_t *ub) 3067{ 3068 uint64_t import_intervals = MAX(zfs_multihost_import_intervals, 1); 3069 uint64_t multihost_interval = MSEC2NSEC( 3070 MMP_INTERVAL_OK(zfs_multihost_interval)); 3071 uint64_t import_delay = MAX(NANOSEC, import_intervals * 3072 multihost_interval); 3073 3074 /* 3075 * Local tunables determine a minimum duration except for the case 3076 * where we know when the remote host will suspend the pool if MMP 3077 * writes do not land. 3078 * 3079 * See Big Theory comment at the top of mmp.c for the reasoning behind 3080 * these cases and times. 3081 */ 3082 3083 ASSERT(MMP_IMPORT_SAFETY_FACTOR >= 100); 3084 3085 if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3086 MMP_FAIL_INT(ub) > 0) { 3087 3088 /* MMP on remote host will suspend pool after failed writes */ 3089 import_delay = MMP_FAIL_INT(ub) * MSEC2NSEC(MMP_INTERVAL(ub)) * 3090 MMP_IMPORT_SAFETY_FACTOR / 100; 3091 3092 zfs_dbgmsg("fail_intvals>0 import_delay=%llu ub_mmp " 3093 "mmp_fails=%llu ub_mmp mmp_interval=%llu " 3094 "import_intervals=%u", import_delay, MMP_FAIL_INT(ub), 3095 MMP_INTERVAL(ub), import_intervals); 3096 3097 } else if (MMP_INTERVAL_VALID(ub) && MMP_FAIL_INT_VALID(ub) && 3098 MMP_FAIL_INT(ub) == 0) { 3099 3100 /* MMP on remote host will never suspend pool */ 3101 import_delay = MAX(import_delay, (MSEC2NSEC(MMP_INTERVAL(ub)) + 3102 ub->ub_mmp_delay) * import_intervals); 3103 3104 zfs_dbgmsg("fail_intvals=0 import_delay=%llu ub_mmp " 3105 "mmp_interval=%llu ub_mmp_delay=%llu " 3106 "import_intervals=%u", import_delay, MMP_INTERVAL(ub), 3107 ub->ub_mmp_delay, import_intervals); 3108 3109 } else if (MMP_VALID(ub)) { 3110 /* 3111 * zfs-0.7 compatibility case 3112 */ 3113 3114 import_delay = MAX(import_delay, (multihost_interval + 3115 ub->ub_mmp_delay) * import_intervals); 3116 3117 zfs_dbgmsg("import_delay=%llu ub_mmp_delay=%llu " 3118 "import_intervals=%u leaves=%u", import_delay, 3119 ub->ub_mmp_delay, import_intervals, 3120 vdev_count_leaves(spa)); 3121 } else { 3122 /* Using local tunings is the only reasonable option */ 3123 zfs_dbgmsg("pool last imported on non-MMP aware " 3124 "host using import_delay=%llu multihost_interval=%llu " 3125 "import_intervals=%u", import_delay, multihost_interval, 3126 import_intervals); 3127 } 3128 3129 return (import_delay); 3130} 3131 3132/* 3133 * Perform the import activity check. If the user canceled the import or 3134 * we detected activity then fail. 3135 */ 3136static int 3137spa_activity_check(spa_t *spa, uberblock_t *ub, nvlist_t *config) 3138{ 3139 uint64_t txg = ub->ub_txg; 3140 uint64_t timestamp = ub->ub_timestamp; 3141 uint64_t mmp_config = ub->ub_mmp_config; 3142 uint16_t mmp_seq = MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0; 3143 uint64_t import_delay; 3144 hrtime_t import_expire; 3145 nvlist_t *mmp_label = NULL; 3146 vdev_t *rvd = spa->spa_root_vdev; 3147 kcondvar_t cv; 3148 kmutex_t mtx; 3149 int error = 0; 3150 3151 cv_init(&cv, NULL, CV_DEFAULT, NULL); 3152 mutex_init(&mtx, NULL, MUTEX_DEFAULT, NULL); 3153 mutex_enter(&mtx); 3154 3155 /* 3156 * If ZPOOL_CONFIG_MMP_TXG is present an activity check was performed 3157 * during the earlier tryimport. If the txg recorded there is 0 then 3158 * the pool is known to be active on another host. 3159 * 3160 * Otherwise, the pool might be in use on another host. Check for 3161 * changes in the uberblocks on disk if necessary. 3162 */ 3163 if (nvlist_exists(config, ZPOOL_CONFIG_LOAD_INFO)) { 3164 nvlist_t *nvinfo = fnvlist_lookup_nvlist(config, 3165 ZPOOL_CONFIG_LOAD_INFO); 3166 3167 if (nvlist_exists(nvinfo, ZPOOL_CONFIG_MMP_TXG) && 3168 fnvlist_lookup_uint64(nvinfo, ZPOOL_CONFIG_MMP_TXG) == 0) { 3169 vdev_uberblock_load(rvd, ub, &mmp_label); 3170 error = SET_ERROR(EREMOTEIO); 3171 goto out; 3172 } 3173 } 3174 3175 import_delay = spa_activity_check_duration(spa, ub); 3176 3177 /* Add a small random factor in case of simultaneous imports (0-25%) */ 3178 import_delay += import_delay * spa_get_random(250) / 1000; 3179 3180 import_expire = gethrtime() + import_delay; 3181 3182 while (gethrtime() < import_expire) { 3183 (void) spa_import_progress_set_mmp_check(spa_guid(spa), 3184 NSEC2SEC(import_expire - gethrtime())); 3185 3186 vdev_uberblock_load(rvd, ub, &mmp_label); 3187 3188 if (txg != ub->ub_txg || timestamp != ub->ub_timestamp || 3189 mmp_seq != (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)) { 3190 zfs_dbgmsg("multihost activity detected " 3191 "txg %llu ub_txg %llu " 3192 "timestamp %llu ub_timestamp %llu " 3193 "mmp_config %#llx ub_mmp_config %#llx", 3194 txg, ub->ub_txg, timestamp, ub->ub_timestamp, 3195 mmp_config, ub->ub_mmp_config); 3196 3197 error = SET_ERROR(EREMOTEIO); 3198 break; 3199 } 3200 3201 if (mmp_label) { 3202 nvlist_free(mmp_label); 3203 mmp_label = NULL; 3204 } 3205 3206 error = cv_timedwait_sig(&cv, &mtx, ddi_get_lbolt() + hz); 3207 if (error != -1) { 3208 error = SET_ERROR(EINTR); 3209 break; 3210 } 3211 error = 0; 3212 } 3213 3214out: 3215 mutex_exit(&mtx); 3216 mutex_destroy(&mtx); 3217 cv_destroy(&cv); 3218 3219 /* 3220 * If the pool is determined to be active store the status in the 3221 * spa->spa_load_info nvlist. If the remote hostname or hostid are 3222 * available from configuration read from disk store them as well. 3223 * This allows 'zpool import' to generate a more useful message. 3224 * 3225 * ZPOOL_CONFIG_MMP_STATE - observed pool status (mandatory) 3226 * ZPOOL_CONFIG_MMP_HOSTNAME - hostname from the active pool 3227 * ZPOOL_CONFIG_MMP_HOSTID - hostid from the active pool 3228 */ 3229 if (error == EREMOTEIO) { 3230 char *hostname = "<unknown>"; 3231 uint64_t hostid = 0; 3232 3233 if (mmp_label) { 3234 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTNAME)) { 3235 hostname = fnvlist_lookup_string(mmp_label, 3236 ZPOOL_CONFIG_HOSTNAME); 3237 fnvlist_add_string(spa->spa_load_info, 3238 ZPOOL_CONFIG_MMP_HOSTNAME, hostname); 3239 } 3240 3241 if (nvlist_exists(mmp_label, ZPOOL_CONFIG_HOSTID)) { 3242 hostid = fnvlist_lookup_uint64(mmp_label, 3243 ZPOOL_CONFIG_HOSTID); 3244 fnvlist_add_uint64(spa->spa_load_info, 3245 ZPOOL_CONFIG_MMP_HOSTID, hostid); 3246 } 3247 } 3248 3249 fnvlist_add_uint64(spa->spa_load_info, 3250 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_ACTIVE); 3251 fnvlist_add_uint64(spa->spa_load_info, 3252 ZPOOL_CONFIG_MMP_TXG, 0); 3253 3254 error = spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO); 3255 } 3256 3257 if (mmp_label) 3258 nvlist_free(mmp_label); 3259 3260 return (error); 3261} 3262 3263static int 3264spa_verify_host(spa_t *spa, nvlist_t *mos_config) 3265{ 3266 uint64_t hostid; 3267 char *hostname; 3268 uint64_t myhostid = 0; 3269 3270 if (!spa_is_root(spa) && nvlist_lookup_uint64(mos_config, 3271 ZPOOL_CONFIG_HOSTID, &hostid) == 0) { 3272 hostname = fnvlist_lookup_string(mos_config, 3273 ZPOOL_CONFIG_HOSTNAME); 3274 3275 myhostid = zone_get_hostid(NULL); 3276 3277 if (hostid != 0 && myhostid != 0 && hostid != myhostid) { 3278 cmn_err(CE_WARN, "pool '%s' could not be " 3279 "loaded as it was last accessed by " 3280 "another system (host: %s hostid: 0x%llx). " 3281 "See: https://openzfs.github.io/openzfs-docs/msg/" 3282 "ZFS-8000-EY", 3283 spa_name(spa), hostname, (u_longlong_t)hostid); 3284 spa_load_failed(spa, "hostid verification failed: pool " 3285 "last accessed by host: %s (hostid: 0x%llx)", 3286 hostname, (u_longlong_t)hostid); 3287 return (SET_ERROR(EBADF)); 3288 } 3289 } 3290 3291 return (0); 3292} 3293 3294static int 3295spa_ld_parse_config(spa_t *spa, spa_import_type_t type) 3296{ 3297 int error = 0; 3298 nvlist_t *nvtree, *nvl, *config = spa->spa_config; 3299 int parse; 3300 vdev_t *rvd; 3301 uint64_t pool_guid; 3302 char *comment; 3303 char *compatibility; 3304 3305 /* 3306 * Versioning wasn't explicitly added to the label until later, so if 3307 * it's not present treat it as the initial version. 3308 */ 3309 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION, 3310 &spa->spa_ubsync.ub_version) != 0) 3311 spa->spa_ubsync.ub_version = SPA_VERSION_INITIAL; 3312 3313 if (nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_GUID, &pool_guid)) { 3314 spa_load_failed(spa, "invalid config provided: '%s' missing", 3315 ZPOOL_CONFIG_POOL_GUID); 3316 return (SET_ERROR(EINVAL)); 3317 } 3318 3319 /* 3320 * If we are doing an import, ensure that the pool is not already 3321 * imported by checking if its pool guid already exists in the 3322 * spa namespace. 3323 * 3324 * The only case that we allow an already imported pool to be 3325 * imported again, is when the pool is checkpointed and we want to 3326 * look at its checkpointed state from userland tools like zdb. 3327 */ 3328#ifdef _KERNEL 3329 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3330 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3331 spa_guid_exists(pool_guid, 0)) { 3332#else 3333 if ((spa->spa_load_state == SPA_LOAD_IMPORT || 3334 spa->spa_load_state == SPA_LOAD_TRYIMPORT) && 3335 spa_guid_exists(pool_guid, 0) && 3336 !spa_importing_readonly_checkpoint(spa)) { 3337#endif 3338 spa_load_failed(spa, "a pool with guid %llu is already open", 3339 (u_longlong_t)pool_guid); 3340 return (SET_ERROR(EEXIST)); 3341 } 3342 3343 spa->spa_config_guid = pool_guid; 3344 3345 nvlist_free(spa->spa_load_info); 3346 spa->spa_load_info = fnvlist_alloc(); 3347 3348 ASSERT(spa->spa_comment == NULL); 3349 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMMENT, &comment) == 0) 3350 spa->spa_comment = spa_strdup(comment); 3351 3352 ASSERT(spa->spa_compatibility == NULL); 3353 if (nvlist_lookup_string(config, ZPOOL_CONFIG_COMPATIBILITY, 3354 &compatibility) == 0) 3355 spa->spa_compatibility = spa_strdup(compatibility); 3356 3357 (void) nvlist_lookup_uint64(config, ZPOOL_CONFIG_POOL_TXG, 3358 &spa->spa_config_txg); 3359 3360 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) 3361 spa->spa_config_splitting = fnvlist_dup(nvl); 3362 3363 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvtree)) { 3364 spa_load_failed(spa, "invalid config provided: '%s' missing", 3365 ZPOOL_CONFIG_VDEV_TREE); 3366 return (SET_ERROR(EINVAL)); 3367 } 3368 3369 /* 3370 * Create "The Godfather" zio to hold all async IOs 3371 */ 3372 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 3373 KM_SLEEP); 3374 for (int i = 0; i < max_ncpus; i++) { 3375 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 3376 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3377 ZIO_FLAG_GODFATHER); 3378 } 3379 3380 /* 3381 * Parse the configuration into a vdev tree. We explicitly set the 3382 * value that will be returned by spa_version() since parsing the 3383 * configuration requires knowing the version number. 3384 */ 3385 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3386 parse = (type == SPA_IMPORT_EXISTING ? 3387 VDEV_ALLOC_LOAD : VDEV_ALLOC_SPLIT); 3388 error = spa_config_parse(spa, &rvd, nvtree, NULL, 0, parse); 3389 spa_config_exit(spa, SCL_ALL, FTAG); 3390 3391 if (error != 0) { 3392 spa_load_failed(spa, "unable to parse config [error=%d]", 3393 error); 3394 return (error); 3395 } 3396 3397 ASSERT(spa->spa_root_vdev == rvd); 3398 ASSERT3U(spa->spa_min_ashift, >=, SPA_MINBLOCKSHIFT); 3399 ASSERT3U(spa->spa_max_ashift, <=, SPA_MAXBLOCKSHIFT); 3400 3401 if (type != SPA_IMPORT_ASSEMBLE) { 3402 ASSERT(spa_guid(spa) == pool_guid); 3403 } 3404 3405 return (0); 3406} 3407 3408/* 3409 * Recursively open all vdevs in the vdev tree. This function is called twice: 3410 * first with the untrusted config, then with the trusted config. 3411 */ 3412static int 3413spa_ld_open_vdevs(spa_t *spa) 3414{ 3415 int error = 0; 3416 3417 /* 3418 * spa_missing_tvds_allowed defines how many top-level vdevs can be 3419 * missing/unopenable for the root vdev to be still considered openable. 3420 */ 3421 if (spa->spa_trust_config) { 3422 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds; 3423 } else if (spa->spa_config_source == SPA_CONFIG_SRC_CACHEFILE) { 3424 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_cachefile; 3425 } else if (spa->spa_config_source == SPA_CONFIG_SRC_SCAN) { 3426 spa->spa_missing_tvds_allowed = zfs_max_missing_tvds_scan; 3427 } else { 3428 spa->spa_missing_tvds_allowed = 0; 3429 } 3430 3431 spa->spa_missing_tvds_allowed = 3432 MAX(zfs_max_missing_tvds, spa->spa_missing_tvds_allowed); 3433 3434 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3435 error = vdev_open(spa->spa_root_vdev); 3436 spa_config_exit(spa, SCL_ALL, FTAG); 3437 3438 if (spa->spa_missing_tvds != 0) { 3439 spa_load_note(spa, "vdev tree has %lld missing top-level " 3440 "vdevs.", (u_longlong_t)spa->spa_missing_tvds); 3441 if (spa->spa_trust_config && (spa->spa_mode & SPA_MODE_WRITE)) { 3442 /* 3443 * Although theoretically we could allow users to open 3444 * incomplete pools in RW mode, we'd need to add a lot 3445 * of extra logic (e.g. adjust pool space to account 3446 * for missing vdevs). 3447 * This limitation also prevents users from accidentally 3448 * opening the pool in RW mode during data recovery and 3449 * damaging it further. 3450 */ 3451 spa_load_note(spa, "pools with missing top-level " 3452 "vdevs can only be opened in read-only mode."); 3453 error = SET_ERROR(ENXIO); 3454 } else { 3455 spa_load_note(spa, "current settings allow for maximum " 3456 "%lld missing top-level vdevs at this stage.", 3457 (u_longlong_t)spa->spa_missing_tvds_allowed); 3458 } 3459 } 3460 if (error != 0) { 3461 spa_load_failed(spa, "unable to open vdev tree [error=%d]", 3462 error); 3463 } 3464 if (spa->spa_missing_tvds != 0 || error != 0) 3465 vdev_dbgmsg_print_tree(spa->spa_root_vdev, 2); 3466 3467 return (error); 3468} 3469 3470/* 3471 * We need to validate the vdev labels against the configuration that 3472 * we have in hand. This function is called twice: first with an untrusted 3473 * config, then with a trusted config. The validation is more strict when the 3474 * config is trusted. 3475 */ 3476static int 3477spa_ld_validate_vdevs(spa_t *spa) 3478{ 3479 int error = 0; 3480 vdev_t *rvd = spa->spa_root_vdev; 3481 3482 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3483 error = vdev_validate(rvd); 3484 spa_config_exit(spa, SCL_ALL, FTAG); 3485 3486 if (error != 0) { 3487 spa_load_failed(spa, "vdev_validate failed [error=%d]", error); 3488 return (error); 3489 } 3490 3491 if (rvd->vdev_state <= VDEV_STATE_CANT_OPEN) { 3492 spa_load_failed(spa, "cannot open vdev tree after invalidating " 3493 "some vdevs"); 3494 vdev_dbgmsg_print_tree(rvd, 2); 3495 return (SET_ERROR(ENXIO)); 3496 } 3497 3498 return (0); 3499} 3500 3501static void 3502spa_ld_select_uberblock_done(spa_t *spa, uberblock_t *ub) 3503{ 3504 spa->spa_state = POOL_STATE_ACTIVE; 3505 spa->spa_ubsync = spa->spa_uberblock; 3506 spa->spa_verify_min_txg = spa->spa_extreme_rewind ? 3507 TXG_INITIAL - 1 : spa_last_synced_txg(spa) - TXG_DEFER_SIZE - 1; 3508 spa->spa_first_txg = spa->spa_last_ubsync_txg ? 3509 spa->spa_last_ubsync_txg : spa_last_synced_txg(spa) + 1; 3510 spa->spa_claim_max_txg = spa->spa_first_txg; 3511 spa->spa_prev_software_version = ub->ub_software_version; 3512} 3513 3514static int 3515spa_ld_select_uberblock(spa_t *spa, spa_import_type_t type) 3516{ 3517 vdev_t *rvd = spa->spa_root_vdev; 3518 nvlist_t *label; 3519 uberblock_t *ub = &spa->spa_uberblock; 3520 boolean_t activity_check = B_FALSE; 3521 3522 /* 3523 * If we are opening the checkpointed state of the pool by 3524 * rewinding to it, at this point we will have written the 3525 * checkpointed uberblock to the vdev labels, so searching 3526 * the labels will find the right uberblock. However, if 3527 * we are opening the checkpointed state read-only, we have 3528 * not modified the labels. Therefore, we must ignore the 3529 * labels and continue using the spa_uberblock that was set 3530 * by spa_ld_checkpoint_rewind. 3531 * 3532 * Note that it would be fine to ignore the labels when 3533 * rewinding (opening writeable) as well. However, if we 3534 * crash just after writing the labels, we will end up 3535 * searching the labels. Doing so in the common case means 3536 * that this code path gets exercised normally, rather than 3537 * just in the edge case. 3538 */ 3539 if (ub->ub_checkpoint_txg != 0 && 3540 spa_importing_readonly_checkpoint(spa)) { 3541 spa_ld_select_uberblock_done(spa, ub); 3542 return (0); 3543 } 3544 3545 /* 3546 * Find the best uberblock. 3547 */ 3548 vdev_uberblock_load(rvd, ub, &label); 3549 3550 /* 3551 * If we weren't able to find a single valid uberblock, return failure. 3552 */ 3553 if (ub->ub_txg == 0) { 3554 nvlist_free(label); 3555 spa_load_failed(spa, "no valid uberblock found"); 3556 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, ENXIO)); 3557 } 3558 3559 if (spa->spa_load_max_txg != UINT64_MAX) { 3560 (void) spa_import_progress_set_max_txg(spa_guid(spa), 3561 (u_longlong_t)spa->spa_load_max_txg); 3562 } 3563 spa_load_note(spa, "using uberblock with txg=%llu", 3564 (u_longlong_t)ub->ub_txg); 3565 3566 3567 /* 3568 * For pools which have the multihost property on determine if the 3569 * pool is truly inactive and can be safely imported. Prevent 3570 * hosts which don't have a hostid set from importing the pool. 3571 */ 3572 activity_check = spa_activity_check_required(spa, ub, label, 3573 spa->spa_config); 3574 if (activity_check) { 3575 if (ub->ub_mmp_magic == MMP_MAGIC && ub->ub_mmp_delay && 3576 spa_get_hostid(spa) == 0) { 3577 nvlist_free(label); 3578 fnvlist_add_uint64(spa->spa_load_info, 3579 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 3580 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 3581 } 3582 3583 int error = spa_activity_check(spa, ub, spa->spa_config); 3584 if (error) { 3585 nvlist_free(label); 3586 return (error); 3587 } 3588 3589 fnvlist_add_uint64(spa->spa_load_info, 3590 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_INACTIVE); 3591 fnvlist_add_uint64(spa->spa_load_info, 3592 ZPOOL_CONFIG_MMP_TXG, ub->ub_txg); 3593 fnvlist_add_uint16(spa->spa_load_info, 3594 ZPOOL_CONFIG_MMP_SEQ, 3595 (MMP_SEQ_VALID(ub) ? MMP_SEQ(ub) : 0)); 3596 } 3597 3598 /* 3599 * If the pool has an unsupported version we can't open it. 3600 */ 3601 if (!SPA_VERSION_IS_SUPPORTED(ub->ub_version)) { 3602 nvlist_free(label); 3603 spa_load_failed(spa, "version %llu is not supported", 3604 (u_longlong_t)ub->ub_version); 3605 return (spa_vdev_err(rvd, VDEV_AUX_VERSION_NEWER, ENOTSUP)); 3606 } 3607 3608 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3609 nvlist_t *features; 3610 3611 /* 3612 * If we weren't able to find what's necessary for reading the 3613 * MOS in the label, return failure. 3614 */ 3615 if (label == NULL) { 3616 spa_load_failed(spa, "label config unavailable"); 3617 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3618 ENXIO)); 3619 } 3620 3621 if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_FEATURES_FOR_READ, 3622 &features) != 0) { 3623 nvlist_free(label); 3624 spa_load_failed(spa, "invalid label: '%s' missing", 3625 ZPOOL_CONFIG_FEATURES_FOR_READ); 3626 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 3627 ENXIO)); 3628 } 3629 3630 /* 3631 * Update our in-core representation with the definitive values 3632 * from the label. 3633 */ 3634 nvlist_free(spa->spa_label_features); 3635 VERIFY(nvlist_dup(features, &spa->spa_label_features, 0) == 0); 3636 } 3637 3638 nvlist_free(label); 3639 3640 /* 3641 * Look through entries in the label nvlist's features_for_read. If 3642 * there is a feature listed there which we don't understand then we 3643 * cannot open a pool. 3644 */ 3645 if (ub->ub_version >= SPA_VERSION_FEATURES) { 3646 nvlist_t *unsup_feat; 3647 3648 VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) == 3649 0); 3650 3651 for (nvpair_t *nvp = nvlist_next_nvpair(spa->spa_label_features, 3652 NULL); nvp != NULL; 3653 nvp = nvlist_next_nvpair(spa->spa_label_features, nvp)) { 3654 if (!zfeature_is_supported(nvpair_name(nvp))) { 3655 VERIFY(nvlist_add_string(unsup_feat, 3656 nvpair_name(nvp), "") == 0); 3657 } 3658 } 3659 3660 if (!nvlist_empty(unsup_feat)) { 3661 VERIFY(nvlist_add_nvlist(spa->spa_load_info, 3662 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0); 3663 nvlist_free(unsup_feat); 3664 spa_load_failed(spa, "some features are unsupported"); 3665 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3666 ENOTSUP)); 3667 } 3668 3669 nvlist_free(unsup_feat); 3670 } 3671 3672 if (type != SPA_IMPORT_ASSEMBLE && spa->spa_config_splitting) { 3673 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3674 spa_try_repair(spa, spa->spa_config); 3675 spa_config_exit(spa, SCL_ALL, FTAG); 3676 nvlist_free(spa->spa_config_splitting); 3677 spa->spa_config_splitting = NULL; 3678 } 3679 3680 /* 3681 * Initialize internal SPA structures. 3682 */ 3683 spa_ld_select_uberblock_done(spa, ub); 3684 3685 return (0); 3686} 3687 3688static int 3689spa_ld_open_rootbp(spa_t *spa) 3690{ 3691 int error = 0; 3692 vdev_t *rvd = spa->spa_root_vdev; 3693 3694 error = dsl_pool_init(spa, spa->spa_first_txg, &spa->spa_dsl_pool); 3695 if (error != 0) { 3696 spa_load_failed(spa, "unable to open rootbp in dsl_pool_init " 3697 "[error=%d]", error); 3698 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3699 } 3700 spa->spa_meta_objset = spa->spa_dsl_pool->dp_meta_objset; 3701 3702 return (0); 3703} 3704 3705static int 3706spa_ld_trusted_config(spa_t *spa, spa_import_type_t type, 3707 boolean_t reloading) 3708{ 3709 vdev_t *mrvd, *rvd = spa->spa_root_vdev; 3710 nvlist_t *nv, *mos_config, *policy; 3711 int error = 0, copy_error; 3712 uint64_t healthy_tvds, healthy_tvds_mos; 3713 uint64_t mos_config_txg; 3714 3715 if (spa_dir_prop(spa, DMU_POOL_CONFIG, &spa->spa_config_object, B_TRUE) 3716 != 0) 3717 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3718 3719 /* 3720 * If we're assembling a pool from a split, the config provided is 3721 * already trusted so there is nothing to do. 3722 */ 3723 if (type == SPA_IMPORT_ASSEMBLE) 3724 return (0); 3725 3726 healthy_tvds = spa_healthy_core_tvds(spa); 3727 3728 if (load_nvlist(spa, spa->spa_config_object, &mos_config) 3729 != 0) { 3730 spa_load_failed(spa, "unable to retrieve MOS config"); 3731 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3732 } 3733 3734 /* 3735 * If we are doing an open, pool owner wasn't verified yet, thus do 3736 * the verification here. 3737 */ 3738 if (spa->spa_load_state == SPA_LOAD_OPEN) { 3739 error = spa_verify_host(spa, mos_config); 3740 if (error != 0) { 3741 nvlist_free(mos_config); 3742 return (error); 3743 } 3744 } 3745 3746 nv = fnvlist_lookup_nvlist(mos_config, ZPOOL_CONFIG_VDEV_TREE); 3747 3748 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 3749 3750 /* 3751 * Build a new vdev tree from the trusted config 3752 */ 3753 error = spa_config_parse(spa, &mrvd, nv, NULL, 0, VDEV_ALLOC_LOAD); 3754 if (error != 0) { 3755 nvlist_free(mos_config); 3756 spa_config_exit(spa, SCL_ALL, FTAG); 3757 spa_load_failed(spa, "spa_config_parse failed [error=%d]", 3758 error); 3759 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3760 } 3761 3762 /* 3763 * Vdev paths in the MOS may be obsolete. If the untrusted config was 3764 * obtained by scanning /dev/dsk, then it will have the right vdev 3765 * paths. We update the trusted MOS config with this information. 3766 * We first try to copy the paths with vdev_copy_path_strict, which 3767 * succeeds only when both configs have exactly the same vdev tree. 3768 * If that fails, we fall back to a more flexible method that has a 3769 * best effort policy. 3770 */ 3771 copy_error = vdev_copy_path_strict(rvd, mrvd); 3772 if (copy_error != 0 || spa_load_print_vdev_tree) { 3773 spa_load_note(spa, "provided vdev tree:"); 3774 vdev_dbgmsg_print_tree(rvd, 2); 3775 spa_load_note(spa, "MOS vdev tree:"); 3776 vdev_dbgmsg_print_tree(mrvd, 2); 3777 } 3778 if (copy_error != 0) { 3779 spa_load_note(spa, "vdev_copy_path_strict failed, falling " 3780 "back to vdev_copy_path_relaxed"); 3781 vdev_copy_path_relaxed(rvd, mrvd); 3782 } 3783 3784 vdev_close(rvd); 3785 vdev_free(rvd); 3786 spa->spa_root_vdev = mrvd; 3787 rvd = mrvd; 3788 spa_config_exit(spa, SCL_ALL, FTAG); 3789 3790 /* 3791 * We will use spa_config if we decide to reload the spa or if spa_load 3792 * fails and we rewind. We must thus regenerate the config using the 3793 * MOS information with the updated paths. ZPOOL_LOAD_POLICY is used to 3794 * pass settings on how to load the pool and is not stored in the MOS. 3795 * We copy it over to our new, trusted config. 3796 */ 3797 mos_config_txg = fnvlist_lookup_uint64(mos_config, 3798 ZPOOL_CONFIG_POOL_TXG); 3799 nvlist_free(mos_config); 3800 mos_config = spa_config_generate(spa, NULL, mos_config_txg, B_FALSE); 3801 if (nvlist_lookup_nvlist(spa->spa_config, ZPOOL_LOAD_POLICY, 3802 &policy) == 0) 3803 fnvlist_add_nvlist(mos_config, ZPOOL_LOAD_POLICY, policy); 3804 spa_config_set(spa, mos_config); 3805 spa->spa_config_source = SPA_CONFIG_SRC_MOS; 3806 3807 /* 3808 * Now that we got the config from the MOS, we should be more strict 3809 * in checking blkptrs and can make assumptions about the consistency 3810 * of the vdev tree. spa_trust_config must be set to true before opening 3811 * vdevs in order for them to be writeable. 3812 */ 3813 spa->spa_trust_config = B_TRUE; 3814 3815 /* 3816 * Open and validate the new vdev tree 3817 */ 3818 error = spa_ld_open_vdevs(spa); 3819 if (error != 0) 3820 return (error); 3821 3822 error = spa_ld_validate_vdevs(spa); 3823 if (error != 0) 3824 return (error); 3825 3826 if (copy_error != 0 || spa_load_print_vdev_tree) { 3827 spa_load_note(spa, "final vdev tree:"); 3828 vdev_dbgmsg_print_tree(rvd, 2); 3829 } 3830 3831 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT && 3832 !spa->spa_extreme_rewind && zfs_max_missing_tvds == 0) { 3833 /* 3834 * Sanity check to make sure that we are indeed loading the 3835 * latest uberblock. If we missed SPA_SYNC_MIN_VDEVS tvds 3836 * in the config provided and they happened to be the only ones 3837 * to have the latest uberblock, we could involuntarily perform 3838 * an extreme rewind. 3839 */ 3840 healthy_tvds_mos = spa_healthy_core_tvds(spa); 3841 if (healthy_tvds_mos - healthy_tvds >= 3842 SPA_SYNC_MIN_VDEVS) { 3843 spa_load_note(spa, "config provided misses too many " 3844 "top-level vdevs compared to MOS (%lld vs %lld). ", 3845 (u_longlong_t)healthy_tvds, 3846 (u_longlong_t)healthy_tvds_mos); 3847 spa_load_note(spa, "vdev tree:"); 3848 vdev_dbgmsg_print_tree(rvd, 2); 3849 if (reloading) { 3850 spa_load_failed(spa, "config was already " 3851 "provided from MOS. Aborting."); 3852 return (spa_vdev_err(rvd, 3853 VDEV_AUX_CORRUPT_DATA, EIO)); 3854 } 3855 spa_load_note(spa, "spa must be reloaded using MOS " 3856 "config"); 3857 return (SET_ERROR(EAGAIN)); 3858 } 3859 } 3860 3861 error = spa_check_for_missing_logs(spa); 3862 if (error != 0) 3863 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, ENXIO)); 3864 3865 if (rvd->vdev_guid_sum != spa->spa_uberblock.ub_guid_sum) { 3866 spa_load_failed(spa, "uberblock guid sum doesn't match MOS " 3867 "guid sum (%llu != %llu)", 3868 (u_longlong_t)spa->spa_uberblock.ub_guid_sum, 3869 (u_longlong_t)rvd->vdev_guid_sum); 3870 return (spa_vdev_err(rvd, VDEV_AUX_BAD_GUID_SUM, 3871 ENXIO)); 3872 } 3873 3874 return (0); 3875} 3876 3877static int 3878spa_ld_open_indirect_vdev_metadata(spa_t *spa) 3879{ 3880 int error = 0; 3881 vdev_t *rvd = spa->spa_root_vdev; 3882 3883 /* 3884 * Everything that we read before spa_remove_init() must be stored 3885 * on concreted vdevs. Therefore we do this as early as possible. 3886 */ 3887 error = spa_remove_init(spa); 3888 if (error != 0) { 3889 spa_load_failed(spa, "spa_remove_init failed [error=%d]", 3890 error); 3891 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3892 } 3893 3894 /* 3895 * Retrieve information needed to condense indirect vdev mappings. 3896 */ 3897 error = spa_condense_init(spa); 3898 if (error != 0) { 3899 spa_load_failed(spa, "spa_condense_init failed [error=%d]", 3900 error); 3901 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 3902 } 3903 3904 return (0); 3905} 3906 3907static int 3908spa_ld_check_features(spa_t *spa, boolean_t *missing_feat_writep) 3909{ 3910 int error = 0; 3911 vdev_t *rvd = spa->spa_root_vdev; 3912 3913 if (spa_version(spa) >= SPA_VERSION_FEATURES) { 3914 boolean_t missing_feat_read = B_FALSE; 3915 nvlist_t *unsup_feat, *enabled_feat; 3916 3917 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ, 3918 &spa->spa_feat_for_read_obj, B_TRUE) != 0) { 3919 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3920 } 3921 3922 if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_WRITE, 3923 &spa->spa_feat_for_write_obj, B_TRUE) != 0) { 3924 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3925 } 3926 3927 if (spa_dir_prop(spa, DMU_POOL_FEATURE_DESCRIPTIONS, 3928 &spa->spa_feat_desc_obj, B_TRUE) != 0) { 3929 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 3930 } 3931 3932 enabled_feat = fnvlist_alloc(); 3933 unsup_feat = fnvlist_alloc(); 3934 3935 if (!spa_features_check(spa, B_FALSE, 3936 unsup_feat, enabled_feat)) 3937 missing_feat_read = B_TRUE; 3938 3939 if (spa_writeable(spa) || 3940 spa->spa_load_state == SPA_LOAD_TRYIMPORT) { 3941 if (!spa_features_check(spa, B_TRUE, 3942 unsup_feat, enabled_feat)) { 3943 *missing_feat_writep = B_TRUE; 3944 } 3945 } 3946 3947 fnvlist_add_nvlist(spa->spa_load_info, 3948 ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat); 3949 3950 if (!nvlist_empty(unsup_feat)) { 3951 fnvlist_add_nvlist(spa->spa_load_info, 3952 ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat); 3953 } 3954 3955 fnvlist_free(enabled_feat); 3956 fnvlist_free(unsup_feat); 3957 3958 if (!missing_feat_read) { 3959 fnvlist_add_boolean(spa->spa_load_info, 3960 ZPOOL_CONFIG_CAN_RDONLY); 3961 } 3962 3963 /* 3964 * If the state is SPA_LOAD_TRYIMPORT, our objective is 3965 * twofold: to determine whether the pool is available for 3966 * import in read-write mode and (if it is not) whether the 3967 * pool is available for import in read-only mode. If the pool 3968 * is available for import in read-write mode, it is displayed 3969 * as available in userland; if it is not available for import 3970 * in read-only mode, it is displayed as unavailable in 3971 * userland. If the pool is available for import in read-only 3972 * mode but not read-write mode, it is displayed as unavailable 3973 * in userland with a special note that the pool is actually 3974 * available for open in read-only mode. 3975 * 3976 * As a result, if the state is SPA_LOAD_TRYIMPORT and we are 3977 * missing a feature for write, we must first determine whether 3978 * the pool can be opened read-only before returning to 3979 * userland in order to know whether to display the 3980 * abovementioned note. 3981 */ 3982 if (missing_feat_read || (*missing_feat_writep && 3983 spa_writeable(spa))) { 3984 spa_load_failed(spa, "pool uses unsupported features"); 3985 return (spa_vdev_err(rvd, VDEV_AUX_UNSUP_FEAT, 3986 ENOTSUP)); 3987 } 3988 3989 /* 3990 * Load refcounts for ZFS features from disk into an in-memory 3991 * cache during SPA initialization. 3992 */ 3993 for (spa_feature_t i = 0; i < SPA_FEATURES; i++) { 3994 uint64_t refcount; 3995 3996 error = feature_get_refcount_from_disk(spa, 3997 &spa_feature_table[i], &refcount); 3998 if (error == 0) { 3999 spa->spa_feat_refcount_cache[i] = refcount; 4000 } else if (error == ENOTSUP) { 4001 spa->spa_feat_refcount_cache[i] = 4002 SPA_FEATURE_DISABLED; 4003 } else { 4004 spa_load_failed(spa, "error getting refcount " 4005 "for feature %s [error=%d]", 4006 spa_feature_table[i].fi_guid, error); 4007 return (spa_vdev_err(rvd, 4008 VDEV_AUX_CORRUPT_DATA, EIO)); 4009 } 4010 } 4011 } 4012 4013 if (spa_feature_is_active(spa, SPA_FEATURE_ENABLED_TXG)) { 4014 if (spa_dir_prop(spa, DMU_POOL_FEATURE_ENABLED_TXG, 4015 &spa->spa_feat_enabled_txg_obj, B_TRUE) != 0) 4016 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4017 } 4018 4019 /* 4020 * Encryption was added before bookmark_v2, even though bookmark_v2 4021 * is now a dependency. If this pool has encryption enabled without 4022 * bookmark_v2, trigger an errata message. 4023 */ 4024 if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) && 4025 !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) { 4026 spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION; 4027 } 4028 4029 return (0); 4030} 4031 4032static int 4033spa_ld_load_special_directories(spa_t *spa) 4034{ 4035 int error = 0; 4036 vdev_t *rvd = spa->spa_root_vdev; 4037 4038 spa->spa_is_initializing = B_TRUE; 4039 error = dsl_pool_open(spa->spa_dsl_pool); 4040 spa->spa_is_initializing = B_FALSE; 4041 if (error != 0) { 4042 spa_load_failed(spa, "dsl_pool_open failed [error=%d]", error); 4043 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4044 } 4045 4046 return (0); 4047} 4048 4049static int 4050spa_ld_get_props(spa_t *spa) 4051{ 4052 int error = 0; 4053 uint64_t obj; 4054 vdev_t *rvd = spa->spa_root_vdev; 4055 4056 /* Grab the checksum salt from the MOS. */ 4057 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4058 DMU_POOL_CHECKSUM_SALT, 1, 4059 sizeof (spa->spa_cksum_salt.zcs_bytes), 4060 spa->spa_cksum_salt.zcs_bytes); 4061 if (error == ENOENT) { 4062 /* Generate a new salt for subsequent use */ 4063 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 4064 sizeof (spa->spa_cksum_salt.zcs_bytes)); 4065 } else if (error != 0) { 4066 spa_load_failed(spa, "unable to retrieve checksum salt from " 4067 "MOS [error=%d]", error); 4068 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4069 } 4070 4071 if (spa_dir_prop(spa, DMU_POOL_SYNC_BPOBJ, &obj, B_TRUE) != 0) 4072 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4073 error = bpobj_open(&spa->spa_deferred_bpobj, spa->spa_meta_objset, obj); 4074 if (error != 0) { 4075 spa_load_failed(spa, "error opening deferred-frees bpobj " 4076 "[error=%d]", error); 4077 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4078 } 4079 4080 /* 4081 * Load the bit that tells us to use the new accounting function 4082 * (raid-z deflation). If we have an older pool, this will not 4083 * be present. 4084 */ 4085 error = spa_dir_prop(spa, DMU_POOL_DEFLATE, &spa->spa_deflate, B_FALSE); 4086 if (error != 0 && error != ENOENT) 4087 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4088 4089 error = spa_dir_prop(spa, DMU_POOL_CREATION_VERSION, 4090 &spa->spa_creation_version, B_FALSE); 4091 if (error != 0 && error != ENOENT) 4092 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4093 4094 /* 4095 * Load the persistent error log. If we have an older pool, this will 4096 * not be present. 4097 */ 4098 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_LAST, &spa->spa_errlog_last, 4099 B_FALSE); 4100 if (error != 0 && error != ENOENT) 4101 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4102 4103 error = spa_dir_prop(spa, DMU_POOL_ERRLOG_SCRUB, 4104 &spa->spa_errlog_scrub, B_FALSE); 4105 if (error != 0 && error != ENOENT) 4106 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4107 4108 /* 4109 * Load the livelist deletion field. If a livelist is queued for 4110 * deletion, indicate that in the spa 4111 */ 4112 error = spa_dir_prop(spa, DMU_POOL_DELETED_CLONES, 4113 &spa->spa_livelists_to_delete, B_FALSE); 4114 if (error != 0 && error != ENOENT) 4115 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4116 4117 /* 4118 * Load the history object. If we have an older pool, this 4119 * will not be present. 4120 */ 4121 error = spa_dir_prop(spa, DMU_POOL_HISTORY, &spa->spa_history, B_FALSE); 4122 if (error != 0 && error != ENOENT) 4123 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4124 4125 /* 4126 * Load the per-vdev ZAP map. If we have an older pool, this will not 4127 * be present; in this case, defer its creation to a later time to 4128 * avoid dirtying the MOS this early / out of sync context. See 4129 * spa_sync_config_object. 4130 */ 4131 4132 /* The sentinel is only available in the MOS config. */ 4133 nvlist_t *mos_config; 4134 if (load_nvlist(spa, spa->spa_config_object, &mos_config) != 0) { 4135 spa_load_failed(spa, "unable to retrieve MOS config"); 4136 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4137 } 4138 4139 error = spa_dir_prop(spa, DMU_POOL_VDEV_ZAP_MAP, 4140 &spa->spa_all_vdev_zaps, B_FALSE); 4141 4142 if (error == ENOENT) { 4143 VERIFY(!nvlist_exists(mos_config, 4144 ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 4145 spa->spa_avz_action = AVZ_ACTION_INITIALIZE; 4146 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4147 } else if (error != 0) { 4148 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4149 } else if (!nvlist_exists(mos_config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)) { 4150 /* 4151 * An older version of ZFS overwrote the sentinel value, so 4152 * we have orphaned per-vdev ZAPs in the MOS. Defer their 4153 * destruction to later; see spa_sync_config_object. 4154 */ 4155 spa->spa_avz_action = AVZ_ACTION_DESTROY; 4156 /* 4157 * We're assuming that no vdevs have had their ZAPs created 4158 * before this. Better be sure of it. 4159 */ 4160 ASSERT0(vdev_count_verify_zaps(spa->spa_root_vdev)); 4161 } 4162 nvlist_free(mos_config); 4163 4164 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 4165 4166 error = spa_dir_prop(spa, DMU_POOL_PROPS, &spa->spa_pool_props_object, 4167 B_FALSE); 4168 if (error && error != ENOENT) 4169 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4170 4171 if (error == 0) { 4172 uint64_t autoreplace; 4173 4174 spa_prop_find(spa, ZPOOL_PROP_BOOTFS, &spa->spa_bootfs); 4175 spa_prop_find(spa, ZPOOL_PROP_AUTOREPLACE, &autoreplace); 4176 spa_prop_find(spa, ZPOOL_PROP_DELEGATION, &spa->spa_delegation); 4177 spa_prop_find(spa, ZPOOL_PROP_FAILUREMODE, &spa->spa_failmode); 4178 spa_prop_find(spa, ZPOOL_PROP_AUTOEXPAND, &spa->spa_autoexpand); 4179 spa_prop_find(spa, ZPOOL_PROP_MULTIHOST, &spa->spa_multihost); 4180 spa_prop_find(spa, ZPOOL_PROP_AUTOTRIM, &spa->spa_autotrim); 4181 spa->spa_autoreplace = (autoreplace != 0); 4182 } 4183 4184 /* 4185 * If we are importing a pool with missing top-level vdevs, 4186 * we enforce that the pool doesn't panic or get suspended on 4187 * error since the likelihood of missing data is extremely high. 4188 */ 4189 if (spa->spa_missing_tvds > 0 && 4190 spa->spa_failmode != ZIO_FAILURE_MODE_CONTINUE && 4191 spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4192 spa_load_note(spa, "forcing failmode to 'continue' " 4193 "as some top level vdevs are missing"); 4194 spa->spa_failmode = ZIO_FAILURE_MODE_CONTINUE; 4195 } 4196 4197 return (0); 4198} 4199 4200static int 4201spa_ld_open_aux_vdevs(spa_t *spa, spa_import_type_t type) 4202{ 4203 int error = 0; 4204 vdev_t *rvd = spa->spa_root_vdev; 4205 4206 /* 4207 * If we're assembling the pool from the split-off vdevs of 4208 * an existing pool, we don't want to attach the spares & cache 4209 * devices. 4210 */ 4211 4212 /* 4213 * Load any hot spares for this pool. 4214 */ 4215 error = spa_dir_prop(spa, DMU_POOL_SPARES, &spa->spa_spares.sav_object, 4216 B_FALSE); 4217 if (error != 0 && error != ENOENT) 4218 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4219 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4220 ASSERT(spa_version(spa) >= SPA_VERSION_SPARES); 4221 if (load_nvlist(spa, spa->spa_spares.sav_object, 4222 &spa->spa_spares.sav_config) != 0) { 4223 spa_load_failed(spa, "error loading spares nvlist"); 4224 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4225 } 4226 4227 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4228 spa_load_spares(spa); 4229 spa_config_exit(spa, SCL_ALL, FTAG); 4230 } else if (error == 0) { 4231 spa->spa_spares.sav_sync = B_TRUE; 4232 } 4233 4234 /* 4235 * Load any level 2 ARC devices for this pool. 4236 */ 4237 error = spa_dir_prop(spa, DMU_POOL_L2CACHE, 4238 &spa->spa_l2cache.sav_object, B_FALSE); 4239 if (error != 0 && error != ENOENT) 4240 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4241 if (error == 0 && type != SPA_IMPORT_ASSEMBLE) { 4242 ASSERT(spa_version(spa) >= SPA_VERSION_L2CACHE); 4243 if (load_nvlist(spa, spa->spa_l2cache.sav_object, 4244 &spa->spa_l2cache.sav_config) != 0) { 4245 spa_load_failed(spa, "error loading l2cache nvlist"); 4246 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4247 } 4248 4249 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4250 spa_load_l2cache(spa); 4251 spa_config_exit(spa, SCL_ALL, FTAG); 4252 } else if (error == 0) { 4253 spa->spa_l2cache.sav_sync = B_TRUE; 4254 } 4255 4256 return (0); 4257} 4258 4259static int 4260spa_ld_load_vdev_metadata(spa_t *spa) 4261{ 4262 int error = 0; 4263 vdev_t *rvd = spa->spa_root_vdev; 4264 4265 /* 4266 * If the 'multihost' property is set, then never allow a pool to 4267 * be imported when the system hostid is zero. The exception to 4268 * this rule is zdb which is always allowed to access pools. 4269 */ 4270 if (spa_multihost(spa) && spa_get_hostid(spa) == 0 && 4271 (spa->spa_import_flags & ZFS_IMPORT_SKIP_MMP) == 0) { 4272 fnvlist_add_uint64(spa->spa_load_info, 4273 ZPOOL_CONFIG_MMP_STATE, MMP_STATE_NO_HOSTID); 4274 return (spa_vdev_err(rvd, VDEV_AUX_ACTIVE, EREMOTEIO)); 4275 } 4276 4277 /* 4278 * If the 'autoreplace' property is set, then post a resource notifying 4279 * the ZFS DE that it should not issue any faults for unopenable 4280 * devices. We also iterate over the vdevs, and post a sysevent for any 4281 * unopenable vdevs so that the normal autoreplace handler can take 4282 * over. 4283 */ 4284 if (spa->spa_autoreplace && spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4285 spa_check_removed(spa->spa_root_vdev); 4286 /* 4287 * For the import case, this is done in spa_import(), because 4288 * at this point we're using the spare definitions from 4289 * the MOS config, not necessarily from the userland config. 4290 */ 4291 if (spa->spa_load_state != SPA_LOAD_IMPORT) { 4292 spa_aux_check_removed(&spa->spa_spares); 4293 spa_aux_check_removed(&spa->spa_l2cache); 4294 } 4295 } 4296 4297 /* 4298 * Load the vdev metadata such as metaslabs, DTLs, spacemap object, etc. 4299 */ 4300 error = vdev_load(rvd); 4301 if (error != 0) { 4302 spa_load_failed(spa, "vdev_load failed [error=%d]", error); 4303 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4304 } 4305 4306 error = spa_ld_log_spacemaps(spa); 4307 if (error != 0) { 4308 spa_load_failed(spa, "spa_ld_log_sm_data failed [error=%d]", 4309 error); 4310 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, error)); 4311 } 4312 4313 /* 4314 * Propagate the leaf DTLs we just loaded all the way up the vdev tree. 4315 */ 4316 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4317 vdev_dtl_reassess(rvd, 0, 0, B_FALSE, B_FALSE); 4318 spa_config_exit(spa, SCL_ALL, FTAG); 4319 4320 return (0); 4321} 4322 4323static int 4324spa_ld_load_dedup_tables(spa_t *spa) 4325{ 4326 int error = 0; 4327 vdev_t *rvd = spa->spa_root_vdev; 4328 4329 error = ddt_load(spa); 4330 if (error != 0) { 4331 spa_load_failed(spa, "ddt_load failed [error=%d]", error); 4332 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO)); 4333 } 4334 4335 return (0); 4336} 4337 4338static int 4339spa_ld_verify_logs(spa_t *spa, spa_import_type_t type, char **ereport) 4340{ 4341 vdev_t *rvd = spa->spa_root_vdev; 4342 4343 if (type != SPA_IMPORT_ASSEMBLE && spa_writeable(spa)) { 4344 boolean_t missing = spa_check_logs(spa); 4345 if (missing) { 4346 if (spa->spa_missing_tvds != 0) { 4347 spa_load_note(spa, "spa_check_logs failed " 4348 "so dropping the logs"); 4349 } else { 4350 *ereport = FM_EREPORT_ZFS_LOG_REPLAY; 4351 spa_load_failed(spa, "spa_check_logs failed"); 4352 return (spa_vdev_err(rvd, VDEV_AUX_BAD_LOG, 4353 ENXIO)); 4354 } 4355 } 4356 } 4357 4358 return (0); 4359} 4360 4361static int 4362spa_ld_verify_pool_data(spa_t *spa) 4363{ 4364 int error = 0; 4365 vdev_t *rvd = spa->spa_root_vdev; 4366 4367 /* 4368 * We've successfully opened the pool, verify that we're ready 4369 * to start pushing transactions. 4370 */ 4371 if (spa->spa_load_state != SPA_LOAD_TRYIMPORT) { 4372 error = spa_load_verify(spa); 4373 if (error != 0) { 4374 spa_load_failed(spa, "spa_load_verify failed " 4375 "[error=%d]", error); 4376 return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, 4377 error)); 4378 } 4379 } 4380 4381 return (0); 4382} 4383 4384static void 4385spa_ld_claim_log_blocks(spa_t *spa) 4386{ 4387 dmu_tx_t *tx; 4388 dsl_pool_t *dp = spa_get_dsl(spa); 4389 4390 /* 4391 * Claim log blocks that haven't been committed yet. 4392 * This must all happen in a single txg. 4393 * Note: spa_claim_max_txg is updated by spa_claim_notify(), 4394 * invoked from zil_claim_log_block()'s i/o done callback. 4395 * Price of rollback is that we abandon the log. 4396 */ 4397 spa->spa_claiming = B_TRUE; 4398 4399 tx = dmu_tx_create_assigned(dp, spa_first_txg(spa)); 4400 (void) dmu_objset_find_dp(dp, dp->dp_root_dir_obj, 4401 zil_claim, tx, DS_FIND_CHILDREN); 4402 dmu_tx_commit(tx); 4403 4404 spa->spa_claiming = B_FALSE; 4405 4406 spa_set_log_state(spa, SPA_LOG_GOOD); 4407} 4408 4409static void 4410spa_ld_check_for_config_update(spa_t *spa, uint64_t config_cache_txg, 4411 boolean_t update_config_cache) 4412{ 4413 vdev_t *rvd = spa->spa_root_vdev; 4414 int need_update = B_FALSE; 4415 4416 /* 4417 * If the config cache is stale, or we have uninitialized 4418 * metaslabs (see spa_vdev_add()), then update the config. 4419 * 4420 * If this is a verbatim import, trust the current 4421 * in-core spa_config and update the disk labels. 4422 */ 4423 if (update_config_cache || config_cache_txg != spa->spa_config_txg || 4424 spa->spa_load_state == SPA_LOAD_IMPORT || 4425 spa->spa_load_state == SPA_LOAD_RECOVER || 4426 (spa->spa_import_flags & ZFS_IMPORT_VERBATIM)) 4427 need_update = B_TRUE; 4428 4429 for (int c = 0; c < rvd->vdev_children; c++) 4430 if (rvd->vdev_child[c]->vdev_ms_array == 0) 4431 need_update = B_TRUE; 4432 4433 /* 4434 * Update the config cache asynchronously in case we're the 4435 * root pool, in which case the config cache isn't writable yet. 4436 */ 4437 if (need_update) 4438 spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE); 4439} 4440 4441static void 4442spa_ld_prepare_for_reload(spa_t *spa) 4443{ 4444 spa_mode_t mode = spa->spa_mode; 4445 int async_suspended = spa->spa_async_suspended; 4446 4447 spa_unload(spa); 4448 spa_deactivate(spa); 4449 spa_activate(spa, mode); 4450 4451 /* 4452 * We save the value of spa_async_suspended as it gets reset to 0 by 4453 * spa_unload(). We want to restore it back to the original value before 4454 * returning as we might be calling spa_async_resume() later. 4455 */ 4456 spa->spa_async_suspended = async_suspended; 4457} 4458 4459static int 4460spa_ld_read_checkpoint_txg(spa_t *spa) 4461{ 4462 uberblock_t checkpoint; 4463 int error = 0; 4464 4465 ASSERT0(spa->spa_checkpoint_txg); 4466 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4467 4468 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4469 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4470 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4471 4472 if (error == ENOENT) 4473 return (0); 4474 4475 if (error != 0) 4476 return (error); 4477 4478 ASSERT3U(checkpoint.ub_txg, !=, 0); 4479 ASSERT3U(checkpoint.ub_checkpoint_txg, !=, 0); 4480 ASSERT3U(checkpoint.ub_timestamp, !=, 0); 4481 spa->spa_checkpoint_txg = checkpoint.ub_txg; 4482 spa->spa_checkpoint_info.sci_timestamp = checkpoint.ub_timestamp; 4483 4484 return (0); 4485} 4486 4487static int 4488spa_ld_mos_init(spa_t *spa, spa_import_type_t type) 4489{ 4490 int error = 0; 4491 4492 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4493 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4494 4495 /* 4496 * Never trust the config that is provided unless we are assembling 4497 * a pool following a split. 4498 * This means don't trust blkptrs and the vdev tree in general. This 4499 * also effectively puts the spa in read-only mode since 4500 * spa_writeable() checks for spa_trust_config to be true. 4501 * We will later load a trusted config from the MOS. 4502 */ 4503 if (type != SPA_IMPORT_ASSEMBLE) 4504 spa->spa_trust_config = B_FALSE; 4505 4506 /* 4507 * Parse the config provided to create a vdev tree. 4508 */ 4509 error = spa_ld_parse_config(spa, type); 4510 if (error != 0) 4511 return (error); 4512 4513 spa_import_progress_add(spa); 4514 4515 /* 4516 * Now that we have the vdev tree, try to open each vdev. This involves 4517 * opening the underlying physical device, retrieving its geometry and 4518 * probing the vdev with a dummy I/O. The state of each vdev will be set 4519 * based on the success of those operations. After this we'll be ready 4520 * to read from the vdevs. 4521 */ 4522 error = spa_ld_open_vdevs(spa); 4523 if (error != 0) 4524 return (error); 4525 4526 /* 4527 * Read the label of each vdev and make sure that the GUIDs stored 4528 * there match the GUIDs in the config provided. 4529 * If we're assembling a new pool that's been split off from an 4530 * existing pool, the labels haven't yet been updated so we skip 4531 * validation for now. 4532 */ 4533 if (type != SPA_IMPORT_ASSEMBLE) { 4534 error = spa_ld_validate_vdevs(spa); 4535 if (error != 0) 4536 return (error); 4537 } 4538 4539 /* 4540 * Read all vdev labels to find the best uberblock (i.e. latest, 4541 * unless spa_load_max_txg is set) and store it in spa_uberblock. We 4542 * get the list of features required to read blkptrs in the MOS from 4543 * the vdev label with the best uberblock and verify that our version 4544 * of zfs supports them all. 4545 */ 4546 error = spa_ld_select_uberblock(spa, type); 4547 if (error != 0) 4548 return (error); 4549 4550 /* 4551 * Pass that uberblock to the dsl_pool layer which will open the root 4552 * blkptr. This blkptr points to the latest version of the MOS and will 4553 * allow us to read its contents. 4554 */ 4555 error = spa_ld_open_rootbp(spa); 4556 if (error != 0) 4557 return (error); 4558 4559 return (0); 4560} 4561 4562static int 4563spa_ld_checkpoint_rewind(spa_t *spa) 4564{ 4565 uberblock_t checkpoint; 4566 int error = 0; 4567 4568 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4569 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4570 4571 error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 4572 DMU_POOL_ZPOOL_CHECKPOINT, sizeof (uint64_t), 4573 sizeof (uberblock_t) / sizeof (uint64_t), &checkpoint); 4574 4575 if (error != 0) { 4576 spa_load_failed(spa, "unable to retrieve checkpointed " 4577 "uberblock from the MOS config [error=%d]", error); 4578 4579 if (error == ENOENT) 4580 error = ZFS_ERR_NO_CHECKPOINT; 4581 4582 return (error); 4583 } 4584 4585 ASSERT3U(checkpoint.ub_txg, <, spa->spa_uberblock.ub_txg); 4586 ASSERT3U(checkpoint.ub_txg, ==, checkpoint.ub_checkpoint_txg); 4587 4588 /* 4589 * We need to update the txg and timestamp of the checkpointed 4590 * uberblock to be higher than the latest one. This ensures that 4591 * the checkpointed uberblock is selected if we were to close and 4592 * reopen the pool right after we've written it in the vdev labels. 4593 * (also see block comment in vdev_uberblock_compare) 4594 */ 4595 checkpoint.ub_txg = spa->spa_uberblock.ub_txg + 1; 4596 checkpoint.ub_timestamp = gethrestime_sec(); 4597 4598 /* 4599 * Set current uberblock to be the checkpointed uberblock. 4600 */ 4601 spa->spa_uberblock = checkpoint; 4602 4603 /* 4604 * If we are doing a normal rewind, then the pool is open for 4605 * writing and we sync the "updated" checkpointed uberblock to 4606 * disk. Once this is done, we've basically rewound the whole 4607 * pool and there is no way back. 4608 * 4609 * There are cases when we don't want to attempt and sync the 4610 * checkpointed uberblock to disk because we are opening a 4611 * pool as read-only. Specifically, verifying the checkpointed 4612 * state with zdb, and importing the checkpointed state to get 4613 * a "preview" of its content. 4614 */ 4615 if (spa_writeable(spa)) { 4616 vdev_t *rvd = spa->spa_root_vdev; 4617 4618 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 4619 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 4620 int svdcount = 0; 4621 int children = rvd->vdev_children; 4622 int c0 = spa_get_random(children); 4623 4624 for (int c = 0; c < children; c++) { 4625 vdev_t *vd = rvd->vdev_child[(c0 + c) % children]; 4626 4627 /* Stop when revisiting the first vdev */ 4628 if (c > 0 && svd[0] == vd) 4629 break; 4630 4631 if (vd->vdev_ms_array == 0 || vd->vdev_islog || 4632 !vdev_is_concrete(vd)) 4633 continue; 4634 4635 svd[svdcount++] = vd; 4636 if (svdcount == SPA_SYNC_MIN_VDEVS) 4637 break; 4638 } 4639 error = vdev_config_sync(svd, svdcount, spa->spa_first_txg); 4640 if (error == 0) 4641 spa->spa_last_synced_guid = rvd->vdev_guid; 4642 spa_config_exit(spa, SCL_ALL, FTAG); 4643 4644 if (error != 0) { 4645 spa_load_failed(spa, "failed to write checkpointed " 4646 "uberblock to the vdev labels [error=%d]", error); 4647 return (error); 4648 } 4649 } 4650 4651 return (0); 4652} 4653 4654static int 4655spa_ld_mos_with_trusted_config(spa_t *spa, spa_import_type_t type, 4656 boolean_t *update_config_cache) 4657{ 4658 int error; 4659 4660 /* 4661 * Parse the config for pool, open and validate vdevs, 4662 * select an uberblock, and use that uberblock to open 4663 * the MOS. 4664 */ 4665 error = spa_ld_mos_init(spa, type); 4666 if (error != 0) 4667 return (error); 4668 4669 /* 4670 * Retrieve the trusted config stored in the MOS and use it to create 4671 * a new, exact version of the vdev tree, then reopen all vdevs. 4672 */ 4673 error = spa_ld_trusted_config(spa, type, B_FALSE); 4674 if (error == EAGAIN) { 4675 if (update_config_cache != NULL) 4676 *update_config_cache = B_TRUE; 4677 4678 /* 4679 * Redo the loading process with the trusted config if it is 4680 * too different from the untrusted config. 4681 */ 4682 spa_ld_prepare_for_reload(spa); 4683 spa_load_note(spa, "RELOADING"); 4684 error = spa_ld_mos_init(spa, type); 4685 if (error != 0) 4686 return (error); 4687 4688 error = spa_ld_trusted_config(spa, type, B_TRUE); 4689 if (error != 0) 4690 return (error); 4691 4692 } else if (error != 0) { 4693 return (error); 4694 } 4695 4696 return (0); 4697} 4698 4699/* 4700 * Load an existing storage pool, using the config provided. This config 4701 * describes which vdevs are part of the pool and is later validated against 4702 * partial configs present in each vdev's label and an entire copy of the 4703 * config stored in the MOS. 4704 */ 4705static int 4706spa_load_impl(spa_t *spa, spa_import_type_t type, char **ereport) 4707{ 4708 int error = 0; 4709 boolean_t missing_feat_write = B_FALSE; 4710 boolean_t checkpoint_rewind = 4711 (spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 4712 boolean_t update_config_cache = B_FALSE; 4713 4714 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 4715 ASSERT(spa->spa_config_source != SPA_CONFIG_SRC_NONE); 4716 4717 spa_load_note(spa, "LOADING"); 4718 4719 error = spa_ld_mos_with_trusted_config(spa, type, &update_config_cache); 4720 if (error != 0) 4721 return (error); 4722 4723 /* 4724 * If we are rewinding to the checkpoint then we need to repeat 4725 * everything we've done so far in this function but this time 4726 * selecting the checkpointed uberblock and using that to open 4727 * the MOS. 4728 */ 4729 if (checkpoint_rewind) { 4730 /* 4731 * If we are rewinding to the checkpoint update config cache 4732 * anyway. 4733 */ 4734 update_config_cache = B_TRUE; 4735 4736 /* 4737 * Extract the checkpointed uberblock from the current MOS 4738 * and use this as the pool's uberblock from now on. If the 4739 * pool is imported as writeable we also write the checkpoint 4740 * uberblock to the labels, making the rewind permanent. 4741 */ 4742 error = spa_ld_checkpoint_rewind(spa); 4743 if (error != 0) 4744 return (error); 4745 4746 /* 4747 * Redo the loading process again with the 4748 * checkpointed uberblock. 4749 */ 4750 spa_ld_prepare_for_reload(spa); 4751 spa_load_note(spa, "LOADING checkpointed uberblock"); 4752 error = spa_ld_mos_with_trusted_config(spa, type, NULL); 4753 if (error != 0) 4754 return (error); 4755 } 4756 4757 /* 4758 * Retrieve the checkpoint txg if the pool has a checkpoint. 4759 */ 4760 error = spa_ld_read_checkpoint_txg(spa); 4761 if (error != 0) 4762 return (error); 4763 4764 /* 4765 * Retrieve the mapping of indirect vdevs. Those vdevs were removed 4766 * from the pool and their contents were re-mapped to other vdevs. Note 4767 * that everything that we read before this step must have been 4768 * rewritten on concrete vdevs after the last device removal was 4769 * initiated. Otherwise we could be reading from indirect vdevs before 4770 * we have loaded their mappings. 4771 */ 4772 error = spa_ld_open_indirect_vdev_metadata(spa); 4773 if (error != 0) 4774 return (error); 4775 4776 /* 4777 * Retrieve the full list of active features from the MOS and check if 4778 * they are all supported. 4779 */ 4780 error = spa_ld_check_features(spa, &missing_feat_write); 4781 if (error != 0) 4782 return (error); 4783 4784 /* 4785 * Load several special directories from the MOS needed by the dsl_pool 4786 * layer. 4787 */ 4788 error = spa_ld_load_special_directories(spa); 4789 if (error != 0) 4790 return (error); 4791 4792 /* 4793 * Retrieve pool properties from the MOS. 4794 */ 4795 error = spa_ld_get_props(spa); 4796 if (error != 0) 4797 return (error); 4798 4799 /* 4800 * Retrieve the list of auxiliary devices - cache devices and spares - 4801 * and open them. 4802 */ 4803 error = spa_ld_open_aux_vdevs(spa, type); 4804 if (error != 0) 4805 return (error); 4806 4807 /* 4808 * Load the metadata for all vdevs. Also check if unopenable devices 4809 * should be autoreplaced. 4810 */ 4811 error = spa_ld_load_vdev_metadata(spa); 4812 if (error != 0) 4813 return (error); 4814 4815 error = spa_ld_load_dedup_tables(spa); 4816 if (error != 0) 4817 return (error); 4818 4819 /* 4820 * Verify the logs now to make sure we don't have any unexpected errors 4821 * when we claim log blocks later. 4822 */ 4823 error = spa_ld_verify_logs(spa, type, ereport); 4824 if (error != 0) 4825 return (error); 4826 4827 if (missing_feat_write) { 4828 ASSERT(spa->spa_load_state == SPA_LOAD_TRYIMPORT); 4829 4830 /* 4831 * At this point, we know that we can open the pool in 4832 * read-only mode but not read-write mode. We now have enough 4833 * information and can return to userland. 4834 */ 4835 return (spa_vdev_err(spa->spa_root_vdev, VDEV_AUX_UNSUP_FEAT, 4836 ENOTSUP)); 4837 } 4838 4839 /* 4840 * Traverse the last txgs to make sure the pool was left off in a safe 4841 * state. When performing an extreme rewind, we verify the whole pool, 4842 * which can take a very long time. 4843 */ 4844 error = spa_ld_verify_pool_data(spa); 4845 if (error != 0) 4846 return (error); 4847 4848 /* 4849 * Calculate the deflated space for the pool. This must be done before 4850 * we write anything to the pool because we'd need to update the space 4851 * accounting using the deflated sizes. 4852 */ 4853 spa_update_dspace(spa); 4854 4855 /* 4856 * We have now retrieved all the information we needed to open the 4857 * pool. If we are importing the pool in read-write mode, a few 4858 * additional steps must be performed to finish the import. 4859 */ 4860 if (spa_writeable(spa) && (spa->spa_load_state == SPA_LOAD_RECOVER || 4861 spa->spa_load_max_txg == UINT64_MAX)) { 4862 uint64_t config_cache_txg = spa->spa_config_txg; 4863 4864 ASSERT(spa->spa_load_state != SPA_LOAD_TRYIMPORT); 4865 4866 /* 4867 * In case of a checkpoint rewind, log the original txg 4868 * of the checkpointed uberblock. 4869 */ 4870 if (checkpoint_rewind) { 4871 spa_history_log_internal(spa, "checkpoint rewind", 4872 NULL, "rewound state to txg=%llu", 4873 (u_longlong_t)spa->spa_uberblock.ub_checkpoint_txg); 4874 } 4875 4876 /* 4877 * Traverse the ZIL and claim all blocks. 4878 */ 4879 spa_ld_claim_log_blocks(spa); 4880 4881 /* 4882 * Kick-off the syncing thread. 4883 */ 4884 spa->spa_sync_on = B_TRUE; 4885 txg_sync_start(spa->spa_dsl_pool); 4886 mmp_thread_start(spa); 4887 4888 /* 4889 * Wait for all claims to sync. We sync up to the highest 4890 * claimed log block birth time so that claimed log blocks 4891 * don't appear to be from the future. spa_claim_max_txg 4892 * will have been set for us by ZIL traversal operations 4893 * performed above. 4894 */ 4895 txg_wait_synced(spa->spa_dsl_pool, spa->spa_claim_max_txg); 4896 4897 /* 4898 * Check if we need to request an update of the config. On the 4899 * next sync, we would update the config stored in vdev labels 4900 * and the cachefile (by default /etc/zfs/zpool.cache). 4901 */ 4902 spa_ld_check_for_config_update(spa, config_cache_txg, 4903 update_config_cache); 4904 4905 /* 4906 * Check if a rebuild was in progress and if so resume it. 4907 * Then check all DTLs to see if anything needs resilvering. 4908 * The resilver will be deferred if a rebuild was started. 4909 */ 4910 if (vdev_rebuild_active(spa->spa_root_vdev)) { 4911 vdev_rebuild_restart(spa); 4912 } else if (!dsl_scan_resilvering(spa->spa_dsl_pool) && 4913 vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 4914 spa_async_request(spa, SPA_ASYNC_RESILVER); 4915 } 4916 4917 /* 4918 * Log the fact that we booted up (so that we can detect if 4919 * we rebooted in the middle of an operation). 4920 */ 4921 spa_history_log_version(spa, "open", NULL); 4922 4923 spa_restart_removal(spa); 4924 spa_spawn_aux_threads(spa); 4925 4926 /* 4927 * Delete any inconsistent datasets. 4928 * 4929 * Note: 4930 * Since we may be issuing deletes for clones here, 4931 * we make sure to do so after we've spawned all the 4932 * auxiliary threads above (from which the livelist 4933 * deletion zthr is part of). 4934 */ 4935 (void) dmu_objset_find(spa_name(spa), 4936 dsl_destroy_inconsistent, NULL, DS_FIND_CHILDREN); 4937 4938 /* 4939 * Clean up any stale temporary dataset userrefs. 4940 */ 4941 dsl_pool_clean_tmp_userrefs(spa->spa_dsl_pool); 4942 4943 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 4944 vdev_initialize_restart(spa->spa_root_vdev); 4945 vdev_trim_restart(spa->spa_root_vdev); 4946 vdev_autotrim_restart(spa); 4947 spa_config_exit(spa, SCL_CONFIG, FTAG); 4948 } 4949 4950 spa_import_progress_remove(spa_guid(spa)); 4951 spa_async_request(spa, SPA_ASYNC_L2CACHE_REBUILD); 4952 4953 spa_load_note(spa, "LOADED"); 4954 4955 return (0); 4956} 4957 4958static int 4959spa_load_retry(spa_t *spa, spa_load_state_t state) 4960{ 4961 spa_mode_t mode = spa->spa_mode; 4962 4963 spa_unload(spa); 4964 spa_deactivate(spa); 4965 4966 spa->spa_load_max_txg = spa->spa_uberblock.ub_txg - 1; 4967 4968 spa_activate(spa, mode); 4969 spa_async_suspend(spa); 4970 4971 spa_load_note(spa, "spa_load_retry: rewind, max txg: %llu", 4972 (u_longlong_t)spa->spa_load_max_txg); 4973 4974 return (spa_load(spa, state, SPA_IMPORT_EXISTING)); 4975} 4976 4977/* 4978 * If spa_load() fails this function will try loading prior txg's. If 4979 * 'state' is SPA_LOAD_RECOVER and one of these loads succeeds the pool 4980 * will be rewound to that txg. If 'state' is not SPA_LOAD_RECOVER this 4981 * function will not rewind the pool and will return the same error as 4982 * spa_load(). 4983 */ 4984static int 4985spa_load_best(spa_t *spa, spa_load_state_t state, uint64_t max_request, 4986 int rewind_flags) 4987{ 4988 nvlist_t *loadinfo = NULL; 4989 nvlist_t *config = NULL; 4990 int load_error, rewind_error; 4991 uint64_t safe_rewind_txg; 4992 uint64_t min_txg; 4993 4994 if (spa->spa_load_txg && state == SPA_LOAD_RECOVER) { 4995 spa->spa_load_max_txg = spa->spa_load_txg; 4996 spa_set_log_state(spa, SPA_LOG_CLEAR); 4997 } else { 4998 spa->spa_load_max_txg = max_request; 4999 if (max_request != UINT64_MAX) 5000 spa->spa_extreme_rewind = B_TRUE; 5001 } 5002 5003 load_error = rewind_error = spa_load(spa, state, SPA_IMPORT_EXISTING); 5004 if (load_error == 0) 5005 return (0); 5006 if (load_error == ZFS_ERR_NO_CHECKPOINT) { 5007 /* 5008 * When attempting checkpoint-rewind on a pool with no 5009 * checkpoint, we should not attempt to load uberblocks 5010 * from previous txgs when spa_load fails. 5011 */ 5012 ASSERT(spa->spa_import_flags & ZFS_IMPORT_CHECKPOINT); 5013 spa_import_progress_remove(spa_guid(spa)); 5014 return (load_error); 5015 } 5016 5017 if (spa->spa_root_vdev != NULL) 5018 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5019 5020 spa->spa_last_ubsync_txg = spa->spa_uberblock.ub_txg; 5021 spa->spa_last_ubsync_txg_ts = spa->spa_uberblock.ub_timestamp; 5022 5023 if (rewind_flags & ZPOOL_NEVER_REWIND) { 5024 nvlist_free(config); 5025 spa_import_progress_remove(spa_guid(spa)); 5026 return (load_error); 5027 } 5028 5029 if (state == SPA_LOAD_RECOVER) { 5030 /* Price of rolling back is discarding txgs, including log */ 5031 spa_set_log_state(spa, SPA_LOG_CLEAR); 5032 } else { 5033 /* 5034 * If we aren't rolling back save the load info from our first 5035 * import attempt so that we can restore it after attempting 5036 * to rewind. 5037 */ 5038 loadinfo = spa->spa_load_info; 5039 spa->spa_load_info = fnvlist_alloc(); 5040 } 5041 5042 spa->spa_load_max_txg = spa->spa_last_ubsync_txg; 5043 safe_rewind_txg = spa->spa_last_ubsync_txg - TXG_DEFER_SIZE; 5044 min_txg = (rewind_flags & ZPOOL_EXTREME_REWIND) ? 5045 TXG_INITIAL : safe_rewind_txg; 5046 5047 /* 5048 * Continue as long as we're finding errors, we're still within 5049 * the acceptable rewind range, and we're still finding uberblocks 5050 */ 5051 while (rewind_error && spa->spa_uberblock.ub_txg >= min_txg && 5052 spa->spa_uberblock.ub_txg <= spa->spa_load_max_txg) { 5053 if (spa->spa_load_max_txg < safe_rewind_txg) 5054 spa->spa_extreme_rewind = B_TRUE; 5055 rewind_error = spa_load_retry(spa, state); 5056 } 5057 5058 spa->spa_extreme_rewind = B_FALSE; 5059 spa->spa_load_max_txg = UINT64_MAX; 5060 5061 if (config && (rewind_error || state != SPA_LOAD_RECOVER)) 5062 spa_config_set(spa, config); 5063 else 5064 nvlist_free(config); 5065 5066 if (state == SPA_LOAD_RECOVER) { 5067 ASSERT3P(loadinfo, ==, NULL); 5068 spa_import_progress_remove(spa_guid(spa)); 5069 return (rewind_error); 5070 } else { 5071 /* Store the rewind info as part of the initial load info */ 5072 fnvlist_add_nvlist(loadinfo, ZPOOL_CONFIG_REWIND_INFO, 5073 spa->spa_load_info); 5074 5075 /* Restore the initial load info */ 5076 fnvlist_free(spa->spa_load_info); 5077 spa->spa_load_info = loadinfo; 5078 5079 spa_import_progress_remove(spa_guid(spa)); 5080 return (load_error); 5081 } 5082} 5083 5084/* 5085 * Pool Open/Import 5086 * 5087 * The import case is identical to an open except that the configuration is sent 5088 * down from userland, instead of grabbed from the configuration cache. For the 5089 * case of an open, the pool configuration will exist in the 5090 * POOL_STATE_UNINITIALIZED state. 5091 * 5092 * The stats information (gen/count/ustats) is used to gather vdev statistics at 5093 * the same time open the pool, without having to keep around the spa_t in some 5094 * ambiguous state. 5095 */ 5096static int 5097spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, 5098 nvlist_t **config) 5099{ 5100 spa_t *spa; 5101 spa_load_state_t state = SPA_LOAD_OPEN; 5102 int error; 5103 int locked = B_FALSE; 5104 int firstopen = B_FALSE; 5105 5106 *spapp = NULL; 5107 5108 /* 5109 * As disgusting as this is, we need to support recursive calls to this 5110 * function because dsl_dir_open() is called during spa_load(), and ends 5111 * up calling spa_open() again. The real fix is to figure out how to 5112 * avoid dsl_dir_open() calling this in the first place. 5113 */ 5114 if (MUTEX_NOT_HELD(&spa_namespace_lock)) { 5115 mutex_enter(&spa_namespace_lock); 5116 locked = B_TRUE; 5117 } 5118 5119 if ((spa = spa_lookup(pool)) == NULL) { 5120 if (locked) 5121 mutex_exit(&spa_namespace_lock); 5122 return (SET_ERROR(ENOENT)); 5123 } 5124 5125 if (spa->spa_state == POOL_STATE_UNINITIALIZED) { 5126 zpool_load_policy_t policy; 5127 5128 firstopen = B_TRUE; 5129 5130 zpool_get_load_policy(nvpolicy ? nvpolicy : spa->spa_config, 5131 &policy); 5132 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 5133 state = SPA_LOAD_RECOVER; 5134 5135 spa_activate(spa, spa_mode_global); 5136 5137 if (state != SPA_LOAD_RECOVER) 5138 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 5139 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 5140 5141 zfs_dbgmsg("spa_open_common: opening %s", pool); 5142 error = spa_load_best(spa, state, policy.zlp_txg, 5143 policy.zlp_rewind); 5144 5145 if (error == EBADF) { 5146 /* 5147 * If vdev_validate() returns failure (indicated by 5148 * EBADF), it indicates that one of the vdevs indicates 5149 * that the pool has been exported or destroyed. If 5150 * this is the case, the config cache is out of sync and 5151 * we should remove the pool from the namespace. 5152 */ 5153 spa_unload(spa); 5154 spa_deactivate(spa); 5155 spa_write_cachefile(spa, B_TRUE, B_TRUE); 5156 spa_remove(spa); 5157 if (locked) 5158 mutex_exit(&spa_namespace_lock); 5159 return (SET_ERROR(ENOENT)); 5160 } 5161 5162 if (error) { 5163 /* 5164 * We can't open the pool, but we still have useful 5165 * information: the state of each vdev after the 5166 * attempted vdev_open(). Return this to the user. 5167 */ 5168 if (config != NULL && spa->spa_config) { 5169 VERIFY(nvlist_dup(spa->spa_config, config, 5170 KM_SLEEP) == 0); 5171 VERIFY(nvlist_add_nvlist(*config, 5172 ZPOOL_CONFIG_LOAD_INFO, 5173 spa->spa_load_info) == 0); 5174 } 5175 spa_unload(spa); 5176 spa_deactivate(spa); 5177 spa->spa_last_open_failed = error; 5178 if (locked) 5179 mutex_exit(&spa_namespace_lock); 5180 *spapp = NULL; 5181 return (error); 5182 } 5183 } 5184 5185 spa_open_ref(spa, tag); 5186 5187 if (config != NULL) 5188 *config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 5189 5190 /* 5191 * If we've recovered the pool, pass back any information we 5192 * gathered while doing the load. 5193 */ 5194 if (state == SPA_LOAD_RECOVER) { 5195 VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, 5196 spa->spa_load_info) == 0); 5197 } 5198 5199 if (locked) { 5200 spa->spa_last_open_failed = 0; 5201 spa->spa_last_ubsync_txg = 0; 5202 spa->spa_load_txg = 0; 5203 mutex_exit(&spa_namespace_lock); 5204 } 5205 5206 if (firstopen) 5207 zvol_create_minors_recursive(spa_name(spa)); 5208 5209 *spapp = spa; 5210 5211 return (0); 5212} 5213 5214int 5215spa_open_rewind(const char *name, spa_t **spapp, void *tag, nvlist_t *policy, 5216 nvlist_t **config) 5217{ 5218 return (spa_open_common(name, spapp, tag, policy, config)); 5219} 5220 5221int 5222spa_open(const char *name, spa_t **spapp, void *tag) 5223{ 5224 return (spa_open_common(name, spapp, tag, NULL, NULL)); 5225} 5226 5227/* 5228 * Lookup the given spa_t, incrementing the inject count in the process, 5229 * preventing it from being exported or destroyed. 5230 */ 5231spa_t * 5232spa_inject_addref(char *name) 5233{ 5234 spa_t *spa; 5235 5236 mutex_enter(&spa_namespace_lock); 5237 if ((spa = spa_lookup(name)) == NULL) { 5238 mutex_exit(&spa_namespace_lock); 5239 return (NULL); 5240 } 5241 spa->spa_inject_ref++; 5242 mutex_exit(&spa_namespace_lock); 5243 5244 return (spa); 5245} 5246 5247void 5248spa_inject_delref(spa_t *spa) 5249{ 5250 mutex_enter(&spa_namespace_lock); 5251 spa->spa_inject_ref--; 5252 mutex_exit(&spa_namespace_lock); 5253} 5254 5255/* 5256 * Add spares device information to the nvlist. 5257 */ 5258static void 5259spa_add_spares(spa_t *spa, nvlist_t *config) 5260{ 5261 nvlist_t **spares; 5262 uint_t i, nspares; 5263 nvlist_t *nvroot; 5264 uint64_t guid; 5265 vdev_stat_t *vs; 5266 uint_t vsc; 5267 uint64_t pool; 5268 5269 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5270 5271 if (spa->spa_spares.sav_count == 0) 5272 return; 5273 5274 VERIFY(nvlist_lookup_nvlist(config, 5275 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5276 VERIFY(nvlist_lookup_nvlist_array(spa->spa_spares.sav_config, 5277 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5278 if (nspares != 0) { 5279 VERIFY(nvlist_add_nvlist_array(nvroot, 5280 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5281 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5282 ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0); 5283 5284 /* 5285 * Go through and find any spares which have since been 5286 * repurposed as an active spare. If this is the case, update 5287 * their status appropriately. 5288 */ 5289 for (i = 0; i < nspares; i++) { 5290 VERIFY(nvlist_lookup_uint64(spares[i], 5291 ZPOOL_CONFIG_GUID, &guid) == 0); 5292 if (spa_spare_exists(guid, &pool, NULL) && 5293 pool != 0ULL) { 5294 VERIFY(nvlist_lookup_uint64_array( 5295 spares[i], ZPOOL_CONFIG_VDEV_STATS, 5296 (uint64_t **)&vs, &vsc) == 0); 5297 vs->vs_state = VDEV_STATE_CANT_OPEN; 5298 vs->vs_aux = VDEV_AUX_SPARED; 5299 } 5300 } 5301 } 5302} 5303 5304/* 5305 * Add l2cache device information to the nvlist, including vdev stats. 5306 */ 5307static void 5308spa_add_l2cache(spa_t *spa, nvlist_t *config) 5309{ 5310 nvlist_t **l2cache; 5311 uint_t i, j, nl2cache; 5312 nvlist_t *nvroot; 5313 uint64_t guid; 5314 vdev_t *vd; 5315 vdev_stat_t *vs; 5316 uint_t vsc; 5317 5318 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5319 5320 if (spa->spa_l2cache.sav_count == 0) 5321 return; 5322 5323 VERIFY(nvlist_lookup_nvlist(config, 5324 ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); 5325 VERIFY(nvlist_lookup_nvlist_array(spa->spa_l2cache.sav_config, 5326 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5327 if (nl2cache != 0) { 5328 VERIFY(nvlist_add_nvlist_array(nvroot, 5329 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5330 VERIFY(nvlist_lookup_nvlist_array(nvroot, 5331 ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); 5332 5333 /* 5334 * Update level 2 cache device stats. 5335 */ 5336 5337 for (i = 0; i < nl2cache; i++) { 5338 VERIFY(nvlist_lookup_uint64(l2cache[i], 5339 ZPOOL_CONFIG_GUID, &guid) == 0); 5340 5341 vd = NULL; 5342 for (j = 0; j < spa->spa_l2cache.sav_count; j++) { 5343 if (guid == 5344 spa->spa_l2cache.sav_vdevs[j]->vdev_guid) { 5345 vd = spa->spa_l2cache.sav_vdevs[j]; 5346 break; 5347 } 5348 } 5349 ASSERT(vd != NULL); 5350 5351 VERIFY(nvlist_lookup_uint64_array(l2cache[i], 5352 ZPOOL_CONFIG_VDEV_STATS, (uint64_t **)&vs, &vsc) 5353 == 0); 5354 vdev_get_stats(vd, vs); 5355 vdev_config_generate_stats(vd, l2cache[i]); 5356 5357 } 5358 } 5359} 5360 5361static void 5362spa_feature_stats_from_disk(spa_t *spa, nvlist_t *features) 5363{ 5364 zap_cursor_t zc; 5365 zap_attribute_t za; 5366 5367 if (spa->spa_feat_for_read_obj != 0) { 5368 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5369 spa->spa_feat_for_read_obj); 5370 zap_cursor_retrieve(&zc, &za) == 0; 5371 zap_cursor_advance(&zc)) { 5372 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5373 za.za_num_integers == 1); 5374 VERIFY0(nvlist_add_uint64(features, za.za_name, 5375 za.za_first_integer)); 5376 } 5377 zap_cursor_fini(&zc); 5378 } 5379 5380 if (spa->spa_feat_for_write_obj != 0) { 5381 for (zap_cursor_init(&zc, spa->spa_meta_objset, 5382 spa->spa_feat_for_write_obj); 5383 zap_cursor_retrieve(&zc, &za) == 0; 5384 zap_cursor_advance(&zc)) { 5385 ASSERT(za.za_integer_length == sizeof (uint64_t) && 5386 za.za_num_integers == 1); 5387 VERIFY0(nvlist_add_uint64(features, za.za_name, 5388 za.za_first_integer)); 5389 } 5390 zap_cursor_fini(&zc); 5391 } 5392} 5393 5394static void 5395spa_feature_stats_from_cache(spa_t *spa, nvlist_t *features) 5396{ 5397 int i; 5398 5399 for (i = 0; i < SPA_FEATURES; i++) { 5400 zfeature_info_t feature = spa_feature_table[i]; 5401 uint64_t refcount; 5402 5403 if (feature_get_refcount(spa, &feature, &refcount) != 0) 5404 continue; 5405 5406 VERIFY0(nvlist_add_uint64(features, feature.fi_guid, refcount)); 5407 } 5408} 5409 5410/* 5411 * Store a list of pool features and their reference counts in the 5412 * config. 5413 * 5414 * The first time this is called on a spa, allocate a new nvlist, fetch 5415 * the pool features and reference counts from disk, then save the list 5416 * in the spa. In subsequent calls on the same spa use the saved nvlist 5417 * and refresh its values from the cached reference counts. This 5418 * ensures we don't block here on I/O on a suspended pool so 'zpool 5419 * clear' can resume the pool. 5420 */ 5421static void 5422spa_add_feature_stats(spa_t *spa, nvlist_t *config) 5423{ 5424 nvlist_t *features; 5425 5426 ASSERT(spa_config_held(spa, SCL_CONFIG, RW_READER)); 5427 5428 mutex_enter(&spa->spa_feat_stats_lock); 5429 features = spa->spa_feat_stats; 5430 5431 if (features != NULL) { 5432 spa_feature_stats_from_cache(spa, features); 5433 } else { 5434 VERIFY0(nvlist_alloc(&features, NV_UNIQUE_NAME, KM_SLEEP)); 5435 spa->spa_feat_stats = features; 5436 spa_feature_stats_from_disk(spa, features); 5437 } 5438 5439 VERIFY0(nvlist_add_nvlist(config, ZPOOL_CONFIG_FEATURE_STATS, 5440 features)); 5441 5442 mutex_exit(&spa->spa_feat_stats_lock); 5443} 5444 5445int 5446spa_get_stats(const char *name, nvlist_t **config, 5447 char *altroot, size_t buflen) 5448{ 5449 int error; 5450 spa_t *spa; 5451 5452 *config = NULL; 5453 error = spa_open_common(name, &spa, FTAG, NULL, config); 5454 5455 if (spa != NULL) { 5456 /* 5457 * This still leaves a window of inconsistency where the spares 5458 * or l2cache devices could change and the config would be 5459 * self-inconsistent. 5460 */ 5461 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 5462 5463 if (*config != NULL) { 5464 uint64_t loadtimes[2]; 5465 5466 loadtimes[0] = spa->spa_loaded_ts.tv_sec; 5467 loadtimes[1] = spa->spa_loaded_ts.tv_nsec; 5468 VERIFY(nvlist_add_uint64_array(*config, 5469 ZPOOL_CONFIG_LOADED_TIME, loadtimes, 2) == 0); 5470 5471 VERIFY(nvlist_add_uint64(*config, 5472 ZPOOL_CONFIG_ERRCOUNT, 5473 spa_get_errlog_size(spa)) == 0); 5474 5475 if (spa_suspended(spa)) { 5476 VERIFY(nvlist_add_uint64(*config, 5477 ZPOOL_CONFIG_SUSPENDED, 5478 spa->spa_failmode) == 0); 5479 VERIFY(nvlist_add_uint64(*config, 5480 ZPOOL_CONFIG_SUSPENDED_REASON, 5481 spa->spa_suspended) == 0); 5482 } 5483 5484 spa_add_spares(spa, *config); 5485 spa_add_l2cache(spa, *config); 5486 spa_add_feature_stats(spa, *config); 5487 } 5488 } 5489 5490 /* 5491 * We want to get the alternate root even for faulted pools, so we cheat 5492 * and call spa_lookup() directly. 5493 */ 5494 if (altroot) { 5495 if (spa == NULL) { 5496 mutex_enter(&spa_namespace_lock); 5497 spa = spa_lookup(name); 5498 if (spa) 5499 spa_altroot(spa, altroot, buflen); 5500 else 5501 altroot[0] = '\0'; 5502 spa = NULL; 5503 mutex_exit(&spa_namespace_lock); 5504 } else { 5505 spa_altroot(spa, altroot, buflen); 5506 } 5507 } 5508 5509 if (spa != NULL) { 5510 spa_config_exit(spa, SCL_CONFIG, FTAG); 5511 spa_close(spa, FTAG); 5512 } 5513 5514 return (error); 5515} 5516 5517/* 5518 * Validate that the auxiliary device array is well formed. We must have an 5519 * array of nvlists, each which describes a valid leaf vdev. If this is an 5520 * import (mode is VDEV_ALLOC_SPARE), then we allow corrupted spares to be 5521 * specified, as long as they are well-formed. 5522 */ 5523static int 5524spa_validate_aux_devs(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode, 5525 spa_aux_vdev_t *sav, const char *config, uint64_t version, 5526 vdev_labeltype_t label) 5527{ 5528 nvlist_t **dev; 5529 uint_t i, ndev; 5530 vdev_t *vd; 5531 int error; 5532 5533 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5534 5535 /* 5536 * It's acceptable to have no devs specified. 5537 */ 5538 if (nvlist_lookup_nvlist_array(nvroot, config, &dev, &ndev) != 0) 5539 return (0); 5540 5541 if (ndev == 0) 5542 return (SET_ERROR(EINVAL)); 5543 5544 /* 5545 * Make sure the pool is formatted with a version that supports this 5546 * device type. 5547 */ 5548 if (spa_version(spa) < version) 5549 return (SET_ERROR(ENOTSUP)); 5550 5551 /* 5552 * Set the pending device list so we correctly handle device in-use 5553 * checking. 5554 */ 5555 sav->sav_pending = dev; 5556 sav->sav_npending = ndev; 5557 5558 for (i = 0; i < ndev; i++) { 5559 if ((error = spa_config_parse(spa, &vd, dev[i], NULL, 0, 5560 mode)) != 0) 5561 goto out; 5562 5563 if (!vd->vdev_ops->vdev_op_leaf) { 5564 vdev_free(vd); 5565 error = SET_ERROR(EINVAL); 5566 goto out; 5567 } 5568 5569 vd->vdev_top = vd; 5570 5571 if ((error = vdev_open(vd)) == 0 && 5572 (error = vdev_label_init(vd, crtxg, label)) == 0) { 5573 VERIFY(nvlist_add_uint64(dev[i], ZPOOL_CONFIG_GUID, 5574 vd->vdev_guid) == 0); 5575 } 5576 5577 vdev_free(vd); 5578 5579 if (error && 5580 (mode != VDEV_ALLOC_SPARE && mode != VDEV_ALLOC_L2CACHE)) 5581 goto out; 5582 else 5583 error = 0; 5584 } 5585 5586out: 5587 sav->sav_pending = NULL; 5588 sav->sav_npending = 0; 5589 return (error); 5590} 5591 5592static int 5593spa_validate_aux(spa_t *spa, nvlist_t *nvroot, uint64_t crtxg, int mode) 5594{ 5595 int error; 5596 5597 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL); 5598 5599 if ((error = spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5600 &spa->spa_spares, ZPOOL_CONFIG_SPARES, SPA_VERSION_SPARES, 5601 VDEV_LABEL_SPARE)) != 0) { 5602 return (error); 5603 } 5604 5605 return (spa_validate_aux_devs(spa, nvroot, crtxg, mode, 5606 &spa->spa_l2cache, ZPOOL_CONFIG_L2CACHE, SPA_VERSION_L2CACHE, 5607 VDEV_LABEL_L2CACHE)); 5608} 5609 5610static void 5611spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, 5612 const char *config) 5613{ 5614 int i; 5615 5616 if (sav->sav_config != NULL) { 5617 nvlist_t **olddevs; 5618 uint_t oldndevs; 5619 nvlist_t **newdevs; 5620 5621 /* 5622 * Generate new dev list by concatenating with the 5623 * current dev list. 5624 */ 5625 VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, config, 5626 &olddevs, &oldndevs) == 0); 5627 5628 newdevs = kmem_alloc(sizeof (void *) * 5629 (ndevs + oldndevs), KM_SLEEP); 5630 for (i = 0; i < oldndevs; i++) 5631 VERIFY(nvlist_dup(olddevs[i], &newdevs[i], 5632 KM_SLEEP) == 0); 5633 for (i = 0; i < ndevs; i++) 5634 VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], 5635 KM_SLEEP) == 0); 5636 5637 VERIFY(nvlist_remove(sav->sav_config, config, 5638 DATA_TYPE_NVLIST_ARRAY) == 0); 5639 5640 VERIFY(nvlist_add_nvlist_array(sav->sav_config, 5641 config, newdevs, ndevs + oldndevs) == 0); 5642 for (i = 0; i < oldndevs + ndevs; i++) 5643 nvlist_free(newdevs[i]); 5644 kmem_free(newdevs, (oldndevs + ndevs) * sizeof (void *)); 5645 } else { 5646 /* 5647 * Generate a new dev list. 5648 */ 5649 VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, 5650 KM_SLEEP) == 0); 5651 VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, 5652 devs, ndevs) == 0); 5653 } 5654} 5655 5656/* 5657 * Stop and drop level 2 ARC devices 5658 */ 5659void 5660spa_l2cache_drop(spa_t *spa) 5661{ 5662 vdev_t *vd; 5663 int i; 5664 spa_aux_vdev_t *sav = &spa->spa_l2cache; 5665 5666 for (i = 0; i < sav->sav_count; i++) { 5667 uint64_t pool; 5668 5669 vd = sav->sav_vdevs[i]; 5670 ASSERT(vd != NULL); 5671 5672 if (spa_l2cache_exists(vd->vdev_guid, &pool) && 5673 pool != 0ULL && l2arc_vdev_present(vd)) 5674 l2arc_remove_vdev(vd); 5675 } 5676} 5677 5678/* 5679 * Verify encryption parameters for spa creation. If we are encrypting, we must 5680 * have the encryption feature flag enabled. 5681 */ 5682static int 5683spa_create_check_encryption_params(dsl_crypto_params_t *dcp, 5684 boolean_t has_encryption) 5685{ 5686 if (dcp->cp_crypt != ZIO_CRYPT_OFF && 5687 dcp->cp_crypt != ZIO_CRYPT_INHERIT && 5688 !has_encryption) 5689 return (SET_ERROR(ENOTSUP)); 5690 5691 return (dmu_objset_create_crypt_check(NULL, dcp, NULL)); 5692} 5693 5694/* 5695 * Pool Creation 5696 */ 5697int 5698spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, 5699 nvlist_t *zplprops, dsl_crypto_params_t *dcp) 5700{ 5701 spa_t *spa; 5702 char *altroot = NULL; 5703 vdev_t *rvd; 5704 dsl_pool_t *dp; 5705 dmu_tx_t *tx; 5706 int error = 0; 5707 uint64_t txg = TXG_INITIAL; 5708 nvlist_t **spares, **l2cache; 5709 uint_t nspares, nl2cache; 5710 uint64_t version, obj, ndraid = 0; 5711 boolean_t has_features; 5712 boolean_t has_encryption; 5713 boolean_t has_allocclass; 5714 spa_feature_t feat; 5715 char *feat_name; 5716 char *poolname; 5717 nvlist_t *nvl; 5718 5719 if (props == NULL || 5720 nvlist_lookup_string(props, "tname", &poolname) != 0) 5721 poolname = (char *)pool; 5722 5723 /* 5724 * If this pool already exists, return failure. 5725 */ 5726 mutex_enter(&spa_namespace_lock); 5727 if (spa_lookup(poolname) != NULL) { 5728 mutex_exit(&spa_namespace_lock); 5729 return (SET_ERROR(EEXIST)); 5730 } 5731 5732 /* 5733 * Allocate a new spa_t structure. 5734 */ 5735 nvl = fnvlist_alloc(); 5736 fnvlist_add_string(nvl, ZPOOL_CONFIG_POOL_NAME, pool); 5737 (void) nvlist_lookup_string(props, 5738 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 5739 spa = spa_add(poolname, nvl, altroot); 5740 fnvlist_free(nvl); 5741 spa_activate(spa, spa_mode_global); 5742 5743 if (props && (error = spa_prop_validate(spa, props))) { 5744 spa_deactivate(spa); 5745 spa_remove(spa); 5746 mutex_exit(&spa_namespace_lock); 5747 return (error); 5748 } 5749 5750 /* 5751 * Temporary pool names should never be written to disk. 5752 */ 5753 if (poolname != pool) 5754 spa->spa_import_flags |= ZFS_IMPORT_TEMP_NAME; 5755 5756 has_features = B_FALSE; 5757 has_encryption = B_FALSE; 5758 has_allocclass = B_FALSE; 5759 for (nvpair_t *elem = nvlist_next_nvpair(props, NULL); 5760 elem != NULL; elem = nvlist_next_nvpair(props, elem)) { 5761 if (zpool_prop_feature(nvpair_name(elem))) { 5762 has_features = B_TRUE; 5763 5764 feat_name = strchr(nvpair_name(elem), '@') + 1; 5765 VERIFY0(zfeature_lookup_name(feat_name, &feat)); 5766 if (feat == SPA_FEATURE_ENCRYPTION) 5767 has_encryption = B_TRUE; 5768 if (feat == SPA_FEATURE_ALLOCATION_CLASSES) 5769 has_allocclass = B_TRUE; 5770 } 5771 } 5772 5773 /* verify encryption params, if they were provided */ 5774 if (dcp != NULL) { 5775 error = spa_create_check_encryption_params(dcp, has_encryption); 5776 if (error != 0) { 5777 spa_deactivate(spa); 5778 spa_remove(spa); 5779 mutex_exit(&spa_namespace_lock); 5780 return (error); 5781 } 5782 } 5783 if (!has_allocclass && zfs_special_devs(nvroot, NULL)) { 5784 spa_deactivate(spa); 5785 spa_remove(spa); 5786 mutex_exit(&spa_namespace_lock); 5787 return (ENOTSUP); 5788 } 5789 5790 if (has_features || nvlist_lookup_uint64(props, 5791 zpool_prop_to_name(ZPOOL_PROP_VERSION), &version) != 0) { 5792 version = SPA_VERSION; 5793 } 5794 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 5795 5796 spa->spa_first_txg = txg; 5797 spa->spa_uberblock.ub_txg = txg - 1; 5798 spa->spa_uberblock.ub_version = version; 5799 spa->spa_ubsync = spa->spa_uberblock; 5800 spa->spa_load_state = SPA_LOAD_CREATE; 5801 spa->spa_removing_phys.sr_state = DSS_NONE; 5802 spa->spa_removing_phys.sr_removing_vdev = -1; 5803 spa->spa_removing_phys.sr_prev_indirect_vdev = -1; 5804 spa->spa_indirect_vdevs_loaded = B_TRUE; 5805 5806 /* 5807 * Create "The Godfather" zio to hold all async IOs 5808 */ 5809 spa->spa_async_zio_root = kmem_alloc(max_ncpus * sizeof (void *), 5810 KM_SLEEP); 5811 for (int i = 0; i < max_ncpus; i++) { 5812 spa->spa_async_zio_root[i] = zio_root(spa, NULL, NULL, 5813 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 5814 ZIO_FLAG_GODFATHER); 5815 } 5816 5817 /* 5818 * Create the root vdev. 5819 */ 5820 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5821 5822 error = spa_config_parse(spa, &rvd, nvroot, NULL, 0, VDEV_ALLOC_ADD); 5823 5824 ASSERT(error != 0 || rvd != NULL); 5825 ASSERT(error != 0 || spa->spa_root_vdev == rvd); 5826 5827 if (error == 0 && !zfs_allocatable_devs(nvroot)) 5828 error = SET_ERROR(EINVAL); 5829 5830 if (error == 0 && 5831 (error = vdev_create(rvd, txg, B_FALSE)) == 0 && 5832 (error = vdev_draid_spare_create(nvroot, rvd, &ndraid, 0)) == 0 && 5833 (error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) == 0) { 5834 /* 5835 * instantiate the metaslab groups (this will dirty the vdevs) 5836 * we can no longer error exit past this point 5837 */ 5838 for (int c = 0; error == 0 && c < rvd->vdev_children; c++) { 5839 vdev_t *vd = rvd->vdev_child[c]; 5840 5841 vdev_metaslab_set_size(vd); 5842 vdev_expand(vd, txg); 5843 } 5844 } 5845 5846 spa_config_exit(spa, SCL_ALL, FTAG); 5847 5848 if (error != 0) { 5849 spa_unload(spa); 5850 spa_deactivate(spa); 5851 spa_remove(spa); 5852 mutex_exit(&spa_namespace_lock); 5853 return (error); 5854 } 5855 5856 /* 5857 * Get the list of spares, if specified. 5858 */ 5859 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 5860 &spares, &nspares) == 0) { 5861 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, 5862 KM_SLEEP) == 0); 5863 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 5864 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 5865 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5866 spa_load_spares(spa); 5867 spa_config_exit(spa, SCL_ALL, FTAG); 5868 spa->spa_spares.sav_sync = B_TRUE; 5869 } 5870 5871 /* 5872 * Get the list of level 2 cache devices, if specified. 5873 */ 5874 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 5875 &l2cache, &nl2cache) == 0) { 5876 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 5877 NV_UNIQUE_NAME, KM_SLEEP) == 0); 5878 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 5879 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 5880 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 5881 spa_load_l2cache(spa); 5882 spa_config_exit(spa, SCL_ALL, FTAG); 5883 spa->spa_l2cache.sav_sync = B_TRUE; 5884 } 5885 5886 spa->spa_is_initializing = B_TRUE; 5887 spa->spa_dsl_pool = dp = dsl_pool_create(spa, zplprops, dcp, txg); 5888 spa->spa_is_initializing = B_FALSE; 5889 5890 /* 5891 * Create DDTs (dedup tables). 5892 */ 5893 ddt_create(spa); 5894 5895 spa_update_dspace(spa); 5896 5897 tx = dmu_tx_create_assigned(dp, txg); 5898 5899 /* 5900 * Create the pool's history object. 5901 */ 5902 if (version >= SPA_VERSION_ZPOOL_HISTORY && !spa->spa_history) 5903 spa_history_create_obj(spa, tx); 5904 5905 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_CREATE); 5906 spa_history_log_version(spa, "create", tx); 5907 5908 /* 5909 * Create the pool config object. 5910 */ 5911 spa->spa_config_object = dmu_object_alloc(spa->spa_meta_objset, 5912 DMU_OT_PACKED_NVLIST, SPA_CONFIG_BLOCKSIZE, 5913 DMU_OT_PACKED_NVLIST_SIZE, sizeof (uint64_t), tx); 5914 5915 if (zap_add(spa->spa_meta_objset, 5916 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CONFIG, 5917 sizeof (uint64_t), 1, &spa->spa_config_object, tx) != 0) { 5918 cmn_err(CE_PANIC, "failed to add pool config"); 5919 } 5920 5921 if (zap_add(spa->spa_meta_objset, 5922 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CREATION_VERSION, 5923 sizeof (uint64_t), 1, &version, tx) != 0) { 5924 cmn_err(CE_PANIC, "failed to add pool version"); 5925 } 5926 5927 /* Newly created pools with the right version are always deflated. */ 5928 if (version >= SPA_VERSION_RAIDZ_DEFLATE) { 5929 spa->spa_deflate = TRUE; 5930 if (zap_add(spa->spa_meta_objset, 5931 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 5932 sizeof (uint64_t), 1, &spa->spa_deflate, tx) != 0) { 5933 cmn_err(CE_PANIC, "failed to add deflate"); 5934 } 5935 } 5936 5937 /* 5938 * Create the deferred-free bpobj. Turn off compression 5939 * because sync-to-convergence takes longer if the blocksize 5940 * keeps changing. 5941 */ 5942 obj = bpobj_alloc(spa->spa_meta_objset, 1 << 14, tx); 5943 dmu_object_set_compress(spa->spa_meta_objset, obj, 5944 ZIO_COMPRESS_OFF, tx); 5945 if (zap_add(spa->spa_meta_objset, 5946 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_SYNC_BPOBJ, 5947 sizeof (uint64_t), 1, &obj, tx) != 0) { 5948 cmn_err(CE_PANIC, "failed to add bpobj"); 5949 } 5950 VERIFY3U(0, ==, bpobj_open(&spa->spa_deferred_bpobj, 5951 spa->spa_meta_objset, obj)); 5952 5953 /* 5954 * Generate some random noise for salted checksums to operate on. 5955 */ 5956 (void) random_get_pseudo_bytes(spa->spa_cksum_salt.zcs_bytes, 5957 sizeof (spa->spa_cksum_salt.zcs_bytes)); 5958 5959 /* 5960 * Set pool properties. 5961 */ 5962 spa->spa_bootfs = zpool_prop_default_numeric(ZPOOL_PROP_BOOTFS); 5963 spa->spa_delegation = zpool_prop_default_numeric(ZPOOL_PROP_DELEGATION); 5964 spa->spa_failmode = zpool_prop_default_numeric(ZPOOL_PROP_FAILUREMODE); 5965 spa->spa_autoexpand = zpool_prop_default_numeric(ZPOOL_PROP_AUTOEXPAND); 5966 spa->spa_multihost = zpool_prop_default_numeric(ZPOOL_PROP_MULTIHOST); 5967 spa->spa_autotrim = zpool_prop_default_numeric(ZPOOL_PROP_AUTOTRIM); 5968 5969 if (props != NULL) { 5970 spa_configfile_set(spa, props, B_FALSE); 5971 spa_sync_props(props, tx); 5972 } 5973 5974 for (int i = 0; i < ndraid; i++) 5975 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 5976 5977 dmu_tx_commit(tx); 5978 5979 spa->spa_sync_on = B_TRUE; 5980 txg_sync_start(dp); 5981 mmp_thread_start(spa); 5982 txg_wait_synced(dp, txg); 5983 5984 spa_spawn_aux_threads(spa); 5985 5986 spa_write_cachefile(spa, B_FALSE, B_TRUE); 5987 5988 /* 5989 * Don't count references from objsets that are already closed 5990 * and are making their way through the eviction process. 5991 */ 5992 spa_evicting_os_wait(spa); 5993 spa->spa_minref = zfs_refcount_count(&spa->spa_refcount); 5994 spa->spa_load_state = SPA_LOAD_NONE; 5995 5996 mutex_exit(&spa_namespace_lock); 5997 5998 return (0); 5999} 6000 6001/* 6002 * Import a non-root pool into the system. 6003 */ 6004int 6005spa_import(char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) 6006{ 6007 spa_t *spa; 6008 char *altroot = NULL; 6009 spa_load_state_t state = SPA_LOAD_IMPORT; 6010 zpool_load_policy_t policy; 6011 spa_mode_t mode = spa_mode_global; 6012 uint64_t readonly = B_FALSE; 6013 int error; 6014 nvlist_t *nvroot; 6015 nvlist_t **spares, **l2cache; 6016 uint_t nspares, nl2cache; 6017 6018 /* 6019 * If a pool with this name exists, return failure. 6020 */ 6021 mutex_enter(&spa_namespace_lock); 6022 if (spa_lookup(pool) != NULL) { 6023 mutex_exit(&spa_namespace_lock); 6024 return (SET_ERROR(EEXIST)); 6025 } 6026 6027 /* 6028 * Create and initialize the spa structure. 6029 */ 6030 (void) nvlist_lookup_string(props, 6031 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 6032 (void) nvlist_lookup_uint64(props, 6033 zpool_prop_to_name(ZPOOL_PROP_READONLY), &readonly); 6034 if (readonly) 6035 mode = SPA_MODE_READ; 6036 spa = spa_add(pool, config, altroot); 6037 spa->spa_import_flags = flags; 6038 6039 /* 6040 * Verbatim import - Take a pool and insert it into the namespace 6041 * as if it had been loaded at boot. 6042 */ 6043 if (spa->spa_import_flags & ZFS_IMPORT_VERBATIM) { 6044 if (props != NULL) 6045 spa_configfile_set(spa, props, B_FALSE); 6046 6047 spa_write_cachefile(spa, B_FALSE, B_TRUE); 6048 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6049 zfs_dbgmsg("spa_import: verbatim import of %s", pool); 6050 mutex_exit(&spa_namespace_lock); 6051 return (0); 6052 } 6053 6054 spa_activate(spa, mode); 6055 6056 /* 6057 * Don't start async tasks until we know everything is healthy. 6058 */ 6059 spa_async_suspend(spa); 6060 6061 zpool_get_load_policy(config, &policy); 6062 if (policy.zlp_rewind & ZPOOL_DO_REWIND) 6063 state = SPA_LOAD_RECOVER; 6064 6065 spa->spa_config_source = SPA_CONFIG_SRC_TRYIMPORT; 6066 6067 if (state != SPA_LOAD_RECOVER) { 6068 spa->spa_last_ubsync_txg = spa->spa_load_txg = 0; 6069 zfs_dbgmsg("spa_import: importing %s", pool); 6070 } else { 6071 zfs_dbgmsg("spa_import: importing %s, max_txg=%lld " 6072 "(RECOVERY MODE)", pool, (longlong_t)policy.zlp_txg); 6073 } 6074 error = spa_load_best(spa, state, policy.zlp_txg, policy.zlp_rewind); 6075 6076 /* 6077 * Propagate anything learned while loading the pool and pass it 6078 * back to caller (i.e. rewind info, missing devices, etc). 6079 */ 6080 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6081 spa->spa_load_info) == 0); 6082 6083 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6084 /* 6085 * Toss any existing sparelist, as it doesn't have any validity 6086 * anymore, and conflicts with spa_has_spare(). 6087 */ 6088 if (spa->spa_spares.sav_config) { 6089 nvlist_free(spa->spa_spares.sav_config); 6090 spa->spa_spares.sav_config = NULL; 6091 spa_load_spares(spa); 6092 } 6093 if (spa->spa_l2cache.sav_config) { 6094 nvlist_free(spa->spa_l2cache.sav_config); 6095 spa->spa_l2cache.sav_config = NULL; 6096 spa_load_l2cache(spa); 6097 } 6098 6099 VERIFY(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, 6100 &nvroot) == 0); 6101 spa_config_exit(spa, SCL_ALL, FTAG); 6102 6103 if (props != NULL) 6104 spa_configfile_set(spa, props, B_FALSE); 6105 6106 if (error != 0 || (props && spa_writeable(spa) && 6107 (error = spa_prop_set(spa, props)))) { 6108 spa_unload(spa); 6109 spa_deactivate(spa); 6110 spa_remove(spa); 6111 mutex_exit(&spa_namespace_lock); 6112 return (error); 6113 } 6114 6115 spa_async_resume(spa); 6116 6117 /* 6118 * Override any spares and level 2 cache devices as specified by 6119 * the user, as these may have correct device names/devids, etc. 6120 */ 6121 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, 6122 &spares, &nspares) == 0) { 6123 if (spa->spa_spares.sav_config) 6124 VERIFY(nvlist_remove(spa->spa_spares.sav_config, 6125 ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); 6126 else 6127 VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, 6128 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6129 VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, 6130 ZPOOL_CONFIG_SPARES, spares, nspares) == 0); 6131 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6132 spa_load_spares(spa); 6133 spa_config_exit(spa, SCL_ALL, FTAG); 6134 spa->spa_spares.sav_sync = B_TRUE; 6135 } 6136 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, 6137 &l2cache, &nl2cache) == 0) { 6138 if (spa->spa_l2cache.sav_config) 6139 VERIFY(nvlist_remove(spa->spa_l2cache.sav_config, 6140 ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); 6141 else 6142 VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, 6143 NV_UNIQUE_NAME, KM_SLEEP) == 0); 6144 VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, 6145 ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); 6146 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6147 spa_load_l2cache(spa); 6148 spa_config_exit(spa, SCL_ALL, FTAG); 6149 spa->spa_l2cache.sav_sync = B_TRUE; 6150 } 6151 6152 /* 6153 * Check for any removed devices. 6154 */ 6155 if (spa->spa_autoreplace) { 6156 spa_aux_check_removed(&spa->spa_spares); 6157 spa_aux_check_removed(&spa->spa_l2cache); 6158 } 6159 6160 if (spa_writeable(spa)) { 6161 /* 6162 * Update the config cache to include the newly-imported pool. 6163 */ 6164 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6165 } 6166 6167 /* 6168 * It's possible that the pool was expanded while it was exported. 6169 * We kick off an async task to handle this for us. 6170 */ 6171 spa_async_request(spa, SPA_ASYNC_AUTOEXPAND); 6172 6173 spa_history_log_version(spa, "import", NULL); 6174 6175 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_IMPORT); 6176 6177 mutex_exit(&spa_namespace_lock); 6178 6179 zvol_create_minors_recursive(pool); 6180 6181 return (0); 6182} 6183 6184nvlist_t * 6185spa_tryimport(nvlist_t *tryconfig) 6186{ 6187 nvlist_t *config = NULL; 6188 char *poolname, *cachefile; 6189 spa_t *spa; 6190 uint64_t state; 6191 int error; 6192 zpool_load_policy_t policy; 6193 6194 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_POOL_NAME, &poolname)) 6195 return (NULL); 6196 6197 if (nvlist_lookup_uint64(tryconfig, ZPOOL_CONFIG_POOL_STATE, &state)) 6198 return (NULL); 6199 6200 /* 6201 * Create and initialize the spa structure. 6202 */ 6203 mutex_enter(&spa_namespace_lock); 6204 spa = spa_add(TRYIMPORT_NAME, tryconfig, NULL); 6205 spa_activate(spa, SPA_MODE_READ); 6206 6207 /* 6208 * Rewind pool if a max txg was provided. 6209 */ 6210 zpool_get_load_policy(spa->spa_config, &policy); 6211 if (policy.zlp_txg != UINT64_MAX) { 6212 spa->spa_load_max_txg = policy.zlp_txg; 6213 spa->spa_extreme_rewind = B_TRUE; 6214 zfs_dbgmsg("spa_tryimport: importing %s, max_txg=%lld", 6215 poolname, (longlong_t)policy.zlp_txg); 6216 } else { 6217 zfs_dbgmsg("spa_tryimport: importing %s", poolname); 6218 } 6219 6220 if (nvlist_lookup_string(tryconfig, ZPOOL_CONFIG_CACHEFILE, &cachefile) 6221 == 0) { 6222 zfs_dbgmsg("spa_tryimport: using cachefile '%s'", cachefile); 6223 spa->spa_config_source = SPA_CONFIG_SRC_CACHEFILE; 6224 } else { 6225 spa->spa_config_source = SPA_CONFIG_SRC_SCAN; 6226 } 6227 6228 error = spa_load(spa, SPA_LOAD_TRYIMPORT, SPA_IMPORT_EXISTING); 6229 6230 /* 6231 * If 'tryconfig' was at least parsable, return the current config. 6232 */ 6233 if (spa->spa_root_vdev != NULL) { 6234 config = spa_config_generate(spa, NULL, -1ULL, B_TRUE); 6235 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, 6236 poolname) == 0); 6237 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 6238 state) == 0); 6239 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_TIMESTAMP, 6240 spa->spa_uberblock.ub_timestamp) == 0); 6241 VERIFY(nvlist_add_nvlist(config, ZPOOL_CONFIG_LOAD_INFO, 6242 spa->spa_load_info) == 0); 6243 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_ERRATA, 6244 spa->spa_errata) == 0); 6245 6246 /* 6247 * If the bootfs property exists on this pool then we 6248 * copy it out so that external consumers can tell which 6249 * pools are bootable. 6250 */ 6251 if ((!error || error == EEXIST) && spa->spa_bootfs) { 6252 char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6253 6254 /* 6255 * We have to play games with the name since the 6256 * pool was opened as TRYIMPORT_NAME. 6257 */ 6258 if (dsl_dsobj_to_dsname(spa_name(spa), 6259 spa->spa_bootfs, tmpname) == 0) { 6260 char *cp; 6261 char *dsname; 6262 6263 dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); 6264 6265 cp = strchr(tmpname, '/'); 6266 if (cp == NULL) { 6267 (void) strlcpy(dsname, tmpname, 6268 MAXPATHLEN); 6269 } else { 6270 (void) snprintf(dsname, MAXPATHLEN, 6271 "%s/%s", poolname, ++cp); 6272 } 6273 VERIFY(nvlist_add_string(config, 6274 ZPOOL_CONFIG_BOOTFS, dsname) == 0); 6275 kmem_free(dsname, MAXPATHLEN); 6276 } 6277 kmem_free(tmpname, MAXPATHLEN); 6278 } 6279 6280 /* 6281 * Add the list of hot spares and level 2 cache devices. 6282 */ 6283 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 6284 spa_add_spares(spa, config); 6285 spa_add_l2cache(spa, config); 6286 spa_config_exit(spa, SCL_CONFIG, FTAG); 6287 } 6288 6289 spa_unload(spa); 6290 spa_deactivate(spa); 6291 spa_remove(spa); 6292 mutex_exit(&spa_namespace_lock); 6293 6294 return (config); 6295} 6296 6297/* 6298 * Pool export/destroy 6299 * 6300 * The act of destroying or exporting a pool is very simple. We make sure there 6301 * is no more pending I/O and any references to the pool are gone. Then, we 6302 * update the pool state and sync all the labels to disk, removing the 6303 * configuration from the cache afterwards. If the 'hardforce' flag is set, then 6304 * we don't sync the labels or remove the configuration cache. 6305 */ 6306static int 6307spa_export_common(const char *pool, int new_state, nvlist_t **oldconfig, 6308 boolean_t force, boolean_t hardforce) 6309{ 6310 int error; 6311 spa_t *spa; 6312 6313 if (oldconfig) 6314 *oldconfig = NULL; 6315 6316 if (!(spa_mode_global & SPA_MODE_WRITE)) 6317 return (SET_ERROR(EROFS)); 6318 6319 mutex_enter(&spa_namespace_lock); 6320 if ((spa = spa_lookup(pool)) == NULL) { 6321 mutex_exit(&spa_namespace_lock); 6322 return (SET_ERROR(ENOENT)); 6323 } 6324 6325 if (spa->spa_is_exporting) { 6326 /* the pool is being exported by another thread */ 6327 mutex_exit(&spa_namespace_lock); 6328 return (SET_ERROR(ZFS_ERR_EXPORT_IN_PROGRESS)); 6329 } 6330 spa->spa_is_exporting = B_TRUE; 6331 6332 /* 6333 * Put a hold on the pool, drop the namespace lock, stop async tasks, 6334 * reacquire the namespace lock, and see if we can export. 6335 */ 6336 spa_open_ref(spa, FTAG); 6337 mutex_exit(&spa_namespace_lock); 6338 spa_async_suspend(spa); 6339 if (spa->spa_zvol_taskq) { 6340 zvol_remove_minors(spa, spa_name(spa), B_TRUE); 6341 taskq_wait(spa->spa_zvol_taskq); 6342 } 6343 mutex_enter(&spa_namespace_lock); 6344 spa_close(spa, FTAG); 6345 6346 if (spa->spa_state == POOL_STATE_UNINITIALIZED) 6347 goto export_spa; 6348 /* 6349 * The pool will be in core if it's openable, in which case we can 6350 * modify its state. Objsets may be open only because they're dirty, 6351 * so we have to force it to sync before checking spa_refcnt. 6352 */ 6353 if (spa->spa_sync_on) { 6354 txg_wait_synced(spa->spa_dsl_pool, 0); 6355 spa_evicting_os_wait(spa); 6356 } 6357 6358 /* 6359 * A pool cannot be exported or destroyed if there are active 6360 * references. If we are resetting a pool, allow references by 6361 * fault injection handlers. 6362 */ 6363 if (!spa_refcount_zero(spa) || (spa->spa_inject_ref != 0)) { 6364 error = SET_ERROR(EBUSY); 6365 goto fail; 6366 } 6367 6368 if (spa->spa_sync_on) { 6369 /* 6370 * A pool cannot be exported if it has an active shared spare. 6371 * This is to prevent other pools stealing the active spare 6372 * from an exported pool. At user's own will, such pool can 6373 * be forcedly exported. 6374 */ 6375 if (!force && new_state == POOL_STATE_EXPORTED && 6376 spa_has_active_shared_spare(spa)) { 6377 error = SET_ERROR(EXDEV); 6378 goto fail; 6379 } 6380 6381 /* 6382 * We're about to export or destroy this pool. Make sure 6383 * we stop all initialization and trim activity here before 6384 * we set the spa_final_txg. This will ensure that all 6385 * dirty data resulting from the initialization is 6386 * committed to disk before we unload the pool. 6387 */ 6388 if (spa->spa_root_vdev != NULL) { 6389 vdev_t *rvd = spa->spa_root_vdev; 6390 vdev_initialize_stop_all(rvd, VDEV_INITIALIZE_ACTIVE); 6391 vdev_trim_stop_all(rvd, VDEV_TRIM_ACTIVE); 6392 vdev_autotrim_stop_all(spa); 6393 vdev_rebuild_stop_all(spa); 6394 } 6395 6396 /* 6397 * We want this to be reflected on every label, 6398 * so mark them all dirty. spa_unload() will do the 6399 * final sync that pushes these changes out. 6400 */ 6401 if (new_state != POOL_STATE_UNINITIALIZED && !hardforce) { 6402 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 6403 spa->spa_state = new_state; 6404 spa->spa_final_txg = spa_last_synced_txg(spa) + 6405 TXG_DEFER_SIZE + 1; 6406 vdev_config_dirty(spa->spa_root_vdev); 6407 spa_config_exit(spa, SCL_ALL, FTAG); 6408 } 6409 } 6410 6411export_spa: 6412 if (new_state == POOL_STATE_DESTROYED) 6413 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_DESTROY); 6414 else if (new_state == POOL_STATE_EXPORTED) 6415 spa_event_notify(spa, NULL, NULL, ESC_ZFS_POOL_EXPORT); 6416 6417 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 6418 spa_unload(spa); 6419 spa_deactivate(spa); 6420 } 6421 6422 if (oldconfig && spa->spa_config) 6423 VERIFY(nvlist_dup(spa->spa_config, oldconfig, 0) == 0); 6424 6425 if (new_state != POOL_STATE_UNINITIALIZED) { 6426 if (!hardforce) 6427 spa_write_cachefile(spa, B_TRUE, B_TRUE); 6428 spa_remove(spa); 6429 } else { 6430 /* 6431 * If spa_remove() is not called for this spa_t and 6432 * there is any possibility that it can be reused, 6433 * we make sure to reset the exporting flag. 6434 */ 6435 spa->spa_is_exporting = B_FALSE; 6436 } 6437 6438 mutex_exit(&spa_namespace_lock); 6439 return (0); 6440 6441fail: 6442 spa->spa_is_exporting = B_FALSE; 6443 spa_async_resume(spa); 6444 mutex_exit(&spa_namespace_lock); 6445 return (error); 6446} 6447 6448/* 6449 * Destroy a storage pool. 6450 */ 6451int 6452spa_destroy(const char *pool) 6453{ 6454 return (spa_export_common(pool, POOL_STATE_DESTROYED, NULL, 6455 B_FALSE, B_FALSE)); 6456} 6457 6458/* 6459 * Export a storage pool. 6460 */ 6461int 6462spa_export(const char *pool, nvlist_t **oldconfig, boolean_t force, 6463 boolean_t hardforce) 6464{ 6465 return (spa_export_common(pool, POOL_STATE_EXPORTED, oldconfig, 6466 force, hardforce)); 6467} 6468 6469/* 6470 * Similar to spa_export(), this unloads the spa_t without actually removing it 6471 * from the namespace in any way. 6472 */ 6473int 6474spa_reset(const char *pool) 6475{ 6476 return (spa_export_common(pool, POOL_STATE_UNINITIALIZED, NULL, 6477 B_FALSE, B_FALSE)); 6478} 6479 6480/* 6481 * ========================================================================== 6482 * Device manipulation 6483 * ========================================================================== 6484 */ 6485 6486/* 6487 * This is called as a synctask to increment the draid feature flag 6488 */ 6489static void 6490spa_draid_feature_incr(void *arg, dmu_tx_t *tx) 6491{ 6492 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 6493 int draid = (int)(uintptr_t)arg; 6494 6495 for (int c = 0; c < draid; c++) 6496 spa_feature_incr(spa, SPA_FEATURE_DRAID, tx); 6497} 6498 6499/* 6500 * Add a device to a storage pool. 6501 */ 6502int 6503spa_vdev_add(spa_t *spa, nvlist_t *nvroot) 6504{ 6505 uint64_t txg, ndraid = 0; 6506 int error; 6507 vdev_t *rvd = spa->spa_root_vdev; 6508 vdev_t *vd, *tvd; 6509 nvlist_t **spares, **l2cache; 6510 uint_t nspares, nl2cache; 6511 6512 ASSERT(spa_writeable(spa)); 6513 6514 txg = spa_vdev_enter(spa); 6515 6516 if ((error = spa_config_parse(spa, &vd, nvroot, NULL, 0, 6517 VDEV_ALLOC_ADD)) != 0) 6518 return (spa_vdev_exit(spa, NULL, txg, error)); 6519 6520 spa->spa_pending_vdev = vd; /* spa_vdev_exit() will clear this */ 6521 6522 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, 6523 &nspares) != 0) 6524 nspares = 0; 6525 6526 if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, 6527 &nl2cache) != 0) 6528 nl2cache = 0; 6529 6530 if (vd->vdev_children == 0 && nspares == 0 && nl2cache == 0) 6531 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6532 6533 if (vd->vdev_children != 0 && 6534 (error = vdev_create(vd, txg, B_FALSE)) != 0) { 6535 return (spa_vdev_exit(spa, vd, txg, error)); 6536 } 6537 6538 /* 6539 * The virtual dRAID spares must be added after vdev tree is created 6540 * and the vdev guids are generated. The guid of their associated 6541 * dRAID is stored in the config and used when opening the spare. 6542 */ 6543 if ((error = vdev_draid_spare_create(nvroot, vd, &ndraid, 6544 rvd->vdev_children)) == 0) { 6545 if (ndraid > 0 && nvlist_lookup_nvlist_array(nvroot, 6546 ZPOOL_CONFIG_SPARES, &spares, &nspares) != 0) 6547 nspares = 0; 6548 } else { 6549 return (spa_vdev_exit(spa, vd, txg, error)); 6550 } 6551 6552 /* 6553 * We must validate the spares and l2cache devices after checking the 6554 * children. Otherwise, vdev_inuse() will blindly overwrite the spare. 6555 */ 6556 if ((error = spa_validate_aux(spa, nvroot, txg, VDEV_ALLOC_ADD)) != 0) 6557 return (spa_vdev_exit(spa, vd, txg, error)); 6558 6559 /* 6560 * If we are in the middle of a device removal, we can only add 6561 * devices which match the existing devices in the pool. 6562 * If we are in the middle of a removal, or have some indirect 6563 * vdevs, we can not add raidz or dRAID top levels. 6564 */ 6565 if (spa->spa_vdev_removal != NULL || 6566 spa->spa_removing_phys.sr_prev_indirect_vdev != -1) { 6567 for (int c = 0; c < vd->vdev_children; c++) { 6568 tvd = vd->vdev_child[c]; 6569 if (spa->spa_vdev_removal != NULL && 6570 tvd->vdev_ashift != spa->spa_max_ashift) { 6571 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6572 } 6573 /* Fail if top level vdev is raidz or a dRAID */ 6574 if (vdev_get_nparity(tvd) != 0) 6575 return (spa_vdev_exit(spa, vd, txg, EINVAL)); 6576 6577 /* 6578 * Need the top level mirror to be 6579 * a mirror of leaf vdevs only 6580 */ 6581 if (tvd->vdev_ops == &vdev_mirror_ops) { 6582 for (uint64_t cid = 0; 6583 cid < tvd->vdev_children; cid++) { 6584 vdev_t *cvd = tvd->vdev_child[cid]; 6585 if (!cvd->vdev_ops->vdev_op_leaf) { 6586 return (spa_vdev_exit(spa, vd, 6587 txg, EINVAL)); 6588 } 6589 } 6590 } 6591 } 6592 } 6593 6594 for (int c = 0; c < vd->vdev_children; c++) { 6595 tvd = vd->vdev_child[c]; 6596 vdev_remove_child(vd, tvd); 6597 tvd->vdev_id = rvd->vdev_children; 6598 vdev_add_child(rvd, tvd); 6599 vdev_config_dirty(tvd); 6600 } 6601 6602 if (nspares != 0) { 6603 spa_set_aux_vdevs(&spa->spa_spares, spares, nspares, 6604 ZPOOL_CONFIG_SPARES); 6605 spa_load_spares(spa); 6606 spa->spa_spares.sav_sync = B_TRUE; 6607 } 6608 6609 if (nl2cache != 0) { 6610 spa_set_aux_vdevs(&spa->spa_l2cache, l2cache, nl2cache, 6611 ZPOOL_CONFIG_L2CACHE); 6612 spa_load_l2cache(spa); 6613 spa->spa_l2cache.sav_sync = B_TRUE; 6614 } 6615 6616 /* 6617 * We can't increment a feature while holding spa_vdev so we 6618 * have to do it in a synctask. 6619 */ 6620 if (ndraid != 0) { 6621 dmu_tx_t *tx; 6622 6623 tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg); 6624 dsl_sync_task_nowait(spa->spa_dsl_pool, spa_draid_feature_incr, 6625 (void *)(uintptr_t)ndraid, tx); 6626 dmu_tx_commit(tx); 6627 } 6628 6629 /* 6630 * We have to be careful when adding new vdevs to an existing pool. 6631 * If other threads start allocating from these vdevs before we 6632 * sync the config cache, and we lose power, then upon reboot we may 6633 * fail to open the pool because there are DVAs that the config cache 6634 * can't translate. Therefore, we first add the vdevs without 6635 * initializing metaslabs; sync the config cache (via spa_vdev_exit()); 6636 * and then let spa_config_update() initialize the new metaslabs. 6637 * 6638 * spa_load() checks for added-but-not-initialized vdevs, so that 6639 * if we lose power at any point in this sequence, the remaining 6640 * steps will be completed the next time we load the pool. 6641 */ 6642 (void) spa_vdev_exit(spa, vd, txg, 0); 6643 6644 mutex_enter(&spa_namespace_lock); 6645 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 6646 spa_event_notify(spa, NULL, NULL, ESC_ZFS_VDEV_ADD); 6647 mutex_exit(&spa_namespace_lock); 6648 6649 return (0); 6650} 6651 6652/* 6653 * Attach a device to a mirror. The arguments are the path to any device 6654 * in the mirror, and the nvroot for the new device. If the path specifies 6655 * a device that is not mirrored, we automatically insert the mirror vdev. 6656 * 6657 * If 'replacing' is specified, the new device is intended to replace the 6658 * existing device; in this case the two devices are made into their own 6659 * mirror using the 'replacing' vdev, which is functionally identical to 6660 * the mirror vdev (it actually reuses all the same ops) but has a few 6661 * extra rules: you can't attach to it after it's been created, and upon 6662 * completion of resilvering, the first disk (the one being replaced) 6663 * is automatically detached. 6664 * 6665 * If 'rebuild' is specified, then sequential reconstruction (a.ka. rebuild) 6666 * should be performed instead of traditional healing reconstruction. From 6667 * an administrators perspective these are both resilver operations. 6668 */ 6669int 6670spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing, 6671 int rebuild) 6672{ 6673 uint64_t txg, dtl_max_txg; 6674 vdev_t *rvd = spa->spa_root_vdev; 6675 vdev_t *oldvd, *newvd, *newrootvd, *pvd, *tvd; 6676 vdev_ops_t *pvops; 6677 char *oldvdpath, *newvdpath; 6678 int newvd_isspare; 6679 int error; 6680 6681 ASSERT(spa_writeable(spa)); 6682 6683 txg = spa_vdev_enter(spa); 6684 6685 oldvd = spa_lookup_by_guid(spa, guid, B_FALSE); 6686 6687 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6688 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6689 error = (spa_has_checkpoint(spa)) ? 6690 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6691 return (spa_vdev_exit(spa, NULL, txg, error)); 6692 } 6693 6694 if (rebuild) { 6695 if (!spa_feature_is_enabled(spa, SPA_FEATURE_DEVICE_REBUILD)) 6696 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6697 6698 if (dsl_scan_resilvering(spa_get_dsl(spa))) 6699 return (spa_vdev_exit(spa, NULL, txg, 6700 ZFS_ERR_RESILVER_IN_PROGRESS)); 6701 } else { 6702 if (vdev_rebuild_active(rvd)) 6703 return (spa_vdev_exit(spa, NULL, txg, 6704 ZFS_ERR_REBUILD_IN_PROGRESS)); 6705 } 6706 6707 if (spa->spa_vdev_removal != NULL) 6708 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6709 6710 if (oldvd == NULL) 6711 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6712 6713 if (!oldvd->vdev_ops->vdev_op_leaf) 6714 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6715 6716 pvd = oldvd->vdev_parent; 6717 6718 if ((error = spa_config_parse(spa, &newrootvd, nvroot, NULL, 0, 6719 VDEV_ALLOC_ATTACH)) != 0) 6720 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 6721 6722 if (newrootvd->vdev_children != 1) 6723 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6724 6725 newvd = newrootvd->vdev_child[0]; 6726 6727 if (!newvd->vdev_ops->vdev_op_leaf) 6728 return (spa_vdev_exit(spa, newrootvd, txg, EINVAL)); 6729 6730 if ((error = vdev_create(newrootvd, txg, replacing)) != 0) 6731 return (spa_vdev_exit(spa, newrootvd, txg, error)); 6732 6733 /* 6734 * Spares can't replace logs 6735 */ 6736 if (oldvd->vdev_top->vdev_islog && newvd->vdev_isspare) 6737 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6738 6739 /* 6740 * A dRAID spare can only replace a child of its parent dRAID vdev. 6741 */ 6742 if (newvd->vdev_ops == &vdev_draid_spare_ops && 6743 oldvd->vdev_top != vdev_draid_spare_get_parent(newvd)) { 6744 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6745 } 6746 6747 if (rebuild) { 6748 /* 6749 * For rebuilds, the top vdev must support reconstruction 6750 * using only space maps. This means the only allowable 6751 * vdevs types are the root vdev, a mirror, or dRAID. 6752 */ 6753 tvd = pvd; 6754 if (pvd->vdev_top != NULL) 6755 tvd = pvd->vdev_top; 6756 6757 if (tvd->vdev_ops != &vdev_mirror_ops && 6758 tvd->vdev_ops != &vdev_root_ops && 6759 tvd->vdev_ops != &vdev_draid_ops) { 6760 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6761 } 6762 } 6763 6764 if (!replacing) { 6765 /* 6766 * For attach, the only allowable parent is a mirror or the root 6767 * vdev. 6768 */ 6769 if (pvd->vdev_ops != &vdev_mirror_ops && 6770 pvd->vdev_ops != &vdev_root_ops) 6771 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6772 6773 pvops = &vdev_mirror_ops; 6774 } else { 6775 /* 6776 * Active hot spares can only be replaced by inactive hot 6777 * spares. 6778 */ 6779 if (pvd->vdev_ops == &vdev_spare_ops && 6780 oldvd->vdev_isspare && 6781 !spa_has_spare(spa, newvd->vdev_guid)) 6782 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6783 6784 /* 6785 * If the source is a hot spare, and the parent isn't already a 6786 * spare, then we want to create a new hot spare. Otherwise, we 6787 * want to create a replacing vdev. The user is not allowed to 6788 * attach to a spared vdev child unless the 'isspare' state is 6789 * the same (spare replaces spare, non-spare replaces 6790 * non-spare). 6791 */ 6792 if (pvd->vdev_ops == &vdev_replacing_ops && 6793 spa_version(spa) < SPA_VERSION_MULTI_REPLACE) { 6794 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6795 } else if (pvd->vdev_ops == &vdev_spare_ops && 6796 newvd->vdev_isspare != oldvd->vdev_isspare) { 6797 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6798 } 6799 6800 if (newvd->vdev_isspare) 6801 pvops = &vdev_spare_ops; 6802 else 6803 pvops = &vdev_replacing_ops; 6804 } 6805 6806 /* 6807 * Make sure the new device is big enough. 6808 */ 6809 if (newvd->vdev_asize < vdev_get_min_asize(oldvd)) 6810 return (spa_vdev_exit(spa, newrootvd, txg, EOVERFLOW)); 6811 6812 /* 6813 * The new device cannot have a higher alignment requirement 6814 * than the top-level vdev. 6815 */ 6816 if (newvd->vdev_ashift > oldvd->vdev_top->vdev_ashift) 6817 return (spa_vdev_exit(spa, newrootvd, txg, ENOTSUP)); 6818 6819 /* 6820 * If this is an in-place replacement, update oldvd's path and devid 6821 * to make it distinguishable from newvd, and unopenable from now on. 6822 */ 6823 if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { 6824 spa_strfree(oldvd->vdev_path); 6825 oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, 6826 KM_SLEEP); 6827 (void) snprintf(oldvd->vdev_path, strlen(newvd->vdev_path) + 5, 6828 "%s/%s", newvd->vdev_path, "old"); 6829 if (oldvd->vdev_devid != NULL) { 6830 spa_strfree(oldvd->vdev_devid); 6831 oldvd->vdev_devid = NULL; 6832 } 6833 } 6834 6835 /* 6836 * If the parent is not a mirror, or if we're replacing, insert the new 6837 * mirror/replacing/spare vdev above oldvd. 6838 */ 6839 if (pvd->vdev_ops != pvops) 6840 pvd = vdev_add_parent(oldvd, pvops); 6841 6842 ASSERT(pvd->vdev_top->vdev_parent == rvd); 6843 ASSERT(pvd->vdev_ops == pvops); 6844 ASSERT(oldvd->vdev_parent == pvd); 6845 6846 /* 6847 * Extract the new device from its root and add it to pvd. 6848 */ 6849 vdev_remove_child(newrootvd, newvd); 6850 newvd->vdev_id = pvd->vdev_children; 6851 newvd->vdev_crtxg = oldvd->vdev_crtxg; 6852 vdev_add_child(pvd, newvd); 6853 6854 /* 6855 * Reevaluate the parent vdev state. 6856 */ 6857 vdev_propagate_state(pvd); 6858 6859 tvd = newvd->vdev_top; 6860 ASSERT(pvd->vdev_top == tvd); 6861 ASSERT(tvd->vdev_parent == rvd); 6862 6863 vdev_config_dirty(tvd); 6864 6865 /* 6866 * Set newvd's DTL to [TXG_INITIAL, dtl_max_txg) so that we account 6867 * for any dmu_sync-ed blocks. It will propagate upward when 6868 * spa_vdev_exit() calls vdev_dtl_reassess(). 6869 */ 6870 dtl_max_txg = txg + TXG_CONCURRENT_STATES; 6871 6872 vdev_dtl_dirty(newvd, DTL_MISSING, 6873 TXG_INITIAL, dtl_max_txg - TXG_INITIAL); 6874 6875 if (newvd->vdev_isspare) { 6876 spa_spare_activate(newvd); 6877 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_SPARE); 6878 } 6879 6880 oldvdpath = spa_strdup(oldvd->vdev_path); 6881 newvdpath = spa_strdup(newvd->vdev_path); 6882 newvd_isspare = newvd->vdev_isspare; 6883 6884 /* 6885 * Mark newvd's DTL dirty in this txg. 6886 */ 6887 vdev_dirty(tvd, VDD_DTL, newvd, txg); 6888 6889 /* 6890 * Schedule the resilver or rebuild to restart in the future. We do 6891 * this to ensure that dmu_sync-ed blocks have been stitched into the 6892 * respective datasets. 6893 */ 6894 if (rebuild) { 6895 newvd->vdev_rebuild_txg = txg; 6896 6897 vdev_rebuild(tvd); 6898 } else { 6899 newvd->vdev_resilver_txg = txg; 6900 6901 if (dsl_scan_resilvering(spa_get_dsl(spa)) && 6902 spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) { 6903 vdev_defer_resilver(newvd); 6904 } else { 6905 dsl_scan_restart_resilver(spa->spa_dsl_pool, 6906 dtl_max_txg); 6907 } 6908 } 6909 6910 if (spa->spa_bootfs) 6911 spa_event_notify(spa, newvd, NULL, ESC_ZFS_BOOTFS_VDEV_ATTACH); 6912 6913 spa_event_notify(spa, newvd, NULL, ESC_ZFS_VDEV_ATTACH); 6914 6915 /* 6916 * Commit the config 6917 */ 6918 (void) spa_vdev_exit(spa, newrootvd, dtl_max_txg, 0); 6919 6920 spa_history_log_internal(spa, "vdev attach", NULL, 6921 "%s vdev=%s %s vdev=%s", 6922 replacing && newvd_isspare ? "spare in" : 6923 replacing ? "replace" : "attach", newvdpath, 6924 replacing ? "for" : "to", oldvdpath); 6925 6926 spa_strfree(oldvdpath); 6927 spa_strfree(newvdpath); 6928 6929 return (0); 6930} 6931 6932/* 6933 * Detach a device from a mirror or replacing vdev. 6934 * 6935 * If 'replace_done' is specified, only detach if the parent 6936 * is a replacing vdev. 6937 */ 6938int 6939spa_vdev_detach(spa_t *spa, uint64_t guid, uint64_t pguid, int replace_done) 6940{ 6941 uint64_t txg; 6942 int error; 6943 vdev_t *rvd __maybe_unused = spa->spa_root_vdev; 6944 vdev_t *vd, *pvd, *cvd, *tvd; 6945 boolean_t unspare = B_FALSE; 6946 uint64_t unspare_guid = 0; 6947 char *vdpath; 6948 6949 ASSERT(spa_writeable(spa)); 6950 6951 txg = spa_vdev_detach_enter(spa, guid); 6952 6953 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 6954 6955 /* 6956 * Besides being called directly from the userland through the 6957 * ioctl interface, spa_vdev_detach() can be potentially called 6958 * at the end of spa_vdev_resilver_done(). 6959 * 6960 * In the regular case, when we have a checkpoint this shouldn't 6961 * happen as we never empty the DTLs of a vdev during the scrub 6962 * [see comment in dsl_scan_done()]. Thus spa_vdev_resilvering_done() 6963 * should never get here when we have a checkpoint. 6964 * 6965 * That said, even in a case when we checkpoint the pool exactly 6966 * as spa_vdev_resilver_done() calls this function everything 6967 * should be fine as the resilver will return right away. 6968 */ 6969 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 6970 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 6971 error = (spa_has_checkpoint(spa)) ? 6972 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 6973 return (spa_vdev_exit(spa, NULL, txg, error)); 6974 } 6975 6976 if (vd == NULL) 6977 return (spa_vdev_exit(spa, NULL, txg, ENODEV)); 6978 6979 if (!vd->vdev_ops->vdev_op_leaf) 6980 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 6981 6982 pvd = vd->vdev_parent; 6983 6984 /* 6985 * If the parent/child relationship is not as expected, don't do it. 6986 * Consider M(A,R(B,C)) -- that is, a mirror of A with a replacing 6987 * vdev that's replacing B with C. The user's intent in replacing 6988 * is to go from M(A,B) to M(A,C). If the user decides to cancel 6989 * the replace by detaching C, the expected behavior is to end up 6990 * M(A,B). But suppose that right after deciding to detach C, 6991 * the replacement of B completes. We would have M(A,C), and then 6992 * ask to detach C, which would leave us with just A -- not what 6993 * the user wanted. To prevent this, we make sure that the 6994 * parent/child relationship hasn't changed -- in this example, 6995 * that C's parent is still the replacing vdev R. 6996 */ 6997 if (pvd->vdev_guid != pguid && pguid != 0) 6998 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 6999 7000 /* 7001 * Only 'replacing' or 'spare' vdevs can be replaced. 7002 */ 7003 if (replace_done && pvd->vdev_ops != &vdev_replacing_ops && 7004 pvd->vdev_ops != &vdev_spare_ops) 7005 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7006 7007 ASSERT(pvd->vdev_ops != &vdev_spare_ops || 7008 spa_version(spa) >= SPA_VERSION_SPARES); 7009 7010 /* 7011 * Only mirror, replacing, and spare vdevs support detach. 7012 */ 7013 if (pvd->vdev_ops != &vdev_replacing_ops && 7014 pvd->vdev_ops != &vdev_mirror_ops && 7015 pvd->vdev_ops != &vdev_spare_ops) 7016 return (spa_vdev_exit(spa, NULL, txg, ENOTSUP)); 7017 7018 /* 7019 * If this device has the only valid copy of some data, 7020 * we cannot safely detach it. 7021 */ 7022 if (vdev_dtl_required(vd)) 7023 return (spa_vdev_exit(spa, NULL, txg, EBUSY)); 7024 7025 ASSERT(pvd->vdev_children >= 2); 7026 7027 /* 7028 * If we are detaching the second disk from a replacing vdev, then 7029 * check to see if we changed the original vdev's path to have "/old" 7030 * at the end in spa_vdev_attach(). If so, undo that change now. 7031 */ 7032 if (pvd->vdev_ops == &vdev_replacing_ops && vd->vdev_id > 0 && 7033 vd->vdev_path != NULL) { 7034 size_t len = strlen(vd->vdev_path); 7035 7036 for (int c = 0; c < pvd->vdev_children; c++) { 7037 cvd = pvd->vdev_child[c]; 7038 7039 if (cvd == vd || cvd->vdev_path == NULL) 7040 continue; 7041 7042 if (strncmp(cvd->vdev_path, vd->vdev_path, len) == 0 && 7043 strcmp(cvd->vdev_path + len, "/old") == 0) { 7044 spa_strfree(cvd->vdev_path); 7045 cvd->vdev_path = spa_strdup(vd->vdev_path); 7046 break; 7047 } 7048 } 7049 } 7050 7051 /* 7052 * If we are detaching the original disk from a normal spare, then it 7053 * implies that the spare should become a real disk, and be removed 7054 * from the active spare list for the pool. dRAID spares on the 7055 * other hand are coupled to the pool and thus should never be removed 7056 * from the spares list. 7057 */ 7058 if (pvd->vdev_ops == &vdev_spare_ops && vd->vdev_id == 0) { 7059 vdev_t *last_cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7060 7061 if (last_cvd->vdev_isspare && 7062 last_cvd->vdev_ops != &vdev_draid_spare_ops) { 7063 unspare = B_TRUE; 7064 } 7065 } 7066 7067 /* 7068 * Erase the disk labels so the disk can be used for other things. 7069 * This must be done after all other error cases are handled, 7070 * but before we disembowel vd (so we can still do I/O to it). 7071 * But if we can't do it, don't treat the error as fatal -- 7072 * it may be that the unwritability of the disk is the reason 7073 * it's being detached! 7074 */ 7075 error = vdev_label_init(vd, 0, VDEV_LABEL_REMOVE); 7076 7077 /* 7078 * Remove vd from its parent and compact the parent's children. 7079 */ 7080 vdev_remove_child(pvd, vd); 7081 vdev_compact_children(pvd); 7082 7083 /* 7084 * Remember one of the remaining children so we can get tvd below. 7085 */ 7086 cvd = pvd->vdev_child[pvd->vdev_children - 1]; 7087 7088 /* 7089 * If we need to remove the remaining child from the list of hot spares, 7090 * do it now, marking the vdev as no longer a spare in the process. 7091 * We must do this before vdev_remove_parent(), because that can 7092 * change the GUID if it creates a new toplevel GUID. For a similar 7093 * reason, we must remove the spare now, in the same txg as the detach; 7094 * otherwise someone could attach a new sibling, change the GUID, and 7095 * the subsequent attempt to spa_vdev_remove(unspare_guid) would fail. 7096 */ 7097 if (unspare) { 7098 ASSERT(cvd->vdev_isspare); 7099 spa_spare_remove(cvd); 7100 unspare_guid = cvd->vdev_guid; 7101 (void) spa_vdev_remove(spa, unspare_guid, B_TRUE); 7102 cvd->vdev_unspare = B_TRUE; 7103 } 7104 7105 /* 7106 * If the parent mirror/replacing vdev only has one child, 7107 * the parent is no longer needed. Remove it from the tree. 7108 */ 7109 if (pvd->vdev_children == 1) { 7110 if (pvd->vdev_ops == &vdev_spare_ops) 7111 cvd->vdev_unspare = B_FALSE; 7112 vdev_remove_parent(cvd); 7113 } 7114 7115 /* 7116 * We don't set tvd until now because the parent we just removed 7117 * may have been the previous top-level vdev. 7118 */ 7119 tvd = cvd->vdev_top; 7120 ASSERT(tvd->vdev_parent == rvd); 7121 7122 /* 7123 * Reevaluate the parent vdev state. 7124 */ 7125 vdev_propagate_state(cvd); 7126 7127 /* 7128 * If the 'autoexpand' property is set on the pool then automatically 7129 * try to expand the size of the pool. For example if the device we 7130 * just detached was smaller than the others, it may be possible to 7131 * add metaslabs (i.e. grow the pool). We need to reopen the vdev 7132 * first so that we can obtain the updated sizes of the leaf vdevs. 7133 */ 7134 if (spa->spa_autoexpand) { 7135 vdev_reopen(tvd); 7136 vdev_expand(tvd, txg); 7137 } 7138 7139 vdev_config_dirty(tvd); 7140 7141 /* 7142 * Mark vd's DTL as dirty in this txg. vdev_dtl_sync() will see that 7143 * vd->vdev_detached is set and free vd's DTL object in syncing context. 7144 * But first make sure we're not on any *other* txg's DTL list, to 7145 * prevent vd from being accessed after it's freed. 7146 */ 7147 vdpath = spa_strdup(vd->vdev_path ? vd->vdev_path : "none"); 7148 for (int t = 0; t < TXG_SIZE; t++) 7149 (void) txg_list_remove_this(&tvd->vdev_dtl_list, vd, t); 7150 vd->vdev_detached = B_TRUE; 7151 vdev_dirty(tvd, VDD_DTL, vd, txg); 7152 7153 spa_event_notify(spa, vd, NULL, ESC_ZFS_VDEV_REMOVE); 7154 spa_notify_waiters(spa); 7155 7156 /* hang on to the spa before we release the lock */ 7157 spa_open_ref(spa, FTAG); 7158 7159 error = spa_vdev_exit(spa, vd, txg, 0); 7160 7161 spa_history_log_internal(spa, "detach", NULL, 7162 "vdev=%s", vdpath); 7163 spa_strfree(vdpath); 7164 7165 /* 7166 * If this was the removal of the original device in a hot spare vdev, 7167 * then we want to go through and remove the device from the hot spare 7168 * list of every other pool. 7169 */ 7170 if (unspare) { 7171 spa_t *altspa = NULL; 7172 7173 mutex_enter(&spa_namespace_lock); 7174 while ((altspa = spa_next(altspa)) != NULL) { 7175 if (altspa->spa_state != POOL_STATE_ACTIVE || 7176 altspa == spa) 7177 continue; 7178 7179 spa_open_ref(altspa, FTAG); 7180 mutex_exit(&spa_namespace_lock); 7181 (void) spa_vdev_remove(altspa, unspare_guid, B_TRUE); 7182 mutex_enter(&spa_namespace_lock); 7183 spa_close(altspa, FTAG); 7184 } 7185 mutex_exit(&spa_namespace_lock); 7186 7187 /* search the rest of the vdevs for spares to remove */ 7188 spa_vdev_resilver_done(spa); 7189 } 7190 7191 /* all done with the spa; OK to release */ 7192 mutex_enter(&spa_namespace_lock); 7193 spa_close(spa, FTAG); 7194 mutex_exit(&spa_namespace_lock); 7195 7196 return (error); 7197} 7198 7199static int 7200spa_vdev_initialize_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7201 list_t *vd_list) 7202{ 7203 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7204 7205 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7206 7207 /* Look up vdev and ensure it's a leaf. */ 7208 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7209 if (vd == NULL || vd->vdev_detached) { 7210 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7211 return (SET_ERROR(ENODEV)); 7212 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7213 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7214 return (SET_ERROR(EINVAL)); 7215 } else if (!vdev_writeable(vd)) { 7216 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7217 return (SET_ERROR(EROFS)); 7218 } 7219 mutex_enter(&vd->vdev_initialize_lock); 7220 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7221 7222 /* 7223 * When we activate an initialize action we check to see 7224 * if the vdev_initialize_thread is NULL. We do this instead 7225 * of using the vdev_initialize_state since there might be 7226 * a previous initialization process which has completed but 7227 * the thread is not exited. 7228 */ 7229 if (cmd_type == POOL_INITIALIZE_START && 7230 (vd->vdev_initialize_thread != NULL || 7231 vd->vdev_top->vdev_removing)) { 7232 mutex_exit(&vd->vdev_initialize_lock); 7233 return (SET_ERROR(EBUSY)); 7234 } else if (cmd_type == POOL_INITIALIZE_CANCEL && 7235 (vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE && 7236 vd->vdev_initialize_state != VDEV_INITIALIZE_SUSPENDED)) { 7237 mutex_exit(&vd->vdev_initialize_lock); 7238 return (SET_ERROR(ESRCH)); 7239 } else if (cmd_type == POOL_INITIALIZE_SUSPEND && 7240 vd->vdev_initialize_state != VDEV_INITIALIZE_ACTIVE) { 7241 mutex_exit(&vd->vdev_initialize_lock); 7242 return (SET_ERROR(ESRCH)); 7243 } 7244 7245 switch (cmd_type) { 7246 case POOL_INITIALIZE_START: 7247 vdev_initialize(vd); 7248 break; 7249 case POOL_INITIALIZE_CANCEL: 7250 vdev_initialize_stop(vd, VDEV_INITIALIZE_CANCELED, vd_list); 7251 break; 7252 case POOL_INITIALIZE_SUSPEND: 7253 vdev_initialize_stop(vd, VDEV_INITIALIZE_SUSPENDED, vd_list); 7254 break; 7255 default: 7256 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7257 } 7258 mutex_exit(&vd->vdev_initialize_lock); 7259 7260 return (0); 7261} 7262 7263int 7264spa_vdev_initialize(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, 7265 nvlist_t *vdev_errlist) 7266{ 7267 int total_errors = 0; 7268 list_t vd_list; 7269 7270 list_create(&vd_list, sizeof (vdev_t), 7271 offsetof(vdev_t, vdev_initialize_node)); 7272 7273 /* 7274 * We hold the namespace lock through the whole function 7275 * to prevent any changes to the pool while we're starting or 7276 * stopping initialization. The config and state locks are held so that 7277 * we can properly assess the vdev state before we commit to 7278 * the initializing operation. 7279 */ 7280 mutex_enter(&spa_namespace_lock); 7281 7282 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7283 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7284 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7285 7286 int error = spa_vdev_initialize_impl(spa, vdev_guid, cmd_type, 7287 &vd_list); 7288 if (error != 0) { 7289 char guid_as_str[MAXNAMELEN]; 7290 7291 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7292 "%llu", (unsigned long long)vdev_guid); 7293 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7294 total_errors++; 7295 } 7296 } 7297 7298 /* Wait for all initialize threads to stop. */ 7299 vdev_initialize_stop_wait(spa, &vd_list); 7300 7301 /* Sync out the initializing state */ 7302 txg_wait_synced(spa->spa_dsl_pool, 0); 7303 mutex_exit(&spa_namespace_lock); 7304 7305 list_destroy(&vd_list); 7306 7307 return (total_errors); 7308} 7309 7310static int 7311spa_vdev_trim_impl(spa_t *spa, uint64_t guid, uint64_t cmd_type, 7312 uint64_t rate, boolean_t partial, boolean_t secure, list_t *vd_list) 7313{ 7314 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7315 7316 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 7317 7318 /* Look up vdev and ensure it's a leaf. */ 7319 vdev_t *vd = spa_lookup_by_guid(spa, guid, B_FALSE); 7320 if (vd == NULL || vd->vdev_detached) { 7321 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7322 return (SET_ERROR(ENODEV)); 7323 } else if (!vd->vdev_ops->vdev_op_leaf || !vdev_is_concrete(vd)) { 7324 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7325 return (SET_ERROR(EINVAL)); 7326 } else if (!vdev_writeable(vd)) { 7327 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7328 return (SET_ERROR(EROFS)); 7329 } else if (!vd->vdev_has_trim) { 7330 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7331 return (SET_ERROR(EOPNOTSUPP)); 7332 } else if (secure && !vd->vdev_has_securetrim) { 7333 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7334 return (SET_ERROR(EOPNOTSUPP)); 7335 } 7336 mutex_enter(&vd->vdev_trim_lock); 7337 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 7338 7339 /* 7340 * When we activate a TRIM action we check to see if the 7341 * vdev_trim_thread is NULL. We do this instead of using the 7342 * vdev_trim_state since there might be a previous TRIM process 7343 * which has completed but the thread is not exited. 7344 */ 7345 if (cmd_type == POOL_TRIM_START && 7346 (vd->vdev_trim_thread != NULL || vd->vdev_top->vdev_removing)) { 7347 mutex_exit(&vd->vdev_trim_lock); 7348 return (SET_ERROR(EBUSY)); 7349 } else if (cmd_type == POOL_TRIM_CANCEL && 7350 (vd->vdev_trim_state != VDEV_TRIM_ACTIVE && 7351 vd->vdev_trim_state != VDEV_TRIM_SUSPENDED)) { 7352 mutex_exit(&vd->vdev_trim_lock); 7353 return (SET_ERROR(ESRCH)); 7354 } else if (cmd_type == POOL_TRIM_SUSPEND && 7355 vd->vdev_trim_state != VDEV_TRIM_ACTIVE) { 7356 mutex_exit(&vd->vdev_trim_lock); 7357 return (SET_ERROR(ESRCH)); 7358 } 7359 7360 switch (cmd_type) { 7361 case POOL_TRIM_START: 7362 vdev_trim(vd, rate, partial, secure); 7363 break; 7364 case POOL_TRIM_CANCEL: 7365 vdev_trim_stop(vd, VDEV_TRIM_CANCELED, vd_list); 7366 break; 7367 case POOL_TRIM_SUSPEND: 7368 vdev_trim_stop(vd, VDEV_TRIM_SUSPENDED, vd_list); 7369 break; 7370 default: 7371 panic("invalid cmd_type %llu", (unsigned long long)cmd_type); 7372 } 7373 mutex_exit(&vd->vdev_trim_lock); 7374 7375 return (0); 7376} 7377 7378/* 7379 * Initiates a manual TRIM for the requested vdevs. This kicks off individual 7380 * TRIM threads for each child vdev. These threads pass over all of the free 7381 * space in the vdev's metaslabs and issues TRIM commands for that space. 7382 */ 7383int 7384spa_vdev_trim(spa_t *spa, nvlist_t *nv, uint64_t cmd_type, uint64_t rate, 7385 boolean_t partial, boolean_t secure, nvlist_t *vdev_errlist) 7386{ 7387 int total_errors = 0; 7388 list_t vd_list; 7389 7390 list_create(&vd_list, sizeof (vdev_t), 7391 offsetof(vdev_t, vdev_trim_node)); 7392 7393 /* 7394 * We hold the namespace lock through the whole function 7395 * to prevent any changes to the pool while we're starting or 7396 * stopping TRIM. The config and state locks are held so that 7397 * we can properly assess the vdev state before we commit to 7398 * the TRIM operation. 7399 */ 7400 mutex_enter(&spa_namespace_lock); 7401 7402 for (nvpair_t *pair = nvlist_next_nvpair(nv, NULL); 7403 pair != NULL; pair = nvlist_next_nvpair(nv, pair)) { 7404 uint64_t vdev_guid = fnvpair_value_uint64(pair); 7405 7406 int error = spa_vdev_trim_impl(spa, vdev_guid, cmd_type, 7407 rate, partial, secure, &vd_list); 7408 if (error != 0) { 7409 char guid_as_str[MAXNAMELEN]; 7410 7411 (void) snprintf(guid_as_str, sizeof (guid_as_str), 7412 "%llu", (unsigned long long)vdev_guid); 7413 fnvlist_add_int64(vdev_errlist, guid_as_str, error); 7414 total_errors++; 7415 } 7416 } 7417 7418 /* Wait for all TRIM threads to stop. */ 7419 vdev_trim_stop_wait(spa, &vd_list); 7420 7421 /* Sync out the TRIM state */ 7422 txg_wait_synced(spa->spa_dsl_pool, 0); 7423 mutex_exit(&spa_namespace_lock); 7424 7425 list_destroy(&vd_list); 7426 7427 return (total_errors); 7428} 7429 7430/* 7431 * Split a set of devices from their mirrors, and create a new pool from them. 7432 */ 7433int 7434spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, 7435 nvlist_t *props, boolean_t exp) 7436{ 7437 int error = 0; 7438 uint64_t txg, *glist; 7439 spa_t *newspa; 7440 uint_t c, children, lastlog; 7441 nvlist_t **child, *nvl, *tmp; 7442 dmu_tx_t *tx; 7443 char *altroot = NULL; 7444 vdev_t *rvd, **vml = NULL; /* vdev modify list */ 7445 boolean_t activate_slog; 7446 7447 ASSERT(spa_writeable(spa)); 7448 7449 txg = spa_vdev_enter(spa); 7450 7451 ASSERT(MUTEX_HELD(&spa_namespace_lock)); 7452 if (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT)) { 7453 error = (spa_has_checkpoint(spa)) ? 7454 ZFS_ERR_CHECKPOINT_EXISTS : ZFS_ERR_DISCARDING_CHECKPOINT; 7455 return (spa_vdev_exit(spa, NULL, txg, error)); 7456 } 7457 7458 /* clear the log and flush everything up to now */ 7459 activate_slog = spa_passivate_log(spa); 7460 (void) spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7461 error = spa_reset_logs(spa); 7462 txg = spa_vdev_config_enter(spa); 7463 7464 if (activate_slog) 7465 spa_activate_log(spa); 7466 7467 if (error != 0) 7468 return (spa_vdev_exit(spa, NULL, txg, error)); 7469 7470 /* check new spa name before going any further */ 7471 if (spa_lookup(newname) != NULL) 7472 return (spa_vdev_exit(spa, NULL, txg, EEXIST)); 7473 7474 /* 7475 * scan through all the children to ensure they're all mirrors 7476 */ 7477 if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvl) != 0 || 7478 nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child, 7479 &children) != 0) 7480 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7481 7482 /* first, check to ensure we've got the right child count */ 7483 rvd = spa->spa_root_vdev; 7484 lastlog = 0; 7485 for (c = 0; c < rvd->vdev_children; c++) { 7486 vdev_t *vd = rvd->vdev_child[c]; 7487 7488 /* don't count the holes & logs as children */ 7489 if (vd->vdev_islog || (vd->vdev_ops != &vdev_indirect_ops && 7490 !vdev_is_concrete(vd))) { 7491 if (lastlog == 0) 7492 lastlog = c; 7493 continue; 7494 } 7495 7496 lastlog = 0; 7497 } 7498 if (children != (lastlog != 0 ? lastlog : rvd->vdev_children)) 7499 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7500 7501 /* next, ensure no spare or cache devices are part of the split */ 7502 if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_SPARES, &tmp) == 0 || 7503 nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) 7504 return (spa_vdev_exit(spa, NULL, txg, EINVAL)); 7505 7506 vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); 7507 glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); 7508 7509 /* then, loop over each vdev and validate it */ 7510 for (c = 0; c < children; c++) { 7511 uint64_t is_hole = 0; 7512 7513 (void) nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_IS_HOLE, 7514 &is_hole); 7515 7516 if (is_hole != 0) { 7517 if (spa->spa_root_vdev->vdev_child[c]->vdev_ishole || 7518 spa->spa_root_vdev->vdev_child[c]->vdev_islog) { 7519 continue; 7520 } else { 7521 error = SET_ERROR(EINVAL); 7522 break; 7523 } 7524 } 7525 7526 /* deal with indirect vdevs */ 7527 if (spa->spa_root_vdev->vdev_child[c]->vdev_ops == 7528 &vdev_indirect_ops) 7529 continue; 7530 7531 /* which disk is going to be split? */ 7532 if (nvlist_lookup_uint64(child[c], ZPOOL_CONFIG_GUID, 7533 &glist[c]) != 0) { 7534 error = SET_ERROR(EINVAL); 7535 break; 7536 } 7537 7538 /* look it up in the spa */ 7539 vml[c] = spa_lookup_by_guid(spa, glist[c], B_FALSE); 7540 if (vml[c] == NULL) { 7541 error = SET_ERROR(ENODEV); 7542 break; 7543 } 7544 7545 /* make sure there's nothing stopping the split */ 7546 if (vml[c]->vdev_parent->vdev_ops != &vdev_mirror_ops || 7547 vml[c]->vdev_islog || 7548 !vdev_is_concrete(vml[c]) || 7549 vml[c]->vdev_isspare || 7550 vml[c]->vdev_isl2cache || 7551 !vdev_writeable(vml[c]) || 7552 vml[c]->vdev_children != 0 || 7553 vml[c]->vdev_state != VDEV_STATE_HEALTHY || 7554 c != spa->spa_root_vdev->vdev_child[c]->vdev_id) { 7555 error = SET_ERROR(EINVAL); 7556 break; 7557 } 7558 7559 if (vdev_dtl_required(vml[c]) || 7560 vdev_resilver_needed(vml[c], NULL, NULL)) { 7561 error = SET_ERROR(EBUSY); 7562 break; 7563 } 7564 7565 /* we need certain info from the top level */ 7566 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_ARRAY, 7567 vml[c]->vdev_top->vdev_ms_array) == 0); 7568 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_METASLAB_SHIFT, 7569 vml[c]->vdev_top->vdev_ms_shift) == 0); 7570 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASIZE, 7571 vml[c]->vdev_top->vdev_asize) == 0); 7572 VERIFY(nvlist_add_uint64(child[c], ZPOOL_CONFIG_ASHIFT, 7573 vml[c]->vdev_top->vdev_ashift) == 0); 7574 7575 /* transfer per-vdev ZAPs */ 7576 ASSERT3U(vml[c]->vdev_leaf_zap, !=, 0); 7577 VERIFY0(nvlist_add_uint64(child[c], 7578 ZPOOL_CONFIG_VDEV_LEAF_ZAP, vml[c]->vdev_leaf_zap)); 7579 7580 ASSERT3U(vml[c]->vdev_top->vdev_top_zap, !=, 0); 7581 VERIFY0(nvlist_add_uint64(child[c], 7582 ZPOOL_CONFIG_VDEV_TOP_ZAP, 7583 vml[c]->vdev_parent->vdev_top_zap)); 7584 } 7585 7586 if (error != 0) { 7587 kmem_free(vml, children * sizeof (vdev_t *)); 7588 kmem_free(glist, children * sizeof (uint64_t)); 7589 return (spa_vdev_exit(spa, NULL, txg, error)); 7590 } 7591 7592 /* stop writers from using the disks */ 7593 for (c = 0; c < children; c++) { 7594 if (vml[c] != NULL) 7595 vml[c]->vdev_offline = B_TRUE; 7596 } 7597 vdev_reopen(spa->spa_root_vdev); 7598 7599 /* 7600 * Temporarily record the splitting vdevs in the spa config. This 7601 * will disappear once the config is regenerated. 7602 */ 7603 VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); 7604 VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, 7605 glist, children) == 0); 7606 kmem_free(glist, children * sizeof (uint64_t)); 7607 7608 mutex_enter(&spa->spa_props_lock); 7609 VERIFY(nvlist_add_nvlist(spa->spa_config, ZPOOL_CONFIG_SPLIT, 7610 nvl) == 0); 7611 mutex_exit(&spa->spa_props_lock); 7612 spa->spa_config_splitting = nvl; 7613 vdev_config_dirty(spa->spa_root_vdev); 7614 7615 /* configure and create the new pool */ 7616 VERIFY(nvlist_add_string(config, ZPOOL_CONFIG_POOL_NAME, newname) == 0); 7617 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_STATE, 7618 exp ? POOL_STATE_EXPORTED : POOL_STATE_ACTIVE) == 0); 7619 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 7620 spa_version(spa)) == 0); 7621 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_TXG, 7622 spa->spa_config_txg) == 0); 7623 VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_POOL_GUID, 7624 spa_generate_guid(NULL)) == 0); 7625 VERIFY0(nvlist_add_boolean(config, ZPOOL_CONFIG_HAS_PER_VDEV_ZAPS)); 7626 (void) nvlist_lookup_string(props, 7627 zpool_prop_to_name(ZPOOL_PROP_ALTROOT), &altroot); 7628 7629 /* add the new pool to the namespace */ 7630 newspa = spa_add(newname, config, altroot); 7631 newspa->spa_avz_action = AVZ_ACTION_REBUILD; 7632 newspa->spa_config_txg = spa->spa_config_txg; 7633 spa_set_log_state(newspa, SPA_LOG_CLEAR); 7634 7635 /* release the spa config lock, retaining the namespace lock */ 7636 spa_vdev_config_exit(spa, NULL, txg, 0, FTAG); 7637 7638 if (zio_injection_enabled) 7639 zio_handle_panic_injection(spa, FTAG, 1); 7640 7641 spa_activate(newspa, spa_mode_global); 7642 spa_async_suspend(newspa); 7643 7644 /* 7645 * Temporarily stop the initializing and TRIM activity. We set the 7646 * state to ACTIVE so that we know to resume initializing or TRIM 7647 * once the split has completed. 7648 */ 7649 list_t vd_initialize_list; 7650 list_create(&vd_initialize_list, sizeof (vdev_t), 7651 offsetof(vdev_t, vdev_initialize_node)); 7652 7653 list_t vd_trim_list; 7654 list_create(&vd_trim_list, sizeof (vdev_t), 7655 offsetof(vdev_t, vdev_trim_node)); 7656 7657 for (c = 0; c < children; c++) { 7658 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7659 mutex_enter(&vml[c]->vdev_initialize_lock); 7660 vdev_initialize_stop(vml[c], 7661 VDEV_INITIALIZE_ACTIVE, &vd_initialize_list); 7662 mutex_exit(&vml[c]->vdev_initialize_lock); 7663 7664 mutex_enter(&vml[c]->vdev_trim_lock); 7665 vdev_trim_stop(vml[c], VDEV_TRIM_ACTIVE, &vd_trim_list); 7666 mutex_exit(&vml[c]->vdev_trim_lock); 7667 } 7668 } 7669 7670 vdev_initialize_stop_wait(spa, &vd_initialize_list); 7671 vdev_trim_stop_wait(spa, &vd_trim_list); 7672 7673 list_destroy(&vd_initialize_list); 7674 list_destroy(&vd_trim_list); 7675 7676 newspa->spa_config_source = SPA_CONFIG_SRC_SPLIT; 7677 newspa->spa_is_splitting = B_TRUE; 7678 7679 /* create the new pool from the disks of the original pool */ 7680 error = spa_load(newspa, SPA_LOAD_IMPORT, SPA_IMPORT_ASSEMBLE); 7681 if (error) 7682 goto out; 7683 7684 /* if that worked, generate a real config for the new pool */ 7685 if (newspa->spa_root_vdev != NULL) { 7686 VERIFY(nvlist_alloc(&newspa->spa_config_splitting, 7687 NV_UNIQUE_NAME, KM_SLEEP) == 0); 7688 VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, 7689 ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); 7690 spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, 7691 B_TRUE)); 7692 } 7693 7694 /* set the props */ 7695 if (props != NULL) { 7696 spa_configfile_set(newspa, props, B_FALSE); 7697 error = spa_prop_set(newspa, props); 7698 if (error) 7699 goto out; 7700 } 7701 7702 /* flush everything */ 7703 txg = spa_vdev_config_enter(newspa); 7704 vdev_config_dirty(newspa->spa_root_vdev); 7705 (void) spa_vdev_config_exit(newspa, NULL, txg, 0, FTAG); 7706 7707 if (zio_injection_enabled) 7708 zio_handle_panic_injection(spa, FTAG, 2); 7709 7710 spa_async_resume(newspa); 7711 7712 /* finally, update the original pool's config */ 7713 txg = spa_vdev_config_enter(spa); 7714 tx = dmu_tx_create_dd(spa_get_dsl(spa)->dp_mos_dir); 7715 error = dmu_tx_assign(tx, TXG_WAIT); 7716 if (error != 0) 7717 dmu_tx_abort(tx); 7718 for (c = 0; c < children; c++) { 7719 if (vml[c] != NULL && vml[c]->vdev_ops != &vdev_indirect_ops) { 7720 vdev_t *tvd = vml[c]->vdev_top; 7721 7722 /* 7723 * Need to be sure the detachable VDEV is not 7724 * on any *other* txg's DTL list to prevent it 7725 * from being accessed after it's freed. 7726 */ 7727 for (int t = 0; t < TXG_SIZE; t++) { 7728 (void) txg_list_remove_this( 7729 &tvd->vdev_dtl_list, vml[c], t); 7730 } 7731 7732 vdev_split(vml[c]); 7733 if (error == 0) 7734 spa_history_log_internal(spa, "detach", tx, 7735 "vdev=%s", vml[c]->vdev_path); 7736 7737 vdev_free(vml[c]); 7738 } 7739 } 7740 spa->spa_avz_action = AVZ_ACTION_REBUILD; 7741 vdev_config_dirty(spa->spa_root_vdev); 7742 spa->spa_config_splitting = NULL; 7743 nvlist_free(nvl); 7744 if (error == 0) 7745 dmu_tx_commit(tx); 7746 (void) spa_vdev_exit(spa, NULL, txg, 0); 7747 7748 if (zio_injection_enabled) 7749 zio_handle_panic_injection(spa, FTAG, 3); 7750 7751 /* split is complete; log a history record */ 7752 spa_history_log_internal(newspa, "split", NULL, 7753 "from pool %s", spa_name(spa)); 7754 7755 newspa->spa_is_splitting = B_FALSE; 7756 kmem_free(vml, children * sizeof (vdev_t *)); 7757 7758 /* if we're not going to mount the filesystems in userland, export */ 7759 if (exp) 7760 error = spa_export_common(newname, POOL_STATE_EXPORTED, NULL, 7761 B_FALSE, B_FALSE); 7762 7763 return (error); 7764 7765out: 7766 spa_unload(newspa); 7767 spa_deactivate(newspa); 7768 spa_remove(newspa); 7769 7770 txg = spa_vdev_config_enter(spa); 7771 7772 /* re-online all offlined disks */ 7773 for (c = 0; c < children; c++) { 7774 if (vml[c] != NULL) 7775 vml[c]->vdev_offline = B_FALSE; 7776 } 7777 7778 /* restart initializing or trimming disks as necessary */ 7779 spa_async_request(spa, SPA_ASYNC_INITIALIZE_RESTART); 7780 spa_async_request(spa, SPA_ASYNC_TRIM_RESTART); 7781 spa_async_request(spa, SPA_ASYNC_AUTOTRIM_RESTART); 7782 7783 vdev_reopen(spa->spa_root_vdev); 7784 7785 nvlist_free(spa->spa_config_splitting); 7786 spa->spa_config_splitting = NULL; 7787 (void) spa_vdev_exit(spa, NULL, txg, error); 7788 7789 kmem_free(vml, children * sizeof (vdev_t *)); 7790 return (error); 7791} 7792 7793/* 7794 * Find any device that's done replacing, or a vdev marked 'unspare' that's 7795 * currently spared, so we can detach it. 7796 */ 7797static vdev_t * 7798spa_vdev_resilver_done_hunt(vdev_t *vd) 7799{ 7800 vdev_t *newvd, *oldvd; 7801 7802 for (int c = 0; c < vd->vdev_children; c++) { 7803 oldvd = spa_vdev_resilver_done_hunt(vd->vdev_child[c]); 7804 if (oldvd != NULL) 7805 return (oldvd); 7806 } 7807 7808 /* 7809 * Check for a completed replacement. We always consider the first 7810 * vdev in the list to be the oldest vdev, and the last one to be 7811 * the newest (see spa_vdev_attach() for how that works). In 7812 * the case where the newest vdev is faulted, we will not automatically 7813 * remove it after a resilver completes. This is OK as it will require 7814 * user intervention to determine which disk the admin wishes to keep. 7815 */ 7816 if (vd->vdev_ops == &vdev_replacing_ops) { 7817 ASSERT(vd->vdev_children > 1); 7818 7819 newvd = vd->vdev_child[vd->vdev_children - 1]; 7820 oldvd = vd->vdev_child[0]; 7821 7822 if (vdev_dtl_empty(newvd, DTL_MISSING) && 7823 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7824 !vdev_dtl_required(oldvd)) 7825 return (oldvd); 7826 } 7827 7828 /* 7829 * Check for a completed resilver with the 'unspare' flag set. 7830 * Also potentially update faulted state. 7831 */ 7832 if (vd->vdev_ops == &vdev_spare_ops) { 7833 vdev_t *first = vd->vdev_child[0]; 7834 vdev_t *last = vd->vdev_child[vd->vdev_children - 1]; 7835 7836 if (last->vdev_unspare) { 7837 oldvd = first; 7838 newvd = last; 7839 } else if (first->vdev_unspare) { 7840 oldvd = last; 7841 newvd = first; 7842 } else { 7843 oldvd = NULL; 7844 } 7845 7846 if (oldvd != NULL && 7847 vdev_dtl_empty(newvd, DTL_MISSING) && 7848 vdev_dtl_empty(newvd, DTL_OUTAGE) && 7849 !vdev_dtl_required(oldvd)) 7850 return (oldvd); 7851 7852 vdev_propagate_state(vd); 7853 7854 /* 7855 * If there are more than two spares attached to a disk, 7856 * and those spares are not required, then we want to 7857 * attempt to free them up now so that they can be used 7858 * by other pools. Once we're back down to a single 7859 * disk+spare, we stop removing them. 7860 */ 7861 if (vd->vdev_children > 2) { 7862 newvd = vd->vdev_child[1]; 7863 7864 if (newvd->vdev_isspare && last->vdev_isspare && 7865 vdev_dtl_empty(last, DTL_MISSING) && 7866 vdev_dtl_empty(last, DTL_OUTAGE) && 7867 !vdev_dtl_required(newvd)) 7868 return (newvd); 7869 } 7870 } 7871 7872 return (NULL); 7873} 7874 7875static void 7876spa_vdev_resilver_done(spa_t *spa) 7877{ 7878 vdev_t *vd, *pvd, *ppvd; 7879 uint64_t guid, sguid, pguid, ppguid; 7880 7881 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7882 7883 while ((vd = spa_vdev_resilver_done_hunt(spa->spa_root_vdev)) != NULL) { 7884 pvd = vd->vdev_parent; 7885 ppvd = pvd->vdev_parent; 7886 guid = vd->vdev_guid; 7887 pguid = pvd->vdev_guid; 7888 ppguid = ppvd->vdev_guid; 7889 sguid = 0; 7890 /* 7891 * If we have just finished replacing a hot spared device, then 7892 * we need to detach the parent's first child (the original hot 7893 * spare) as well. 7894 */ 7895 if (ppvd->vdev_ops == &vdev_spare_ops && pvd->vdev_id == 0 && 7896 ppvd->vdev_children == 2) { 7897 ASSERT(pvd->vdev_ops == &vdev_replacing_ops); 7898 sguid = ppvd->vdev_child[1]->vdev_guid; 7899 } 7900 ASSERT(vd->vdev_resilver_txg == 0 || !vdev_dtl_required(vd)); 7901 7902 spa_config_exit(spa, SCL_ALL, FTAG); 7903 if (spa_vdev_detach(spa, guid, pguid, B_TRUE) != 0) 7904 return; 7905 if (sguid && spa_vdev_detach(spa, sguid, ppguid, B_TRUE) != 0) 7906 return; 7907 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 7908 } 7909 7910 spa_config_exit(spa, SCL_ALL, FTAG); 7911 7912 /* 7913 * If a detach was not performed above replace waiters will not have 7914 * been notified. In which case we must do so now. 7915 */ 7916 spa_notify_waiters(spa); 7917} 7918 7919/* 7920 * Update the stored path or FRU for this vdev. 7921 */ 7922static int 7923spa_vdev_set_common(spa_t *spa, uint64_t guid, const char *value, 7924 boolean_t ispath) 7925{ 7926 vdev_t *vd; 7927 boolean_t sync = B_FALSE; 7928 7929 ASSERT(spa_writeable(spa)); 7930 7931 spa_vdev_state_enter(spa, SCL_ALL); 7932 7933 if ((vd = spa_lookup_by_guid(spa, guid, B_TRUE)) == NULL) 7934 return (spa_vdev_state_exit(spa, NULL, ENOENT)); 7935 7936 if (!vd->vdev_ops->vdev_op_leaf) 7937 return (spa_vdev_state_exit(spa, NULL, ENOTSUP)); 7938 7939 if (ispath) { 7940 if (strcmp(value, vd->vdev_path) != 0) { 7941 spa_strfree(vd->vdev_path); 7942 vd->vdev_path = spa_strdup(value); 7943 sync = B_TRUE; 7944 } 7945 } else { 7946 if (vd->vdev_fru == NULL) { 7947 vd->vdev_fru = spa_strdup(value); 7948 sync = B_TRUE; 7949 } else if (strcmp(value, vd->vdev_fru) != 0) { 7950 spa_strfree(vd->vdev_fru); 7951 vd->vdev_fru = spa_strdup(value); 7952 sync = B_TRUE; 7953 } 7954 } 7955 7956 return (spa_vdev_state_exit(spa, sync ? vd : NULL, 0)); 7957} 7958 7959int 7960spa_vdev_setpath(spa_t *spa, uint64_t guid, const char *newpath) 7961{ 7962 return (spa_vdev_set_common(spa, guid, newpath, B_TRUE)); 7963} 7964 7965int 7966spa_vdev_setfru(spa_t *spa, uint64_t guid, const char *newfru) 7967{ 7968 return (spa_vdev_set_common(spa, guid, newfru, B_FALSE)); 7969} 7970 7971/* 7972 * ========================================================================== 7973 * SPA Scanning 7974 * ========================================================================== 7975 */ 7976int 7977spa_scrub_pause_resume(spa_t *spa, pool_scrub_cmd_t cmd) 7978{ 7979 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7980 7981 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7982 return (SET_ERROR(EBUSY)); 7983 7984 return (dsl_scrub_set_pause_resume(spa->spa_dsl_pool, cmd)); 7985} 7986 7987int 7988spa_scan_stop(spa_t *spa) 7989{ 7990 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 7991 if (dsl_scan_resilvering(spa->spa_dsl_pool)) 7992 return (SET_ERROR(EBUSY)); 7993 return (dsl_scan_cancel(spa->spa_dsl_pool)); 7994} 7995 7996int 7997spa_scan(spa_t *spa, pool_scan_func_t func) 7998{ 7999 ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == 0); 8000 8001 if (func >= POOL_SCAN_FUNCS || func == POOL_SCAN_NONE) 8002 return (SET_ERROR(ENOTSUP)); 8003 8004 if (func == POOL_SCAN_RESILVER && 8005 !spa_feature_is_enabled(spa, SPA_FEATURE_RESILVER_DEFER)) 8006 return (SET_ERROR(ENOTSUP)); 8007 8008 /* 8009 * If a resilver was requested, but there is no DTL on a 8010 * writeable leaf device, we have nothing to do. 8011 */ 8012 if (func == POOL_SCAN_RESILVER && 8013 !vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) { 8014 spa_async_request(spa, SPA_ASYNC_RESILVER_DONE); 8015 return (0); 8016 } 8017 8018 return (dsl_scan(spa->spa_dsl_pool, func)); 8019} 8020 8021/* 8022 * ========================================================================== 8023 * SPA async task processing 8024 * ========================================================================== 8025 */ 8026 8027static void 8028spa_async_remove(spa_t *spa, vdev_t *vd) 8029{ 8030 if (vd->vdev_remove_wanted) { 8031 vd->vdev_remove_wanted = B_FALSE; 8032 vd->vdev_delayed_close = B_FALSE; 8033 vdev_set_state(vd, B_FALSE, VDEV_STATE_REMOVED, VDEV_AUX_NONE); 8034 8035 /* 8036 * We want to clear the stats, but we don't want to do a full 8037 * vdev_clear() as that will cause us to throw away 8038 * degraded/faulted state as well as attempt to reopen the 8039 * device, all of which is a waste. 8040 */ 8041 vd->vdev_stat.vs_read_errors = 0; 8042 vd->vdev_stat.vs_write_errors = 0; 8043 vd->vdev_stat.vs_checksum_errors = 0; 8044 8045 vdev_state_dirty(vd->vdev_top); 8046 8047 /* Tell userspace that the vdev is gone. */ 8048 zfs_post_remove(spa, vd); 8049 } 8050 8051 for (int c = 0; c < vd->vdev_children; c++) 8052 spa_async_remove(spa, vd->vdev_child[c]); 8053} 8054 8055static void 8056spa_async_probe(spa_t *spa, vdev_t *vd) 8057{ 8058 if (vd->vdev_probe_wanted) { 8059 vd->vdev_probe_wanted = B_FALSE; 8060 vdev_reopen(vd); /* vdev_open() does the actual probe */ 8061 } 8062 8063 for (int c = 0; c < vd->vdev_children; c++) 8064 spa_async_probe(spa, vd->vdev_child[c]); 8065} 8066 8067static void 8068spa_async_autoexpand(spa_t *spa, vdev_t *vd) 8069{ 8070 if (!spa->spa_autoexpand) 8071 return; 8072 8073 for (int c = 0; c < vd->vdev_children; c++) { 8074 vdev_t *cvd = vd->vdev_child[c]; 8075 spa_async_autoexpand(spa, cvd); 8076 } 8077 8078 if (!vd->vdev_ops->vdev_op_leaf || vd->vdev_physpath == NULL) 8079 return; 8080 8081 spa_event_notify(vd->vdev_spa, vd, NULL, ESC_ZFS_VDEV_AUTOEXPAND); 8082} 8083 8084static void 8085spa_async_thread(void *arg) 8086{ 8087 spa_t *spa = (spa_t *)arg; 8088 dsl_pool_t *dp = spa->spa_dsl_pool; 8089 int tasks; 8090 8091 ASSERT(spa->spa_sync_on); 8092 8093 mutex_enter(&spa->spa_async_lock); 8094 tasks = spa->spa_async_tasks; 8095 spa->spa_async_tasks = 0; 8096 mutex_exit(&spa->spa_async_lock); 8097 8098 /* 8099 * See if the config needs to be updated. 8100 */ 8101 if (tasks & SPA_ASYNC_CONFIG_UPDATE) { 8102 uint64_t old_space, new_space; 8103 8104 mutex_enter(&spa_namespace_lock); 8105 old_space = metaslab_class_get_space(spa_normal_class(spa)); 8106 old_space += metaslab_class_get_space(spa_special_class(spa)); 8107 old_space += metaslab_class_get_space(spa_dedup_class(spa)); 8108 old_space += metaslab_class_get_space( 8109 spa_embedded_log_class(spa)); 8110 8111 spa_config_update(spa, SPA_CONFIG_UPDATE_POOL); 8112 8113 new_space = metaslab_class_get_space(spa_normal_class(spa)); 8114 new_space += metaslab_class_get_space(spa_special_class(spa)); 8115 new_space += metaslab_class_get_space(spa_dedup_class(spa)); 8116 new_space += metaslab_class_get_space( 8117 spa_embedded_log_class(spa)); 8118 mutex_exit(&spa_namespace_lock); 8119 8120 /* 8121 * If the pool grew as a result of the config update, 8122 * then log an internal history event. 8123 */ 8124 if (new_space != old_space) { 8125 spa_history_log_internal(spa, "vdev online", NULL, 8126 "pool '%s' size: %llu(+%llu)", 8127 spa_name(spa), (u_longlong_t)new_space, 8128 (u_longlong_t)(new_space - old_space)); 8129 } 8130 } 8131 8132 /* 8133 * See if any devices need to be marked REMOVED. 8134 */ 8135 if (tasks & SPA_ASYNC_REMOVE) { 8136 spa_vdev_state_enter(spa, SCL_NONE); 8137 spa_async_remove(spa, spa->spa_root_vdev); 8138 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) 8139 spa_async_remove(spa, spa->spa_l2cache.sav_vdevs[i]); 8140 for (int i = 0; i < spa->spa_spares.sav_count; i++) 8141 spa_async_remove(spa, spa->spa_spares.sav_vdevs[i]); 8142 (void) spa_vdev_state_exit(spa, NULL, 0); 8143 } 8144 8145 if ((tasks & SPA_ASYNC_AUTOEXPAND) && !spa_suspended(spa)) { 8146 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8147 spa_async_autoexpand(spa, spa->spa_root_vdev); 8148 spa_config_exit(spa, SCL_CONFIG, FTAG); 8149 } 8150 8151 /* 8152 * See if any devices need to be probed. 8153 */ 8154 if (tasks & SPA_ASYNC_PROBE) { 8155 spa_vdev_state_enter(spa, SCL_NONE); 8156 spa_async_probe(spa, spa->spa_root_vdev); 8157 (void) spa_vdev_state_exit(spa, NULL, 0); 8158 } 8159 8160 /* 8161 * If any devices are done replacing, detach them. 8162 */ 8163 if (tasks & SPA_ASYNC_RESILVER_DONE || 8164 tasks & SPA_ASYNC_REBUILD_DONE) { 8165 spa_vdev_resilver_done(spa); 8166 } 8167 8168 /* 8169 * Kick off a resilver. 8170 */ 8171 if (tasks & SPA_ASYNC_RESILVER && 8172 !vdev_rebuild_active(spa->spa_root_vdev) && 8173 (!dsl_scan_resilvering(dp) || 8174 !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_RESILVER_DEFER))) 8175 dsl_scan_restart_resilver(dp, 0); 8176 8177 if (tasks & SPA_ASYNC_INITIALIZE_RESTART) { 8178 mutex_enter(&spa_namespace_lock); 8179 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8180 vdev_initialize_restart(spa->spa_root_vdev); 8181 spa_config_exit(spa, SCL_CONFIG, FTAG); 8182 mutex_exit(&spa_namespace_lock); 8183 } 8184 8185 if (tasks & SPA_ASYNC_TRIM_RESTART) { 8186 mutex_enter(&spa_namespace_lock); 8187 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8188 vdev_trim_restart(spa->spa_root_vdev); 8189 spa_config_exit(spa, SCL_CONFIG, FTAG); 8190 mutex_exit(&spa_namespace_lock); 8191 } 8192 8193 if (tasks & SPA_ASYNC_AUTOTRIM_RESTART) { 8194 mutex_enter(&spa_namespace_lock); 8195 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8196 vdev_autotrim_restart(spa); 8197 spa_config_exit(spa, SCL_CONFIG, FTAG); 8198 mutex_exit(&spa_namespace_lock); 8199 } 8200 8201 /* 8202 * Kick off L2 cache whole device TRIM. 8203 */ 8204 if (tasks & SPA_ASYNC_L2CACHE_TRIM) { 8205 mutex_enter(&spa_namespace_lock); 8206 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 8207 vdev_trim_l2arc(spa); 8208 spa_config_exit(spa, SCL_CONFIG, FTAG); 8209 mutex_exit(&spa_namespace_lock); 8210 } 8211 8212 /* 8213 * Kick off L2 cache rebuilding. 8214 */ 8215 if (tasks & SPA_ASYNC_L2CACHE_REBUILD) { 8216 mutex_enter(&spa_namespace_lock); 8217 spa_config_enter(spa, SCL_L2ARC, FTAG, RW_READER); 8218 l2arc_spa_rebuild_start(spa); 8219 spa_config_exit(spa, SCL_L2ARC, FTAG); 8220 mutex_exit(&spa_namespace_lock); 8221 } 8222 8223 /* 8224 * Let the world know that we're done. 8225 */ 8226 mutex_enter(&spa->spa_async_lock); 8227 spa->spa_async_thread = NULL; 8228 cv_broadcast(&spa->spa_async_cv); 8229 mutex_exit(&spa->spa_async_lock); 8230 thread_exit(); 8231} 8232 8233void 8234spa_async_suspend(spa_t *spa) 8235{ 8236 mutex_enter(&spa->spa_async_lock); 8237 spa->spa_async_suspended++; 8238 while (spa->spa_async_thread != NULL) 8239 cv_wait(&spa->spa_async_cv, &spa->spa_async_lock); 8240 mutex_exit(&spa->spa_async_lock); 8241 8242 spa_vdev_remove_suspend(spa); 8243 8244 zthr_t *condense_thread = spa->spa_condense_zthr; 8245 if (condense_thread != NULL) 8246 zthr_cancel(condense_thread); 8247 8248 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8249 if (discard_thread != NULL) 8250 zthr_cancel(discard_thread); 8251 8252 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8253 if (ll_delete_thread != NULL) 8254 zthr_cancel(ll_delete_thread); 8255 8256 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8257 if (ll_condense_thread != NULL) 8258 zthr_cancel(ll_condense_thread); 8259} 8260 8261void 8262spa_async_resume(spa_t *spa) 8263{ 8264 mutex_enter(&spa->spa_async_lock); 8265 ASSERT(spa->spa_async_suspended != 0); 8266 spa->spa_async_suspended--; 8267 mutex_exit(&spa->spa_async_lock); 8268 spa_restart_removal(spa); 8269 8270 zthr_t *condense_thread = spa->spa_condense_zthr; 8271 if (condense_thread != NULL) 8272 zthr_resume(condense_thread); 8273 8274 zthr_t *discard_thread = spa->spa_checkpoint_discard_zthr; 8275 if (discard_thread != NULL) 8276 zthr_resume(discard_thread); 8277 8278 zthr_t *ll_delete_thread = spa->spa_livelist_delete_zthr; 8279 if (ll_delete_thread != NULL) 8280 zthr_resume(ll_delete_thread); 8281 8282 zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr; 8283 if (ll_condense_thread != NULL) 8284 zthr_resume(ll_condense_thread); 8285} 8286 8287static boolean_t 8288spa_async_tasks_pending(spa_t *spa) 8289{ 8290 uint_t non_config_tasks; 8291 uint_t config_task; 8292 boolean_t config_task_suspended; 8293 8294 non_config_tasks = spa->spa_async_tasks & ~SPA_ASYNC_CONFIG_UPDATE; 8295 config_task = spa->spa_async_tasks & SPA_ASYNC_CONFIG_UPDATE; 8296 if (spa->spa_ccw_fail_time == 0) { 8297 config_task_suspended = B_FALSE; 8298 } else { 8299 config_task_suspended = 8300 (gethrtime() - spa->spa_ccw_fail_time) < 8301 ((hrtime_t)zfs_ccw_retry_interval * NANOSEC); 8302 } 8303 8304 return (non_config_tasks || (config_task && !config_task_suspended)); 8305} 8306 8307static void 8308spa_async_dispatch(spa_t *spa) 8309{ 8310 mutex_enter(&spa->spa_async_lock); 8311 if (spa_async_tasks_pending(spa) && 8312 !spa->spa_async_suspended && 8313 spa->spa_async_thread == NULL) 8314 spa->spa_async_thread = thread_create(NULL, 0, 8315 spa_async_thread, spa, 0, &p0, TS_RUN, maxclsyspri); 8316 mutex_exit(&spa->spa_async_lock); 8317} 8318 8319void 8320spa_async_request(spa_t *spa, int task) 8321{ 8322 zfs_dbgmsg("spa=%s async request task=%u", spa->spa_name, task); 8323 mutex_enter(&spa->spa_async_lock); 8324 spa->spa_async_tasks |= task; 8325 mutex_exit(&spa->spa_async_lock); 8326} 8327 8328int 8329spa_async_tasks(spa_t *spa) 8330{ 8331 return (spa->spa_async_tasks); 8332} 8333 8334/* 8335 * ========================================================================== 8336 * SPA syncing routines 8337 * ========================================================================== 8338 */ 8339 8340 8341static int 8342bpobj_enqueue_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8343 dmu_tx_t *tx) 8344{ 8345 bpobj_t *bpo = arg; 8346 bpobj_enqueue(bpo, bp, bp_freed, tx); 8347 return (0); 8348} 8349 8350int 8351bpobj_enqueue_alloc_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8352{ 8353 return (bpobj_enqueue_cb(arg, bp, B_FALSE, tx)); 8354} 8355 8356int 8357bpobj_enqueue_free_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8358{ 8359 return (bpobj_enqueue_cb(arg, bp, B_TRUE, tx)); 8360} 8361 8362static int 8363spa_free_sync_cb(void *arg, const blkptr_t *bp, dmu_tx_t *tx) 8364{ 8365 zio_t *pio = arg; 8366 8367 zio_nowait(zio_free_sync(pio, pio->io_spa, dmu_tx_get_txg(tx), bp, 8368 pio->io_flags)); 8369 return (0); 8370} 8371 8372static int 8373bpobj_spa_free_sync_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, 8374 dmu_tx_t *tx) 8375{ 8376 ASSERT(!bp_freed); 8377 return (spa_free_sync_cb(arg, bp, tx)); 8378} 8379 8380/* 8381 * Note: this simple function is not inlined to make it easier to dtrace the 8382 * amount of time spent syncing frees. 8383 */ 8384static void 8385spa_sync_frees(spa_t *spa, bplist_t *bpl, dmu_tx_t *tx) 8386{ 8387 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8388 bplist_iterate(bpl, spa_free_sync_cb, zio, tx); 8389 VERIFY(zio_wait(zio) == 0); 8390} 8391 8392/* 8393 * Note: this simple function is not inlined to make it easier to dtrace the 8394 * amount of time spent syncing deferred frees. 8395 */ 8396static void 8397spa_sync_deferred_frees(spa_t *spa, dmu_tx_t *tx) 8398{ 8399 if (spa_sync_pass(spa) != 1) 8400 return; 8401 8402 /* 8403 * Note: 8404 * If the log space map feature is active, we stop deferring 8405 * frees to the next TXG and therefore running this function 8406 * would be considered a no-op as spa_deferred_bpobj should 8407 * not have any entries. 8408 * 8409 * That said we run this function anyway (instead of returning 8410 * immediately) for the edge-case scenario where we just 8411 * activated the log space map feature in this TXG but we have 8412 * deferred frees from the previous TXG. 8413 */ 8414 zio_t *zio = zio_root(spa, NULL, NULL, 0); 8415 VERIFY3U(bpobj_iterate(&spa->spa_deferred_bpobj, 8416 bpobj_spa_free_sync_cb, zio, tx), ==, 0); 8417 VERIFY0(zio_wait(zio)); 8418} 8419 8420static void 8421spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) 8422{ 8423 char *packed = NULL; 8424 size_t bufsize; 8425 size_t nvsize = 0; 8426 dmu_buf_t *db; 8427 8428 VERIFY(nvlist_size(nv, &nvsize, NV_ENCODE_XDR) == 0); 8429 8430 /* 8431 * Write full (SPA_CONFIG_BLOCKSIZE) blocks of configuration 8432 * information. This avoids the dmu_buf_will_dirty() path and 8433 * saves us a pre-read to get data we don't actually care about. 8434 */ 8435 bufsize = P2ROUNDUP((uint64_t)nvsize, SPA_CONFIG_BLOCKSIZE); 8436 packed = vmem_alloc(bufsize, KM_SLEEP); 8437 8438 VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, 8439 KM_SLEEP) == 0); 8440 bzero(packed + nvsize, bufsize - nvsize); 8441 8442 dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); 8443 8444 vmem_free(packed, bufsize); 8445 8446 VERIFY(0 == dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db)); 8447 dmu_buf_will_dirty(db, tx); 8448 *(uint64_t *)db->db_data = nvsize; 8449 dmu_buf_rele(db, FTAG); 8450} 8451 8452static void 8453spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, 8454 const char *config, const char *entry) 8455{ 8456 nvlist_t *nvroot; 8457 nvlist_t **list; 8458 int i; 8459 8460 if (!sav->sav_sync) 8461 return; 8462 8463 /* 8464 * Update the MOS nvlist describing the list of available devices. 8465 * spa_validate_aux() will have already made sure this nvlist is 8466 * valid and the vdevs are labeled appropriately. 8467 */ 8468 if (sav->sav_object == 0) { 8469 sav->sav_object = dmu_object_alloc(spa->spa_meta_objset, 8470 DMU_OT_PACKED_NVLIST, 1 << 14, DMU_OT_PACKED_NVLIST_SIZE, 8471 sizeof (uint64_t), tx); 8472 VERIFY(zap_update(spa->spa_meta_objset, 8473 DMU_POOL_DIRECTORY_OBJECT, entry, sizeof (uint64_t), 1, 8474 &sav->sav_object, tx) == 0); 8475 } 8476 8477 VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); 8478 if (sav->sav_count == 0) { 8479 VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); 8480 } else { 8481 list = kmem_alloc(sav->sav_count*sizeof (void *), KM_SLEEP); 8482 for (i = 0; i < sav->sav_count; i++) 8483 list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], 8484 B_FALSE, VDEV_CONFIG_L2CACHE); 8485 VERIFY(nvlist_add_nvlist_array(nvroot, config, list, 8486 sav->sav_count) == 0); 8487 for (i = 0; i < sav->sav_count; i++) 8488 nvlist_free(list[i]); 8489 kmem_free(list, sav->sav_count * sizeof (void *)); 8490 } 8491 8492 spa_sync_nvlist(spa, sav->sav_object, nvroot, tx); 8493 nvlist_free(nvroot); 8494 8495 sav->sav_sync = B_FALSE; 8496} 8497 8498/* 8499 * Rebuild spa's all-vdev ZAP from the vdev ZAPs indicated in each vdev_t. 8500 * The all-vdev ZAP must be empty. 8501 */ 8502static void 8503spa_avz_build(vdev_t *vd, uint64_t avz, dmu_tx_t *tx) 8504{ 8505 spa_t *spa = vd->vdev_spa; 8506 8507 if (vd->vdev_top_zap != 0) { 8508 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8509 vd->vdev_top_zap, tx)); 8510 } 8511 if (vd->vdev_leaf_zap != 0) { 8512 VERIFY0(zap_add_int(spa->spa_meta_objset, avz, 8513 vd->vdev_leaf_zap, tx)); 8514 } 8515 for (uint64_t i = 0; i < vd->vdev_children; i++) { 8516 spa_avz_build(vd->vdev_child[i], avz, tx); 8517 } 8518} 8519 8520static void 8521spa_sync_config_object(spa_t *spa, dmu_tx_t *tx) 8522{ 8523 nvlist_t *config; 8524 8525 /* 8526 * If the pool is being imported from a pre-per-vdev-ZAP version of ZFS, 8527 * its config may not be dirty but we still need to build per-vdev ZAPs. 8528 * Similarly, if the pool is being assembled (e.g. after a split), we 8529 * need to rebuild the AVZ although the config may not be dirty. 8530 */ 8531 if (list_is_empty(&spa->spa_config_dirty_list) && 8532 spa->spa_avz_action == AVZ_ACTION_NONE) 8533 return; 8534 8535 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 8536 8537 ASSERT(spa->spa_avz_action == AVZ_ACTION_NONE || 8538 spa->spa_avz_action == AVZ_ACTION_INITIALIZE || 8539 spa->spa_all_vdev_zaps != 0); 8540 8541 if (spa->spa_avz_action == AVZ_ACTION_REBUILD) { 8542 /* Make and build the new AVZ */ 8543 uint64_t new_avz = zap_create(spa->spa_meta_objset, 8544 DMU_OTN_ZAP_METADATA, DMU_OT_NONE, 0, tx); 8545 spa_avz_build(spa->spa_root_vdev, new_avz, tx); 8546 8547 /* Diff old AVZ with new one */ 8548 zap_cursor_t zc; 8549 zap_attribute_t za; 8550 8551 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8552 spa->spa_all_vdev_zaps); 8553 zap_cursor_retrieve(&zc, &za) == 0; 8554 zap_cursor_advance(&zc)) { 8555 uint64_t vdzap = za.za_first_integer; 8556 if (zap_lookup_int(spa->spa_meta_objset, new_avz, 8557 vdzap) == ENOENT) { 8558 /* 8559 * ZAP is listed in old AVZ but not in new one; 8560 * destroy it 8561 */ 8562 VERIFY0(zap_destroy(spa->spa_meta_objset, vdzap, 8563 tx)); 8564 } 8565 } 8566 8567 zap_cursor_fini(&zc); 8568 8569 /* Destroy the old AVZ */ 8570 VERIFY0(zap_destroy(spa->spa_meta_objset, 8571 spa->spa_all_vdev_zaps, tx)); 8572 8573 /* Replace the old AVZ in the dir obj with the new one */ 8574 VERIFY0(zap_update(spa->spa_meta_objset, 8575 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, 8576 sizeof (new_avz), 1, &new_avz, tx)); 8577 8578 spa->spa_all_vdev_zaps = new_avz; 8579 } else if (spa->spa_avz_action == AVZ_ACTION_DESTROY) { 8580 zap_cursor_t zc; 8581 zap_attribute_t za; 8582 8583 /* Walk through the AVZ and destroy all listed ZAPs */ 8584 for (zap_cursor_init(&zc, spa->spa_meta_objset, 8585 spa->spa_all_vdev_zaps); 8586 zap_cursor_retrieve(&zc, &za) == 0; 8587 zap_cursor_advance(&zc)) { 8588 uint64_t zap = za.za_first_integer; 8589 VERIFY0(zap_destroy(spa->spa_meta_objset, zap, tx)); 8590 } 8591 8592 zap_cursor_fini(&zc); 8593 8594 /* Destroy and unlink the AVZ itself */ 8595 VERIFY0(zap_destroy(spa->spa_meta_objset, 8596 spa->spa_all_vdev_zaps, tx)); 8597 VERIFY0(zap_remove(spa->spa_meta_objset, 8598 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_VDEV_ZAP_MAP, tx)); 8599 spa->spa_all_vdev_zaps = 0; 8600 } 8601 8602 if (spa->spa_all_vdev_zaps == 0) { 8603 spa->spa_all_vdev_zaps = zap_create_link(spa->spa_meta_objset, 8604 DMU_OTN_ZAP_METADATA, DMU_POOL_DIRECTORY_OBJECT, 8605 DMU_POOL_VDEV_ZAP_MAP, tx); 8606 } 8607 spa->spa_avz_action = AVZ_ACTION_NONE; 8608 8609 /* Create ZAPs for vdevs that don't have them. */ 8610 vdev_construct_zaps(spa->spa_root_vdev, tx); 8611 8612 config = spa_config_generate(spa, spa->spa_root_vdev, 8613 dmu_tx_get_txg(tx), B_FALSE); 8614 8615 /* 8616 * If we're upgrading the spa version then make sure that 8617 * the config object gets updated with the correct version. 8618 */ 8619 if (spa->spa_ubsync.ub_version < spa->spa_uberblock.ub_version) 8620 fnvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, 8621 spa->spa_uberblock.ub_version); 8622 8623 spa_config_exit(spa, SCL_STATE, FTAG); 8624 8625 nvlist_free(spa->spa_config_syncing); 8626 spa->spa_config_syncing = config; 8627 8628 spa_sync_nvlist(spa, spa->spa_config_object, config, tx); 8629} 8630 8631static void 8632spa_sync_version(void *arg, dmu_tx_t *tx) 8633{ 8634 uint64_t *versionp = arg; 8635 uint64_t version = *versionp; 8636 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8637 8638 /* 8639 * Setting the version is special cased when first creating the pool. 8640 */ 8641 ASSERT(tx->tx_txg != TXG_INITIAL); 8642 8643 ASSERT(SPA_VERSION_IS_SUPPORTED(version)); 8644 ASSERT(version >= spa_version(spa)); 8645 8646 spa->spa_uberblock.ub_version = version; 8647 vdev_config_dirty(spa->spa_root_vdev); 8648 spa_history_log_internal(spa, "set", tx, "version=%lld", 8649 (longlong_t)version); 8650} 8651 8652/* 8653 * Set zpool properties. 8654 */ 8655static void 8656spa_sync_props(void *arg, dmu_tx_t *tx) 8657{ 8658 nvlist_t *nvp = arg; 8659 spa_t *spa = dmu_tx_pool(tx)->dp_spa; 8660 objset_t *mos = spa->spa_meta_objset; 8661 nvpair_t *elem = NULL; 8662 8663 mutex_enter(&spa->spa_props_lock); 8664 8665 while ((elem = nvlist_next_nvpair(nvp, elem))) { 8666 uint64_t intval; 8667 char *strval, *fname; 8668 zpool_prop_t prop; 8669 const char *propname; 8670 zprop_type_t proptype; 8671 spa_feature_t fid; 8672 8673 switch (prop = zpool_name_to_prop(nvpair_name(elem))) { 8674 case ZPOOL_PROP_INVAL: 8675 /* 8676 * We checked this earlier in spa_prop_validate(). 8677 */ 8678 ASSERT(zpool_prop_feature(nvpair_name(elem))); 8679 8680 fname = strchr(nvpair_name(elem), '@') + 1; 8681 VERIFY0(zfeature_lookup_name(fname, &fid)); 8682 8683 spa_feature_enable(spa, fid, tx); 8684 spa_history_log_internal(spa, "set", tx, 8685 "%s=enabled", nvpair_name(elem)); 8686 break; 8687 8688 case ZPOOL_PROP_VERSION: 8689 intval = fnvpair_value_uint64(elem); 8690 /* 8691 * The version is synced separately before other 8692 * properties and should be correct by now. 8693 */ 8694 ASSERT3U(spa_version(spa), >=, intval); 8695 break; 8696 8697 case ZPOOL_PROP_ALTROOT: 8698 /* 8699 * 'altroot' is a non-persistent property. It should 8700 * have been set temporarily at creation or import time. 8701 */ 8702 ASSERT(spa->spa_root != NULL); 8703 break; 8704 8705 case ZPOOL_PROP_READONLY: 8706 case ZPOOL_PROP_CACHEFILE: 8707 /* 8708 * 'readonly' and 'cachefile' are also non-persistent 8709 * properties. 8710 */ 8711 break; 8712 case ZPOOL_PROP_COMMENT: 8713 strval = fnvpair_value_string(elem); 8714 if (spa->spa_comment != NULL) 8715 spa_strfree(spa->spa_comment); 8716 spa->spa_comment = spa_strdup(strval); 8717 /* 8718 * We need to dirty the configuration on all the vdevs 8719 * so that their labels get updated. It's unnecessary 8720 * to do this for pool creation since the vdev's 8721 * configuration has already been dirtied. 8722 */ 8723 if (tx->tx_txg != TXG_INITIAL) 8724 vdev_config_dirty(spa->spa_root_vdev); 8725 spa_history_log_internal(spa, "set", tx, 8726 "%s=%s", nvpair_name(elem), strval); 8727 break; 8728 case ZPOOL_PROP_COMPATIBILITY: 8729 strval = fnvpair_value_string(elem); 8730 if (spa->spa_compatibility != NULL) 8731 spa_strfree(spa->spa_compatibility); 8732 spa->spa_compatibility = spa_strdup(strval); 8733 /* 8734 * Dirty the configuration on vdevs as above. 8735 */ 8736 if (tx->tx_txg != TXG_INITIAL) 8737 vdev_config_dirty(spa->spa_root_vdev); 8738 spa_history_log_internal(spa, "set", tx, 8739 "%s=%s", nvpair_name(elem), strval); 8740 break; 8741 8742 default: 8743 /* 8744 * Set pool property values in the poolprops mos object. 8745 */ 8746 if (spa->spa_pool_props_object == 0) { 8747 spa->spa_pool_props_object = 8748 zap_create_link(mos, DMU_OT_POOL_PROPS, 8749 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_PROPS, 8750 tx); 8751 } 8752 8753 /* normalize the property name */ 8754 propname = zpool_prop_to_name(prop); 8755 proptype = zpool_prop_get_type(prop); 8756 8757 if (nvpair_type(elem) == DATA_TYPE_STRING) { 8758 ASSERT(proptype == PROP_TYPE_STRING); 8759 strval = fnvpair_value_string(elem); 8760 VERIFY0(zap_update(mos, 8761 spa->spa_pool_props_object, propname, 8762 1, strlen(strval) + 1, strval, tx)); 8763 spa_history_log_internal(spa, "set", tx, 8764 "%s=%s", nvpair_name(elem), strval); 8765 } else if (nvpair_type(elem) == DATA_TYPE_UINT64) { 8766 intval = fnvpair_value_uint64(elem); 8767 8768 if (proptype == PROP_TYPE_INDEX) { 8769 const char *unused; 8770 VERIFY0(zpool_prop_index_to_string( 8771 prop, intval, &unused)); 8772 } 8773 VERIFY0(zap_update(mos, 8774 spa->spa_pool_props_object, propname, 8775 8, 1, &intval, tx)); 8776 spa_history_log_internal(spa, "set", tx, 8777 "%s=%lld", nvpair_name(elem), 8778 (longlong_t)intval); 8779 } else { 8780 ASSERT(0); /* not allowed */ 8781 } 8782 8783 switch (prop) { 8784 case ZPOOL_PROP_DELEGATION: 8785 spa->spa_delegation = intval; 8786 break; 8787 case ZPOOL_PROP_BOOTFS: 8788 spa->spa_bootfs = intval; 8789 break; 8790 case ZPOOL_PROP_FAILUREMODE: 8791 spa->spa_failmode = intval; 8792 break; 8793 case ZPOOL_PROP_AUTOTRIM: 8794 spa->spa_autotrim = intval; 8795 spa_async_request(spa, 8796 SPA_ASYNC_AUTOTRIM_RESTART); 8797 break; 8798 case ZPOOL_PROP_AUTOEXPAND: 8799 spa->spa_autoexpand = intval; 8800 if (tx->tx_txg != TXG_INITIAL) 8801 spa_async_request(spa, 8802 SPA_ASYNC_AUTOEXPAND); 8803 break; 8804 case ZPOOL_PROP_MULTIHOST: 8805 spa->spa_multihost = intval; 8806 break; 8807 default: 8808 break; 8809 } 8810 } 8811 8812 } 8813 8814 mutex_exit(&spa->spa_props_lock); 8815} 8816 8817/* 8818 * Perform one-time upgrade on-disk changes. spa_version() does not 8819 * reflect the new version this txg, so there must be no changes this 8820 * txg to anything that the upgrade code depends on after it executes. 8821 * Therefore this must be called after dsl_pool_sync() does the sync 8822 * tasks. 8823 */ 8824static void 8825spa_sync_upgrades(spa_t *spa, dmu_tx_t *tx) 8826{ 8827 if (spa_sync_pass(spa) != 1) 8828 return; 8829 8830 dsl_pool_t *dp = spa->spa_dsl_pool; 8831 rrw_enter(&dp->dp_config_rwlock, RW_WRITER, FTAG); 8832 8833 if (spa->spa_ubsync.ub_version < SPA_VERSION_ORIGIN && 8834 spa->spa_uberblock.ub_version >= SPA_VERSION_ORIGIN) { 8835 dsl_pool_create_origin(dp, tx); 8836 8837 /* Keeping the origin open increases spa_minref */ 8838 spa->spa_minref += 3; 8839 } 8840 8841 if (spa->spa_ubsync.ub_version < SPA_VERSION_NEXT_CLONES && 8842 spa->spa_uberblock.ub_version >= SPA_VERSION_NEXT_CLONES) { 8843 dsl_pool_upgrade_clones(dp, tx); 8844 } 8845 8846 if (spa->spa_ubsync.ub_version < SPA_VERSION_DIR_CLONES && 8847 spa->spa_uberblock.ub_version >= SPA_VERSION_DIR_CLONES) { 8848 dsl_pool_upgrade_dir_clones(dp, tx); 8849 8850 /* Keeping the freedir open increases spa_minref */ 8851 spa->spa_minref += 3; 8852 } 8853 8854 if (spa->spa_ubsync.ub_version < SPA_VERSION_FEATURES && 8855 spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8856 spa_feature_create_zap_objects(spa, tx); 8857 } 8858 8859 /* 8860 * LZ4_COMPRESS feature's behaviour was changed to activate_on_enable 8861 * when possibility to use lz4 compression for metadata was added 8862 * Old pools that have this feature enabled must be upgraded to have 8863 * this feature active 8864 */ 8865 if (spa->spa_uberblock.ub_version >= SPA_VERSION_FEATURES) { 8866 boolean_t lz4_en = spa_feature_is_enabled(spa, 8867 SPA_FEATURE_LZ4_COMPRESS); 8868 boolean_t lz4_ac = spa_feature_is_active(spa, 8869 SPA_FEATURE_LZ4_COMPRESS); 8870 8871 if (lz4_en && !lz4_ac) 8872 spa_feature_incr(spa, SPA_FEATURE_LZ4_COMPRESS, tx); 8873 } 8874 8875 /* 8876 * If we haven't written the salt, do so now. Note that the 8877 * feature may not be activated yet, but that's fine since 8878 * the presence of this ZAP entry is backwards compatible. 8879 */ 8880 if (zap_contains(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT, 8881 DMU_POOL_CHECKSUM_SALT) == ENOENT) { 8882 VERIFY0(zap_add(spa->spa_meta_objset, 8883 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_CHECKSUM_SALT, 1, 8884 sizeof (spa->spa_cksum_salt.zcs_bytes), 8885 spa->spa_cksum_salt.zcs_bytes, tx)); 8886 } 8887 8888 rrw_exit(&dp->dp_config_rwlock, FTAG); 8889} 8890 8891static void 8892vdev_indirect_state_sync_verify(vdev_t *vd) 8893{ 8894 vdev_indirect_mapping_t *vim __maybe_unused = vd->vdev_indirect_mapping; 8895 vdev_indirect_births_t *vib __maybe_unused = vd->vdev_indirect_births; 8896 8897 if (vd->vdev_ops == &vdev_indirect_ops) { 8898 ASSERT(vim != NULL); 8899 ASSERT(vib != NULL); 8900 } 8901 8902 uint64_t obsolete_sm_object = 0; 8903 ASSERT0(vdev_obsolete_sm_object(vd, &obsolete_sm_object)); 8904 if (obsolete_sm_object != 0) { 8905 ASSERT(vd->vdev_obsolete_sm != NULL); 8906 ASSERT(vd->vdev_removing || 8907 vd->vdev_ops == &vdev_indirect_ops); 8908 ASSERT(vdev_indirect_mapping_num_entries(vim) > 0); 8909 ASSERT(vdev_indirect_mapping_bytes_mapped(vim) > 0); 8910 ASSERT3U(obsolete_sm_object, ==, 8911 space_map_object(vd->vdev_obsolete_sm)); 8912 ASSERT3U(vdev_indirect_mapping_bytes_mapped(vim), >=, 8913 space_map_allocated(vd->vdev_obsolete_sm)); 8914 } 8915 ASSERT(vd->vdev_obsolete_segments != NULL); 8916 8917 /* 8918 * Since frees / remaps to an indirect vdev can only 8919 * happen in syncing context, the obsolete segments 8920 * tree must be empty when we start syncing. 8921 */ 8922 ASSERT0(range_tree_space(vd->vdev_obsolete_segments)); 8923} 8924 8925/* 8926 * Set the top-level vdev's max queue depth. Evaluate each top-level's 8927 * async write queue depth in case it changed. The max queue depth will 8928 * not change in the middle of syncing out this txg. 8929 */ 8930static void 8931spa_sync_adjust_vdev_max_queue_depth(spa_t *spa) 8932{ 8933 ASSERT(spa_writeable(spa)); 8934 8935 vdev_t *rvd = spa->spa_root_vdev; 8936 uint32_t max_queue_depth = zfs_vdev_async_write_max_active * 8937 zfs_vdev_queue_depth_pct / 100; 8938 metaslab_class_t *normal = spa_normal_class(spa); 8939 metaslab_class_t *special = spa_special_class(spa); 8940 metaslab_class_t *dedup = spa_dedup_class(spa); 8941 8942 uint64_t slots_per_allocator = 0; 8943 for (int c = 0; c < rvd->vdev_children; c++) { 8944 vdev_t *tvd = rvd->vdev_child[c]; 8945 8946 metaslab_group_t *mg = tvd->vdev_mg; 8947 if (mg == NULL || !metaslab_group_initialized(mg)) 8948 continue; 8949 8950 metaslab_class_t *mc = mg->mg_class; 8951 if (mc != normal && mc != special && mc != dedup) 8952 continue; 8953 8954 /* 8955 * It is safe to do a lock-free check here because only async 8956 * allocations look at mg_max_alloc_queue_depth, and async 8957 * allocations all happen from spa_sync(). 8958 */ 8959 for (int i = 0; i < mg->mg_allocators; i++) { 8960 ASSERT0(zfs_refcount_count( 8961 &(mg->mg_allocator[i].mga_alloc_queue_depth))); 8962 } 8963 mg->mg_max_alloc_queue_depth = max_queue_depth; 8964 8965 for (int i = 0; i < mg->mg_allocators; i++) { 8966 mg->mg_allocator[i].mga_cur_max_alloc_queue_depth = 8967 zfs_vdev_def_queue_depth; 8968 } 8969 slots_per_allocator += zfs_vdev_def_queue_depth; 8970 } 8971 8972 for (int i = 0; i < spa->spa_alloc_count; i++) { 8973 ASSERT0(zfs_refcount_count(&normal->mc_allocator[i]. 8974 mca_alloc_slots)); 8975 ASSERT0(zfs_refcount_count(&special->mc_allocator[i]. 8976 mca_alloc_slots)); 8977 ASSERT0(zfs_refcount_count(&dedup->mc_allocator[i]. 8978 mca_alloc_slots)); 8979 normal->mc_allocator[i].mca_alloc_max_slots = 8980 slots_per_allocator; 8981 special->mc_allocator[i].mca_alloc_max_slots = 8982 slots_per_allocator; 8983 dedup->mc_allocator[i].mca_alloc_max_slots = 8984 slots_per_allocator; 8985 } 8986 normal->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 8987 special->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 8988 dedup->mc_alloc_throttle_enabled = zio_dva_throttle_enabled; 8989} 8990 8991static void 8992spa_sync_condense_indirect(spa_t *spa, dmu_tx_t *tx) 8993{ 8994 ASSERT(spa_writeable(spa)); 8995 8996 vdev_t *rvd = spa->spa_root_vdev; 8997 for (int c = 0; c < rvd->vdev_children; c++) { 8998 vdev_t *vd = rvd->vdev_child[c]; 8999 vdev_indirect_state_sync_verify(vd); 9000 9001 if (vdev_indirect_should_condense(vd)) { 9002 spa_condense_indirect_start_sync(vd, tx); 9003 break; 9004 } 9005 } 9006} 9007 9008static void 9009spa_sync_iterate_to_convergence(spa_t *spa, dmu_tx_t *tx) 9010{ 9011 objset_t *mos = spa->spa_meta_objset; 9012 dsl_pool_t *dp = spa->spa_dsl_pool; 9013 uint64_t txg = tx->tx_txg; 9014 bplist_t *free_bpl = &spa->spa_free_bplist[txg & TXG_MASK]; 9015 9016 do { 9017 int pass = ++spa->spa_sync_pass; 9018 9019 spa_sync_config_object(spa, tx); 9020 spa_sync_aux_dev(spa, &spa->spa_spares, tx, 9021 ZPOOL_CONFIG_SPARES, DMU_POOL_SPARES); 9022 spa_sync_aux_dev(spa, &spa->spa_l2cache, tx, 9023 ZPOOL_CONFIG_L2CACHE, DMU_POOL_L2CACHE); 9024 spa_errlog_sync(spa, txg); 9025 dsl_pool_sync(dp, txg); 9026 9027 if (pass < zfs_sync_pass_deferred_free || 9028 spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) { 9029 /* 9030 * If the log space map feature is active we don't 9031 * care about deferred frees and the deferred bpobj 9032 * as the log space map should effectively have the 9033 * same results (i.e. appending only to one object). 9034 */ 9035 spa_sync_frees(spa, free_bpl, tx); 9036 } else { 9037 /* 9038 * We can not defer frees in pass 1, because 9039 * we sync the deferred frees later in pass 1. 9040 */ 9041 ASSERT3U(pass, >, 1); 9042 bplist_iterate(free_bpl, bpobj_enqueue_alloc_cb, 9043 &spa->spa_deferred_bpobj, tx); 9044 } 9045 9046 ddt_sync(spa, txg); 9047 dsl_scan_sync(dp, tx); 9048 svr_sync(spa, tx); 9049 spa_sync_upgrades(spa, tx); 9050 9051 spa_flush_metaslabs(spa, tx); 9052 9053 vdev_t *vd = NULL; 9054 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, txg)) 9055 != NULL) 9056 vdev_sync(vd, txg); 9057 9058 /* 9059 * Note: We need to check if the MOS is dirty because we could 9060 * have marked the MOS dirty without updating the uberblock 9061 * (e.g. if we have sync tasks but no dirty user data). We need 9062 * to check the uberblock's rootbp because it is updated if we 9063 * have synced out dirty data (though in this case the MOS will 9064 * most likely also be dirty due to second order effects, we 9065 * don't want to rely on that here). 9066 */ 9067 if (pass == 1 && 9068 spa->spa_uberblock.ub_rootbp.blk_birth < txg && 9069 !dmu_objset_is_dirty(mos, txg)) { 9070 /* 9071 * Nothing changed on the first pass, therefore this 9072 * TXG is a no-op. Avoid syncing deferred frees, so 9073 * that we can keep this TXG as a no-op. 9074 */ 9075 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9076 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9077 ASSERT(txg_list_empty(&dp->dp_sync_tasks, txg)); 9078 ASSERT(txg_list_empty(&dp->dp_early_sync_tasks, txg)); 9079 break; 9080 } 9081 9082 spa_sync_deferred_frees(spa, tx); 9083 } while (dmu_objset_is_dirty(mos, txg)); 9084} 9085 9086/* 9087 * Rewrite the vdev configuration (which includes the uberblock) to 9088 * commit the transaction group. 9089 * 9090 * If there are no dirty vdevs, we sync the uberblock to a few random 9091 * top-level vdevs that are known to be visible in the config cache 9092 * (see spa_vdev_add() for a complete description). If there *are* dirty 9093 * vdevs, sync the uberblock to all vdevs. 9094 */ 9095static void 9096spa_sync_rewrite_vdev_config(spa_t *spa, dmu_tx_t *tx) 9097{ 9098 vdev_t *rvd = spa->spa_root_vdev; 9099 uint64_t txg = tx->tx_txg; 9100 9101 for (;;) { 9102 int error = 0; 9103 9104 /* 9105 * We hold SCL_STATE to prevent vdev open/close/etc. 9106 * while we're attempting to write the vdev labels. 9107 */ 9108 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9109 9110 if (list_is_empty(&spa->spa_config_dirty_list)) { 9111 vdev_t *svd[SPA_SYNC_MIN_VDEVS] = { NULL }; 9112 int svdcount = 0; 9113 int children = rvd->vdev_children; 9114 int c0 = spa_get_random(children); 9115 9116 for (int c = 0; c < children; c++) { 9117 vdev_t *vd = 9118 rvd->vdev_child[(c0 + c) % children]; 9119 9120 /* Stop when revisiting the first vdev */ 9121 if (c > 0 && svd[0] == vd) 9122 break; 9123 9124 if (vd->vdev_ms_array == 0 || 9125 vd->vdev_islog || 9126 !vdev_is_concrete(vd)) 9127 continue; 9128 9129 svd[svdcount++] = vd; 9130 if (svdcount == SPA_SYNC_MIN_VDEVS) 9131 break; 9132 } 9133 error = vdev_config_sync(svd, svdcount, txg); 9134 } else { 9135 error = vdev_config_sync(rvd->vdev_child, 9136 rvd->vdev_children, txg); 9137 } 9138 9139 if (error == 0) 9140 spa->spa_last_synced_guid = rvd->vdev_guid; 9141 9142 spa_config_exit(spa, SCL_STATE, FTAG); 9143 9144 if (error == 0) 9145 break; 9146 zio_suspend(spa, NULL, ZIO_SUSPEND_IOERR); 9147 zio_resume_wait(spa); 9148 } 9149} 9150 9151/* 9152 * Sync the specified transaction group. New blocks may be dirtied as 9153 * part of the process, so we iterate until it converges. 9154 */ 9155void 9156spa_sync(spa_t *spa, uint64_t txg) 9157{ 9158 vdev_t *vd = NULL; 9159 9160 VERIFY(spa_writeable(spa)); 9161 9162 /* 9163 * Wait for i/os issued in open context that need to complete 9164 * before this txg syncs. 9165 */ 9166 (void) zio_wait(spa->spa_txg_zio[txg & TXG_MASK]); 9167 spa->spa_txg_zio[txg & TXG_MASK] = zio_root(spa, NULL, NULL, 9168 ZIO_FLAG_CANFAIL); 9169 9170 /* 9171 * Lock out configuration changes. 9172 */ 9173 spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER); 9174 9175 spa->spa_syncing_txg = txg; 9176 spa->spa_sync_pass = 0; 9177 9178 for (int i = 0; i < spa->spa_alloc_count; i++) { 9179 mutex_enter(&spa->spa_alloc_locks[i]); 9180 VERIFY0(avl_numnodes(&spa->spa_alloc_trees[i])); 9181 mutex_exit(&spa->spa_alloc_locks[i]); 9182 } 9183 9184 /* 9185 * If there are any pending vdev state changes, convert them 9186 * into config changes that go out with this transaction group. 9187 */ 9188 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 9189 while (list_head(&spa->spa_state_dirty_list) != NULL) { 9190 /* 9191 * We need the write lock here because, for aux vdevs, 9192 * calling vdev_config_dirty() modifies sav_config. 9193 * This is ugly and will become unnecessary when we 9194 * eliminate the aux vdev wart by integrating all vdevs 9195 * into the root vdev tree. 9196 */ 9197 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9198 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_WRITER); 9199 while ((vd = list_head(&spa->spa_state_dirty_list)) != NULL) { 9200 vdev_state_clean(vd); 9201 vdev_config_dirty(vd); 9202 } 9203 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9204 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9205 } 9206 spa_config_exit(spa, SCL_STATE, FTAG); 9207 9208 dsl_pool_t *dp = spa->spa_dsl_pool; 9209 dmu_tx_t *tx = dmu_tx_create_assigned(dp, txg); 9210 9211 spa->spa_sync_starttime = gethrtime(); 9212 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9213 spa->spa_deadman_tqid = taskq_dispatch_delay(system_delay_taskq, 9214 spa_deadman, spa, TQ_SLEEP, ddi_get_lbolt() + 9215 NSEC_TO_TICK(spa->spa_deadman_synctime)); 9216 9217 /* 9218 * If we are upgrading to SPA_VERSION_RAIDZ_DEFLATE this txg, 9219 * set spa_deflate if we have no raid-z vdevs. 9220 */ 9221 if (spa->spa_ubsync.ub_version < SPA_VERSION_RAIDZ_DEFLATE && 9222 spa->spa_uberblock.ub_version >= SPA_VERSION_RAIDZ_DEFLATE) { 9223 vdev_t *rvd = spa->spa_root_vdev; 9224 9225 int i; 9226 for (i = 0; i < rvd->vdev_children; i++) { 9227 vd = rvd->vdev_child[i]; 9228 if (vd->vdev_deflate_ratio != SPA_MINBLOCKSIZE) 9229 break; 9230 } 9231 if (i == rvd->vdev_children) { 9232 spa->spa_deflate = TRUE; 9233 VERIFY0(zap_add(spa->spa_meta_objset, 9234 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_DEFLATE, 9235 sizeof (uint64_t), 1, &spa->spa_deflate, tx)); 9236 } 9237 } 9238 9239 spa_sync_adjust_vdev_max_queue_depth(spa); 9240 9241 spa_sync_condense_indirect(spa, tx); 9242 9243 spa_sync_iterate_to_convergence(spa, tx); 9244 9245#ifdef ZFS_DEBUG 9246 if (!list_is_empty(&spa->spa_config_dirty_list)) { 9247 /* 9248 * Make sure that the number of ZAPs for all the vdevs matches 9249 * the number of ZAPs in the per-vdev ZAP list. This only gets 9250 * called if the config is dirty; otherwise there may be 9251 * outstanding AVZ operations that weren't completed in 9252 * spa_sync_config_object. 9253 */ 9254 uint64_t all_vdev_zap_entry_count; 9255 ASSERT0(zap_count(spa->spa_meta_objset, 9256 spa->spa_all_vdev_zaps, &all_vdev_zap_entry_count)); 9257 ASSERT3U(vdev_count_verify_zaps(spa->spa_root_vdev), ==, 9258 all_vdev_zap_entry_count); 9259 } 9260#endif 9261 9262 if (spa->spa_vdev_removal != NULL) { 9263 ASSERT0(spa->spa_vdev_removal->svr_bytes_done[txg & TXG_MASK]); 9264 } 9265 9266 spa_sync_rewrite_vdev_config(spa, tx); 9267 dmu_tx_commit(tx); 9268 9269 taskq_cancel_id(system_delay_taskq, spa->spa_deadman_tqid); 9270 spa->spa_deadman_tqid = 0; 9271 9272 /* 9273 * Clear the dirty config list. 9274 */ 9275 while ((vd = list_head(&spa->spa_config_dirty_list)) != NULL) 9276 vdev_config_clean(vd); 9277 9278 /* 9279 * Now that the new config has synced transactionally, 9280 * let it become visible to the config cache. 9281 */ 9282 if (spa->spa_config_syncing != NULL) { 9283 spa_config_set(spa, spa->spa_config_syncing); 9284 spa->spa_config_txg = txg; 9285 spa->spa_config_syncing = NULL; 9286 } 9287 9288 dsl_pool_sync_done(dp, txg); 9289 9290 for (int i = 0; i < spa->spa_alloc_count; i++) { 9291 mutex_enter(&spa->spa_alloc_locks[i]); 9292 VERIFY0(avl_numnodes(&spa->spa_alloc_trees[i])); 9293 mutex_exit(&spa->spa_alloc_locks[i]); 9294 } 9295 9296 /* 9297 * Update usable space statistics. 9298 */ 9299 while ((vd = txg_list_remove(&spa->spa_vdev_txg_list, TXG_CLEAN(txg))) 9300 != NULL) 9301 vdev_sync_done(vd, txg); 9302 9303 metaslab_class_evict_old(spa->spa_normal_class, txg); 9304 metaslab_class_evict_old(spa->spa_log_class, txg); 9305 9306 spa_sync_close_syncing_log_sm(spa); 9307 9308 spa_update_dspace(spa); 9309 9310 /* 9311 * It had better be the case that we didn't dirty anything 9312 * since vdev_config_sync(). 9313 */ 9314 ASSERT(txg_list_empty(&dp->dp_dirty_datasets, txg)); 9315 ASSERT(txg_list_empty(&dp->dp_dirty_dirs, txg)); 9316 ASSERT(txg_list_empty(&spa->spa_vdev_txg_list, txg)); 9317 9318 while (zfs_pause_spa_sync) 9319 delay(1); 9320 9321 spa->spa_sync_pass = 0; 9322 9323 /* 9324 * Update the last synced uberblock here. We want to do this at 9325 * the end of spa_sync() so that consumers of spa_last_synced_txg() 9326 * will be guaranteed that all the processing associated with 9327 * that txg has been completed. 9328 */ 9329 spa->spa_ubsync = spa->spa_uberblock; 9330 spa_config_exit(spa, SCL_CONFIG, FTAG); 9331 9332 spa_handle_ignored_writes(spa); 9333 9334 /* 9335 * If any async tasks have been requested, kick them off. 9336 */ 9337 spa_async_dispatch(spa); 9338} 9339 9340/* 9341 * Sync all pools. We don't want to hold the namespace lock across these 9342 * operations, so we take a reference on the spa_t and drop the lock during the 9343 * sync. 9344 */ 9345void 9346spa_sync_allpools(void) 9347{ 9348 spa_t *spa = NULL; 9349 mutex_enter(&spa_namespace_lock); 9350 while ((spa = spa_next(spa)) != NULL) { 9351 if (spa_state(spa) != POOL_STATE_ACTIVE || 9352 !spa_writeable(spa) || spa_suspended(spa)) 9353 continue; 9354 spa_open_ref(spa, FTAG); 9355 mutex_exit(&spa_namespace_lock); 9356 txg_wait_synced(spa_get_dsl(spa), 0); 9357 mutex_enter(&spa_namespace_lock); 9358 spa_close(spa, FTAG); 9359 } 9360 mutex_exit(&spa_namespace_lock); 9361} 9362 9363/* 9364 * ========================================================================== 9365 * Miscellaneous routines 9366 * ========================================================================== 9367 */ 9368 9369/* 9370 * Remove all pools in the system. 9371 */ 9372void 9373spa_evict_all(void) 9374{ 9375 spa_t *spa; 9376 9377 /* 9378 * Remove all cached state. All pools should be closed now, 9379 * so every spa in the AVL tree should be unreferenced. 9380 */ 9381 mutex_enter(&spa_namespace_lock); 9382 while ((spa = spa_next(NULL)) != NULL) { 9383 /* 9384 * Stop async tasks. The async thread may need to detach 9385 * a device that's been replaced, which requires grabbing 9386 * spa_namespace_lock, so we must drop it here. 9387 */ 9388 spa_open_ref(spa, FTAG); 9389 mutex_exit(&spa_namespace_lock); 9390 spa_async_suspend(spa); 9391 mutex_enter(&spa_namespace_lock); 9392 spa_close(spa, FTAG); 9393 9394 if (spa->spa_state != POOL_STATE_UNINITIALIZED) { 9395 spa_unload(spa); 9396 spa_deactivate(spa); 9397 } 9398 spa_remove(spa); 9399 } 9400 mutex_exit(&spa_namespace_lock); 9401} 9402 9403vdev_t * 9404spa_lookup_by_guid(spa_t *spa, uint64_t guid, boolean_t aux) 9405{ 9406 vdev_t *vd; 9407 int i; 9408 9409 if ((vd = vdev_lookup_by_guid(spa->spa_root_vdev, guid)) != NULL) 9410 return (vd); 9411 9412 if (aux) { 9413 for (i = 0; i < spa->spa_l2cache.sav_count; i++) { 9414 vd = spa->spa_l2cache.sav_vdevs[i]; 9415 if (vd->vdev_guid == guid) 9416 return (vd); 9417 } 9418 9419 for (i = 0; i < spa->spa_spares.sav_count; i++) { 9420 vd = spa->spa_spares.sav_vdevs[i]; 9421 if (vd->vdev_guid == guid) 9422 return (vd); 9423 } 9424 } 9425 9426 return (NULL); 9427} 9428 9429void 9430spa_upgrade(spa_t *spa, uint64_t version) 9431{ 9432 ASSERT(spa_writeable(spa)); 9433 9434 spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); 9435 9436 /* 9437 * This should only be called for a non-faulted pool, and since a 9438 * future version would result in an unopenable pool, this shouldn't be 9439 * possible. 9440 */ 9441 ASSERT(SPA_VERSION_IS_SUPPORTED(spa->spa_uberblock.ub_version)); 9442 ASSERT3U(version, >=, spa->spa_uberblock.ub_version); 9443 9444 spa->spa_uberblock.ub_version = version; 9445 vdev_config_dirty(spa->spa_root_vdev); 9446 9447 spa_config_exit(spa, SCL_ALL, FTAG); 9448 9449 txg_wait_synced(spa_get_dsl(spa), 0); 9450} 9451 9452boolean_t 9453spa_has_spare(spa_t *spa, uint64_t guid) 9454{ 9455 int i; 9456 uint64_t spareguid; 9457 spa_aux_vdev_t *sav = &spa->spa_spares; 9458 9459 for (i = 0; i < sav->sav_count; i++) 9460 if (sav->sav_vdevs[i]->vdev_guid == guid) 9461 return (B_TRUE); 9462 9463 for (i = 0; i < sav->sav_npending; i++) { 9464 if (nvlist_lookup_uint64(sav->sav_pending[i], ZPOOL_CONFIG_GUID, 9465 &spareguid) == 0 && spareguid == guid) 9466 return (B_TRUE); 9467 } 9468 9469 return (B_FALSE); 9470} 9471 9472/* 9473 * Check if a pool has an active shared spare device. 9474 * Note: reference count of an active spare is 2, as a spare and as a replace 9475 */ 9476static boolean_t 9477spa_has_active_shared_spare(spa_t *spa) 9478{ 9479 int i, refcnt; 9480 uint64_t pool; 9481 spa_aux_vdev_t *sav = &spa->spa_spares; 9482 9483 for (i = 0; i < sav->sav_count; i++) { 9484 if (spa_spare_exists(sav->sav_vdevs[i]->vdev_guid, &pool, 9485 &refcnt) && pool != 0ULL && pool == spa_guid(spa) && 9486 refcnt > 2) 9487 return (B_TRUE); 9488 } 9489 9490 return (B_FALSE); 9491} 9492 9493uint64_t 9494spa_total_metaslabs(spa_t *spa) 9495{ 9496 vdev_t *rvd = spa->spa_root_vdev; 9497 9498 uint64_t m = 0; 9499 for (uint64_t c = 0; c < rvd->vdev_children; c++) { 9500 vdev_t *vd = rvd->vdev_child[c]; 9501 if (!vdev_is_concrete(vd)) 9502 continue; 9503 m += vd->vdev_ms_count; 9504 } 9505 return (m); 9506} 9507 9508/* 9509 * Notify any waiting threads that some activity has switched from being in- 9510 * progress to not-in-progress so that the thread can wake up and determine 9511 * whether it is finished waiting. 9512 */ 9513void 9514spa_notify_waiters(spa_t *spa) 9515{ 9516 /* 9517 * Acquiring spa_activities_lock here prevents the cv_broadcast from 9518 * happening between the waiting thread's check and cv_wait. 9519 */ 9520 mutex_enter(&spa->spa_activities_lock); 9521 cv_broadcast(&spa->spa_activities_cv); 9522 mutex_exit(&spa->spa_activities_lock); 9523} 9524 9525/* 9526 * Notify any waiting threads that the pool is exporting, and then block until 9527 * they are finished using the spa_t. 9528 */ 9529void 9530spa_wake_waiters(spa_t *spa) 9531{ 9532 mutex_enter(&spa->spa_activities_lock); 9533 spa->spa_waiters_cancel = B_TRUE; 9534 cv_broadcast(&spa->spa_activities_cv); 9535 while (spa->spa_waiters != 0) 9536 cv_wait(&spa->spa_waiters_cv, &spa->spa_activities_lock); 9537 spa->spa_waiters_cancel = B_FALSE; 9538 mutex_exit(&spa->spa_activities_lock); 9539} 9540 9541/* Whether the vdev or any of its descendants are being initialized/trimmed. */ 9542static boolean_t 9543spa_vdev_activity_in_progress_impl(vdev_t *vd, zpool_wait_activity_t activity) 9544{ 9545 spa_t *spa = vd->vdev_spa; 9546 9547 ASSERT(spa_config_held(spa, SCL_CONFIG | SCL_STATE, RW_READER)); 9548 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9549 ASSERT(activity == ZPOOL_WAIT_INITIALIZE || 9550 activity == ZPOOL_WAIT_TRIM); 9551 9552 kmutex_t *lock = activity == ZPOOL_WAIT_INITIALIZE ? 9553 &vd->vdev_initialize_lock : &vd->vdev_trim_lock; 9554 9555 mutex_exit(&spa->spa_activities_lock); 9556 mutex_enter(lock); 9557 mutex_enter(&spa->spa_activities_lock); 9558 9559 boolean_t in_progress = (activity == ZPOOL_WAIT_INITIALIZE) ? 9560 (vd->vdev_initialize_state == VDEV_INITIALIZE_ACTIVE) : 9561 (vd->vdev_trim_state == VDEV_TRIM_ACTIVE); 9562 mutex_exit(lock); 9563 9564 if (in_progress) 9565 return (B_TRUE); 9566 9567 for (int i = 0; i < vd->vdev_children; i++) { 9568 if (spa_vdev_activity_in_progress_impl(vd->vdev_child[i], 9569 activity)) 9570 return (B_TRUE); 9571 } 9572 9573 return (B_FALSE); 9574} 9575 9576/* 9577 * If use_guid is true, this checks whether the vdev specified by guid is 9578 * being initialized/trimmed. Otherwise, it checks whether any vdev in the pool 9579 * is being initialized/trimmed. The caller must hold the config lock and 9580 * spa_activities_lock. 9581 */ 9582static int 9583spa_vdev_activity_in_progress(spa_t *spa, boolean_t use_guid, uint64_t guid, 9584 zpool_wait_activity_t activity, boolean_t *in_progress) 9585{ 9586 mutex_exit(&spa->spa_activities_lock); 9587 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9588 mutex_enter(&spa->spa_activities_lock); 9589 9590 vdev_t *vd; 9591 if (use_guid) { 9592 vd = spa_lookup_by_guid(spa, guid, B_FALSE); 9593 if (vd == NULL || !vd->vdev_ops->vdev_op_leaf) { 9594 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9595 return (EINVAL); 9596 } 9597 } else { 9598 vd = spa->spa_root_vdev; 9599 } 9600 9601 *in_progress = spa_vdev_activity_in_progress_impl(vd, activity); 9602 9603 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9604 return (0); 9605} 9606 9607/* 9608 * Locking for waiting threads 9609 * --------------------------- 9610 * 9611 * Waiting threads need a way to check whether a given activity is in progress, 9612 * and then, if it is, wait for it to complete. Each activity will have some 9613 * in-memory representation of the relevant on-disk state which can be used to 9614 * determine whether or not the activity is in progress. The in-memory state and 9615 * the locking used to protect it will be different for each activity, and may 9616 * not be suitable for use with a cvar (e.g., some state is protected by the 9617 * config lock). To allow waiting threads to wait without any races, another 9618 * lock, spa_activities_lock, is used. 9619 * 9620 * When the state is checked, both the activity-specific lock (if there is one) 9621 * and spa_activities_lock are held. In some cases, the activity-specific lock 9622 * is acquired explicitly (e.g. the config lock). In others, the locking is 9623 * internal to some check (e.g. bpobj_is_empty). After checking, the waiting 9624 * thread releases the activity-specific lock and, if the activity is in 9625 * progress, then cv_waits using spa_activities_lock. 9626 * 9627 * The waiting thread is woken when another thread, one completing some 9628 * activity, updates the state of the activity and then calls 9629 * spa_notify_waiters, which will cv_broadcast. This 'completing' thread only 9630 * needs to hold its activity-specific lock when updating the state, and this 9631 * lock can (but doesn't have to) be dropped before calling spa_notify_waiters. 9632 * 9633 * Because spa_notify_waiters acquires spa_activities_lock before broadcasting, 9634 * and because it is held when the waiting thread checks the state of the 9635 * activity, it can never be the case that the completing thread both updates 9636 * the activity state and cv_broadcasts in between the waiting thread's check 9637 * and cv_wait. Thus, a waiting thread can never miss a wakeup. 9638 * 9639 * In order to prevent deadlock, when the waiting thread does its check, in some 9640 * cases it will temporarily drop spa_activities_lock in order to acquire the 9641 * activity-specific lock. The order in which spa_activities_lock and the 9642 * activity specific lock are acquired in the waiting thread is determined by 9643 * the order in which they are acquired in the completing thread; if the 9644 * completing thread calls spa_notify_waiters with the activity-specific lock 9645 * held, then the waiting thread must also acquire the activity-specific lock 9646 * first. 9647 */ 9648 9649static int 9650spa_activity_in_progress(spa_t *spa, zpool_wait_activity_t activity, 9651 boolean_t use_tag, uint64_t tag, boolean_t *in_progress) 9652{ 9653 int error = 0; 9654 9655 ASSERT(MUTEX_HELD(&spa->spa_activities_lock)); 9656 9657 switch (activity) { 9658 case ZPOOL_WAIT_CKPT_DISCARD: 9659 *in_progress = 9660 (spa_feature_is_active(spa, SPA_FEATURE_POOL_CHECKPOINT) && 9661 zap_contains(spa_meta_objset(spa), 9662 DMU_POOL_DIRECTORY_OBJECT, DMU_POOL_ZPOOL_CHECKPOINT) == 9663 ENOENT); 9664 break; 9665 case ZPOOL_WAIT_FREE: 9666 *in_progress = ((spa_version(spa) >= SPA_VERSION_DEADLISTS && 9667 !bpobj_is_empty(&spa->spa_dsl_pool->dp_free_bpobj)) || 9668 spa_feature_is_active(spa, SPA_FEATURE_ASYNC_DESTROY) || 9669 spa_livelist_delete_check(spa)); 9670 break; 9671 case ZPOOL_WAIT_INITIALIZE: 9672 case ZPOOL_WAIT_TRIM: 9673 error = spa_vdev_activity_in_progress(spa, use_tag, tag, 9674 activity, in_progress); 9675 break; 9676 case ZPOOL_WAIT_REPLACE: 9677 mutex_exit(&spa->spa_activities_lock); 9678 spa_config_enter(spa, SCL_CONFIG | SCL_STATE, FTAG, RW_READER); 9679 mutex_enter(&spa->spa_activities_lock); 9680 9681 *in_progress = vdev_replace_in_progress(spa->spa_root_vdev); 9682 spa_config_exit(spa, SCL_CONFIG | SCL_STATE, FTAG); 9683 break; 9684 case ZPOOL_WAIT_REMOVE: 9685 *in_progress = (spa->spa_removing_phys.sr_state == 9686 DSS_SCANNING); 9687 break; 9688 case ZPOOL_WAIT_RESILVER: 9689 if ((*in_progress = vdev_rebuild_active(spa->spa_root_vdev))) 9690 break; 9691 /* fall through */ 9692 case ZPOOL_WAIT_SCRUB: 9693 { 9694 boolean_t scanning, paused, is_scrub; 9695 dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan; 9696 9697 is_scrub = (scn->scn_phys.scn_func == POOL_SCAN_SCRUB); 9698 scanning = (scn->scn_phys.scn_state == DSS_SCANNING); 9699 paused = dsl_scan_is_paused_scrub(scn); 9700 *in_progress = (scanning && !paused && 9701 is_scrub == (activity == ZPOOL_WAIT_SCRUB)); 9702 break; 9703 } 9704 default: 9705 panic("unrecognized value for activity %d", activity); 9706 } 9707 9708 return (error); 9709} 9710 9711static int 9712spa_wait_common(const char *pool, zpool_wait_activity_t activity, 9713 boolean_t use_tag, uint64_t tag, boolean_t *waited) 9714{ 9715 /* 9716 * The tag is used to distinguish between instances of an activity. 9717 * 'initialize' and 'trim' are the only activities that we use this for. 9718 * The other activities can only have a single instance in progress in a 9719 * pool at one time, making the tag unnecessary. 9720 * 9721 * There can be multiple devices being replaced at once, but since they 9722 * all finish once resilvering finishes, we don't bother keeping track 9723 * of them individually, we just wait for them all to finish. 9724 */ 9725 if (use_tag && activity != ZPOOL_WAIT_INITIALIZE && 9726 activity != ZPOOL_WAIT_TRIM) 9727 return (EINVAL); 9728 9729 if (activity < 0 || activity >= ZPOOL_WAIT_NUM_ACTIVITIES) 9730 return (EINVAL); 9731 9732 spa_t *spa; 9733 int error = spa_open(pool, &spa, FTAG); 9734 if (error != 0) 9735 return (error); 9736 9737 /* 9738 * Increment the spa's waiter count so that we can call spa_close and 9739 * still ensure that the spa_t doesn't get freed before this thread is 9740 * finished with it when the pool is exported. We want to call spa_close 9741 * before we start waiting because otherwise the additional ref would 9742 * prevent the pool from being exported or destroyed throughout the 9743 * potentially long wait. 9744 */ 9745 mutex_enter(&spa->spa_activities_lock); 9746 spa->spa_waiters++; 9747 spa_close(spa, FTAG); 9748 9749 *waited = B_FALSE; 9750 for (;;) { 9751 boolean_t in_progress; 9752 error = spa_activity_in_progress(spa, activity, use_tag, tag, 9753 &in_progress); 9754 9755 if (error || !in_progress || spa->spa_waiters_cancel) 9756 break; 9757 9758 *waited = B_TRUE; 9759 9760 if (cv_wait_sig(&spa->spa_activities_cv, 9761 &spa->spa_activities_lock) == 0) { 9762 error = EINTR; 9763 break; 9764 } 9765 } 9766 9767 spa->spa_waiters--; 9768 cv_signal(&spa->spa_waiters_cv); 9769 mutex_exit(&spa->spa_activities_lock); 9770 9771 return (error); 9772} 9773 9774/* 9775 * Wait for a particular instance of the specified activity to complete, where 9776 * the instance is identified by 'tag' 9777 */ 9778int 9779spa_wait_tag(const char *pool, zpool_wait_activity_t activity, uint64_t tag, 9780 boolean_t *waited) 9781{ 9782 return (spa_wait_common(pool, activity, B_TRUE, tag, waited)); 9783} 9784 9785/* 9786 * Wait for all instances of the specified activity complete 9787 */ 9788int 9789spa_wait(const char *pool, zpool_wait_activity_t activity, boolean_t *waited) 9790{ 9791 9792 return (spa_wait_common(pool, activity, B_FALSE, 0, waited)); 9793} 9794 9795sysevent_t * 9796spa_event_create(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9797{ 9798 sysevent_t *ev = NULL; 9799#ifdef _KERNEL 9800 nvlist_t *resource; 9801 9802 resource = zfs_event_create(spa, vd, FM_SYSEVENT_CLASS, name, hist_nvl); 9803 if (resource) { 9804 ev = kmem_alloc(sizeof (sysevent_t), KM_SLEEP); 9805 ev->resource = resource; 9806 } 9807#endif 9808 return (ev); 9809} 9810 9811void 9812spa_event_post(sysevent_t *ev) 9813{ 9814#ifdef _KERNEL 9815 if (ev) { 9816 zfs_zevent_post(ev->resource, NULL, zfs_zevent_post_cb); 9817 kmem_free(ev, sizeof (*ev)); 9818 } 9819#endif 9820} 9821 9822/* 9823 * Post a zevent corresponding to the given sysevent. The 'name' must be one 9824 * of the event definitions in sys/sysevent/eventdefs.h. The payload will be 9825 * filled in from the spa and (optionally) the vdev. This doesn't do anything 9826 * in the userland libzpool, as we don't want consumers to misinterpret ztest 9827 * or zdb as real changes. 9828 */ 9829void 9830spa_event_notify(spa_t *spa, vdev_t *vd, nvlist_t *hist_nvl, const char *name) 9831{ 9832 spa_event_post(spa_event_create(spa, vd, hist_nvl, name)); 9833} 9834 9835/* state manipulation functions */ 9836EXPORT_SYMBOL(spa_open); 9837EXPORT_SYMBOL(spa_open_rewind); 9838EXPORT_SYMBOL(spa_get_stats); 9839EXPORT_SYMBOL(spa_create); 9840EXPORT_SYMBOL(spa_import); 9841EXPORT_SYMBOL(spa_tryimport); 9842EXPORT_SYMBOL(spa_destroy); 9843EXPORT_SYMBOL(spa_export); 9844EXPORT_SYMBOL(spa_reset); 9845EXPORT_SYMBOL(spa_async_request); 9846EXPORT_SYMBOL(spa_async_suspend); 9847EXPORT_SYMBOL(spa_async_resume); 9848EXPORT_SYMBOL(spa_inject_addref); 9849EXPORT_SYMBOL(spa_inject_delref); 9850EXPORT_SYMBOL(spa_scan_stat_init); 9851EXPORT_SYMBOL(spa_scan_get_stats); 9852 9853/* device manipulation */ 9854EXPORT_SYMBOL(spa_vdev_add); 9855EXPORT_SYMBOL(spa_vdev_attach); 9856EXPORT_SYMBOL(spa_vdev_detach); 9857EXPORT_SYMBOL(spa_vdev_setpath); 9858EXPORT_SYMBOL(spa_vdev_setfru); 9859EXPORT_SYMBOL(spa_vdev_split_mirror); 9860 9861/* spare statech is global across all pools) */ 9862EXPORT_SYMBOL(spa_spare_add); 9863EXPORT_SYMBOL(spa_spare_remove); 9864EXPORT_SYMBOL(spa_spare_exists); 9865EXPORT_SYMBOL(spa_spare_activate); 9866 9867/* L2ARC statech is global across all pools) */ 9868EXPORT_SYMBOL(spa_l2cache_add); 9869EXPORT_SYMBOL(spa_l2cache_remove); 9870EXPORT_SYMBOL(spa_l2cache_exists); 9871EXPORT_SYMBOL(spa_l2cache_activate); 9872EXPORT_SYMBOL(spa_l2cache_drop); 9873 9874/* scanning */ 9875EXPORT_SYMBOL(spa_scan); 9876EXPORT_SYMBOL(spa_scan_stop); 9877 9878/* spa syncing */ 9879EXPORT_SYMBOL(spa_sync); /* only for DMU use */ 9880EXPORT_SYMBOL(spa_sync_allpools); 9881 9882/* properties */ 9883EXPORT_SYMBOL(spa_prop_set); 9884EXPORT_SYMBOL(spa_prop_get); 9885EXPORT_SYMBOL(spa_prop_clear_bootfs); 9886 9887/* asynchronous event notification */ 9888EXPORT_SYMBOL(spa_event_notify); 9889 9890/* BEGIN CSTYLED */ 9891ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_shift, INT, ZMOD_RW, 9892 "log2 fraction of arc that can be used by inflight I/Os when " 9893 "verifying pool during import"); 9894 9895ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_metadata, INT, ZMOD_RW, 9896 "Set to traverse metadata on pool import"); 9897 9898ZFS_MODULE_PARAM(zfs_spa, spa_, load_verify_data, INT, ZMOD_RW, 9899 "Set to traverse data on pool import"); 9900 9901ZFS_MODULE_PARAM(zfs_spa, spa_, load_print_vdev_tree, INT, ZMOD_RW, 9902 "Print vdev tree to zfs_dbgmsg during pool import"); 9903 9904ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_pct, UINT, ZMOD_RD, 9905 "Percentage of CPUs to run an IO worker thread"); 9906 9907ZFS_MODULE_PARAM(zfs_zio, zio_, taskq_batch_tpq, UINT, ZMOD_RD, 9908 "Number of threads per IO worker taskqueue"); 9909 9910ZFS_MODULE_PARAM(zfs, zfs_, max_missing_tvds, ULONG, ZMOD_RW, 9911 "Allow importing pool with up to this number of missing top-level " 9912 "vdevs (in read-only mode)"); 9913 9914ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_pause, INT, ZMOD_RW, 9915 "Set the livelist condense zthr to pause"); 9916 9917ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_pause, INT, ZMOD_RW, 9918 "Set the livelist condense synctask to pause"); 9919 9920ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, sync_cancel, INT, ZMOD_RW, 9921 "Whether livelist condensing was canceled in the synctask"); 9922 9923ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, zthr_cancel, INT, ZMOD_RW, 9924 "Whether livelist condensing was canceled in the zthr function"); 9925 9926ZFS_MODULE_PARAM(zfs_livelist_condense, zfs_livelist_condense_, new_alloc, INT, ZMOD_RW, 9927 "Whether extra ALLOC blkptrs were added to a livelist entry while it " 9928 "was being condensed"); 9929/* END CSTYLED */ 9930