zil.c revision 213197
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26#include <sys/zfs_context.h> 27#include <sys/spa.h> 28#include <sys/spa_impl.h> 29#include <sys/dmu.h> 30#include <sys/zap.h> 31#include <sys/arc.h> 32#include <sys/stat.h> 33#include <sys/resource.h> 34#include <sys/zil.h> 35#include <sys/zil_impl.h> 36#include <sys/dsl_dataset.h> 37#include <sys/vdev.h> 38#include <sys/dmu_tx.h> 39 40/* 41 * The zfs intent log (ZIL) saves transaction records of system calls 42 * that change the file system in memory with enough information 43 * to be able to replay them. These are stored in memory until 44 * either the DMU transaction group (txg) commits them to the stable pool 45 * and they can be discarded, or they are flushed to the stable log 46 * (also in the pool) due to a fsync, O_DSYNC or other synchronous 47 * requirement. In the event of a panic or power fail then those log 48 * records (transactions) are replayed. 49 * 50 * There is one ZIL per file system. Its on-disk (pool) format consists 51 * of 3 parts: 52 * 53 * - ZIL header 54 * - ZIL blocks 55 * - ZIL records 56 * 57 * A log record holds a system call transaction. Log blocks can 58 * hold many log records and the blocks are chained together. 59 * Each ZIL block contains a block pointer (blkptr_t) to the next 60 * ZIL block in the chain. The ZIL header points to the first 61 * block in the chain. Note there is not a fixed place in the pool 62 * to hold blocks. They are dynamically allocated and freed as 63 * needed from the blocks available. Figure X shows the ZIL structure: 64 */ 65 66/* 67 * This global ZIL switch affects all pools 68 */ 69int zil_disable = 0; /* disable intent logging */ 70SYSCTL_DECL(_vfs_zfs); 71TUNABLE_INT("vfs.zfs.zil_disable", &zil_disable); 72SYSCTL_INT(_vfs_zfs, OID_AUTO, zil_disable, CTLFLAG_RW, &zil_disable, 0, 73 "Disable ZFS Intent Log (ZIL)"); 74 75/* 76 * Tunable parameter for debugging or performance analysis. Setting 77 * zfs_nocacheflush will cause corruption on power loss if a volatile 78 * out-of-order write cache is enabled. 79 */ 80boolean_t zfs_nocacheflush = B_FALSE; 81TUNABLE_INT("vfs.zfs.cache_flush_disable", &zfs_nocacheflush); 82SYSCTL_INT(_vfs_zfs, OID_AUTO, cache_flush_disable, CTLFLAG_RDTUN, 83 &zfs_nocacheflush, 0, "Disable cache flush"); 84 85static kmem_cache_t *zil_lwb_cache; 86 87static int 88zil_dva_compare(const void *x1, const void *x2) 89{ 90 const dva_t *dva1 = x1; 91 const dva_t *dva2 = x2; 92 93 if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 94 return (-1); 95 if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 96 return (1); 97 98 if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 99 return (-1); 100 if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 101 return (1); 102 103 return (0); 104} 105 106static void 107zil_dva_tree_init(avl_tree_t *t) 108{ 109 avl_create(t, zil_dva_compare, sizeof (zil_dva_node_t), 110 offsetof(zil_dva_node_t, zn_node)); 111} 112 113static void 114zil_dva_tree_fini(avl_tree_t *t) 115{ 116 zil_dva_node_t *zn; 117 void *cookie = NULL; 118 119 while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 120 kmem_free(zn, sizeof (zil_dva_node_t)); 121 122 avl_destroy(t); 123} 124 125static int 126zil_dva_tree_add(avl_tree_t *t, dva_t *dva) 127{ 128 zil_dva_node_t *zn; 129 avl_index_t where; 130 131 if (avl_find(t, dva, &where) != NULL) 132 return (EEXIST); 133 134 zn = kmem_alloc(sizeof (zil_dva_node_t), KM_SLEEP); 135 zn->zn_dva = *dva; 136 avl_insert(t, zn, where); 137 138 return (0); 139} 140 141static zil_header_t * 142zil_header_in_syncing_context(zilog_t *zilog) 143{ 144 return ((zil_header_t *)zilog->zl_header); 145} 146 147static void 148zil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 149{ 150 zio_cksum_t *zc = &bp->blk_cksum; 151 152 zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 153 zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 154 zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 155 zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 156} 157 158/* 159 * Read a log block, make sure it's valid, and byteswap it if necessary. 160 */ 161static int 162zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, arc_buf_t **abufpp) 163{ 164 blkptr_t blk = *bp; 165 zbookmark_t zb; 166 uint32_t aflags = ARC_WAIT; 167 int error; 168 169 zb.zb_objset = bp->blk_cksum.zc_word[ZIL_ZC_OBJSET]; 170 zb.zb_object = 0; 171 zb.zb_level = -1; 172 zb.zb_blkid = bp->blk_cksum.zc_word[ZIL_ZC_SEQ]; 173 174 *abufpp = NULL; 175 176 /* 177 * We shouldn't be doing any scrubbing while we're doing log 178 * replay, it's OK to not lock. 179 */ 180 error = arc_read_nolock(NULL, zilog->zl_spa, &blk, 181 arc_getbuf_func, abufpp, ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL | 182 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB, &aflags, &zb); 183 184 if (error == 0) { 185 char *data = (*abufpp)->b_data; 186 uint64_t blksz = BP_GET_LSIZE(bp); 187 zil_trailer_t *ztp = (zil_trailer_t *)(data + blksz) - 1; 188 zio_cksum_t cksum = bp->blk_cksum; 189 190 /* 191 * Validate the checksummed log block. 192 * 193 * Sequence numbers should be... sequential. The checksum 194 * verifier for the next block should be bp's checksum plus 1. 195 * 196 * Also check the log chain linkage and size used. 197 */ 198 cksum.zc_word[ZIL_ZC_SEQ]++; 199 200 if (bcmp(&cksum, &ztp->zit_next_blk.blk_cksum, 201 sizeof (cksum)) || BP_IS_HOLE(&ztp->zit_next_blk) || 202 (ztp->zit_nused > (blksz - sizeof (zil_trailer_t)))) { 203 error = ECKSUM; 204 } 205 206 if (error) { 207 VERIFY(arc_buf_remove_ref(*abufpp, abufpp) == 1); 208 *abufpp = NULL; 209 } 210 } 211 212 dprintf("error %d on %llu:%llu\n", error, zb.zb_objset, zb.zb_blkid); 213 214 return (error); 215} 216 217/* 218 * Parse the intent log, and call parse_func for each valid record within. 219 * Return the highest sequence number. 220 */ 221uint64_t 222zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 223 zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 224{ 225 const zil_header_t *zh = zilog->zl_header; 226 uint64_t claim_seq = zh->zh_claim_seq; 227 uint64_t seq = 0; 228 uint64_t max_seq = 0; 229 blkptr_t blk = zh->zh_log; 230 arc_buf_t *abuf; 231 char *lrbuf, *lrp; 232 zil_trailer_t *ztp; 233 int reclen, error; 234 235 if (BP_IS_HOLE(&blk)) 236 return (max_seq); 237 238 /* 239 * Starting at the block pointed to by zh_log we read the log chain. 240 * For each block in the chain we strongly check that block to 241 * ensure its validity. We stop when an invalid block is found. 242 * For each block pointer in the chain we call parse_blk_func(). 243 * For each record in each valid block we call parse_lr_func(). 244 * If the log has been claimed, stop if we encounter a sequence 245 * number greater than the highest claimed sequence number. 246 */ 247 zil_dva_tree_init(&zilog->zl_dva_tree); 248 for (;;) { 249 seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 250 251 if (claim_seq != 0 && seq > claim_seq) 252 break; 253 254 ASSERT(max_seq < seq); 255 max_seq = seq; 256 257 error = zil_read_log_block(zilog, &blk, &abuf); 258 259 if (parse_blk_func != NULL) 260 parse_blk_func(zilog, &blk, arg, txg); 261 262 if (error) 263 break; 264 265 lrbuf = abuf->b_data; 266 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 267 blk = ztp->zit_next_blk; 268 269 if (parse_lr_func == NULL) { 270 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 271 continue; 272 } 273 274 for (lrp = lrbuf; lrp < lrbuf + ztp->zit_nused; lrp += reclen) { 275 lr_t *lr = (lr_t *)lrp; 276 reclen = lr->lrc_reclen; 277 ASSERT3U(reclen, >=, sizeof (lr_t)); 278 parse_lr_func(zilog, lr, arg, txg); 279 } 280 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 281 } 282 zil_dva_tree_fini(&zilog->zl_dva_tree); 283 284 return (max_seq); 285} 286 287/* ARGSUSED */ 288static void 289zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 290{ 291 spa_t *spa = zilog->zl_spa; 292 int err; 293 294 /* 295 * Claim log block if not already committed and not already claimed. 296 */ 297 if (bp->blk_birth >= first_txg && 298 zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp)) == 0) { 299 err = zio_wait(zio_claim(NULL, spa, first_txg, bp, NULL, NULL, 300 ZIO_FLAG_MUSTSUCCEED)); 301 ASSERT(err == 0); 302 } 303} 304 305static void 306zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 307{ 308 if (lrc->lrc_txtype == TX_WRITE) { 309 lr_write_t *lr = (lr_write_t *)lrc; 310 zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg); 311 } 312} 313 314/* ARGSUSED */ 315static void 316zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 317{ 318 zio_free_blk(zilog->zl_spa, bp, dmu_tx_get_txg(tx)); 319} 320 321static void 322zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 323{ 324 /* 325 * If we previously claimed it, we need to free it. 326 */ 327 if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE) { 328 lr_write_t *lr = (lr_write_t *)lrc; 329 blkptr_t *bp = &lr->lr_blkptr; 330 if (bp->blk_birth >= claim_txg && 331 !zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp))) { 332 (void) arc_free(NULL, zilog->zl_spa, 333 dmu_tx_get_txg(tx), bp, NULL, NULL, ARC_WAIT); 334 } 335 } 336} 337 338/* 339 * Create an on-disk intent log. 340 */ 341static void 342zil_create(zilog_t *zilog) 343{ 344 const zil_header_t *zh = zilog->zl_header; 345 lwb_t *lwb; 346 uint64_t txg = 0; 347 dmu_tx_t *tx = NULL; 348 blkptr_t blk; 349 int error = 0; 350 351 /* 352 * Wait for any previous destroy to complete. 353 */ 354 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 355 356 ASSERT(zh->zh_claim_txg == 0); 357 ASSERT(zh->zh_replay_seq == 0); 358 359 blk = zh->zh_log; 360 361 /* 362 * If we don't already have an initial log block or we have one 363 * but it's the wrong endianness then allocate one. 364 */ 365 if (BP_IS_HOLE(&blk) || BP_SHOULD_BYTESWAP(&blk)) { 366 tx = dmu_tx_create(zilog->zl_os); 367 (void) dmu_tx_assign(tx, TXG_WAIT); 368 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 369 txg = dmu_tx_get_txg(tx); 370 371 if (!BP_IS_HOLE(&blk)) { 372 zio_free_blk(zilog->zl_spa, &blk, txg); 373 BP_ZERO(&blk); 374 } 375 376 error = zio_alloc_blk(zilog->zl_spa, ZIL_MIN_BLKSZ, &blk, 377 NULL, txg); 378 379 if (error == 0) 380 zil_init_log_chain(zilog, &blk); 381 } 382 383 /* 384 * Allocate a log write buffer (lwb) for the first log block. 385 */ 386 if (error == 0) { 387 lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 388 lwb->lwb_zilog = zilog; 389 lwb->lwb_blk = blk; 390 lwb->lwb_nused = 0; 391 lwb->lwb_sz = BP_GET_LSIZE(&lwb->lwb_blk); 392 lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz); 393 lwb->lwb_max_txg = txg; 394 lwb->lwb_zio = NULL; 395 396 mutex_enter(&zilog->zl_lock); 397 list_insert_tail(&zilog->zl_lwb_list, lwb); 398 mutex_exit(&zilog->zl_lock); 399 } 400 401 /* 402 * If we just allocated the first log block, commit our transaction 403 * and wait for zil_sync() to stuff the block poiner into zh_log. 404 * (zh is part of the MOS, so we cannot modify it in open context.) 405 */ 406 if (tx != NULL) { 407 dmu_tx_commit(tx); 408 txg_wait_synced(zilog->zl_dmu_pool, txg); 409 } 410 411 ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 412} 413 414/* 415 * In one tx, free all log blocks and clear the log header. 416 * If keep_first is set, then we're replaying a log with no content. 417 * We want to keep the first block, however, so that the first 418 * synchronous transaction doesn't require a txg_wait_synced() 419 * in zil_create(). We don't need to txg_wait_synced() here either 420 * when keep_first is set, because both zil_create() and zil_destroy() 421 * will wait for any in-progress destroys to complete. 422 */ 423void 424zil_destroy(zilog_t *zilog, boolean_t keep_first) 425{ 426 const zil_header_t *zh = zilog->zl_header; 427 lwb_t *lwb; 428 dmu_tx_t *tx; 429 uint64_t txg; 430 431 /* 432 * Wait for any previous destroy to complete. 433 */ 434 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 435 436 if (BP_IS_HOLE(&zh->zh_log)) 437 return; 438 439 tx = dmu_tx_create(zilog->zl_os); 440 (void) dmu_tx_assign(tx, TXG_WAIT); 441 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 442 txg = dmu_tx_get_txg(tx); 443 444 mutex_enter(&zilog->zl_lock); 445 446 /* 447 * It is possible for the ZIL to get the previously mounted zilog 448 * structure of the same dataset if quickly remounted and the dbuf 449 * eviction has not completed. In this case we can see a non 450 * empty lwb list and keep_first will be set. We fix this by 451 * clearing the keep_first. This will be slower but it's very rare. 452 */ 453 if (!list_is_empty(&zilog->zl_lwb_list) && keep_first) 454 keep_first = B_FALSE; 455 456 ASSERT3U(zilog->zl_destroy_txg, <, txg); 457 zilog->zl_destroy_txg = txg; 458 zilog->zl_keep_first = keep_first; 459 460 if (!list_is_empty(&zilog->zl_lwb_list)) { 461 ASSERT(zh->zh_claim_txg == 0); 462 ASSERT(!keep_first); 463 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 464 list_remove(&zilog->zl_lwb_list, lwb); 465 if (lwb->lwb_buf != NULL) 466 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 467 zio_free_blk(zilog->zl_spa, &lwb->lwb_blk, txg); 468 kmem_cache_free(zil_lwb_cache, lwb); 469 } 470 } else { 471 if (!keep_first) { 472 (void) zil_parse(zilog, zil_free_log_block, 473 zil_free_log_record, tx, zh->zh_claim_txg); 474 } 475 } 476 mutex_exit(&zilog->zl_lock); 477 478 dmu_tx_commit(tx); 479} 480 481/* 482 * return true if the initial log block is not valid 483 */ 484static boolean_t 485zil_empty(zilog_t *zilog) 486{ 487 const zil_header_t *zh = zilog->zl_header; 488 arc_buf_t *abuf = NULL; 489 490 if (BP_IS_HOLE(&zh->zh_log)) 491 return (B_TRUE); 492 493 if (zil_read_log_block(zilog, &zh->zh_log, &abuf) != 0) 494 return (B_TRUE); 495 496 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 497 return (B_FALSE); 498} 499 500int 501zil_claim(char *osname, void *txarg) 502{ 503 dmu_tx_t *tx = txarg; 504 uint64_t first_txg = dmu_tx_get_txg(tx); 505 zilog_t *zilog; 506 zil_header_t *zh; 507 objset_t *os; 508 int error; 509 510 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 511 if (error) { 512 cmn_err(CE_WARN, "can't open objset for %s", osname); 513 return (0); 514 } 515 516 zilog = dmu_objset_zil(os); 517 zh = zil_header_in_syncing_context(zilog); 518 519 if (zilog->zl_spa->spa_log_state == SPA_LOG_CLEAR) { 520 if (!BP_IS_HOLE(&zh->zh_log)) 521 zio_free_blk(zilog->zl_spa, &zh->zh_log, first_txg); 522 BP_ZERO(&zh->zh_log); 523 dsl_dataset_dirty(dmu_objset_ds(os), tx); 524 } 525 526 /* 527 * Record here whether the zil has any records to replay. 528 * If the header block pointer is null or the block points 529 * to the stubby then we know there are no valid log records. 530 * We use the header to store this state as the the zilog gets 531 * freed later in dmu_objset_close(). 532 * The flags (and the rest of the header fields) are cleared in 533 * zil_sync() as a result of a zil_destroy(), after replaying the log. 534 * 535 * Note, the intent log can be empty but still need the 536 * stubby to be claimed. 537 */ 538 if (!zil_empty(zilog)) { 539 zh->zh_flags |= ZIL_REPLAY_NEEDED; 540 dsl_dataset_dirty(dmu_objset_ds(os), tx); 541 } 542 543 /* 544 * Claim all log blocks if we haven't already done so, and remember 545 * the highest claimed sequence number. This ensures that if we can 546 * read only part of the log now (e.g. due to a missing device), 547 * but we can read the entire log later, we will not try to replay 548 * or destroy beyond the last block we successfully claimed. 549 */ 550 ASSERT3U(zh->zh_claim_txg, <=, first_txg); 551 if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 552 zh->zh_claim_txg = first_txg; 553 zh->zh_claim_seq = zil_parse(zilog, zil_claim_log_block, 554 zil_claim_log_record, tx, first_txg); 555 dsl_dataset_dirty(dmu_objset_ds(os), tx); 556 } 557 558 ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 559 dmu_objset_close(os); 560 return (0); 561} 562 563/* 564 * Check the log by walking the log chain. 565 * Checksum errors are ok as they indicate the end of the chain. 566 * Any other error (no device or read failure) returns an error. 567 */ 568/* ARGSUSED */ 569int 570zil_check_log_chain(char *osname, void *txarg) 571{ 572 zilog_t *zilog; 573 zil_header_t *zh; 574 blkptr_t blk; 575 arc_buf_t *abuf; 576 objset_t *os; 577 char *lrbuf; 578 zil_trailer_t *ztp; 579 int error; 580 581 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 582 if (error) { 583 cmn_err(CE_WARN, "can't open objset for %s", osname); 584 return (0); 585 } 586 587 zilog = dmu_objset_zil(os); 588 zh = zil_header_in_syncing_context(zilog); 589 blk = zh->zh_log; 590 if (BP_IS_HOLE(&blk)) { 591 dmu_objset_close(os); 592 return (0); /* no chain */ 593 } 594 595 for (;;) { 596 error = zil_read_log_block(zilog, &blk, &abuf); 597 if (error) 598 break; 599 lrbuf = abuf->b_data; 600 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 601 blk = ztp->zit_next_blk; 602 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 603 } 604 dmu_objset_close(os); 605 if (error == ECKSUM) 606 return (0); /* normal end of chain */ 607 return (error); 608} 609 610static int 611zil_vdev_compare(const void *x1, const void *x2) 612{ 613 uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev; 614 uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev; 615 616 if (v1 < v2) 617 return (-1); 618 if (v1 > v2) 619 return (1); 620 621 return (0); 622} 623 624void 625zil_add_block(zilog_t *zilog, blkptr_t *bp) 626{ 627 avl_tree_t *t = &zilog->zl_vdev_tree; 628 avl_index_t where; 629 zil_vdev_node_t *zv, zvsearch; 630 int ndvas = BP_GET_NDVAS(bp); 631 int i; 632 633 if (zfs_nocacheflush) 634 return; 635 636 ASSERT(zilog->zl_writer); 637 638 /* 639 * Even though we're zl_writer, we still need a lock because the 640 * zl_get_data() callbacks may have dmu_sync() done callbacks 641 * that will run concurrently. 642 */ 643 mutex_enter(&zilog->zl_vdev_lock); 644 for (i = 0; i < ndvas; i++) { 645 zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 646 if (avl_find(t, &zvsearch, &where) == NULL) { 647 zv = kmem_alloc(sizeof (*zv), KM_SLEEP); 648 zv->zv_vdev = zvsearch.zv_vdev; 649 avl_insert(t, zv, where); 650 } 651 } 652 mutex_exit(&zilog->zl_vdev_lock); 653} 654 655void 656zil_flush_vdevs(zilog_t *zilog) 657{ 658 spa_t *spa = zilog->zl_spa; 659 avl_tree_t *t = &zilog->zl_vdev_tree; 660 void *cookie = NULL; 661 zil_vdev_node_t *zv; 662 zio_t *zio; 663 664 ASSERT(zilog->zl_writer); 665 666 /* 667 * We don't need zl_vdev_lock here because we're the zl_writer, 668 * and all zl_get_data() callbacks are done. 669 */ 670 if (avl_numnodes(t) == 0) 671 return; 672 673 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 674 675 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 676 677 while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) { 678 vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev); 679 if (vd != NULL) 680 zio_flush(zio, vd); 681 kmem_free(zv, sizeof (*zv)); 682 } 683 684 /* 685 * Wait for all the flushes to complete. Not all devices actually 686 * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 687 */ 688 (void) zio_wait(zio); 689 690 spa_config_exit(spa, SCL_STATE, FTAG); 691} 692 693/* 694 * Function called when a log block write completes 695 */ 696static void 697zil_lwb_write_done(zio_t *zio) 698{ 699 lwb_t *lwb = zio->io_private; 700 zilog_t *zilog = lwb->lwb_zilog; 701 702 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 703 ASSERT(BP_GET_CHECKSUM(zio->io_bp) == ZIO_CHECKSUM_ZILOG); 704 ASSERT(BP_GET_TYPE(zio->io_bp) == DMU_OT_INTENT_LOG); 705 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 706 ASSERT(BP_GET_BYTEORDER(zio->io_bp) == ZFS_HOST_BYTEORDER); 707 ASSERT(!BP_IS_GANG(zio->io_bp)); 708 ASSERT(!BP_IS_HOLE(zio->io_bp)); 709 ASSERT(zio->io_bp->blk_fill == 0); 710 711 /* 712 * Ensure the lwb buffer pointer is cleared before releasing 713 * the txg. If we have had an allocation failure and 714 * the txg is waiting to sync then we want want zil_sync() 715 * to remove the lwb so that it's not picked up as the next new 716 * one in zil_commit_writer(). zil_sync() will only remove 717 * the lwb if lwb_buf is null. 718 */ 719 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 720 mutex_enter(&zilog->zl_lock); 721 lwb->lwb_buf = NULL; 722 if (zio->io_error) 723 zilog->zl_log_error = B_TRUE; 724 725 /* 726 * Now that we've written this log block, we have a stable pointer 727 * to the next block in the chain, so it's OK to let the txg in 728 * which we allocated the next block sync. We still have the 729 * zl_lock to ensure zil_sync doesn't kmem free the lwb. 730 */ 731 txg_rele_to_sync(&lwb->lwb_txgh); 732 mutex_exit(&zilog->zl_lock); 733} 734 735/* 736 * Initialize the io for a log block. 737 */ 738static void 739zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 740{ 741 zbookmark_t zb; 742 743 zb.zb_objset = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET]; 744 zb.zb_object = 0; 745 zb.zb_level = -1; 746 zb.zb_blkid = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 747 748 if (zilog->zl_root_zio == NULL) { 749 zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 750 ZIO_FLAG_CANFAIL); 751 } 752 if (lwb->lwb_zio == NULL) { 753 lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 754 0, &lwb->lwb_blk, lwb->lwb_buf, lwb->lwb_sz, 755 zil_lwb_write_done, lwb, ZIO_PRIORITY_LOG_WRITE, 756 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, &zb); 757 } 758} 759 760/* 761 * Start a log block write and advance to the next log block. 762 * Calls are serialized. 763 */ 764static lwb_t * 765zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 766{ 767 lwb_t *nlwb; 768 zil_trailer_t *ztp = (zil_trailer_t *)(lwb->lwb_buf + lwb->lwb_sz) - 1; 769 spa_t *spa = zilog->zl_spa; 770 blkptr_t *bp = &ztp->zit_next_blk; 771 uint64_t txg; 772 uint64_t zil_blksz; 773 int error; 774 775 ASSERT(lwb->lwb_nused <= ZIL_BLK_DATA_SZ(lwb)); 776 777 /* 778 * Allocate the next block and save its address in this block 779 * before writing it in order to establish the log chain. 780 * Note that if the allocation of nlwb synced before we wrote 781 * the block that points at it (lwb), we'd leak it if we crashed. 782 * Therefore, we don't do txg_rele_to_sync() until zil_lwb_write_done(). 783 */ 784 txg = txg_hold_open(zilog->zl_dmu_pool, &lwb->lwb_txgh); 785 txg_rele_to_quiesce(&lwb->lwb_txgh); 786 787 /* 788 * Pick a ZIL blocksize. We request a size that is the 789 * maximum of the previous used size, the current used size and 790 * the amount waiting in the queue. 791 */ 792 zil_blksz = MAX(zilog->zl_prev_used, 793 zilog->zl_cur_used + sizeof (*ztp)); 794 zil_blksz = MAX(zil_blksz, zilog->zl_itx_list_sz + sizeof (*ztp)); 795 zil_blksz = P2ROUNDUP_TYPED(zil_blksz, ZIL_MIN_BLKSZ, uint64_t); 796 if (zil_blksz > ZIL_MAX_BLKSZ) 797 zil_blksz = ZIL_MAX_BLKSZ; 798 799 BP_ZERO(bp); 800 /* pass the old blkptr in order to spread log blocks across devs */ 801 error = zio_alloc_blk(spa, zil_blksz, bp, &lwb->lwb_blk, txg); 802 if (error) { 803 dmu_tx_t *tx = dmu_tx_create_assigned(zilog->zl_dmu_pool, txg); 804 805 /* 806 * We dirty the dataset to ensure that zil_sync() will 807 * be called to remove this lwb from our zl_lwb_list. 808 * Failing to do so, may leave an lwb with a NULL lwb_buf 809 * hanging around on the zl_lwb_list. 810 */ 811 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 812 dmu_tx_commit(tx); 813 814 /* 815 * Since we've just experienced an allocation failure so we 816 * terminate the current lwb and send it on its way. 817 */ 818 ztp->zit_pad = 0; 819 ztp->zit_nused = lwb->lwb_nused; 820 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 821 zio_nowait(lwb->lwb_zio); 822 823 /* 824 * By returning NULL the caller will call tx_wait_synced() 825 */ 826 return (NULL); 827 } 828 829 ASSERT3U(bp->blk_birth, ==, txg); 830 ztp->zit_pad = 0; 831 ztp->zit_nused = lwb->lwb_nused; 832 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 833 bp->blk_cksum = lwb->lwb_blk.blk_cksum; 834 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 835 836 /* 837 * Allocate a new log write buffer (lwb). 838 */ 839 nlwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 840 841 nlwb->lwb_zilog = zilog; 842 nlwb->lwb_blk = *bp; 843 nlwb->lwb_nused = 0; 844 nlwb->lwb_sz = BP_GET_LSIZE(&nlwb->lwb_blk); 845 nlwb->lwb_buf = zio_buf_alloc(nlwb->lwb_sz); 846 nlwb->lwb_max_txg = txg; 847 nlwb->lwb_zio = NULL; 848 849 /* 850 * Put new lwb at the end of the log chain 851 */ 852 mutex_enter(&zilog->zl_lock); 853 list_insert_tail(&zilog->zl_lwb_list, nlwb); 854 mutex_exit(&zilog->zl_lock); 855 856 /* Record the block for later vdev flushing */ 857 zil_add_block(zilog, &lwb->lwb_blk); 858 859 /* 860 * kick off the write for the old log block 861 */ 862 dprintf_bp(&lwb->lwb_blk, "lwb %p txg %llu: ", lwb, txg); 863 ASSERT(lwb->lwb_zio); 864 zio_nowait(lwb->lwb_zio); 865 866 return (nlwb); 867} 868 869static lwb_t * 870zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 871{ 872 lr_t *lrc = &itx->itx_lr; /* common log record */ 873 lr_write_t *lr = (lr_write_t *)lrc; 874 uint64_t txg = lrc->lrc_txg; 875 uint64_t reclen = lrc->lrc_reclen; 876 uint64_t dlen; 877 878 if (lwb == NULL) 879 return (NULL); 880 ASSERT(lwb->lwb_buf != NULL); 881 882 if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 883 dlen = P2ROUNDUP_TYPED( 884 lr->lr_length, sizeof (uint64_t), uint64_t); 885 else 886 dlen = 0; 887 888 zilog->zl_cur_used += (reclen + dlen); 889 890 zil_lwb_write_init(zilog, lwb); 891 892 /* 893 * If this record won't fit in the current log block, start a new one. 894 */ 895 if (lwb->lwb_nused + reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 896 lwb = zil_lwb_write_start(zilog, lwb); 897 if (lwb == NULL) 898 return (NULL); 899 zil_lwb_write_init(zilog, lwb); 900 ASSERT(lwb->lwb_nused == 0); 901 if (reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 902 txg_wait_synced(zilog->zl_dmu_pool, txg); 903 return (lwb); 904 } 905 } 906 907 /* 908 * Update the lrc_seq, to be log record sequence number. See zil.h 909 * Then copy the record to the log buffer. 910 */ 911 lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 912 bcopy(lrc, lwb->lwb_buf + lwb->lwb_nused, reclen); 913 914 /* 915 * If it's a write, fetch the data or get its blkptr as appropriate. 916 */ 917 if (lrc->lrc_txtype == TX_WRITE) { 918 if (txg > spa_freeze_txg(zilog->zl_spa)) 919 txg_wait_synced(zilog->zl_dmu_pool, txg); 920 if (itx->itx_wr_state != WR_COPIED) { 921 char *dbuf; 922 int error; 923 924 /* alignment is guaranteed */ 925 lr = (lr_write_t *)(lwb->lwb_buf + lwb->lwb_nused); 926 if (dlen) { 927 ASSERT(itx->itx_wr_state == WR_NEED_COPY); 928 dbuf = lwb->lwb_buf + lwb->lwb_nused + reclen; 929 lr->lr_common.lrc_reclen += dlen; 930 } else { 931 ASSERT(itx->itx_wr_state == WR_INDIRECT); 932 dbuf = NULL; 933 } 934 error = zilog->zl_get_data( 935 itx->itx_private, lr, dbuf, lwb->lwb_zio); 936 if (error) { 937 ASSERT(error == ENOENT || error == EEXIST || 938 error == EALREADY); 939 return (lwb); 940 } 941 } 942 } 943 944 lwb->lwb_nused += reclen + dlen; 945 lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 946 ASSERT3U(lwb->lwb_nused, <=, ZIL_BLK_DATA_SZ(lwb)); 947 ASSERT3U(P2PHASE(lwb->lwb_nused, sizeof (uint64_t)), ==, 0); 948 949 return (lwb); 950} 951 952itx_t * 953zil_itx_create(uint64_t txtype, size_t lrsize) 954{ 955 itx_t *itx; 956 957 lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 958 959 itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 960 itx->itx_lr.lrc_txtype = txtype; 961 itx->itx_lr.lrc_reclen = lrsize; 962 itx->itx_sod = lrsize; /* if write & WR_NEED_COPY will be increased */ 963 itx->itx_lr.lrc_seq = 0; /* defensive */ 964 965 return (itx); 966} 967 968uint64_t 969zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 970{ 971 uint64_t seq; 972 973 ASSERT(itx->itx_lr.lrc_seq == 0); 974 975 mutex_enter(&zilog->zl_lock); 976 list_insert_tail(&zilog->zl_itx_list, itx); 977 zilog->zl_itx_list_sz += itx->itx_sod; 978 itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 979 itx->itx_lr.lrc_seq = seq = ++zilog->zl_itx_seq; 980 mutex_exit(&zilog->zl_lock); 981 982 return (seq); 983} 984 985/* 986 * Free up all in-memory intent log transactions that have now been synced. 987 */ 988static void 989zil_itx_clean(zilog_t *zilog) 990{ 991 uint64_t synced_txg = spa_last_synced_txg(zilog->zl_spa); 992 uint64_t freeze_txg = spa_freeze_txg(zilog->zl_spa); 993 list_t clean_list; 994 itx_t *itx; 995 996 list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 997 998 mutex_enter(&zilog->zl_lock); 999 /* wait for a log writer to finish walking list */ 1000 while (zilog->zl_writer) { 1001 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1002 } 1003 1004 /* 1005 * Move the sync'd log transactions to a separate list so we can call 1006 * kmem_free without holding the zl_lock. 1007 * 1008 * There is no need to set zl_writer as we don't drop zl_lock here 1009 */ 1010 while ((itx = list_head(&zilog->zl_itx_list)) != NULL && 1011 itx->itx_lr.lrc_txg <= MIN(synced_txg, freeze_txg)) { 1012 list_remove(&zilog->zl_itx_list, itx); 1013 zilog->zl_itx_list_sz -= itx->itx_sod; 1014 list_insert_tail(&clean_list, itx); 1015 } 1016 cv_broadcast(&zilog->zl_cv_writer); 1017 mutex_exit(&zilog->zl_lock); 1018 1019 /* destroy sync'd log transactions */ 1020 while ((itx = list_head(&clean_list)) != NULL) { 1021 list_remove(&clean_list, itx); 1022 kmem_free(itx, offsetof(itx_t, itx_lr) 1023 + itx->itx_lr.lrc_reclen); 1024 } 1025 list_destroy(&clean_list); 1026} 1027 1028/* 1029 * If there are any in-memory intent log transactions which have now been 1030 * synced then start up a taskq to free them. 1031 */ 1032void 1033zil_clean(zilog_t *zilog) 1034{ 1035 itx_t *itx; 1036 1037 mutex_enter(&zilog->zl_lock); 1038 itx = list_head(&zilog->zl_itx_list); 1039 if ((itx != NULL) && 1040 (itx->itx_lr.lrc_txg <= spa_last_synced_txg(zilog->zl_spa))) { 1041 (void) taskq_dispatch(zilog->zl_clean_taskq, 1042 (task_func_t *)zil_itx_clean, zilog, TQ_SLEEP); 1043 } 1044 mutex_exit(&zilog->zl_lock); 1045} 1046 1047static void 1048zil_commit_writer(zilog_t *zilog, uint64_t seq, uint64_t foid) 1049{ 1050 uint64_t txg; 1051 uint64_t commit_seq = 0; 1052 itx_t *itx, *itx_next = (itx_t *)-1; 1053 lwb_t *lwb; 1054 spa_t *spa; 1055 1056 zilog->zl_writer = B_TRUE; 1057 ASSERT(zilog->zl_root_zio == NULL); 1058 spa = zilog->zl_spa; 1059 1060 if (zilog->zl_suspend) { 1061 lwb = NULL; 1062 } else { 1063 lwb = list_tail(&zilog->zl_lwb_list); 1064 if (lwb == NULL) { 1065 /* 1066 * Return if there's nothing to flush before we 1067 * dirty the fs by calling zil_create() 1068 */ 1069 if (list_is_empty(&zilog->zl_itx_list)) { 1070 zilog->zl_writer = B_FALSE; 1071 return; 1072 } 1073 mutex_exit(&zilog->zl_lock); 1074 zil_create(zilog); 1075 mutex_enter(&zilog->zl_lock); 1076 lwb = list_tail(&zilog->zl_lwb_list); 1077 } 1078 } 1079 1080 /* Loop through in-memory log transactions filling log blocks. */ 1081 DTRACE_PROBE1(zil__cw1, zilog_t *, zilog); 1082 for (;;) { 1083 /* 1084 * Find the next itx to push: 1085 * Push all transactions related to specified foid and all 1086 * other transactions except TX_WRITE, TX_TRUNCATE, 1087 * TX_SETATTR and TX_ACL for all other files. 1088 */ 1089 if (itx_next != (itx_t *)-1) 1090 itx = itx_next; 1091 else 1092 itx = list_head(&zilog->zl_itx_list); 1093 for (; itx != NULL; itx = list_next(&zilog->zl_itx_list, itx)) { 1094 if (foid == 0) /* push all foids? */ 1095 break; 1096 if (itx->itx_sync) /* push all O_[D]SYNC */ 1097 break; 1098 switch (itx->itx_lr.lrc_txtype) { 1099 case TX_SETATTR: 1100 case TX_WRITE: 1101 case TX_TRUNCATE: 1102 case TX_ACL: 1103 /* lr_foid is same offset for these records */ 1104 if (((lr_write_t *)&itx->itx_lr)->lr_foid 1105 != foid) { 1106 continue; /* skip this record */ 1107 } 1108 } 1109 break; 1110 } 1111 if (itx == NULL) 1112 break; 1113 1114 if ((itx->itx_lr.lrc_seq > seq) && 1115 ((lwb == NULL) || (lwb->lwb_nused == 0) || 1116 (lwb->lwb_nused + itx->itx_sod > ZIL_BLK_DATA_SZ(lwb)))) { 1117 break; 1118 } 1119 1120 /* 1121 * Save the next pointer. Even though we soon drop 1122 * zl_lock all threads that may change the list 1123 * (another writer or zil_itx_clean) can't do so until 1124 * they have zl_writer. 1125 */ 1126 itx_next = list_next(&zilog->zl_itx_list, itx); 1127 list_remove(&zilog->zl_itx_list, itx); 1128 zilog->zl_itx_list_sz -= itx->itx_sod; 1129 mutex_exit(&zilog->zl_lock); 1130 txg = itx->itx_lr.lrc_txg; 1131 ASSERT(txg); 1132 1133 if (txg > spa_last_synced_txg(spa) || 1134 txg > spa_freeze_txg(spa)) 1135 lwb = zil_lwb_commit(zilog, itx, lwb); 1136 kmem_free(itx, offsetof(itx_t, itx_lr) 1137 + itx->itx_lr.lrc_reclen); 1138 mutex_enter(&zilog->zl_lock); 1139 } 1140 DTRACE_PROBE1(zil__cw2, zilog_t *, zilog); 1141 /* determine commit sequence number */ 1142 itx = list_head(&zilog->zl_itx_list); 1143 if (itx) 1144 commit_seq = itx->itx_lr.lrc_seq; 1145 else 1146 commit_seq = zilog->zl_itx_seq; 1147 mutex_exit(&zilog->zl_lock); 1148 1149 /* write the last block out */ 1150 if (lwb != NULL && lwb->lwb_zio != NULL) 1151 lwb = zil_lwb_write_start(zilog, lwb); 1152 1153 zilog->zl_prev_used = zilog->zl_cur_used; 1154 zilog->zl_cur_used = 0; 1155 1156 /* 1157 * Wait if necessary for the log blocks to be on stable storage. 1158 */ 1159 if (zilog->zl_root_zio) { 1160 DTRACE_PROBE1(zil__cw3, zilog_t *, zilog); 1161 (void) zio_wait(zilog->zl_root_zio); 1162 zilog->zl_root_zio = NULL; 1163 DTRACE_PROBE1(zil__cw4, zilog_t *, zilog); 1164 zil_flush_vdevs(zilog); 1165 } 1166 1167 if (zilog->zl_log_error || lwb == NULL) { 1168 zilog->zl_log_error = 0; 1169 txg_wait_synced(zilog->zl_dmu_pool, 0); 1170 } 1171 1172 mutex_enter(&zilog->zl_lock); 1173 zilog->zl_writer = B_FALSE; 1174 1175 ASSERT3U(commit_seq, >=, zilog->zl_commit_seq); 1176 zilog->zl_commit_seq = commit_seq; 1177} 1178 1179/* 1180 * Push zfs transactions to stable storage up to the supplied sequence number. 1181 * If foid is 0 push out all transactions, otherwise push only those 1182 * for that file or might have been used to create that file. 1183 */ 1184void 1185zil_commit(zilog_t *zilog, uint64_t seq, uint64_t foid) 1186{ 1187 if (zilog == NULL || seq == 0) 1188 return; 1189 1190 mutex_enter(&zilog->zl_lock); 1191 1192 seq = MIN(seq, zilog->zl_itx_seq); /* cap seq at largest itx seq */ 1193 1194 while (zilog->zl_writer) { 1195 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1196 if (seq < zilog->zl_commit_seq) { 1197 mutex_exit(&zilog->zl_lock); 1198 return; 1199 } 1200 } 1201 zil_commit_writer(zilog, seq, foid); /* drops zl_lock */ 1202 /* wake up others waiting on the commit */ 1203 cv_broadcast(&zilog->zl_cv_writer); 1204 mutex_exit(&zilog->zl_lock); 1205} 1206 1207/* 1208 * Called in syncing context to free committed log blocks and update log header. 1209 */ 1210void 1211zil_sync(zilog_t *zilog, dmu_tx_t *tx) 1212{ 1213 zil_header_t *zh = zil_header_in_syncing_context(zilog); 1214 uint64_t txg = dmu_tx_get_txg(tx); 1215 spa_t *spa = zilog->zl_spa; 1216 lwb_t *lwb; 1217 1218 /* 1219 * We don't zero out zl_destroy_txg, so make sure we don't try 1220 * to destroy it twice. 1221 */ 1222 if (spa_sync_pass(spa) != 1) 1223 return; 1224 1225 mutex_enter(&zilog->zl_lock); 1226 1227 ASSERT(zilog->zl_stop_sync == 0); 1228 1229 zh->zh_replay_seq = zilog->zl_replayed_seq[txg & TXG_MASK]; 1230 1231 if (zilog->zl_destroy_txg == txg) { 1232 blkptr_t blk = zh->zh_log; 1233 1234 ASSERT(list_head(&zilog->zl_lwb_list) == NULL); 1235 1236 bzero(zh, sizeof (zil_header_t)); 1237 bzero(zilog->zl_replayed_seq, sizeof (zilog->zl_replayed_seq)); 1238 1239 if (zilog->zl_keep_first) { 1240 /* 1241 * If this block was part of log chain that couldn't 1242 * be claimed because a device was missing during 1243 * zil_claim(), but that device later returns, 1244 * then this block could erroneously appear valid. 1245 * To guard against this, assign a new GUID to the new 1246 * log chain so it doesn't matter what blk points to. 1247 */ 1248 zil_init_log_chain(zilog, &blk); 1249 zh->zh_log = blk; 1250 } 1251 } 1252 1253 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1254 zh->zh_log = lwb->lwb_blk; 1255 if (lwb->lwb_buf != NULL || lwb->lwb_max_txg > txg) 1256 break; 1257 list_remove(&zilog->zl_lwb_list, lwb); 1258 zio_free_blk(spa, &lwb->lwb_blk, txg); 1259 kmem_cache_free(zil_lwb_cache, lwb); 1260 1261 /* 1262 * If we don't have anything left in the lwb list then 1263 * we've had an allocation failure and we need to zero 1264 * out the zil_header blkptr so that we don't end 1265 * up freeing the same block twice. 1266 */ 1267 if (list_head(&zilog->zl_lwb_list) == NULL) 1268 BP_ZERO(&zh->zh_log); 1269 } 1270 mutex_exit(&zilog->zl_lock); 1271} 1272 1273void 1274zil_init(void) 1275{ 1276 zil_lwb_cache = kmem_cache_create("zil_lwb_cache", 1277 sizeof (struct lwb), 0, NULL, NULL, NULL, NULL, NULL, 0); 1278} 1279 1280void 1281zil_fini(void) 1282{ 1283 kmem_cache_destroy(zil_lwb_cache); 1284} 1285 1286zilog_t * 1287zil_alloc(objset_t *os, zil_header_t *zh_phys) 1288{ 1289 zilog_t *zilog; 1290 1291 zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); 1292 1293 zilog->zl_header = zh_phys; 1294 zilog->zl_os = os; 1295 zilog->zl_spa = dmu_objset_spa(os); 1296 zilog->zl_dmu_pool = dmu_objset_pool(os); 1297 zilog->zl_destroy_txg = TXG_INITIAL - 1; 1298 1299 mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL); 1300 1301 list_create(&zilog->zl_itx_list, sizeof (itx_t), 1302 offsetof(itx_t, itx_node)); 1303 1304 list_create(&zilog->zl_lwb_list, sizeof (lwb_t), 1305 offsetof(lwb_t, lwb_node)); 1306 1307 mutex_init(&zilog->zl_vdev_lock, NULL, MUTEX_DEFAULT, NULL); 1308 1309 avl_create(&zilog->zl_vdev_tree, zil_vdev_compare, 1310 sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node)); 1311 1312 cv_init(&zilog->zl_cv_writer, NULL, CV_DEFAULT, NULL); 1313 cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL); 1314 1315 return (zilog); 1316} 1317 1318void 1319zil_free(zilog_t *zilog) 1320{ 1321 lwb_t *lwb; 1322 1323 zilog->zl_stop_sync = 1; 1324 1325 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1326 list_remove(&zilog->zl_lwb_list, lwb); 1327 if (lwb->lwb_buf != NULL) 1328 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 1329 kmem_cache_free(zil_lwb_cache, lwb); 1330 } 1331 list_destroy(&zilog->zl_lwb_list); 1332 1333 avl_destroy(&zilog->zl_vdev_tree); 1334 mutex_destroy(&zilog->zl_vdev_lock); 1335 1336 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1337 list_destroy(&zilog->zl_itx_list); 1338 mutex_destroy(&zilog->zl_lock); 1339 1340 cv_destroy(&zilog->zl_cv_writer); 1341 cv_destroy(&zilog->zl_cv_suspend); 1342 1343 kmem_free(zilog, sizeof (zilog_t)); 1344} 1345 1346/* 1347 * Open an intent log. 1348 */ 1349zilog_t * 1350zil_open(objset_t *os, zil_get_data_t *get_data) 1351{ 1352 zilog_t *zilog = dmu_objset_zil(os); 1353 1354 zilog->zl_get_data = get_data; 1355 zilog->zl_clean_taskq = taskq_create("zil_clean", 1, minclsyspri, 1356 2, 2, TASKQ_PREPOPULATE); 1357 1358 return (zilog); 1359} 1360 1361/* 1362 * Close an intent log. 1363 */ 1364void 1365zil_close(zilog_t *zilog) 1366{ 1367 /* 1368 * If the log isn't already committed, mark the objset dirty 1369 * (so zil_sync() will be called) and wait for that txg to sync. 1370 */ 1371 if (!zil_is_committed(zilog)) { 1372 uint64_t txg; 1373 dmu_tx_t *tx = dmu_tx_create(zilog->zl_os); 1374 (void) dmu_tx_assign(tx, TXG_WAIT); 1375 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 1376 txg = dmu_tx_get_txg(tx); 1377 dmu_tx_commit(tx); 1378 txg_wait_synced(zilog->zl_dmu_pool, txg); 1379 } 1380 1381 taskq_destroy(zilog->zl_clean_taskq); 1382 zilog->zl_clean_taskq = NULL; 1383 zilog->zl_get_data = NULL; 1384 1385 zil_itx_clean(zilog); 1386 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1387} 1388 1389/* 1390 * Suspend an intent log. While in suspended mode, we still honor 1391 * synchronous semantics, but we rely on txg_wait_synced() to do it. 1392 * We suspend the log briefly when taking a snapshot so that the snapshot 1393 * contains all the data it's supposed to, and has an empty intent log. 1394 */ 1395int 1396zil_suspend(zilog_t *zilog) 1397{ 1398 const zil_header_t *zh = zilog->zl_header; 1399 1400 mutex_enter(&zilog->zl_lock); 1401 if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */ 1402 mutex_exit(&zilog->zl_lock); 1403 return (EBUSY); 1404 } 1405 if (zilog->zl_suspend++ != 0) { 1406 /* 1407 * Someone else already began a suspend. 1408 * Just wait for them to finish. 1409 */ 1410 while (zilog->zl_suspending) 1411 cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); 1412 mutex_exit(&zilog->zl_lock); 1413 return (0); 1414 } 1415 zilog->zl_suspending = B_TRUE; 1416 mutex_exit(&zilog->zl_lock); 1417 1418 zil_commit(zilog, UINT64_MAX, 0); 1419 1420 /* 1421 * Wait for any in-flight log writes to complete. 1422 */ 1423 mutex_enter(&zilog->zl_lock); 1424 while (zilog->zl_writer) 1425 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1426 mutex_exit(&zilog->zl_lock); 1427 1428 zil_destroy(zilog, B_FALSE); 1429 1430 mutex_enter(&zilog->zl_lock); 1431 zilog->zl_suspending = B_FALSE; 1432 cv_broadcast(&zilog->zl_cv_suspend); 1433 mutex_exit(&zilog->zl_lock); 1434 1435 return (0); 1436} 1437 1438void 1439zil_resume(zilog_t *zilog) 1440{ 1441 mutex_enter(&zilog->zl_lock); 1442 ASSERT(zilog->zl_suspend != 0); 1443 zilog->zl_suspend--; 1444 mutex_exit(&zilog->zl_lock); 1445} 1446 1447/* 1448 * Read in the data for the dmu_sync()ed block, and change the log 1449 * record to write this whole block. 1450 */ 1451void 1452zil_get_replay_data(zilog_t *zilog, lr_write_t *lr) 1453{ 1454 blkptr_t *wbp = &lr->lr_blkptr; 1455 char *wbuf = (char *)(lr + 1); /* data follows lr_write_t */ 1456 uint64_t blksz; 1457 1458 if (BP_IS_HOLE(wbp)) { /* compressed to a hole */ 1459 blksz = BP_GET_LSIZE(&lr->lr_blkptr); 1460 /* 1461 * If the blksz is zero then we must be replaying a log 1462 * from an version prior to setting the blksize of null blocks. 1463 * So we just zero the actual write size reqeusted. 1464 */ 1465 if (blksz == 0) { 1466 bzero(wbuf, lr->lr_length); 1467 return; 1468 } 1469 bzero(wbuf, blksz); 1470 } else { 1471 /* 1472 * A subsequent write may have overwritten this block, in which 1473 * case wbp may have been been freed and reallocated, and our 1474 * read of wbp may fail with a checksum error. We can safely 1475 * ignore this because the later write will provide the 1476 * correct data. 1477 */ 1478 zbookmark_t zb; 1479 1480 zb.zb_objset = dmu_objset_id(zilog->zl_os); 1481 zb.zb_object = lr->lr_foid; 1482 zb.zb_level = 0; 1483 zb.zb_blkid = -1; /* unknown */ 1484 1485 blksz = BP_GET_LSIZE(&lr->lr_blkptr); 1486 (void) zio_wait(zio_read(NULL, zilog->zl_spa, wbp, wbuf, blksz, 1487 NULL, NULL, ZIO_PRIORITY_SYNC_READ, 1488 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, &zb)); 1489 } 1490 lr->lr_offset -= lr->lr_offset % blksz; 1491 lr->lr_length = blksz; 1492} 1493 1494typedef struct zil_replay_arg { 1495 objset_t *zr_os; 1496 zil_replay_func_t **zr_replay; 1497 void *zr_arg; 1498 boolean_t zr_byteswap; 1499 char *zr_lrbuf; 1500} zil_replay_arg_t; 1501 1502static void 1503zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) 1504{ 1505 zil_replay_arg_t *zr = zra; 1506 const zil_header_t *zh = zilog->zl_header; 1507 uint64_t reclen = lr->lrc_reclen; 1508 uint64_t txtype = lr->lrc_txtype; 1509 char *name; 1510 int pass, error; 1511 1512 if (!zilog->zl_replay) /* giving up */ 1513 return; 1514 1515 if (lr->lrc_txg < claim_txg) /* already committed */ 1516 return; 1517 1518 if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */ 1519 return; 1520 1521 /* Strip case-insensitive bit, still present in log record */ 1522 txtype &= ~TX_CI; 1523 1524 if (txtype == 0 || txtype >= TX_MAX_TYPE) { 1525 error = EINVAL; 1526 goto bad; 1527 } 1528 1529 /* 1530 * Make a copy of the data so we can revise and extend it. 1531 */ 1532 bcopy(lr, zr->zr_lrbuf, reclen); 1533 1534 /* 1535 * The log block containing this lr may have been byteswapped 1536 * so that we can easily examine common fields like lrc_txtype. 1537 * However, the log is a mix of different data types, and only the 1538 * replay vectors know how to byteswap their records. Therefore, if 1539 * the lr was byteswapped, undo it before invoking the replay vector. 1540 */ 1541 if (zr->zr_byteswap) 1542 byteswap_uint64_array(zr->zr_lrbuf, reclen); 1543 1544 /* 1545 * We must now do two things atomically: replay this log record, 1546 * and update the log header sequence number to reflect the fact that 1547 * we did so. At the end of each replay function the sequence number 1548 * is updated if we are in replay mode. 1549 */ 1550 for (pass = 1; pass <= 2; pass++) { 1551 zilog->zl_replaying_seq = lr->lrc_seq; 1552 /* Only byteswap (if needed) on the 1st pass. */ 1553 error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lrbuf, 1554 zr->zr_byteswap && pass == 1); 1555 1556 if (!error) 1557 return; 1558 1559 /* 1560 * The DMU's dnode layer doesn't see removes until the txg 1561 * commits, so a subsequent claim can spuriously fail with 1562 * EEXIST. So if we receive any error we try syncing out 1563 * any removes then retry the transaction. 1564 */ 1565 if (pass == 1) 1566 txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0); 1567 } 1568 1569bad: 1570 ASSERT(error); 1571 name = kmem_alloc(MAXNAMELEN, KM_SLEEP); 1572 dmu_objset_name(zr->zr_os, name); 1573 cmn_err(CE_WARN, "ZFS replay transaction error %d, " 1574 "dataset %s, seq 0x%llx, txtype %llu %s\n", 1575 error, name, (u_longlong_t)lr->lrc_seq, (u_longlong_t)txtype, 1576 (lr->lrc_txtype & TX_CI) ? "CI" : ""); 1577 zilog->zl_replay = B_FALSE; 1578 kmem_free(name, MAXNAMELEN); 1579} 1580 1581/* ARGSUSED */ 1582static void 1583zil_incr_blks(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 1584{ 1585 zilog->zl_replay_blks++; 1586} 1587 1588/* 1589 * If this dataset has a non-empty intent log, replay it and destroy it. 1590 */ 1591void 1592zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) 1593{ 1594 zilog_t *zilog = dmu_objset_zil(os); 1595 const zil_header_t *zh = zilog->zl_header; 1596 zil_replay_arg_t zr; 1597 1598 if ((zh->zh_flags & ZIL_REPLAY_NEEDED) == 0) { 1599 zil_destroy(zilog, B_TRUE); 1600 return; 1601 } 1602 //printf("ZFS: Replaying ZIL on %s...\n", os->os->os_spa->spa_name); 1603 1604 zr.zr_os = os; 1605 zr.zr_replay = replay_func; 1606 zr.zr_arg = arg; 1607 zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); 1608 zr.zr_lrbuf = kmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); 1609 1610 /* 1611 * Wait for in-progress removes to sync before starting replay. 1612 */ 1613 txg_wait_synced(zilog->zl_dmu_pool, 0); 1614 1615 zilog->zl_replay = B_TRUE; 1616 zilog->zl_replay_time = LBOLT; 1617 ASSERT(zilog->zl_replay_blks == 0); 1618 (void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr, 1619 zh->zh_claim_txg); 1620 kmem_free(zr.zr_lrbuf, 2 * SPA_MAXBLOCKSIZE); 1621 1622 zil_destroy(zilog, B_FALSE); 1623 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 1624 zilog->zl_replay = B_FALSE; 1625 //printf("ZFS: Replay of ZIL on %s finished.\n", os->os->os_spa->spa_name); 1626} 1627 1628/* 1629 * Report whether all transactions are committed 1630 */ 1631int 1632zil_is_committed(zilog_t *zilog) 1633{ 1634 lwb_t *lwb; 1635 int ret; 1636 1637 mutex_enter(&zilog->zl_lock); 1638 while (zilog->zl_writer) 1639 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1640 1641 /* recent unpushed intent log transactions? */ 1642 if (!list_is_empty(&zilog->zl_itx_list)) { 1643 ret = B_FALSE; 1644 goto out; 1645 } 1646 1647 /* intent log never used? */ 1648 lwb = list_head(&zilog->zl_lwb_list); 1649 if (lwb == NULL) { 1650 ret = B_TRUE; 1651 goto out; 1652 } 1653 1654 /* 1655 * more than 1 log buffer means zil_sync() hasn't yet freed 1656 * entries after a txg has committed 1657 */ 1658 if (list_next(&zilog->zl_lwb_list, lwb)) { 1659 ret = B_FALSE; 1660 goto out; 1661 } 1662 1663 ASSERT(zil_empty(zilog)); 1664 ret = B_TRUE; 1665out: 1666 cv_broadcast(&zilog->zl_cv_writer); 1667 mutex_exit(&zilog->zl_lock); 1668 return (ret); 1669} 1670 1671/* ARGSUSED */ 1672int 1673zil_vdev_offline(char *osname, void *arg) 1674{ 1675 objset_t *os; 1676 zilog_t *zilog; 1677 int error; 1678 1679 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 1680 if (error) 1681 return (error); 1682 1683 zilog = dmu_objset_zil(os); 1684 if (zil_suspend(zilog) != 0) 1685 error = EEXIST; 1686 else 1687 zil_resume(zilog); 1688 dmu_objset_close(os); 1689 return (error); 1690} 1691