zil.c revision 214378
1234287Sdim/* 2234287Sdim * CDDL HEADER START 3234287Sdim * 4234287Sdim * The contents of this file are subject to the terms of the 5234287Sdim * Common Development and Distribution License (the "License"). 6234287Sdim * You may not use this file except in compliance with the License. 7234287Sdim * 8234287Sdim * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9234287Sdim * or http://www.opensolaris.org/os/licensing. 10234287Sdim * See the License for the specific language governing permissions 11234287Sdim * and limitations under the License. 12234287Sdim * 13234287Sdim * When distributing Covered Code, include this CDDL HEADER in each 14234287Sdim * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15234287Sdim * If applicable, add the following below this CDDL HEADER, with the 16234287Sdim * fields enclosed by brackets "[]" replaced with your own identifying 17234287Sdim * information: Portions Copyright [yyyy] [name of copyright owner] 18249423Sdim * 19249423Sdim * CDDL HEADER END 20234287Sdim */ 21234287Sdim/* 22249423Sdim * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23249423Sdim * Use is subject to license terms. 24234287Sdim */ 25234287Sdim 26234287Sdim#include <sys/zfs_context.h> 27234287Sdim#include <sys/spa.h> 28234287Sdim#include <sys/spa_impl.h> 29234287Sdim#include <sys/dmu.h> 30234287Sdim#include <sys/zap.h> 31234287Sdim#include <sys/arc.h> 32234287Sdim#include <sys/stat.h> 33234287Sdim#include <sys/resource.h> 34234287Sdim#include <sys/zil.h> 35234287Sdim#include <sys/zil_impl.h> 36234287Sdim#include <sys/dsl_dataset.h> 37234287Sdim#include <sys/vdev.h> 38234287Sdim#include <sys/dmu_tx.h> 39234287Sdim 40234287Sdim/* 41234287Sdim * The zfs intent log (ZIL) saves transaction records of system calls 42234287Sdim * that change the file system in memory with enough information 43234287Sdim * to be able to replay them. These are stored in memory until 44234287Sdim * either the DMU transaction group (txg) commits them to the stable pool 45234287Sdim * and they can be discarded, or they are flushed to the stable log 46234287Sdim * (also in the pool) due to a fsync, O_DSYNC or other synchronous 47234287Sdim * requirement. In the event of a panic or power fail then those log 48234287Sdim * records (transactions) are replayed. 49234287Sdim * 50239462Sdim * There is one ZIL per file system. Its on-disk (pool) format consists 51234287Sdim * of 3 parts: 52234287Sdim * 53234287Sdim * - ZIL header 54234287Sdim * - ZIL blocks 55234287Sdim * - ZIL records 56234287Sdim * 57234287Sdim * A log record holds a system call transaction. Log blocks can 58234287Sdim * hold many log records and the blocks are chained together. 59234287Sdim * Each ZIL block contains a block pointer (blkptr_t) to the next 60234287Sdim * ZIL block in the chain. The ZIL header points to the first 61234287Sdim * block in the chain. Note there is not a fixed place in the pool 62234287Sdim * to hold blocks. They are dynamically allocated and freed as 63234287Sdim * needed from the blocks available. Figure X shows the ZIL structure: 64234287Sdim */ 65234287Sdim 66234287Sdim/* 67234287Sdim * This global ZIL switch affects all pools 68234287Sdim */ 69234287Sdimint zil_disable = 0; /* disable intent logging */ 70234287SdimSYSCTL_DECL(_vfs_zfs); 71234287SdimTUNABLE_INT("vfs.zfs.zil_disable", &zil_disable); 72234287SdimSYSCTL_INT(_vfs_zfs, OID_AUTO, zil_disable, CTLFLAG_RW, &zil_disable, 0, 73234287Sdim "Disable ZFS Intent Log (ZIL)"); 74234287Sdim 75234287Sdim/* 76234287Sdim * Tunable parameter for debugging or performance analysis. Setting 77234287Sdim * zfs_nocacheflush will cause corruption on power loss if a volatile 78234287Sdim * out-of-order write cache is enabled. 79234287Sdim */ 80234287Sdimboolean_t zfs_nocacheflush = B_FALSE; 81234287SdimTUNABLE_INT("vfs.zfs.cache_flush_disable", &zfs_nocacheflush); 82234287SdimSYSCTL_INT(_vfs_zfs, OID_AUTO, cache_flush_disable, CTLFLAG_RDTUN, 83234287Sdim &zfs_nocacheflush, 0, "Disable cache flush"); 84234287Sdim 85234287Sdimstatic kmem_cache_t *zil_lwb_cache; 86234287Sdim 87234287Sdimstatic int 88234287Sdimzil_dva_compare(const void *x1, const void *x2) 89234287Sdim{ 90234287Sdim const dva_t *dva1 = x1; 91234287Sdim const dva_t *dva2 = x2; 92234287Sdim 93234287Sdim if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 94234287Sdim return (-1); 95234287Sdim if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 96234287Sdim return (1); 97234287Sdim 98234287Sdim if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 99234287Sdim return (-1); 100234287Sdim if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 101234287Sdim return (1); 102234287Sdim 103234287Sdim return (0); 104234287Sdim} 105234287Sdim 106234287Sdimstatic void 107234287Sdimzil_dva_tree_init(avl_tree_t *t) 108234287Sdim{ 109234287Sdim avl_create(t, zil_dva_compare, sizeof (zil_dva_node_t), 110234287Sdim offsetof(zil_dva_node_t, zn_node)); 111234287Sdim} 112234287Sdim 113234287Sdimstatic void 114234287Sdimzil_dva_tree_fini(avl_tree_t *t) 115234287Sdim{ 116234287Sdim zil_dva_node_t *zn; 117234287Sdim void *cookie = NULL; 118234287Sdim 119234287Sdim while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 120234287Sdim kmem_free(zn, sizeof (zil_dva_node_t)); 121234287Sdim 122234287Sdim avl_destroy(t); 123234287Sdim} 124234287Sdim 125234287Sdimstatic int 126234287Sdimzil_dva_tree_add(avl_tree_t *t, dva_t *dva) 127234287Sdim{ 128234287Sdim zil_dva_node_t *zn; 129234287Sdim avl_index_t where; 130234287Sdim 131234287Sdim if (avl_find(t, dva, &where) != NULL) 132234287Sdim return (EEXIST); 133234287Sdim 134234287Sdim zn = kmem_alloc(sizeof (zil_dva_node_t), KM_SLEEP); 135234287Sdim zn->zn_dva = *dva; 136234287Sdim avl_insert(t, zn, where); 137234287Sdim 138234287Sdim return (0); 139234287Sdim} 140234287Sdim 141234287Sdimstatic zil_header_t * 142234287Sdimzil_header_in_syncing_context(zilog_t *zilog) 143234287Sdim{ 144234287Sdim return ((zil_header_t *)zilog->zl_header); 145234287Sdim} 146234287Sdim 147234287Sdimstatic void 148234287Sdimzil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 149234287Sdim{ 150234287Sdim zio_cksum_t *zc = &bp->blk_cksum; 151234287Sdim 152234287Sdim zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 153234287Sdim zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 154234287Sdim zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 155234287Sdim zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 156234287Sdim} 157234287Sdim 158234287Sdim/* 159234287Sdim * Read a log block, make sure it's valid, and byteswap it if necessary. 160234287Sdim */ 161234287Sdimstatic int 162234287Sdimzil_read_log_block(zilog_t *zilog, const blkptr_t *bp, arc_buf_t **abufpp) 163234287Sdim{ 164234287Sdim blkptr_t blk = *bp; 165234287Sdim zbookmark_t zb; 166234287Sdim uint32_t aflags = ARC_WAIT; 167234287Sdim int error; 168234287Sdim 169234287Sdim zb.zb_objset = bp->blk_cksum.zc_word[ZIL_ZC_OBJSET]; 170234287Sdim zb.zb_object = 0; 171234287Sdim zb.zb_level = -1; 172234287Sdim zb.zb_blkid = bp->blk_cksum.zc_word[ZIL_ZC_SEQ]; 173234287Sdim 174234287Sdim *abufpp = NULL; 175234287Sdim 176234287Sdim /* 177234287Sdim * We shouldn't be doing any scrubbing while we're doing log 178234287Sdim * replay, it's OK to not lock. 179234287Sdim */ 180234287Sdim error = arc_read_nolock(NULL, zilog->zl_spa, &blk, 181234287Sdim arc_getbuf_func, abufpp, ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL | 182234287Sdim ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB, &aflags, &zb); 183234287Sdim 184234287Sdim if (error == 0) { 185234287Sdim char *data = (*abufpp)->b_data; 186234287Sdim uint64_t blksz = BP_GET_LSIZE(bp); 187234287Sdim zil_trailer_t *ztp = (zil_trailer_t *)(data + blksz) - 1; 188234287Sdim zio_cksum_t cksum = bp->blk_cksum; 189234287Sdim 190234287Sdim /* 191234287Sdim * Validate the checksummed log block. 192234287Sdim * 193234287Sdim * Sequence numbers should be... sequential. The checksum 194263508Sdim * verifier for the next block should be bp's checksum plus 1. 195234287Sdim * 196234287Sdim * Also check the log chain linkage and size used. 197234287Sdim */ 198234287Sdim cksum.zc_word[ZIL_ZC_SEQ]++; 199234287Sdim 200234287Sdim if (bcmp(&cksum, &ztp->zit_next_blk.blk_cksum, 201234287Sdim sizeof (cksum)) || BP_IS_HOLE(&ztp->zit_next_blk) || 202234287Sdim (ztp->zit_nused > (blksz - sizeof (zil_trailer_t)))) { 203234287Sdim error = ECKSUM; 204263508Sdim } 205234287Sdim 206234287Sdim if (error) { 207234287Sdim VERIFY(arc_buf_remove_ref(*abufpp, abufpp) == 1); 208234287Sdim *abufpp = NULL; 209234287Sdim } 210234287Sdim } 211234287Sdim 212234287Sdim dprintf("error %d on %llu:%llu\n", error, zb.zb_objset, zb.zb_blkid); 213234287Sdim 214234287Sdim return (error); 215234287Sdim} 216234287Sdim 217234287Sdim/* 218234287Sdim * Parse the intent log, and call parse_func for each valid record within. 219234287Sdim * Return the highest sequence number. 220234287Sdim */ 221234287Sdimuint64_t 222234287Sdimzil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 223234287Sdim zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 224234287Sdim{ 225234287Sdim const zil_header_t *zh = zilog->zl_header; 226234287Sdim uint64_t claim_seq = zh->zh_claim_seq; 227234287Sdim uint64_t seq = 0; 228234287Sdim uint64_t max_seq = 0; 229234287Sdim blkptr_t blk = zh->zh_log; 230234287Sdim arc_buf_t *abuf; 231234287Sdim char *lrbuf, *lrp; 232234287Sdim zil_trailer_t *ztp; 233234287Sdim int reclen, error; 234234287Sdim 235234287Sdim if (BP_IS_HOLE(&blk)) 236234287Sdim return (max_seq); 237234287Sdim 238234287Sdim /* 239234287Sdim * Starting at the block pointed to by zh_log we read the log chain. 240234287Sdim * For each block in the chain we strongly check that block to 241234287Sdim * ensure its validity. We stop when an invalid block is found. 242234287Sdim * For each block pointer in the chain we call parse_blk_func(). 243 * For each record in each valid block we call parse_lr_func(). 244 * If the log has been claimed, stop if we encounter a sequence 245 * number greater than the highest claimed sequence number. 246 */ 247 zil_dva_tree_init(&zilog->zl_dva_tree); 248 for (;;) { 249 seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 250 251 if (claim_seq != 0 && seq > claim_seq) 252 break; 253 254 ASSERT(max_seq < seq); 255 max_seq = seq; 256 257 error = zil_read_log_block(zilog, &blk, &abuf); 258 259 if (parse_blk_func != NULL) 260 parse_blk_func(zilog, &blk, arg, txg); 261 262 if (error) 263 break; 264 265 lrbuf = abuf->b_data; 266 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 267 blk = ztp->zit_next_blk; 268 269 if (parse_lr_func == NULL) { 270 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 271 continue; 272 } 273 274 for (lrp = lrbuf; lrp < lrbuf + ztp->zit_nused; lrp += reclen) { 275 lr_t *lr = (lr_t *)lrp; 276 reclen = lr->lrc_reclen; 277 ASSERT3U(reclen, >=, sizeof (lr_t)); 278 parse_lr_func(zilog, lr, arg, txg); 279 } 280 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 281 } 282 zil_dva_tree_fini(&zilog->zl_dva_tree); 283 284 return (max_seq); 285} 286 287/* ARGSUSED */ 288static void 289zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 290{ 291 spa_t *spa = zilog->zl_spa; 292 int err; 293 294 /* 295 * Claim log block if not already committed and not already claimed. 296 */ 297 if (bp->blk_birth >= first_txg && 298 zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp)) == 0) { 299 err = zio_wait(zio_claim(NULL, spa, first_txg, bp, NULL, NULL, 300 ZIO_FLAG_MUSTSUCCEED)); 301 ASSERT(err == 0); 302 } 303} 304 305static void 306zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 307{ 308 if (lrc->lrc_txtype == TX_WRITE) { 309 lr_write_t *lr = (lr_write_t *)lrc; 310 zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg); 311 } 312} 313 314/* ARGSUSED */ 315static void 316zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 317{ 318 zio_free_blk(zilog->zl_spa, bp, dmu_tx_get_txg(tx)); 319} 320 321static void 322zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 323{ 324 /* 325 * If we previously claimed it, we need to free it. 326 */ 327 if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE) { 328 lr_write_t *lr = (lr_write_t *)lrc; 329 blkptr_t *bp = &lr->lr_blkptr; 330 if (bp->blk_birth >= claim_txg && 331 !zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp))) { 332 (void) arc_free(NULL, zilog->zl_spa, 333 dmu_tx_get_txg(tx), bp, NULL, NULL, ARC_WAIT); 334 } 335 } 336} 337 338/* 339 * Create an on-disk intent log. 340 */ 341static void 342zil_create(zilog_t *zilog) 343{ 344 const zil_header_t *zh = zilog->zl_header; 345 lwb_t *lwb; 346 uint64_t txg = 0; 347 dmu_tx_t *tx = NULL; 348 blkptr_t blk; 349 int error = 0; 350 351 /* 352 * Wait for any previous destroy to complete. 353 */ 354 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 355 356 ASSERT(zh->zh_claim_txg == 0); 357 ASSERT(zh->zh_replay_seq == 0); 358 359 blk = zh->zh_log; 360 361 /* 362 * If we don't already have an initial log block or we have one 363 * but it's the wrong endianness then allocate one. 364 */ 365 if (BP_IS_HOLE(&blk) || BP_SHOULD_BYTESWAP(&blk)) { 366 tx = dmu_tx_create(zilog->zl_os); 367 (void) dmu_tx_assign(tx, TXG_WAIT); 368 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 369 txg = dmu_tx_get_txg(tx); 370 371 if (!BP_IS_HOLE(&blk)) { 372 zio_free_blk(zilog->zl_spa, &blk, txg); 373 BP_ZERO(&blk); 374 } 375 376 error = zio_alloc_blk(zilog->zl_spa, ZIL_MIN_BLKSZ, &blk, 377 NULL, txg); 378 379 if (error == 0) 380 zil_init_log_chain(zilog, &blk); 381 } 382 383 /* 384 * Allocate a log write buffer (lwb) for the first log block. 385 */ 386 if (error == 0) { 387 lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 388 lwb->lwb_zilog = zilog; 389 lwb->lwb_blk = blk; 390 lwb->lwb_nused = 0; 391 lwb->lwb_sz = BP_GET_LSIZE(&lwb->lwb_blk); 392 lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz); 393 lwb->lwb_max_txg = txg; 394 lwb->lwb_zio = NULL; 395 396 mutex_enter(&zilog->zl_lock); 397 list_insert_tail(&zilog->zl_lwb_list, lwb); 398 mutex_exit(&zilog->zl_lock); 399 } 400 401 /* 402 * If we just allocated the first log block, commit our transaction 403 * and wait for zil_sync() to stuff the block poiner into zh_log. 404 * (zh is part of the MOS, so we cannot modify it in open context.) 405 */ 406 if (tx != NULL) { 407 dmu_tx_commit(tx); 408 txg_wait_synced(zilog->zl_dmu_pool, txg); 409 } 410 411 ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 412} 413 414/* 415 * In one tx, free all log blocks and clear the log header. 416 * If keep_first is set, then we're replaying a log with no content. 417 * We want to keep the first block, however, so that the first 418 * synchronous transaction doesn't require a txg_wait_synced() 419 * in zil_create(). We don't need to txg_wait_synced() here either 420 * when keep_first is set, because both zil_create() and zil_destroy() 421 * will wait for any in-progress destroys to complete. 422 */ 423void 424zil_destroy(zilog_t *zilog, boolean_t keep_first) 425{ 426 const zil_header_t *zh = zilog->zl_header; 427 lwb_t *lwb; 428 dmu_tx_t *tx; 429 uint64_t txg; 430 431 /* 432 * Wait for any previous destroy to complete. 433 */ 434 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 435 436 if (BP_IS_HOLE(&zh->zh_log)) 437 return; 438 439 tx = dmu_tx_create(zilog->zl_os); 440 (void) dmu_tx_assign(tx, TXG_WAIT); 441 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 442 txg = dmu_tx_get_txg(tx); 443 444 mutex_enter(&zilog->zl_lock); 445 446 /* 447 * It is possible for the ZIL to get the previously mounted zilog 448 * structure of the same dataset if quickly remounted and the dbuf 449 * eviction has not completed. In this case we can see a non 450 * empty lwb list and keep_first will be set. We fix this by 451 * clearing the keep_first. This will be slower but it's very rare. 452 */ 453 if (!list_is_empty(&zilog->zl_lwb_list) && keep_first) 454 keep_first = B_FALSE; 455 456 ASSERT3U(zilog->zl_destroy_txg, <, txg); 457 zilog->zl_destroy_txg = txg; 458 zilog->zl_keep_first = keep_first; 459 460 if (!list_is_empty(&zilog->zl_lwb_list)) { 461 ASSERT(zh->zh_claim_txg == 0); 462 ASSERT(!keep_first); 463 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 464 list_remove(&zilog->zl_lwb_list, lwb); 465 if (lwb->lwb_buf != NULL) 466 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 467 zio_free_blk(zilog->zl_spa, &lwb->lwb_blk, txg); 468 kmem_cache_free(zil_lwb_cache, lwb); 469 } 470 } else { 471 if (!keep_first) { 472 (void) zil_parse(zilog, zil_free_log_block, 473 zil_free_log_record, tx, zh->zh_claim_txg); 474 } 475 } 476 mutex_exit(&zilog->zl_lock); 477 478 dmu_tx_commit(tx); 479} 480 481/* 482 * return true if the initial log block is not valid 483 */ 484static boolean_t 485zil_empty(zilog_t *zilog) 486{ 487 const zil_header_t *zh = zilog->zl_header; 488 arc_buf_t *abuf = NULL; 489 490 if (BP_IS_HOLE(&zh->zh_log)) 491 return (B_TRUE); 492 493 if (zil_read_log_block(zilog, &zh->zh_log, &abuf) != 0) 494 return (B_TRUE); 495 496 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 497 return (B_FALSE); 498} 499 500int 501zil_claim(char *osname, void *txarg) 502{ 503 dmu_tx_t *tx = txarg; 504 uint64_t first_txg = dmu_tx_get_txg(tx); 505 zilog_t *zilog; 506 zil_header_t *zh; 507 objset_t *os; 508 int error; 509 510 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 511 if (error) { 512 cmn_err(CE_WARN, "can't open objset for %s", osname); 513 return (0); 514 } 515 516 zilog = dmu_objset_zil(os); 517 zh = zil_header_in_syncing_context(zilog); 518 519 if (zilog->zl_spa->spa_log_state == SPA_LOG_CLEAR) { 520 if (!BP_IS_HOLE(&zh->zh_log)) 521 zio_free_blk(zilog->zl_spa, &zh->zh_log, first_txg); 522 BP_ZERO(&zh->zh_log); 523 dsl_dataset_dirty(dmu_objset_ds(os), tx); 524 } 525 526 /* 527 * Record here whether the zil has any records to replay. 528 * If the header block pointer is null or the block points 529 * to the stubby then we know there are no valid log records. 530 * We use the header to store this state as the the zilog gets 531 * freed later in dmu_objset_close(). 532 * The flags (and the rest of the header fields) are cleared in 533 * zil_sync() as a result of a zil_destroy(), after replaying the log. 534 * 535 * Note, the intent log can be empty but still need the 536 * stubby to be claimed. 537 */ 538 if (!zil_empty(zilog)) { 539 zh->zh_flags |= ZIL_REPLAY_NEEDED; 540 dsl_dataset_dirty(dmu_objset_ds(os), tx); 541 } 542 543 /* 544 * Claim all log blocks if we haven't already done so, and remember 545 * the highest claimed sequence number. This ensures that if we can 546 * read only part of the log now (e.g. due to a missing device), 547 * but we can read the entire log later, we will not try to replay 548 * or destroy beyond the last block we successfully claimed. 549 */ 550 ASSERT3U(zh->zh_claim_txg, <=, first_txg); 551 if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 552 zh->zh_claim_txg = first_txg; 553 zh->zh_claim_seq = zil_parse(zilog, zil_claim_log_block, 554 zil_claim_log_record, tx, first_txg); 555 dsl_dataset_dirty(dmu_objset_ds(os), tx); 556 } 557 558 ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 559 dmu_objset_close(os); 560 return (0); 561} 562 563/* 564 * Check the log by walking the log chain. 565 * Checksum errors are ok as they indicate the end of the chain. 566 * Any other error (no device or read failure) returns an error. 567 */ 568/* ARGSUSED */ 569int 570zil_check_log_chain(char *osname, void *txarg) 571{ 572 zilog_t *zilog; 573 zil_header_t *zh; 574 blkptr_t blk; 575 arc_buf_t *abuf; 576 objset_t *os; 577 char *lrbuf; 578 zil_trailer_t *ztp; 579 int error; 580 581 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 582 if (error) { 583 cmn_err(CE_WARN, "can't open objset for %s", osname); 584 return (0); 585 } 586 587 zilog = dmu_objset_zil(os); 588 zh = zil_header_in_syncing_context(zilog); 589 blk = zh->zh_log; 590 if (BP_IS_HOLE(&blk)) { 591 dmu_objset_close(os); 592 return (0); /* no chain */ 593 } 594 595 for (;;) { 596 error = zil_read_log_block(zilog, &blk, &abuf); 597 if (error) 598 break; 599 lrbuf = abuf->b_data; 600 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 601 blk = ztp->zit_next_blk; 602 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 603 } 604 dmu_objset_close(os); 605 if (error == ECKSUM) 606 return (0); /* normal end of chain */ 607 return (error); 608} 609 610static int 611zil_vdev_compare(const void *x1, const void *x2) 612{ 613 uint64_t v1 = ((zil_vdev_node_t *)x1)->zv_vdev; 614 uint64_t v2 = ((zil_vdev_node_t *)x2)->zv_vdev; 615 616 if (v1 < v2) 617 return (-1); 618 if (v1 > v2) 619 return (1); 620 621 return (0); 622} 623 624void 625zil_add_block(zilog_t *zilog, blkptr_t *bp) 626{ 627 avl_tree_t *t = &zilog->zl_vdev_tree; 628 avl_index_t where; 629 zil_vdev_node_t *zv, zvsearch; 630 int ndvas = BP_GET_NDVAS(bp); 631 int i; 632 633 if (zfs_nocacheflush) 634 return; 635 636 ASSERT(zilog->zl_writer); 637 638 /* 639 * Even though we're zl_writer, we still need a lock because the 640 * zl_get_data() callbacks may have dmu_sync() done callbacks 641 * that will run concurrently. 642 */ 643 mutex_enter(&zilog->zl_vdev_lock); 644 for (i = 0; i < ndvas; i++) { 645 zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); 646 if (avl_find(t, &zvsearch, &where) == NULL) { 647 zv = kmem_alloc(sizeof (*zv), KM_SLEEP); 648 zv->zv_vdev = zvsearch.zv_vdev; 649 avl_insert(t, zv, where); 650 } 651 } 652 mutex_exit(&zilog->zl_vdev_lock); 653} 654 655void 656zil_flush_vdevs(zilog_t *zilog) 657{ 658 spa_t *spa = zilog->zl_spa; 659 avl_tree_t *t = &zilog->zl_vdev_tree; 660 void *cookie = NULL; 661 zil_vdev_node_t *zv; 662 zio_t *zio; 663 664 ASSERT(zilog->zl_writer); 665 666 /* 667 * We don't need zl_vdev_lock here because we're the zl_writer, 668 * and all zl_get_data() callbacks are done. 669 */ 670 if (avl_numnodes(t) == 0) 671 return; 672 673 spa_config_enter(spa, SCL_STATE, FTAG, RW_READER); 674 675 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 676 677 while ((zv = avl_destroy_nodes(t, &cookie)) != NULL) { 678 vdev_t *vd = vdev_lookup_top(spa, zv->zv_vdev); 679 if (vd != NULL) 680 zio_flush(zio, vd); 681 kmem_free(zv, sizeof (*zv)); 682 } 683 684 /* 685 * Wait for all the flushes to complete. Not all devices actually 686 * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 687 */ 688 (void) zio_wait(zio); 689 690 spa_config_exit(spa, SCL_STATE, FTAG); 691} 692 693/* 694 * Function called when a log block write completes 695 */ 696static void 697zil_lwb_write_done(zio_t *zio) 698{ 699 lwb_t *lwb = zio->io_private; 700 zilog_t *zilog = lwb->lwb_zilog; 701 702 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 703 ASSERT(BP_GET_CHECKSUM(zio->io_bp) == ZIO_CHECKSUM_ZILOG); 704 ASSERT(BP_GET_TYPE(zio->io_bp) == DMU_OT_INTENT_LOG); 705 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 706 ASSERT(BP_GET_BYTEORDER(zio->io_bp) == ZFS_HOST_BYTEORDER); 707 ASSERT(!BP_IS_GANG(zio->io_bp)); 708 ASSERT(!BP_IS_HOLE(zio->io_bp)); 709 ASSERT(zio->io_bp->blk_fill == 0); 710 711 /* 712 * Ensure the lwb buffer pointer is cleared before releasing 713 * the txg. If we have had an allocation failure and 714 * the txg is waiting to sync then we want want zil_sync() 715 * to remove the lwb so that it's not picked up as the next new 716 * one in zil_commit_writer(). zil_sync() will only remove 717 * the lwb if lwb_buf is null. 718 */ 719 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 720 mutex_enter(&zilog->zl_lock); 721 lwb->lwb_buf = NULL; 722 if (zio->io_error) 723 zilog->zl_log_error = B_TRUE; 724 725 /* 726 * Now that we've written this log block, we have a stable pointer 727 * to the next block in the chain, so it's OK to let the txg in 728 * which we allocated the next block sync. We still have the 729 * zl_lock to ensure zil_sync doesn't kmem free the lwb. 730 */ 731 txg_rele_to_sync(&lwb->lwb_txgh); 732 mutex_exit(&zilog->zl_lock); 733} 734 735/* 736 * Initialize the io for a log block. 737 */ 738static void 739zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 740{ 741 zbookmark_t zb; 742 743 zb.zb_objset = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET]; 744 zb.zb_object = 0; 745 zb.zb_level = -1; 746 zb.zb_blkid = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 747 748 if (zilog->zl_root_zio == NULL) { 749 zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 750 ZIO_FLAG_CANFAIL); 751 } 752 if (lwb->lwb_zio == NULL) { 753 lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 754 0, &lwb->lwb_blk, lwb->lwb_buf, lwb->lwb_sz, 755 zil_lwb_write_done, lwb, ZIO_PRIORITY_LOG_WRITE, 756 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, &zb); 757 } 758} 759 760/* 761 * Start a log block write and advance to the next log block. 762 * Calls are serialized. 763 */ 764static lwb_t * 765zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 766{ 767 lwb_t *nlwb; 768 zil_trailer_t *ztp = (zil_trailer_t *)(lwb->lwb_buf + lwb->lwb_sz) - 1; 769 spa_t *spa = zilog->zl_spa; 770 blkptr_t *bp = &ztp->zit_next_blk; 771 uint64_t txg; 772 uint64_t zil_blksz; 773 int error; 774 775 ASSERT(lwb->lwb_nused <= ZIL_BLK_DATA_SZ(lwb)); 776 777 /* 778 * Allocate the next block and save its address in this block 779 * before writing it in order to establish the log chain. 780 * Note that if the allocation of nlwb synced before we wrote 781 * the block that points at it (lwb), we'd leak it if we crashed. 782 * Therefore, we don't do txg_rele_to_sync() until zil_lwb_write_done(). 783 */ 784 txg = txg_hold_open(zilog->zl_dmu_pool, &lwb->lwb_txgh); 785 txg_rele_to_quiesce(&lwb->lwb_txgh); 786 787 /* 788 * Pick a ZIL blocksize. We request a size that is the 789 * maximum of the previous used size, the current used size and 790 * the amount waiting in the queue. 791 */ 792 zil_blksz = MAX(zilog->zl_prev_used, 793 zilog->zl_cur_used + sizeof (*ztp)); 794 zil_blksz = MAX(zil_blksz, zilog->zl_itx_list_sz + sizeof (*ztp)); 795 zil_blksz = P2ROUNDUP_TYPED(zil_blksz, ZIL_MIN_BLKSZ, uint64_t); 796 if (zil_blksz > ZIL_MAX_BLKSZ) 797 zil_blksz = ZIL_MAX_BLKSZ; 798 799 BP_ZERO(bp); 800 /* pass the old blkptr in order to spread log blocks across devs */ 801 error = zio_alloc_blk(spa, zil_blksz, bp, &lwb->lwb_blk, txg); 802 if (error) { 803 dmu_tx_t *tx = dmu_tx_create_assigned(zilog->zl_dmu_pool, txg); 804 805 /* 806 * We dirty the dataset to ensure that zil_sync() will 807 * be called to remove this lwb from our zl_lwb_list. 808 * Failing to do so, may leave an lwb with a NULL lwb_buf 809 * hanging around on the zl_lwb_list. 810 */ 811 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 812 dmu_tx_commit(tx); 813 814 /* 815 * Since we've just experienced an allocation failure so we 816 * terminate the current lwb and send it on its way. 817 */ 818 ztp->zit_pad = 0; 819 ztp->zit_nused = lwb->lwb_nused; 820 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 821 zio_nowait(lwb->lwb_zio); 822 823 /* 824 * By returning NULL the caller will call tx_wait_synced() 825 */ 826 return (NULL); 827 } 828 829 ASSERT3U(bp->blk_birth, ==, txg); 830 ztp->zit_pad = 0; 831 ztp->zit_nused = lwb->lwb_nused; 832 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 833 bp->blk_cksum = lwb->lwb_blk.blk_cksum; 834 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 835 836 /* 837 * Allocate a new log write buffer (lwb). 838 */ 839 nlwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 840 841 nlwb->lwb_zilog = zilog; 842 nlwb->lwb_blk = *bp; 843 nlwb->lwb_nused = 0; 844 nlwb->lwb_sz = BP_GET_LSIZE(&nlwb->lwb_blk); 845 nlwb->lwb_buf = zio_buf_alloc(nlwb->lwb_sz); 846 nlwb->lwb_max_txg = txg; 847 nlwb->lwb_zio = NULL; 848 849 /* 850 * Put new lwb at the end of the log chain 851 */ 852 mutex_enter(&zilog->zl_lock); 853 list_insert_tail(&zilog->zl_lwb_list, nlwb); 854 mutex_exit(&zilog->zl_lock); 855 856 /* Record the block for later vdev flushing */ 857 zil_add_block(zilog, &lwb->lwb_blk); 858 859 /* 860 * kick off the write for the old log block 861 */ 862 dprintf_bp(&lwb->lwb_blk, "lwb %p txg %llu: ", lwb, txg); 863 ASSERT(lwb->lwb_zio); 864 zio_nowait(lwb->lwb_zio); 865 866 return (nlwb); 867} 868 869static lwb_t * 870zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 871{ 872 lr_t *lrc = &itx->itx_lr; /* common log record */ 873 lr_write_t *lr = (lr_write_t *)lrc; 874 uint64_t txg = lrc->lrc_txg; 875 uint64_t reclen = lrc->lrc_reclen; 876 uint64_t dlen; 877 878 if (lwb == NULL) 879 return (NULL); 880 ASSERT(lwb->lwb_buf != NULL); 881 882 if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 883 dlen = P2ROUNDUP_TYPED( 884 lr->lr_length, sizeof (uint64_t), uint64_t); 885 else 886 dlen = 0; 887 888 zilog->zl_cur_used += (reclen + dlen); 889 890 zil_lwb_write_init(zilog, lwb); 891 892 /* 893 * If this record won't fit in the current log block, start a new one. 894 */ 895 if (lwb->lwb_nused + reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 896 lwb = zil_lwb_write_start(zilog, lwb); 897 if (lwb == NULL) 898 return (NULL); 899 zil_lwb_write_init(zilog, lwb); 900 ASSERT(lwb->lwb_nused == 0); 901 if (reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 902 txg_wait_synced(zilog->zl_dmu_pool, txg); 903 return (lwb); 904 } 905 } 906 907 /* 908 * Update the lrc_seq, to be log record sequence number. See zil.h 909 * Then copy the record to the log buffer. 910 */ 911 lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 912 bcopy(lrc, lwb->lwb_buf + lwb->lwb_nused, reclen); 913 914 /* 915 * If it's a write, fetch the data or get its blkptr as appropriate. 916 */ 917 if (lrc->lrc_txtype == TX_WRITE) { 918 if (txg > spa_freeze_txg(zilog->zl_spa)) 919 txg_wait_synced(zilog->zl_dmu_pool, txg); 920 if (itx->itx_wr_state != WR_COPIED) { 921 char *dbuf; 922 int error; 923 924 /* alignment is guaranteed */ 925 lr = (lr_write_t *)(lwb->lwb_buf + lwb->lwb_nused); 926 if (dlen) { 927 ASSERT(itx->itx_wr_state == WR_NEED_COPY); 928 dbuf = lwb->lwb_buf + lwb->lwb_nused + reclen; 929 lr->lr_common.lrc_reclen += dlen; 930 } else { 931 ASSERT(itx->itx_wr_state == WR_INDIRECT); 932 dbuf = NULL; 933 } 934 error = zilog->zl_get_data( 935 itx->itx_private, lr, dbuf, lwb->lwb_zio); 936 if (error == EIO) { 937 txg_wait_synced(zilog->zl_dmu_pool, txg); 938 return (lwb); 939 } 940 if (error) { 941 ASSERT(error == ENOENT || error == EEXIST || 942 error == EALREADY); 943 return (lwb); 944 } 945 } 946 } 947 948 lwb->lwb_nused += reclen + dlen; 949 lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 950 ASSERT3U(lwb->lwb_nused, <=, ZIL_BLK_DATA_SZ(lwb)); 951 ASSERT3U(P2PHASE(lwb->lwb_nused, sizeof (uint64_t)), ==, 0); 952 953 return (lwb); 954} 955 956itx_t * 957zil_itx_create(uint64_t txtype, size_t lrsize) 958{ 959 itx_t *itx; 960 961 lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 962 963 itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 964 itx->itx_lr.lrc_txtype = txtype; 965 itx->itx_lr.lrc_reclen = lrsize; 966 itx->itx_sod = lrsize; /* if write & WR_NEED_COPY will be increased */ 967 itx->itx_lr.lrc_seq = 0; /* defensive */ 968 969 return (itx); 970} 971 972uint64_t 973zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 974{ 975 uint64_t seq; 976 977 ASSERT(itx->itx_lr.lrc_seq == 0); 978 979 mutex_enter(&zilog->zl_lock); 980 list_insert_tail(&zilog->zl_itx_list, itx); 981 zilog->zl_itx_list_sz += itx->itx_sod; 982 itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 983 itx->itx_lr.lrc_seq = seq = ++zilog->zl_itx_seq; 984 mutex_exit(&zilog->zl_lock); 985 986 return (seq); 987} 988 989/* 990 * Free up all in-memory intent log transactions that have now been synced. 991 */ 992static void 993zil_itx_clean(zilog_t *zilog) 994{ 995 uint64_t synced_txg = spa_last_synced_txg(zilog->zl_spa); 996 uint64_t freeze_txg = spa_freeze_txg(zilog->zl_spa); 997 list_t clean_list; 998 itx_t *itx; 999 1000 list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 1001 1002 mutex_enter(&zilog->zl_lock); 1003 /* wait for a log writer to finish walking list */ 1004 while (zilog->zl_writer) { 1005 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1006 } 1007 1008 /* 1009 * Move the sync'd log transactions to a separate list so we can call 1010 * kmem_free without holding the zl_lock. 1011 * 1012 * There is no need to set zl_writer as we don't drop zl_lock here 1013 */ 1014 while ((itx = list_head(&zilog->zl_itx_list)) != NULL && 1015 itx->itx_lr.lrc_txg <= MIN(synced_txg, freeze_txg)) { 1016 list_remove(&zilog->zl_itx_list, itx); 1017 zilog->zl_itx_list_sz -= itx->itx_sod; 1018 list_insert_tail(&clean_list, itx); 1019 } 1020 cv_broadcast(&zilog->zl_cv_writer); 1021 mutex_exit(&zilog->zl_lock); 1022 1023 /* destroy sync'd log transactions */ 1024 while ((itx = list_head(&clean_list)) != NULL) { 1025 list_remove(&clean_list, itx); 1026 kmem_free(itx, offsetof(itx_t, itx_lr) 1027 + itx->itx_lr.lrc_reclen); 1028 } 1029 list_destroy(&clean_list); 1030} 1031 1032/* 1033 * If there are any in-memory intent log transactions which have now been 1034 * synced then start up a taskq to free them. 1035 */ 1036void 1037zil_clean(zilog_t *zilog) 1038{ 1039 itx_t *itx; 1040 1041 mutex_enter(&zilog->zl_lock); 1042 itx = list_head(&zilog->zl_itx_list); 1043 if ((itx != NULL) && 1044 (itx->itx_lr.lrc_txg <= spa_last_synced_txg(zilog->zl_spa))) { 1045 (void) taskq_dispatch(zilog->zl_clean_taskq, 1046 (task_func_t *)zil_itx_clean, zilog, TQ_SLEEP); 1047 } 1048 mutex_exit(&zilog->zl_lock); 1049} 1050 1051static void 1052zil_commit_writer(zilog_t *zilog, uint64_t seq, uint64_t foid) 1053{ 1054 uint64_t txg; 1055 uint64_t commit_seq = 0; 1056 itx_t *itx, *itx_next = (itx_t *)-1; 1057 lwb_t *lwb; 1058 spa_t *spa; 1059 1060 zilog->zl_writer = B_TRUE; 1061 ASSERT(zilog->zl_root_zio == NULL); 1062 spa = zilog->zl_spa; 1063 1064 if (zilog->zl_suspend) { 1065 lwb = NULL; 1066 } else { 1067 lwb = list_tail(&zilog->zl_lwb_list); 1068 if (lwb == NULL) { 1069 /* 1070 * Return if there's nothing to flush before we 1071 * dirty the fs by calling zil_create() 1072 */ 1073 if (list_is_empty(&zilog->zl_itx_list)) { 1074 zilog->zl_writer = B_FALSE; 1075 return; 1076 } 1077 mutex_exit(&zilog->zl_lock); 1078 zil_create(zilog); 1079 mutex_enter(&zilog->zl_lock); 1080 lwb = list_tail(&zilog->zl_lwb_list); 1081 } 1082 } 1083 1084 /* Loop through in-memory log transactions filling log blocks. */ 1085 DTRACE_PROBE1(zil__cw1, zilog_t *, zilog); 1086 for (;;) { 1087 /* 1088 * Find the next itx to push: 1089 * Push all transactions related to specified foid and all 1090 * other transactions except TX_WRITE, TX_TRUNCATE, 1091 * TX_SETATTR and TX_ACL for all other files. 1092 */ 1093 if (itx_next != (itx_t *)-1) 1094 itx = itx_next; 1095 else 1096 itx = list_head(&zilog->zl_itx_list); 1097 for (; itx != NULL; itx = list_next(&zilog->zl_itx_list, itx)) { 1098 if (foid == 0) /* push all foids? */ 1099 break; 1100 if (itx->itx_sync) /* push all O_[D]SYNC */ 1101 break; 1102 switch (itx->itx_lr.lrc_txtype) { 1103 case TX_SETATTR: 1104 case TX_WRITE: 1105 case TX_TRUNCATE: 1106 case TX_ACL: 1107 /* lr_foid is same offset for these records */ 1108 if (((lr_write_t *)&itx->itx_lr)->lr_foid 1109 != foid) { 1110 continue; /* skip this record */ 1111 } 1112 } 1113 break; 1114 } 1115 if (itx == NULL) 1116 break; 1117 1118 if ((itx->itx_lr.lrc_seq > seq) && 1119 ((lwb == NULL) || (lwb->lwb_nused == 0) || 1120 (lwb->lwb_nused + itx->itx_sod > ZIL_BLK_DATA_SZ(lwb)))) { 1121 break; 1122 } 1123 1124 /* 1125 * Save the next pointer. Even though we soon drop 1126 * zl_lock all threads that may change the list 1127 * (another writer or zil_itx_clean) can't do so until 1128 * they have zl_writer. 1129 */ 1130 itx_next = list_next(&zilog->zl_itx_list, itx); 1131 list_remove(&zilog->zl_itx_list, itx); 1132 zilog->zl_itx_list_sz -= itx->itx_sod; 1133 mutex_exit(&zilog->zl_lock); 1134 txg = itx->itx_lr.lrc_txg; 1135 ASSERT(txg); 1136 1137 if (txg > spa_last_synced_txg(spa) || 1138 txg > spa_freeze_txg(spa)) 1139 lwb = zil_lwb_commit(zilog, itx, lwb); 1140 kmem_free(itx, offsetof(itx_t, itx_lr) 1141 + itx->itx_lr.lrc_reclen); 1142 mutex_enter(&zilog->zl_lock); 1143 } 1144 DTRACE_PROBE1(zil__cw2, zilog_t *, zilog); 1145 /* determine commit sequence number */ 1146 itx = list_head(&zilog->zl_itx_list); 1147 if (itx) 1148 commit_seq = itx->itx_lr.lrc_seq; 1149 else 1150 commit_seq = zilog->zl_itx_seq; 1151 mutex_exit(&zilog->zl_lock); 1152 1153 /* write the last block out */ 1154 if (lwb != NULL && lwb->lwb_zio != NULL) 1155 lwb = zil_lwb_write_start(zilog, lwb); 1156 1157 zilog->zl_prev_used = zilog->zl_cur_used; 1158 zilog->zl_cur_used = 0; 1159 1160 /* 1161 * Wait if necessary for the log blocks to be on stable storage. 1162 */ 1163 if (zilog->zl_root_zio) { 1164 DTRACE_PROBE1(zil__cw3, zilog_t *, zilog); 1165 (void) zio_wait(zilog->zl_root_zio); 1166 zilog->zl_root_zio = NULL; 1167 DTRACE_PROBE1(zil__cw4, zilog_t *, zilog); 1168 zil_flush_vdevs(zilog); 1169 } 1170 1171 if (zilog->zl_log_error || lwb == NULL) { 1172 zilog->zl_log_error = 0; 1173 txg_wait_synced(zilog->zl_dmu_pool, 0); 1174 } 1175 1176 mutex_enter(&zilog->zl_lock); 1177 zilog->zl_writer = B_FALSE; 1178 1179 ASSERT3U(commit_seq, >=, zilog->zl_commit_seq); 1180 zilog->zl_commit_seq = commit_seq; 1181} 1182 1183/* 1184 * Push zfs transactions to stable storage up to the supplied sequence number. 1185 * If foid is 0 push out all transactions, otherwise push only those 1186 * for that file or might have been used to create that file. 1187 */ 1188void 1189zil_commit(zilog_t *zilog, uint64_t seq, uint64_t foid) 1190{ 1191 if (zilog == NULL || seq == 0) 1192 return; 1193 1194 mutex_enter(&zilog->zl_lock); 1195 1196 seq = MIN(seq, zilog->zl_itx_seq); /* cap seq at largest itx seq */ 1197 1198 while (zilog->zl_writer) { 1199 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1200 if (seq < zilog->zl_commit_seq) { 1201 mutex_exit(&zilog->zl_lock); 1202 return; 1203 } 1204 } 1205 zil_commit_writer(zilog, seq, foid); /* drops zl_lock */ 1206 /* wake up others waiting on the commit */ 1207 cv_broadcast(&zilog->zl_cv_writer); 1208 mutex_exit(&zilog->zl_lock); 1209} 1210 1211/* 1212 * Called in syncing context to free committed log blocks and update log header. 1213 */ 1214void 1215zil_sync(zilog_t *zilog, dmu_tx_t *tx) 1216{ 1217 zil_header_t *zh = zil_header_in_syncing_context(zilog); 1218 uint64_t txg = dmu_tx_get_txg(tx); 1219 spa_t *spa = zilog->zl_spa; 1220 lwb_t *lwb; 1221 1222 /* 1223 * We don't zero out zl_destroy_txg, so make sure we don't try 1224 * to destroy it twice. 1225 */ 1226 if (spa_sync_pass(spa) != 1) 1227 return; 1228 1229 mutex_enter(&zilog->zl_lock); 1230 1231 ASSERT(zilog->zl_stop_sync == 0); 1232 1233 zh->zh_replay_seq = zilog->zl_replayed_seq[txg & TXG_MASK]; 1234 1235 if (zilog->zl_destroy_txg == txg) { 1236 blkptr_t blk = zh->zh_log; 1237 1238 ASSERT(list_head(&zilog->zl_lwb_list) == NULL); 1239 1240 bzero(zh, sizeof (zil_header_t)); 1241 bzero(zilog->zl_replayed_seq, sizeof (zilog->zl_replayed_seq)); 1242 1243 if (zilog->zl_keep_first) { 1244 /* 1245 * If this block was part of log chain that couldn't 1246 * be claimed because a device was missing during 1247 * zil_claim(), but that device later returns, 1248 * then this block could erroneously appear valid. 1249 * To guard against this, assign a new GUID to the new 1250 * log chain so it doesn't matter what blk points to. 1251 */ 1252 zil_init_log_chain(zilog, &blk); 1253 zh->zh_log = blk; 1254 } 1255 } 1256 1257 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1258 zh->zh_log = lwb->lwb_blk; 1259 if (lwb->lwb_buf != NULL || lwb->lwb_max_txg > txg) 1260 break; 1261 list_remove(&zilog->zl_lwb_list, lwb); 1262 zio_free_blk(spa, &lwb->lwb_blk, txg); 1263 kmem_cache_free(zil_lwb_cache, lwb); 1264 1265 /* 1266 * If we don't have anything left in the lwb list then 1267 * we've had an allocation failure and we need to zero 1268 * out the zil_header blkptr so that we don't end 1269 * up freeing the same block twice. 1270 */ 1271 if (list_head(&zilog->zl_lwb_list) == NULL) 1272 BP_ZERO(&zh->zh_log); 1273 } 1274 mutex_exit(&zilog->zl_lock); 1275} 1276 1277void 1278zil_init(void) 1279{ 1280 zil_lwb_cache = kmem_cache_create("zil_lwb_cache", 1281 sizeof (struct lwb), 0, NULL, NULL, NULL, NULL, NULL, 0); 1282} 1283 1284void 1285zil_fini(void) 1286{ 1287 kmem_cache_destroy(zil_lwb_cache); 1288} 1289 1290zilog_t * 1291zil_alloc(objset_t *os, zil_header_t *zh_phys) 1292{ 1293 zilog_t *zilog; 1294 1295 zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); 1296 1297 zilog->zl_header = zh_phys; 1298 zilog->zl_os = os; 1299 zilog->zl_spa = dmu_objset_spa(os); 1300 zilog->zl_dmu_pool = dmu_objset_pool(os); 1301 zilog->zl_destroy_txg = TXG_INITIAL - 1; 1302 1303 mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL); 1304 1305 list_create(&zilog->zl_itx_list, sizeof (itx_t), 1306 offsetof(itx_t, itx_node)); 1307 1308 list_create(&zilog->zl_lwb_list, sizeof (lwb_t), 1309 offsetof(lwb_t, lwb_node)); 1310 1311 mutex_init(&zilog->zl_vdev_lock, NULL, MUTEX_DEFAULT, NULL); 1312 1313 avl_create(&zilog->zl_vdev_tree, zil_vdev_compare, 1314 sizeof (zil_vdev_node_t), offsetof(zil_vdev_node_t, zv_node)); 1315 1316 cv_init(&zilog->zl_cv_writer, NULL, CV_DEFAULT, NULL); 1317 cv_init(&zilog->zl_cv_suspend, NULL, CV_DEFAULT, NULL); 1318 1319 return (zilog); 1320} 1321 1322void 1323zil_free(zilog_t *zilog) 1324{ 1325 lwb_t *lwb; 1326 1327 zilog->zl_stop_sync = 1; 1328 1329 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1330 list_remove(&zilog->zl_lwb_list, lwb); 1331 if (lwb->lwb_buf != NULL) 1332 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 1333 kmem_cache_free(zil_lwb_cache, lwb); 1334 } 1335 list_destroy(&zilog->zl_lwb_list); 1336 1337 avl_destroy(&zilog->zl_vdev_tree); 1338 mutex_destroy(&zilog->zl_vdev_lock); 1339 1340 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1341 list_destroy(&zilog->zl_itx_list); 1342 mutex_destroy(&zilog->zl_lock); 1343 1344 cv_destroy(&zilog->zl_cv_writer); 1345 cv_destroy(&zilog->zl_cv_suspend); 1346 1347 kmem_free(zilog, sizeof (zilog_t)); 1348} 1349 1350/* 1351 * Open an intent log. 1352 */ 1353zilog_t * 1354zil_open(objset_t *os, zil_get_data_t *get_data) 1355{ 1356 zilog_t *zilog = dmu_objset_zil(os); 1357 1358 zilog->zl_get_data = get_data; 1359 zilog->zl_clean_taskq = taskq_create("zil_clean", 1, minclsyspri, 1360 2, 2, TASKQ_PREPOPULATE); 1361 1362 return (zilog); 1363} 1364 1365/* 1366 * Close an intent log. 1367 */ 1368void 1369zil_close(zilog_t *zilog) 1370{ 1371 /* 1372 * If the log isn't already committed, mark the objset dirty 1373 * (so zil_sync() will be called) and wait for that txg to sync. 1374 */ 1375 if (!zil_is_committed(zilog)) { 1376 uint64_t txg; 1377 dmu_tx_t *tx = dmu_tx_create(zilog->zl_os); 1378 (void) dmu_tx_assign(tx, TXG_WAIT); 1379 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 1380 txg = dmu_tx_get_txg(tx); 1381 dmu_tx_commit(tx); 1382 txg_wait_synced(zilog->zl_dmu_pool, txg); 1383 } 1384 1385 taskq_destroy(zilog->zl_clean_taskq); 1386 zilog->zl_clean_taskq = NULL; 1387 zilog->zl_get_data = NULL; 1388 1389 zil_itx_clean(zilog); 1390 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1391} 1392 1393/* 1394 * Suspend an intent log. While in suspended mode, we still honor 1395 * synchronous semantics, but we rely on txg_wait_synced() to do it. 1396 * We suspend the log briefly when taking a snapshot so that the snapshot 1397 * contains all the data it's supposed to, and has an empty intent log. 1398 */ 1399int 1400zil_suspend(zilog_t *zilog) 1401{ 1402 const zil_header_t *zh = zilog->zl_header; 1403 1404 mutex_enter(&zilog->zl_lock); 1405 if (zh->zh_flags & ZIL_REPLAY_NEEDED) { /* unplayed log */ 1406 mutex_exit(&zilog->zl_lock); 1407 return (EBUSY); 1408 } 1409 if (zilog->zl_suspend++ != 0) { 1410 /* 1411 * Someone else already began a suspend. 1412 * Just wait for them to finish. 1413 */ 1414 while (zilog->zl_suspending) 1415 cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); 1416 mutex_exit(&zilog->zl_lock); 1417 return (0); 1418 } 1419 zilog->zl_suspending = B_TRUE; 1420 mutex_exit(&zilog->zl_lock); 1421 1422 zil_commit(zilog, UINT64_MAX, 0); 1423 1424 /* 1425 * Wait for any in-flight log writes to complete. 1426 */ 1427 mutex_enter(&zilog->zl_lock); 1428 while (zilog->zl_writer) 1429 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1430 mutex_exit(&zilog->zl_lock); 1431 1432 zil_destroy(zilog, B_FALSE); 1433 1434 mutex_enter(&zilog->zl_lock); 1435 zilog->zl_suspending = B_FALSE; 1436 cv_broadcast(&zilog->zl_cv_suspend); 1437 mutex_exit(&zilog->zl_lock); 1438 1439 return (0); 1440} 1441 1442void 1443zil_resume(zilog_t *zilog) 1444{ 1445 mutex_enter(&zilog->zl_lock); 1446 ASSERT(zilog->zl_suspend != 0); 1447 zilog->zl_suspend--; 1448 mutex_exit(&zilog->zl_lock); 1449} 1450 1451/* 1452 * Read in the data for the dmu_sync()ed block, and change the log 1453 * record to write this whole block. 1454 */ 1455void 1456zil_get_replay_data(zilog_t *zilog, lr_write_t *lr) 1457{ 1458 blkptr_t *wbp = &lr->lr_blkptr; 1459 char *wbuf = (char *)(lr + 1); /* data follows lr_write_t */ 1460 uint64_t blksz; 1461 1462 if (BP_IS_HOLE(wbp)) { /* compressed to a hole */ 1463 blksz = BP_GET_LSIZE(&lr->lr_blkptr); 1464 /* 1465 * If the blksz is zero then we must be replaying a log 1466 * from an version prior to setting the blksize of null blocks. 1467 * So we just zero the actual write size reqeusted. 1468 */ 1469 if (blksz == 0) { 1470 bzero(wbuf, lr->lr_length); 1471 return; 1472 } 1473 bzero(wbuf, blksz); 1474 } else { 1475 /* 1476 * A subsequent write may have overwritten this block, in which 1477 * case wbp may have been been freed and reallocated, and our 1478 * read of wbp may fail with a checksum error. We can safely 1479 * ignore this because the later write will provide the 1480 * correct data. 1481 */ 1482 zbookmark_t zb; 1483 1484 zb.zb_objset = dmu_objset_id(zilog->zl_os); 1485 zb.zb_object = lr->lr_foid; 1486 zb.zb_level = 0; 1487 zb.zb_blkid = -1; /* unknown */ 1488 1489 blksz = BP_GET_LSIZE(&lr->lr_blkptr); 1490 (void) zio_wait(zio_read(NULL, zilog->zl_spa, wbp, wbuf, blksz, 1491 NULL, NULL, ZIO_PRIORITY_SYNC_READ, 1492 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, &zb)); 1493 } 1494 lr->lr_offset -= lr->lr_offset % blksz; 1495 lr->lr_length = blksz; 1496} 1497 1498typedef struct zil_replay_arg { 1499 objset_t *zr_os; 1500 zil_replay_func_t **zr_replay; 1501 void *zr_arg; 1502 boolean_t zr_byteswap; 1503 char *zr_lrbuf; 1504} zil_replay_arg_t; 1505 1506static void 1507zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) 1508{ 1509 zil_replay_arg_t *zr = zra; 1510 const zil_header_t *zh = zilog->zl_header; 1511 uint64_t reclen = lr->lrc_reclen; 1512 uint64_t txtype = lr->lrc_txtype; 1513 char *name; 1514 int pass, error; 1515 1516 if (!zilog->zl_replay) /* giving up */ 1517 return; 1518 1519 if (lr->lrc_txg < claim_txg) /* already committed */ 1520 return; 1521 1522 if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */ 1523 return; 1524 1525 /* Strip case-insensitive bit, still present in log record */ 1526 txtype &= ~TX_CI; 1527 1528 if (txtype == 0 || txtype >= TX_MAX_TYPE) { 1529 error = EINVAL; 1530 goto bad; 1531 } 1532 1533 /* 1534 * Make a copy of the data so we can revise and extend it. 1535 */ 1536 bcopy(lr, zr->zr_lrbuf, reclen); 1537 1538 /* 1539 * The log block containing this lr may have been byteswapped 1540 * so that we can easily examine common fields like lrc_txtype. 1541 * However, the log is a mix of different data types, and only the 1542 * replay vectors know how to byteswap their records. Therefore, if 1543 * the lr was byteswapped, undo it before invoking the replay vector. 1544 */ 1545 if (zr->zr_byteswap) 1546 byteswap_uint64_array(zr->zr_lrbuf, reclen); 1547 1548 /* 1549 * We must now do two things atomically: replay this log record, 1550 * and update the log header sequence number to reflect the fact that 1551 * we did so. At the end of each replay function the sequence number 1552 * is updated if we are in replay mode. 1553 */ 1554 for (pass = 1; pass <= 2; pass++) { 1555 zilog->zl_replaying_seq = lr->lrc_seq; 1556 /* Only byteswap (if needed) on the 1st pass. */ 1557 error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lrbuf, 1558 zr->zr_byteswap && pass == 1); 1559 1560 if (!error) 1561 return; 1562 1563 /* 1564 * The DMU's dnode layer doesn't see removes until the txg 1565 * commits, so a subsequent claim can spuriously fail with 1566 * EEXIST. So if we receive any error we try syncing out 1567 * any removes then retry the transaction. 1568 */ 1569 if (pass == 1) 1570 txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0); 1571 } 1572 1573bad: 1574 ASSERT(error); 1575 name = kmem_alloc(MAXNAMELEN, KM_SLEEP); 1576 dmu_objset_name(zr->zr_os, name); 1577 cmn_err(CE_WARN, "ZFS replay transaction error %d, " 1578 "dataset %s, seq 0x%llx, txtype %llu %s\n", 1579 error, name, (u_longlong_t)lr->lrc_seq, (u_longlong_t)txtype, 1580 (lr->lrc_txtype & TX_CI) ? "CI" : ""); 1581 zilog->zl_replay = B_FALSE; 1582 kmem_free(name, MAXNAMELEN); 1583} 1584 1585/* ARGSUSED */ 1586static void 1587zil_incr_blks(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 1588{ 1589 zilog->zl_replay_blks++; 1590} 1591 1592/* 1593 * If this dataset has a non-empty intent log, replay it and destroy it. 1594 */ 1595void 1596zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) 1597{ 1598 zilog_t *zilog = dmu_objset_zil(os); 1599 const zil_header_t *zh = zilog->zl_header; 1600 zil_replay_arg_t zr; 1601 1602 if ((zh->zh_flags & ZIL_REPLAY_NEEDED) == 0) { 1603 zil_destroy(zilog, B_TRUE); 1604 return; 1605 } 1606 //printf("ZFS: Replaying ZIL on %s...\n", os->os->os_spa->spa_name); 1607 1608 zr.zr_os = os; 1609 zr.zr_replay = replay_func; 1610 zr.zr_arg = arg; 1611 zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); 1612 zr.zr_lrbuf = kmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); 1613 1614 /* 1615 * Wait for in-progress removes to sync before starting replay. 1616 */ 1617 txg_wait_synced(zilog->zl_dmu_pool, 0); 1618 1619 zilog->zl_replay = B_TRUE; 1620 zilog->zl_replay_time = LBOLT; 1621 ASSERT(zilog->zl_replay_blks == 0); 1622 (void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr, 1623 zh->zh_claim_txg); 1624 kmem_free(zr.zr_lrbuf, 2 * SPA_MAXBLOCKSIZE); 1625 1626 zil_destroy(zilog, B_FALSE); 1627 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 1628 zilog->zl_replay = B_FALSE; 1629 //printf("ZFS: Replay of ZIL on %s finished.\n", os->os->os_spa->spa_name); 1630} 1631 1632/* 1633 * Report whether all transactions are committed 1634 */ 1635int 1636zil_is_committed(zilog_t *zilog) 1637{ 1638 lwb_t *lwb; 1639 int ret; 1640 1641 mutex_enter(&zilog->zl_lock); 1642 while (zilog->zl_writer) 1643 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1644 1645 /* recent unpushed intent log transactions? */ 1646 if (!list_is_empty(&zilog->zl_itx_list)) { 1647 ret = B_FALSE; 1648 goto out; 1649 } 1650 1651 /* intent log never used? */ 1652 lwb = list_head(&zilog->zl_lwb_list); 1653 if (lwb == NULL) { 1654 ret = B_TRUE; 1655 goto out; 1656 } 1657 1658 /* 1659 * more than 1 log buffer means zil_sync() hasn't yet freed 1660 * entries after a txg has committed 1661 */ 1662 if (list_next(&zilog->zl_lwb_list, lwb)) { 1663 ret = B_FALSE; 1664 goto out; 1665 } 1666 1667 ASSERT(zil_empty(zilog)); 1668 ret = B_TRUE; 1669out: 1670 cv_broadcast(&zilog->zl_cv_writer); 1671 mutex_exit(&zilog->zl_lock); 1672 return (ret); 1673} 1674 1675/* ARGSUSED */ 1676int 1677zil_vdev_offline(char *osname, void *arg) 1678{ 1679 objset_t *os; 1680 zilog_t *zilog; 1681 int error; 1682 1683 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_USER, &os); 1684 if (error) 1685 return (error); 1686 1687 zilog = dmu_objset_zil(os); 1688 if (zil_suspend(zilog) != 0) 1689 error = EEXIST; 1690 else 1691 zil_resume(zilog); 1692 dmu_objset_close(os); 1693 return (error); 1694} 1695