1// SPDX-License-Identifier: GPL-2.0 2 3/* 4 * fs/ext4/fast_commit.c 5 * 6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com> 7 * 8 * Ext4 fast commits routines. 9 */ 10#include "ext4.h" 11#include "ext4_jbd2.h" 12#include "ext4_extents.h" 13#include "mballoc.h" 14 15/* 16 * Ext4 Fast Commits 17 * ----------------- 18 * 19 * Ext4 fast commits implement fine grained journalling for Ext4. 20 * 21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See 22 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by 23 * TLV during the recovery phase. For the scenarios for which we currently 24 * don't have replay code, fast commit falls back to full commits. 25 * Fast commits record delta in one of the following three categories. 26 * 27 * (A) Directory entry updates: 28 * 29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink 30 * - EXT4_FC_TAG_LINK - records directory entry link 31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation 32 * 33 * (B) File specific data range updates: 34 * 35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode 36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode 37 * 38 * (C) Inode metadata (mtime / ctime etc): 39 * 40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed 41 * during recovery. Note that iblocks field is 42 * not replayed and instead derived during 43 * replay. 44 * Commit Operation 45 * ---------------- 46 * With fast commits, we maintain all the directory entry operations in the 47 * order in which they are issued in an in-memory queue. This queue is flushed 48 * to disk during the commit operation. We also maintain a list of inodes 49 * that need to be committed during a fast commit in another in memory queue of 50 * inodes. During the commit operation, we commit in the following order: 51 * 52 * [1] Lock inodes for any further data updates by setting COMMITTING state 53 * [2] Submit data buffers of all the inodes 54 * [3] Wait for [2] to complete 55 * [4] Commit all the directory entry updates in the fast commit space 56 * [5] Commit all the changed inode structures 57 * [6] Write tail tag (this tag ensures the atomicity, please read the following 58 * section for more details). 59 * [7] Wait for [4], [5] and [6] to complete. 60 * 61 * All the inode updates must call ext4_fc_start_update() before starting an 62 * update. If such an ongoing update is present, fast commit waits for it to 63 * complete. The completion of such an update is marked by 64 * ext4_fc_stop_update(). 65 * 66 * Fast Commit Ineligibility 67 * ------------------------- 68 * 69 * Not all operations are supported by fast commits today (e.g extended 70 * attributes). Fast commit ineligibility is marked by calling 71 * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back 72 * to full commit. 73 * 74 * Atomicity of commits 75 * -------------------- 76 * In order to guarantee atomicity during the commit operation, fast commit 77 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail 78 * tag contains CRC of the contents and TID of the transaction after which 79 * this fast commit should be applied. Recovery code replays fast commit 80 * logs only if there's at least 1 valid tail present. For every fast commit 81 * operation, there is 1 tail. This means, we may end up with multiple tails 82 * in the fast commit space. Here's an example: 83 * 84 * - Create a new file A and remove existing file B 85 * - fsync() 86 * - Append contents to file A 87 * - Truncate file A 88 * - fsync() 89 * 90 * The fast commit space at the end of above operations would look like this: 91 * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL] 92 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->| 93 * 94 * Replay code should thus check for all the valid tails in the FC area. 95 * 96 * Fast Commit Replay Idempotence 97 * ------------------------------ 98 * 99 * Fast commits tags are idempotent in nature provided the recovery code follows 100 * certain rules. The guiding principle that the commit path follows while 101 * committing is that it stores the result of a particular operation instead of 102 * storing the procedure. 103 * 104 * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a' 105 * was associated with inode 10. During fast commit, instead of storing this 106 * operation as a procedure "rename a to b", we store the resulting file system 107 * state as a "series" of outcomes: 108 * 109 * - Link dirent b to inode 10 110 * - Unlink dirent a 111 * - Inode <10> with valid refcount 112 * 113 * Now when recovery code runs, it needs "enforce" this state on the file 114 * system. This is what guarantees idempotence of fast commit replay. 115 * 116 * Let's take an example of a procedure that is not idempotent and see how fast 117 * commits make it idempotent. Consider following sequence of operations: 118 * 119 * rm A; mv B A; read A 120 * (x) (y) (z) 121 * 122 * (x), (y) and (z) are the points at which we can crash. If we store this 123 * sequence of operations as is then the replay is not idempotent. Let's say 124 * while in replay, we crash at (z). During the second replay, file A (which was 125 * actually created as a result of "mv B A" operation) would get deleted. Thus, 126 * file named A would be absent when we try to read A. So, this sequence of 127 * operations is not idempotent. However, as mentioned above, instead of storing 128 * the procedure fast commits store the outcome of each procedure. Thus the fast 129 * commit log for above procedure would be as follows: 130 * 131 * (Let's assume dirent A was linked to inode 10 and dirent B was linked to 132 * inode 11 before the replay) 133 * 134 * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11] 135 * (w) (x) (y) (z) 136 * 137 * If we crash at (z), we will have file A linked to inode 11. During the second 138 * replay, we will remove file A (inode 11). But we will create it back and make 139 * it point to inode 11. We won't find B, so we'll just skip that step. At this 140 * point, the refcount for inode 11 is not reliable, but that gets fixed by the 141 * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled 142 * similarly. Thus, by converting a non-idempotent procedure into a series of 143 * idempotent outcomes, fast commits ensured idempotence during the replay. 144 * 145 * TODOs 146 * ----- 147 * 148 * 0) Fast commit replay path hardening: Fast commit replay code should use 149 * journal handles to make sure all the updates it does during the replay 150 * path are atomic. With that if we crash during fast commit replay, after 151 * trying to do recovery again, we will find a file system where fast commit 152 * area is invalid (because new full commit would be found). In order to deal 153 * with that, fast commit replay code should ensure that the "FC_REPLAY" 154 * superblock state is persisted before starting the replay, so that after 155 * the crash, fast commit recovery code can look at that flag and perform 156 * fast commit recovery even if that area is invalidated by later full 157 * commits. 158 * 159 * 1) Fast commit's commit path locks the entire file system during fast 160 * commit. This has significant performance penalty. Instead of that, we 161 * should use ext4_fc_start/stop_update functions to start inode level 162 * updates from ext4_journal_start/stop. Once we do that we can drop file 163 * system locking during commit path. 164 * 165 * 2) Handle more ineligible cases. 166 */ 167 168#include <trace/events/ext4.h> 169static struct kmem_cache *ext4_fc_dentry_cachep; 170 171static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate) 172{ 173 BUFFER_TRACE(bh, ""); 174 if (uptodate) { 175 ext4_debug("%s: Block %lld up-to-date", 176 __func__, bh->b_blocknr); 177 set_buffer_uptodate(bh); 178 } else { 179 ext4_debug("%s: Block %lld not up-to-date", 180 __func__, bh->b_blocknr); 181 clear_buffer_uptodate(bh); 182 } 183 184 unlock_buffer(bh); 185} 186 187static inline void ext4_fc_reset_inode(struct inode *inode) 188{ 189 struct ext4_inode_info *ei = EXT4_I(inode); 190 191 ei->i_fc_lblk_start = 0; 192 ei->i_fc_lblk_len = 0; 193} 194 195void ext4_fc_init_inode(struct inode *inode) 196{ 197 struct ext4_inode_info *ei = EXT4_I(inode); 198 199 ext4_fc_reset_inode(inode); 200 ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); 201 INIT_LIST_HEAD(&ei->i_fc_list); 202 INIT_LIST_HEAD(&ei->i_fc_dilist); 203 init_waitqueue_head(&ei->i_fc_wait); 204 atomic_set(&ei->i_fc_updates, 0); 205} 206 207/* This function must be called with sbi->s_fc_lock held. */ 208static void ext4_fc_wait_committing_inode(struct inode *inode) 209__releases(&EXT4_SB(inode->i_sb)->s_fc_lock) 210{ 211 wait_queue_head_t *wq; 212 struct ext4_inode_info *ei = EXT4_I(inode); 213 214#if (BITS_PER_LONG < 64) 215 DEFINE_WAIT_BIT(wait, &ei->i_state_flags, 216 EXT4_STATE_FC_COMMITTING); 217 wq = bit_waitqueue(&ei->i_state_flags, 218 EXT4_STATE_FC_COMMITTING); 219#else 220 DEFINE_WAIT_BIT(wait, &ei->i_flags, 221 EXT4_STATE_FC_COMMITTING); 222 wq = bit_waitqueue(&ei->i_flags, 223 EXT4_STATE_FC_COMMITTING); 224#endif 225 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock); 226 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); 227 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 228 schedule(); 229 finish_wait(wq, &wait.wq_entry); 230} 231 232static bool ext4_fc_disabled(struct super_block *sb) 233{ 234 return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || 235 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); 236} 237 238/* 239 * Inform Ext4's fast about start of an inode update 240 * 241 * This function is called by the high level call VFS callbacks before 242 * performing any inode update. This function blocks if there's an ongoing 243 * fast commit on the inode in question. 244 */ 245void ext4_fc_start_update(struct inode *inode) 246{ 247 struct ext4_inode_info *ei = EXT4_I(inode); 248 249 if (ext4_fc_disabled(inode->i_sb)) 250 return; 251 252restart: 253 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); 254 if (list_empty(&ei->i_fc_list)) 255 goto out; 256 257 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { 258 ext4_fc_wait_committing_inode(inode); 259 goto restart; 260 } 261out: 262 atomic_inc(&ei->i_fc_updates); 263 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 264} 265 266/* 267 * Stop inode update and wake up waiting fast commits if any. 268 */ 269void ext4_fc_stop_update(struct inode *inode) 270{ 271 struct ext4_inode_info *ei = EXT4_I(inode); 272 273 if (ext4_fc_disabled(inode->i_sb)) 274 return; 275 276 if (atomic_dec_and_test(&ei->i_fc_updates)) 277 wake_up_all(&ei->i_fc_wait); 278} 279 280/* 281 * Remove inode from fast commit list. If the inode is being committed 282 * we wait until inode commit is done. 283 */ 284void ext4_fc_del(struct inode *inode) 285{ 286 struct ext4_inode_info *ei = EXT4_I(inode); 287 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 288 struct ext4_fc_dentry_update *fc_dentry; 289 290 if (ext4_fc_disabled(inode->i_sb)) 291 return; 292 293restart: 294 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); 295 if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { 296 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); 297 return; 298 } 299 300 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) { 301 ext4_fc_wait_committing_inode(inode); 302 goto restart; 303 } 304 305 if (!list_empty(&ei->i_fc_list)) 306 list_del_init(&ei->i_fc_list); 307 308 /* 309 * Since this inode is getting removed, let's also remove all FC 310 * dentry create references, since it is not needed to log it anyways. 311 */ 312 if (list_empty(&ei->i_fc_dilist)) { 313 spin_unlock(&sbi->s_fc_lock); 314 return; 315 } 316 317 fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist); 318 WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); 319 list_del_init(&fc_dentry->fcd_list); 320 list_del_init(&fc_dentry->fcd_dilist); 321 322 WARN_ON(!list_empty(&ei->i_fc_dilist)); 323 spin_unlock(&sbi->s_fc_lock); 324 325 if (fc_dentry->fcd_name.name && 326 fc_dentry->fcd_name.len > DNAME_INLINE_LEN) 327 kfree(fc_dentry->fcd_name.name); 328 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); 329 330 return; 331} 332 333/* 334 * Mark file system as fast commit ineligible, and record latest 335 * ineligible transaction tid. This means until the recorded 336 * transaction, commit operation would result in a full jbd2 commit. 337 */ 338void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle) 339{ 340 struct ext4_sb_info *sbi = EXT4_SB(sb); 341 tid_t tid; 342 343 if (ext4_fc_disabled(sb)) 344 return; 345 346 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); 347 if (handle && !IS_ERR(handle)) 348 tid = handle->h_transaction->t_tid; 349 else { 350 read_lock(&sbi->s_journal->j_state_lock); 351 tid = sbi->s_journal->j_running_transaction ? 352 sbi->s_journal->j_running_transaction->t_tid : 0; 353 read_unlock(&sbi->s_journal->j_state_lock); 354 } 355 spin_lock(&sbi->s_fc_lock); 356 if (sbi->s_fc_ineligible_tid < tid) 357 sbi->s_fc_ineligible_tid = tid; 358 spin_unlock(&sbi->s_fc_lock); 359 WARN_ON(reason >= EXT4_FC_REASON_MAX); 360 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; 361} 362 363/* 364 * Generic fast commit tracking function. If this is the first time this we are 365 * called after a full commit, we initialize fast commit fields and then call 366 * __fc_track_fn() with update = 0. If we have already been called after a full 367 * commit, we pass update = 1. Based on that, the track function can determine 368 * if it needs to track a field for the first time or if it needs to just 369 * update the previously tracked value. 370 * 371 * If enqueue is set, this function enqueues the inode in fast commit list. 372 */ 373static int ext4_fc_track_template( 374 handle_t *handle, struct inode *inode, 375 int (*__fc_track_fn)(struct inode *, void *, bool), 376 void *args, int enqueue) 377{ 378 bool update = false; 379 struct ext4_inode_info *ei = EXT4_I(inode); 380 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 381 tid_t tid = 0; 382 int ret; 383 384 tid = handle->h_transaction->t_tid; 385 mutex_lock(&ei->i_fc_lock); 386 if (tid == ei->i_sync_tid) { 387 update = true; 388 } else { 389 ext4_fc_reset_inode(inode); 390 ei->i_sync_tid = tid; 391 } 392 ret = __fc_track_fn(inode, args, update); 393 mutex_unlock(&ei->i_fc_lock); 394 395 if (!enqueue) 396 return ret; 397 398 spin_lock(&sbi->s_fc_lock); 399 if (list_empty(&EXT4_I(inode)->i_fc_list)) 400 list_add_tail(&EXT4_I(inode)->i_fc_list, 401 (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || 402 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? 403 &sbi->s_fc_q[FC_Q_STAGING] : 404 &sbi->s_fc_q[FC_Q_MAIN]); 405 spin_unlock(&sbi->s_fc_lock); 406 407 return ret; 408} 409 410struct __track_dentry_update_args { 411 struct dentry *dentry; 412 int op; 413}; 414 415/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */ 416static int __track_dentry_update(struct inode *inode, void *arg, bool update) 417{ 418 struct ext4_fc_dentry_update *node; 419 struct ext4_inode_info *ei = EXT4_I(inode); 420 struct __track_dentry_update_args *dentry_update = 421 (struct __track_dentry_update_args *)arg; 422 struct dentry *dentry = dentry_update->dentry; 423 struct inode *dir = dentry->d_parent->d_inode; 424 struct super_block *sb = inode->i_sb; 425 struct ext4_sb_info *sbi = EXT4_SB(sb); 426 427 mutex_unlock(&ei->i_fc_lock); 428 429 if (IS_ENCRYPTED(dir)) { 430 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME, 431 NULL); 432 mutex_lock(&ei->i_fc_lock); 433 return -EOPNOTSUPP; 434 } 435 436 node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS); 437 if (!node) { 438 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL); 439 mutex_lock(&ei->i_fc_lock); 440 return -ENOMEM; 441 } 442 443 node->fcd_op = dentry_update->op; 444 node->fcd_parent = dir->i_ino; 445 node->fcd_ino = inode->i_ino; 446 if (dentry->d_name.len > DNAME_INLINE_LEN) { 447 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS); 448 if (!node->fcd_name.name) { 449 kmem_cache_free(ext4_fc_dentry_cachep, node); 450 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL); 451 mutex_lock(&ei->i_fc_lock); 452 return -ENOMEM; 453 } 454 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name, 455 dentry->d_name.len); 456 } else { 457 memcpy(node->fcd_iname, dentry->d_name.name, 458 dentry->d_name.len); 459 node->fcd_name.name = node->fcd_iname; 460 } 461 node->fcd_name.len = dentry->d_name.len; 462 INIT_LIST_HEAD(&node->fcd_dilist); 463 spin_lock(&sbi->s_fc_lock); 464 if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || 465 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) 466 list_add_tail(&node->fcd_list, 467 &sbi->s_fc_dentry_q[FC_Q_STAGING]); 468 else 469 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); 470 471 /* 472 * This helps us keep a track of all fc_dentry updates which is part of 473 * this ext4 inode. So in case the inode is getting unlinked, before 474 * even we get a chance to fsync, we could remove all fc_dentry 475 * references while evicting the inode in ext4_fc_del(). 476 * Also with this, we don't need to loop over all the inodes in 477 * sbi->s_fc_q to get the corresponding inode in 478 * ext4_fc_commit_dentry_updates(). 479 */ 480 if (dentry_update->op == EXT4_FC_TAG_CREAT) { 481 WARN_ON(!list_empty(&ei->i_fc_dilist)); 482 list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); 483 } 484 spin_unlock(&sbi->s_fc_lock); 485 mutex_lock(&ei->i_fc_lock); 486 487 return 0; 488} 489 490void __ext4_fc_track_unlink(handle_t *handle, 491 struct inode *inode, struct dentry *dentry) 492{ 493 struct __track_dentry_update_args args; 494 int ret; 495 496 args.dentry = dentry; 497 args.op = EXT4_FC_TAG_UNLINK; 498 499 ret = ext4_fc_track_template(handle, inode, __track_dentry_update, 500 (void *)&args, 0); 501 trace_ext4_fc_track_unlink(handle, inode, dentry, ret); 502} 503 504void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry) 505{ 506 struct inode *inode = d_inode(dentry); 507 508 if (ext4_fc_disabled(inode->i_sb)) 509 return; 510 511 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) 512 return; 513 514 __ext4_fc_track_unlink(handle, inode, dentry); 515} 516 517void __ext4_fc_track_link(handle_t *handle, 518 struct inode *inode, struct dentry *dentry) 519{ 520 struct __track_dentry_update_args args; 521 int ret; 522 523 args.dentry = dentry; 524 args.op = EXT4_FC_TAG_LINK; 525 526 ret = ext4_fc_track_template(handle, inode, __track_dentry_update, 527 (void *)&args, 0); 528 trace_ext4_fc_track_link(handle, inode, dentry, ret); 529} 530 531void ext4_fc_track_link(handle_t *handle, struct dentry *dentry) 532{ 533 struct inode *inode = d_inode(dentry); 534 535 if (ext4_fc_disabled(inode->i_sb)) 536 return; 537 538 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) 539 return; 540 541 __ext4_fc_track_link(handle, inode, dentry); 542} 543 544void __ext4_fc_track_create(handle_t *handle, struct inode *inode, 545 struct dentry *dentry) 546{ 547 struct __track_dentry_update_args args; 548 int ret; 549 550 args.dentry = dentry; 551 args.op = EXT4_FC_TAG_CREAT; 552 553 ret = ext4_fc_track_template(handle, inode, __track_dentry_update, 554 (void *)&args, 0); 555 trace_ext4_fc_track_create(handle, inode, dentry, ret); 556} 557 558void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) 559{ 560 struct inode *inode = d_inode(dentry); 561 562 if (ext4_fc_disabled(inode->i_sb)) 563 return; 564 565 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) 566 return; 567 568 __ext4_fc_track_create(handle, inode, dentry); 569} 570 571/* __track_fn for inode tracking */ 572static int __track_inode(struct inode *inode, void *arg, bool update) 573{ 574 if (update) 575 return -EEXIST; 576 577 EXT4_I(inode)->i_fc_lblk_len = 0; 578 579 return 0; 580} 581 582void ext4_fc_track_inode(handle_t *handle, struct inode *inode) 583{ 584 int ret; 585 586 if (S_ISDIR(inode->i_mode)) 587 return; 588 589 if (ext4_fc_disabled(inode->i_sb)) 590 return; 591 592 if (ext4_should_journal_data(inode)) { 593 ext4_fc_mark_ineligible(inode->i_sb, 594 EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); 595 return; 596 } 597 598 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) 599 return; 600 601 ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1); 602 trace_ext4_fc_track_inode(handle, inode, ret); 603} 604 605struct __track_range_args { 606 ext4_lblk_t start, end; 607}; 608 609/* __track_fn for tracking data updates */ 610static int __track_range(struct inode *inode, void *arg, bool update) 611{ 612 struct ext4_inode_info *ei = EXT4_I(inode); 613 ext4_lblk_t oldstart; 614 struct __track_range_args *__arg = 615 (struct __track_range_args *)arg; 616 617 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { 618 ext4_debug("Special inode %ld being modified\n", inode->i_ino); 619 return -ECANCELED; 620 } 621 622 oldstart = ei->i_fc_lblk_start; 623 624 if (update && ei->i_fc_lblk_len > 0) { 625 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); 626 ei->i_fc_lblk_len = 627 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - 628 ei->i_fc_lblk_start + 1; 629 } else { 630 ei->i_fc_lblk_start = __arg->start; 631 ei->i_fc_lblk_len = __arg->end - __arg->start + 1; 632 } 633 634 return 0; 635} 636 637void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, 638 ext4_lblk_t end) 639{ 640 struct __track_range_args args; 641 int ret; 642 643 if (S_ISDIR(inode->i_mode)) 644 return; 645 646 if (ext4_fc_disabled(inode->i_sb)) 647 return; 648 649 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE)) 650 return; 651 652 args.start = start; 653 args.end = end; 654 655 ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1); 656 657 trace_ext4_fc_track_range(handle, inode, start, end, ret); 658} 659 660static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) 661{ 662 blk_opf_t write_flags = REQ_SYNC; 663 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; 664 665 /* Add REQ_FUA | REQ_PREFLUSH only its tail */ 666 if (test_opt(sb, BARRIER) && is_tail) 667 write_flags |= REQ_FUA | REQ_PREFLUSH; 668 lock_buffer(bh); 669 set_buffer_dirty(bh); 670 set_buffer_uptodate(bh); 671 bh->b_end_io = ext4_end_buffer_io_sync; 672 submit_bh(REQ_OP_WRITE | write_flags, bh); 673 EXT4_SB(sb)->s_fc_bh = NULL; 674} 675 676/* Ext4 commit path routines */ 677 678/* 679 * Allocate len bytes on a fast commit buffer. 680 * 681 * During the commit time this function is used to manage fast commit 682 * block space. We don't split a fast commit log onto different 683 * blocks. So this function makes sure that if there's not enough space 684 * on the current block, the remaining space in the current block is 685 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case, 686 * new block is from jbd2 and CRC is updated to reflect the padding 687 * we added. 688 */ 689static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) 690{ 691 struct ext4_fc_tl tl; 692 struct ext4_sb_info *sbi = EXT4_SB(sb); 693 struct buffer_head *bh; 694 int bsize = sbi->s_journal->j_blocksize; 695 int ret, off = sbi->s_fc_bytes % bsize; 696 int remaining; 697 u8 *dst; 698 699 /* 700 * If 'len' is too long to fit in any block alongside a PAD tlv, then we 701 * cannot fulfill the request. 702 */ 703 if (len > bsize - EXT4_FC_TAG_BASE_LEN) 704 return NULL; 705 706 if (!sbi->s_fc_bh) { 707 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); 708 if (ret) 709 return NULL; 710 sbi->s_fc_bh = bh; 711 } 712 dst = sbi->s_fc_bh->b_data + off; 713 714 /* 715 * Allocate the bytes in the current block if we can do so while still 716 * leaving enough space for a PAD tlv. 717 */ 718 remaining = bsize - EXT4_FC_TAG_BASE_LEN - off; 719 if (len <= remaining) { 720 sbi->s_fc_bytes += len; 721 return dst; 722 } 723 724 /* 725 * Else, terminate the current block with a PAD tlv, then allocate a new 726 * block and allocate the bytes at the start of that new block. 727 */ 728 729 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); 730 tl.fc_len = cpu_to_le16(remaining); 731 memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); 732 memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining); 733 *crc = ext4_chksum(sbi, *crc, sbi->s_fc_bh->b_data, bsize); 734 735 ext4_fc_submit_bh(sb, false); 736 737 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); 738 if (ret) 739 return NULL; 740 sbi->s_fc_bh = bh; 741 sbi->s_fc_bytes += bsize - off + len; 742 return sbi->s_fc_bh->b_data; 743} 744 745/* 746 * Complete a fast commit by writing tail tag. 747 * 748 * Writing tail tag marks the end of a fast commit. In order to guarantee 749 * atomicity, after writing tail tag, even if there's space remaining 750 * in the block, next commit shouldn't use it. That's why tail tag 751 * has the length as that of the remaining space on the block. 752 */ 753static int ext4_fc_write_tail(struct super_block *sb, u32 crc) 754{ 755 struct ext4_sb_info *sbi = EXT4_SB(sb); 756 struct ext4_fc_tl tl; 757 struct ext4_fc_tail tail; 758 int off, bsize = sbi->s_journal->j_blocksize; 759 u8 *dst; 760 761 /* 762 * ext4_fc_reserve_space takes care of allocating an extra block if 763 * there's no enough space on this block for accommodating this tail. 764 */ 765 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc); 766 if (!dst) 767 return -ENOSPC; 768 769 off = sbi->s_fc_bytes % bsize; 770 771 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL); 772 tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail)); 773 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); 774 775 memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); 776 dst += EXT4_FC_TAG_BASE_LEN; 777 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); 778 memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid)); 779 dst += sizeof(tail.fc_tid); 780 crc = ext4_chksum(sbi, crc, sbi->s_fc_bh->b_data, 781 dst - (u8 *)sbi->s_fc_bh->b_data); 782 tail.fc_crc = cpu_to_le32(crc); 783 memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc)); 784 dst += sizeof(tail.fc_crc); 785 memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */ 786 787 ext4_fc_submit_bh(sb, true); 788 789 return 0; 790} 791 792/* 793 * Adds tag, length, value and updates CRC. Returns true if tlv was added. 794 * Returns false if there's not enough space. 795 */ 796static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, 797 u32 *crc) 798{ 799 struct ext4_fc_tl tl; 800 u8 *dst; 801 802 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc); 803 if (!dst) 804 return false; 805 806 tl.fc_tag = cpu_to_le16(tag); 807 tl.fc_len = cpu_to_le16(len); 808 809 memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); 810 memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len); 811 812 return true; 813} 814 815/* Same as above, but adds dentry tlv. */ 816static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, 817 struct ext4_fc_dentry_update *fc_dentry) 818{ 819 struct ext4_fc_dentry_info fcd; 820 struct ext4_fc_tl tl; 821 int dlen = fc_dentry->fcd_name.len; 822 u8 *dst = ext4_fc_reserve_space(sb, 823 EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc); 824 825 if (!dst) 826 return false; 827 828 fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent); 829 fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); 830 tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); 831 tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); 832 memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); 833 dst += EXT4_FC_TAG_BASE_LEN; 834 memcpy(dst, &fcd, sizeof(fcd)); 835 dst += sizeof(fcd); 836 memcpy(dst, fc_dentry->fcd_name.name, dlen); 837 838 return true; 839} 840 841/* 842 * Writes inode in the fast commit space under TLV with tag @tag. 843 * Returns 0 on success, error on failure. 844 */ 845static int ext4_fc_write_inode(struct inode *inode, u32 *crc) 846{ 847 struct ext4_inode_info *ei = EXT4_I(inode); 848 int inode_len = EXT4_GOOD_OLD_INODE_SIZE; 849 int ret; 850 struct ext4_iloc iloc; 851 struct ext4_fc_inode fc_inode; 852 struct ext4_fc_tl tl; 853 u8 *dst; 854 855 ret = ext4_get_inode_loc(inode, &iloc); 856 if (ret) 857 return ret; 858 859 if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) 860 inode_len = EXT4_INODE_SIZE(inode->i_sb); 861 else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) 862 inode_len += ei->i_extra_isize; 863 864 fc_inode.fc_ino = cpu_to_le32(inode->i_ino); 865 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); 866 tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); 867 868 ret = -ECANCELED; 869 dst = ext4_fc_reserve_space(inode->i_sb, 870 EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc); 871 if (!dst) 872 goto err; 873 874 memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); 875 dst += EXT4_FC_TAG_BASE_LEN; 876 memcpy(dst, &fc_inode, sizeof(fc_inode)); 877 dst += sizeof(fc_inode); 878 memcpy(dst, (u8 *)ext4_raw_inode(&iloc), inode_len); 879 ret = 0; 880err: 881 brelse(iloc.bh); 882 return ret; 883} 884 885/* 886 * Writes updated data ranges for the inode in question. Updates CRC. 887 * Returns 0 on success, error otherwise. 888 */ 889static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc) 890{ 891 ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size; 892 struct ext4_inode_info *ei = EXT4_I(inode); 893 struct ext4_map_blocks map; 894 struct ext4_fc_add_range fc_ext; 895 struct ext4_fc_del_range lrange; 896 struct ext4_extent *ex; 897 int ret; 898 899 mutex_lock(&ei->i_fc_lock); 900 if (ei->i_fc_lblk_len == 0) { 901 mutex_unlock(&ei->i_fc_lock); 902 return 0; 903 } 904 old_blk_size = ei->i_fc_lblk_start; 905 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; 906 ei->i_fc_lblk_len = 0; 907 mutex_unlock(&ei->i_fc_lock); 908 909 cur_lblk_off = old_blk_size; 910 ext4_debug("will try writing %d to %d for inode %ld\n", 911 cur_lblk_off, new_blk_size, inode->i_ino); 912 913 while (cur_lblk_off <= new_blk_size) { 914 map.m_lblk = cur_lblk_off; 915 map.m_len = new_blk_size - cur_lblk_off + 1; 916 ret = ext4_map_blocks(NULL, inode, &map, 0); 917 if (ret < 0) 918 return -ECANCELED; 919 920 if (map.m_len == 0) { 921 cur_lblk_off++; 922 continue; 923 } 924 925 if (ret == 0) { 926 lrange.fc_ino = cpu_to_le32(inode->i_ino); 927 lrange.fc_lblk = cpu_to_le32(map.m_lblk); 928 lrange.fc_len = cpu_to_le32(map.m_len); 929 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, 930 sizeof(lrange), (u8 *)&lrange, crc)) 931 return -ENOSPC; 932 } else { 933 unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ? 934 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN; 935 936 /* Limit the number of blocks in one extent */ 937 map.m_len = min(max, map.m_len); 938 939 fc_ext.fc_ino = cpu_to_le32(inode->i_ino); 940 ex = (struct ext4_extent *)&fc_ext.fc_ex; 941 ex->ee_block = cpu_to_le32(map.m_lblk); 942 ex->ee_len = cpu_to_le16(map.m_len); 943 ext4_ext_store_pblock(ex, map.m_pblk); 944 if (map.m_flags & EXT4_MAP_UNWRITTEN) 945 ext4_ext_mark_unwritten(ex); 946 else 947 ext4_ext_mark_initialized(ex); 948 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, 949 sizeof(fc_ext), (u8 *)&fc_ext, crc)) 950 return -ENOSPC; 951 } 952 953 cur_lblk_off += map.m_len; 954 } 955 956 return 0; 957} 958 959 960/* Submit data for all the fast commit inodes */ 961static int ext4_fc_submit_inode_data_all(journal_t *journal) 962{ 963 struct super_block *sb = journal->j_private; 964 struct ext4_sb_info *sbi = EXT4_SB(sb); 965 struct ext4_inode_info *ei; 966 int ret = 0; 967 968 spin_lock(&sbi->s_fc_lock); 969 list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { 970 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); 971 while (atomic_read(&ei->i_fc_updates)) { 972 DEFINE_WAIT(wait); 973 974 prepare_to_wait(&ei->i_fc_wait, &wait, 975 TASK_UNINTERRUPTIBLE); 976 if (atomic_read(&ei->i_fc_updates)) { 977 spin_unlock(&sbi->s_fc_lock); 978 schedule(); 979 spin_lock(&sbi->s_fc_lock); 980 } 981 finish_wait(&ei->i_fc_wait, &wait); 982 } 983 spin_unlock(&sbi->s_fc_lock); 984 ret = jbd2_submit_inode_data(journal, ei->jinode); 985 if (ret) 986 return ret; 987 spin_lock(&sbi->s_fc_lock); 988 } 989 spin_unlock(&sbi->s_fc_lock); 990 991 return ret; 992} 993 994/* Wait for completion of data for all the fast commit inodes */ 995static int ext4_fc_wait_inode_data_all(journal_t *journal) 996{ 997 struct super_block *sb = journal->j_private; 998 struct ext4_sb_info *sbi = EXT4_SB(sb); 999 struct ext4_inode_info *pos, *n; 1000 int ret = 0; 1001 1002 spin_lock(&sbi->s_fc_lock); 1003 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { 1004 if (!ext4_test_inode_state(&pos->vfs_inode, 1005 EXT4_STATE_FC_COMMITTING)) 1006 continue; 1007 spin_unlock(&sbi->s_fc_lock); 1008 1009 ret = jbd2_wait_inode_data(journal, pos->jinode); 1010 if (ret) 1011 return ret; 1012 spin_lock(&sbi->s_fc_lock); 1013 } 1014 spin_unlock(&sbi->s_fc_lock); 1015 1016 return 0; 1017} 1018 1019/* Commit all the directory entry updates */ 1020static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) 1021__acquires(&sbi->s_fc_lock) 1022__releases(&sbi->s_fc_lock) 1023{ 1024 struct super_block *sb = journal->j_private; 1025 struct ext4_sb_info *sbi = EXT4_SB(sb); 1026 struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n; 1027 struct inode *inode; 1028 struct ext4_inode_info *ei; 1029 int ret; 1030 1031 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) 1032 return 0; 1033 list_for_each_entry_safe(fc_dentry, fc_dentry_n, 1034 &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { 1035 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { 1036 spin_unlock(&sbi->s_fc_lock); 1037 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) { 1038 ret = -ENOSPC; 1039 goto lock_and_exit; 1040 } 1041 spin_lock(&sbi->s_fc_lock); 1042 continue; 1043 } 1044 /* 1045 * With fcd_dilist we need not loop in sbi->s_fc_q to get the 1046 * corresponding inode pointer 1047 */ 1048 WARN_ON(list_empty(&fc_dentry->fcd_dilist)); 1049 ei = list_first_entry(&fc_dentry->fcd_dilist, 1050 struct ext4_inode_info, i_fc_dilist); 1051 inode = &ei->vfs_inode; 1052 WARN_ON(inode->i_ino != fc_dentry->fcd_ino); 1053 1054 spin_unlock(&sbi->s_fc_lock); 1055 1056 /* 1057 * We first write the inode and then the create dirent. This 1058 * allows the recovery code to create an unnamed inode first 1059 * and then link it to a directory entry. This allows us 1060 * to use namei.c routines almost as is and simplifies 1061 * the recovery code. 1062 */ 1063 ret = ext4_fc_write_inode(inode, crc); 1064 if (ret) 1065 goto lock_and_exit; 1066 1067 ret = ext4_fc_write_inode_data(inode, crc); 1068 if (ret) 1069 goto lock_and_exit; 1070 1071 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) { 1072 ret = -ENOSPC; 1073 goto lock_and_exit; 1074 } 1075 1076 spin_lock(&sbi->s_fc_lock); 1077 } 1078 return 0; 1079lock_and_exit: 1080 spin_lock(&sbi->s_fc_lock); 1081 return ret; 1082} 1083 1084static int ext4_fc_perform_commit(journal_t *journal) 1085{ 1086 struct super_block *sb = journal->j_private; 1087 struct ext4_sb_info *sbi = EXT4_SB(sb); 1088 struct ext4_inode_info *iter; 1089 struct ext4_fc_head head; 1090 struct inode *inode; 1091 struct blk_plug plug; 1092 int ret = 0; 1093 u32 crc = 0; 1094 1095 ret = ext4_fc_submit_inode_data_all(journal); 1096 if (ret) 1097 return ret; 1098 1099 ret = ext4_fc_wait_inode_data_all(journal); 1100 if (ret) 1101 return ret; 1102 1103 /* 1104 * If file system device is different from journal device, issue a cache 1105 * flush before we start writing fast commit blocks. 1106 */ 1107 if (journal->j_fs_dev != journal->j_dev) 1108 blkdev_issue_flush(journal->j_fs_dev); 1109 1110 blk_start_plug(&plug); 1111 if (sbi->s_fc_bytes == 0) { 1112 /* 1113 * Add a head tag only if this is the first fast commit 1114 * in this TID. 1115 */ 1116 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES); 1117 head.fc_tid = cpu_to_le32( 1118 sbi->s_journal->j_running_transaction->t_tid); 1119 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head), 1120 (u8 *)&head, &crc)) { 1121 ret = -ENOSPC; 1122 goto out; 1123 } 1124 } 1125 1126 spin_lock(&sbi->s_fc_lock); 1127 ret = ext4_fc_commit_dentry_updates(journal, &crc); 1128 if (ret) { 1129 spin_unlock(&sbi->s_fc_lock); 1130 goto out; 1131 } 1132 1133 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { 1134 inode = &iter->vfs_inode; 1135 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) 1136 continue; 1137 1138 spin_unlock(&sbi->s_fc_lock); 1139 ret = ext4_fc_write_inode_data(inode, &crc); 1140 if (ret) 1141 goto out; 1142 ret = ext4_fc_write_inode(inode, &crc); 1143 if (ret) 1144 goto out; 1145 spin_lock(&sbi->s_fc_lock); 1146 } 1147 spin_unlock(&sbi->s_fc_lock); 1148 1149 ret = ext4_fc_write_tail(sb, crc); 1150 1151out: 1152 blk_finish_plug(&plug); 1153 return ret; 1154} 1155 1156static void ext4_fc_update_stats(struct super_block *sb, int status, 1157 u64 commit_time, int nblks, tid_t commit_tid) 1158{ 1159 struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats; 1160 1161 ext4_debug("Fast commit ended with status = %d for tid %u", 1162 status, commit_tid); 1163 if (status == EXT4_FC_STATUS_OK) { 1164 stats->fc_num_commits++; 1165 stats->fc_numblks += nblks; 1166 if (likely(stats->s_fc_avg_commit_time)) 1167 stats->s_fc_avg_commit_time = 1168 (commit_time + 1169 stats->s_fc_avg_commit_time * 3) / 4; 1170 else 1171 stats->s_fc_avg_commit_time = commit_time; 1172 } else if (status == EXT4_FC_STATUS_FAILED || 1173 status == EXT4_FC_STATUS_INELIGIBLE) { 1174 if (status == EXT4_FC_STATUS_FAILED) 1175 stats->fc_failed_commits++; 1176 stats->fc_ineligible_commits++; 1177 } else { 1178 stats->fc_skipped_commits++; 1179 } 1180 trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid); 1181} 1182 1183/* 1184 * The main commit entry point. Performs a fast commit for transaction 1185 * commit_tid if needed. If it's not possible to perform a fast commit 1186 * due to various reasons, we fall back to full commit. Returns 0 1187 * on success, error otherwise. 1188 */ 1189int ext4_fc_commit(journal_t *journal, tid_t commit_tid) 1190{ 1191 struct super_block *sb = journal->j_private; 1192 struct ext4_sb_info *sbi = EXT4_SB(sb); 1193 int nblks = 0, ret, bsize = journal->j_blocksize; 1194 int subtid = atomic_read(&sbi->s_fc_subtid); 1195 int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0; 1196 ktime_t start_time, commit_time; 1197 1198 if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) 1199 return jbd2_complete_transaction(journal, commit_tid); 1200 1201 trace_ext4_fc_commit_start(sb, commit_tid); 1202 1203 start_time = ktime_get(); 1204 1205restart_fc: 1206 ret = jbd2_fc_begin_commit(journal, commit_tid); 1207 if (ret == -EALREADY) { 1208 /* There was an ongoing commit, check if we need to restart */ 1209 if (atomic_read(&sbi->s_fc_subtid) <= subtid && 1210 commit_tid > journal->j_commit_sequence) 1211 goto restart_fc; 1212 ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0, 1213 commit_tid); 1214 return 0; 1215 } else if (ret) { 1216 /* 1217 * Commit couldn't start. Just update stats and perform a 1218 * full commit. 1219 */ 1220 ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0, 1221 commit_tid); 1222 return jbd2_complete_transaction(journal, commit_tid); 1223 } 1224 1225 /* 1226 * After establishing journal barrier via jbd2_fc_begin_commit(), check 1227 * if we are fast commit ineligible. 1228 */ 1229 if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) { 1230 status = EXT4_FC_STATUS_INELIGIBLE; 1231 goto fallback; 1232 } 1233 1234 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; 1235 ret = ext4_fc_perform_commit(journal); 1236 if (ret < 0) { 1237 status = EXT4_FC_STATUS_FAILED; 1238 goto fallback; 1239 } 1240 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; 1241 ret = jbd2_fc_wait_bufs(journal, nblks); 1242 if (ret < 0) { 1243 status = EXT4_FC_STATUS_FAILED; 1244 goto fallback; 1245 } 1246 atomic_inc(&sbi->s_fc_subtid); 1247 ret = jbd2_fc_end_commit(journal); 1248 /* 1249 * weight the commit time higher than the average time so we 1250 * don't react too strongly to vast changes in the commit time 1251 */ 1252 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); 1253 ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid); 1254 return ret; 1255 1256fallback: 1257 ret = jbd2_fc_end_commit_fallback(journal); 1258 ext4_fc_update_stats(sb, status, 0, 0, commit_tid); 1259 return ret; 1260} 1261 1262/* 1263 * Fast commit cleanup routine. This is called after every fast commit and 1264 * full commit. full is true if we are called after a full commit. 1265 */ 1266static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) 1267{ 1268 struct super_block *sb = journal->j_private; 1269 struct ext4_sb_info *sbi = EXT4_SB(sb); 1270 struct ext4_inode_info *iter, *iter_n; 1271 struct ext4_fc_dentry_update *fc_dentry; 1272 1273 if (full && sbi->s_fc_bh) 1274 sbi->s_fc_bh = NULL; 1275 1276 trace_ext4_fc_cleanup(journal, full, tid); 1277 jbd2_fc_release_bufs(journal); 1278 1279 spin_lock(&sbi->s_fc_lock); 1280 list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN], 1281 i_fc_list) { 1282 list_del_init(&iter->i_fc_list); 1283 ext4_clear_inode_state(&iter->vfs_inode, 1284 EXT4_STATE_FC_COMMITTING); 1285 if (iter->i_sync_tid <= tid) 1286 ext4_fc_reset_inode(&iter->vfs_inode); 1287 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */ 1288 smp_mb(); 1289#if (BITS_PER_LONG < 64) 1290 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING); 1291#else 1292 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING); 1293#endif 1294 } 1295 1296 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { 1297 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], 1298 struct ext4_fc_dentry_update, 1299 fcd_list); 1300 list_del_init(&fc_dentry->fcd_list); 1301 list_del_init(&fc_dentry->fcd_dilist); 1302 spin_unlock(&sbi->s_fc_lock); 1303 1304 if (fc_dentry->fcd_name.name && 1305 fc_dentry->fcd_name.len > DNAME_INLINE_LEN) 1306 kfree(fc_dentry->fcd_name.name); 1307 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); 1308 spin_lock(&sbi->s_fc_lock); 1309 } 1310 1311 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], 1312 &sbi->s_fc_dentry_q[FC_Q_MAIN]); 1313 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], 1314 &sbi->s_fc_q[FC_Q_MAIN]); 1315 1316 if (tid >= sbi->s_fc_ineligible_tid) { 1317 sbi->s_fc_ineligible_tid = 0; 1318 ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); 1319 } 1320 1321 if (full) 1322 sbi->s_fc_bytes = 0; 1323 spin_unlock(&sbi->s_fc_lock); 1324 trace_ext4_fc_stats(sb); 1325} 1326 1327/* Ext4 Replay Path Routines */ 1328 1329/* Helper struct for dentry replay routines */ 1330struct dentry_info_args { 1331 int parent_ino, dname_len, ino, inode_len; 1332 char *dname; 1333}; 1334 1335/* Same as struct ext4_fc_tl, but uses native endianness fields */ 1336struct ext4_fc_tl_mem { 1337 u16 fc_tag; 1338 u16 fc_len; 1339}; 1340 1341static inline void tl_to_darg(struct dentry_info_args *darg, 1342 struct ext4_fc_tl_mem *tl, u8 *val) 1343{ 1344 struct ext4_fc_dentry_info fcd; 1345 1346 memcpy(&fcd, val, sizeof(fcd)); 1347 1348 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); 1349 darg->ino = le32_to_cpu(fcd.fc_ino); 1350 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); 1351 darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); 1352} 1353 1354static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val) 1355{ 1356 struct ext4_fc_tl tl_disk; 1357 1358 memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN); 1359 tl->fc_len = le16_to_cpu(tl_disk.fc_len); 1360 tl->fc_tag = le16_to_cpu(tl_disk.fc_tag); 1361} 1362 1363/* Unlink replay function */ 1364static int ext4_fc_replay_unlink(struct super_block *sb, 1365 struct ext4_fc_tl_mem *tl, u8 *val) 1366{ 1367 struct inode *inode, *old_parent; 1368 struct qstr entry; 1369 struct dentry_info_args darg; 1370 int ret = 0; 1371 1372 tl_to_darg(&darg, tl, val); 1373 1374 trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino, 1375 darg.parent_ino, darg.dname_len); 1376 1377 entry.name = darg.dname; 1378 entry.len = darg.dname_len; 1379 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); 1380 1381 if (IS_ERR(inode)) { 1382 ext4_debug("Inode %d not found", darg.ino); 1383 return 0; 1384 } 1385 1386 old_parent = ext4_iget(sb, darg.parent_ino, 1387 EXT4_IGET_NORMAL); 1388 if (IS_ERR(old_parent)) { 1389 ext4_debug("Dir with inode %d not found", darg.parent_ino); 1390 iput(inode); 1391 return 0; 1392 } 1393 1394 ret = __ext4_unlink(old_parent, &entry, inode, NULL); 1395 /* -ENOENT ok coz it might not exist anymore. */ 1396 if (ret == -ENOENT) 1397 ret = 0; 1398 iput(old_parent); 1399 iput(inode); 1400 return ret; 1401} 1402 1403static int ext4_fc_replay_link_internal(struct super_block *sb, 1404 struct dentry_info_args *darg, 1405 struct inode *inode) 1406{ 1407 struct inode *dir = NULL; 1408 struct dentry *dentry_dir = NULL, *dentry_inode = NULL; 1409 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); 1410 int ret = 0; 1411 1412 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL); 1413 if (IS_ERR(dir)) { 1414 ext4_debug("Dir with inode %d not found.", darg->parent_ino); 1415 dir = NULL; 1416 goto out; 1417 } 1418 1419 dentry_dir = d_obtain_alias(dir); 1420 if (IS_ERR(dentry_dir)) { 1421 ext4_debug("Failed to obtain dentry"); 1422 dentry_dir = NULL; 1423 goto out; 1424 } 1425 1426 dentry_inode = d_alloc(dentry_dir, &qstr_dname); 1427 if (!dentry_inode) { 1428 ext4_debug("Inode dentry not created."); 1429 ret = -ENOMEM; 1430 goto out; 1431 } 1432 1433 ret = __ext4_link(dir, inode, dentry_inode); 1434 /* 1435 * It's possible that link already existed since data blocks 1436 * for the dir in question got persisted before we crashed OR 1437 * we replayed this tag and crashed before the entire replay 1438 * could complete. 1439 */ 1440 if (ret && ret != -EEXIST) { 1441 ext4_debug("Failed to link\n"); 1442 goto out; 1443 } 1444 1445 ret = 0; 1446out: 1447 if (dentry_dir) { 1448 d_drop(dentry_dir); 1449 dput(dentry_dir); 1450 } else if (dir) { 1451 iput(dir); 1452 } 1453 if (dentry_inode) { 1454 d_drop(dentry_inode); 1455 dput(dentry_inode); 1456 } 1457 1458 return ret; 1459} 1460 1461/* Link replay function */ 1462static int ext4_fc_replay_link(struct super_block *sb, 1463 struct ext4_fc_tl_mem *tl, u8 *val) 1464{ 1465 struct inode *inode; 1466 struct dentry_info_args darg; 1467 int ret = 0; 1468 1469 tl_to_darg(&darg, tl, val); 1470 trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino, 1471 darg.parent_ino, darg.dname_len); 1472 1473 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); 1474 if (IS_ERR(inode)) { 1475 ext4_debug("Inode not found."); 1476 return 0; 1477 } 1478 1479 ret = ext4_fc_replay_link_internal(sb, &darg, inode); 1480 iput(inode); 1481 return ret; 1482} 1483 1484/* 1485 * Record all the modified inodes during replay. We use this later to setup 1486 * block bitmaps correctly. 1487 */ 1488static int ext4_fc_record_modified_inode(struct super_block *sb, int ino) 1489{ 1490 struct ext4_fc_replay_state *state; 1491 int i; 1492 1493 state = &EXT4_SB(sb)->s_fc_replay_state; 1494 for (i = 0; i < state->fc_modified_inodes_used; i++) 1495 if (state->fc_modified_inodes[i] == ino) 1496 return 0; 1497 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { 1498 int *fc_modified_inodes; 1499 1500 fc_modified_inodes = krealloc(state->fc_modified_inodes, 1501 sizeof(int) * (state->fc_modified_inodes_size + 1502 EXT4_FC_REPLAY_REALLOC_INCREMENT), 1503 GFP_KERNEL); 1504 if (!fc_modified_inodes) 1505 return -ENOMEM; 1506 state->fc_modified_inodes = fc_modified_inodes; 1507 state->fc_modified_inodes_size += 1508 EXT4_FC_REPLAY_REALLOC_INCREMENT; 1509 } 1510 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino; 1511 return 0; 1512} 1513 1514/* 1515 * Inode replay function 1516 */ 1517static int ext4_fc_replay_inode(struct super_block *sb, 1518 struct ext4_fc_tl_mem *tl, u8 *val) 1519{ 1520 struct ext4_fc_inode fc_inode; 1521 struct ext4_inode *raw_inode; 1522 struct ext4_inode *raw_fc_inode; 1523 struct inode *inode = NULL; 1524 struct ext4_iloc iloc; 1525 int inode_len, ino, ret, tag = tl->fc_tag; 1526 struct ext4_extent_header *eh; 1527 size_t off_gen = offsetof(struct ext4_inode, i_generation); 1528 1529 memcpy(&fc_inode, val, sizeof(fc_inode)); 1530 1531 ino = le32_to_cpu(fc_inode.fc_ino); 1532 trace_ext4_fc_replay(sb, tag, ino, 0, 0); 1533 1534 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); 1535 if (!IS_ERR(inode)) { 1536 ext4_ext_clear_bb(inode); 1537 iput(inode); 1538 } 1539 inode = NULL; 1540 1541 ret = ext4_fc_record_modified_inode(sb, ino); 1542 if (ret) 1543 goto out; 1544 1545 raw_fc_inode = (struct ext4_inode *) 1546 (val + offsetof(struct ext4_fc_inode, fc_raw_inode)); 1547 ret = ext4_get_fc_inode_loc(sb, ino, &iloc); 1548 if (ret) 1549 goto out; 1550 1551 inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); 1552 raw_inode = ext4_raw_inode(&iloc); 1553 1554 memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); 1555 memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen, 1556 inode_len - off_gen); 1557 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { 1558 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); 1559 if (eh->eh_magic != EXT4_EXT_MAGIC) { 1560 memset(eh, 0, sizeof(*eh)); 1561 eh->eh_magic = EXT4_EXT_MAGIC; 1562 eh->eh_max = cpu_to_le16( 1563 (sizeof(raw_inode->i_block) - 1564 sizeof(struct ext4_extent_header)) 1565 / sizeof(struct ext4_extent)); 1566 } 1567 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) { 1568 memcpy(raw_inode->i_block, raw_fc_inode->i_block, 1569 sizeof(raw_inode->i_block)); 1570 } 1571 1572 /* Immediately update the inode on disk. */ 1573 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); 1574 if (ret) 1575 goto out; 1576 ret = sync_dirty_buffer(iloc.bh); 1577 if (ret) 1578 goto out; 1579 ret = ext4_mark_inode_used(sb, ino); 1580 if (ret) 1581 goto out; 1582 1583 /* Given that we just wrote the inode on disk, this SHOULD succeed. */ 1584 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); 1585 if (IS_ERR(inode)) { 1586 ext4_debug("Inode not found."); 1587 return -EFSCORRUPTED; 1588 } 1589 1590 /* 1591 * Our allocator could have made different decisions than before 1592 * crashing. This should be fixed but until then, we calculate 1593 * the number of blocks the inode. 1594 */ 1595 if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) 1596 ext4_ext_replay_set_iblocks(inode); 1597 1598 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation); 1599 ext4_reset_inode_seed(inode); 1600 1601 ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); 1602 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); 1603 sync_dirty_buffer(iloc.bh); 1604 brelse(iloc.bh); 1605out: 1606 iput(inode); 1607 if (!ret) 1608 blkdev_issue_flush(sb->s_bdev); 1609 1610 return 0; 1611} 1612 1613/* 1614 * Dentry create replay function. 1615 * 1616 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the 1617 * inode for which we are trying to create a dentry here, should already have 1618 * been replayed before we start here. 1619 */ 1620static int ext4_fc_replay_create(struct super_block *sb, 1621 struct ext4_fc_tl_mem *tl, u8 *val) 1622{ 1623 int ret = 0; 1624 struct inode *inode = NULL; 1625 struct inode *dir = NULL; 1626 struct dentry_info_args darg; 1627 1628 tl_to_darg(&darg, tl, val); 1629 1630 trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino, 1631 darg.parent_ino, darg.dname_len); 1632 1633 /* This takes care of update group descriptor and other metadata */ 1634 ret = ext4_mark_inode_used(sb, darg.ino); 1635 if (ret) 1636 goto out; 1637 1638 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); 1639 if (IS_ERR(inode)) { 1640 ext4_debug("inode %d not found.", darg.ino); 1641 inode = NULL; 1642 ret = -EINVAL; 1643 goto out; 1644 } 1645 1646 if (S_ISDIR(inode->i_mode)) { 1647 /* 1648 * If we are creating a directory, we need to make sure that the 1649 * dot and dot dot dirents are setup properly. 1650 */ 1651 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL); 1652 if (IS_ERR(dir)) { 1653 ext4_debug("Dir %d not found.", darg.ino); 1654 goto out; 1655 } 1656 ret = ext4_init_new_dir(NULL, dir, inode); 1657 iput(dir); 1658 if (ret) { 1659 ret = 0; 1660 goto out; 1661 } 1662 } 1663 ret = ext4_fc_replay_link_internal(sb, &darg, inode); 1664 if (ret) 1665 goto out; 1666 set_nlink(inode, 1); 1667 ext4_mark_inode_dirty(NULL, inode); 1668out: 1669 iput(inode); 1670 return ret; 1671} 1672 1673/* 1674 * Record physical disk regions which are in use as per fast commit area, 1675 * and used by inodes during replay phase. Our simple replay phase 1676 * allocator excludes these regions from allocation. 1677 */ 1678int ext4_fc_record_regions(struct super_block *sb, int ino, 1679 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay) 1680{ 1681 struct ext4_fc_replay_state *state; 1682 struct ext4_fc_alloc_region *region; 1683 1684 state = &EXT4_SB(sb)->s_fc_replay_state; 1685 /* 1686 * during replay phase, the fc_regions_valid may not same as 1687 * fc_regions_used, update it when do new additions. 1688 */ 1689 if (replay && state->fc_regions_used != state->fc_regions_valid) 1690 state->fc_regions_used = state->fc_regions_valid; 1691 if (state->fc_regions_used == state->fc_regions_size) { 1692 struct ext4_fc_alloc_region *fc_regions; 1693 1694 fc_regions = krealloc(state->fc_regions, 1695 sizeof(struct ext4_fc_alloc_region) * 1696 (state->fc_regions_size + 1697 EXT4_FC_REPLAY_REALLOC_INCREMENT), 1698 GFP_KERNEL); 1699 if (!fc_regions) 1700 return -ENOMEM; 1701 state->fc_regions_size += 1702 EXT4_FC_REPLAY_REALLOC_INCREMENT; 1703 state->fc_regions = fc_regions; 1704 } 1705 region = &state->fc_regions[state->fc_regions_used++]; 1706 region->ino = ino; 1707 region->lblk = lblk; 1708 region->pblk = pblk; 1709 region->len = len; 1710 1711 if (replay) 1712 state->fc_regions_valid++; 1713 1714 return 0; 1715} 1716 1717/* Replay add range tag */ 1718static int ext4_fc_replay_add_range(struct super_block *sb, 1719 struct ext4_fc_tl_mem *tl, u8 *val) 1720{ 1721 struct ext4_fc_add_range fc_add_ex; 1722 struct ext4_extent newex, *ex; 1723 struct inode *inode; 1724 ext4_lblk_t start, cur; 1725 int remaining, len; 1726 ext4_fsblk_t start_pblk; 1727 struct ext4_map_blocks map; 1728 struct ext4_ext_path *path = NULL; 1729 int ret; 1730 1731 memcpy(&fc_add_ex, val, sizeof(fc_add_ex)); 1732 ex = (struct ext4_extent *)&fc_add_ex.fc_ex; 1733 1734 trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE, 1735 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block), 1736 ext4_ext_get_actual_len(ex)); 1737 1738 inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL); 1739 if (IS_ERR(inode)) { 1740 ext4_debug("Inode not found."); 1741 return 0; 1742 } 1743 1744 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); 1745 if (ret) 1746 goto out; 1747 1748 start = le32_to_cpu(ex->ee_block); 1749 start_pblk = ext4_ext_pblock(ex); 1750 len = ext4_ext_get_actual_len(ex); 1751 1752 cur = start; 1753 remaining = len; 1754 ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n", 1755 start, start_pblk, len, ext4_ext_is_unwritten(ex), 1756 inode->i_ino); 1757 1758 while (remaining > 0) { 1759 map.m_lblk = cur; 1760 map.m_len = remaining; 1761 map.m_pblk = 0; 1762 ret = ext4_map_blocks(NULL, inode, &map, 0); 1763 1764 if (ret < 0) 1765 goto out; 1766 1767 if (ret == 0) { 1768 /* Range is not mapped */ 1769 path = ext4_find_extent(inode, cur, NULL, 0); 1770 if (IS_ERR(path)) 1771 goto out; 1772 memset(&newex, 0, sizeof(newex)); 1773 newex.ee_block = cpu_to_le32(cur); 1774 ext4_ext_store_pblock( 1775 &newex, start_pblk + cur - start); 1776 newex.ee_len = cpu_to_le16(map.m_len); 1777 if (ext4_ext_is_unwritten(ex)) 1778 ext4_ext_mark_unwritten(&newex); 1779 down_write(&EXT4_I(inode)->i_data_sem); 1780 ret = ext4_ext_insert_extent( 1781 NULL, inode, &path, &newex, 0); 1782 up_write((&EXT4_I(inode)->i_data_sem)); 1783 ext4_free_ext_path(path); 1784 if (ret) 1785 goto out; 1786 goto next; 1787 } 1788 1789 if (start_pblk + cur - start != map.m_pblk) { 1790 /* 1791 * Logical to physical mapping changed. This can happen 1792 * if this range was removed and then reallocated to 1793 * map to new physical blocks during a fast commit. 1794 */ 1795 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, 1796 ext4_ext_is_unwritten(ex), 1797 start_pblk + cur - start); 1798 if (ret) 1799 goto out; 1800 /* 1801 * Mark the old blocks as free since they aren't used 1802 * anymore. We maintain an array of all the modified 1803 * inodes. In case these blocks are still used at either 1804 * a different logical range in the same inode or in 1805 * some different inode, we will mark them as allocated 1806 * at the end of the FC replay using our array of 1807 * modified inodes. 1808 */ 1809 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); 1810 goto next; 1811 } 1812 1813 /* Range is mapped and needs a state change */ 1814 ext4_debug("Converting from %ld to %d %lld", 1815 map.m_flags & EXT4_MAP_UNWRITTEN, 1816 ext4_ext_is_unwritten(ex), map.m_pblk); 1817 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, 1818 ext4_ext_is_unwritten(ex), map.m_pblk); 1819 if (ret) 1820 goto out; 1821 /* 1822 * We may have split the extent tree while toggling the state. 1823 * Try to shrink the extent tree now. 1824 */ 1825 ext4_ext_replay_shrink_inode(inode, start + len); 1826next: 1827 cur += map.m_len; 1828 remaining -= map.m_len; 1829 } 1830 ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> 1831 sb->s_blocksize_bits); 1832out: 1833 iput(inode); 1834 return 0; 1835} 1836 1837/* Replay DEL_RANGE tag */ 1838static int 1839ext4_fc_replay_del_range(struct super_block *sb, 1840 struct ext4_fc_tl_mem *tl, u8 *val) 1841{ 1842 struct inode *inode; 1843 struct ext4_fc_del_range lrange; 1844 struct ext4_map_blocks map; 1845 ext4_lblk_t cur, remaining; 1846 int ret; 1847 1848 memcpy(&lrange, val, sizeof(lrange)); 1849 cur = le32_to_cpu(lrange.fc_lblk); 1850 remaining = le32_to_cpu(lrange.fc_len); 1851 1852 trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE, 1853 le32_to_cpu(lrange.fc_ino), cur, remaining); 1854 1855 inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL); 1856 if (IS_ERR(inode)) { 1857 ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino)); 1858 return 0; 1859 } 1860 1861 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); 1862 if (ret) 1863 goto out; 1864 1865 ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n", 1866 inode->i_ino, le32_to_cpu(lrange.fc_lblk), 1867 le32_to_cpu(lrange.fc_len)); 1868 while (remaining > 0) { 1869 map.m_lblk = cur; 1870 map.m_len = remaining; 1871 1872 ret = ext4_map_blocks(NULL, inode, &map, 0); 1873 if (ret < 0) 1874 goto out; 1875 if (ret > 0) { 1876 remaining -= ret; 1877 cur += ret; 1878 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); 1879 } else { 1880 remaining -= map.m_len; 1881 cur += map.m_len; 1882 } 1883 } 1884 1885 down_write(&EXT4_I(inode)->i_data_sem); 1886 ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk), 1887 le32_to_cpu(lrange.fc_lblk) + 1888 le32_to_cpu(lrange.fc_len) - 1); 1889 up_write(&EXT4_I(inode)->i_data_sem); 1890 if (ret) 1891 goto out; 1892 ext4_ext_replay_shrink_inode(inode, 1893 i_size_read(inode) >> sb->s_blocksize_bits); 1894 ext4_mark_inode_dirty(NULL, inode); 1895out: 1896 iput(inode); 1897 return 0; 1898} 1899 1900static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) 1901{ 1902 struct ext4_fc_replay_state *state; 1903 struct inode *inode; 1904 struct ext4_ext_path *path = NULL; 1905 struct ext4_map_blocks map; 1906 int i, ret, j; 1907 ext4_lblk_t cur, end; 1908 1909 state = &EXT4_SB(sb)->s_fc_replay_state; 1910 for (i = 0; i < state->fc_modified_inodes_used; i++) { 1911 inode = ext4_iget(sb, state->fc_modified_inodes[i], 1912 EXT4_IGET_NORMAL); 1913 if (IS_ERR(inode)) { 1914 ext4_debug("Inode %d not found.", 1915 state->fc_modified_inodes[i]); 1916 continue; 1917 } 1918 cur = 0; 1919 end = EXT_MAX_BLOCKS; 1920 if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) { 1921 iput(inode); 1922 continue; 1923 } 1924 while (cur < end) { 1925 map.m_lblk = cur; 1926 map.m_len = end - cur; 1927 1928 ret = ext4_map_blocks(NULL, inode, &map, 0); 1929 if (ret < 0) 1930 break; 1931 1932 if (ret > 0) { 1933 path = ext4_find_extent(inode, map.m_lblk, NULL, 0); 1934 if (!IS_ERR(path)) { 1935 for (j = 0; j < path->p_depth; j++) 1936 ext4_mb_mark_bb(inode->i_sb, 1937 path[j].p_block, 1, true); 1938 ext4_free_ext_path(path); 1939 } 1940 cur += ret; 1941 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, 1942 map.m_len, true); 1943 } else { 1944 cur = cur + (map.m_len ? map.m_len : 1); 1945 } 1946 } 1947 iput(inode); 1948 } 1949} 1950 1951/* 1952 * Check if block is in excluded regions for block allocation. The simple 1953 * allocator that runs during replay phase is calls this function to see 1954 * if it is okay to use a block. 1955 */ 1956bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk) 1957{ 1958 int i; 1959 struct ext4_fc_replay_state *state; 1960 1961 state = &EXT4_SB(sb)->s_fc_replay_state; 1962 for (i = 0; i < state->fc_regions_valid; i++) { 1963 if (state->fc_regions[i].ino == 0 || 1964 state->fc_regions[i].len == 0) 1965 continue; 1966 if (in_range(blk, state->fc_regions[i].pblk, 1967 state->fc_regions[i].len)) 1968 return true; 1969 } 1970 return false; 1971} 1972 1973/* Cleanup function called after replay */ 1974void ext4_fc_replay_cleanup(struct super_block *sb) 1975{ 1976 struct ext4_sb_info *sbi = EXT4_SB(sb); 1977 1978 sbi->s_mount_state &= ~EXT4_FC_REPLAY; 1979 kfree(sbi->s_fc_replay_state.fc_regions); 1980 kfree(sbi->s_fc_replay_state.fc_modified_inodes); 1981} 1982 1983static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi, 1984 int tag, int len) 1985{ 1986 switch (tag) { 1987 case EXT4_FC_TAG_ADD_RANGE: 1988 return len == sizeof(struct ext4_fc_add_range); 1989 case EXT4_FC_TAG_DEL_RANGE: 1990 return len == sizeof(struct ext4_fc_del_range); 1991 case EXT4_FC_TAG_CREAT: 1992 case EXT4_FC_TAG_LINK: 1993 case EXT4_FC_TAG_UNLINK: 1994 len -= sizeof(struct ext4_fc_dentry_info); 1995 return len >= 1 && len <= EXT4_NAME_LEN; 1996 case EXT4_FC_TAG_INODE: 1997 len -= sizeof(struct ext4_fc_inode); 1998 return len >= EXT4_GOOD_OLD_INODE_SIZE && 1999 len <= sbi->s_inode_size; 2000 case EXT4_FC_TAG_PAD: 2001 return true; /* padding can have any length */ 2002 case EXT4_FC_TAG_TAIL: 2003 return len >= sizeof(struct ext4_fc_tail); 2004 case EXT4_FC_TAG_HEAD: 2005 return len == sizeof(struct ext4_fc_head); 2006 } 2007 return false; 2008} 2009 2010/* 2011 * Recovery Scan phase handler 2012 * 2013 * This function is called during the scan phase and is responsible 2014 * for doing following things: 2015 * - Make sure the fast commit area has valid tags for replay 2016 * - Count number of tags that need to be replayed by the replay handler 2017 * - Verify CRC 2018 * - Create a list of excluded blocks for allocation during replay phase 2019 * 2020 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is 2021 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP 2022 * to indicate that scan has finished and JBD2 can now start replay phase. 2023 * It returns a negative error to indicate that there was an error. At the end 2024 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set 2025 * to indicate the number of tags that need to replayed during the replay phase. 2026 */ 2027static int ext4_fc_replay_scan(journal_t *journal, 2028 struct buffer_head *bh, int off, 2029 tid_t expected_tid) 2030{ 2031 struct super_block *sb = journal->j_private; 2032 struct ext4_sb_info *sbi = EXT4_SB(sb); 2033 struct ext4_fc_replay_state *state; 2034 int ret = JBD2_FC_REPLAY_CONTINUE; 2035 struct ext4_fc_add_range ext; 2036 struct ext4_fc_tl_mem tl; 2037 struct ext4_fc_tail tail; 2038 __u8 *start, *end, *cur, *val; 2039 struct ext4_fc_head head; 2040 struct ext4_extent *ex; 2041 2042 state = &sbi->s_fc_replay_state; 2043 2044 start = (u8 *)bh->b_data; 2045 end = start + journal->j_blocksize; 2046 2047 if (state->fc_replay_expected_off == 0) { 2048 state->fc_cur_tag = 0; 2049 state->fc_replay_num_tags = 0; 2050 state->fc_crc = 0; 2051 state->fc_regions = NULL; 2052 state->fc_regions_valid = state->fc_regions_used = 2053 state->fc_regions_size = 0; 2054 /* Check if we can stop early */ 2055 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag) 2056 != EXT4_FC_TAG_HEAD) 2057 return 0; 2058 } 2059 2060 if (off != state->fc_replay_expected_off) { 2061 ret = -EFSCORRUPTED; 2062 goto out_err; 2063 } 2064 2065 state->fc_replay_expected_off++; 2066 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; 2067 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { 2068 ext4_fc_get_tl(&tl, cur); 2069 val = cur + EXT4_FC_TAG_BASE_LEN; 2070 if (tl.fc_len > end - val || 2071 !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) { 2072 ret = state->fc_replay_num_tags ? 2073 JBD2_FC_REPLAY_STOP : -ECANCELED; 2074 goto out_err; 2075 } 2076 ext4_debug("Scan phase, tag:%s, blk %lld\n", 2077 tag2str(tl.fc_tag), bh->b_blocknr); 2078 switch (tl.fc_tag) { 2079 case EXT4_FC_TAG_ADD_RANGE: 2080 memcpy(&ext, val, sizeof(ext)); 2081 ex = (struct ext4_extent *)&ext.fc_ex; 2082 ret = ext4_fc_record_regions(sb, 2083 le32_to_cpu(ext.fc_ino), 2084 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex), 2085 ext4_ext_get_actual_len(ex), 0); 2086 if (ret < 0) 2087 break; 2088 ret = JBD2_FC_REPLAY_CONTINUE; 2089 fallthrough; 2090 case EXT4_FC_TAG_DEL_RANGE: 2091 case EXT4_FC_TAG_LINK: 2092 case EXT4_FC_TAG_UNLINK: 2093 case EXT4_FC_TAG_CREAT: 2094 case EXT4_FC_TAG_INODE: 2095 case EXT4_FC_TAG_PAD: 2096 state->fc_cur_tag++; 2097 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, 2098 EXT4_FC_TAG_BASE_LEN + tl.fc_len); 2099 break; 2100 case EXT4_FC_TAG_TAIL: 2101 state->fc_cur_tag++; 2102 memcpy(&tail, val, sizeof(tail)); 2103 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, 2104 EXT4_FC_TAG_BASE_LEN + 2105 offsetof(struct ext4_fc_tail, 2106 fc_crc)); 2107 if (le32_to_cpu(tail.fc_tid) == expected_tid && 2108 le32_to_cpu(tail.fc_crc) == state->fc_crc) { 2109 state->fc_replay_num_tags = state->fc_cur_tag; 2110 state->fc_regions_valid = 2111 state->fc_regions_used; 2112 } else { 2113 ret = state->fc_replay_num_tags ? 2114 JBD2_FC_REPLAY_STOP : -EFSBADCRC; 2115 } 2116 state->fc_crc = 0; 2117 break; 2118 case EXT4_FC_TAG_HEAD: 2119 memcpy(&head, val, sizeof(head)); 2120 if (le32_to_cpu(head.fc_features) & 2121 ~EXT4_FC_SUPPORTED_FEATURES) { 2122 ret = -EOPNOTSUPP; 2123 break; 2124 } 2125 if (le32_to_cpu(head.fc_tid) != expected_tid) { 2126 ret = JBD2_FC_REPLAY_STOP; 2127 break; 2128 } 2129 state->fc_cur_tag++; 2130 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, 2131 EXT4_FC_TAG_BASE_LEN + tl.fc_len); 2132 break; 2133 default: 2134 ret = state->fc_replay_num_tags ? 2135 JBD2_FC_REPLAY_STOP : -ECANCELED; 2136 } 2137 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP) 2138 break; 2139 } 2140 2141out_err: 2142 trace_ext4_fc_replay_scan(sb, ret, off); 2143 return ret; 2144} 2145 2146/* 2147 * Main recovery path entry point. 2148 * The meaning of return codes is similar as above. 2149 */ 2150static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, 2151 enum passtype pass, int off, tid_t expected_tid) 2152{ 2153 struct super_block *sb = journal->j_private; 2154 struct ext4_sb_info *sbi = EXT4_SB(sb); 2155 struct ext4_fc_tl_mem tl; 2156 __u8 *start, *end, *cur, *val; 2157 int ret = JBD2_FC_REPLAY_CONTINUE; 2158 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state; 2159 struct ext4_fc_tail tail; 2160 2161 if (pass == PASS_SCAN) { 2162 state->fc_current_pass = PASS_SCAN; 2163 return ext4_fc_replay_scan(journal, bh, off, expected_tid); 2164 } 2165 2166 if (state->fc_current_pass != pass) { 2167 state->fc_current_pass = pass; 2168 sbi->s_mount_state |= EXT4_FC_REPLAY; 2169 } 2170 if (!sbi->s_fc_replay_state.fc_replay_num_tags) { 2171 ext4_debug("Replay stops\n"); 2172 ext4_fc_set_bitmaps_and_counters(sb); 2173 return 0; 2174 } 2175 2176#ifdef CONFIG_EXT4_DEBUG 2177 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) { 2178 pr_warn("Dropping fc block %d because max_replay set\n", off); 2179 return JBD2_FC_REPLAY_STOP; 2180 } 2181#endif 2182 2183 start = (u8 *)bh->b_data; 2184 end = start + journal->j_blocksize; 2185 2186 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; 2187 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { 2188 ext4_fc_get_tl(&tl, cur); 2189 val = cur + EXT4_FC_TAG_BASE_LEN; 2190 2191 if (state->fc_replay_num_tags == 0) { 2192 ret = JBD2_FC_REPLAY_STOP; 2193 ext4_fc_set_bitmaps_and_counters(sb); 2194 break; 2195 } 2196 2197 ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag)); 2198 state->fc_replay_num_tags--; 2199 switch (tl.fc_tag) { 2200 case EXT4_FC_TAG_LINK: 2201 ret = ext4_fc_replay_link(sb, &tl, val); 2202 break; 2203 case EXT4_FC_TAG_UNLINK: 2204 ret = ext4_fc_replay_unlink(sb, &tl, val); 2205 break; 2206 case EXT4_FC_TAG_ADD_RANGE: 2207 ret = ext4_fc_replay_add_range(sb, &tl, val); 2208 break; 2209 case EXT4_FC_TAG_CREAT: 2210 ret = ext4_fc_replay_create(sb, &tl, val); 2211 break; 2212 case EXT4_FC_TAG_DEL_RANGE: 2213 ret = ext4_fc_replay_del_range(sb, &tl, val); 2214 break; 2215 case EXT4_FC_TAG_INODE: 2216 ret = ext4_fc_replay_inode(sb, &tl, val); 2217 break; 2218 case EXT4_FC_TAG_PAD: 2219 trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, 2220 tl.fc_len, 0); 2221 break; 2222 case EXT4_FC_TAG_TAIL: 2223 trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 2224 0, tl.fc_len, 0); 2225 memcpy(&tail, val, sizeof(tail)); 2226 WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid); 2227 break; 2228 case EXT4_FC_TAG_HEAD: 2229 break; 2230 default: 2231 trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0); 2232 ret = -ECANCELED; 2233 break; 2234 } 2235 if (ret < 0) 2236 break; 2237 ret = JBD2_FC_REPLAY_CONTINUE; 2238 } 2239 return ret; 2240} 2241 2242void ext4_fc_init(struct super_block *sb, journal_t *journal) 2243{ 2244 /* 2245 * We set replay callback even if fast commit disabled because we may 2246 * could still have fast commit blocks that need to be replayed even if 2247 * fast commit has now been turned off. 2248 */ 2249 journal->j_fc_replay_callback = ext4_fc_replay; 2250 if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) 2251 return; 2252 journal->j_fc_cleanup_callback = ext4_fc_cleanup; 2253} 2254 2255static const char * const fc_ineligible_reasons[] = { 2256 [EXT4_FC_REASON_XATTR] = "Extended attributes changed", 2257 [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename", 2258 [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed", 2259 [EXT4_FC_REASON_NOMEM] = "Insufficient memory", 2260 [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot", 2261 [EXT4_FC_REASON_RESIZE] = "Resize", 2262 [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed", 2263 [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op", 2264 [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling", 2265 [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename", 2266}; 2267 2268int ext4_fc_info_show(struct seq_file *seq, void *v) 2269{ 2270 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); 2271 struct ext4_fc_stats *stats = &sbi->s_fc_stats; 2272 int i; 2273 2274 if (v != SEQ_START_TOKEN) 2275 return 0; 2276 2277 seq_printf(seq, 2278 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", 2279 stats->fc_num_commits, stats->fc_ineligible_commits, 2280 stats->fc_numblks, 2281 div_u64(stats->s_fc_avg_commit_time, 1000)); 2282 seq_puts(seq, "Ineligible reasons:\n"); 2283 for (i = 0; i < EXT4_FC_REASON_MAX; i++) 2284 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i], 2285 stats->fc_ineligible_reason_count[i]); 2286 2287 return 0; 2288} 2289 2290int __init ext4_fc_init_dentry_cache(void) 2291{ 2292 ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, 2293 SLAB_RECLAIM_ACCOUNT); 2294 2295 if (ext4_fc_dentry_cachep == NULL) 2296 return -ENOMEM; 2297 2298 return 0; 2299} 2300 2301void ext4_fc_destroy_dentry_cache(void) 2302{ 2303 kmem_cache_destroy(ext4_fc_dentry_cachep); 2304} 2305