1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (c) 2018-2024 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6#include "xfs.h" 7#include "xfs_fs.h" 8#include "xfs_shared.h" 9#include "xfs_format.h" 10#include "xfs_trans_resv.h" 11#include "xfs_mount.h" 12#include "xfs_defer.h" 13#include "xfs_btree.h" 14#include "xfs_btree_staging.h" 15#include "xfs_buf_mem.h" 16#include "xfs_btree_mem.h" 17#include "xfs_bit.h" 18#include "xfs_log_format.h" 19#include "xfs_trans.h" 20#include "xfs_sb.h" 21#include "xfs_alloc.h" 22#include "xfs_alloc_btree.h" 23#include "xfs_ialloc.h" 24#include "xfs_ialloc_btree.h" 25#include "xfs_rmap.h" 26#include "xfs_rmap_btree.h" 27#include "xfs_inode.h" 28#include "xfs_icache.h" 29#include "xfs_bmap.h" 30#include "xfs_bmap_btree.h" 31#include "xfs_refcount.h" 32#include "xfs_refcount_btree.h" 33#include "xfs_ag.h" 34#include "scrub/xfs_scrub.h" 35#include "scrub/scrub.h" 36#include "scrub/common.h" 37#include "scrub/btree.h" 38#include "scrub/trace.h" 39#include "scrub/repair.h" 40#include "scrub/bitmap.h" 41#include "scrub/agb_bitmap.h" 42#include "scrub/xfile.h" 43#include "scrub/xfarray.h" 44#include "scrub/iscan.h" 45#include "scrub/newbt.h" 46#include "scrub/reap.h" 47 48/* 49 * Reverse Mapping Btree Repair 50 * ============================ 51 * 52 * This is the most involved of all the AG space btree rebuilds. Everywhere 53 * else in XFS we lock inodes and then AG data structures, but generating the 54 * list of rmap records requires that we be able to scan both block mapping 55 * btrees of every inode in the filesystem to see if it owns any extents in 56 * this AG. We can't tolerate any inode updates while we do this, so we 57 * freeze the filesystem to lock everyone else out, and grant ourselves 58 * special privileges to run transactions with regular background reclamation 59 * turned off. 60 * 61 * We also have to be very careful not to allow inode reclaim to start a 62 * transaction because all transactions (other than our own) will block. 63 * Deferred inode inactivation helps us out there. 64 * 65 * I) Reverse mappings for all non-space metadata and file data are collected 66 * according to the following algorithm: 67 * 68 * 1. For each fork of each inode: 69 * 1.1. Create a bitmap BMBIT to track bmbt blocks if necessary. 70 * 1.2. If the incore extent map isn't loaded, walk the bmbt to accumulate 71 * bmaps into rmap records (see 1.1.4). Set bits in BMBIT for each btree 72 * block. 73 * 1.3. If the incore extent map is loaded but the fork is in btree format, 74 * just visit the bmbt blocks to set the corresponding BMBIT areas. 75 * 1.4. From the incore extent map, accumulate each bmap that falls into our 76 * target AG. Remember, multiple bmap records can map to a single rmap 77 * record, so we cannot simply emit rmap records 1:1. 78 * 1.5. Emit rmap records for each extent in BMBIT and free it. 79 * 2. Create bitmaps INOBIT and ICHUNKBIT. 80 * 3. For each record in the inobt, set the corresponding areas in ICHUNKBIT, 81 * and set bits in INOBIT for each btree block. If the inobt has no records 82 * at all, we must be careful to record its root in INOBIT. 83 * 4. For each block in the finobt, set the corresponding INOBIT area. 84 * 5. Emit rmap records for each extent in INOBIT and ICHUNKBIT and free them. 85 * 6. Create bitmaps REFCBIT and COWBIT. 86 * 7. For each CoW staging extent in the refcountbt, set the corresponding 87 * areas in COWBIT. 88 * 8. For each block in the refcountbt, set the corresponding REFCBIT area. 89 * 9. Emit rmap records for each extent in REFCBIT and COWBIT and free them. 90 * A. Emit rmap for the AG headers. 91 * B. Emit rmap for the log, if there is one. 92 * 93 * II) The rmapbt shape and space metadata rmaps are computed as follows: 94 * 95 * 1. Count the rmaps collected in the previous step. (= NR) 96 * 2. Estimate the number of rmapbt blocks needed to store NR records. (= RMB) 97 * 3. Reserve RMB blocks through the newbt using the allocator in normap mode. 98 * 4. Create bitmap AGBIT. 99 * 5. For each reservation in the newbt, set the corresponding areas in AGBIT. 100 * 6. For each block in the AGFL, bnobt, and cntbt, set the bits in AGBIT. 101 * 7. Count the extents in AGBIT. (= AGNR) 102 * 8. Estimate the number of rmapbt blocks needed for NR + AGNR rmaps. (= RMB') 103 * 9. If RMB' >= RMB, reserve RMB' - RMB more newbt blocks, set RMB = RMB', 104 * and clear AGBIT. Go to step 5. 105 * A. Emit rmaps for each extent in AGBIT. 106 * 107 * III) The rmapbt is constructed and set in place as follows: 108 * 109 * 1. Sort the rmap records. 110 * 2. Bulk load the rmaps. 111 * 112 * IV) Reap the old btree blocks. 113 * 114 * 1. Create a bitmap OLDRMBIT. 115 * 2. For each gap in the new rmapbt, set the corresponding areas of OLDRMBIT. 116 * 3. For each extent in the bnobt, clear the corresponding parts of OLDRMBIT. 117 * 4. Reap the extents corresponding to the set areas in OLDRMBIT. These are 118 * the parts of the AG that the rmap didn't find during its scan of the 119 * primary metadata and aren't known to be in the free space, which implies 120 * that they were the old rmapbt blocks. 121 * 5. Commit. 122 * 123 * We use the 'xrep_rmap' prefix for all the rmap functions. 124 */ 125 126/* Context for collecting rmaps */ 127struct xrep_rmap { 128 /* new rmapbt information */ 129 struct xrep_newbt new_btree; 130 131 /* lock for the xfbtree and xfile */ 132 struct mutex lock; 133 134 /* rmap records generated from primary metadata */ 135 struct xfbtree rmap_btree; 136 137 struct xfs_scrub *sc; 138 139 /* in-memory btree cursor for the xfs_btree_bload iteration */ 140 struct xfs_btree_cur *mcur; 141 142 /* Hooks into rmap update code. */ 143 struct xfs_rmap_hook rhook; 144 145 /* inode scan cursor */ 146 struct xchk_iscan iscan; 147 148 /* Number of non-freespace records found. */ 149 unsigned long long nr_records; 150 151 /* bnobt/cntbt contribution to btreeblks */ 152 xfs_agblock_t freesp_btblocks; 153 154 /* old agf_rmap_blocks counter */ 155 unsigned int old_rmapbt_fsbcount; 156}; 157 158/* Set us up to repair reverse mapping btrees. */ 159int 160xrep_setup_ag_rmapbt( 161 struct xfs_scrub *sc) 162{ 163 struct xrep_rmap *rr; 164 char *descr; 165 int error; 166 167 xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); 168 169 descr = xchk_xfile_ag_descr(sc, "reverse mapping records"); 170 error = xrep_setup_xfbtree(sc, descr); 171 kfree(descr); 172 if (error) 173 return error; 174 175 rr = kzalloc(sizeof(struct xrep_rmap), XCHK_GFP_FLAGS); 176 if (!rr) 177 return -ENOMEM; 178 179 rr->sc = sc; 180 sc->buf = rr; 181 return 0; 182} 183 184/* Make sure there's nothing funny about this mapping. */ 185STATIC int 186xrep_rmap_check_mapping( 187 struct xfs_scrub *sc, 188 const struct xfs_rmap_irec *rec) 189{ 190 enum xbtree_recpacking outcome; 191 int error; 192 193 if (xfs_rmap_check_irec(sc->sa.pag, rec) != NULL) 194 return -EFSCORRUPTED; 195 196 /* Make sure this isn't free space. */ 197 error = xfs_alloc_has_records(sc->sa.bno_cur, rec->rm_startblock, 198 rec->rm_blockcount, &outcome); 199 if (error) 200 return error; 201 if (outcome != XBTREE_RECPACKING_EMPTY) 202 return -EFSCORRUPTED; 203 204 return 0; 205} 206 207/* Store a reverse-mapping record. */ 208static inline int 209xrep_rmap_stash( 210 struct xrep_rmap *rr, 211 xfs_agblock_t startblock, 212 xfs_extlen_t blockcount, 213 uint64_t owner, 214 uint64_t offset, 215 unsigned int flags) 216{ 217 struct xfs_rmap_irec rmap = { 218 .rm_startblock = startblock, 219 .rm_blockcount = blockcount, 220 .rm_owner = owner, 221 .rm_offset = offset, 222 .rm_flags = flags, 223 }; 224 struct xfs_scrub *sc = rr->sc; 225 struct xfs_btree_cur *mcur; 226 int error = 0; 227 228 if (xchk_should_terminate(sc, &error)) 229 return error; 230 231 if (xchk_iscan_aborted(&rr->iscan)) 232 return -EFSCORRUPTED; 233 234 trace_xrep_rmap_found(sc->mp, sc->sa.pag->pag_agno, &rmap); 235 236 mutex_lock(&rr->lock); 237 mcur = xfs_rmapbt_mem_cursor(sc->sa.pag, sc->tp, &rr->rmap_btree); 238 error = xfs_rmap_map_raw(mcur, &rmap); 239 xfs_btree_del_cursor(mcur, error); 240 if (error) 241 goto out_cancel; 242 243 error = xfbtree_trans_commit(&rr->rmap_btree, sc->tp); 244 if (error) 245 goto out_abort; 246 247 mutex_unlock(&rr->lock); 248 return 0; 249 250out_cancel: 251 xfbtree_trans_cancel(&rr->rmap_btree, sc->tp); 252out_abort: 253 xchk_iscan_abort(&rr->iscan); 254 mutex_unlock(&rr->lock); 255 return error; 256} 257 258struct xrep_rmap_stash_run { 259 struct xrep_rmap *rr; 260 uint64_t owner; 261 unsigned int rmap_flags; 262}; 263 264static int 265xrep_rmap_stash_run( 266 uint32_t start, 267 uint32_t len, 268 void *priv) 269{ 270 struct xrep_rmap_stash_run *rsr = priv; 271 struct xrep_rmap *rr = rsr->rr; 272 273 return xrep_rmap_stash(rr, start, len, rsr->owner, 0, rsr->rmap_flags); 274} 275 276/* 277 * Emit rmaps for every extent of bits set in the bitmap. Caller must ensure 278 * that the ranges are in units of FS blocks. 279 */ 280STATIC int 281xrep_rmap_stash_bitmap( 282 struct xrep_rmap *rr, 283 struct xagb_bitmap *bitmap, 284 const struct xfs_owner_info *oinfo) 285{ 286 struct xrep_rmap_stash_run rsr = { 287 .rr = rr, 288 .owner = oinfo->oi_owner, 289 .rmap_flags = 0, 290 }; 291 292 if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK) 293 rsr.rmap_flags |= XFS_RMAP_ATTR_FORK; 294 if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK) 295 rsr.rmap_flags |= XFS_RMAP_BMBT_BLOCK; 296 297 return xagb_bitmap_walk(bitmap, xrep_rmap_stash_run, &rsr); 298} 299 300/* Section (I): Finding all file and bmbt extents. */ 301 302/* Context for accumulating rmaps for an inode fork. */ 303struct xrep_rmap_ifork { 304 /* 305 * Accumulate rmap data here to turn multiple adjacent bmaps into a 306 * single rmap. 307 */ 308 struct xfs_rmap_irec accum; 309 310 /* Bitmap of bmbt blocks in this AG. */ 311 struct xagb_bitmap bmbt_blocks; 312 313 struct xrep_rmap *rr; 314 315 /* Which inode fork? */ 316 int whichfork; 317}; 318 319/* Stash an rmap that we accumulated while walking an inode fork. */ 320STATIC int 321xrep_rmap_stash_accumulated( 322 struct xrep_rmap_ifork *rf) 323{ 324 if (rf->accum.rm_blockcount == 0) 325 return 0; 326 327 return xrep_rmap_stash(rf->rr, rf->accum.rm_startblock, 328 rf->accum.rm_blockcount, rf->accum.rm_owner, 329 rf->accum.rm_offset, rf->accum.rm_flags); 330} 331 332/* Accumulate a bmbt record. */ 333STATIC int 334xrep_rmap_visit_bmbt( 335 struct xfs_btree_cur *cur, 336 struct xfs_bmbt_irec *rec, 337 void *priv) 338{ 339 struct xrep_rmap_ifork *rf = priv; 340 struct xfs_mount *mp = rf->rr->sc->mp; 341 struct xfs_rmap_irec *accum = &rf->accum; 342 xfs_agblock_t agbno; 343 unsigned int rmap_flags = 0; 344 int error; 345 346 if (XFS_FSB_TO_AGNO(mp, rec->br_startblock) != 347 rf->rr->sc->sa.pag->pag_agno) 348 return 0; 349 350 agbno = XFS_FSB_TO_AGBNO(mp, rec->br_startblock); 351 if (rf->whichfork == XFS_ATTR_FORK) 352 rmap_flags |= XFS_RMAP_ATTR_FORK; 353 if (rec->br_state == XFS_EXT_UNWRITTEN) 354 rmap_flags |= XFS_RMAP_UNWRITTEN; 355 356 /* If this bmap is adjacent to the previous one, just add it. */ 357 if (accum->rm_blockcount > 0 && 358 rec->br_startoff == accum->rm_offset + accum->rm_blockcount && 359 agbno == accum->rm_startblock + accum->rm_blockcount && 360 rmap_flags == accum->rm_flags) { 361 accum->rm_blockcount += rec->br_blockcount; 362 return 0; 363 } 364 365 /* Otherwise stash the old rmap and start accumulating a new one. */ 366 error = xrep_rmap_stash_accumulated(rf); 367 if (error) 368 return error; 369 370 accum->rm_startblock = agbno; 371 accum->rm_blockcount = rec->br_blockcount; 372 accum->rm_offset = rec->br_startoff; 373 accum->rm_flags = rmap_flags; 374 return 0; 375} 376 377/* Add a btree block to the bitmap. */ 378STATIC int 379xrep_rmap_visit_iroot_btree_block( 380 struct xfs_btree_cur *cur, 381 int level, 382 void *priv) 383{ 384 struct xrep_rmap_ifork *rf = priv; 385 struct xfs_buf *bp; 386 xfs_fsblock_t fsbno; 387 xfs_agblock_t agbno; 388 389 xfs_btree_get_block(cur, level, &bp); 390 if (!bp) 391 return 0; 392 393 fsbno = XFS_DADDR_TO_FSB(cur->bc_mp, xfs_buf_daddr(bp)); 394 if (XFS_FSB_TO_AGNO(cur->bc_mp, fsbno) != rf->rr->sc->sa.pag->pag_agno) 395 return 0; 396 397 agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); 398 return xagb_bitmap_set(&rf->bmbt_blocks, agbno, 1); 399} 400 401/* 402 * Iterate a metadata btree rooted in an inode to collect rmap records for 403 * anything in this fork that matches the AG. 404 */ 405STATIC int 406xrep_rmap_scan_iroot_btree( 407 struct xrep_rmap_ifork *rf, 408 struct xfs_btree_cur *cur) 409{ 410 struct xfs_owner_info oinfo; 411 struct xrep_rmap *rr = rf->rr; 412 int error; 413 414 xagb_bitmap_init(&rf->bmbt_blocks); 415 416 /* Record all the blocks in the btree itself. */ 417 error = xfs_btree_visit_blocks(cur, xrep_rmap_visit_iroot_btree_block, 418 XFS_BTREE_VISIT_ALL, rf); 419 if (error) 420 goto out; 421 422 /* Emit rmaps for the btree blocks. */ 423 xfs_rmap_ino_bmbt_owner(&oinfo, rf->accum.rm_owner, rf->whichfork); 424 error = xrep_rmap_stash_bitmap(rr, &rf->bmbt_blocks, &oinfo); 425 if (error) 426 goto out; 427 428 /* Stash any remaining accumulated rmaps. */ 429 error = xrep_rmap_stash_accumulated(rf); 430out: 431 xagb_bitmap_destroy(&rf->bmbt_blocks); 432 return error; 433} 434 435static inline bool 436is_rt_data_fork( 437 struct xfs_inode *ip, 438 int whichfork) 439{ 440 return XFS_IS_REALTIME_INODE(ip) && whichfork == XFS_DATA_FORK; 441} 442 443/* 444 * Iterate the block mapping btree to collect rmap records for anything in this 445 * fork that matches the AG. Sets @mappings_done to true if we've scanned the 446 * block mappings in this fork. 447 */ 448STATIC int 449xrep_rmap_scan_bmbt( 450 struct xrep_rmap_ifork *rf, 451 struct xfs_inode *ip, 452 bool *mappings_done) 453{ 454 struct xrep_rmap *rr = rf->rr; 455 struct xfs_btree_cur *cur; 456 struct xfs_ifork *ifp; 457 int error; 458 459 *mappings_done = false; 460 ifp = xfs_ifork_ptr(ip, rf->whichfork); 461 cur = xfs_bmbt_init_cursor(rr->sc->mp, rr->sc->tp, ip, rf->whichfork); 462 463 if (!xfs_ifork_is_realtime(ip, rf->whichfork) && 464 xfs_need_iread_extents(ifp)) { 465 /* 466 * If the incore extent cache isn't loaded, scan the bmbt for 467 * mapping records. This avoids loading the incore extent 468 * tree, which will increase memory pressure at a time when 469 * we're trying to run as quickly as we possibly can. Ignore 470 * realtime extents. 471 */ 472 error = xfs_bmap_query_all(cur, xrep_rmap_visit_bmbt, rf); 473 if (error) 474 goto out_cur; 475 476 *mappings_done = true; 477 } 478 479 /* Scan for the bmbt blocks, which always live on the data device. */ 480 error = xrep_rmap_scan_iroot_btree(rf, cur); 481out_cur: 482 xfs_btree_del_cursor(cur, error); 483 return error; 484} 485 486/* 487 * Iterate the in-core extent cache to collect rmap records for anything in 488 * this fork that matches the AG. 489 */ 490STATIC int 491xrep_rmap_scan_iext( 492 struct xrep_rmap_ifork *rf, 493 struct xfs_ifork *ifp) 494{ 495 struct xfs_bmbt_irec rec; 496 struct xfs_iext_cursor icur; 497 int error; 498 499 for_each_xfs_iext(ifp, &icur, &rec) { 500 if (isnullstartblock(rec.br_startblock)) 501 continue; 502 error = xrep_rmap_visit_bmbt(NULL, &rec, rf); 503 if (error) 504 return error; 505 } 506 507 return xrep_rmap_stash_accumulated(rf); 508} 509 510/* Find all the extents from a given AG in an inode fork. */ 511STATIC int 512xrep_rmap_scan_ifork( 513 struct xrep_rmap *rr, 514 struct xfs_inode *ip, 515 int whichfork) 516{ 517 struct xrep_rmap_ifork rf = { 518 .accum = { .rm_owner = ip->i_ino, }, 519 .rr = rr, 520 .whichfork = whichfork, 521 }; 522 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); 523 int error = 0; 524 525 if (!ifp) 526 return 0; 527 528 if (ifp->if_format == XFS_DINODE_FMT_BTREE) { 529 bool mappings_done; 530 531 /* 532 * Scan the bmap btree for data device mappings. This includes 533 * the btree blocks themselves, even if this is a realtime 534 * file. 535 */ 536 error = xrep_rmap_scan_bmbt(&rf, ip, &mappings_done); 537 if (error || mappings_done) 538 return error; 539 } else if (ifp->if_format != XFS_DINODE_FMT_EXTENTS) { 540 return 0; 541 } 542 543 /* Scan incore extent cache if this isn't a realtime file. */ 544 if (xfs_ifork_is_realtime(ip, whichfork)) 545 return 0; 546 547 return xrep_rmap_scan_iext(&rf, ifp); 548} 549 550/* 551 * Take ILOCK on a file that we want to scan. 552 * 553 * Select ILOCK_EXCL if the file has an unloaded data bmbt or has an unloaded 554 * attr bmbt. Otherwise, take ILOCK_SHARED. 555 */ 556static inline unsigned int 557xrep_rmap_scan_ilock( 558 struct xfs_inode *ip) 559{ 560 uint lock_mode = XFS_ILOCK_SHARED; 561 562 if (xfs_need_iread_extents(&ip->i_df)) { 563 lock_mode = XFS_ILOCK_EXCL; 564 goto lock; 565 } 566 567 if (xfs_inode_has_attr_fork(ip) && xfs_need_iread_extents(&ip->i_af)) 568 lock_mode = XFS_ILOCK_EXCL; 569 570lock: 571 xfs_ilock(ip, lock_mode); 572 return lock_mode; 573} 574 575/* Record reverse mappings for a file. */ 576STATIC int 577xrep_rmap_scan_inode( 578 struct xrep_rmap *rr, 579 struct xfs_inode *ip) 580{ 581 unsigned int lock_mode = 0; 582 int error; 583 584 /* 585 * Directory updates (create/link/unlink/rename) drop the directory's 586 * ILOCK before finishing any rmapbt updates associated with directory 587 * shape changes. For this scan to coordinate correctly with the live 588 * update hook, we must take the only lock (i_rwsem) that is held all 589 * the way to dir op completion. This will get fixed by the parent 590 * pointer patchset. 591 */ 592 if (S_ISDIR(VFS_I(ip)->i_mode)) { 593 lock_mode = XFS_IOLOCK_SHARED; 594 xfs_ilock(ip, lock_mode); 595 } 596 lock_mode |= xrep_rmap_scan_ilock(ip); 597 598 /* Check the data fork. */ 599 error = xrep_rmap_scan_ifork(rr, ip, XFS_DATA_FORK); 600 if (error) 601 goto out_unlock; 602 603 /* Check the attr fork. */ 604 error = xrep_rmap_scan_ifork(rr, ip, XFS_ATTR_FORK); 605 if (error) 606 goto out_unlock; 607 608 /* COW fork extents are "owned" by the refcount btree. */ 609 610 xchk_iscan_mark_visited(&rr->iscan, ip); 611out_unlock: 612 xfs_iunlock(ip, lock_mode); 613 return error; 614} 615 616/* Section (I): Find all AG metadata extents except for free space metadata. */ 617 618struct xrep_rmap_inodes { 619 struct xrep_rmap *rr; 620 struct xagb_bitmap inobt_blocks; /* INOBIT */ 621 struct xagb_bitmap ichunk_blocks; /* ICHUNKBIT */ 622}; 623 624/* Record inode btree rmaps. */ 625STATIC int 626xrep_rmap_walk_inobt( 627 struct xfs_btree_cur *cur, 628 const union xfs_btree_rec *rec, 629 void *priv) 630{ 631 struct xfs_inobt_rec_incore irec; 632 struct xrep_rmap_inodes *ri = priv; 633 struct xfs_mount *mp = cur->bc_mp; 634 xfs_agblock_t agbno; 635 xfs_extlen_t aglen; 636 xfs_agino_t agino; 637 xfs_agino_t iperhole; 638 unsigned int i; 639 int error; 640 641 /* Record the inobt blocks. */ 642 error = xagb_bitmap_set_btcur_path(&ri->inobt_blocks, cur); 643 if (error) 644 return error; 645 646 xfs_inobt_btrec_to_irec(mp, rec, &irec); 647 if (xfs_inobt_check_irec(cur->bc_ag.pag, &irec) != NULL) 648 return -EFSCORRUPTED; 649 650 agino = irec.ir_startino; 651 652 /* Record a non-sparse inode chunk. */ 653 if (!xfs_inobt_issparse(irec.ir_holemask)) { 654 agbno = XFS_AGINO_TO_AGBNO(mp, agino); 655 aglen = max_t(xfs_extlen_t, 1, 656 XFS_INODES_PER_CHUNK / mp->m_sb.sb_inopblock); 657 658 return xagb_bitmap_set(&ri->ichunk_blocks, agbno, aglen); 659 } 660 661 /* Iterate each chunk. */ 662 iperhole = max_t(xfs_agino_t, mp->m_sb.sb_inopblock, 663 XFS_INODES_PER_HOLEMASK_BIT); 664 aglen = iperhole / mp->m_sb.sb_inopblock; 665 for (i = 0, agino = irec.ir_startino; 666 i < XFS_INOBT_HOLEMASK_BITS; 667 i += iperhole / XFS_INODES_PER_HOLEMASK_BIT, agino += iperhole) { 668 /* Skip holes. */ 669 if (irec.ir_holemask & (1 << i)) 670 continue; 671 672 /* Record the inode chunk otherwise. */ 673 agbno = XFS_AGINO_TO_AGBNO(mp, agino); 674 error = xagb_bitmap_set(&ri->ichunk_blocks, agbno, aglen); 675 if (error) 676 return error; 677 } 678 679 return 0; 680} 681 682/* Collect rmaps for the blocks containing inode btrees and the inode chunks. */ 683STATIC int 684xrep_rmap_find_inode_rmaps( 685 struct xrep_rmap *rr) 686{ 687 struct xrep_rmap_inodes ri = { 688 .rr = rr, 689 }; 690 struct xfs_scrub *sc = rr->sc; 691 int error; 692 693 xagb_bitmap_init(&ri.inobt_blocks); 694 xagb_bitmap_init(&ri.ichunk_blocks); 695 696 /* 697 * Iterate every record in the inobt so we can capture all the inode 698 * chunks and the blocks in the inobt itself. 699 */ 700 error = xfs_btree_query_all(sc->sa.ino_cur, xrep_rmap_walk_inobt, &ri); 701 if (error) 702 goto out_bitmap; 703 704 /* 705 * Note that if there are zero records in the inobt then query_all does 706 * nothing and we have to account the empty inobt root manually. 707 */ 708 if (xagb_bitmap_empty(&ri.ichunk_blocks)) { 709 struct xfs_agi *agi = sc->sa.agi_bp->b_addr; 710 711 error = xagb_bitmap_set(&ri.inobt_blocks, 712 be32_to_cpu(agi->agi_root), 1); 713 if (error) 714 goto out_bitmap; 715 } 716 717 /* Scan the finobt too. */ 718 if (xfs_has_finobt(sc->mp)) { 719 error = xagb_bitmap_set_btblocks(&ri.inobt_blocks, 720 sc->sa.fino_cur); 721 if (error) 722 goto out_bitmap; 723 } 724 725 /* Generate rmaps for everything. */ 726 error = xrep_rmap_stash_bitmap(rr, &ri.inobt_blocks, 727 &XFS_RMAP_OINFO_INOBT); 728 if (error) 729 goto out_bitmap; 730 error = xrep_rmap_stash_bitmap(rr, &ri.ichunk_blocks, 731 &XFS_RMAP_OINFO_INODES); 732 733out_bitmap: 734 xagb_bitmap_destroy(&ri.inobt_blocks); 735 xagb_bitmap_destroy(&ri.ichunk_blocks); 736 return error; 737} 738 739/* Record a CoW staging extent. */ 740STATIC int 741xrep_rmap_walk_cowblocks( 742 struct xfs_btree_cur *cur, 743 const struct xfs_refcount_irec *irec, 744 void *priv) 745{ 746 struct xagb_bitmap *bitmap = priv; 747 748 if (!xfs_refcount_check_domain(irec) || 749 irec->rc_domain != XFS_REFC_DOMAIN_COW) 750 return -EFSCORRUPTED; 751 752 return xagb_bitmap_set(bitmap, irec->rc_startblock, irec->rc_blockcount); 753} 754 755/* 756 * Collect rmaps for the blocks containing the refcount btree, and all CoW 757 * staging extents. 758 */ 759STATIC int 760xrep_rmap_find_refcount_rmaps( 761 struct xrep_rmap *rr) 762{ 763 struct xagb_bitmap refcountbt_blocks; /* REFCBIT */ 764 struct xagb_bitmap cow_blocks; /* COWBIT */ 765 struct xfs_refcount_irec low = { 766 .rc_startblock = 0, 767 .rc_domain = XFS_REFC_DOMAIN_COW, 768 }; 769 struct xfs_refcount_irec high = { 770 .rc_startblock = -1U, 771 .rc_domain = XFS_REFC_DOMAIN_COW, 772 }; 773 struct xfs_scrub *sc = rr->sc; 774 int error; 775 776 if (!xfs_has_reflink(sc->mp)) 777 return 0; 778 779 xagb_bitmap_init(&refcountbt_blocks); 780 xagb_bitmap_init(&cow_blocks); 781 782 /* refcountbt */ 783 error = xagb_bitmap_set_btblocks(&refcountbt_blocks, sc->sa.refc_cur); 784 if (error) 785 goto out_bitmap; 786 787 /* Collect rmaps for CoW staging extents. */ 788 error = xfs_refcount_query_range(sc->sa.refc_cur, &low, &high, 789 xrep_rmap_walk_cowblocks, &cow_blocks); 790 if (error) 791 goto out_bitmap; 792 793 /* Generate rmaps for everything. */ 794 error = xrep_rmap_stash_bitmap(rr, &cow_blocks, &XFS_RMAP_OINFO_COW); 795 if (error) 796 goto out_bitmap; 797 error = xrep_rmap_stash_bitmap(rr, &refcountbt_blocks, 798 &XFS_RMAP_OINFO_REFC); 799 800out_bitmap: 801 xagb_bitmap_destroy(&cow_blocks); 802 xagb_bitmap_destroy(&refcountbt_blocks); 803 return error; 804} 805 806/* Generate rmaps for the AG headers (AGI/AGF/AGFL) */ 807STATIC int 808xrep_rmap_find_agheader_rmaps( 809 struct xrep_rmap *rr) 810{ 811 struct xfs_scrub *sc = rr->sc; 812 813 /* Create a record for the AG sb->agfl. */ 814 return xrep_rmap_stash(rr, XFS_SB_BLOCK(sc->mp), 815 XFS_AGFL_BLOCK(sc->mp) - XFS_SB_BLOCK(sc->mp) + 1, 816 XFS_RMAP_OWN_FS, 0, 0); 817} 818 819/* Generate rmaps for the log, if it's in this AG. */ 820STATIC int 821xrep_rmap_find_log_rmaps( 822 struct xrep_rmap *rr) 823{ 824 struct xfs_scrub *sc = rr->sc; 825 826 if (!xfs_ag_contains_log(sc->mp, sc->sa.pag->pag_agno)) 827 return 0; 828 829 return xrep_rmap_stash(rr, 830 XFS_FSB_TO_AGBNO(sc->mp, sc->mp->m_sb.sb_logstart), 831 sc->mp->m_sb.sb_logblocks, XFS_RMAP_OWN_LOG, 0, 0); 832} 833 834/* Check and count all the records that we gathered. */ 835STATIC int 836xrep_rmap_check_record( 837 struct xfs_btree_cur *cur, 838 const struct xfs_rmap_irec *rec, 839 void *priv) 840{ 841 struct xrep_rmap *rr = priv; 842 int error; 843 844 error = xrep_rmap_check_mapping(rr->sc, rec); 845 if (error) 846 return error; 847 848 rr->nr_records++; 849 return 0; 850} 851 852/* 853 * Generate all the reverse-mappings for this AG, a list of the old rmapbt 854 * blocks, and the new btreeblks count. Figure out if we have enough free 855 * space to reconstruct the inode btrees. The caller must clean up the lists 856 * if anything goes wrong. This implements section (I) above. 857 */ 858STATIC int 859xrep_rmap_find_rmaps( 860 struct xrep_rmap *rr) 861{ 862 struct xfs_scrub *sc = rr->sc; 863 struct xchk_ag *sa = &sc->sa; 864 struct xfs_inode *ip; 865 struct xfs_btree_cur *mcur; 866 int error; 867 868 /* Find all the per-AG metadata. */ 869 xrep_ag_btcur_init(sc, &sc->sa); 870 871 error = xrep_rmap_find_inode_rmaps(rr); 872 if (error) 873 goto end_agscan; 874 875 error = xrep_rmap_find_refcount_rmaps(rr); 876 if (error) 877 goto end_agscan; 878 879 error = xrep_rmap_find_agheader_rmaps(rr); 880 if (error) 881 goto end_agscan; 882 883 error = xrep_rmap_find_log_rmaps(rr); 884end_agscan: 885 xchk_ag_btcur_free(&sc->sa); 886 if (error) 887 return error; 888 889 /* 890 * Set up for a potentially lengthy filesystem scan by reducing our 891 * transaction resource usage for the duration. Specifically: 892 * 893 * Unlock the AG header buffers and cancel the transaction to release 894 * the log grant space while we scan the filesystem. 895 * 896 * Create a new empty transaction to eliminate the possibility of the 897 * inode scan deadlocking on cyclical metadata. 898 * 899 * We pass the empty transaction to the file scanning function to avoid 900 * repeatedly cycling empty transactions. This can be done even though 901 * we take the IOLOCK to quiesce the file because empty transactions 902 * do not take sb_internal. 903 */ 904 sa->agf_bp = NULL; 905 sa->agi_bp = NULL; 906 xchk_trans_cancel(sc); 907 error = xchk_trans_alloc_empty(sc); 908 if (error) 909 return error; 910 911 /* Iterate all AGs for inodes rmaps. */ 912 while ((error = xchk_iscan_iter(&rr->iscan, &ip)) == 1) { 913 error = xrep_rmap_scan_inode(rr, ip); 914 xchk_irele(sc, ip); 915 if (error) 916 break; 917 918 if (xchk_should_terminate(sc, &error)) 919 break; 920 } 921 xchk_iscan_iter_finish(&rr->iscan); 922 if (error) 923 return error; 924 925 /* 926 * Switch out for a real transaction and lock the AG headers in 927 * preparation for building a new tree. 928 */ 929 xchk_trans_cancel(sc); 930 error = xchk_setup_fs(sc); 931 if (error) 932 return error; 933 error = xchk_perag_drain_and_lock(sc); 934 if (error) 935 return error; 936 937 /* 938 * If a hook failed to update the in-memory btree, we lack the data to 939 * continue the repair. 940 */ 941 if (xchk_iscan_aborted(&rr->iscan)) 942 return -EFSCORRUPTED; 943 944 /* 945 * Now that we have everything locked again, we need to count the 946 * number of rmap records stashed in the btree. This should reflect 947 * all actively-owned space in the filesystem. At the same time, check 948 * all our records before we start building a new btree, which requires 949 * a bnobt cursor. 950 */ 951 mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, &rr->rmap_btree); 952 sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, 953 sc->sa.pag); 954 955 rr->nr_records = 0; 956 error = xfs_rmap_query_all(mcur, xrep_rmap_check_record, rr); 957 958 xfs_btree_del_cursor(sc->sa.bno_cur, error); 959 sc->sa.bno_cur = NULL; 960 xfs_btree_del_cursor(mcur, error); 961 962 return error; 963} 964 965/* Section (II): Reserving space for new rmapbt and setting free space bitmap */ 966 967struct xrep_rmap_agfl { 968 struct xagb_bitmap *bitmap; 969 xfs_agnumber_t agno; 970}; 971 972/* Add an AGFL block to the rmap list. */ 973STATIC int 974xrep_rmap_walk_agfl( 975 struct xfs_mount *mp, 976 xfs_agblock_t agbno, 977 void *priv) 978{ 979 struct xrep_rmap_agfl *ra = priv; 980 981 return xagb_bitmap_set(ra->bitmap, agbno, 1); 982} 983 984/* 985 * Run one round of reserving space for the new rmapbt and recomputing the 986 * number of blocks needed to store the previously observed rmapbt records and 987 * the ones we'll create for the free space metadata. When we don't need more 988 * blocks, return a bitmap of OWN_AG extents in @freesp_blocks and set @done to 989 * true. 990 */ 991STATIC int 992xrep_rmap_try_reserve( 993 struct xrep_rmap *rr, 994 struct xfs_btree_cur *rmap_cur, 995 struct xagb_bitmap *freesp_blocks, 996 uint64_t *blocks_reserved, 997 bool *done) 998{ 999 struct xrep_rmap_agfl ra = { 1000 .bitmap = freesp_blocks, 1001 .agno = rr->sc->sa.pag->pag_agno, 1002 }; 1003 struct xfs_scrub *sc = rr->sc; 1004 struct xrep_newbt_resv *resv, *n; 1005 struct xfs_agf *agf = sc->sa.agf_bp->b_addr; 1006 struct xfs_buf *agfl_bp; 1007 uint64_t nr_blocks; /* RMB */ 1008 uint64_t freesp_records; 1009 int error; 1010 1011 /* 1012 * We're going to recompute new_btree.bload.nr_blocks at the end of 1013 * this function to reflect however many btree blocks we need to store 1014 * all the rmap records (including the ones that reflect the changes we 1015 * made to support the new rmapbt blocks), so we save the old value 1016 * here so we can decide if we've reserved enough blocks. 1017 */ 1018 nr_blocks = rr->new_btree.bload.nr_blocks; 1019 1020 /* 1021 * Make sure we've reserved enough space for the new btree. This can 1022 * change the shape of the free space btrees, which can cause secondary 1023 * interactions with the rmap records because all three space btrees 1024 * have the same rmap owner. We'll account for all that below. 1025 */ 1026 error = xrep_newbt_alloc_blocks(&rr->new_btree, 1027 nr_blocks - *blocks_reserved); 1028 if (error) 1029 return error; 1030 1031 *blocks_reserved = rr->new_btree.bload.nr_blocks; 1032 1033 /* Clear everything in the bitmap. */ 1034 xagb_bitmap_destroy(freesp_blocks); 1035 1036 /* Set all the bnobt blocks in the bitmap. */ 1037 sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, 1038 sc->sa.pag); 1039 error = xagb_bitmap_set_btblocks(freesp_blocks, sc->sa.bno_cur); 1040 xfs_btree_del_cursor(sc->sa.bno_cur, error); 1041 sc->sa.bno_cur = NULL; 1042 if (error) 1043 return error; 1044 1045 /* Set all the cntbt blocks in the bitmap. */ 1046 sc->sa.cnt_cur = xfs_cntbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, 1047 sc->sa.pag); 1048 error = xagb_bitmap_set_btblocks(freesp_blocks, sc->sa.cnt_cur); 1049 xfs_btree_del_cursor(sc->sa.cnt_cur, error); 1050 sc->sa.cnt_cur = NULL; 1051 if (error) 1052 return error; 1053 1054 /* Record our new btreeblks value. */ 1055 rr->freesp_btblocks = xagb_bitmap_hweight(freesp_blocks) - 2; 1056 1057 /* Set all the new rmapbt blocks in the bitmap. */ 1058 list_for_each_entry_safe(resv, n, &rr->new_btree.resv_list, list) { 1059 error = xagb_bitmap_set(freesp_blocks, resv->agbno, resv->len); 1060 if (error) 1061 return error; 1062 } 1063 1064 /* Set all the AGFL blocks in the bitmap. */ 1065 error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); 1066 if (error) 1067 return error; 1068 1069 error = xfs_agfl_walk(sc->mp, agf, agfl_bp, xrep_rmap_walk_agfl, &ra); 1070 if (error) 1071 return error; 1072 1073 /* Count the extents in the bitmap. */ 1074 freesp_records = xagb_bitmap_count_set_regions(freesp_blocks); 1075 1076 /* Compute how many blocks we'll need for all the rmaps. */ 1077 error = xfs_btree_bload_compute_geometry(rmap_cur, 1078 &rr->new_btree.bload, rr->nr_records + freesp_records); 1079 if (error) 1080 return error; 1081 1082 /* We're done when we don't need more blocks. */ 1083 *done = nr_blocks >= rr->new_btree.bload.nr_blocks; 1084 return 0; 1085} 1086 1087/* 1088 * Iteratively reserve space for rmap btree while recording OWN_AG rmaps for 1089 * the free space metadata. This implements section (II) above. 1090 */ 1091STATIC int 1092xrep_rmap_reserve_space( 1093 struct xrep_rmap *rr, 1094 struct xfs_btree_cur *rmap_cur) 1095{ 1096 struct xagb_bitmap freesp_blocks; /* AGBIT */ 1097 uint64_t blocks_reserved = 0; 1098 bool done = false; 1099 int error; 1100 1101 /* Compute how many blocks we'll need for the rmaps collected so far. */ 1102 error = xfs_btree_bload_compute_geometry(rmap_cur, 1103 &rr->new_btree.bload, rr->nr_records); 1104 if (error) 1105 return error; 1106 1107 /* Last chance to abort before we start committing fixes. */ 1108 if (xchk_should_terminate(rr->sc, &error)) 1109 return error; 1110 1111 xagb_bitmap_init(&freesp_blocks); 1112 1113 /* 1114 * Iteratively reserve space for the new rmapbt and recompute the 1115 * number of blocks needed to store the previously observed rmapbt 1116 * records and the ones we'll create for the free space metadata. 1117 * Finish when we don't need more blocks. 1118 */ 1119 do { 1120 error = xrep_rmap_try_reserve(rr, rmap_cur, &freesp_blocks, 1121 &blocks_reserved, &done); 1122 if (error) 1123 goto out_bitmap; 1124 } while (!done); 1125 1126 /* Emit rmaps for everything in the free space bitmap. */ 1127 xrep_ag_btcur_init(rr->sc, &rr->sc->sa); 1128 error = xrep_rmap_stash_bitmap(rr, &freesp_blocks, &XFS_RMAP_OINFO_AG); 1129 xchk_ag_btcur_free(&rr->sc->sa); 1130 1131out_bitmap: 1132 xagb_bitmap_destroy(&freesp_blocks); 1133 return error; 1134} 1135 1136/* Section (III): Building the new rmap btree. */ 1137 1138/* Update the AGF counters. */ 1139STATIC int 1140xrep_rmap_reset_counters( 1141 struct xrep_rmap *rr) 1142{ 1143 struct xfs_scrub *sc = rr->sc; 1144 struct xfs_perag *pag = sc->sa.pag; 1145 struct xfs_agf *agf = sc->sa.agf_bp->b_addr; 1146 xfs_agblock_t rmap_btblocks; 1147 1148 /* 1149 * The AGF header contains extra information related to the reverse 1150 * mapping btree, so we must update those fields here. 1151 */ 1152 rmap_btblocks = rr->new_btree.afake.af_blocks - 1; 1153 agf->agf_btreeblks = cpu_to_be32(rr->freesp_btblocks + rmap_btblocks); 1154 xfs_alloc_log_agf(sc->tp, sc->sa.agf_bp, XFS_AGF_BTREEBLKS); 1155 1156 /* 1157 * After we commit the new btree to disk, it is possible that the 1158 * process to reap the old btree blocks will race with the AIL trying 1159 * to checkpoint the old btree blocks into the filesystem. If the new 1160 * tree is shorter than the old one, the rmapbt write verifier will 1161 * fail and the AIL will shut down the filesystem. 1162 * 1163 * To avoid this, save the old incore btree height values as the alt 1164 * height values before re-initializing the perag info from the updated 1165 * AGF to capture all the new values. 1166 */ 1167 pag->pagf_repair_rmap_level = pag->pagf_rmap_level; 1168 1169 /* Reinitialize with the values we just logged. */ 1170 return xrep_reinit_pagf(sc); 1171} 1172 1173/* Retrieve rmapbt data for bulk load. */ 1174STATIC int 1175xrep_rmap_get_records( 1176 struct xfs_btree_cur *cur, 1177 unsigned int idx, 1178 struct xfs_btree_block *block, 1179 unsigned int nr_wanted, 1180 void *priv) 1181{ 1182 struct xrep_rmap *rr = priv; 1183 union xfs_btree_rec *block_rec; 1184 unsigned int loaded; 1185 int error; 1186 1187 for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { 1188 int stat = 0; 1189 1190 error = xfs_btree_increment(rr->mcur, 0, &stat); 1191 if (error) 1192 return error; 1193 if (!stat) 1194 return -EFSCORRUPTED; 1195 1196 error = xfs_rmap_get_rec(rr->mcur, &cur->bc_rec.r, &stat); 1197 if (error) 1198 return error; 1199 if (!stat) 1200 return -EFSCORRUPTED; 1201 1202 block_rec = xfs_btree_rec_addr(cur, idx, block); 1203 cur->bc_ops->init_rec_from_cur(cur, block_rec); 1204 } 1205 1206 return loaded; 1207} 1208 1209/* Feed one of the new btree blocks to the bulk loader. */ 1210STATIC int 1211xrep_rmap_claim_block( 1212 struct xfs_btree_cur *cur, 1213 union xfs_btree_ptr *ptr, 1214 void *priv) 1215{ 1216 struct xrep_rmap *rr = priv; 1217 1218 return xrep_newbt_claim_block(cur, &rr->new_btree, ptr); 1219} 1220 1221/* Custom allocation function for new rmap btrees. */ 1222STATIC int 1223xrep_rmap_alloc_vextent( 1224 struct xfs_scrub *sc, 1225 struct xfs_alloc_arg *args, 1226 xfs_fsblock_t alloc_hint) 1227{ 1228 int error; 1229 1230 /* 1231 * We don't want an rmap update on the allocation, since we iteratively 1232 * compute the OWN_AG records /after/ allocating blocks for the records 1233 * that we already know we need to store. Therefore, fix the freelist 1234 * with the NORMAP flag set so that we don't also try to create an rmap 1235 * for new AGFL blocks. 1236 */ 1237 error = xrep_fix_freelist(sc, XFS_ALLOC_FLAG_NORMAP); 1238 if (error) 1239 return error; 1240 1241 /* 1242 * If xrep_fix_freelist fixed the freelist by moving blocks from the 1243 * free space btrees or by removing blocks from the AGFL and queueing 1244 * an EFI to free the block, the transaction will be dirty. This 1245 * second case is of interest to us. 1246 * 1247 * Later on, we will need to compare gaps in the new recordset against 1248 * the block usage of all OWN_AG owners in order to free the old 1249 * btree's blocks, which means that we can't have EFIs for former AGFL 1250 * blocks attached to the repair transaction when we commit the new 1251 * btree. 1252 * 1253 * xrep_newbt_alloc_blocks guarantees this for us by calling 1254 * xrep_defer_finish to commit anything that fix_freelist may have 1255 * added to the transaction. 1256 */ 1257 return xfs_alloc_vextent_near_bno(args, alloc_hint); 1258} 1259 1260 1261/* Count the records in this btree. */ 1262STATIC int 1263xrep_rmap_count_records( 1264 struct xfs_btree_cur *cur, 1265 unsigned long long *nr) 1266{ 1267 int running = 1; 1268 int error; 1269 1270 *nr = 0; 1271 1272 error = xfs_btree_goto_left_edge(cur); 1273 if (error) 1274 return error; 1275 1276 while (running && !(error = xfs_btree_increment(cur, 0, &running))) { 1277 if (running) 1278 (*nr)++; 1279 } 1280 1281 return error; 1282} 1283/* 1284 * Use the collected rmap information to stage a new rmap btree. If this is 1285 * successful we'll return with the new btree root information logged to the 1286 * repair transaction but not yet committed. This implements section (III) 1287 * above. 1288 */ 1289STATIC int 1290xrep_rmap_build_new_tree( 1291 struct xrep_rmap *rr) 1292{ 1293 struct xfs_scrub *sc = rr->sc; 1294 struct xfs_perag *pag = sc->sa.pag; 1295 struct xfs_agf *agf = sc->sa.agf_bp->b_addr; 1296 struct xfs_btree_cur *rmap_cur; 1297 xfs_fsblock_t fsbno; 1298 int error; 1299 1300 /* 1301 * Preserve the old rmapbt block count so that we can adjust the 1302 * per-AG rmapbt reservation after we commit the new btree root and 1303 * want to dispose of the old btree blocks. 1304 */ 1305 rr->old_rmapbt_fsbcount = be32_to_cpu(agf->agf_rmap_blocks); 1306 1307 /* 1308 * Prepare to construct the new btree by reserving disk space for the 1309 * new btree and setting up all the accounting information we'll need 1310 * to root the new btree while it's under construction and before we 1311 * attach it to the AG header. The new blocks are accounted to the 1312 * rmapbt per-AG reservation, which we will adjust further after 1313 * committing the new btree. 1314 */ 1315 fsbno = XFS_AGB_TO_FSB(sc->mp, pag->pag_agno, XFS_RMAP_BLOCK(sc->mp)); 1316 xrep_newbt_init_ag(&rr->new_btree, sc, &XFS_RMAP_OINFO_SKIP_UPDATE, 1317 fsbno, XFS_AG_RESV_RMAPBT); 1318 rr->new_btree.bload.get_records = xrep_rmap_get_records; 1319 rr->new_btree.bload.claim_block = xrep_rmap_claim_block; 1320 rr->new_btree.alloc_vextent = xrep_rmap_alloc_vextent; 1321 rmap_cur = xfs_rmapbt_init_cursor(sc->mp, NULL, NULL, pag); 1322 xfs_btree_stage_afakeroot(rmap_cur, &rr->new_btree.afake); 1323 1324 /* 1325 * Initialize @rr->new_btree, reserve space for the new rmapbt, 1326 * and compute OWN_AG rmaps. 1327 */ 1328 error = xrep_rmap_reserve_space(rr, rmap_cur); 1329 if (error) 1330 goto err_cur; 1331 1332 /* 1333 * Count the rmapbt records again, because the space reservation 1334 * for the rmapbt itself probably added more records to the btree. 1335 */ 1336 rr->mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, 1337 &rr->rmap_btree); 1338 1339 error = xrep_rmap_count_records(rr->mcur, &rr->nr_records); 1340 if (error) 1341 goto err_mcur; 1342 1343 /* 1344 * Due to btree slack factors, it's possible for a new btree to be one 1345 * level taller than the old btree. Update the incore btree height so 1346 * that we don't trip the verifiers when writing the new btree blocks 1347 * to disk. 1348 */ 1349 pag->pagf_repair_rmap_level = rr->new_btree.bload.btree_height; 1350 1351 /* 1352 * Move the cursor to the left edge of the tree so that the first 1353 * increment in ->get_records positions us at the first record. 1354 */ 1355 error = xfs_btree_goto_left_edge(rr->mcur); 1356 if (error) 1357 goto err_level; 1358 1359 /* Add all observed rmap records. */ 1360 error = xfs_btree_bload(rmap_cur, &rr->new_btree.bload, rr); 1361 if (error) 1362 goto err_level; 1363 1364 /* 1365 * Install the new btree in the AG header. After this point the old 1366 * btree is no longer accessible and the new tree is live. 1367 */ 1368 xfs_rmapbt_commit_staged_btree(rmap_cur, sc->tp, sc->sa.agf_bp); 1369 xfs_btree_del_cursor(rmap_cur, 0); 1370 xfs_btree_del_cursor(rr->mcur, 0); 1371 rr->mcur = NULL; 1372 1373 /* 1374 * Now that we've written the new btree to disk, we don't need to keep 1375 * updating the in-memory btree. Abort the scan to stop live updates. 1376 */ 1377 xchk_iscan_abort(&rr->iscan); 1378 1379 /* 1380 * The newly committed rmap recordset includes mappings for the blocks 1381 * that we reserved to build the new btree. If there is excess space 1382 * reservation to be freed, the corresponding rmap records must also be 1383 * removed. 1384 */ 1385 rr->new_btree.oinfo = XFS_RMAP_OINFO_AG; 1386 1387 /* Reset the AGF counters now that we've changed the btree shape. */ 1388 error = xrep_rmap_reset_counters(rr); 1389 if (error) 1390 goto err_newbt; 1391 1392 /* Dispose of any unused blocks and the accounting information. */ 1393 error = xrep_newbt_commit(&rr->new_btree); 1394 if (error) 1395 return error; 1396 1397 return xrep_roll_ag_trans(sc); 1398 1399err_level: 1400 pag->pagf_repair_rmap_level = 0; 1401err_mcur: 1402 xfs_btree_del_cursor(rr->mcur, error); 1403err_cur: 1404 xfs_btree_del_cursor(rmap_cur, error); 1405err_newbt: 1406 xrep_newbt_cancel(&rr->new_btree); 1407 return error; 1408} 1409 1410/* Section (IV): Reaping the old btree. */ 1411 1412struct xrep_rmap_find_gaps { 1413 struct xagb_bitmap rmap_gaps; 1414 xfs_agblock_t next_agbno; 1415}; 1416 1417/* Subtract each free extent in the bnobt from the rmap gaps. */ 1418STATIC int 1419xrep_rmap_find_freesp( 1420 struct xfs_btree_cur *cur, 1421 const struct xfs_alloc_rec_incore *rec, 1422 void *priv) 1423{ 1424 struct xrep_rmap_find_gaps *rfg = priv; 1425 1426 return xagb_bitmap_clear(&rfg->rmap_gaps, rec->ar_startblock, 1427 rec->ar_blockcount); 1428} 1429 1430/* Record the free space we find, as part of cleaning out the btree. */ 1431STATIC int 1432xrep_rmap_find_gaps( 1433 struct xfs_btree_cur *cur, 1434 const struct xfs_rmap_irec *rec, 1435 void *priv) 1436{ 1437 struct xrep_rmap_find_gaps *rfg = priv; 1438 int error; 1439 1440 if (rec->rm_startblock > rfg->next_agbno) { 1441 error = xagb_bitmap_set(&rfg->rmap_gaps, rfg->next_agbno, 1442 rec->rm_startblock - rfg->next_agbno); 1443 if (error) 1444 return error; 1445 } 1446 1447 rfg->next_agbno = max_t(xfs_agblock_t, rfg->next_agbno, 1448 rec->rm_startblock + rec->rm_blockcount); 1449 return 0; 1450} 1451 1452/* 1453 * Reap the old rmapbt blocks. Now that the rmapbt is fully rebuilt, we make 1454 * a list of gaps in the rmap records and a list of the extents mentioned in 1455 * the bnobt. Any block that's in the new rmapbt gap list but not mentioned 1456 * in the bnobt is a block from the old rmapbt and can be removed. 1457 */ 1458STATIC int 1459xrep_rmap_remove_old_tree( 1460 struct xrep_rmap *rr) 1461{ 1462 struct xrep_rmap_find_gaps rfg = { 1463 .next_agbno = 0, 1464 }; 1465 struct xfs_scrub *sc = rr->sc; 1466 struct xfs_agf *agf = sc->sa.agf_bp->b_addr; 1467 struct xfs_perag *pag = sc->sa.pag; 1468 struct xfs_btree_cur *mcur; 1469 xfs_agblock_t agend; 1470 int error; 1471 1472 xagb_bitmap_init(&rfg.rmap_gaps); 1473 1474 /* Compute free space from the new rmapbt. */ 1475 mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, &rr->rmap_btree); 1476 1477 error = xfs_rmap_query_all(mcur, xrep_rmap_find_gaps, &rfg); 1478 xfs_btree_del_cursor(mcur, error); 1479 if (error) 1480 goto out_bitmap; 1481 1482 /* Insert a record for space between the last rmap and EOAG. */ 1483 agend = be32_to_cpu(agf->agf_length); 1484 if (rfg.next_agbno < agend) { 1485 error = xagb_bitmap_set(&rfg.rmap_gaps, rfg.next_agbno, 1486 agend - rfg.next_agbno); 1487 if (error) 1488 goto out_bitmap; 1489 } 1490 1491 /* Compute free space from the existing bnobt. */ 1492 sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, 1493 sc->sa.pag); 1494 error = xfs_alloc_query_all(sc->sa.bno_cur, xrep_rmap_find_freesp, 1495 &rfg); 1496 xfs_btree_del_cursor(sc->sa.bno_cur, error); 1497 sc->sa.bno_cur = NULL; 1498 if (error) 1499 goto out_bitmap; 1500 1501 /* 1502 * Free the "free" blocks that the new rmapbt knows about but the bnobt 1503 * doesn't--these are the old rmapbt blocks. Credit the old rmapbt 1504 * block usage count back to the per-AG rmapbt reservation (and not 1505 * fdblocks, since the rmap btree lives in free space) to keep the 1506 * reservation and free space accounting correct. 1507 */ 1508 error = xrep_reap_agblocks(sc, &rfg.rmap_gaps, 1509 &XFS_RMAP_OINFO_ANY_OWNER, XFS_AG_RESV_RMAPBT); 1510 if (error) 1511 goto out_bitmap; 1512 1513 /* 1514 * Now that we've zapped all the old rmapbt blocks we can turn off 1515 * the alternate height mechanism and reset the per-AG space 1516 * reservation. 1517 */ 1518 pag->pagf_repair_rmap_level = 0; 1519 sc->flags |= XREP_RESET_PERAG_RESV; 1520out_bitmap: 1521 xagb_bitmap_destroy(&rfg.rmap_gaps); 1522 return error; 1523} 1524 1525static inline bool 1526xrep_rmapbt_want_live_update( 1527 struct xchk_iscan *iscan, 1528 const struct xfs_owner_info *oi) 1529{ 1530 if (xchk_iscan_aborted(iscan)) 1531 return false; 1532 1533 /* 1534 * Before unlocking the AG header to perform the inode scan, we 1535 * recorded reverse mappings for all AG metadata except for the OWN_AG 1536 * metadata. IOWs, the in-memory btree knows about the AG headers, the 1537 * two inode btrees, the CoW staging extents, and the refcount btrees. 1538 * For these types of metadata, we need to record the live updates in 1539 * the in-memory rmap btree. 1540 * 1541 * However, we do not scan the free space btrees or the AGFL until we 1542 * have re-locked the AGF and are ready to reserve space for the new 1543 * rmap btree, so we do not want live updates for OWN_AG metadata. 1544 */ 1545 if (XFS_RMAP_NON_INODE_OWNER(oi->oi_owner)) 1546 return oi->oi_owner != XFS_RMAP_OWN_AG; 1547 1548 /* Ignore updates to files that the scanner hasn't visited yet. */ 1549 return xchk_iscan_want_live_update(iscan, oi->oi_owner); 1550} 1551 1552/* 1553 * Apply a rmapbt update from the regular filesystem into our shadow btree. 1554 * We're running from the thread that owns the AGF buffer and is generating 1555 * the update, so we must be careful about which parts of the struct xrep_rmap 1556 * that we change. 1557 */ 1558static int 1559xrep_rmapbt_live_update( 1560 struct notifier_block *nb, 1561 unsigned long action, 1562 void *data) 1563{ 1564 struct xfs_rmap_update_params *p = data; 1565 struct xrep_rmap *rr; 1566 struct xfs_mount *mp; 1567 struct xfs_btree_cur *mcur; 1568 struct xfs_trans *tp; 1569 void *txcookie; 1570 int error; 1571 1572 rr = container_of(nb, struct xrep_rmap, rhook.rmap_hook.nb); 1573 mp = rr->sc->mp; 1574 1575 if (!xrep_rmapbt_want_live_update(&rr->iscan, &p->oinfo)) 1576 goto out_unlock; 1577 1578 trace_xrep_rmap_live_update(mp, rr->sc->sa.pag->pag_agno, action, p); 1579 1580 error = xrep_trans_alloc_hook_dummy(mp, &txcookie, &tp); 1581 if (error) 1582 goto out_abort; 1583 1584 mutex_lock(&rr->lock); 1585 mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, tp, &rr->rmap_btree); 1586 error = __xfs_rmap_finish_intent(mcur, action, p->startblock, 1587 p->blockcount, &p->oinfo, p->unwritten); 1588 xfs_btree_del_cursor(mcur, error); 1589 if (error) 1590 goto out_cancel; 1591 1592 error = xfbtree_trans_commit(&rr->rmap_btree, tp); 1593 if (error) 1594 goto out_cancel; 1595 1596 xrep_trans_cancel_hook_dummy(&txcookie, tp); 1597 mutex_unlock(&rr->lock); 1598 return NOTIFY_DONE; 1599 1600out_cancel: 1601 xfbtree_trans_cancel(&rr->rmap_btree, tp); 1602 xrep_trans_cancel_hook_dummy(&txcookie, tp); 1603out_abort: 1604 mutex_unlock(&rr->lock); 1605 xchk_iscan_abort(&rr->iscan); 1606out_unlock: 1607 return NOTIFY_DONE; 1608} 1609 1610/* Set up the filesystem scan components. */ 1611STATIC int 1612xrep_rmap_setup_scan( 1613 struct xrep_rmap *rr) 1614{ 1615 struct xfs_scrub *sc = rr->sc; 1616 int error; 1617 1618 mutex_init(&rr->lock); 1619 1620 /* Set up in-memory rmap btree */ 1621 error = xfs_rmapbt_mem_init(sc->mp, &rr->rmap_btree, sc->xmbtp, 1622 sc->sa.pag->pag_agno); 1623 if (error) 1624 goto out_mutex; 1625 1626 /* Retry iget every tenth of a second for up to 30 seconds. */ 1627 xchk_iscan_start(sc, 30000, 100, &rr->iscan); 1628 1629 /* 1630 * Hook into live rmap operations so that we can update our in-memory 1631 * btree to reflect live changes on the filesystem. Since we drop the 1632 * AGF buffer to scan all the inodes, we need this piece to avoid 1633 * installing a stale btree. 1634 */ 1635 ASSERT(sc->flags & XCHK_FSGATES_RMAP); 1636 xfs_rmap_hook_setup(&rr->rhook, xrep_rmapbt_live_update); 1637 error = xfs_rmap_hook_add(sc->sa.pag, &rr->rhook); 1638 if (error) 1639 goto out_iscan; 1640 return 0; 1641 1642out_iscan: 1643 xchk_iscan_teardown(&rr->iscan); 1644 xfbtree_destroy(&rr->rmap_btree); 1645out_mutex: 1646 mutex_destroy(&rr->lock); 1647 return error; 1648} 1649 1650/* Tear down scan components. */ 1651STATIC void 1652xrep_rmap_teardown( 1653 struct xrep_rmap *rr) 1654{ 1655 struct xfs_scrub *sc = rr->sc; 1656 1657 xchk_iscan_abort(&rr->iscan); 1658 xfs_rmap_hook_del(sc->sa.pag, &rr->rhook); 1659 xchk_iscan_teardown(&rr->iscan); 1660 xfbtree_destroy(&rr->rmap_btree); 1661 mutex_destroy(&rr->lock); 1662} 1663 1664/* Repair the rmap btree for some AG. */ 1665int 1666xrep_rmapbt( 1667 struct xfs_scrub *sc) 1668{ 1669 struct xrep_rmap *rr = sc->buf; 1670 int error; 1671 1672 error = xrep_rmap_setup_scan(rr); 1673 if (error) 1674 return error; 1675 1676 /* 1677 * Collect rmaps for everything in this AG that isn't space metadata. 1678 * These rmaps won't change even as we try to allocate blocks. 1679 */ 1680 error = xrep_rmap_find_rmaps(rr); 1681 if (error) 1682 goto out_records; 1683 1684 /* Rebuild the rmap information. */ 1685 error = xrep_rmap_build_new_tree(rr); 1686 if (error) 1687 goto out_records; 1688 1689 /* Kill the old tree. */ 1690 error = xrep_rmap_remove_old_tree(rr); 1691 if (error) 1692 goto out_records; 1693 1694out_records: 1695 xrep_rmap_teardown(rr); 1696 return error; 1697} 1698