1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (c) 2020-2024 Oracle. All Rights Reserved. 4 * Author: Darrick J. Wong <djwong@kernel.org> 5 */ 6#include "xfs.h" 7#include "xfs_fs.h" 8#include "xfs_shared.h" 9#include "xfs_format.h" 10#include "xfs_log_format.h" 11#include "xfs_trans_resv.h" 12#include "xfs_mount.h" 13#include "xfs_defer.h" 14#include "xfs_inode.h" 15#include "xfs_trans.h" 16#include "xfs_bmap.h" 17#include "xfs_icache.h" 18#include "xfs_quota.h" 19#include "xfs_exchmaps.h" 20#include "xfs_trace.h" 21#include "xfs_bmap_btree.h" 22#include "xfs_trans_space.h" 23#include "xfs_error.h" 24#include "xfs_errortag.h" 25#include "xfs_health.h" 26#include "xfs_exchmaps_item.h" 27#include "xfs_da_format.h" 28#include "xfs_da_btree.h" 29#include "xfs_attr_leaf.h" 30#include "xfs_attr.h" 31#include "xfs_dir2_priv.h" 32#include "xfs_dir2.h" 33#include "xfs_symlink_remote.h" 34 35struct kmem_cache *xfs_exchmaps_intent_cache; 36 37/* bmbt mappings adjacent to a pair of records. */ 38struct xfs_exchmaps_adjacent { 39 struct xfs_bmbt_irec left1; 40 struct xfs_bmbt_irec right1; 41 struct xfs_bmbt_irec left2; 42 struct xfs_bmbt_irec right2; 43}; 44 45#define ADJACENT_INIT { \ 46 .left1 = { .br_startblock = HOLESTARTBLOCK }, \ 47 .right1 = { .br_startblock = HOLESTARTBLOCK }, \ 48 .left2 = { .br_startblock = HOLESTARTBLOCK }, \ 49 .right2 = { .br_startblock = HOLESTARTBLOCK }, \ 50} 51 52/* Information to reset reflink flag / CoW fork state after an exchange. */ 53 54/* 55 * If the reflink flag is set on either inode, make sure it has an incore CoW 56 * fork, since all reflink inodes must have them. If there's a CoW fork and it 57 * has mappings in it, make sure the inodes are tagged appropriately so that 58 * speculative preallocations can be GC'd if we run low of space. 59 */ 60static inline void 61xfs_exchmaps_ensure_cowfork( 62 struct xfs_inode *ip) 63{ 64 struct xfs_ifork *cfork; 65 66 if (xfs_is_reflink_inode(ip)) 67 xfs_ifork_init_cow(ip); 68 69 cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); 70 if (!cfork) 71 return; 72 if (cfork->if_bytes > 0) 73 xfs_inode_set_cowblocks_tag(ip); 74 else 75 xfs_inode_clear_cowblocks_tag(ip); 76} 77 78/* 79 * Adjust the on-disk inode size upwards if needed so that we never add 80 * mappings into the file past EOF. This is crucial so that log recovery won't 81 * get confused by the sudden appearance of post-eof mappings. 82 */ 83STATIC void 84xfs_exchmaps_update_size( 85 struct xfs_trans *tp, 86 struct xfs_inode *ip, 87 struct xfs_bmbt_irec *imap, 88 xfs_fsize_t new_isize) 89{ 90 struct xfs_mount *mp = tp->t_mountp; 91 xfs_fsize_t len; 92 93 if (new_isize < 0) 94 return; 95 96 len = min(XFS_FSB_TO_B(mp, imap->br_startoff + imap->br_blockcount), 97 new_isize); 98 99 if (len <= ip->i_disk_size) 100 return; 101 102 trace_xfs_exchmaps_update_inode_size(ip, len); 103 104 ip->i_disk_size = len; 105 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 106} 107 108/* Advance the incore state tracking after exchanging a mapping. */ 109static inline void 110xmi_advance( 111 struct xfs_exchmaps_intent *xmi, 112 const struct xfs_bmbt_irec *irec) 113{ 114 xmi->xmi_startoff1 += irec->br_blockcount; 115 xmi->xmi_startoff2 += irec->br_blockcount; 116 xmi->xmi_blockcount -= irec->br_blockcount; 117} 118 119/* Do we still have more mappings to exchange? */ 120static inline bool 121xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) 122{ 123 return xmi->xmi_blockcount > 0; 124} 125 126/* Do we have post-operation cleanups to perform? */ 127static inline bool 128xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) 129{ 130 return xmi->xmi_flags & (XFS_EXCHMAPS_CLEAR_INO1_REFLINK | 131 XFS_EXCHMAPS_CLEAR_INO2_REFLINK | 132 __XFS_EXCHMAPS_INO2_SHORTFORM); 133} 134 135/* Check all mappings to make sure we can actually exchange them. */ 136int 137xfs_exchmaps_check_forks( 138 struct xfs_mount *mp, 139 const struct xfs_exchmaps_req *req) 140{ 141 struct xfs_ifork *ifp1, *ifp2; 142 int whichfork = xfs_exchmaps_reqfork(req); 143 144 /* No fork? */ 145 ifp1 = xfs_ifork_ptr(req->ip1, whichfork); 146 ifp2 = xfs_ifork_ptr(req->ip2, whichfork); 147 if (!ifp1 || !ifp2) 148 return -EINVAL; 149 150 /* We don't know how to exchange local format forks. */ 151 if (ifp1->if_format == XFS_DINODE_FMT_LOCAL || 152 ifp2->if_format == XFS_DINODE_FMT_LOCAL) 153 return -EINVAL; 154 155 return 0; 156} 157 158#ifdef CONFIG_XFS_QUOTA 159/* Log the actual updates to the quota accounting. */ 160static inline void 161xfs_exchmaps_update_quota( 162 struct xfs_trans *tp, 163 struct xfs_exchmaps_intent *xmi, 164 struct xfs_bmbt_irec *irec1, 165 struct xfs_bmbt_irec *irec2) 166{ 167 int64_t ip1_delta = 0, ip2_delta = 0; 168 unsigned int qflag; 169 170 qflag = XFS_IS_REALTIME_INODE(xmi->xmi_ip1) ? XFS_TRANS_DQ_RTBCOUNT : 171 XFS_TRANS_DQ_BCOUNT; 172 173 if (xfs_bmap_is_real_extent(irec1)) { 174 ip1_delta -= irec1->br_blockcount; 175 ip2_delta += irec1->br_blockcount; 176 } 177 178 if (xfs_bmap_is_real_extent(irec2)) { 179 ip1_delta += irec2->br_blockcount; 180 ip2_delta -= irec2->br_blockcount; 181 } 182 183 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip1, qflag, ip1_delta); 184 xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip2, qflag, ip2_delta); 185} 186#else 187# define xfs_exchmaps_update_quota(tp, xmi, irec1, irec2) ((void)0) 188#endif 189 190/* Decide if we want to skip this mapping from file1. */ 191static inline bool 192xfs_exchmaps_can_skip_mapping( 193 struct xfs_exchmaps_intent *xmi, 194 struct xfs_bmbt_irec *irec) 195{ 196 struct xfs_mount *mp = xmi->xmi_ip1->i_mount; 197 198 /* Do not skip this mapping if the caller did not tell us to. */ 199 if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) 200 return false; 201 202 /* Do not skip mapped, written mappings. */ 203 if (xfs_bmap_is_written_extent(irec)) 204 return false; 205 206 /* 207 * The mapping is unwritten or a hole. It cannot be a delalloc 208 * reservation because we already excluded those. It cannot be an 209 * unwritten extent with dirty page cache because we flushed the page 210 * cache. For files where the allocation unit is 1FSB (files on the 211 * data dev, rt files if the extent size is 1FSB), we can safely 212 * skip this mapping. 213 */ 214 if (!xfs_inode_has_bigrtalloc(xmi->xmi_ip1)) 215 return true; 216 217 /* 218 * For a realtime file with a multi-fsb allocation unit, the decision 219 * is trickier because we can only swap full allocation units. 220 * Unwritten mappings can appear in the middle of an rtx if the rtx is 221 * partially written, but they can also appear for preallocations. 222 * 223 * If the mapping is a hole, skip it entirely. Holes should align with 224 * rtx boundaries. 225 */ 226 if (!xfs_bmap_is_real_extent(irec)) 227 return true; 228 229 /* 230 * All mappings below this point are unwritten. 231 * 232 * - If the beginning is not aligned to an rtx, trim the end of the 233 * mapping so that it does not cross an rtx boundary, and swap it. 234 * 235 * - If both ends are aligned to an rtx, skip the entire mapping. 236 */ 237 if (!isaligned_64(irec->br_startoff, mp->m_sb.sb_rextsize)) { 238 xfs_fileoff_t new_end; 239 240 new_end = roundup_64(irec->br_startoff, mp->m_sb.sb_rextsize); 241 irec->br_blockcount = min(irec->br_blockcount, 242 new_end - irec->br_startoff); 243 return false; 244 } 245 if (isaligned_64(irec->br_blockcount, mp->m_sb.sb_rextsize)) 246 return true; 247 248 /* 249 * All mappings below this point are unwritten, start on an rtx 250 * boundary, and do not end on an rtx boundary. 251 * 252 * - If the mapping is longer than one rtx, trim the end of the mapping 253 * down to an rtx boundary and skip it. 254 * 255 * - The mapping is shorter than one rtx. Swap it. 256 */ 257 if (irec->br_blockcount > mp->m_sb.sb_rextsize) { 258 xfs_fileoff_t new_end; 259 260 new_end = rounddown_64(irec->br_startoff + irec->br_blockcount, 261 mp->m_sb.sb_rextsize); 262 irec->br_blockcount = new_end - irec->br_startoff; 263 return true; 264 } 265 266 return false; 267} 268 269/* 270 * Walk forward through the file ranges in @xmi until we find two different 271 * mappings to exchange. If there is work to do, return the mappings; 272 * otherwise we've reached the end of the range and xmi_blockcount will be 273 * zero. 274 * 275 * If the walk skips over a pair of mappings to the same storage, save them as 276 * the left records in @adj (if provided) so that the simulation phase can 277 * avoid an extra lookup. 278 */ 279static int 280xfs_exchmaps_find_mappings( 281 struct xfs_exchmaps_intent *xmi, 282 struct xfs_bmbt_irec *irec1, 283 struct xfs_bmbt_irec *irec2, 284 struct xfs_exchmaps_adjacent *adj) 285{ 286 int nimaps; 287 int bmap_flags; 288 int error; 289 290 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_whichfork(xmi)); 291 292 for (; xmi_has_more_exchange_work(xmi); xmi_advance(xmi, irec1)) { 293 /* Read mapping from the first file */ 294 nimaps = 1; 295 error = xfs_bmapi_read(xmi->xmi_ip1, xmi->xmi_startoff1, 296 xmi->xmi_blockcount, irec1, &nimaps, 297 bmap_flags); 298 if (error) 299 return error; 300 if (nimaps != 1 || 301 irec1->br_startblock == DELAYSTARTBLOCK || 302 irec1->br_startoff != xmi->xmi_startoff1) { 303 /* 304 * We should never get no mapping or a delalloc mapping 305 * or something that doesn't match what we asked for, 306 * since the caller flushed both inodes and we hold the 307 * ILOCKs for both inodes. 308 */ 309 ASSERT(0); 310 return -EINVAL; 311 } 312 313 if (xfs_exchmaps_can_skip_mapping(xmi, irec1)) { 314 trace_xfs_exchmaps_mapping1_skip(xmi->xmi_ip1, irec1); 315 continue; 316 } 317 318 /* Read mapping from the second file */ 319 nimaps = 1; 320 error = xfs_bmapi_read(xmi->xmi_ip2, xmi->xmi_startoff2, 321 irec1->br_blockcount, irec2, &nimaps, 322 bmap_flags); 323 if (error) 324 return error; 325 if (nimaps != 1 || 326 irec2->br_startblock == DELAYSTARTBLOCK || 327 irec2->br_startoff != xmi->xmi_startoff2) { 328 /* 329 * We should never get no mapping or a delalloc mapping 330 * or something that doesn't match what we asked for, 331 * since the caller flushed both inodes and we hold the 332 * ILOCKs for both inodes. 333 */ 334 ASSERT(0); 335 return -EINVAL; 336 } 337 338 /* 339 * We can only exchange as many blocks as the smaller of the 340 * two mapping maps. 341 */ 342 irec1->br_blockcount = min(irec1->br_blockcount, 343 irec2->br_blockcount); 344 345 trace_xfs_exchmaps_mapping1(xmi->xmi_ip1, irec1); 346 trace_xfs_exchmaps_mapping2(xmi->xmi_ip2, irec2); 347 348 /* We found something to exchange, so return it. */ 349 if (irec1->br_startblock != irec2->br_startblock) 350 return 0; 351 352 /* 353 * Two mappings pointing to the same physical block must not 354 * have different states; that's filesystem corruption. Move 355 * on to the next mapping if they're both holes or both point 356 * to the same physical space extent. 357 */ 358 if (irec1->br_state != irec2->br_state) { 359 xfs_bmap_mark_sick(xmi->xmi_ip1, 360 xfs_exchmaps_whichfork(xmi)); 361 xfs_bmap_mark_sick(xmi->xmi_ip2, 362 xfs_exchmaps_whichfork(xmi)); 363 return -EFSCORRUPTED; 364 } 365 366 /* 367 * Save the mappings if we're estimating work and skipping 368 * these identical mappings. 369 */ 370 if (adj) { 371 memcpy(&adj->left1, irec1, sizeof(*irec1)); 372 memcpy(&adj->left2, irec2, sizeof(*irec2)); 373 } 374 } 375 376 return 0; 377} 378 379/* Exchange these two mappings. */ 380static void 381xfs_exchmaps_one_step( 382 struct xfs_trans *tp, 383 struct xfs_exchmaps_intent *xmi, 384 struct xfs_bmbt_irec *irec1, 385 struct xfs_bmbt_irec *irec2) 386{ 387 int whichfork = xfs_exchmaps_whichfork(xmi); 388 389 xfs_exchmaps_update_quota(tp, xmi, irec1, irec2); 390 391 /* Remove both mappings. */ 392 xfs_bmap_unmap_extent(tp, xmi->xmi_ip1, whichfork, irec1); 393 xfs_bmap_unmap_extent(tp, xmi->xmi_ip2, whichfork, irec2); 394 395 /* 396 * Re-add both mappings. We exchange the file offsets between the two 397 * maps and add the opposite map, which has the effect of filling the 398 * logical offsets we just unmapped, but with with the physical mapping 399 * information exchanged. 400 */ 401 swap(irec1->br_startoff, irec2->br_startoff); 402 xfs_bmap_map_extent(tp, xmi->xmi_ip1, whichfork, irec2); 403 xfs_bmap_map_extent(tp, xmi->xmi_ip2, whichfork, irec1); 404 405 /* Make sure we're not adding mappings past EOF. */ 406 if (whichfork == XFS_DATA_FORK) { 407 xfs_exchmaps_update_size(tp, xmi->xmi_ip1, irec2, 408 xmi->xmi_isize1); 409 xfs_exchmaps_update_size(tp, xmi->xmi_ip2, irec1, 410 xmi->xmi_isize2); 411 } 412 413 /* 414 * Advance our cursor and exit. The caller (either defer ops or log 415 * recovery) will log the XMD item, and if *blockcount is nonzero, it 416 * will log a new XMI item for the remainder and call us back. 417 */ 418 xmi_advance(xmi, irec1); 419} 420 421/* Convert inode2's leaf attr fork back to shortform, if possible.. */ 422STATIC int 423xfs_exchmaps_attr_to_sf( 424 struct xfs_trans *tp, 425 struct xfs_exchmaps_intent *xmi) 426{ 427 struct xfs_da_args args = { 428 .dp = xmi->xmi_ip2, 429 .geo = tp->t_mountp->m_attr_geo, 430 .whichfork = XFS_ATTR_FORK, 431 .trans = tp, 432 .owner = xmi->xmi_ip2->i_ino, 433 }; 434 struct xfs_buf *bp; 435 int forkoff; 436 int error; 437 438 if (!xfs_attr_is_leaf(xmi->xmi_ip2)) 439 return 0; 440 441 error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, 0, 442 &bp); 443 if (error) 444 return error; 445 446 forkoff = xfs_attr_shortform_allfit(bp, xmi->xmi_ip2); 447 if (forkoff == 0) 448 return 0; 449 450 return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); 451} 452 453/* Convert inode2's block dir fork back to shortform, if possible.. */ 454STATIC int 455xfs_exchmaps_dir_to_sf( 456 struct xfs_trans *tp, 457 struct xfs_exchmaps_intent *xmi) 458{ 459 struct xfs_da_args args = { 460 .dp = xmi->xmi_ip2, 461 .geo = tp->t_mountp->m_dir_geo, 462 .whichfork = XFS_DATA_FORK, 463 .trans = tp, 464 .owner = xmi->xmi_ip2->i_ino, 465 }; 466 struct xfs_dir2_sf_hdr sfh; 467 struct xfs_buf *bp; 468 int size; 469 int error = 0; 470 471 if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) 472 return error; 473 474 error = xfs_dir3_block_read(tp, xmi->xmi_ip2, xmi->xmi_ip2->i_ino, &bp); 475 if (error) 476 return error; 477 478 size = xfs_dir2_block_sfsize(xmi->xmi_ip2, bp->b_addr, &sfh); 479 if (size > xfs_inode_data_fork_size(xmi->xmi_ip2)) 480 return 0; 481 482 return xfs_dir2_block_to_sf(&args, bp, size, &sfh); 483} 484 485/* Convert inode2's remote symlink target back to shortform, if possible. */ 486STATIC int 487xfs_exchmaps_link_to_sf( 488 struct xfs_trans *tp, 489 struct xfs_exchmaps_intent *xmi) 490{ 491 struct xfs_inode *ip = xmi->xmi_ip2; 492 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); 493 char *buf; 494 int error; 495 496 if (ifp->if_format == XFS_DINODE_FMT_LOCAL || 497 ip->i_disk_size > xfs_inode_data_fork_size(ip)) 498 return 0; 499 500 /* Read the current symlink target into a buffer. */ 501 buf = kmalloc(ip->i_disk_size + 1, 502 GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); 503 if (!buf) { 504 ASSERT(0); 505 return -ENOMEM; 506 } 507 508 error = xfs_symlink_remote_read(ip, buf); 509 if (error) 510 goto free; 511 512 /* Remove the blocks. */ 513 error = xfs_symlink_remote_truncate(tp, ip); 514 if (error) 515 goto free; 516 517 /* Convert fork to local format and log our changes. */ 518 xfs_idestroy_fork(ifp); 519 ifp->if_bytes = 0; 520 ifp->if_format = XFS_DINODE_FMT_LOCAL; 521 xfs_init_local_fork(ip, XFS_DATA_FORK, buf, ip->i_disk_size); 522 xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); 523free: 524 kfree(buf); 525 return error; 526} 527 528/* Clear the reflink flag after an exchange. */ 529static inline void 530xfs_exchmaps_clear_reflink( 531 struct xfs_trans *tp, 532 struct xfs_inode *ip) 533{ 534 trace_xfs_reflink_unset_inode_flag(ip); 535 536 ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; 537 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 538} 539 540/* Finish whatever work might come after an exchange operation. */ 541static int 542xfs_exchmaps_do_postop_work( 543 struct xfs_trans *tp, 544 struct xfs_exchmaps_intent *xmi) 545{ 546 if (xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM) { 547 int error = 0; 548 549 if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) 550 error = xfs_exchmaps_attr_to_sf(tp, xmi); 551 else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) 552 error = xfs_exchmaps_dir_to_sf(tp, xmi); 553 else if (S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 554 error = xfs_exchmaps_link_to_sf(tp, xmi); 555 xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; 556 if (error) 557 return error; 558 } 559 560 if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO1_REFLINK) { 561 xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip1); 562 xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO1_REFLINK; 563 } 564 565 if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO2_REFLINK) { 566 xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip2); 567 xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO2_REFLINK; 568 } 569 570 return 0; 571} 572 573/* Finish one step in a mapping exchange operation, possibly relogging. */ 574int 575xfs_exchmaps_finish_one( 576 struct xfs_trans *tp, 577 struct xfs_exchmaps_intent *xmi) 578{ 579 struct xfs_bmbt_irec irec1, irec2; 580 int error; 581 582 if (xmi_has_more_exchange_work(xmi)) { 583 /* 584 * If the operation state says that some range of the files 585 * have not yet been exchanged, look for mappings in that range 586 * to exchange. If we find some mappings, exchange them. 587 */ 588 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, NULL); 589 if (error) 590 return error; 591 592 if (xmi_has_more_exchange_work(xmi)) 593 xfs_exchmaps_one_step(tp, xmi, &irec1, &irec2); 594 595 /* 596 * If the caller asked us to exchange the file sizes after the 597 * exchange and either we just exchanged the last mappings in 598 * the range or we didn't find anything to exchange, update the 599 * ondisk file sizes. 600 */ 601 if ((xmi->xmi_flags & XFS_EXCHMAPS_SET_SIZES) && 602 !xmi_has_more_exchange_work(xmi)) { 603 xmi->xmi_ip1->i_disk_size = xmi->xmi_isize1; 604 xmi->xmi_ip2->i_disk_size = xmi->xmi_isize2; 605 606 xfs_trans_log_inode(tp, xmi->xmi_ip1, XFS_ILOG_CORE); 607 xfs_trans_log_inode(tp, xmi->xmi_ip2, XFS_ILOG_CORE); 608 } 609 } else if (xmi_has_postop_work(xmi)) { 610 /* 611 * Now that we're finished with the exchange operation, 612 * complete the post-op cleanup work. 613 */ 614 error = xfs_exchmaps_do_postop_work(tp, xmi); 615 if (error) 616 return error; 617 } 618 619 if (XFS_TEST_ERROR(false, tp->t_mountp, XFS_ERRTAG_EXCHMAPS_FINISH_ONE)) 620 return -EIO; 621 622 /* If we still have work to do, ask for a new transaction. */ 623 if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { 624 trace_xfs_exchmaps_defer(tp->t_mountp, xmi); 625 return -EAGAIN; 626 } 627 628 /* 629 * If we reach here, we've finished all the exchange work and the post 630 * operation work. The last thing we need to do before returning to 631 * the caller is to make sure that COW forks are set up correctly. 632 */ 633 if (!(xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK)) { 634 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip1); 635 xfs_exchmaps_ensure_cowfork(xmi->xmi_ip2); 636 } 637 638 return 0; 639} 640 641/* 642 * Compute the amount of bmbt blocks we should reserve for each file. In the 643 * worst case, each exchange will fill a hole with a new mapping, which could 644 * result in a btree split every time we add a new leaf block. 645 */ 646static inline uint64_t 647xfs_exchmaps_bmbt_blocks( 648 struct xfs_mount *mp, 649 const struct xfs_exchmaps_req *req) 650{ 651 return howmany_64(req->nr_exchanges, 652 XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp)) * 653 XFS_EXTENTADD_SPACE_RES(mp, xfs_exchmaps_reqfork(req)); 654} 655 656/* Compute the space we should reserve for the rmap btree expansions. */ 657static inline uint64_t 658xfs_exchmaps_rmapbt_blocks( 659 struct xfs_mount *mp, 660 const struct xfs_exchmaps_req *req) 661{ 662 if (!xfs_has_rmapbt(mp)) 663 return 0; 664 if (XFS_IS_REALTIME_INODE(req->ip1)) 665 return 0; 666 667 return howmany_64(req->nr_exchanges, 668 XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * 669 XFS_RMAPADD_SPACE_RES(mp); 670} 671 672/* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ 673int 674xfs_exchmaps_estimate_overhead( 675 struct xfs_exchmaps_req *req) 676{ 677 struct xfs_mount *mp = req->ip1->i_mount; 678 xfs_filblks_t bmbt_blocks; 679 xfs_filblks_t rmapbt_blocks; 680 xfs_filblks_t resblks = req->resblks; 681 682 /* 683 * Compute the number of bmbt and rmapbt blocks we might need to handle 684 * the estimated number of exchanges. 685 */ 686 bmbt_blocks = xfs_exchmaps_bmbt_blocks(mp, req); 687 rmapbt_blocks = xfs_exchmaps_rmapbt_blocks(mp, req); 688 689 trace_xfs_exchmaps_overhead(mp, bmbt_blocks, rmapbt_blocks); 690 691 /* Make sure the change in file block count doesn't overflow. */ 692 if (check_add_overflow(req->ip1_bcount, bmbt_blocks, &req->ip1_bcount)) 693 return -EFBIG; 694 if (check_add_overflow(req->ip2_bcount, bmbt_blocks, &req->ip2_bcount)) 695 return -EFBIG; 696 697 /* 698 * Add together the number of blocks we need to handle btree growth, 699 * then add it to the number of blocks we need to reserve to this 700 * transaction. 701 */ 702 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 703 return -ENOSPC; 704 if (check_add_overflow(resblks, bmbt_blocks, &resblks)) 705 return -ENOSPC; 706 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 707 return -ENOSPC; 708 if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) 709 return -ENOSPC; 710 711 /* Can't actually reserve more than UINT_MAX blocks. */ 712 if (req->resblks > UINT_MAX) 713 return -ENOSPC; 714 715 req->resblks = resblks; 716 trace_xfs_exchmaps_final_estimate(req); 717 return 0; 718} 719 720/* Decide if we can merge two real mappings. */ 721static inline bool 722xmi_can_merge( 723 const struct xfs_bmbt_irec *b1, 724 const struct xfs_bmbt_irec *b2) 725{ 726 /* Don't merge holes. */ 727 if (b1->br_startblock == HOLESTARTBLOCK || 728 b2->br_startblock == HOLESTARTBLOCK) 729 return false; 730 731 /* We don't merge holes. */ 732 if (!xfs_bmap_is_real_extent(b1) || !xfs_bmap_is_real_extent(b2)) 733 return false; 734 735 if (b1->br_startoff + b1->br_blockcount == b2->br_startoff && 736 b1->br_startblock + b1->br_blockcount == b2->br_startblock && 737 b1->br_state == b2->br_state && 738 b1->br_blockcount + b2->br_blockcount <= XFS_MAX_BMBT_EXTLEN) 739 return true; 740 741 return false; 742} 743 744/* 745 * Decide if we can merge three mappings. Caller must ensure all three 746 * mappings must not be holes or delalloc reservations. 747 */ 748static inline bool 749xmi_can_merge_all( 750 const struct xfs_bmbt_irec *l, 751 const struct xfs_bmbt_irec *m, 752 const struct xfs_bmbt_irec *r) 753{ 754 xfs_filblks_t new_len; 755 756 new_len = l->br_blockcount + m->br_blockcount + r->br_blockcount; 757 return new_len <= XFS_MAX_BMBT_EXTLEN; 758} 759 760#define CLEFT_CONTIG 0x01 761#define CRIGHT_CONTIG 0x02 762#define CHOLE 0x04 763#define CBOTH_CONTIG (CLEFT_CONTIG | CRIGHT_CONTIG) 764 765#define NLEFT_CONTIG 0x10 766#define NRIGHT_CONTIG 0x20 767#define NHOLE 0x40 768#define NBOTH_CONTIG (NLEFT_CONTIG | NRIGHT_CONTIG) 769 770/* Estimate the effect of a single exchange on mapping count. */ 771static inline int 772xmi_delta_nextents_step( 773 struct xfs_mount *mp, 774 const struct xfs_bmbt_irec *left, 775 const struct xfs_bmbt_irec *curr, 776 const struct xfs_bmbt_irec *new, 777 const struct xfs_bmbt_irec *right) 778{ 779 bool lhole, rhole, chole, nhole; 780 unsigned int state = 0; 781 int ret = 0; 782 783 lhole = left->br_startblock == HOLESTARTBLOCK; 784 rhole = right->br_startblock == HOLESTARTBLOCK; 785 chole = curr->br_startblock == HOLESTARTBLOCK; 786 nhole = new->br_startblock == HOLESTARTBLOCK; 787 788 if (chole) 789 state |= CHOLE; 790 if (!lhole && !chole && xmi_can_merge(left, curr)) 791 state |= CLEFT_CONTIG; 792 if (!rhole && !chole && xmi_can_merge(curr, right)) 793 state |= CRIGHT_CONTIG; 794 if ((state & CBOTH_CONTIG) == CBOTH_CONTIG && 795 !xmi_can_merge_all(left, curr, right)) 796 state &= ~CRIGHT_CONTIG; 797 798 if (nhole) 799 state |= NHOLE; 800 if (!lhole && !nhole && xmi_can_merge(left, new)) 801 state |= NLEFT_CONTIG; 802 if (!rhole && !nhole && xmi_can_merge(new, right)) 803 state |= NRIGHT_CONTIG; 804 if ((state & NBOTH_CONTIG) == NBOTH_CONTIG && 805 !xmi_can_merge_all(left, new, right)) 806 state &= ~NRIGHT_CONTIG; 807 808 switch (state & (CLEFT_CONTIG | CRIGHT_CONTIG | CHOLE)) { 809 case CLEFT_CONTIG | CRIGHT_CONTIG: 810 /* 811 * left/curr/right are the same mapping, so deleting curr 812 * causes 2 new mappings to be created. 813 */ 814 ret += 2; 815 break; 816 case 0: 817 /* 818 * curr is not contiguous with any mapping, so we remove curr 819 * completely 820 */ 821 ret--; 822 break; 823 case CHOLE: 824 /* hole, do nothing */ 825 break; 826 case CLEFT_CONTIG: 827 case CRIGHT_CONTIG: 828 /* trim either left or right, no change */ 829 break; 830 } 831 832 switch (state & (NLEFT_CONTIG | NRIGHT_CONTIG | NHOLE)) { 833 case NLEFT_CONTIG | NRIGHT_CONTIG: 834 /* 835 * left/curr/right will become the same mapping, so adding 836 * curr causes the deletion of right. 837 */ 838 ret--; 839 break; 840 case 0: 841 /* new is not contiguous with any mapping */ 842 ret++; 843 break; 844 case NHOLE: 845 /* hole, do nothing. */ 846 break; 847 case NLEFT_CONTIG: 848 case NRIGHT_CONTIG: 849 /* new is absorbed into left or right, no change */ 850 break; 851 } 852 853 trace_xfs_exchmaps_delta_nextents_step(mp, left, curr, new, right, ret, 854 state); 855 return ret; 856} 857 858/* Make sure we don't overflow the extent (mapping) counters. */ 859static inline int 860xmi_ensure_delta_nextents( 861 struct xfs_exchmaps_req *req, 862 struct xfs_inode *ip, 863 int64_t delta) 864{ 865 struct xfs_mount *mp = ip->i_mount; 866 int whichfork = xfs_exchmaps_reqfork(req); 867 struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); 868 uint64_t new_nextents; 869 xfs_extnum_t max_nextents; 870 871 if (delta < 0) 872 return 0; 873 874 /* 875 * It's always an error if the delta causes integer overflow. delta 876 * needs an explicit cast here to avoid warnings about implicit casts 877 * coded into the overflow check. 878 */ 879 if (check_add_overflow(ifp->if_nextents, (uint64_t)delta, 880 &new_nextents)) 881 return -EFBIG; 882 883 if (XFS_TEST_ERROR(false, mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && 884 new_nextents > 10) 885 return -EFBIG; 886 887 /* 888 * We always promote both inodes to have large extent counts if the 889 * superblock feature is enabled, so we only need to check against the 890 * theoretical maximum. 891 */ 892 max_nextents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), 893 whichfork); 894 if (new_nextents > max_nextents) 895 return -EFBIG; 896 897 return 0; 898} 899 900/* Find the next mapping after irec. */ 901static inline int 902xmi_next( 903 struct xfs_inode *ip, 904 int bmap_flags, 905 const struct xfs_bmbt_irec *irec, 906 struct xfs_bmbt_irec *nrec) 907{ 908 xfs_fileoff_t off; 909 xfs_filblks_t blockcount; 910 int nimaps = 1; 911 int error; 912 913 off = irec->br_startoff + irec->br_blockcount; 914 blockcount = XFS_MAX_FILEOFF - off; 915 error = xfs_bmapi_read(ip, off, blockcount, nrec, &nimaps, bmap_flags); 916 if (error) 917 return error; 918 if (nrec->br_startblock == DELAYSTARTBLOCK || 919 nrec->br_startoff != off) { 920 /* 921 * If we don't get the mapping we want, return a zero-length 922 * mapping, which our estimator function will pretend is a hole. 923 * We shouldn't get delalloc reservations. 924 */ 925 nrec->br_startblock = HOLESTARTBLOCK; 926 } 927 928 return 0; 929} 930 931int __init 932xfs_exchmaps_intent_init_cache(void) 933{ 934 xfs_exchmaps_intent_cache = kmem_cache_create("xfs_exchmaps_intent", 935 sizeof(struct xfs_exchmaps_intent), 936 0, 0, NULL); 937 938 return xfs_exchmaps_intent_cache != NULL ? 0 : -ENOMEM; 939} 940 941void 942xfs_exchmaps_intent_destroy_cache(void) 943{ 944 kmem_cache_destroy(xfs_exchmaps_intent_cache); 945 xfs_exchmaps_intent_cache = NULL; 946} 947 948/* 949 * Decide if we will exchange the reflink flags between the two files after the 950 * exchange. The only time we want to do this is if we're exchanging all 951 * mappings under EOF and the inode reflink flags have different states. 952 */ 953static inline bool 954xmi_can_exchange_reflink_flags( 955 const struct xfs_exchmaps_req *req, 956 unsigned int reflink_state) 957{ 958 struct xfs_mount *mp = req->ip1->i_mount; 959 960 if (hweight32(reflink_state) != 1) 961 return false; 962 if (req->startoff1 != 0 || req->startoff2 != 0) 963 return false; 964 if (req->blockcount != XFS_B_TO_FSB(mp, req->ip1->i_disk_size)) 965 return false; 966 if (req->blockcount != XFS_B_TO_FSB(mp, req->ip2->i_disk_size)) 967 return false; 968 return true; 969} 970 971 972/* Allocate and initialize a new incore intent item from a request. */ 973struct xfs_exchmaps_intent * 974xfs_exchmaps_init_intent( 975 const struct xfs_exchmaps_req *req) 976{ 977 struct xfs_exchmaps_intent *xmi; 978 unsigned int rs = 0; 979 980 xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, 981 GFP_NOFS | __GFP_NOFAIL); 982 INIT_LIST_HEAD(&xmi->xmi_list); 983 xmi->xmi_ip1 = req->ip1; 984 xmi->xmi_ip2 = req->ip2; 985 xmi->xmi_startoff1 = req->startoff1; 986 xmi->xmi_startoff2 = req->startoff2; 987 xmi->xmi_blockcount = req->blockcount; 988 xmi->xmi_isize1 = xmi->xmi_isize2 = -1; 989 xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; 990 991 if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) { 992 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 993 return xmi; 994 } 995 996 if (req->flags & XFS_EXCHMAPS_SET_SIZES) { 997 xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; 998 xmi->xmi_isize1 = req->ip2->i_disk_size; 999 xmi->xmi_isize2 = req->ip1->i_disk_size; 1000 } 1001 1002 /* Record the state of each inode's reflink flag before the op. */ 1003 if (xfs_is_reflink_inode(req->ip1)) 1004 rs |= 1; 1005 if (xfs_is_reflink_inode(req->ip2)) 1006 rs |= 2; 1007 1008 /* 1009 * Figure out if we're clearing the reflink flags (which effectively 1010 * exchanges them) after the operation. 1011 */ 1012 if (xmi_can_exchange_reflink_flags(req, rs)) { 1013 if (rs & 1) 1014 xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO1_REFLINK; 1015 if (rs & 2) 1016 xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; 1017 } 1018 1019 if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || 1020 S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) 1021 xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; 1022 1023 return xmi; 1024} 1025 1026/* 1027 * Estimate the number of exchange operations and the number of file blocks 1028 * in each file that will be affected by the exchange operation. 1029 */ 1030int 1031xfs_exchmaps_estimate( 1032 struct xfs_exchmaps_req *req) 1033{ 1034 struct xfs_exchmaps_intent *xmi; 1035 struct xfs_bmbt_irec irec1, irec2; 1036 struct xfs_exchmaps_adjacent adj = ADJACENT_INIT; 1037 xfs_filblks_t ip1_blocks = 0, ip2_blocks = 0; 1038 int64_t d_nexts1, d_nexts2; 1039 int bmap_flags; 1040 int error; 1041 1042 ASSERT(!(req->flags & ~XFS_EXCHMAPS_PARAMS)); 1043 1044 bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_reqfork(req)); 1045 xmi = xfs_exchmaps_init_intent(req); 1046 1047 /* 1048 * To guard against the possibility of overflowing the extent counters, 1049 * we have to estimate an upper bound on the potential increase in that 1050 * counter. We can split the mapping at each end of the range, and for 1051 * each step of the exchange we can split the mapping that we're 1052 * working on if the mappings do not align. 1053 */ 1054 d_nexts1 = d_nexts2 = 3; 1055 1056 while (xmi_has_more_exchange_work(xmi)) { 1057 /* 1058 * Walk through the file ranges until we find something to 1059 * exchange. Because we're simulating the exchange, pass in 1060 * adj to capture skipped mappings for correct estimation of 1061 * bmbt record merges. 1062 */ 1063 error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, &adj); 1064 if (error) 1065 goto out_free; 1066 if (!xmi_has_more_exchange_work(xmi)) 1067 break; 1068 1069 /* Update accounting. */ 1070 if (xfs_bmap_is_real_extent(&irec1)) 1071 ip1_blocks += irec1.br_blockcount; 1072 if (xfs_bmap_is_real_extent(&irec2)) 1073 ip2_blocks += irec2.br_blockcount; 1074 req->nr_exchanges++; 1075 1076 /* Read the next mappings from both files. */ 1077 error = xmi_next(req->ip1, bmap_flags, &irec1, &adj.right1); 1078 if (error) 1079 goto out_free; 1080 1081 error = xmi_next(req->ip2, bmap_flags, &irec2, &adj.right2); 1082 if (error) 1083 goto out_free; 1084 1085 /* Update extent count deltas. */ 1086 d_nexts1 += xmi_delta_nextents_step(req->ip1->i_mount, 1087 &adj.left1, &irec1, &irec2, &adj.right1); 1088 1089 d_nexts2 += xmi_delta_nextents_step(req->ip1->i_mount, 1090 &adj.left2, &irec2, &irec1, &adj.right2); 1091 1092 /* Now pretend we exchanged the mappings. */ 1093 if (xmi_can_merge(&adj.left2, &irec1)) 1094 adj.left2.br_blockcount += irec1.br_blockcount; 1095 else 1096 memcpy(&adj.left2, &irec1, sizeof(irec1)); 1097 1098 if (xmi_can_merge(&adj.left1, &irec2)) 1099 adj.left1.br_blockcount += irec2.br_blockcount; 1100 else 1101 memcpy(&adj.left1, &irec2, sizeof(irec2)); 1102 1103 xmi_advance(xmi, &irec1); 1104 } 1105 1106 /* Account for the blocks that are being exchanged. */ 1107 if (XFS_IS_REALTIME_INODE(req->ip1) && 1108 xfs_exchmaps_reqfork(req) == XFS_DATA_FORK) { 1109 req->ip1_rtbcount = ip1_blocks; 1110 req->ip2_rtbcount = ip2_blocks; 1111 } else { 1112 req->ip1_bcount = ip1_blocks; 1113 req->ip2_bcount = ip2_blocks; 1114 } 1115 1116 /* 1117 * Make sure that both forks have enough slack left in their extent 1118 * counters that the exchange operation will not overflow. 1119 */ 1120 trace_xfs_exchmaps_delta_nextents(req, d_nexts1, d_nexts2); 1121 if (req->ip1 == req->ip2) { 1122 error = xmi_ensure_delta_nextents(req, req->ip1, 1123 d_nexts1 + d_nexts2); 1124 } else { 1125 error = xmi_ensure_delta_nextents(req, req->ip1, d_nexts1); 1126 if (error) 1127 goto out_free; 1128 error = xmi_ensure_delta_nextents(req, req->ip2, d_nexts2); 1129 } 1130 if (error) 1131 goto out_free; 1132 1133 trace_xfs_exchmaps_initial_estimate(req); 1134 error = xfs_exchmaps_estimate_overhead(req); 1135out_free: 1136 kmem_cache_free(xfs_exchmaps_intent_cache, xmi); 1137 return error; 1138} 1139 1140/* Set the reflink flag before an operation. */ 1141static inline void 1142xfs_exchmaps_set_reflink( 1143 struct xfs_trans *tp, 1144 struct xfs_inode *ip) 1145{ 1146 trace_xfs_reflink_set_inode_flag(ip); 1147 1148 ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; 1149 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1150} 1151 1152/* 1153 * If either file has shared blocks and we're exchanging data forks, we must 1154 * flag the other file as having shared blocks so that we get the shared-block 1155 * rmap functions if we need to fix up the rmaps. 1156 */ 1157void 1158xfs_exchmaps_ensure_reflink( 1159 struct xfs_trans *tp, 1160 const struct xfs_exchmaps_intent *xmi) 1161{ 1162 unsigned int rs = 0; 1163 1164 if (xfs_is_reflink_inode(xmi->xmi_ip1)) 1165 rs |= 1; 1166 if (xfs_is_reflink_inode(xmi->xmi_ip2)) 1167 rs |= 2; 1168 1169 if ((rs & 1) && !xfs_is_reflink_inode(xmi->xmi_ip2)) 1170 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip2); 1171 1172 if ((rs & 2) && !xfs_is_reflink_inode(xmi->xmi_ip1)) 1173 xfs_exchmaps_set_reflink(tp, xmi->xmi_ip1); 1174} 1175 1176/* Set the large extent count flag before an operation if needed. */ 1177static inline void 1178xfs_exchmaps_ensure_large_extent_counts( 1179 struct xfs_trans *tp, 1180 struct xfs_inode *ip) 1181{ 1182 if (xfs_inode_has_large_extent_counts(ip)) 1183 return; 1184 1185 ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; 1186 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); 1187} 1188 1189/* Widen the extent counter fields of both inodes if necessary. */ 1190void 1191xfs_exchmaps_upgrade_extent_counts( 1192 struct xfs_trans *tp, 1193 const struct xfs_exchmaps_intent *xmi) 1194{ 1195 if (!xfs_has_large_extent_counts(tp->t_mountp)) 1196 return; 1197 1198 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip1); 1199 xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip2); 1200} 1201 1202/* 1203 * Schedule an exchange a range of mappings from one inode to another. 1204 * 1205 * The use of file mapping exchange log intent items ensures the operation can 1206 * be resumed even if the system goes down. The caller must commit the 1207 * transaction to start the work. 1208 * 1209 * The caller must ensure the inodes must be joined to the transaction and 1210 * ILOCKd; they will still be joined to the transaction at exit. 1211 */ 1212void 1213xfs_exchange_mappings( 1214 struct xfs_trans *tp, 1215 const struct xfs_exchmaps_req *req) 1216{ 1217 struct xfs_exchmaps_intent *xmi; 1218 1219 BUILD_BUG_ON(XFS_EXCHMAPS_INTERNAL_FLAGS & XFS_EXCHMAPS_LOGGED_FLAGS); 1220 1221 xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); 1222 xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); 1223 ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); 1224 if (req->flags & XFS_EXCHMAPS_SET_SIZES) 1225 ASSERT(!(req->flags & XFS_EXCHMAPS_ATTR_FORK)); 1226 ASSERT(xfs_has_exchange_range(tp->t_mountp)); 1227 1228 if (req->blockcount == 0) 1229 return; 1230 1231 xmi = xfs_exchmaps_init_intent(req); 1232 xfs_exchmaps_defer_add(tp, xmi); 1233 xfs_exchmaps_ensure_reflink(tp, xmi); 1234 xfs_exchmaps_upgrade_extent_counts(tp, xmi); 1235} 1236