1// SPDX-License-Identifier: LGPL-2.1 2/* 3 * Copyright (c) 2012 Taobao. 4 * Written by Tao Ma <boyu.mt@taobao.com> 5 */ 6 7#include <linux/iomap.h> 8#include <linux/fiemap.h> 9#include <linux/namei.h> 10#include <linux/iversion.h> 11#include <linux/sched/mm.h> 12 13#include "ext4_jbd2.h" 14#include "ext4.h" 15#include "xattr.h" 16#include "truncate.h" 17 18#define EXT4_XATTR_SYSTEM_DATA "data" 19#define EXT4_MIN_INLINE_DATA_SIZE ((sizeof(__le32) * EXT4_N_BLOCKS)) 20#define EXT4_INLINE_DOTDOT_OFFSET 2 21#define EXT4_INLINE_DOTDOT_SIZE 4 22 23static int ext4_get_inline_size(struct inode *inode) 24{ 25 if (EXT4_I(inode)->i_inline_off) 26 return EXT4_I(inode)->i_inline_size; 27 28 return 0; 29} 30 31static int get_max_inline_xattr_value_size(struct inode *inode, 32 struct ext4_iloc *iloc) 33{ 34 struct ext4_xattr_ibody_header *header; 35 struct ext4_xattr_entry *entry; 36 struct ext4_inode *raw_inode; 37 void *end; 38 int free, min_offs; 39 40 if (!EXT4_INODE_HAS_XATTR_SPACE(inode)) 41 return 0; 42 43 min_offs = EXT4_SB(inode->i_sb)->s_inode_size - 44 EXT4_GOOD_OLD_INODE_SIZE - 45 EXT4_I(inode)->i_extra_isize - 46 sizeof(struct ext4_xattr_ibody_header); 47 48 /* 49 * We need to subtract another sizeof(__u32) since an in-inode xattr 50 * needs an empty 4 bytes to indicate the gap between the xattr entry 51 * and the name/value pair. 52 */ 53 if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR)) 54 return EXT4_XATTR_SIZE(min_offs - 55 EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA)) - 56 EXT4_XATTR_ROUND - sizeof(__u32)); 57 58 raw_inode = ext4_raw_inode(iloc); 59 header = IHDR(inode, raw_inode); 60 entry = IFIRST(header); 61 end = (void *)raw_inode + EXT4_SB(inode->i_sb)->s_inode_size; 62 63 /* Compute min_offs. */ 64 while (!IS_LAST_ENTRY(entry)) { 65 void *next = EXT4_XATTR_NEXT(entry); 66 67 if (next >= end) { 68 EXT4_ERROR_INODE(inode, 69 "corrupt xattr in inline inode"); 70 return 0; 71 } 72 if (!entry->e_value_inum && entry->e_value_size) { 73 size_t offs = le16_to_cpu(entry->e_value_offs); 74 if (offs < min_offs) 75 min_offs = offs; 76 } 77 entry = next; 78 } 79 free = min_offs - 80 ((void *)entry - (void *)IFIRST(header)) - sizeof(__u32); 81 82 if (EXT4_I(inode)->i_inline_off) { 83 entry = (struct ext4_xattr_entry *) 84 ((void *)raw_inode + EXT4_I(inode)->i_inline_off); 85 86 free += EXT4_XATTR_SIZE(le32_to_cpu(entry->e_value_size)); 87 goto out; 88 } 89 90 free -= EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA)); 91 92 if (free > EXT4_XATTR_ROUND) 93 free = EXT4_XATTR_SIZE(free - EXT4_XATTR_ROUND); 94 else 95 free = 0; 96 97out: 98 return free; 99} 100 101/* 102 * Get the maximum size we now can store in an inode. 103 * If we can't find the space for a xattr entry, don't use the space 104 * of the extents since we have no space to indicate the inline data. 105 */ 106int ext4_get_max_inline_size(struct inode *inode) 107{ 108 int error, max_inline_size; 109 struct ext4_iloc iloc; 110 111 if (EXT4_I(inode)->i_extra_isize == 0) 112 return 0; 113 114 error = ext4_get_inode_loc(inode, &iloc); 115 if (error) { 116 ext4_error_inode_err(inode, __func__, __LINE__, 0, -error, 117 "can't get inode location %lu", 118 inode->i_ino); 119 return 0; 120 } 121 122 down_read(&EXT4_I(inode)->xattr_sem); 123 max_inline_size = get_max_inline_xattr_value_size(inode, &iloc); 124 up_read(&EXT4_I(inode)->xattr_sem); 125 126 brelse(iloc.bh); 127 128 if (!max_inline_size) 129 return 0; 130 131 return max_inline_size + EXT4_MIN_INLINE_DATA_SIZE; 132} 133 134/* 135 * this function does not take xattr_sem, which is OK because it is 136 * currently only used in a code path coming form ext4_iget, before 137 * the new inode has been unlocked 138 */ 139int ext4_find_inline_data_nolock(struct inode *inode) 140{ 141 struct ext4_xattr_ibody_find is = { 142 .s = { .not_found = -ENODATA, }, 143 }; 144 struct ext4_xattr_info i = { 145 .name_index = EXT4_XATTR_INDEX_SYSTEM, 146 .name = EXT4_XATTR_SYSTEM_DATA, 147 }; 148 int error; 149 150 if (EXT4_I(inode)->i_extra_isize == 0) 151 return 0; 152 153 error = ext4_get_inode_loc(inode, &is.iloc); 154 if (error) 155 return error; 156 157 error = ext4_xattr_ibody_find(inode, &i, &is); 158 if (error) 159 goto out; 160 161 if (!is.s.not_found) { 162 if (is.s.here->e_value_inum) { 163 EXT4_ERROR_INODE(inode, "inline data xattr refers " 164 "to an external xattr inode"); 165 error = -EFSCORRUPTED; 166 goto out; 167 } 168 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - 169 (void *)ext4_raw_inode(&is.iloc)); 170 EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE + 171 le32_to_cpu(is.s.here->e_value_size); 172 } 173out: 174 brelse(is.iloc.bh); 175 return error; 176} 177 178static int ext4_read_inline_data(struct inode *inode, void *buffer, 179 unsigned int len, 180 struct ext4_iloc *iloc) 181{ 182 struct ext4_xattr_entry *entry; 183 struct ext4_xattr_ibody_header *header; 184 int cp_len = 0; 185 struct ext4_inode *raw_inode; 186 187 if (!len) 188 return 0; 189 190 BUG_ON(len > EXT4_I(inode)->i_inline_size); 191 192 cp_len = min_t(unsigned int, len, EXT4_MIN_INLINE_DATA_SIZE); 193 194 raw_inode = ext4_raw_inode(iloc); 195 memcpy(buffer, (void *)(raw_inode->i_block), cp_len); 196 197 len -= cp_len; 198 buffer += cp_len; 199 200 if (!len) 201 goto out; 202 203 header = IHDR(inode, raw_inode); 204 entry = (struct ext4_xattr_entry *)((void *)raw_inode + 205 EXT4_I(inode)->i_inline_off); 206 len = min_t(unsigned int, len, 207 (unsigned int)le32_to_cpu(entry->e_value_size)); 208 209 memcpy(buffer, 210 (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs), len); 211 cp_len += len; 212 213out: 214 return cp_len; 215} 216 217/* 218 * write the buffer to the inline inode. 219 * If 'create' is set, we don't need to do the extra copy in the xattr 220 * value since it is already handled by ext4_xattr_ibody_set. 221 * That saves us one memcpy. 222 */ 223static void ext4_write_inline_data(struct inode *inode, struct ext4_iloc *iloc, 224 void *buffer, loff_t pos, unsigned int len) 225{ 226 struct ext4_xattr_entry *entry; 227 struct ext4_xattr_ibody_header *header; 228 struct ext4_inode *raw_inode; 229 int cp_len = 0; 230 231 if (unlikely(ext4_forced_shutdown(inode->i_sb))) 232 return; 233 234 BUG_ON(!EXT4_I(inode)->i_inline_off); 235 BUG_ON(pos + len > EXT4_I(inode)->i_inline_size); 236 237 raw_inode = ext4_raw_inode(iloc); 238 buffer += pos; 239 240 if (pos < EXT4_MIN_INLINE_DATA_SIZE) { 241 cp_len = pos + len > EXT4_MIN_INLINE_DATA_SIZE ? 242 EXT4_MIN_INLINE_DATA_SIZE - pos : len; 243 memcpy((void *)raw_inode->i_block + pos, buffer, cp_len); 244 245 len -= cp_len; 246 buffer += cp_len; 247 pos += cp_len; 248 } 249 250 if (!len) 251 return; 252 253 pos -= EXT4_MIN_INLINE_DATA_SIZE; 254 header = IHDR(inode, raw_inode); 255 entry = (struct ext4_xattr_entry *)((void *)raw_inode + 256 EXT4_I(inode)->i_inline_off); 257 258 memcpy((void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs) + pos, 259 buffer, len); 260} 261 262static int ext4_create_inline_data(handle_t *handle, 263 struct inode *inode, unsigned len) 264{ 265 int error; 266 void *value = NULL; 267 struct ext4_xattr_ibody_find is = { 268 .s = { .not_found = -ENODATA, }, 269 }; 270 struct ext4_xattr_info i = { 271 .name_index = EXT4_XATTR_INDEX_SYSTEM, 272 .name = EXT4_XATTR_SYSTEM_DATA, 273 }; 274 275 error = ext4_get_inode_loc(inode, &is.iloc); 276 if (error) 277 return error; 278 279 BUFFER_TRACE(is.iloc.bh, "get_write_access"); 280 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, 281 EXT4_JTR_NONE); 282 if (error) 283 goto out; 284 285 if (len > EXT4_MIN_INLINE_DATA_SIZE) { 286 value = EXT4_ZERO_XATTR_VALUE; 287 len -= EXT4_MIN_INLINE_DATA_SIZE; 288 } else { 289 value = ""; 290 len = 0; 291 } 292 293 /* Insert the xttr entry. */ 294 i.value = value; 295 i.value_len = len; 296 297 error = ext4_xattr_ibody_find(inode, &i, &is); 298 if (error) 299 goto out; 300 301 BUG_ON(!is.s.not_found); 302 303 error = ext4_xattr_ibody_set(handle, inode, &i, &is); 304 if (error) { 305 if (error == -ENOSPC) 306 ext4_clear_inode_state(inode, 307 EXT4_STATE_MAY_INLINE_DATA); 308 goto out; 309 } 310 311 memset((void *)ext4_raw_inode(&is.iloc)->i_block, 312 0, EXT4_MIN_INLINE_DATA_SIZE); 313 314 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - 315 (void *)ext4_raw_inode(&is.iloc)); 316 EXT4_I(inode)->i_inline_size = len + EXT4_MIN_INLINE_DATA_SIZE; 317 ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); 318 ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA); 319 get_bh(is.iloc.bh); 320 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); 321 322out: 323 brelse(is.iloc.bh); 324 return error; 325} 326 327static int ext4_update_inline_data(handle_t *handle, struct inode *inode, 328 unsigned int len) 329{ 330 int error; 331 void *value = NULL; 332 struct ext4_xattr_ibody_find is = { 333 .s = { .not_found = -ENODATA, }, 334 }; 335 struct ext4_xattr_info i = { 336 .name_index = EXT4_XATTR_INDEX_SYSTEM, 337 .name = EXT4_XATTR_SYSTEM_DATA, 338 }; 339 340 /* If the old space is ok, write the data directly. */ 341 if (len <= EXT4_I(inode)->i_inline_size) 342 return 0; 343 344 error = ext4_get_inode_loc(inode, &is.iloc); 345 if (error) 346 return error; 347 348 error = ext4_xattr_ibody_find(inode, &i, &is); 349 if (error) 350 goto out; 351 352 BUG_ON(is.s.not_found); 353 354 len -= EXT4_MIN_INLINE_DATA_SIZE; 355 value = kzalloc(len, GFP_NOFS); 356 if (!value) { 357 error = -ENOMEM; 358 goto out; 359 } 360 361 error = ext4_xattr_ibody_get(inode, i.name_index, i.name, 362 value, len); 363 if (error < 0) 364 goto out; 365 366 BUFFER_TRACE(is.iloc.bh, "get_write_access"); 367 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, 368 EXT4_JTR_NONE); 369 if (error) 370 goto out; 371 372 /* Update the xattr entry. */ 373 i.value = value; 374 i.value_len = len; 375 376 error = ext4_xattr_ibody_set(handle, inode, &i, &is); 377 if (error) 378 goto out; 379 380 EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - 381 (void *)ext4_raw_inode(&is.iloc)); 382 EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE + 383 le32_to_cpu(is.s.here->e_value_size); 384 ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 385 get_bh(is.iloc.bh); 386 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); 387 388out: 389 kfree(value); 390 brelse(is.iloc.bh); 391 return error; 392} 393 394static int ext4_prepare_inline_data(handle_t *handle, struct inode *inode, 395 unsigned int len) 396{ 397 int ret, size, no_expand; 398 struct ext4_inode_info *ei = EXT4_I(inode); 399 400 if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) 401 return -ENOSPC; 402 403 size = ext4_get_max_inline_size(inode); 404 if (size < len) 405 return -ENOSPC; 406 407 ext4_write_lock_xattr(inode, &no_expand); 408 409 if (ei->i_inline_off) 410 ret = ext4_update_inline_data(handle, inode, len); 411 else 412 ret = ext4_create_inline_data(handle, inode, len); 413 414 ext4_write_unlock_xattr(inode, &no_expand); 415 return ret; 416} 417 418static int ext4_destroy_inline_data_nolock(handle_t *handle, 419 struct inode *inode) 420{ 421 struct ext4_inode_info *ei = EXT4_I(inode); 422 struct ext4_xattr_ibody_find is = { 423 .s = { .not_found = 0, }, 424 }; 425 struct ext4_xattr_info i = { 426 .name_index = EXT4_XATTR_INDEX_SYSTEM, 427 .name = EXT4_XATTR_SYSTEM_DATA, 428 .value = NULL, 429 .value_len = 0, 430 }; 431 int error; 432 433 if (!ei->i_inline_off) 434 return 0; 435 436 error = ext4_get_inode_loc(inode, &is.iloc); 437 if (error) 438 return error; 439 440 error = ext4_xattr_ibody_find(inode, &i, &is); 441 if (error) 442 goto out; 443 444 BUFFER_TRACE(is.iloc.bh, "get_write_access"); 445 error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, 446 EXT4_JTR_NONE); 447 if (error) 448 goto out; 449 450 error = ext4_xattr_ibody_set(handle, inode, &i, &is); 451 if (error) 452 goto out; 453 454 memset((void *)ext4_raw_inode(&is.iloc)->i_block, 455 0, EXT4_MIN_INLINE_DATA_SIZE); 456 memset(ei->i_data, 0, EXT4_MIN_INLINE_DATA_SIZE); 457 458 if (ext4_has_feature_extents(inode->i_sb)) { 459 if (S_ISDIR(inode->i_mode) || 460 S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) { 461 ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); 462 ext4_ext_tree_init(handle, inode); 463 } 464 } 465 ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA); 466 467 get_bh(is.iloc.bh); 468 error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); 469 470 EXT4_I(inode)->i_inline_off = 0; 471 EXT4_I(inode)->i_inline_size = 0; 472 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 473out: 474 brelse(is.iloc.bh); 475 if (error == -ENODATA) 476 error = 0; 477 return error; 478} 479 480static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) 481{ 482 void *kaddr; 483 int ret = 0; 484 size_t len; 485 struct ext4_iloc iloc; 486 487 BUG_ON(!folio_test_locked(folio)); 488 BUG_ON(!ext4_has_inline_data(inode)); 489 BUG_ON(folio->index); 490 491 if (!EXT4_I(inode)->i_inline_off) { 492 ext4_warning(inode->i_sb, "inode %lu doesn't have inline data.", 493 inode->i_ino); 494 goto out; 495 } 496 497 ret = ext4_get_inode_loc(inode, &iloc); 498 if (ret) 499 goto out; 500 501 len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); 502 BUG_ON(len > PAGE_SIZE); 503 kaddr = kmap_local_folio(folio, 0); 504 ret = ext4_read_inline_data(inode, kaddr, len, &iloc); 505 kaddr = folio_zero_tail(folio, len, kaddr + len); 506 kunmap_local(kaddr); 507 folio_mark_uptodate(folio); 508 brelse(iloc.bh); 509 510out: 511 return ret; 512} 513 514int ext4_readpage_inline(struct inode *inode, struct folio *folio) 515{ 516 int ret = 0; 517 518 down_read(&EXT4_I(inode)->xattr_sem); 519 if (!ext4_has_inline_data(inode)) { 520 up_read(&EXT4_I(inode)->xattr_sem); 521 return -EAGAIN; 522 } 523 524 /* 525 * Current inline data can only exist in the 1st page, 526 * So for all the other pages, just set them uptodate. 527 */ 528 if (!folio->index) 529 ret = ext4_read_inline_folio(inode, folio); 530 else if (!folio_test_uptodate(folio)) { 531 folio_zero_segment(folio, 0, folio_size(folio)); 532 folio_mark_uptodate(folio); 533 } 534 535 up_read(&EXT4_I(inode)->xattr_sem); 536 537 folio_unlock(folio); 538 return ret >= 0 ? 0 : ret; 539} 540 541static int ext4_convert_inline_data_to_extent(struct address_space *mapping, 542 struct inode *inode) 543{ 544 int ret, needed_blocks, no_expand; 545 handle_t *handle = NULL; 546 int retries = 0, sem_held = 0; 547 struct folio *folio = NULL; 548 unsigned from, to; 549 struct ext4_iloc iloc; 550 551 if (!ext4_has_inline_data(inode)) { 552 /* 553 * clear the flag so that no new write 554 * will trap here again. 555 */ 556 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 557 return 0; 558 } 559 560 needed_blocks = ext4_writepage_trans_blocks(inode); 561 562 ret = ext4_get_inode_loc(inode, &iloc); 563 if (ret) 564 return ret; 565 566retry: 567 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); 568 if (IS_ERR(handle)) { 569 ret = PTR_ERR(handle); 570 handle = NULL; 571 goto out; 572 } 573 574 /* We cannot recurse into the filesystem as the transaction is already 575 * started */ 576 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, 577 mapping_gfp_mask(mapping)); 578 if (IS_ERR(folio)) { 579 ret = PTR_ERR(folio); 580 goto out_nofolio; 581 } 582 583 ext4_write_lock_xattr(inode, &no_expand); 584 sem_held = 1; 585 /* If some one has already done this for us, just exit. */ 586 if (!ext4_has_inline_data(inode)) { 587 ret = 0; 588 goto out; 589 } 590 591 from = 0; 592 to = ext4_get_inline_size(inode); 593 if (!folio_test_uptodate(folio)) { 594 ret = ext4_read_inline_folio(inode, folio); 595 if (ret < 0) 596 goto out; 597 } 598 599 ret = ext4_destroy_inline_data_nolock(handle, inode); 600 if (ret) 601 goto out; 602 603 if (ext4_should_dioread_nolock(inode)) { 604 ret = __block_write_begin(&folio->page, from, to, 605 ext4_get_block_unwritten); 606 } else 607 ret = __block_write_begin(&folio->page, from, to, ext4_get_block); 608 609 if (!ret && ext4_should_journal_data(inode)) { 610 ret = ext4_walk_page_buffers(handle, inode, 611 folio_buffers(folio), from, to, 612 NULL, do_journal_get_write_access); 613 } 614 615 if (ret) { 616 folio_unlock(folio); 617 folio_put(folio); 618 folio = NULL; 619 ext4_orphan_add(handle, inode); 620 ext4_write_unlock_xattr(inode, &no_expand); 621 sem_held = 0; 622 ext4_journal_stop(handle); 623 handle = NULL; 624 ext4_truncate_failed_write(inode); 625 /* 626 * If truncate failed early the inode might 627 * still be on the orphan list; we need to 628 * make sure the inode is removed from the 629 * orphan list in that case. 630 */ 631 if (inode->i_nlink) 632 ext4_orphan_del(NULL, inode); 633 } 634 635 if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) 636 goto retry; 637 638 if (folio) 639 block_commit_write(&folio->page, from, to); 640out: 641 if (folio) { 642 folio_unlock(folio); 643 folio_put(folio); 644 } 645out_nofolio: 646 if (sem_held) 647 ext4_write_unlock_xattr(inode, &no_expand); 648 if (handle) 649 ext4_journal_stop(handle); 650 brelse(iloc.bh); 651 return ret; 652} 653 654/* 655 * Try to write data in the inode. 656 * If the inode has inline data, check whether the new write can be 657 * in the inode also. If not, create the page the handle, move the data 658 * to the page make it update and let the later codes create extent for it. 659 */ 660int ext4_try_to_write_inline_data(struct address_space *mapping, 661 struct inode *inode, 662 loff_t pos, unsigned len, 663 struct page **pagep) 664{ 665 int ret; 666 handle_t *handle; 667 struct folio *folio; 668 struct ext4_iloc iloc; 669 670 if (pos + len > ext4_get_max_inline_size(inode)) 671 goto convert; 672 673 ret = ext4_get_inode_loc(inode, &iloc); 674 if (ret) 675 return ret; 676 677 /* 678 * The possible write could happen in the inode, 679 * so try to reserve the space in inode first. 680 */ 681 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 682 if (IS_ERR(handle)) { 683 ret = PTR_ERR(handle); 684 handle = NULL; 685 goto out; 686 } 687 688 ret = ext4_prepare_inline_data(handle, inode, pos + len); 689 if (ret && ret != -ENOSPC) 690 goto out; 691 692 /* We don't have space in inline inode, so convert it to extent. */ 693 if (ret == -ENOSPC) { 694 ext4_journal_stop(handle); 695 brelse(iloc.bh); 696 goto convert; 697 } 698 699 ret = ext4_journal_get_write_access(handle, inode->i_sb, iloc.bh, 700 EXT4_JTR_NONE); 701 if (ret) 702 goto out; 703 704 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, 705 mapping_gfp_mask(mapping)); 706 if (IS_ERR(folio)) { 707 ret = PTR_ERR(folio); 708 goto out; 709 } 710 711 *pagep = &folio->page; 712 down_read(&EXT4_I(inode)->xattr_sem); 713 if (!ext4_has_inline_data(inode)) { 714 ret = 0; 715 folio_unlock(folio); 716 folio_put(folio); 717 goto out_up_read; 718 } 719 720 if (!folio_test_uptodate(folio)) { 721 ret = ext4_read_inline_folio(inode, folio); 722 if (ret < 0) { 723 folio_unlock(folio); 724 folio_put(folio); 725 goto out_up_read; 726 } 727 } 728 729 ret = 1; 730 handle = NULL; 731out_up_read: 732 up_read(&EXT4_I(inode)->xattr_sem); 733out: 734 if (handle && (ret != 1)) 735 ext4_journal_stop(handle); 736 brelse(iloc.bh); 737 return ret; 738convert: 739 return ext4_convert_inline_data_to_extent(mapping, inode); 740} 741 742int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, 743 unsigned copied, struct folio *folio) 744{ 745 handle_t *handle = ext4_journal_current_handle(); 746 int no_expand; 747 void *kaddr; 748 struct ext4_iloc iloc; 749 int ret = 0, ret2; 750 751 if (unlikely(copied < len) && !folio_test_uptodate(folio)) 752 copied = 0; 753 754 if (likely(copied)) { 755 ret = ext4_get_inode_loc(inode, &iloc); 756 if (ret) { 757 folio_unlock(folio); 758 folio_put(folio); 759 ext4_std_error(inode->i_sb, ret); 760 goto out; 761 } 762 ext4_write_lock_xattr(inode, &no_expand); 763 BUG_ON(!ext4_has_inline_data(inode)); 764 765 /* 766 * ei->i_inline_off may have changed since 767 * ext4_write_begin() called 768 * ext4_try_to_write_inline_data() 769 */ 770 (void) ext4_find_inline_data_nolock(inode); 771 772 kaddr = kmap_local_folio(folio, 0); 773 ext4_write_inline_data(inode, &iloc, kaddr, pos, copied); 774 kunmap_local(kaddr); 775 folio_mark_uptodate(folio); 776 /* clear dirty flag so that writepages wouldn't work for us. */ 777 folio_clear_dirty(folio); 778 779 ext4_write_unlock_xattr(inode, &no_expand); 780 brelse(iloc.bh); 781 782 /* 783 * It's important to update i_size while still holding folio 784 * lock: page writeout could otherwise come in and zero 785 * beyond i_size. 786 */ 787 ext4_update_inode_size(inode, pos + copied); 788 } 789 folio_unlock(folio); 790 folio_put(folio); 791 792 /* 793 * Don't mark the inode dirty under folio lock. First, it unnecessarily 794 * makes the holding time of folio lock longer. Second, it forces lock 795 * ordering of folio lock and transaction start for journaling 796 * filesystems. 797 */ 798 if (likely(copied)) 799 mark_inode_dirty(inode); 800out: 801 /* 802 * If we didn't copy as much data as expected, we need to trim back 803 * size of xattr containing inline data. 804 */ 805 if (pos + len > inode->i_size && ext4_can_truncate(inode)) 806 ext4_orphan_add(handle, inode); 807 808 ret2 = ext4_journal_stop(handle); 809 if (!ret) 810 ret = ret2; 811 if (pos + len > inode->i_size) { 812 ext4_truncate_failed_write(inode); 813 /* 814 * If truncate failed early the inode might still be 815 * on the orphan list; we need to make sure the inode 816 * is removed from the orphan list in that case. 817 */ 818 if (inode->i_nlink) 819 ext4_orphan_del(NULL, inode); 820 } 821 return ret ? ret : copied; 822} 823 824/* 825 * Try to make the page cache and handle ready for the inline data case. 826 * We can call this function in 2 cases: 827 * 1. The inode is created and the first write exceeds inline size. We can 828 * clear the inode state safely. 829 * 2. The inode has inline data, then we need to read the data, make it 830 * update and dirty so that ext4_da_writepages can handle it. We don't 831 * need to start the journal since the file's metadata isn't changed now. 832 */ 833static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, 834 struct inode *inode, 835 void **fsdata) 836{ 837 int ret = 0, inline_size; 838 struct folio *folio; 839 840 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN, 841 mapping_gfp_mask(mapping)); 842 if (IS_ERR(folio)) 843 return PTR_ERR(folio); 844 845 down_read(&EXT4_I(inode)->xattr_sem); 846 if (!ext4_has_inline_data(inode)) { 847 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 848 goto out; 849 } 850 851 inline_size = ext4_get_inline_size(inode); 852 853 if (!folio_test_uptodate(folio)) { 854 ret = ext4_read_inline_folio(inode, folio); 855 if (ret < 0) 856 goto out; 857 } 858 859 ret = __block_write_begin(&folio->page, 0, inline_size, 860 ext4_da_get_block_prep); 861 if (ret) { 862 up_read(&EXT4_I(inode)->xattr_sem); 863 folio_unlock(folio); 864 folio_put(folio); 865 ext4_truncate_failed_write(inode); 866 return ret; 867 } 868 869 folio_mark_dirty(folio); 870 folio_mark_uptodate(folio); 871 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 872 *fsdata = (void *)CONVERT_INLINE_DATA; 873 874out: 875 up_read(&EXT4_I(inode)->xattr_sem); 876 if (folio) { 877 folio_unlock(folio); 878 folio_put(folio); 879 } 880 return ret; 881} 882 883/* 884 * Prepare the write for the inline data. 885 * If the data can be written into the inode, we just read 886 * the page and make it uptodate, and start the journal. 887 * Otherwise read the page, makes it dirty so that it can be 888 * handle in writepages(the i_disksize update is left to the 889 * normal ext4_da_write_end). 890 */ 891int ext4_da_write_inline_data_begin(struct address_space *mapping, 892 struct inode *inode, 893 loff_t pos, unsigned len, 894 struct page **pagep, 895 void **fsdata) 896{ 897 int ret; 898 handle_t *handle; 899 struct folio *folio; 900 struct ext4_iloc iloc; 901 int retries = 0; 902 903 ret = ext4_get_inode_loc(inode, &iloc); 904 if (ret) 905 return ret; 906 907retry_journal: 908 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); 909 if (IS_ERR(handle)) { 910 ret = PTR_ERR(handle); 911 goto out; 912 } 913 914 ret = ext4_prepare_inline_data(handle, inode, pos + len); 915 if (ret && ret != -ENOSPC) 916 goto out_journal; 917 918 if (ret == -ENOSPC) { 919 ext4_journal_stop(handle); 920 ret = ext4_da_convert_inline_data_to_extent(mapping, 921 inode, 922 fsdata); 923 if (ret == -ENOSPC && 924 ext4_should_retry_alloc(inode->i_sb, &retries)) 925 goto retry_journal; 926 goto out; 927 } 928 929 /* 930 * We cannot recurse into the filesystem as the transaction 931 * is already started. 932 */ 933 folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, 934 mapping_gfp_mask(mapping)); 935 if (IS_ERR(folio)) { 936 ret = PTR_ERR(folio); 937 goto out_journal; 938 } 939 940 down_read(&EXT4_I(inode)->xattr_sem); 941 if (!ext4_has_inline_data(inode)) { 942 ret = 0; 943 goto out_release_page; 944 } 945 946 if (!folio_test_uptodate(folio)) { 947 ret = ext4_read_inline_folio(inode, folio); 948 if (ret < 0) 949 goto out_release_page; 950 } 951 ret = ext4_journal_get_write_access(handle, inode->i_sb, iloc.bh, 952 EXT4_JTR_NONE); 953 if (ret) 954 goto out_release_page; 955 956 up_read(&EXT4_I(inode)->xattr_sem); 957 *pagep = &folio->page; 958 brelse(iloc.bh); 959 return 1; 960out_release_page: 961 up_read(&EXT4_I(inode)->xattr_sem); 962 folio_unlock(folio); 963 folio_put(folio); 964out_journal: 965 ext4_journal_stop(handle); 966out: 967 brelse(iloc.bh); 968 return ret; 969} 970 971#ifdef INLINE_DIR_DEBUG 972void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh, 973 void *inline_start, int inline_size) 974{ 975 int offset; 976 unsigned short de_len; 977 struct ext4_dir_entry_2 *de = inline_start; 978 void *dlimit = inline_start + inline_size; 979 980 trace_printk("inode %lu\n", dir->i_ino); 981 offset = 0; 982 while ((void *)de < dlimit) { 983 de_len = ext4_rec_len_from_disk(de->rec_len, inline_size); 984 trace_printk("de: off %u rlen %u name %.*s nlen %u ino %u\n", 985 offset, de_len, de->name_len, de->name, 986 de->name_len, le32_to_cpu(de->inode)); 987 if (ext4_check_dir_entry(dir, NULL, de, bh, 988 inline_start, inline_size, offset)) 989 BUG(); 990 991 offset += de_len; 992 de = (struct ext4_dir_entry_2 *) ((char *) de + de_len); 993 } 994} 995#else 996#define ext4_show_inline_dir(dir, bh, inline_start, inline_size) 997#endif 998 999/* 1000 * Add a new entry into a inline dir. 1001 * It will return -ENOSPC if no space is available, and -EIO 1002 * and -EEXIST if directory entry already exists. 1003 */ 1004static int ext4_add_dirent_to_inline(handle_t *handle, 1005 struct ext4_filename *fname, 1006 struct inode *dir, 1007 struct inode *inode, 1008 struct ext4_iloc *iloc, 1009 void *inline_start, int inline_size) 1010{ 1011 int err; 1012 struct ext4_dir_entry_2 *de; 1013 1014 err = ext4_find_dest_de(dir, inode, iloc->bh, inline_start, 1015 inline_size, fname, &de); 1016 if (err) 1017 return err; 1018 1019 BUFFER_TRACE(iloc->bh, "get_write_access"); 1020 err = ext4_journal_get_write_access(handle, dir->i_sb, iloc->bh, 1021 EXT4_JTR_NONE); 1022 if (err) 1023 return err; 1024 ext4_insert_dentry(dir, inode, de, inline_size, fname); 1025 1026 ext4_show_inline_dir(dir, iloc->bh, inline_start, inline_size); 1027 1028 /* 1029 * XXX shouldn't update any times until successful 1030 * completion of syscall, but too many callers depend 1031 * on this. 1032 * 1033 * XXX similarly, too many callers depend on 1034 * ext4_new_inode() setting the times, but error 1035 * recovery deletes the inode, so the worst that can 1036 * happen is that the times are slightly out of date 1037 * and/or different from the directory change time. 1038 */ 1039 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); 1040 ext4_update_dx_flag(dir); 1041 inode_inc_iversion(dir); 1042 return 1; 1043} 1044 1045static void *ext4_get_inline_xattr_pos(struct inode *inode, 1046 struct ext4_iloc *iloc) 1047{ 1048 struct ext4_xattr_entry *entry; 1049 struct ext4_xattr_ibody_header *header; 1050 1051 BUG_ON(!EXT4_I(inode)->i_inline_off); 1052 1053 header = IHDR(inode, ext4_raw_inode(iloc)); 1054 entry = (struct ext4_xattr_entry *)((void *)ext4_raw_inode(iloc) + 1055 EXT4_I(inode)->i_inline_off); 1056 1057 return (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs); 1058} 1059 1060/* Set the final de to cover the whole block. */ 1061static void ext4_update_final_de(void *de_buf, int old_size, int new_size) 1062{ 1063 struct ext4_dir_entry_2 *de, *prev_de; 1064 void *limit; 1065 int de_len; 1066 1067 de = de_buf; 1068 if (old_size) { 1069 limit = de_buf + old_size; 1070 do { 1071 prev_de = de; 1072 de_len = ext4_rec_len_from_disk(de->rec_len, old_size); 1073 de_buf += de_len; 1074 de = de_buf; 1075 } while (de_buf < limit); 1076 1077 prev_de->rec_len = ext4_rec_len_to_disk(de_len + new_size - 1078 old_size, new_size); 1079 } else { 1080 /* this is just created, so create an empty entry. */ 1081 de->inode = 0; 1082 de->rec_len = ext4_rec_len_to_disk(new_size, new_size); 1083 } 1084} 1085 1086static int ext4_update_inline_dir(handle_t *handle, struct inode *dir, 1087 struct ext4_iloc *iloc) 1088{ 1089 int ret; 1090 int old_size = EXT4_I(dir)->i_inline_size - EXT4_MIN_INLINE_DATA_SIZE; 1091 int new_size = get_max_inline_xattr_value_size(dir, iloc); 1092 1093 if (new_size - old_size <= ext4_dir_rec_len(1, NULL)) 1094 return -ENOSPC; 1095 1096 ret = ext4_update_inline_data(handle, dir, 1097 new_size + EXT4_MIN_INLINE_DATA_SIZE); 1098 if (ret) 1099 return ret; 1100 1101 ext4_update_final_de(ext4_get_inline_xattr_pos(dir, iloc), old_size, 1102 EXT4_I(dir)->i_inline_size - 1103 EXT4_MIN_INLINE_DATA_SIZE); 1104 dir->i_size = EXT4_I(dir)->i_disksize = EXT4_I(dir)->i_inline_size; 1105 return 0; 1106} 1107 1108static void ext4_restore_inline_data(handle_t *handle, struct inode *inode, 1109 struct ext4_iloc *iloc, 1110 void *buf, int inline_size) 1111{ 1112 int ret; 1113 1114 ret = ext4_create_inline_data(handle, inode, inline_size); 1115 if (ret) { 1116 ext4_msg(inode->i_sb, KERN_EMERG, 1117 "error restoring inline_data for inode -- potential data loss! (inode %lu, error %d)", 1118 inode->i_ino, ret); 1119 return; 1120 } 1121 ext4_write_inline_data(inode, iloc, buf, 0, inline_size); 1122 ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 1123} 1124 1125static int ext4_finish_convert_inline_dir(handle_t *handle, 1126 struct inode *inode, 1127 struct buffer_head *dir_block, 1128 void *buf, 1129 int inline_size) 1130{ 1131 int err, csum_size = 0, header_size = 0; 1132 struct ext4_dir_entry_2 *de; 1133 void *target = dir_block->b_data; 1134 1135 /* 1136 * First create "." and ".." and then copy the dir information 1137 * back to the block. 1138 */ 1139 de = target; 1140 de = ext4_init_dot_dotdot(inode, de, 1141 inode->i_sb->s_blocksize, csum_size, 1142 le32_to_cpu(((struct ext4_dir_entry_2 *)buf)->inode), 1); 1143 header_size = (void *)de - target; 1144 1145 memcpy((void *)de, buf + EXT4_INLINE_DOTDOT_SIZE, 1146 inline_size - EXT4_INLINE_DOTDOT_SIZE); 1147 1148 if (ext4_has_metadata_csum(inode->i_sb)) 1149 csum_size = sizeof(struct ext4_dir_entry_tail); 1150 1151 inode->i_size = inode->i_sb->s_blocksize; 1152 i_size_write(inode, inode->i_sb->s_blocksize); 1153 EXT4_I(inode)->i_disksize = inode->i_sb->s_blocksize; 1154 ext4_update_final_de(dir_block->b_data, 1155 inline_size - EXT4_INLINE_DOTDOT_SIZE + header_size, 1156 inode->i_sb->s_blocksize - csum_size); 1157 1158 if (csum_size) 1159 ext4_initialize_dirent_tail(dir_block, 1160 inode->i_sb->s_blocksize); 1161 set_buffer_uptodate(dir_block); 1162 unlock_buffer(dir_block); 1163 err = ext4_handle_dirty_dirblock(handle, inode, dir_block); 1164 if (err) 1165 return err; 1166 set_buffer_verified(dir_block); 1167 return ext4_mark_inode_dirty(handle, inode); 1168} 1169 1170static int ext4_convert_inline_data_nolock(handle_t *handle, 1171 struct inode *inode, 1172 struct ext4_iloc *iloc) 1173{ 1174 int error; 1175 void *buf = NULL; 1176 struct buffer_head *data_bh = NULL; 1177 struct ext4_map_blocks map; 1178 int inline_size; 1179 1180 inline_size = ext4_get_inline_size(inode); 1181 buf = kmalloc(inline_size, GFP_NOFS); 1182 if (!buf) { 1183 error = -ENOMEM; 1184 goto out; 1185 } 1186 1187 error = ext4_read_inline_data(inode, buf, inline_size, iloc); 1188 if (error < 0) 1189 goto out; 1190 1191 /* 1192 * Make sure the inline directory entries pass checks before we try to 1193 * convert them, so that we avoid touching stuff that needs fsck. 1194 */ 1195 if (S_ISDIR(inode->i_mode)) { 1196 error = ext4_check_all_de(inode, iloc->bh, 1197 buf + EXT4_INLINE_DOTDOT_SIZE, 1198 inline_size - EXT4_INLINE_DOTDOT_SIZE); 1199 if (error) 1200 goto out; 1201 } 1202 1203 error = ext4_destroy_inline_data_nolock(handle, inode); 1204 if (error) 1205 goto out; 1206 1207 map.m_lblk = 0; 1208 map.m_len = 1; 1209 map.m_flags = 0; 1210 error = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_CREATE); 1211 if (error < 0) 1212 goto out_restore; 1213 if (!(map.m_flags & EXT4_MAP_MAPPED)) { 1214 error = -EIO; 1215 goto out_restore; 1216 } 1217 1218 data_bh = sb_getblk(inode->i_sb, map.m_pblk); 1219 if (!data_bh) { 1220 error = -ENOMEM; 1221 goto out_restore; 1222 } 1223 1224 lock_buffer(data_bh); 1225 error = ext4_journal_get_create_access(handle, inode->i_sb, data_bh, 1226 EXT4_JTR_NONE); 1227 if (error) { 1228 unlock_buffer(data_bh); 1229 error = -EIO; 1230 goto out_restore; 1231 } 1232 memset(data_bh->b_data, 0, inode->i_sb->s_blocksize); 1233 1234 if (!S_ISDIR(inode->i_mode)) { 1235 memcpy(data_bh->b_data, buf, inline_size); 1236 set_buffer_uptodate(data_bh); 1237 unlock_buffer(data_bh); 1238 error = ext4_handle_dirty_metadata(handle, 1239 inode, data_bh); 1240 } else { 1241 error = ext4_finish_convert_inline_dir(handle, inode, data_bh, 1242 buf, inline_size); 1243 } 1244 1245out_restore: 1246 if (error) 1247 ext4_restore_inline_data(handle, inode, iloc, buf, inline_size); 1248 1249out: 1250 brelse(data_bh); 1251 kfree(buf); 1252 return error; 1253} 1254 1255/* 1256 * Try to add the new entry to the inline data. 1257 * If succeeds, return 0. If not, extended the inline dir and copied data to 1258 * the new created block. 1259 */ 1260int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname, 1261 struct inode *dir, struct inode *inode) 1262{ 1263 int ret, ret2, inline_size, no_expand; 1264 void *inline_start; 1265 struct ext4_iloc iloc; 1266 1267 ret = ext4_get_inode_loc(dir, &iloc); 1268 if (ret) 1269 return ret; 1270 1271 ext4_write_lock_xattr(dir, &no_expand); 1272 if (!ext4_has_inline_data(dir)) 1273 goto out; 1274 1275 inline_start = (void *)ext4_raw_inode(&iloc)->i_block + 1276 EXT4_INLINE_DOTDOT_SIZE; 1277 inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE; 1278 1279 ret = ext4_add_dirent_to_inline(handle, fname, dir, inode, &iloc, 1280 inline_start, inline_size); 1281 if (ret != -ENOSPC) 1282 goto out; 1283 1284 /* check whether it can be inserted to inline xattr space. */ 1285 inline_size = EXT4_I(dir)->i_inline_size - 1286 EXT4_MIN_INLINE_DATA_SIZE; 1287 if (!inline_size) { 1288 /* Try to use the xattr space.*/ 1289 ret = ext4_update_inline_dir(handle, dir, &iloc); 1290 if (ret && ret != -ENOSPC) 1291 goto out; 1292 1293 inline_size = EXT4_I(dir)->i_inline_size - 1294 EXT4_MIN_INLINE_DATA_SIZE; 1295 } 1296 1297 if (inline_size) { 1298 inline_start = ext4_get_inline_xattr_pos(dir, &iloc); 1299 1300 ret = ext4_add_dirent_to_inline(handle, fname, dir, 1301 inode, &iloc, inline_start, 1302 inline_size); 1303 1304 if (ret != -ENOSPC) 1305 goto out; 1306 } 1307 1308 /* 1309 * The inline space is filled up, so create a new block for it. 1310 * As the extent tree will be created, we have to save the inline 1311 * dir first. 1312 */ 1313 ret = ext4_convert_inline_data_nolock(handle, dir, &iloc); 1314 1315out: 1316 ext4_write_unlock_xattr(dir, &no_expand); 1317 ret2 = ext4_mark_inode_dirty(handle, dir); 1318 if (unlikely(ret2 && !ret)) 1319 ret = ret2; 1320 brelse(iloc.bh); 1321 return ret; 1322} 1323 1324/* 1325 * This function fills a red-black tree with information from an 1326 * inlined dir. It returns the number directory entries loaded 1327 * into the tree. If there is an error it is returned in err. 1328 */ 1329int ext4_inlinedir_to_tree(struct file *dir_file, 1330 struct inode *dir, ext4_lblk_t block, 1331 struct dx_hash_info *hinfo, 1332 __u32 start_hash, __u32 start_minor_hash, 1333 int *has_inline_data) 1334{ 1335 int err = 0, count = 0; 1336 unsigned int parent_ino; 1337 int pos; 1338 struct ext4_dir_entry_2 *de; 1339 struct inode *inode = file_inode(dir_file); 1340 int ret, inline_size = 0; 1341 struct ext4_iloc iloc; 1342 void *dir_buf = NULL; 1343 struct ext4_dir_entry_2 fake; 1344 struct fscrypt_str tmp_str; 1345 1346 ret = ext4_get_inode_loc(inode, &iloc); 1347 if (ret) 1348 return ret; 1349 1350 down_read(&EXT4_I(inode)->xattr_sem); 1351 if (!ext4_has_inline_data(inode)) { 1352 up_read(&EXT4_I(inode)->xattr_sem); 1353 *has_inline_data = 0; 1354 goto out; 1355 } 1356 1357 inline_size = ext4_get_inline_size(inode); 1358 dir_buf = kmalloc(inline_size, GFP_NOFS); 1359 if (!dir_buf) { 1360 ret = -ENOMEM; 1361 up_read(&EXT4_I(inode)->xattr_sem); 1362 goto out; 1363 } 1364 1365 ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc); 1366 up_read(&EXT4_I(inode)->xattr_sem); 1367 if (ret < 0) 1368 goto out; 1369 1370 pos = 0; 1371 parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode); 1372 while (pos < inline_size) { 1373 /* 1374 * As inlined dir doesn't store any information about '.' and 1375 * only the inode number of '..' is stored, we have to handle 1376 * them differently. 1377 */ 1378 if (pos == 0) { 1379 fake.inode = cpu_to_le32(inode->i_ino); 1380 fake.name_len = 1; 1381 strcpy(fake.name, "."); 1382 fake.rec_len = ext4_rec_len_to_disk( 1383 ext4_dir_rec_len(fake.name_len, NULL), 1384 inline_size); 1385 ext4_set_de_type(inode->i_sb, &fake, S_IFDIR); 1386 de = &fake; 1387 pos = EXT4_INLINE_DOTDOT_OFFSET; 1388 } else if (pos == EXT4_INLINE_DOTDOT_OFFSET) { 1389 fake.inode = cpu_to_le32(parent_ino); 1390 fake.name_len = 2; 1391 strcpy(fake.name, ".."); 1392 fake.rec_len = ext4_rec_len_to_disk( 1393 ext4_dir_rec_len(fake.name_len, NULL), 1394 inline_size); 1395 ext4_set_de_type(inode->i_sb, &fake, S_IFDIR); 1396 de = &fake; 1397 pos = EXT4_INLINE_DOTDOT_SIZE; 1398 } else { 1399 de = (struct ext4_dir_entry_2 *)(dir_buf + pos); 1400 pos += ext4_rec_len_from_disk(de->rec_len, inline_size); 1401 if (ext4_check_dir_entry(inode, dir_file, de, 1402 iloc.bh, dir_buf, 1403 inline_size, pos)) { 1404 ret = count; 1405 goto out; 1406 } 1407 } 1408 1409 if (ext4_hash_in_dirent(dir)) { 1410 hinfo->hash = EXT4_DIRENT_HASH(de); 1411 hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de); 1412 } else { 1413 ext4fs_dirhash(dir, de->name, de->name_len, hinfo); 1414 } 1415 if ((hinfo->hash < start_hash) || 1416 ((hinfo->hash == start_hash) && 1417 (hinfo->minor_hash < start_minor_hash))) 1418 continue; 1419 if (de->inode == 0) 1420 continue; 1421 tmp_str.name = de->name; 1422 tmp_str.len = de->name_len; 1423 err = ext4_htree_store_dirent(dir_file, hinfo->hash, 1424 hinfo->minor_hash, de, &tmp_str); 1425 if (err) { 1426 ret = err; 1427 goto out; 1428 } 1429 count++; 1430 } 1431 ret = count; 1432out: 1433 kfree(dir_buf); 1434 brelse(iloc.bh); 1435 return ret; 1436} 1437 1438/* 1439 * So this function is called when the volume is mkfsed with 1440 * dir_index disabled. In order to keep f_pos persistent 1441 * after we convert from an inlined dir to a blocked based, 1442 * we just pretend that we are a normal dir and return the 1443 * offset as if '.' and '..' really take place. 1444 * 1445 */ 1446int ext4_read_inline_dir(struct file *file, 1447 struct dir_context *ctx, 1448 int *has_inline_data) 1449{ 1450 unsigned int offset, parent_ino; 1451 int i; 1452 struct ext4_dir_entry_2 *de; 1453 struct super_block *sb; 1454 struct inode *inode = file_inode(file); 1455 int ret, inline_size = 0; 1456 struct ext4_iloc iloc; 1457 void *dir_buf = NULL; 1458 int dotdot_offset, dotdot_size, extra_offset, extra_size; 1459 1460 ret = ext4_get_inode_loc(inode, &iloc); 1461 if (ret) 1462 return ret; 1463 1464 down_read(&EXT4_I(inode)->xattr_sem); 1465 if (!ext4_has_inline_data(inode)) { 1466 up_read(&EXT4_I(inode)->xattr_sem); 1467 *has_inline_data = 0; 1468 goto out; 1469 } 1470 1471 inline_size = ext4_get_inline_size(inode); 1472 dir_buf = kmalloc(inline_size, GFP_NOFS); 1473 if (!dir_buf) { 1474 ret = -ENOMEM; 1475 up_read(&EXT4_I(inode)->xattr_sem); 1476 goto out; 1477 } 1478 1479 ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc); 1480 up_read(&EXT4_I(inode)->xattr_sem); 1481 if (ret < 0) 1482 goto out; 1483 1484 ret = 0; 1485 sb = inode->i_sb; 1486 parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode); 1487 offset = ctx->pos; 1488 1489 /* 1490 * dotdot_offset and dotdot_size is the real offset and 1491 * size for ".." and "." if the dir is block based while 1492 * the real size for them are only EXT4_INLINE_DOTDOT_SIZE. 1493 * So we will use extra_offset and extra_size to indicate them 1494 * during the inline dir iteration. 1495 */ 1496 dotdot_offset = ext4_dir_rec_len(1, NULL); 1497 dotdot_size = dotdot_offset + ext4_dir_rec_len(2, NULL); 1498 extra_offset = dotdot_size - EXT4_INLINE_DOTDOT_SIZE; 1499 extra_size = extra_offset + inline_size; 1500 1501 /* 1502 * If the version has changed since the last call to 1503 * readdir(2), then we might be pointing to an invalid 1504 * dirent right now. Scan from the start of the inline 1505 * dir to make sure. 1506 */ 1507 if (!inode_eq_iversion(inode, file->f_version)) { 1508 for (i = 0; i < extra_size && i < offset;) { 1509 /* 1510 * "." is with offset 0 and 1511 * ".." is dotdot_offset. 1512 */ 1513 if (!i) { 1514 i = dotdot_offset; 1515 continue; 1516 } else if (i == dotdot_offset) { 1517 i = dotdot_size; 1518 continue; 1519 } 1520 /* for other entry, the real offset in 1521 * the buf has to be tuned accordingly. 1522 */ 1523 de = (struct ext4_dir_entry_2 *) 1524 (dir_buf + i - extra_offset); 1525 /* It's too expensive to do a full 1526 * dirent test each time round this 1527 * loop, but we do have to test at 1528 * least that it is non-zero. A 1529 * failure will be detected in the 1530 * dirent test below. */ 1531 if (ext4_rec_len_from_disk(de->rec_len, extra_size) 1532 < ext4_dir_rec_len(1, NULL)) 1533 break; 1534 i += ext4_rec_len_from_disk(de->rec_len, 1535 extra_size); 1536 } 1537 offset = i; 1538 ctx->pos = offset; 1539 file->f_version = inode_query_iversion(inode); 1540 } 1541 1542 while (ctx->pos < extra_size) { 1543 if (ctx->pos == 0) { 1544 if (!dir_emit(ctx, ".", 1, inode->i_ino, DT_DIR)) 1545 goto out; 1546 ctx->pos = dotdot_offset; 1547 continue; 1548 } 1549 1550 if (ctx->pos == dotdot_offset) { 1551 if (!dir_emit(ctx, "..", 2, parent_ino, DT_DIR)) 1552 goto out; 1553 ctx->pos = dotdot_size; 1554 continue; 1555 } 1556 1557 de = (struct ext4_dir_entry_2 *) 1558 (dir_buf + ctx->pos - extra_offset); 1559 if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, 1560 extra_size, ctx->pos)) 1561 goto out; 1562 if (le32_to_cpu(de->inode)) { 1563 if (!dir_emit(ctx, de->name, de->name_len, 1564 le32_to_cpu(de->inode), 1565 get_dtype(sb, de->file_type))) 1566 goto out; 1567 } 1568 ctx->pos += ext4_rec_len_from_disk(de->rec_len, extra_size); 1569 } 1570out: 1571 kfree(dir_buf); 1572 brelse(iloc.bh); 1573 return ret; 1574} 1575 1576void *ext4_read_inline_link(struct inode *inode) 1577{ 1578 struct ext4_iloc iloc; 1579 int ret, inline_size; 1580 void *link; 1581 1582 ret = ext4_get_inode_loc(inode, &iloc); 1583 if (ret) 1584 return ERR_PTR(ret); 1585 1586 ret = -ENOMEM; 1587 inline_size = ext4_get_inline_size(inode); 1588 link = kmalloc(inline_size + 1, GFP_NOFS); 1589 if (!link) 1590 goto out; 1591 1592 ret = ext4_read_inline_data(inode, link, inline_size, &iloc); 1593 if (ret < 0) { 1594 kfree(link); 1595 goto out; 1596 } 1597 nd_terminate_link(link, inode->i_size, ret); 1598out: 1599 if (ret < 0) 1600 link = ERR_PTR(ret); 1601 brelse(iloc.bh); 1602 return link; 1603} 1604 1605struct buffer_head *ext4_get_first_inline_block(struct inode *inode, 1606 struct ext4_dir_entry_2 **parent_de, 1607 int *retval) 1608{ 1609 struct ext4_iloc iloc; 1610 1611 *retval = ext4_get_inode_loc(inode, &iloc); 1612 if (*retval) 1613 return NULL; 1614 1615 *parent_de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; 1616 1617 return iloc.bh; 1618} 1619 1620/* 1621 * Try to create the inline data for the new dir. 1622 * If it succeeds, return 0, otherwise return the error. 1623 * In case of ENOSPC, the caller should create the normal disk layout dir. 1624 */ 1625int ext4_try_create_inline_dir(handle_t *handle, struct inode *parent, 1626 struct inode *inode) 1627{ 1628 int ret, inline_size = EXT4_MIN_INLINE_DATA_SIZE; 1629 struct ext4_iloc iloc; 1630 struct ext4_dir_entry_2 *de; 1631 1632 ret = ext4_get_inode_loc(inode, &iloc); 1633 if (ret) 1634 return ret; 1635 1636 ret = ext4_prepare_inline_data(handle, inode, inline_size); 1637 if (ret) 1638 goto out; 1639 1640 /* 1641 * For inline dir, we only save the inode information for the ".." 1642 * and create a fake dentry to cover the left space. 1643 */ 1644 de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; 1645 de->inode = cpu_to_le32(parent->i_ino); 1646 de = (struct ext4_dir_entry_2 *)((void *)de + EXT4_INLINE_DOTDOT_SIZE); 1647 de->inode = 0; 1648 de->rec_len = ext4_rec_len_to_disk( 1649 inline_size - EXT4_INLINE_DOTDOT_SIZE, 1650 inline_size); 1651 set_nlink(inode, 2); 1652 inode->i_size = EXT4_I(inode)->i_disksize = inline_size; 1653out: 1654 brelse(iloc.bh); 1655 return ret; 1656} 1657 1658struct buffer_head *ext4_find_inline_entry(struct inode *dir, 1659 struct ext4_filename *fname, 1660 struct ext4_dir_entry_2 **res_dir, 1661 int *has_inline_data) 1662{ 1663 int ret; 1664 struct ext4_iloc iloc; 1665 void *inline_start; 1666 int inline_size; 1667 1668 if (ext4_get_inode_loc(dir, &iloc)) 1669 return NULL; 1670 1671 down_read(&EXT4_I(dir)->xattr_sem); 1672 if (!ext4_has_inline_data(dir)) { 1673 *has_inline_data = 0; 1674 goto out; 1675 } 1676 1677 inline_start = (void *)ext4_raw_inode(&iloc)->i_block + 1678 EXT4_INLINE_DOTDOT_SIZE; 1679 inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE; 1680 ret = ext4_search_dir(iloc.bh, inline_start, inline_size, 1681 dir, fname, 0, res_dir); 1682 if (ret == 1) 1683 goto out_find; 1684 if (ret < 0) 1685 goto out; 1686 1687 if (ext4_get_inline_size(dir) == EXT4_MIN_INLINE_DATA_SIZE) 1688 goto out; 1689 1690 inline_start = ext4_get_inline_xattr_pos(dir, &iloc); 1691 inline_size = ext4_get_inline_size(dir) - EXT4_MIN_INLINE_DATA_SIZE; 1692 1693 ret = ext4_search_dir(iloc.bh, inline_start, inline_size, 1694 dir, fname, 0, res_dir); 1695 if (ret == 1) 1696 goto out_find; 1697 1698out: 1699 brelse(iloc.bh); 1700 iloc.bh = NULL; 1701out_find: 1702 up_read(&EXT4_I(dir)->xattr_sem); 1703 return iloc.bh; 1704} 1705 1706int ext4_delete_inline_entry(handle_t *handle, 1707 struct inode *dir, 1708 struct ext4_dir_entry_2 *de_del, 1709 struct buffer_head *bh, 1710 int *has_inline_data) 1711{ 1712 int err, inline_size, no_expand; 1713 struct ext4_iloc iloc; 1714 void *inline_start; 1715 1716 err = ext4_get_inode_loc(dir, &iloc); 1717 if (err) 1718 return err; 1719 1720 ext4_write_lock_xattr(dir, &no_expand); 1721 if (!ext4_has_inline_data(dir)) { 1722 *has_inline_data = 0; 1723 goto out; 1724 } 1725 1726 if ((void *)de_del - ((void *)ext4_raw_inode(&iloc)->i_block) < 1727 EXT4_MIN_INLINE_DATA_SIZE) { 1728 inline_start = (void *)ext4_raw_inode(&iloc)->i_block + 1729 EXT4_INLINE_DOTDOT_SIZE; 1730 inline_size = EXT4_MIN_INLINE_DATA_SIZE - 1731 EXT4_INLINE_DOTDOT_SIZE; 1732 } else { 1733 inline_start = ext4_get_inline_xattr_pos(dir, &iloc); 1734 inline_size = ext4_get_inline_size(dir) - 1735 EXT4_MIN_INLINE_DATA_SIZE; 1736 } 1737 1738 BUFFER_TRACE(bh, "get_write_access"); 1739 err = ext4_journal_get_write_access(handle, dir->i_sb, bh, 1740 EXT4_JTR_NONE); 1741 if (err) 1742 goto out; 1743 1744 err = ext4_generic_delete_entry(dir, de_del, bh, 1745 inline_start, inline_size, 0); 1746 if (err) 1747 goto out; 1748 1749 ext4_show_inline_dir(dir, iloc.bh, inline_start, inline_size); 1750out: 1751 ext4_write_unlock_xattr(dir, &no_expand); 1752 if (likely(err == 0)) 1753 err = ext4_mark_inode_dirty(handle, dir); 1754 brelse(iloc.bh); 1755 if (err != -ENOENT) 1756 ext4_std_error(dir->i_sb, err); 1757 return err; 1758} 1759 1760/* 1761 * Get the inline dentry at offset. 1762 */ 1763static inline struct ext4_dir_entry_2 * 1764ext4_get_inline_entry(struct inode *inode, 1765 struct ext4_iloc *iloc, 1766 unsigned int offset, 1767 void **inline_start, 1768 int *inline_size) 1769{ 1770 void *inline_pos; 1771 1772 BUG_ON(offset > ext4_get_inline_size(inode)); 1773 1774 if (offset < EXT4_MIN_INLINE_DATA_SIZE) { 1775 inline_pos = (void *)ext4_raw_inode(iloc)->i_block; 1776 *inline_size = EXT4_MIN_INLINE_DATA_SIZE; 1777 } else { 1778 inline_pos = ext4_get_inline_xattr_pos(inode, iloc); 1779 offset -= EXT4_MIN_INLINE_DATA_SIZE; 1780 *inline_size = ext4_get_inline_size(inode) - 1781 EXT4_MIN_INLINE_DATA_SIZE; 1782 } 1783 1784 if (inline_start) 1785 *inline_start = inline_pos; 1786 return (struct ext4_dir_entry_2 *)(inline_pos + offset); 1787} 1788 1789bool empty_inline_dir(struct inode *dir, int *has_inline_data) 1790{ 1791 int err, inline_size; 1792 struct ext4_iloc iloc; 1793 size_t inline_len; 1794 void *inline_pos; 1795 unsigned int offset; 1796 struct ext4_dir_entry_2 *de; 1797 bool ret = false; 1798 1799 err = ext4_get_inode_loc(dir, &iloc); 1800 if (err) { 1801 EXT4_ERROR_INODE_ERR(dir, -err, 1802 "error %d getting inode %lu block", 1803 err, dir->i_ino); 1804 return false; 1805 } 1806 1807 down_read(&EXT4_I(dir)->xattr_sem); 1808 if (!ext4_has_inline_data(dir)) { 1809 *has_inline_data = 0; 1810 ret = true; 1811 goto out; 1812 } 1813 1814 de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; 1815 if (!le32_to_cpu(de->inode)) { 1816 ext4_warning(dir->i_sb, 1817 "bad inline directory (dir #%lu) - no `..'", 1818 dir->i_ino); 1819 goto out; 1820 } 1821 1822 inline_len = ext4_get_inline_size(dir); 1823 offset = EXT4_INLINE_DOTDOT_SIZE; 1824 while (offset < inline_len) { 1825 de = ext4_get_inline_entry(dir, &iloc, offset, 1826 &inline_pos, &inline_size); 1827 if (ext4_check_dir_entry(dir, NULL, de, 1828 iloc.bh, inline_pos, 1829 inline_size, offset)) { 1830 ext4_warning(dir->i_sb, 1831 "bad inline directory (dir #%lu) - " 1832 "inode %u, rec_len %u, name_len %d" 1833 "inline size %d", 1834 dir->i_ino, le32_to_cpu(de->inode), 1835 le16_to_cpu(de->rec_len), de->name_len, 1836 inline_size); 1837 goto out; 1838 } 1839 if (le32_to_cpu(de->inode)) { 1840 goto out; 1841 } 1842 offset += ext4_rec_len_from_disk(de->rec_len, inline_size); 1843 } 1844 1845 ret = true; 1846out: 1847 up_read(&EXT4_I(dir)->xattr_sem); 1848 brelse(iloc.bh); 1849 return ret; 1850} 1851 1852int ext4_destroy_inline_data(handle_t *handle, struct inode *inode) 1853{ 1854 int ret, no_expand; 1855 1856 ext4_write_lock_xattr(inode, &no_expand); 1857 ret = ext4_destroy_inline_data_nolock(handle, inode); 1858 ext4_write_unlock_xattr(inode, &no_expand); 1859 1860 return ret; 1861} 1862 1863int ext4_inline_data_iomap(struct inode *inode, struct iomap *iomap) 1864{ 1865 __u64 addr; 1866 int error = -EAGAIN; 1867 struct ext4_iloc iloc; 1868 1869 down_read(&EXT4_I(inode)->xattr_sem); 1870 if (!ext4_has_inline_data(inode)) 1871 goto out; 1872 1873 error = ext4_get_inode_loc(inode, &iloc); 1874 if (error) 1875 goto out; 1876 1877 addr = (__u64)iloc.bh->b_blocknr << inode->i_sb->s_blocksize_bits; 1878 addr += (char *)ext4_raw_inode(&iloc) - iloc.bh->b_data; 1879 addr += offsetof(struct ext4_inode, i_block); 1880 1881 brelse(iloc.bh); 1882 1883 iomap->addr = addr; 1884 iomap->offset = 0; 1885 iomap->length = min_t(loff_t, ext4_get_inline_size(inode), 1886 i_size_read(inode)); 1887 iomap->type = IOMAP_INLINE; 1888 iomap->flags = 0; 1889 1890out: 1891 up_read(&EXT4_I(inode)->xattr_sem); 1892 return error; 1893} 1894 1895int ext4_inline_data_truncate(struct inode *inode, int *has_inline) 1896{ 1897 handle_t *handle; 1898 int inline_size, value_len, needed_blocks, no_expand, err = 0; 1899 size_t i_size; 1900 void *value = NULL; 1901 struct ext4_xattr_ibody_find is = { 1902 .s = { .not_found = -ENODATA, }, 1903 }; 1904 struct ext4_xattr_info i = { 1905 .name_index = EXT4_XATTR_INDEX_SYSTEM, 1906 .name = EXT4_XATTR_SYSTEM_DATA, 1907 }; 1908 1909 1910 needed_blocks = ext4_writepage_trans_blocks(inode); 1911 handle = ext4_journal_start(inode, EXT4_HT_INODE, needed_blocks); 1912 if (IS_ERR(handle)) 1913 return PTR_ERR(handle); 1914 1915 ext4_write_lock_xattr(inode, &no_expand); 1916 if (!ext4_has_inline_data(inode)) { 1917 ext4_write_unlock_xattr(inode, &no_expand); 1918 *has_inline = 0; 1919 ext4_journal_stop(handle); 1920 return 0; 1921 } 1922 1923 if ((err = ext4_orphan_add(handle, inode)) != 0) 1924 goto out; 1925 1926 if ((err = ext4_get_inode_loc(inode, &is.iloc)) != 0) 1927 goto out; 1928 1929 down_write(&EXT4_I(inode)->i_data_sem); 1930 i_size = inode->i_size; 1931 inline_size = ext4_get_inline_size(inode); 1932 EXT4_I(inode)->i_disksize = i_size; 1933 1934 if (i_size < inline_size) { 1935 /* 1936 * if there's inline data to truncate and this file was 1937 * converted to extents after that inline data was written, 1938 * the extent status cache must be cleared to avoid leaving 1939 * behind stale delayed allocated extent entries 1940 */ 1941 if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) 1942 ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); 1943 1944 /* Clear the content in the xattr space. */ 1945 if (inline_size > EXT4_MIN_INLINE_DATA_SIZE) { 1946 if ((err = ext4_xattr_ibody_find(inode, &i, &is)) != 0) 1947 goto out_error; 1948 1949 BUG_ON(is.s.not_found); 1950 1951 value_len = le32_to_cpu(is.s.here->e_value_size); 1952 value = kmalloc(value_len, GFP_NOFS); 1953 if (!value) { 1954 err = -ENOMEM; 1955 goto out_error; 1956 } 1957 1958 err = ext4_xattr_ibody_get(inode, i.name_index, 1959 i.name, value, value_len); 1960 if (err <= 0) 1961 goto out_error; 1962 1963 i.value = value; 1964 i.value_len = i_size > EXT4_MIN_INLINE_DATA_SIZE ? 1965 i_size - EXT4_MIN_INLINE_DATA_SIZE : 0; 1966 err = ext4_xattr_ibody_set(handle, inode, &i, &is); 1967 if (err) 1968 goto out_error; 1969 } 1970 1971 /* Clear the content within i_blocks. */ 1972 if (i_size < EXT4_MIN_INLINE_DATA_SIZE) { 1973 void *p = (void *) ext4_raw_inode(&is.iloc)->i_block; 1974 memset(p + i_size, 0, 1975 EXT4_MIN_INLINE_DATA_SIZE - i_size); 1976 } 1977 1978 EXT4_I(inode)->i_inline_size = i_size < 1979 EXT4_MIN_INLINE_DATA_SIZE ? 1980 EXT4_MIN_INLINE_DATA_SIZE : i_size; 1981 } 1982 1983out_error: 1984 up_write(&EXT4_I(inode)->i_data_sem); 1985out: 1986 brelse(is.iloc.bh); 1987 ext4_write_unlock_xattr(inode, &no_expand); 1988 kfree(value); 1989 if (inode->i_nlink) 1990 ext4_orphan_del(handle, inode); 1991 1992 if (err == 0) { 1993 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); 1994 err = ext4_mark_inode_dirty(handle, inode); 1995 if (IS_SYNC(inode)) 1996 ext4_handle_sync(handle); 1997 } 1998 ext4_journal_stop(handle); 1999 return err; 2000} 2001 2002int ext4_convert_inline_data(struct inode *inode) 2003{ 2004 int error, needed_blocks, no_expand; 2005 handle_t *handle; 2006 struct ext4_iloc iloc; 2007 2008 if (!ext4_has_inline_data(inode)) { 2009 ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); 2010 return 0; 2011 } else if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { 2012 /* 2013 * Inode has inline data but EXT4_STATE_MAY_INLINE_DATA is 2014 * cleared. This means we are in the middle of moving of 2015 * inline data to delay allocated block. Just force writeout 2016 * here to finish conversion. 2017 */ 2018 error = filemap_flush(inode->i_mapping); 2019 if (error) 2020 return error; 2021 if (!ext4_has_inline_data(inode)) 2022 return 0; 2023 } 2024 2025 needed_blocks = ext4_writepage_trans_blocks(inode); 2026 2027 iloc.bh = NULL; 2028 error = ext4_get_inode_loc(inode, &iloc); 2029 if (error) 2030 return error; 2031 2032 handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); 2033 if (IS_ERR(handle)) { 2034 error = PTR_ERR(handle); 2035 goto out_free; 2036 } 2037 2038 ext4_write_lock_xattr(inode, &no_expand); 2039 if (ext4_has_inline_data(inode)) 2040 error = ext4_convert_inline_data_nolock(handle, inode, &iloc); 2041 ext4_write_unlock_xattr(inode, &no_expand); 2042 ext4_journal_stop(handle); 2043out_free: 2044 brelse(iloc.bh); 2045 return error; 2046} 2047