1// SPDX-License-Identifier: GPL-2.0 2#ifndef NO_BCACHEFS_FS 3 4#include "bcachefs.h" 5#include "alloc_foreground.h" 6#include "bkey_buf.h" 7#include "fs-io.h" 8#include "fs-io-buffered.h" 9#include "fs-io-direct.h" 10#include "fs-io-pagecache.h" 11#include "io_read.h" 12#include "io_write.h" 13 14#include <linux/backing-dev.h> 15#include <linux/pagemap.h> 16#include <linux/writeback.h> 17 18static inline bool bio_full(struct bio *bio, unsigned len) 19{ 20 if (bio->bi_vcnt >= bio->bi_max_vecs) 21 return true; 22 if (bio->bi_iter.bi_size > UINT_MAX - len) 23 return true; 24 return false; 25} 26 27/* readpage(s): */ 28 29static void bch2_readpages_end_io(struct bio *bio) 30{ 31 struct folio_iter fi; 32 33 bio_for_each_folio_all(fi, bio) { 34 if (!bio->bi_status) { 35 folio_mark_uptodate(fi.folio); 36 } else { 37 folio_clear_uptodate(fi.folio); 38 folio_set_error(fi.folio); 39 } 40 folio_unlock(fi.folio); 41 } 42 43 bio_put(bio); 44} 45 46struct readpages_iter { 47 struct address_space *mapping; 48 unsigned idx; 49 folios folios; 50}; 51 52static int readpages_iter_init(struct readpages_iter *iter, 53 struct readahead_control *ractl) 54{ 55 struct folio *folio; 56 57 *iter = (struct readpages_iter) { ractl->mapping }; 58 59 while ((folio = __readahead_folio(ractl))) { 60 if (!bch2_folio_create(folio, GFP_KERNEL) || 61 darray_push(&iter->folios, folio)) { 62 bch2_folio_release(folio); 63 ractl->_nr_pages += folio_nr_pages(folio); 64 ractl->_index -= folio_nr_pages(folio); 65 return iter->folios.nr ? 0 : -ENOMEM; 66 } 67 68 folio_put(folio); 69 } 70 71 return 0; 72} 73 74static inline struct folio *readpage_iter_peek(struct readpages_iter *iter) 75{ 76 if (iter->idx >= iter->folios.nr) 77 return NULL; 78 return iter->folios.data[iter->idx]; 79} 80 81static inline void readpage_iter_advance(struct readpages_iter *iter) 82{ 83 iter->idx++; 84} 85 86static bool extent_partial_reads_expensive(struct bkey_s_c k) 87{ 88 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 89 struct bch_extent_crc_unpacked crc; 90 const union bch_extent_entry *i; 91 92 bkey_for_each_crc(k.k, ptrs, crc, i) 93 if (crc.csum_type || crc.compression_type) 94 return true; 95 return false; 96} 97 98static int readpage_bio_extend(struct btree_trans *trans, 99 struct readpages_iter *iter, 100 struct bio *bio, 101 unsigned sectors_this_extent, 102 bool get_more) 103{ 104 /* Don't hold btree locks while allocating memory: */ 105 bch2_trans_unlock(trans); 106 107 while (bio_sectors(bio) < sectors_this_extent && 108 bio->bi_vcnt < bio->bi_max_vecs) { 109 struct folio *folio = readpage_iter_peek(iter); 110 int ret; 111 112 if (folio) { 113 readpage_iter_advance(iter); 114 } else { 115 pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT; 116 117 if (!get_more) 118 break; 119 120 folio = xa_load(&iter->mapping->i_pages, folio_offset); 121 if (folio && !xa_is_value(folio)) 122 break; 123 124 folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0); 125 if (!folio) 126 break; 127 128 if (!__bch2_folio_create(folio, GFP_KERNEL)) { 129 folio_put(folio); 130 break; 131 } 132 133 ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL); 134 if (ret) { 135 __bch2_folio_release(folio); 136 folio_put(folio); 137 break; 138 } 139 140 folio_put(folio); 141 } 142 143 BUG_ON(folio_sector(folio) != bio_end_sector(bio)); 144 145 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0)); 146 } 147 148 return bch2_trans_relock(trans); 149} 150 151static void bchfs_read(struct btree_trans *trans, 152 struct bch_read_bio *rbio, 153 subvol_inum inum, 154 struct readpages_iter *readpages_iter) 155{ 156 struct bch_fs *c = trans->c; 157 struct btree_iter iter; 158 struct bkey_buf sk; 159 int flags = BCH_READ_RETRY_IF_STALE| 160 BCH_READ_MAY_PROMOTE; 161 u32 snapshot; 162 int ret = 0; 163 164 rbio->c = c; 165 rbio->start_time = local_clock(); 166 rbio->subvol = inum.subvol; 167 168 bch2_bkey_buf_init(&sk); 169retry: 170 bch2_trans_begin(trans); 171 iter = (struct btree_iter) { NULL }; 172 173 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); 174 if (ret) 175 goto err; 176 177 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 178 SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot), 179 BTREE_ITER_SLOTS); 180 while (1) { 181 struct bkey_s_c k; 182 unsigned bytes, sectors, offset_into_extent; 183 enum btree_id data_btree = BTREE_ID_extents; 184 185 /* 186 * read_extent -> io_time_reset may cause a transaction restart 187 * without returning an error, we need to check for that here: 188 */ 189 ret = bch2_trans_relock(trans); 190 if (ret) 191 break; 192 193 bch2_btree_iter_set_pos(&iter, 194 POS(inum.inum, rbio->bio.bi_iter.bi_sector)); 195 196 k = bch2_btree_iter_peek_slot(&iter); 197 ret = bkey_err(k); 198 if (ret) 199 break; 200 201 offset_into_extent = iter.pos.offset - 202 bkey_start_offset(k.k); 203 sectors = k.k->size - offset_into_extent; 204 205 bch2_bkey_buf_reassemble(&sk, c, k); 206 207 ret = bch2_read_indirect_extent(trans, &data_btree, 208 &offset_into_extent, &sk); 209 if (ret) 210 break; 211 212 k = bkey_i_to_s_c(sk.k); 213 214 sectors = min(sectors, k.k->size - offset_into_extent); 215 216 if (readpages_iter) { 217 ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors, 218 extent_partial_reads_expensive(k)); 219 if (ret) 220 break; 221 } 222 223 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9; 224 swap(rbio->bio.bi_iter.bi_size, bytes); 225 226 if (rbio->bio.bi_iter.bi_size == bytes) 227 flags |= BCH_READ_LAST_FRAGMENT; 228 229 bch2_bio_page_state_set(&rbio->bio, k); 230 231 bch2_read_extent(trans, rbio, iter.pos, 232 data_btree, k, offset_into_extent, flags); 233 234 if (flags & BCH_READ_LAST_FRAGMENT) 235 break; 236 237 swap(rbio->bio.bi_iter.bi_size, bytes); 238 bio_advance(&rbio->bio, bytes); 239 240 ret = btree_trans_too_many_iters(trans); 241 if (ret) 242 break; 243 } 244err: 245 bch2_trans_iter_exit(trans, &iter); 246 247 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 248 goto retry; 249 250 if (ret) { 251 bch_err_inum_offset_ratelimited(c, 252 iter.pos.inode, 253 iter.pos.offset << 9, 254 "read error %i from btree lookup", ret); 255 rbio->bio.bi_status = BLK_STS_IOERR; 256 bio_endio(&rbio->bio); 257 } 258 259 bch2_bkey_buf_exit(&sk, c); 260} 261 262void bch2_readahead(struct readahead_control *ractl) 263{ 264 struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host); 265 struct bch_fs *c = inode->v.i_sb->s_fs_info; 266 struct bch_io_opts opts; 267 struct btree_trans *trans = bch2_trans_get(c); 268 struct folio *folio; 269 struct readpages_iter readpages_iter; 270 271 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 272 273 int ret = readpages_iter_init(&readpages_iter, ractl); 274 if (ret) 275 return; 276 277 bch2_pagecache_add_get(inode); 278 279 while ((folio = readpage_iter_peek(&readpages_iter))) { 280 unsigned n = min_t(unsigned, 281 readpages_iter.folios.nr - 282 readpages_iter.idx, 283 BIO_MAX_VECS); 284 struct bch_read_bio *rbio = 285 rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ, 286 GFP_KERNEL, &c->bio_read), 287 opts); 288 289 readpage_iter_advance(&readpages_iter); 290 291 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 292 rbio->bio.bi_end_io = bch2_readpages_end_io; 293 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 294 295 bchfs_read(trans, rbio, inode_inum(inode), 296 &readpages_iter); 297 bch2_trans_unlock(trans); 298 } 299 300 bch2_pagecache_add_put(inode); 301 302 bch2_trans_put(trans); 303 darray_exit(&readpages_iter.folios); 304} 305 306static void bch2_read_single_folio_end_io(struct bio *bio) 307{ 308 complete(bio->bi_private); 309} 310 311int bch2_read_single_folio(struct folio *folio, struct address_space *mapping) 312{ 313 struct bch_inode_info *inode = to_bch_ei(mapping->host); 314 struct bch_fs *c = inode->v.i_sb->s_fs_info; 315 struct bch_read_bio *rbio; 316 struct bch_io_opts opts; 317 int ret; 318 DECLARE_COMPLETION_ONSTACK(done); 319 320 if (!bch2_folio_create(folio, GFP_KERNEL)) 321 return -ENOMEM; 322 323 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 324 325 rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read), 326 opts); 327 rbio->bio.bi_private = &done; 328 rbio->bio.bi_end_io = bch2_read_single_folio_end_io; 329 330 rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC; 331 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 332 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 333 334 bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0)); 335 wait_for_completion(&done); 336 337 ret = blk_status_to_errno(rbio->bio.bi_status); 338 bio_put(&rbio->bio); 339 340 if (ret < 0) 341 return ret; 342 343 folio_mark_uptodate(folio); 344 return 0; 345} 346 347int bch2_read_folio(struct file *file, struct folio *folio) 348{ 349 int ret; 350 351 ret = bch2_read_single_folio(folio, folio->mapping); 352 folio_unlock(folio); 353 return bch2_err_class(ret); 354} 355 356/* writepages: */ 357 358struct bch_writepage_io { 359 struct bch_inode_info *inode; 360 361 /* must be last: */ 362 struct bch_write_op op; 363}; 364 365struct bch_writepage_state { 366 struct bch_writepage_io *io; 367 struct bch_io_opts opts; 368 struct bch_folio_sector *tmp; 369 unsigned tmp_sectors; 370}; 371 372static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c, 373 struct bch_inode_info *inode) 374{ 375 struct bch_writepage_state ret = { 0 }; 376 377 bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode); 378 return ret; 379} 380 381/* 382 * Determine when a writepage io is full. We have to limit writepage bios to a 383 * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to 384 * what the bounce path in bch2_write_extent() can handle. In theory we could 385 * loosen this restriction for non-bounce I/O, but we don't have that context 386 * here. Ideally, we can up this limit and make it configurable in the future 387 * when the bounce path can be enhanced to accommodate larger source bios. 388 */ 389static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len) 390{ 391 struct bio *bio = &io->op.wbio.bio; 392 return bio_full(bio, len) || 393 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE); 394} 395 396static void bch2_writepage_io_done(struct bch_write_op *op) 397{ 398 struct bch_writepage_io *io = 399 container_of(op, struct bch_writepage_io, op); 400 struct bch_fs *c = io->op.c; 401 struct bio *bio = &io->op.wbio.bio; 402 struct folio_iter fi; 403 unsigned i; 404 405 if (io->op.error) { 406 set_bit(EI_INODE_ERROR, &io->inode->ei_flags); 407 408 bio_for_each_folio_all(fi, bio) { 409 struct bch_folio *s; 410 411 folio_set_error(fi.folio); 412 mapping_set_error(fi.folio->mapping, -EIO); 413 414 s = __bch2_folio(fi.folio); 415 spin_lock(&s->lock); 416 for (i = 0; i < folio_sectors(fi.folio); i++) 417 s->s[i].nr_replicas = 0; 418 spin_unlock(&s->lock); 419 } 420 } 421 422 if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) { 423 bio_for_each_folio_all(fi, bio) { 424 struct bch_folio *s; 425 426 s = __bch2_folio(fi.folio); 427 spin_lock(&s->lock); 428 for (i = 0; i < folio_sectors(fi.folio); i++) 429 s->s[i].nr_replicas = 0; 430 spin_unlock(&s->lock); 431 } 432 } 433 434 /* 435 * racing with fallocate can cause us to add fewer sectors than 436 * expected - but we shouldn't add more sectors than expected: 437 */ 438 WARN_ON_ONCE(io->op.i_sectors_delta > 0); 439 440 /* 441 * (error (due to going RO) halfway through a page can screw that up 442 * slightly) 443 * XXX wtf? 444 BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS); 445 */ 446 447 /* 448 * PageWriteback is effectively our ref on the inode - fixup i_blocks 449 * before calling end_page_writeback: 450 */ 451 bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); 452 453 bio_for_each_folio_all(fi, bio) { 454 struct bch_folio *s = __bch2_folio(fi.folio); 455 456 if (atomic_dec_and_test(&s->write_count)) 457 folio_end_writeback(fi.folio); 458 } 459 460 bio_put(&io->op.wbio.bio); 461} 462 463static void bch2_writepage_do_io(struct bch_writepage_state *w) 464{ 465 struct bch_writepage_io *io = w->io; 466 467 w->io = NULL; 468 closure_call(&io->op.cl, bch2_write, NULL, NULL); 469} 470 471/* 472 * Get a bch_writepage_io and add @page to it - appending to an existing one if 473 * possible, else allocating a new one: 474 */ 475static void bch2_writepage_io_alloc(struct bch_fs *c, 476 struct writeback_control *wbc, 477 struct bch_writepage_state *w, 478 struct bch_inode_info *inode, 479 u64 sector, 480 unsigned nr_replicas) 481{ 482 struct bch_write_op *op; 483 484 w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS, 485 REQ_OP_WRITE, 486 GFP_KERNEL, 487 &c->writepage_bioset), 488 struct bch_writepage_io, op.wbio.bio); 489 490 w->io->inode = inode; 491 op = &w->io->op; 492 bch2_write_op_init(op, c, w->opts); 493 op->target = w->opts.foreground_target; 494 op->nr_replicas = nr_replicas; 495 op->res.nr_replicas = nr_replicas; 496 op->write_point = writepoint_hashed(inode->ei_last_dirtied); 497 op->subvol = inode->ei_subvol; 498 op->pos = POS(inode->v.i_ino, sector); 499 op->end_io = bch2_writepage_io_done; 500 op->devs_need_flush = &inode->ei_devs_need_flush; 501 op->wbio.bio.bi_iter.bi_sector = sector; 502 op->wbio.bio.bi_opf = wbc_to_write_flags(wbc); 503} 504 505static int __bch2_writepage(struct folio *folio, 506 struct writeback_control *wbc, 507 void *data) 508{ 509 struct bch_inode_info *inode = to_bch_ei(folio->mapping->host); 510 struct bch_fs *c = inode->v.i_sb->s_fs_info; 511 struct bch_writepage_state *w = data; 512 struct bch_folio *s; 513 unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX; 514 loff_t i_size = i_size_read(&inode->v); 515 int ret; 516 517 EBUG_ON(!folio_test_uptodate(folio)); 518 519 /* Is the folio fully inside i_size? */ 520 if (folio_end_pos(folio) <= i_size) 521 goto do_io; 522 523 /* Is the folio fully outside i_size? (truncate in progress) */ 524 if (folio_pos(folio) >= i_size) { 525 folio_unlock(folio); 526 return 0; 527 } 528 529 /* 530 * The folio straddles i_size. It must be zeroed out on each and every 531 * writepage invocation because it may be mmapped. "A file is mapped 532 * in multiples of the folio size. For a file that is not a multiple of 533 * the folio size, the remaining memory is zeroed when mapped, and 534 * writes to that region are not written out to the file." 535 */ 536 folio_zero_segment(folio, 537 i_size - folio_pos(folio), 538 folio_size(folio)); 539do_io: 540 f_sectors = folio_sectors(folio); 541 s = bch2_folio(folio); 542 543 if (f_sectors > w->tmp_sectors) { 544 kfree(w->tmp); 545 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL); 546 w->tmp_sectors = f_sectors; 547 } 548 549 /* 550 * Things get really hairy with errors during writeback: 551 */ 552 ret = bch2_get_folio_disk_reservation(c, inode, folio, false); 553 BUG_ON(ret); 554 555 /* Before unlocking the page, get copy of reservations: */ 556 spin_lock(&s->lock); 557 memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors); 558 559 for (i = 0; i < f_sectors; i++) { 560 if (s->s[i].state < SECTOR_dirty) 561 continue; 562 563 nr_replicas_this_write = 564 min_t(unsigned, nr_replicas_this_write, 565 s->s[i].nr_replicas + 566 s->s[i].replicas_reserved); 567 } 568 569 for (i = 0; i < f_sectors; i++) { 570 if (s->s[i].state < SECTOR_dirty) 571 continue; 572 573 s->s[i].nr_replicas = w->opts.compression 574 ? 0 : nr_replicas_this_write; 575 576 s->s[i].replicas_reserved = 0; 577 bch2_folio_sector_set(folio, s, i, SECTOR_allocated); 578 } 579 spin_unlock(&s->lock); 580 581 BUG_ON(atomic_read(&s->write_count)); 582 atomic_set(&s->write_count, 1); 583 584 BUG_ON(folio_test_writeback(folio)); 585 folio_start_writeback(folio); 586 587 folio_unlock(folio); 588 589 offset = 0; 590 while (1) { 591 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0; 592 u64 sector; 593 594 while (offset < f_sectors && 595 w->tmp[offset].state < SECTOR_dirty) 596 offset++; 597 598 if (offset == f_sectors) 599 break; 600 601 while (offset + sectors < f_sectors && 602 w->tmp[offset + sectors].state >= SECTOR_dirty) { 603 reserved_sectors += w->tmp[offset + sectors].replicas_reserved; 604 dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty; 605 sectors++; 606 } 607 BUG_ON(!sectors); 608 609 sector = folio_sector(folio) + offset; 610 611 if (w->io && 612 (w->io->op.res.nr_replicas != nr_replicas_this_write || 613 bch_io_full(w->io, sectors << 9) || 614 bio_end_sector(&w->io->op.wbio.bio) != sector)) 615 bch2_writepage_do_io(w); 616 617 if (!w->io) 618 bch2_writepage_io_alloc(c, wbc, w, inode, sector, 619 nr_replicas_this_write); 620 621 atomic_inc(&s->write_count); 622 623 BUG_ON(inode != w->io->inode); 624 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio, 625 sectors << 9, offset << 9)); 626 627 /* Check for writing past i_size: */ 628 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) > 629 round_up(i_size, block_bytes(c)) && 630 !test_bit(BCH_FS_emergency_ro, &c->flags), 631 "writing past i_size: %llu > %llu (unrounded %llu)\n", 632 bio_end_sector(&w->io->op.wbio.bio) << 9, 633 round_up(i_size, block_bytes(c)), 634 i_size); 635 636 w->io->op.res.sectors += reserved_sectors; 637 w->io->op.i_sectors_delta -= dirty_sectors; 638 w->io->op.new_i_size = i_size; 639 640 offset += sectors; 641 } 642 643 if (atomic_dec_and_test(&s->write_count)) 644 folio_end_writeback(folio); 645 646 return 0; 647} 648 649int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc) 650{ 651 struct bch_fs *c = mapping->host->i_sb->s_fs_info; 652 struct bch_writepage_state w = 653 bch_writepage_state_init(c, to_bch_ei(mapping->host)); 654 struct blk_plug plug; 655 int ret; 656 657 blk_start_plug(&plug); 658 ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w); 659 if (w.io) 660 bch2_writepage_do_io(&w); 661 blk_finish_plug(&plug); 662 kfree(w.tmp); 663 return bch2_err_class(ret); 664} 665 666/* buffered writes: */ 667 668int bch2_write_begin(struct file *file, struct address_space *mapping, 669 loff_t pos, unsigned len, 670 struct page **pagep, void **fsdata) 671{ 672 struct bch_inode_info *inode = to_bch_ei(mapping->host); 673 struct bch_fs *c = inode->v.i_sb->s_fs_info; 674 struct bch2_folio_reservation *res; 675 struct folio *folio; 676 unsigned offset; 677 int ret = -ENOMEM; 678 679 res = kmalloc(sizeof(*res), GFP_KERNEL); 680 if (!res) 681 return -ENOMEM; 682 683 bch2_folio_reservation_init(c, inode, res); 684 *fsdata = res; 685 686 bch2_pagecache_add_get(inode); 687 688 folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT, 689 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE, 690 mapping_gfp_mask(mapping)); 691 if (IS_ERR_OR_NULL(folio)) 692 goto err_unlock; 693 694 offset = pos - folio_pos(folio); 695 len = min_t(size_t, len, folio_end_pos(folio) - pos); 696 697 if (folio_test_uptodate(folio)) 698 goto out; 699 700 /* If we're writing entire folio, don't need to read it in first: */ 701 if (!offset && len == folio_size(folio)) 702 goto out; 703 704 if (!offset && pos + len >= inode->v.i_size) { 705 folio_zero_segment(folio, len, folio_size(folio)); 706 flush_dcache_folio(folio); 707 goto out; 708 } 709 710 if (folio_pos(folio) >= inode->v.i_size) { 711 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio)); 712 flush_dcache_folio(folio); 713 goto out; 714 } 715readpage: 716 ret = bch2_read_single_folio(folio, mapping); 717 if (ret) 718 goto err; 719out: 720 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1); 721 if (ret) 722 goto err; 723 724 ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len); 725 if (ret) { 726 if (!folio_test_uptodate(folio)) { 727 /* 728 * If the folio hasn't been read in, we won't know if we 729 * actually need a reservation - we don't actually need 730 * to read here, we just need to check if the folio is 731 * fully backed by uncompressed data: 732 */ 733 goto readpage; 734 } 735 736 goto err; 737 } 738 739 *pagep = &folio->page; 740 return 0; 741err: 742 folio_unlock(folio); 743 folio_put(folio); 744 *pagep = NULL; 745err_unlock: 746 bch2_pagecache_add_put(inode); 747 kfree(res); 748 *fsdata = NULL; 749 return bch2_err_class(ret); 750} 751 752int bch2_write_end(struct file *file, struct address_space *mapping, 753 loff_t pos, unsigned len, unsigned copied, 754 struct page *page, void *fsdata) 755{ 756 struct bch_inode_info *inode = to_bch_ei(mapping->host); 757 struct bch_fs *c = inode->v.i_sb->s_fs_info; 758 struct bch2_folio_reservation *res = fsdata; 759 struct folio *folio = page_folio(page); 760 unsigned offset = pos - folio_pos(folio); 761 762 lockdep_assert_held(&inode->v.i_rwsem); 763 BUG_ON(offset + copied > folio_size(folio)); 764 765 if (unlikely(copied < len && !folio_test_uptodate(folio))) { 766 /* 767 * The folio needs to be read in, but that would destroy 768 * our partial write - simplest thing is to just force 769 * userspace to redo the write: 770 */ 771 folio_zero_range(folio, 0, folio_size(folio)); 772 flush_dcache_folio(folio); 773 copied = 0; 774 } 775 776 spin_lock(&inode->v.i_lock); 777 if (pos + copied > inode->v.i_size) 778 i_size_write(&inode->v, pos + copied); 779 spin_unlock(&inode->v.i_lock); 780 781 if (copied) { 782 if (!folio_test_uptodate(folio)) 783 folio_mark_uptodate(folio); 784 785 bch2_set_folio_dirty(c, inode, folio, res, offset, copied); 786 787 inode->ei_last_dirtied = (unsigned long) current; 788 } 789 790 folio_unlock(folio); 791 folio_put(folio); 792 bch2_pagecache_add_put(inode); 793 794 bch2_folio_reservation_put(c, inode, res); 795 kfree(res); 796 797 return copied; 798} 799 800static noinline void folios_trunc(folios *fs, struct folio **fi) 801{ 802 while (fs->data + fs->nr > fi) { 803 struct folio *f = darray_pop(fs); 804 805 folio_unlock(f); 806 folio_put(f); 807 } 808} 809 810static int __bch2_buffered_write(struct bch_inode_info *inode, 811 struct address_space *mapping, 812 struct iov_iter *iter, 813 loff_t pos, unsigned len, 814 bool inode_locked) 815{ 816 struct bch_fs *c = inode->v.i_sb->s_fs_info; 817 struct bch2_folio_reservation res; 818 folios fs; 819 struct folio *f; 820 unsigned copied = 0, f_offset, f_copied; 821 u64 end = pos + len, f_pos, f_len; 822 loff_t last_folio_pos = inode->v.i_size; 823 int ret = 0; 824 825 BUG_ON(!len); 826 827 bch2_folio_reservation_init(c, inode, &res); 828 darray_init(&fs); 829 830 ret = bch2_filemap_get_contig_folios_d(mapping, pos, end, 831 FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT, 832 mapping_gfp_mask(mapping), 833 &fs); 834 if (ret) 835 goto out; 836 837 BUG_ON(!fs.nr); 838 839 /* 840 * If we're not using the inode lock, we need to lock all the folios for 841 * atomiticity of writes vs. other writes: 842 */ 843 if (!inode_locked && folio_end_pos(darray_last(fs)) < end) { 844 ret = -BCH_ERR_need_inode_lock; 845 goto out; 846 } 847 848 f = darray_first(fs); 849 if (pos != folio_pos(f) && !folio_test_uptodate(f)) { 850 ret = bch2_read_single_folio(f, mapping); 851 if (ret) 852 goto out; 853 } 854 855 f = darray_last(fs); 856 end = min(end, folio_end_pos(f)); 857 last_folio_pos = folio_pos(f); 858 if (end != folio_end_pos(f) && !folio_test_uptodate(f)) { 859 if (end >= inode->v.i_size) { 860 folio_zero_range(f, 0, folio_size(f)); 861 } else { 862 ret = bch2_read_single_folio(f, mapping); 863 if (ret) 864 goto out; 865 } 866 } 867 868 ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr); 869 if (ret) 870 goto out; 871 872 f_pos = pos; 873 f_offset = pos - folio_pos(darray_first(fs)); 874 darray_for_each(fs, fi) { 875 f = *fi; 876 f_len = min(end, folio_end_pos(f)) - f_pos; 877 878 /* 879 * XXX: per POSIX and fstests generic/275, on -ENOSPC we're 880 * supposed to write as much as we have disk space for. 881 * 882 * On failure here we should still write out a partial page if 883 * we aren't completely out of disk space - we don't do that 884 * yet: 885 */ 886 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len); 887 if (unlikely(ret)) { 888 folios_trunc(&fs, fi); 889 if (!fs.nr) 890 goto out; 891 892 end = min(end, folio_end_pos(darray_last(fs))); 893 break; 894 } 895 896 f_pos = folio_end_pos(f); 897 f_offset = 0; 898 } 899 900 if (mapping_writably_mapped(mapping)) 901 darray_for_each(fs, fi) 902 flush_dcache_folio(*fi); 903 904 f_pos = pos; 905 f_offset = pos - folio_pos(darray_first(fs)); 906 darray_for_each(fs, fi) { 907 f = *fi; 908 f_len = min(end, folio_end_pos(f)) - f_pos; 909 f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter); 910 if (!f_copied) { 911 folios_trunc(&fs, fi); 912 break; 913 } 914 915 if (!folio_test_uptodate(f) && 916 f_copied != folio_size(f) && 917 pos + copied + f_copied < inode->v.i_size) { 918 iov_iter_revert(iter, f_copied); 919 folio_zero_range(f, 0, folio_size(f)); 920 folios_trunc(&fs, fi); 921 break; 922 } 923 924 flush_dcache_folio(f); 925 copied += f_copied; 926 927 if (f_copied != f_len) { 928 folios_trunc(&fs, fi + 1); 929 break; 930 } 931 932 f_pos = folio_end_pos(f); 933 f_offset = 0; 934 } 935 936 if (!copied) 937 goto out; 938 939 end = pos + copied; 940 941 spin_lock(&inode->v.i_lock); 942 if (end > inode->v.i_size) { 943 BUG_ON(!inode_locked); 944 i_size_write(&inode->v, end); 945 } 946 spin_unlock(&inode->v.i_lock); 947 948 f_pos = pos; 949 f_offset = pos - folio_pos(darray_first(fs)); 950 darray_for_each(fs, fi) { 951 f = *fi; 952 f_len = min(end, folio_end_pos(f)) - f_pos; 953 954 if (!folio_test_uptodate(f)) 955 folio_mark_uptodate(f); 956 957 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len); 958 959 f_pos = folio_end_pos(f); 960 f_offset = 0; 961 } 962 963 inode->ei_last_dirtied = (unsigned long) current; 964out: 965 darray_for_each(fs, fi) { 966 folio_unlock(*fi); 967 folio_put(*fi); 968 } 969 970 /* 971 * If the last folio added to the mapping starts beyond current EOF, we 972 * performed a short write but left around at least one post-EOF folio. 973 * Clean up the mapping before we return. 974 */ 975 if (last_folio_pos >= inode->v.i_size) 976 truncate_pagecache(&inode->v, inode->v.i_size); 977 978 darray_exit(&fs); 979 bch2_folio_reservation_put(c, inode, &res); 980 981 return copied ?: ret; 982} 983 984static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter) 985{ 986 struct file *file = iocb->ki_filp; 987 struct address_space *mapping = file->f_mapping; 988 struct bch_inode_info *inode = file_bch_inode(file); 989 loff_t pos; 990 bool inode_locked = false; 991 ssize_t written = 0, written2 = 0, ret = 0; 992 993 /* 994 * We don't take the inode lock unless i_size will be changing. Folio 995 * locks provide exclusion with other writes, and the pagecache add lock 996 * provides exclusion with truncate and hole punching. 997 * 998 * There is one nasty corner case where atomicity would be broken 999 * without great care: when copying data from userspace to the page 1000 * cache, we do that with faults disable - a page fault would recurse 1001 * back into the filesystem, taking filesystem locks again, and 1002 * deadlock; so it's done with faults disabled, and we fault in the user 1003 * buffer when we aren't holding locks. 1004 * 1005 * If we do part of the write, but we then race and in the userspace 1006 * buffer have been evicted and are no longer resident, then we have to 1007 * drop our folio locks to re-fault them in, breaking write atomicity. 1008 * 1009 * To fix this, we restart the write from the start, if we weren't 1010 * holding the inode lock. 1011 * 1012 * There is another wrinkle after that; if we restart the write from the 1013 * start, and then get an unrecoverable error, we _cannot_ claim to 1014 * userspace that we did not write data we actually did - so we must 1015 * track (written2) the most we ever wrote. 1016 */ 1017 1018 if ((iocb->ki_flags & IOCB_APPEND) || 1019 (iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v))) { 1020 inode_lock(&inode->v); 1021 inode_locked = true; 1022 } 1023 1024 ret = generic_write_checks(iocb, iter); 1025 if (ret <= 0) 1026 goto unlock; 1027 1028 ret = file_remove_privs_flags(file, !inode_locked ? IOCB_NOWAIT : 0); 1029 if (ret) { 1030 if (!inode_locked) { 1031 inode_lock(&inode->v); 1032 inode_locked = true; 1033 ret = file_remove_privs_flags(file, 0); 1034 } 1035 if (ret) 1036 goto unlock; 1037 } 1038 1039 ret = file_update_time(file); 1040 if (ret) 1041 goto unlock; 1042 1043 pos = iocb->ki_pos; 1044 1045 bch2_pagecache_add_get(inode); 1046 1047 if (!inode_locked && 1048 (iocb->ki_pos + iov_iter_count(iter) > i_size_read(&inode->v))) 1049 goto get_inode_lock; 1050 1051 do { 1052 unsigned offset = pos & (PAGE_SIZE - 1); 1053 unsigned bytes = iov_iter_count(iter); 1054again: 1055 /* 1056 * Bring in the user page that we will copy from _first_. 1057 * Otherwise there's a nasty deadlock on copying from the 1058 * same page as we're writing to, without it being marked 1059 * up-to-date. 1060 * 1061 * Not only is this an optimisation, but it is also required 1062 * to check that the address is actually valid, when atomic 1063 * usercopies are used, below. 1064 */ 1065 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1066 bytes = min_t(unsigned long, iov_iter_count(iter), 1067 PAGE_SIZE - offset); 1068 1069 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1070 ret = -EFAULT; 1071 break; 1072 } 1073 } 1074 1075 if (unlikely(bytes != iov_iter_count(iter) && !inode_locked)) 1076 goto get_inode_lock; 1077 1078 if (unlikely(fatal_signal_pending(current))) { 1079 ret = -EINTR; 1080 break; 1081 } 1082 1083 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes, inode_locked); 1084 if (ret == -BCH_ERR_need_inode_lock) 1085 goto get_inode_lock; 1086 if (unlikely(ret < 0)) 1087 break; 1088 1089 cond_resched(); 1090 1091 if (unlikely(ret == 0)) { 1092 /* 1093 * If we were unable to copy any data at all, we must 1094 * fall back to a single segment length write. 1095 * 1096 * If we didn't fallback here, we could livelock 1097 * because not all segments in the iov can be copied at 1098 * once without a pagefault. 1099 */ 1100 bytes = min_t(unsigned long, PAGE_SIZE - offset, 1101 iov_iter_single_seg_count(iter)); 1102 goto again; 1103 } 1104 pos += ret; 1105 written += ret; 1106 written2 = max(written, written2); 1107 1108 if (ret != bytes && !inode_locked) 1109 goto get_inode_lock; 1110 ret = 0; 1111 1112 balance_dirty_pages_ratelimited(mapping); 1113 1114 if (0) { 1115get_inode_lock: 1116 bch2_pagecache_add_put(inode); 1117 inode_lock(&inode->v); 1118 inode_locked = true; 1119 bch2_pagecache_add_get(inode); 1120 1121 iov_iter_revert(iter, written); 1122 pos -= written; 1123 written = 0; 1124 ret = 0; 1125 } 1126 } while (iov_iter_count(iter)); 1127 bch2_pagecache_add_put(inode); 1128unlock: 1129 if (inode_locked) 1130 inode_unlock(&inode->v); 1131 1132 iocb->ki_pos += written; 1133 1134 ret = max(written, written2) ?: ret; 1135 if (ret > 0) 1136 ret = generic_write_sync(iocb, ret); 1137 return ret; 1138} 1139 1140ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *iter) 1141{ 1142 ssize_t ret = iocb->ki_flags & IOCB_DIRECT 1143 ? bch2_direct_write(iocb, iter) 1144 : bch2_buffered_write(iocb, iter); 1145 1146 return bch2_err_class(ret); 1147} 1148 1149void bch2_fs_fs_io_buffered_exit(struct bch_fs *c) 1150{ 1151 bioset_exit(&c->writepage_bioset); 1152} 1153 1154int bch2_fs_fs_io_buffered_init(struct bch_fs *c) 1155{ 1156 if (bioset_init(&c->writepage_bioset, 1157 4, offsetof(struct bch_writepage_io, op.wbio.bio), 1158 BIOSET_NEED_BVECS)) 1159 return -BCH_ERR_ENOMEM_writepage_bioset_init; 1160 1161 return 0; 1162} 1163 1164#endif /* NO_BCACHEFS_FS */ 1165