1// SPDX-License-Identifier: GPL-2.0-only 2/* Network filesystem high-level write support. 3 * 4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8#include <linux/export.h> 9#include <linux/fs.h> 10#include <linux/mm.h> 11#include <linux/pagemap.h> 12#include <linux/slab.h> 13#include <linux/pagevec.h> 14#include "internal.h" 15 16/* 17 * Determined write method. Adjust netfs_folio_traces if this is changed. 18 */ 19enum netfs_how_to_modify { 20 NETFS_FOLIO_IS_UPTODATE, /* Folio is uptodate already */ 21 NETFS_JUST_PREFETCH, /* We have to read the folio anyway */ 22 NETFS_WHOLE_FOLIO_MODIFY, /* We're going to overwrite the whole folio */ 23 NETFS_MODIFY_AND_CLEAR, /* We can assume there is no data to be downloaded. */ 24 NETFS_STREAMING_WRITE, /* Store incomplete data in non-uptodate page. */ 25 NETFS_STREAMING_WRITE_CONT, /* Continue streaming write. */ 26 NETFS_FLUSH_CONTENT, /* Flush incompatible content. */ 27}; 28 29static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq); 30 31static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group) 32{ 33 if (netfs_group && !folio_get_private(folio)) 34 folio_attach_private(folio, netfs_get_group(netfs_group)); 35} 36 37#if IS_ENABLED(CONFIG_FSCACHE) 38static void netfs_folio_start_fscache(bool caching, struct folio *folio) 39{ 40 if (caching) 41 folio_start_fscache(folio); 42} 43#else 44static void netfs_folio_start_fscache(bool caching, struct folio *folio) 45{ 46} 47#endif 48 49/* 50 * Decide how we should modify a folio. We might be attempting to do 51 * write-streaming, in which case we don't want to a local RMW cycle if we can 52 * avoid it. If we're doing local caching or content crypto, we award that 53 * priority over avoiding RMW. If the file is open readably, then we also 54 * assume that we may want to read what we wrote. 55 */ 56static enum netfs_how_to_modify netfs_how_to_modify(struct netfs_inode *ctx, 57 struct file *file, 58 struct folio *folio, 59 void *netfs_group, 60 size_t flen, 61 size_t offset, 62 size_t len, 63 bool maybe_trouble) 64{ 65 struct netfs_folio *finfo = netfs_folio_info(folio); 66 loff_t pos = folio_file_pos(folio); 67 68 _enter(""); 69 70 if (netfs_folio_group(folio) != netfs_group) 71 return NETFS_FLUSH_CONTENT; 72 73 if (folio_test_uptodate(folio)) 74 return NETFS_FOLIO_IS_UPTODATE; 75 76 if (pos >= ctx->zero_point) 77 return NETFS_MODIFY_AND_CLEAR; 78 79 if (!maybe_trouble && offset == 0 && len >= flen) 80 return NETFS_WHOLE_FOLIO_MODIFY; 81 82 if (file->f_mode & FMODE_READ) 83 goto no_write_streaming; 84 if (test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags)) 85 goto no_write_streaming; 86 87 if (netfs_is_cache_enabled(ctx)) { 88 /* We don't want to get a streaming write on a file that loses 89 * caching service temporarily because the backing store got 90 * culled. 91 */ 92 if (!test_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags)) 93 set_bit(NETFS_ICTX_NO_WRITE_STREAMING, &ctx->flags); 94 goto no_write_streaming; 95 } 96 97 if (!finfo) 98 return NETFS_STREAMING_WRITE; 99 100 /* We can continue a streaming write only if it continues on from the 101 * previous. If it overlaps, we must flush lest we suffer a partial 102 * copy and disjoint dirty regions. 103 */ 104 if (offset == finfo->dirty_offset + finfo->dirty_len) 105 return NETFS_STREAMING_WRITE_CONT; 106 return NETFS_FLUSH_CONTENT; 107 108no_write_streaming: 109 if (finfo) { 110 netfs_stat(&netfs_n_wh_wstream_conflict); 111 return NETFS_FLUSH_CONTENT; 112 } 113 return NETFS_JUST_PREFETCH; 114} 115 116/* 117 * Grab a folio for writing and lock it. Attempt to allocate as large a folio 118 * as possible to hold as much of the remaining length as possible in one go. 119 */ 120static struct folio *netfs_grab_folio_for_write(struct address_space *mapping, 121 loff_t pos, size_t part) 122{ 123 pgoff_t index = pos / PAGE_SIZE; 124 fgf_t fgp_flags = FGP_WRITEBEGIN; 125 126 if (mapping_large_folio_support(mapping)) 127 fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part); 128 129 return __filemap_get_folio(mapping, index, fgp_flags, 130 mapping_gfp_mask(mapping)); 131} 132 133/** 134 * netfs_perform_write - Copy data into the pagecache. 135 * @iocb: The operation parameters 136 * @iter: The source buffer 137 * @netfs_group: Grouping for dirty pages (eg. ceph snaps). 138 * 139 * Copy data into pagecache pages attached to the inode specified by @iocb. 140 * The caller must hold appropriate inode locks. 141 * 142 * Dirty pages are tagged with a netfs_folio struct if they're not up to date 143 * to indicate the range modified. Dirty pages may also be tagged with a 144 * netfs-specific grouping such that data from an old group gets flushed before 145 * a new one is started. 146 */ 147ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, 148 struct netfs_group *netfs_group) 149{ 150 struct file *file = iocb->ki_filp; 151 struct inode *inode = file_inode(file); 152 struct address_space *mapping = inode->i_mapping; 153 struct netfs_inode *ctx = netfs_inode(inode); 154 struct writeback_control wbc = { 155 .sync_mode = WB_SYNC_NONE, 156 .for_sync = true, 157 .nr_to_write = LONG_MAX, 158 .range_start = iocb->ki_pos, 159 .range_end = iocb->ki_pos + iter->count, 160 }; 161 struct netfs_io_request *wreq = NULL; 162 struct netfs_folio *finfo; 163 struct folio *folio; 164 enum netfs_how_to_modify howto; 165 enum netfs_folio_trace trace; 166 unsigned int bdp_flags = (iocb->ki_flags & IOCB_SYNC) ? 0: BDP_ASYNC; 167 ssize_t written = 0, ret; 168 loff_t i_size, pos = iocb->ki_pos, from, to; 169 size_t max_chunk = PAGE_SIZE << MAX_PAGECACHE_ORDER; 170 bool maybe_trouble = false; 171 172 if (unlikely(test_bit(NETFS_ICTX_WRITETHROUGH, &ctx->flags) || 173 iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) 174 ) { 175 if (pos < i_size_read(inode)) { 176 ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count); 177 if (ret < 0) { 178 goto out; 179 } 180 } 181 182 wbc_attach_fdatawrite_inode(&wbc, mapping->host); 183 184 wreq = netfs_begin_writethrough(iocb, iter->count); 185 if (IS_ERR(wreq)) { 186 wbc_detach_inode(&wbc); 187 ret = PTR_ERR(wreq); 188 wreq = NULL; 189 goto out; 190 } 191 if (!is_sync_kiocb(iocb)) 192 wreq->iocb = iocb; 193 wreq->cleanup = netfs_cleanup_buffered_write; 194 } 195 196 do { 197 size_t flen; 198 size_t offset; /* Offset into pagecache folio */ 199 size_t part; /* Bytes to write to folio */ 200 size_t copied; /* Bytes copied from user */ 201 202 ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags); 203 if (unlikely(ret < 0)) 204 break; 205 206 offset = pos & (max_chunk - 1); 207 part = min(max_chunk - offset, iov_iter_count(iter)); 208 209 /* Bring in the user pages that we will copy from _first_ lest 210 * we hit a nasty deadlock on copying from the same page as 211 * we're writing to, without it being marked uptodate. 212 * 213 * Not only is this an optimisation, but it is also required to 214 * check that the address is actually valid, when atomic 215 * usercopies are used below. 216 * 217 * We rely on the page being held onto long enough by the LRU 218 * that we can grab it below if this causes it to be read. 219 */ 220 ret = -EFAULT; 221 if (unlikely(fault_in_iov_iter_readable(iter, part) == part)) 222 break; 223 224 folio = netfs_grab_folio_for_write(mapping, pos, part); 225 if (IS_ERR(folio)) { 226 ret = PTR_ERR(folio); 227 break; 228 } 229 230 flen = folio_size(folio); 231 offset = pos & (flen - 1); 232 part = min_t(size_t, flen - offset, part); 233 234 if (signal_pending(current)) { 235 ret = written ? -EINTR : -ERESTARTSYS; 236 goto error_folio_unlock; 237 } 238 239 /* See if we need to prefetch the area we're going to modify. 240 * We need to do this before we get a lock on the folio in case 241 * there's more than one writer competing for the same cache 242 * block. 243 */ 244 howto = netfs_how_to_modify(ctx, file, folio, netfs_group, 245 flen, offset, part, maybe_trouble); 246 _debug("howto %u", howto); 247 switch (howto) { 248 case NETFS_JUST_PREFETCH: 249 ret = netfs_prefetch_for_write(file, folio, offset, part); 250 if (ret < 0) { 251 _debug("prefetch = %zd", ret); 252 goto error_folio_unlock; 253 } 254 break; 255 case NETFS_FOLIO_IS_UPTODATE: 256 case NETFS_WHOLE_FOLIO_MODIFY: 257 case NETFS_STREAMING_WRITE_CONT: 258 break; 259 case NETFS_MODIFY_AND_CLEAR: 260 zero_user_segment(&folio->page, 0, offset); 261 break; 262 case NETFS_STREAMING_WRITE: 263 ret = -EIO; 264 if (WARN_ON(folio_get_private(folio))) 265 goto error_folio_unlock; 266 break; 267 case NETFS_FLUSH_CONTENT: 268 trace_netfs_folio(folio, netfs_flush_content); 269 from = folio_pos(folio); 270 to = from + folio_size(folio) - 1; 271 folio_unlock(folio); 272 folio_put(folio); 273 ret = filemap_write_and_wait_range(mapping, from, to); 274 if (ret < 0) 275 goto error_folio_unlock; 276 continue; 277 } 278 279 if (mapping_writably_mapped(mapping)) 280 flush_dcache_folio(folio); 281 282 copied = copy_folio_from_iter_atomic(folio, offset, part, iter); 283 284 flush_dcache_folio(folio); 285 286 /* Deal with a (partially) failed copy */ 287 if (copied == 0) { 288 ret = -EFAULT; 289 goto error_folio_unlock; 290 } 291 292 trace = (enum netfs_folio_trace)howto; 293 switch (howto) { 294 case NETFS_FOLIO_IS_UPTODATE: 295 case NETFS_JUST_PREFETCH: 296 netfs_set_group(folio, netfs_group); 297 break; 298 case NETFS_MODIFY_AND_CLEAR: 299 zero_user_segment(&folio->page, offset + copied, flen); 300 netfs_set_group(folio, netfs_group); 301 folio_mark_uptodate(folio); 302 break; 303 case NETFS_WHOLE_FOLIO_MODIFY: 304 if (unlikely(copied < part)) { 305 maybe_trouble = true; 306 iov_iter_revert(iter, copied); 307 copied = 0; 308 goto retry; 309 } 310 netfs_set_group(folio, netfs_group); 311 folio_mark_uptodate(folio); 312 break; 313 case NETFS_STREAMING_WRITE: 314 if (offset == 0 && copied == flen) { 315 netfs_set_group(folio, netfs_group); 316 folio_mark_uptodate(folio); 317 trace = netfs_streaming_filled_page; 318 break; 319 } 320 finfo = kzalloc(sizeof(*finfo), GFP_KERNEL); 321 if (!finfo) { 322 iov_iter_revert(iter, copied); 323 ret = -ENOMEM; 324 goto error_folio_unlock; 325 } 326 finfo->netfs_group = netfs_get_group(netfs_group); 327 finfo->dirty_offset = offset; 328 finfo->dirty_len = copied; 329 folio_attach_private(folio, (void *)((unsigned long)finfo | 330 NETFS_FOLIO_INFO)); 331 break; 332 case NETFS_STREAMING_WRITE_CONT: 333 finfo = netfs_folio_info(folio); 334 finfo->dirty_len += copied; 335 if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) { 336 if (finfo->netfs_group) 337 folio_change_private(folio, finfo->netfs_group); 338 else 339 folio_detach_private(folio); 340 folio_mark_uptodate(folio); 341 kfree(finfo); 342 trace = netfs_streaming_cont_filled_page; 343 } 344 break; 345 default: 346 WARN(true, "Unexpected modify type %u ix=%lx\n", 347 howto, folio->index); 348 ret = -EIO; 349 goto error_folio_unlock; 350 } 351 352 trace_netfs_folio(folio, trace); 353 354 /* Update the inode size if we moved the EOF marker */ 355 i_size = i_size_read(inode); 356 pos += copied; 357 if (pos > i_size) { 358 if (ctx->ops->update_i_size) { 359 ctx->ops->update_i_size(inode, pos); 360 } else { 361 i_size_write(inode, pos); 362#if IS_ENABLED(CONFIG_FSCACHE) 363 fscache_update_cookie(ctx->cache, NULL, &pos); 364#endif 365 } 366 } 367 written += copied; 368 369 if (likely(!wreq)) { 370 folio_mark_dirty(folio); 371 } else { 372 if (folio_test_dirty(folio)) 373 /* Sigh. mmap. */ 374 folio_clear_dirty_for_io(folio); 375 /* We make multiple writes to the folio... */ 376 if (!folio_test_writeback(folio)) { 377 folio_wait_fscache(folio); 378 folio_start_writeback(folio); 379 folio_start_fscache(folio); 380 if (wreq->iter.count == 0) 381 trace_netfs_folio(folio, netfs_folio_trace_wthru); 382 else 383 trace_netfs_folio(folio, netfs_folio_trace_wthru_plus); 384 } 385 netfs_advance_writethrough(wreq, copied, 386 offset + copied == flen); 387 } 388 retry: 389 folio_unlock(folio); 390 folio_put(folio); 391 folio = NULL; 392 393 cond_resched(); 394 } while (iov_iter_count(iter)); 395 396out: 397 if (unlikely(wreq)) { 398 ret = netfs_end_writethrough(wreq, iocb); 399 wbc_detach_inode(&wbc); 400 if (ret == -EIOCBQUEUED) 401 return ret; 402 } 403 404 iocb->ki_pos += written; 405 _leave(" = %zd [%zd]", written, ret); 406 return written ? written : ret; 407 408error_folio_unlock: 409 folio_unlock(folio); 410 folio_put(folio); 411 goto out; 412} 413EXPORT_SYMBOL(netfs_perform_write); 414 415/** 416 * netfs_buffered_write_iter_locked - write data to a file 417 * @iocb: IO state structure (file, offset, etc.) 418 * @from: iov_iter with data to write 419 * @netfs_group: Grouping for dirty pages (eg. ceph snaps). 420 * 421 * This function does all the work needed for actually writing data to a 422 * file. It does all basic checks, removes SUID from the file, updates 423 * modification times and calls proper subroutines depending on whether we 424 * do direct IO or a standard buffered write. 425 * 426 * The caller must hold appropriate locks around this function and have called 427 * generic_write_checks() already. The caller is also responsible for doing 428 * any necessary syncing afterwards. 429 * 430 * This function does *not* take care of syncing data in case of O_SYNC write. 431 * A caller has to handle it. This is mainly due to the fact that we want to 432 * avoid syncing under i_rwsem. 433 * 434 * Return: 435 * * number of bytes written, even for truncated writes 436 * * negative error code if no data has been written at all 437 */ 438ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from, 439 struct netfs_group *netfs_group) 440{ 441 struct file *file = iocb->ki_filp; 442 ssize_t ret; 443 444 trace_netfs_write_iter(iocb, from); 445 446 ret = file_remove_privs(file); 447 if (ret) 448 return ret; 449 450 ret = file_update_time(file); 451 if (ret) 452 return ret; 453 454 return netfs_perform_write(iocb, from, netfs_group); 455} 456EXPORT_SYMBOL(netfs_buffered_write_iter_locked); 457 458/** 459 * netfs_file_write_iter - write data to a file 460 * @iocb: IO state structure 461 * @from: iov_iter with data to write 462 * 463 * Perform a write to a file, writing into the pagecache if possible and doing 464 * an unbuffered write instead if not. 465 * 466 * Return: 467 * * Negative error code if no data has been written at all of 468 * vfs_fsync_range() failed for a synchronous write 469 * * Number of bytes written, even for truncated writes 470 */ 471ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 472{ 473 struct file *file = iocb->ki_filp; 474 struct inode *inode = file->f_mapping->host; 475 struct netfs_inode *ictx = netfs_inode(inode); 476 ssize_t ret; 477 478 _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode)); 479 480 if (!iov_iter_count(from)) 481 return 0; 482 483 if ((iocb->ki_flags & IOCB_DIRECT) || 484 test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags)) 485 return netfs_unbuffered_write_iter(iocb, from); 486 487 ret = netfs_start_io_write(inode); 488 if (ret < 0) 489 return ret; 490 491 ret = generic_write_checks(iocb, from); 492 if (ret > 0) 493 ret = netfs_buffered_write_iter_locked(iocb, from, NULL); 494 netfs_end_io_write(inode); 495 if (ret > 0) 496 ret = generic_write_sync(iocb, ret); 497 return ret; 498} 499EXPORT_SYMBOL(netfs_file_write_iter); 500 501/* 502 * Notification that a previously read-only page is about to become writable. 503 * Note that the caller indicates a single page of a multipage folio. 504 */ 505vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group) 506{ 507 struct folio *folio = page_folio(vmf->page); 508 struct file *file = vmf->vma->vm_file; 509 struct inode *inode = file_inode(file); 510 vm_fault_t ret = VM_FAULT_RETRY; 511 int err; 512 513 _enter("%lx", folio->index); 514 515 sb_start_pagefault(inode->i_sb); 516 517 if (folio_wait_writeback_killable(folio)) 518 goto out; 519 520 if (folio_lock_killable(folio) < 0) 521 goto out; 522 523 /* Can we see a streaming write here? */ 524 if (WARN_ON(!folio_test_uptodate(folio))) { 525 ret = VM_FAULT_SIGBUS | VM_FAULT_LOCKED; 526 goto out; 527 } 528 529 if (netfs_folio_group(folio) != netfs_group) { 530 folio_unlock(folio); 531 err = filemap_fdatawait_range(inode->i_mapping, 532 folio_pos(folio), 533 folio_pos(folio) + folio_size(folio)); 534 switch (err) { 535 case 0: 536 ret = VM_FAULT_RETRY; 537 goto out; 538 case -ENOMEM: 539 ret = VM_FAULT_OOM; 540 goto out; 541 default: 542 ret = VM_FAULT_SIGBUS; 543 goto out; 544 } 545 } 546 547 if (folio_test_dirty(folio)) 548 trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus); 549 else 550 trace_netfs_folio(folio, netfs_folio_trace_mkwrite); 551 netfs_set_group(folio, netfs_group); 552 file_update_time(file); 553 ret = VM_FAULT_LOCKED; 554out: 555 sb_end_pagefault(inode->i_sb); 556 return ret; 557} 558EXPORT_SYMBOL(netfs_page_mkwrite); 559 560/* 561 * Kill all the pages in the given range 562 */ 563static void netfs_kill_pages(struct address_space *mapping, 564 loff_t start, loff_t len) 565{ 566 struct folio *folio; 567 pgoff_t index = start / PAGE_SIZE; 568 pgoff_t last = (start + len - 1) / PAGE_SIZE, next; 569 570 _enter("%llx-%llx", start, start + len - 1); 571 572 do { 573 _debug("kill %lx (to %lx)", index, last); 574 575 folio = filemap_get_folio(mapping, index); 576 if (IS_ERR(folio)) { 577 next = index + 1; 578 continue; 579 } 580 581 next = folio_next_index(folio); 582 583 trace_netfs_folio(folio, netfs_folio_trace_kill); 584 folio_clear_uptodate(folio); 585 if (folio_test_fscache(folio)) 586 folio_end_fscache(folio); 587 folio_end_writeback(folio); 588 folio_lock(folio); 589 generic_error_remove_folio(mapping, folio); 590 folio_unlock(folio); 591 folio_put(folio); 592 593 } while (index = next, index <= last); 594 595 _leave(""); 596} 597 598/* 599 * Redirty all the pages in a given range. 600 */ 601static void netfs_redirty_pages(struct address_space *mapping, 602 loff_t start, loff_t len) 603{ 604 struct folio *folio; 605 pgoff_t index = start / PAGE_SIZE; 606 pgoff_t last = (start + len - 1) / PAGE_SIZE, next; 607 608 _enter("%llx-%llx", start, start + len - 1); 609 610 do { 611 _debug("redirty %llx @%llx", len, start); 612 613 folio = filemap_get_folio(mapping, index); 614 if (IS_ERR(folio)) { 615 next = index + 1; 616 continue; 617 } 618 619 next = folio_next_index(folio); 620 trace_netfs_folio(folio, netfs_folio_trace_redirty); 621 filemap_dirty_folio(mapping, folio); 622 if (folio_test_fscache(folio)) 623 folio_end_fscache(folio); 624 folio_end_writeback(folio); 625 folio_put(folio); 626 } while (index = next, index <= last); 627 628 balance_dirty_pages_ratelimited(mapping); 629 630 _leave(""); 631} 632 633/* 634 * Completion of write to server 635 */ 636static void netfs_pages_written_back(struct netfs_io_request *wreq) 637{ 638 struct address_space *mapping = wreq->mapping; 639 struct netfs_folio *finfo; 640 struct netfs_group *group = NULL; 641 struct folio *folio; 642 pgoff_t last; 643 int gcount = 0; 644 645 XA_STATE(xas, &mapping->i_pages, wreq->start / PAGE_SIZE); 646 647 _enter("%llx-%llx", wreq->start, wreq->start + wreq->len); 648 649 rcu_read_lock(); 650 651 last = (wreq->start + wreq->len - 1) / PAGE_SIZE; 652 xas_for_each(&xas, folio, last) { 653 WARN(!folio_test_writeback(folio), 654 "bad %zx @%llx page %lx %lx\n", 655 wreq->len, wreq->start, folio->index, last); 656 657 if ((finfo = netfs_folio_info(folio))) { 658 /* Streaming writes cannot be redirtied whilst under 659 * writeback, so discard the streaming record. 660 */ 661 folio_detach_private(folio); 662 group = finfo->netfs_group; 663 gcount++; 664 trace_netfs_folio(folio, netfs_folio_trace_clear_s); 665 kfree(finfo); 666 } else if ((group = netfs_folio_group(folio))) { 667 /* Need to detach the group pointer if the page didn't 668 * get redirtied. If it has been redirtied, then it 669 * must be within the same group. 670 */ 671 if (folio_test_dirty(folio)) { 672 trace_netfs_folio(folio, netfs_folio_trace_redirtied); 673 goto end_wb; 674 } 675 if (folio_trylock(folio)) { 676 if (!folio_test_dirty(folio)) { 677 folio_detach_private(folio); 678 gcount++; 679 trace_netfs_folio(folio, netfs_folio_trace_clear_g); 680 } else { 681 trace_netfs_folio(folio, netfs_folio_trace_redirtied); 682 } 683 folio_unlock(folio); 684 goto end_wb; 685 } 686 687 xas_pause(&xas); 688 rcu_read_unlock(); 689 folio_lock(folio); 690 if (!folio_test_dirty(folio)) { 691 folio_detach_private(folio); 692 gcount++; 693 trace_netfs_folio(folio, netfs_folio_trace_clear_g); 694 } else { 695 trace_netfs_folio(folio, netfs_folio_trace_redirtied); 696 } 697 folio_unlock(folio); 698 rcu_read_lock(); 699 } else { 700 trace_netfs_folio(folio, netfs_folio_trace_clear); 701 } 702 end_wb: 703 if (folio_test_fscache(folio)) 704 folio_end_fscache(folio); 705 xas_advance(&xas, folio_next_index(folio) - 1); 706 folio_end_writeback(folio); 707 } 708 709 rcu_read_unlock(); 710 netfs_put_group_many(group, gcount); 711 _leave(""); 712} 713 714/* 715 * Deal with the disposition of the folios that are under writeback to close 716 * out the operation. 717 */ 718static void netfs_cleanup_buffered_write(struct netfs_io_request *wreq) 719{ 720 struct address_space *mapping = wreq->mapping; 721 722 _enter(""); 723 724 switch (wreq->error) { 725 case 0: 726 netfs_pages_written_back(wreq); 727 break; 728 729 default: 730 pr_notice("R=%08x Unexpected error %d\n", wreq->debug_id, wreq->error); 731 fallthrough; 732 case -EACCES: 733 case -EPERM: 734 case -ENOKEY: 735 case -EKEYEXPIRED: 736 case -EKEYREJECTED: 737 case -EKEYREVOKED: 738 case -ENETRESET: 739 case -EDQUOT: 740 case -ENOSPC: 741 netfs_redirty_pages(mapping, wreq->start, wreq->len); 742 break; 743 744 case -EROFS: 745 case -EIO: 746 case -EREMOTEIO: 747 case -EFBIG: 748 case -ENOENT: 749 case -ENOMEDIUM: 750 case -ENXIO: 751 netfs_kill_pages(mapping, wreq->start, wreq->len); 752 break; 753 } 754 755 if (wreq->error) 756 mapping_set_error(mapping, wreq->error); 757 if (wreq->netfs_ops->done) 758 wreq->netfs_ops->done(wreq); 759} 760 761/* 762 * Extend the region to be written back to include subsequent contiguously 763 * dirty pages if possible, but don't sleep while doing so. 764 * 765 * If this page holds new content, then we can include filler zeros in the 766 * writeback. 767 */ 768static void netfs_extend_writeback(struct address_space *mapping, 769 struct netfs_group *group, 770 struct xa_state *xas, 771 long *_count, 772 loff_t start, 773 loff_t max_len, 774 bool caching, 775 size_t *_len, 776 size_t *_top) 777{ 778 struct netfs_folio *finfo; 779 struct folio_batch fbatch; 780 struct folio *folio; 781 unsigned int i; 782 pgoff_t index = (start + *_len) / PAGE_SIZE; 783 size_t len; 784 void *priv; 785 bool stop = true; 786 787 folio_batch_init(&fbatch); 788 789 do { 790 /* Firstly, we gather up a batch of contiguous dirty pages 791 * under the RCU read lock - but we can't clear the dirty flags 792 * there if any of those pages are mapped. 793 */ 794 rcu_read_lock(); 795 796 xas_for_each(xas, folio, ULONG_MAX) { 797 stop = true; 798 if (xas_retry(xas, folio)) 799 continue; 800 if (xa_is_value(folio)) 801 break; 802 if (folio->index != index) { 803 xas_reset(xas); 804 break; 805 } 806 807 if (!folio_try_get_rcu(folio)) { 808 xas_reset(xas); 809 continue; 810 } 811 812 /* Has the folio moved or been split? */ 813 if (unlikely(folio != xas_reload(xas))) { 814 folio_put(folio); 815 xas_reset(xas); 816 break; 817 } 818 819 if (!folio_trylock(folio)) { 820 folio_put(folio); 821 xas_reset(xas); 822 break; 823 } 824 if (!folio_test_dirty(folio) || 825 folio_test_writeback(folio) || 826 folio_test_fscache(folio)) { 827 folio_unlock(folio); 828 folio_put(folio); 829 xas_reset(xas); 830 break; 831 } 832 833 stop = false; 834 len = folio_size(folio); 835 priv = folio_get_private(folio); 836 if ((const struct netfs_group *)priv != group) { 837 stop = true; 838 finfo = netfs_folio_info(folio); 839 if (finfo->netfs_group != group || 840 finfo->dirty_offset > 0) { 841 folio_unlock(folio); 842 folio_put(folio); 843 xas_reset(xas); 844 break; 845 } 846 len = finfo->dirty_len; 847 } 848 849 *_top += folio_size(folio); 850 index += folio_nr_pages(folio); 851 *_count -= folio_nr_pages(folio); 852 *_len += len; 853 if (*_len >= max_len || *_count <= 0) 854 stop = true; 855 856 if (!folio_batch_add(&fbatch, folio)) 857 break; 858 if (stop) 859 break; 860 } 861 862 xas_pause(xas); 863 rcu_read_unlock(); 864 865 /* Now, if we obtained any folios, we can shift them to being 866 * writable and mark them for caching. 867 */ 868 if (!folio_batch_count(&fbatch)) 869 break; 870 871 for (i = 0; i < folio_batch_count(&fbatch); i++) { 872 folio = fbatch.folios[i]; 873 trace_netfs_folio(folio, netfs_folio_trace_store_plus); 874 875 if (!folio_clear_dirty_for_io(folio)) 876 BUG(); 877 folio_start_writeback(folio); 878 netfs_folio_start_fscache(caching, folio); 879 folio_unlock(folio); 880 } 881 882 folio_batch_release(&fbatch); 883 cond_resched(); 884 } while (!stop); 885} 886 887/* 888 * Synchronously write back the locked page and any subsequent non-locked dirty 889 * pages. 890 */ 891static ssize_t netfs_write_back_from_locked_folio(struct address_space *mapping, 892 struct writeback_control *wbc, 893 struct netfs_group *group, 894 struct xa_state *xas, 895 struct folio *folio, 896 unsigned long long start, 897 unsigned long long end) 898{ 899 struct netfs_io_request *wreq; 900 struct netfs_folio *finfo; 901 struct netfs_inode *ctx = netfs_inode(mapping->host); 902 unsigned long long i_size = i_size_read(&ctx->inode); 903 size_t len, max_len; 904 bool caching = netfs_is_cache_enabled(ctx); 905 long count = wbc->nr_to_write; 906 int ret; 907 908 _enter(",%lx,%llx-%llx,%u", folio->index, start, end, caching); 909 910 wreq = netfs_alloc_request(mapping, NULL, start, folio_size(folio), 911 NETFS_WRITEBACK); 912 if (IS_ERR(wreq)) { 913 folio_unlock(folio); 914 return PTR_ERR(wreq); 915 } 916 917 if (!folio_clear_dirty_for_io(folio)) 918 BUG(); 919 folio_start_writeback(folio); 920 netfs_folio_start_fscache(caching, folio); 921 922 count -= folio_nr_pages(folio); 923 924 /* Find all consecutive lockable dirty pages that have contiguous 925 * written regions, stopping when we find a page that is not 926 * immediately lockable, is not dirty or is missing, or we reach the 927 * end of the range. 928 */ 929 trace_netfs_folio(folio, netfs_folio_trace_store); 930 931 len = wreq->len; 932 finfo = netfs_folio_info(folio); 933 if (finfo) { 934 start += finfo->dirty_offset; 935 if (finfo->dirty_offset + finfo->dirty_len != len) { 936 len = finfo->dirty_len; 937 goto cant_expand; 938 } 939 len = finfo->dirty_len; 940 } 941 942 if (start < i_size) { 943 /* Trim the write to the EOF; the extra data is ignored. Also 944 * put an upper limit on the size of a single storedata op. 945 */ 946 max_len = 65536 * 4096; 947 max_len = min_t(unsigned long long, max_len, end - start + 1); 948 max_len = min_t(unsigned long long, max_len, i_size - start); 949 950 if (len < max_len) 951 netfs_extend_writeback(mapping, group, xas, &count, start, 952 max_len, caching, &len, &wreq->upper_len); 953 } 954 955cant_expand: 956 len = min_t(unsigned long long, len, i_size - start); 957 958 /* We now have a contiguous set of dirty pages, each with writeback 959 * set; the first page is still locked at this point, but all the rest 960 * have been unlocked. 961 */ 962 folio_unlock(folio); 963 wreq->start = start; 964 wreq->len = len; 965 966 if (start < i_size) { 967 _debug("write back %zx @%llx [%llx]", len, start, i_size); 968 969 /* Speculatively write to the cache. We have to fix this up 970 * later if the store fails. 971 */ 972 wreq->cleanup = netfs_cleanup_buffered_write; 973 974 iov_iter_xarray(&wreq->iter, ITER_SOURCE, &mapping->i_pages, start, 975 wreq->upper_len); 976 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); 977 ret = netfs_begin_write(wreq, true, netfs_write_trace_writeback); 978 if (ret == 0 || ret == -EIOCBQUEUED) 979 wbc->nr_to_write -= len / PAGE_SIZE; 980 } else { 981 _debug("write discard %zx @%llx [%llx]", len, start, i_size); 982 983 /* The dirty region was entirely beyond the EOF. */ 984 fscache_clear_page_bits(mapping, start, len, caching); 985 netfs_pages_written_back(wreq); 986 ret = 0; 987 } 988 989 netfs_put_request(wreq, false, netfs_rreq_trace_put_return); 990 _leave(" = 1"); 991 return 1; 992} 993 994/* 995 * Write a region of pages back to the server 996 */ 997static ssize_t netfs_writepages_begin(struct address_space *mapping, 998 struct writeback_control *wbc, 999 struct netfs_group *group, 1000 struct xa_state *xas, 1001 unsigned long long *_start, 1002 unsigned long long end) 1003{ 1004 const struct netfs_folio *finfo; 1005 struct folio *folio; 1006 unsigned long long start = *_start; 1007 ssize_t ret; 1008 void *priv; 1009 int skips = 0; 1010 1011 _enter("%llx,%llx,", start, end); 1012 1013search_again: 1014 /* Find the first dirty page in the group. */ 1015 rcu_read_lock(); 1016 1017 for (;;) { 1018 folio = xas_find_marked(xas, end / PAGE_SIZE, PAGECACHE_TAG_DIRTY); 1019 if (xas_retry(xas, folio) || xa_is_value(folio)) 1020 continue; 1021 if (!folio) 1022 break; 1023 1024 if (!folio_try_get_rcu(folio)) { 1025 xas_reset(xas); 1026 continue; 1027 } 1028 1029 if (unlikely(folio != xas_reload(xas))) { 1030 folio_put(folio); 1031 xas_reset(xas); 1032 continue; 1033 } 1034 1035 /* Skip any dirty folio that's not in the group of interest. */ 1036 priv = folio_get_private(folio); 1037 if ((const struct netfs_group *)priv != group) { 1038 finfo = netfs_folio_info(folio); 1039 if (finfo->netfs_group != group) { 1040 folio_put(folio); 1041 continue; 1042 } 1043 } 1044 1045 xas_pause(xas); 1046 break; 1047 } 1048 rcu_read_unlock(); 1049 if (!folio) 1050 return 0; 1051 1052 start = folio_pos(folio); /* May regress with THPs */ 1053 1054 _debug("wback %lx", folio->index); 1055 1056 /* At this point we hold neither the i_pages lock nor the page lock: 1057 * the page may be truncated or invalidated (changing page->mapping to 1058 * NULL), or even swizzled back from swapper_space to tmpfs file 1059 * mapping 1060 */ 1061lock_again: 1062 if (wbc->sync_mode != WB_SYNC_NONE) { 1063 ret = folio_lock_killable(folio); 1064 if (ret < 0) 1065 return ret; 1066 } else { 1067 if (!folio_trylock(folio)) 1068 goto search_again; 1069 } 1070 1071 if (folio->mapping != mapping || 1072 !folio_test_dirty(folio)) { 1073 start += folio_size(folio); 1074 folio_unlock(folio); 1075 goto search_again; 1076 } 1077 1078 if (folio_test_writeback(folio) || 1079 folio_test_fscache(folio)) { 1080 folio_unlock(folio); 1081 if (wbc->sync_mode != WB_SYNC_NONE) { 1082 folio_wait_writeback(folio); 1083#ifdef CONFIG_FSCACHE 1084 folio_wait_fscache(folio); 1085#endif 1086 goto lock_again; 1087 } 1088 1089 start += folio_size(folio); 1090 if (wbc->sync_mode == WB_SYNC_NONE) { 1091 if (skips >= 5 || need_resched()) { 1092 ret = 0; 1093 goto out; 1094 } 1095 skips++; 1096 } 1097 goto search_again; 1098 } 1099 1100 ret = netfs_write_back_from_locked_folio(mapping, wbc, group, xas, 1101 folio, start, end); 1102out: 1103 if (ret > 0) 1104 *_start = start + ret; 1105 _leave(" = %zd [%llx]", ret, *_start); 1106 return ret; 1107} 1108 1109/* 1110 * Write a region of pages back to the server 1111 */ 1112static int netfs_writepages_region(struct address_space *mapping, 1113 struct writeback_control *wbc, 1114 struct netfs_group *group, 1115 unsigned long long *_start, 1116 unsigned long long end) 1117{ 1118 ssize_t ret; 1119 1120 XA_STATE(xas, &mapping->i_pages, *_start / PAGE_SIZE); 1121 1122 do { 1123 ret = netfs_writepages_begin(mapping, wbc, group, &xas, 1124 _start, end); 1125 if (ret > 0 && wbc->nr_to_write > 0) 1126 cond_resched(); 1127 } while (ret > 0 && wbc->nr_to_write > 0); 1128 1129 return ret > 0 ? 0 : ret; 1130} 1131 1132/* 1133 * write some of the pending data back to the server 1134 */ 1135int netfs_writepages(struct address_space *mapping, 1136 struct writeback_control *wbc) 1137{ 1138 struct netfs_group *group = NULL; 1139 loff_t start, end; 1140 int ret; 1141 1142 _enter(""); 1143 1144 /* We have to be careful as we can end up racing with setattr() 1145 * truncating the pagecache since the caller doesn't take a lock here 1146 * to prevent it. 1147 */ 1148 1149 if (wbc->range_cyclic && mapping->writeback_index) { 1150 start = mapping->writeback_index * PAGE_SIZE; 1151 ret = netfs_writepages_region(mapping, wbc, group, 1152 &start, LLONG_MAX); 1153 if (ret < 0) 1154 goto out; 1155 1156 if (wbc->nr_to_write <= 0) { 1157 mapping->writeback_index = start / PAGE_SIZE; 1158 goto out; 1159 } 1160 1161 start = 0; 1162 end = mapping->writeback_index * PAGE_SIZE; 1163 mapping->writeback_index = 0; 1164 ret = netfs_writepages_region(mapping, wbc, group, &start, end); 1165 if (ret == 0) 1166 mapping->writeback_index = start / PAGE_SIZE; 1167 } else if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) { 1168 start = 0; 1169 ret = netfs_writepages_region(mapping, wbc, group, 1170 &start, LLONG_MAX); 1171 if (wbc->nr_to_write > 0 && ret == 0) 1172 mapping->writeback_index = start / PAGE_SIZE; 1173 } else { 1174 start = wbc->range_start; 1175 ret = netfs_writepages_region(mapping, wbc, group, 1176 &start, wbc->range_end); 1177 } 1178 1179out: 1180 _leave(" = %d", ret); 1181 return ret; 1182} 1183EXPORT_SYMBOL(netfs_writepages); 1184 1185/* 1186 * Deal with the disposition of a laundered folio. 1187 */ 1188static void netfs_cleanup_launder_folio(struct netfs_io_request *wreq) 1189{ 1190 if (wreq->error) { 1191 pr_notice("R=%08x Laundering error %d\n", wreq->debug_id, wreq->error); 1192 mapping_set_error(wreq->mapping, wreq->error); 1193 } 1194} 1195 1196/** 1197 * netfs_launder_folio - Clean up a dirty folio that's being invalidated 1198 * @folio: The folio to clean 1199 * 1200 * This is called to write back a folio that's being invalidated when an inode 1201 * is getting torn down. Ideally, writepages would be used instead. 1202 */ 1203int netfs_launder_folio(struct folio *folio) 1204{ 1205 struct netfs_io_request *wreq; 1206 struct address_space *mapping = folio->mapping; 1207 struct netfs_folio *finfo = netfs_folio_info(folio); 1208 struct netfs_group *group = netfs_folio_group(folio); 1209 struct bio_vec bvec; 1210 unsigned long long i_size = i_size_read(mapping->host); 1211 unsigned long long start = folio_pos(folio); 1212 size_t offset = 0, len; 1213 int ret = 0; 1214 1215 if (finfo) { 1216 offset = finfo->dirty_offset; 1217 start += offset; 1218 len = finfo->dirty_len; 1219 } else { 1220 len = folio_size(folio); 1221 } 1222 len = min_t(unsigned long long, len, i_size - start); 1223 1224 wreq = netfs_alloc_request(mapping, NULL, start, len, NETFS_LAUNDER_WRITE); 1225 if (IS_ERR(wreq)) { 1226 ret = PTR_ERR(wreq); 1227 goto out; 1228 } 1229 1230 if (!folio_clear_dirty_for_io(folio)) 1231 goto out_put; 1232 1233 trace_netfs_folio(folio, netfs_folio_trace_launder); 1234 1235 _debug("launder %llx-%llx", start, start + len - 1); 1236 1237 /* Speculatively write to the cache. We have to fix this up later if 1238 * the store fails. 1239 */ 1240 wreq->cleanup = netfs_cleanup_launder_folio; 1241 1242 bvec_set_folio(&bvec, folio, len, offset); 1243 iov_iter_bvec(&wreq->iter, ITER_SOURCE, &bvec, 1, len); 1244 __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); 1245 ret = netfs_begin_write(wreq, true, netfs_write_trace_launder); 1246 1247out_put: 1248 folio_detach_private(folio); 1249 netfs_put_group(group); 1250 kfree(finfo); 1251 netfs_put_request(wreq, false, netfs_rreq_trace_put_return); 1252out: 1253 folio_wait_fscache(folio); 1254 _leave(" = %d", ret); 1255 return ret; 1256} 1257EXPORT_SYMBOL(netfs_launder_folio); 1258