1// SPDX-License-Identifier: GPL-2.0 2/* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8#include <linux/fs.h> 9#include <linux/f2fs_fs.h> 10#include <linux/moduleparam.h> 11#include <linux/writeback.h> 12#include <linux/backing-dev.h> 13#include <linux/lzo.h> 14#include <linux/lz4.h> 15#include <linux/zstd.h> 16#include <linux/pagevec.h> 17 18#include "f2fs.h" 19#include "node.h" 20#include "segment.h" 21#include <trace/events/f2fs.h> 22 23static struct kmem_cache *cic_entry_slab; 24static struct kmem_cache *dic_entry_slab; 25 26static void *page_array_alloc(struct inode *inode, int nr) 27{ 28 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 29 unsigned int size = sizeof(struct page *) * nr; 30 31 if (likely(size <= sbi->page_array_slab_size)) 32 return f2fs_kmem_cache_alloc(sbi->page_array_slab, 33 GFP_F2FS_ZERO, false, F2FS_I_SB(inode)); 34 return f2fs_kzalloc(sbi, size, GFP_NOFS); 35} 36 37static void page_array_free(struct inode *inode, void *pages, int nr) 38{ 39 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 40 unsigned int size = sizeof(struct page *) * nr; 41 42 if (!pages) 43 return; 44 45 if (likely(size <= sbi->page_array_slab_size)) 46 kmem_cache_free(sbi->page_array_slab, pages); 47 else 48 kfree(pages); 49} 50 51struct f2fs_compress_ops { 52 int (*init_compress_ctx)(struct compress_ctx *cc); 53 void (*destroy_compress_ctx)(struct compress_ctx *cc); 54 int (*compress_pages)(struct compress_ctx *cc); 55 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 56 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 57 int (*decompress_pages)(struct decompress_io_ctx *dic); 58 bool (*is_level_valid)(int level); 59}; 60 61static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 62{ 63 return index & (cc->cluster_size - 1); 64} 65 66static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 67{ 68 return index >> cc->log_cluster_size; 69} 70 71static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 72{ 73 return cc->cluster_idx << cc->log_cluster_size; 74} 75 76bool f2fs_is_compressed_page(struct page *page) 77{ 78 if (!PagePrivate(page)) 79 return false; 80 if (!page_private(page)) 81 return false; 82 if (page_private_nonpointer(page)) 83 return false; 84 85 f2fs_bug_on(F2FS_M_SB(page->mapping), 86 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC); 87 return true; 88} 89 90static void f2fs_set_compressed_page(struct page *page, 91 struct inode *inode, pgoff_t index, void *data) 92{ 93 attach_page_private(page, (void *)data); 94 95 /* i_crypto_info and iv index */ 96 page->index = index; 97 page->mapping = inode->i_mapping; 98} 99 100static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 101{ 102 int i; 103 104 for (i = 0; i < len; i++) { 105 if (!cc->rpages[i]) 106 continue; 107 if (unlock) 108 unlock_page(cc->rpages[i]); 109 else 110 put_page(cc->rpages[i]); 111 } 112} 113 114static void f2fs_put_rpages(struct compress_ctx *cc) 115{ 116 f2fs_drop_rpages(cc, cc->cluster_size, false); 117} 118 119static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 120{ 121 f2fs_drop_rpages(cc, len, true); 122} 123 124static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 125 struct writeback_control *wbc, bool redirty, int unlock) 126{ 127 unsigned int i; 128 129 for (i = 0; i < cc->cluster_size; i++) { 130 if (!cc->rpages[i]) 131 continue; 132 if (redirty) 133 redirty_page_for_writepage(wbc, cc->rpages[i]); 134 f2fs_put_page(cc->rpages[i], unlock); 135 } 136} 137 138struct page *f2fs_compress_control_page(struct page *page) 139{ 140 return ((struct compress_io_ctx *)page_private(page))->rpages[0]; 141} 142 143int f2fs_init_compress_ctx(struct compress_ctx *cc) 144{ 145 if (cc->rpages) 146 return 0; 147 148 cc->rpages = page_array_alloc(cc->inode, cc->cluster_size); 149 return cc->rpages ? 0 : -ENOMEM; 150} 151 152void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) 153{ 154 page_array_free(cc->inode, cc->rpages, cc->cluster_size); 155 cc->rpages = NULL; 156 cc->nr_rpages = 0; 157 cc->nr_cpages = 0; 158 cc->valid_nr_cpages = 0; 159 if (!reuse) 160 cc->cluster_idx = NULL_CLUSTER; 161} 162 163void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page) 164{ 165 unsigned int cluster_ofs; 166 167 if (!f2fs_cluster_can_merge_page(cc, page->index)) 168 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 169 170 cluster_ofs = offset_in_cluster(cc, page->index); 171 cc->rpages[cluster_ofs] = page; 172 cc->nr_rpages++; 173 cc->cluster_idx = cluster_idx(cc, page->index); 174} 175 176#ifdef CONFIG_F2FS_FS_LZO 177static int lzo_init_compress_ctx(struct compress_ctx *cc) 178{ 179 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 180 LZO1X_MEM_COMPRESS, GFP_NOFS); 181 if (!cc->private) 182 return -ENOMEM; 183 184 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 185 return 0; 186} 187 188static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 189{ 190 kvfree(cc->private); 191 cc->private = NULL; 192} 193 194static int lzo_compress_pages(struct compress_ctx *cc) 195{ 196 int ret; 197 198 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 199 &cc->clen, cc->private); 200 if (ret != LZO_E_OK) { 201 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n", 202 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); 203 return -EIO; 204 } 205 return 0; 206} 207 208static int lzo_decompress_pages(struct decompress_io_ctx *dic) 209{ 210 int ret; 211 212 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 213 dic->rbuf, &dic->rlen); 214 if (ret != LZO_E_OK) { 215 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n", 216 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 217 return -EIO; 218 } 219 220 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 221 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, " 222 "expected:%lu\n", KERN_ERR, 223 F2FS_I_SB(dic->inode)->sb->s_id, 224 dic->rlen, 225 PAGE_SIZE << dic->log_cluster_size); 226 return -EIO; 227 } 228 return 0; 229} 230 231static const struct f2fs_compress_ops f2fs_lzo_ops = { 232 .init_compress_ctx = lzo_init_compress_ctx, 233 .destroy_compress_ctx = lzo_destroy_compress_ctx, 234 .compress_pages = lzo_compress_pages, 235 .decompress_pages = lzo_decompress_pages, 236}; 237#endif 238 239#ifdef CONFIG_F2FS_FS_LZ4 240static int lz4_init_compress_ctx(struct compress_ctx *cc) 241{ 242 unsigned int size = LZ4_MEM_COMPRESS; 243 244#ifdef CONFIG_F2FS_FS_LZ4HC 245 if (F2FS_I(cc->inode)->i_compress_level) 246 size = LZ4HC_MEM_COMPRESS; 247#endif 248 249 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), size, GFP_NOFS); 250 if (!cc->private) 251 return -ENOMEM; 252 253 /* 254 * we do not change cc->clen to LZ4_compressBound(inputsize) to 255 * adapt worst compress case, because lz4 compressor can handle 256 * output budget properly. 257 */ 258 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 259 return 0; 260} 261 262static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 263{ 264 kvfree(cc->private); 265 cc->private = NULL; 266} 267 268static int lz4_compress_pages(struct compress_ctx *cc) 269{ 270 int len = -EINVAL; 271 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 272 273 if (!level) 274 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 275 cc->clen, cc->private); 276#ifdef CONFIG_F2FS_FS_LZ4HC 277 else 278 len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, 279 cc->clen, level, cc->private); 280#endif 281 if (len < 0) 282 return len; 283 if (!len) 284 return -EAGAIN; 285 286 cc->clen = len; 287 return 0; 288} 289 290static int lz4_decompress_pages(struct decompress_io_ctx *dic) 291{ 292 int ret; 293 294 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 295 dic->clen, dic->rlen); 296 if (ret < 0) { 297 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n", 298 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 299 return -EIO; 300 } 301 302 if (ret != PAGE_SIZE << dic->log_cluster_size) { 303 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid ret:%d, " 304 "expected:%lu\n", KERN_ERR, 305 F2FS_I_SB(dic->inode)->sb->s_id, ret, 306 PAGE_SIZE << dic->log_cluster_size); 307 return -EIO; 308 } 309 return 0; 310} 311 312static bool lz4_is_level_valid(int lvl) 313{ 314#ifdef CONFIG_F2FS_FS_LZ4HC 315 return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL); 316#else 317 return lvl == 0; 318#endif 319} 320 321static const struct f2fs_compress_ops f2fs_lz4_ops = { 322 .init_compress_ctx = lz4_init_compress_ctx, 323 .destroy_compress_ctx = lz4_destroy_compress_ctx, 324 .compress_pages = lz4_compress_pages, 325 .decompress_pages = lz4_decompress_pages, 326 .is_level_valid = lz4_is_level_valid, 327}; 328#endif 329 330#ifdef CONFIG_F2FS_FS_ZSTD 331static int zstd_init_compress_ctx(struct compress_ctx *cc) 332{ 333 zstd_parameters params; 334 zstd_cstream *stream; 335 void *workspace; 336 unsigned int workspace_size; 337 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 338 339 /* Need to remain this for backward compatibility */ 340 if (!level) 341 level = F2FS_ZSTD_DEFAULT_CLEVEL; 342 343 params = zstd_get_params(level, cc->rlen); 344 workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); 345 346 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 347 workspace_size, GFP_NOFS); 348 if (!workspace) 349 return -ENOMEM; 350 351 stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); 352 if (!stream) { 353 printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_cstream failed\n", 354 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 355 __func__); 356 kvfree(workspace); 357 return -EIO; 358 } 359 360 cc->private = workspace; 361 cc->private2 = stream; 362 363 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 364 return 0; 365} 366 367static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 368{ 369 kvfree(cc->private); 370 cc->private = NULL; 371 cc->private2 = NULL; 372} 373 374static int zstd_compress_pages(struct compress_ctx *cc) 375{ 376 zstd_cstream *stream = cc->private2; 377 zstd_in_buffer inbuf; 378 zstd_out_buffer outbuf; 379 int src_size = cc->rlen; 380 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 381 int ret; 382 383 inbuf.pos = 0; 384 inbuf.src = cc->rbuf; 385 inbuf.size = src_size; 386 387 outbuf.pos = 0; 388 outbuf.dst = cc->cbuf->cdata; 389 outbuf.size = dst_size; 390 391 ret = zstd_compress_stream(stream, &outbuf, &inbuf); 392 if (zstd_is_error(ret)) { 393 printk_ratelimited("%sF2FS-fs (%s): %s zstd_compress_stream failed, ret: %d\n", 394 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 395 __func__, zstd_get_error_code(ret)); 396 return -EIO; 397 } 398 399 ret = zstd_end_stream(stream, &outbuf); 400 if (zstd_is_error(ret)) { 401 printk_ratelimited("%sF2FS-fs (%s): %s zstd_end_stream returned %d\n", 402 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 403 __func__, zstd_get_error_code(ret)); 404 return -EIO; 405 } 406 407 /* 408 * there is compressed data remained in intermediate buffer due to 409 * no more space in cbuf.cdata 410 */ 411 if (ret) 412 return -EAGAIN; 413 414 cc->clen = outbuf.pos; 415 return 0; 416} 417 418static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 419{ 420 zstd_dstream *stream; 421 void *workspace; 422 unsigned int workspace_size; 423 unsigned int max_window_size = 424 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); 425 426 workspace_size = zstd_dstream_workspace_bound(max_window_size); 427 428 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), 429 workspace_size, GFP_NOFS); 430 if (!workspace) 431 return -ENOMEM; 432 433 stream = zstd_init_dstream(max_window_size, workspace, workspace_size); 434 if (!stream) { 435 printk_ratelimited("%sF2FS-fs (%s): %s zstd_init_dstream failed\n", 436 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 437 __func__); 438 kvfree(workspace); 439 return -EIO; 440 } 441 442 dic->private = workspace; 443 dic->private2 = stream; 444 445 return 0; 446} 447 448static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 449{ 450 kvfree(dic->private); 451 dic->private = NULL; 452 dic->private2 = NULL; 453} 454 455static int zstd_decompress_pages(struct decompress_io_ctx *dic) 456{ 457 zstd_dstream *stream = dic->private2; 458 zstd_in_buffer inbuf; 459 zstd_out_buffer outbuf; 460 int ret; 461 462 inbuf.pos = 0; 463 inbuf.src = dic->cbuf->cdata; 464 inbuf.size = dic->clen; 465 466 outbuf.pos = 0; 467 outbuf.dst = dic->rbuf; 468 outbuf.size = dic->rlen; 469 470 ret = zstd_decompress_stream(stream, &outbuf, &inbuf); 471 if (zstd_is_error(ret)) { 472 printk_ratelimited("%sF2FS-fs (%s): %s zstd_decompress_stream failed, ret: %d\n", 473 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 474 __func__, zstd_get_error_code(ret)); 475 return -EIO; 476 } 477 478 if (dic->rlen != outbuf.pos) { 479 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, " 480 "expected:%lu\n", KERN_ERR, 481 F2FS_I_SB(dic->inode)->sb->s_id, 482 __func__, dic->rlen, 483 PAGE_SIZE << dic->log_cluster_size); 484 return -EIO; 485 } 486 487 return 0; 488} 489 490static bool zstd_is_level_valid(int lvl) 491{ 492 return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel(); 493} 494 495static const struct f2fs_compress_ops f2fs_zstd_ops = { 496 .init_compress_ctx = zstd_init_compress_ctx, 497 .destroy_compress_ctx = zstd_destroy_compress_ctx, 498 .compress_pages = zstd_compress_pages, 499 .init_decompress_ctx = zstd_init_decompress_ctx, 500 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 501 .decompress_pages = zstd_decompress_pages, 502 .is_level_valid = zstd_is_level_valid, 503}; 504#endif 505 506#ifdef CONFIG_F2FS_FS_LZO 507#ifdef CONFIG_F2FS_FS_LZORLE 508static int lzorle_compress_pages(struct compress_ctx *cc) 509{ 510 int ret; 511 512 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 513 &cc->clen, cc->private); 514 if (ret != LZO_E_OK) { 515 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 516 "lzo-rle compress failed, ret:%d", ret); 517 return -EIO; 518 } 519 return 0; 520} 521 522static const struct f2fs_compress_ops f2fs_lzorle_ops = { 523 .init_compress_ctx = lzo_init_compress_ctx, 524 .destroy_compress_ctx = lzo_destroy_compress_ctx, 525 .compress_pages = lzorle_compress_pages, 526 .decompress_pages = lzo_decompress_pages, 527}; 528#endif 529#endif 530 531static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 532#ifdef CONFIG_F2FS_FS_LZO 533 &f2fs_lzo_ops, 534#else 535 NULL, 536#endif 537#ifdef CONFIG_F2FS_FS_LZ4 538 &f2fs_lz4_ops, 539#else 540 NULL, 541#endif 542#ifdef CONFIG_F2FS_FS_ZSTD 543 &f2fs_zstd_ops, 544#else 545 NULL, 546#endif 547#if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 548 &f2fs_lzorle_ops, 549#else 550 NULL, 551#endif 552}; 553 554bool f2fs_is_compress_backend_ready(struct inode *inode) 555{ 556 if (!f2fs_compressed_file(inode)) 557 return true; 558 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 559} 560 561bool f2fs_is_compress_level_valid(int alg, int lvl) 562{ 563 const struct f2fs_compress_ops *cops = f2fs_cops[alg]; 564 565 if (cops->is_level_valid) 566 return cops->is_level_valid(lvl); 567 568 return lvl == 0; 569} 570 571static mempool_t *compress_page_pool; 572static int num_compress_pages = 512; 573module_param(num_compress_pages, uint, 0444); 574MODULE_PARM_DESC(num_compress_pages, 575 "Number of intermediate compress pages to preallocate"); 576 577int __init f2fs_init_compress_mempool(void) 578{ 579 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 580 return compress_page_pool ? 0 : -ENOMEM; 581} 582 583void f2fs_destroy_compress_mempool(void) 584{ 585 mempool_destroy(compress_page_pool); 586} 587 588static struct page *f2fs_compress_alloc_page(void) 589{ 590 struct page *page; 591 592 page = mempool_alloc(compress_page_pool, GFP_NOFS); 593 lock_page(page); 594 595 return page; 596} 597 598static void f2fs_compress_free_page(struct page *page) 599{ 600 if (!page) 601 return; 602 detach_page_private(page); 603 page->mapping = NULL; 604 unlock_page(page); 605 mempool_free(page, compress_page_pool); 606} 607 608#define MAX_VMAP_RETRIES 3 609 610static void *f2fs_vmap(struct page **pages, unsigned int count) 611{ 612 int i; 613 void *buf = NULL; 614 615 for (i = 0; i < MAX_VMAP_RETRIES; i++) { 616 buf = vm_map_ram(pages, count, -1); 617 if (buf) 618 break; 619 vm_unmap_aliases(); 620 } 621 return buf; 622} 623 624static int f2fs_compress_pages(struct compress_ctx *cc) 625{ 626 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 627 const struct f2fs_compress_ops *cops = 628 f2fs_cops[fi->i_compress_algorithm]; 629 unsigned int max_len, new_nr_cpages; 630 u32 chksum = 0; 631 int i, ret; 632 633 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 634 cc->cluster_size, fi->i_compress_algorithm); 635 636 if (cops->init_compress_ctx) { 637 ret = cops->init_compress_ctx(cc); 638 if (ret) 639 goto out; 640 } 641 642 max_len = COMPRESS_HEADER_SIZE + cc->clen; 643 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 644 cc->valid_nr_cpages = cc->nr_cpages; 645 646 cc->cpages = page_array_alloc(cc->inode, cc->nr_cpages); 647 if (!cc->cpages) { 648 ret = -ENOMEM; 649 goto destroy_compress_ctx; 650 } 651 652 for (i = 0; i < cc->nr_cpages; i++) 653 cc->cpages[i] = f2fs_compress_alloc_page(); 654 655 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); 656 if (!cc->rbuf) { 657 ret = -ENOMEM; 658 goto out_free_cpages; 659 } 660 661 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); 662 if (!cc->cbuf) { 663 ret = -ENOMEM; 664 goto out_vunmap_rbuf; 665 } 666 667 ret = cops->compress_pages(cc); 668 if (ret) 669 goto out_vunmap_cbuf; 670 671 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 672 673 if (cc->clen > max_len) { 674 ret = -EAGAIN; 675 goto out_vunmap_cbuf; 676 } 677 678 cc->cbuf->clen = cpu_to_le32(cc->clen); 679 680 if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM)) 681 chksum = f2fs_crc32(F2FS_I_SB(cc->inode), 682 cc->cbuf->cdata, cc->clen); 683 cc->cbuf->chksum = cpu_to_le32(chksum); 684 685 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 686 cc->cbuf->reserved[i] = cpu_to_le32(0); 687 688 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 689 690 /* zero out any unused part of the last page */ 691 memset(&cc->cbuf->cdata[cc->clen], 0, 692 (new_nr_cpages * PAGE_SIZE) - 693 (cc->clen + COMPRESS_HEADER_SIZE)); 694 695 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 696 vm_unmap_ram(cc->rbuf, cc->cluster_size); 697 698 for (i = new_nr_cpages; i < cc->nr_cpages; i++) { 699 f2fs_compress_free_page(cc->cpages[i]); 700 cc->cpages[i] = NULL; 701 } 702 703 if (cops->destroy_compress_ctx) 704 cops->destroy_compress_ctx(cc); 705 706 cc->valid_nr_cpages = new_nr_cpages; 707 708 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 709 cc->clen, ret); 710 return 0; 711 712out_vunmap_cbuf: 713 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 714out_vunmap_rbuf: 715 vm_unmap_ram(cc->rbuf, cc->cluster_size); 716out_free_cpages: 717 for (i = 0; i < cc->nr_cpages; i++) { 718 if (cc->cpages[i]) 719 f2fs_compress_free_page(cc->cpages[i]); 720 } 721 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 722 cc->cpages = NULL; 723destroy_compress_ctx: 724 if (cops->destroy_compress_ctx) 725 cops->destroy_compress_ctx(cc); 726out: 727 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 728 cc->clen, ret); 729 return ret; 730} 731 732static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 733 bool pre_alloc); 734static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 735 bool bypass_destroy_callback, bool pre_alloc); 736 737void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) 738{ 739 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 740 struct f2fs_inode_info *fi = F2FS_I(dic->inode); 741 const struct f2fs_compress_ops *cops = 742 f2fs_cops[fi->i_compress_algorithm]; 743 bool bypass_callback = false; 744 int ret; 745 746 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 747 dic->cluster_size, fi->i_compress_algorithm); 748 749 if (dic->failed) { 750 ret = -EIO; 751 goto out_end_io; 752 } 753 754 ret = f2fs_prepare_decomp_mem(dic, false); 755 if (ret) { 756 bypass_callback = true; 757 goto out_release; 758 } 759 760 dic->clen = le32_to_cpu(dic->cbuf->clen); 761 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 762 763 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 764 ret = -EFSCORRUPTED; 765 766 /* Avoid f2fs_commit_super in irq context */ 767 if (!in_task) 768 f2fs_handle_error_async(sbi, ERROR_FAIL_DECOMPRESSION); 769 else 770 f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); 771 goto out_release; 772 } 773 774 ret = cops->decompress_pages(dic); 775 776 if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) { 777 u32 provided = le32_to_cpu(dic->cbuf->chksum); 778 u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen); 779 780 if (provided != calculated) { 781 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { 782 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); 783 f2fs_info_ratelimited(sbi, 784 "checksum invalid, nid = %lu, %x vs %x", 785 dic->inode->i_ino, 786 provided, calculated); 787 } 788 set_sbi_flag(sbi, SBI_NEED_FSCK); 789 } 790 } 791 792out_release: 793 f2fs_release_decomp_mem(dic, bypass_callback, false); 794 795out_end_io: 796 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 797 dic->clen, ret); 798 f2fs_decompress_end_io(dic, ret, in_task); 799} 800 801/* 802 * This is called when a page of a compressed cluster has been read from disk 803 * (or failed to be read from disk). It checks whether this page was the last 804 * page being waited on in the cluster, and if so, it decompresses the cluster 805 * (or in the case of a failure, cleans up without actually decompressing). 806 */ 807void f2fs_end_read_compressed_page(struct page *page, bool failed, 808 block_t blkaddr, bool in_task) 809{ 810 struct decompress_io_ctx *dic = 811 (struct decompress_io_ctx *)page_private(page); 812 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 813 814 dec_page_count(sbi, F2FS_RD_DATA); 815 816 if (failed) 817 WRITE_ONCE(dic->failed, true); 818 else if (blkaddr && in_task) 819 f2fs_cache_compressed_page(sbi, page, 820 dic->inode->i_ino, blkaddr); 821 822 if (atomic_dec_and_test(&dic->remaining_pages)) 823 f2fs_decompress_cluster(dic, in_task); 824} 825 826static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 827{ 828 if (cc->cluster_idx == NULL_CLUSTER) 829 return true; 830 return cc->cluster_idx == cluster_idx(cc, index); 831} 832 833bool f2fs_cluster_is_empty(struct compress_ctx *cc) 834{ 835 return cc->nr_rpages == 0; 836} 837 838static bool f2fs_cluster_is_full(struct compress_ctx *cc) 839{ 840 return cc->cluster_size == cc->nr_rpages; 841} 842 843bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 844{ 845 if (f2fs_cluster_is_empty(cc)) 846 return true; 847 return is_page_in_cluster(cc, index); 848} 849 850bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, 851 int index, int nr_pages, bool uptodate) 852{ 853 unsigned long pgidx = pages[index]->index; 854 int i = uptodate ? 0 : 1; 855 856 /* 857 * when uptodate set to true, try to check all pages in cluster is 858 * uptodate or not. 859 */ 860 if (uptodate && (pgidx % cc->cluster_size)) 861 return false; 862 863 if (nr_pages - index < cc->cluster_size) 864 return false; 865 866 for (; i < cc->cluster_size; i++) { 867 if (pages[index + i]->index != pgidx + i) 868 return false; 869 if (uptodate && !PageUptodate(pages[index + i])) 870 return false; 871 } 872 873 return true; 874} 875 876static bool cluster_has_invalid_data(struct compress_ctx *cc) 877{ 878 loff_t i_size = i_size_read(cc->inode); 879 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 880 int i; 881 882 for (i = 0; i < cc->cluster_size; i++) { 883 struct page *page = cc->rpages[i]; 884 885 f2fs_bug_on(F2FS_I_SB(cc->inode), !page); 886 887 /* beyond EOF */ 888 if (page->index >= nr_pages) 889 return true; 890 } 891 return false; 892} 893 894bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) 895{ 896#ifdef CONFIG_F2FS_CHECK_FS 897 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 898 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; 899 int cluster_end = 0; 900 unsigned int count; 901 int i; 902 char *reason = ""; 903 904 if (dn->data_blkaddr != COMPRESS_ADDR) 905 return false; 906 907 /* [..., COMPR_ADDR, ...] */ 908 if (dn->ofs_in_node % cluster_size) { 909 reason = "[*|C|*|*]"; 910 goto out; 911 } 912 913 for (i = 1, count = 1; i < cluster_size; i++, count++) { 914 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 915 dn->ofs_in_node + i); 916 917 /* [COMPR_ADDR, ..., COMPR_ADDR] */ 918 if (blkaddr == COMPRESS_ADDR) { 919 reason = "[C|*|C|*]"; 920 goto out; 921 } 922 if (!__is_valid_data_blkaddr(blkaddr)) { 923 if (!cluster_end) 924 cluster_end = i; 925 continue; 926 } 927 /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ 928 if (cluster_end) { 929 reason = "[C|N|N|V]"; 930 goto out; 931 } 932 } 933 934 f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size && 935 !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED)); 936 937 return false; 938out: 939 f2fs_warn(sbi, "access invalid cluster, ino:%lu, nid:%u, ofs_in_node:%u, reason:%s", 940 dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason); 941 set_sbi_flag(sbi, SBI_NEED_FSCK); 942 return true; 943#else 944 return false; 945#endif 946} 947 948static int __f2fs_get_cluster_blocks(struct inode *inode, 949 struct dnode_of_data *dn) 950{ 951 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 952 int count, i; 953 954 for (i = 1, count = 1; i < cluster_size; i++) { 955 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 956 dn->ofs_in_node + i); 957 958 if (__is_valid_data_blkaddr(blkaddr)) 959 count++; 960 } 961 962 return count; 963} 964 965static int __f2fs_cluster_blocks(struct inode *inode, 966 unsigned int cluster_idx, bool compr_blks) 967{ 968 struct dnode_of_data dn; 969 unsigned int start_idx = cluster_idx << 970 F2FS_I(inode)->i_log_cluster_size; 971 int ret; 972 973 set_new_dnode(&dn, inode, NULL, NULL, 0); 974 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 975 if (ret) { 976 if (ret == -ENOENT) 977 ret = 0; 978 goto fail; 979 } 980 981 if (f2fs_sanity_check_cluster(&dn)) { 982 ret = -EFSCORRUPTED; 983 goto fail; 984 } 985 986 if (dn.data_blkaddr == COMPRESS_ADDR) { 987 if (compr_blks) 988 ret = __f2fs_get_cluster_blocks(inode, &dn); 989 else 990 ret = 1; 991 } 992fail: 993 f2fs_put_dnode(&dn); 994 return ret; 995} 996 997/* return # of compressed blocks in compressed cluster */ 998static int f2fs_compressed_blocks(struct compress_ctx *cc) 999{ 1000 return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx, true); 1001} 1002 1003/* return whether cluster is compressed one or not */ 1004int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 1005{ 1006 return __f2fs_cluster_blocks(inode, 1007 index >> F2FS_I(inode)->i_log_cluster_size, 1008 false); 1009} 1010 1011static bool cluster_may_compress(struct compress_ctx *cc) 1012{ 1013 if (!f2fs_need_compress_data(cc->inode)) 1014 return false; 1015 if (f2fs_is_atomic_file(cc->inode)) 1016 return false; 1017 if (!f2fs_cluster_is_full(cc)) 1018 return false; 1019 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 1020 return false; 1021 return !cluster_has_invalid_data(cc); 1022} 1023 1024static void set_cluster_writeback(struct compress_ctx *cc) 1025{ 1026 int i; 1027 1028 for (i = 0; i < cc->cluster_size; i++) { 1029 if (cc->rpages[i]) 1030 set_page_writeback(cc->rpages[i]); 1031 } 1032} 1033 1034static void set_cluster_dirty(struct compress_ctx *cc) 1035{ 1036 int i; 1037 1038 for (i = 0; i < cc->cluster_size; i++) 1039 if (cc->rpages[i]) { 1040 set_page_dirty(cc->rpages[i]); 1041 set_page_private_gcing(cc->rpages[i]); 1042 } 1043} 1044 1045static int prepare_compress_overwrite(struct compress_ctx *cc, 1046 struct page **pagep, pgoff_t index, void **fsdata) 1047{ 1048 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1049 struct address_space *mapping = cc->inode->i_mapping; 1050 struct page *page; 1051 sector_t last_block_in_bio; 1052 fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 1053 pgoff_t start_idx = start_idx_of_cluster(cc); 1054 int i, ret; 1055 1056retry: 1057 ret = f2fs_is_compressed_cluster(cc->inode, start_idx); 1058 if (ret <= 0) 1059 return ret; 1060 1061 ret = f2fs_init_compress_ctx(cc); 1062 if (ret) 1063 return ret; 1064 1065 /* keep page reference to avoid page reclaim */ 1066 for (i = 0; i < cc->cluster_size; i++) { 1067 page = f2fs_pagecache_get_page(mapping, start_idx + i, 1068 fgp_flag, GFP_NOFS); 1069 if (!page) { 1070 ret = -ENOMEM; 1071 goto unlock_pages; 1072 } 1073 1074 if (PageUptodate(page)) 1075 f2fs_put_page(page, 1); 1076 else 1077 f2fs_compress_ctx_add_page(cc, page); 1078 } 1079 1080 if (!f2fs_cluster_is_empty(cc)) { 1081 struct bio *bio = NULL; 1082 1083 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 1084 &last_block_in_bio, false, true); 1085 f2fs_put_rpages(cc); 1086 f2fs_destroy_compress_ctx(cc, true); 1087 if (ret) 1088 goto out; 1089 if (bio) 1090 f2fs_submit_read_bio(sbi, bio, DATA); 1091 1092 ret = f2fs_init_compress_ctx(cc); 1093 if (ret) 1094 goto out; 1095 } 1096 1097 for (i = 0; i < cc->cluster_size; i++) { 1098 f2fs_bug_on(sbi, cc->rpages[i]); 1099 1100 page = find_lock_page(mapping, start_idx + i); 1101 if (!page) { 1102 /* page can be truncated */ 1103 goto release_and_retry; 1104 } 1105 1106 f2fs_wait_on_page_writeback(page, DATA, true, true); 1107 f2fs_compress_ctx_add_page(cc, page); 1108 1109 if (!PageUptodate(page)) { 1110release_and_retry: 1111 f2fs_put_rpages(cc); 1112 f2fs_unlock_rpages(cc, i + 1); 1113 f2fs_destroy_compress_ctx(cc, true); 1114 goto retry; 1115 } 1116 } 1117 1118 if (likely(!ret)) { 1119 *fsdata = cc->rpages; 1120 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1121 return cc->cluster_size; 1122 } 1123 1124unlock_pages: 1125 f2fs_put_rpages(cc); 1126 f2fs_unlock_rpages(cc, i); 1127 f2fs_destroy_compress_ctx(cc, true); 1128out: 1129 return ret; 1130} 1131 1132int f2fs_prepare_compress_overwrite(struct inode *inode, 1133 struct page **pagep, pgoff_t index, void **fsdata) 1134{ 1135 struct compress_ctx cc = { 1136 .inode = inode, 1137 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1138 .cluster_size = F2FS_I(inode)->i_cluster_size, 1139 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1140 .rpages = NULL, 1141 .nr_rpages = 0, 1142 }; 1143 1144 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1145} 1146 1147bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1148 pgoff_t index, unsigned copied) 1149 1150{ 1151 struct compress_ctx cc = { 1152 .inode = inode, 1153 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1154 .cluster_size = F2FS_I(inode)->i_cluster_size, 1155 .rpages = fsdata, 1156 }; 1157 bool first_index = (index == cc.rpages[0]->index); 1158 1159 if (copied) 1160 set_cluster_dirty(&cc); 1161 1162 f2fs_put_rpages_wbc(&cc, NULL, false, 1); 1163 f2fs_destroy_compress_ctx(&cc, false); 1164 1165 return first_index; 1166} 1167 1168int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1169{ 1170 void *fsdata = NULL; 1171 struct page *pagep; 1172 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1173 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1174 log_cluster_size; 1175 int err; 1176 1177 err = f2fs_is_compressed_cluster(inode, start_idx); 1178 if (err < 0) 1179 return err; 1180 1181 /* truncate normal cluster */ 1182 if (!err) 1183 return f2fs_do_truncate_blocks(inode, from, lock); 1184 1185 /* truncate compressed cluster */ 1186 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1187 start_idx, &fsdata); 1188 1189 /* should not be a normal cluster */ 1190 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1191 1192 if (err <= 0) 1193 return err; 1194 1195 if (err > 0) { 1196 struct page **rpages = fsdata; 1197 int cluster_size = F2FS_I(inode)->i_cluster_size; 1198 int i; 1199 1200 for (i = cluster_size - 1; i >= 0; i--) { 1201 loff_t start = rpages[i]->index << PAGE_SHIFT; 1202 1203 if (from <= start) { 1204 zero_user_segment(rpages[i], 0, PAGE_SIZE); 1205 } else { 1206 zero_user_segment(rpages[i], from - start, 1207 PAGE_SIZE); 1208 break; 1209 } 1210 } 1211 1212 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1213 } 1214 return 0; 1215} 1216 1217static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1218 int *submitted, 1219 struct writeback_control *wbc, 1220 enum iostat_type io_type) 1221{ 1222 struct inode *inode = cc->inode; 1223 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1224 struct f2fs_inode_info *fi = F2FS_I(inode); 1225 struct f2fs_io_info fio = { 1226 .sbi = sbi, 1227 .ino = cc->inode->i_ino, 1228 .type = DATA, 1229 .op = REQ_OP_WRITE, 1230 .op_flags = wbc_to_write_flags(wbc), 1231 .old_blkaddr = NEW_ADDR, 1232 .page = NULL, 1233 .encrypted_page = NULL, 1234 .compressed_page = NULL, 1235 .submitted = 0, 1236 .io_type = io_type, 1237 .io_wbc = wbc, 1238 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? 1239 1 : 0, 1240 }; 1241 struct dnode_of_data dn; 1242 struct node_info ni; 1243 struct compress_io_ctx *cic; 1244 pgoff_t start_idx = start_idx_of_cluster(cc); 1245 unsigned int last_index = cc->cluster_size - 1; 1246 loff_t psize; 1247 int i, err; 1248 bool quota_inode = IS_NOQUOTA(inode); 1249 1250 /* we should bypass data pages to proceed the kworker jobs */ 1251 if (unlikely(f2fs_cp_error(sbi))) { 1252 mapping_set_error(cc->rpages[0]->mapping, -EIO); 1253 goto out_free; 1254 } 1255 1256 if (quota_inode) { 1257 /* 1258 * We need to wait for node_write to avoid block allocation during 1259 * checkpoint. This can only happen to quota writes which can cause 1260 * the below discard race condition. 1261 */ 1262 f2fs_down_read(&sbi->node_write); 1263 } else if (!f2fs_trylock_op(sbi)) { 1264 goto out_free; 1265 } 1266 1267 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1268 1269 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1270 if (err) 1271 goto out_unlock_op; 1272 1273 for (i = 0; i < cc->cluster_size; i++) { 1274 if (data_blkaddr(dn.inode, dn.node_page, 1275 dn.ofs_in_node + i) == NULL_ADDR) 1276 goto out_put_dnode; 1277 } 1278 1279 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT; 1280 1281 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); 1282 if (err) 1283 goto out_put_dnode; 1284 1285 fio.version = ni.version; 1286 1287 cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1288 if (!cic) 1289 goto out_put_dnode; 1290 1291 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1292 cic->inode = inode; 1293 atomic_set(&cic->pending_pages, cc->valid_nr_cpages); 1294 cic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1295 if (!cic->rpages) 1296 goto out_put_cic; 1297 1298 cic->nr_rpages = cc->cluster_size; 1299 1300 for (i = 0; i < cc->valid_nr_cpages; i++) { 1301 f2fs_set_compressed_page(cc->cpages[i], inode, 1302 cc->rpages[i + 1]->index, cic); 1303 fio.compressed_page = cc->cpages[i]; 1304 1305 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_page, 1306 dn.ofs_in_node + i + 1); 1307 1308 /* wait for GCed page writeback via META_MAPPING */ 1309 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1310 1311 if (fio.encrypted) { 1312 fio.page = cc->rpages[i + 1]; 1313 err = f2fs_encrypt_one_page(&fio); 1314 if (err) 1315 goto out_destroy_crypt; 1316 cc->cpages[i] = fio.encrypted_page; 1317 } 1318 } 1319 1320 set_cluster_writeback(cc); 1321 1322 for (i = 0; i < cc->cluster_size; i++) 1323 cic->rpages[i] = cc->rpages[i]; 1324 1325 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1326 block_t blkaddr; 1327 1328 blkaddr = f2fs_data_blkaddr(&dn); 1329 fio.page = cc->rpages[i]; 1330 fio.old_blkaddr = blkaddr; 1331 1332 /* cluster header */ 1333 if (i == 0) { 1334 if (blkaddr == COMPRESS_ADDR) 1335 fio.compr_blocks++; 1336 if (__is_valid_data_blkaddr(blkaddr)) 1337 f2fs_invalidate_blocks(sbi, blkaddr); 1338 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1339 goto unlock_continue; 1340 } 1341 1342 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1343 fio.compr_blocks++; 1344 1345 if (i > cc->valid_nr_cpages) { 1346 if (__is_valid_data_blkaddr(blkaddr)) { 1347 f2fs_invalidate_blocks(sbi, blkaddr); 1348 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1349 } 1350 goto unlock_continue; 1351 } 1352 1353 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1354 1355 if (fio.encrypted) 1356 fio.encrypted_page = cc->cpages[i - 1]; 1357 else 1358 fio.compressed_page = cc->cpages[i - 1]; 1359 1360 cc->cpages[i - 1] = NULL; 1361 f2fs_outplace_write_data(&dn, &fio); 1362 (*submitted)++; 1363unlock_continue: 1364 inode_dec_dirty_pages(cc->inode); 1365 unlock_page(fio.page); 1366 } 1367 1368 if (fio.compr_blocks) 1369 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1370 f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true); 1371 add_compr_block_stat(inode, cc->valid_nr_cpages); 1372 1373 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1374 1375 f2fs_put_dnode(&dn); 1376 if (quota_inode) 1377 f2fs_up_read(&sbi->node_write); 1378 else 1379 f2fs_unlock_op(sbi); 1380 1381 spin_lock(&fi->i_size_lock); 1382 if (fi->last_disk_size < psize) 1383 fi->last_disk_size = psize; 1384 spin_unlock(&fi->i_size_lock); 1385 1386 f2fs_put_rpages(cc); 1387 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1388 cc->cpages = NULL; 1389 f2fs_destroy_compress_ctx(cc, false); 1390 return 0; 1391 1392out_destroy_crypt: 1393 page_array_free(cc->inode, cic->rpages, cc->cluster_size); 1394 1395 for (--i; i >= 0; i--) 1396 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1397out_put_cic: 1398 kmem_cache_free(cic_entry_slab, cic); 1399out_put_dnode: 1400 f2fs_put_dnode(&dn); 1401out_unlock_op: 1402 if (quota_inode) 1403 f2fs_up_read(&sbi->node_write); 1404 else 1405 f2fs_unlock_op(sbi); 1406out_free: 1407 for (i = 0; i < cc->valid_nr_cpages; i++) { 1408 f2fs_compress_free_page(cc->cpages[i]); 1409 cc->cpages[i] = NULL; 1410 } 1411 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1412 cc->cpages = NULL; 1413 return -EAGAIN; 1414} 1415 1416void f2fs_compress_write_end_io(struct bio *bio, struct page *page) 1417{ 1418 struct f2fs_sb_info *sbi = bio->bi_private; 1419 struct compress_io_ctx *cic = 1420 (struct compress_io_ctx *)page_private(page); 1421 enum count_type type = WB_DATA_TYPE(page, 1422 f2fs_is_compressed_page(page)); 1423 int i; 1424 1425 if (unlikely(bio->bi_status)) 1426 mapping_set_error(cic->inode->i_mapping, -EIO); 1427 1428 f2fs_compress_free_page(page); 1429 1430 dec_page_count(sbi, type); 1431 1432 if (atomic_dec_return(&cic->pending_pages)) 1433 return; 1434 1435 for (i = 0; i < cic->nr_rpages; i++) { 1436 WARN_ON(!cic->rpages[i]); 1437 clear_page_private_gcing(cic->rpages[i]); 1438 end_page_writeback(cic->rpages[i]); 1439 } 1440 1441 page_array_free(cic->inode, cic->rpages, cic->nr_rpages); 1442 kmem_cache_free(cic_entry_slab, cic); 1443} 1444 1445static int f2fs_write_raw_pages(struct compress_ctx *cc, 1446 int *submitted_p, 1447 struct writeback_control *wbc, 1448 enum iostat_type io_type) 1449{ 1450 struct address_space *mapping = cc->inode->i_mapping; 1451 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 1452 int submitted, compr_blocks, i; 1453 int ret = 0; 1454 1455 compr_blocks = f2fs_compressed_blocks(cc); 1456 1457 for (i = 0; i < cc->cluster_size; i++) { 1458 if (!cc->rpages[i]) 1459 continue; 1460 1461 redirty_page_for_writepage(wbc, cc->rpages[i]); 1462 unlock_page(cc->rpages[i]); 1463 } 1464 1465 if (compr_blocks < 0) 1466 return compr_blocks; 1467 1468 /* overwrite compressed cluster w/ normal cluster */ 1469 if (compr_blocks > 0) 1470 f2fs_lock_op(sbi); 1471 1472 for (i = 0; i < cc->cluster_size; i++) { 1473 if (!cc->rpages[i]) 1474 continue; 1475retry_write: 1476 lock_page(cc->rpages[i]); 1477 1478 if (cc->rpages[i]->mapping != mapping) { 1479continue_unlock: 1480 unlock_page(cc->rpages[i]); 1481 continue; 1482 } 1483 1484 if (!PageDirty(cc->rpages[i])) 1485 goto continue_unlock; 1486 1487 if (PageWriteback(cc->rpages[i])) { 1488 if (wbc->sync_mode == WB_SYNC_NONE) 1489 goto continue_unlock; 1490 f2fs_wait_on_page_writeback(cc->rpages[i], DATA, true, true); 1491 } 1492 1493 if (!clear_page_dirty_for_io(cc->rpages[i])) 1494 goto continue_unlock; 1495 1496 ret = f2fs_write_single_data_page(cc->rpages[i], &submitted, 1497 NULL, NULL, wbc, io_type, 1498 compr_blocks, false); 1499 if (ret) { 1500 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1501 unlock_page(cc->rpages[i]); 1502 ret = 0; 1503 } else if (ret == -EAGAIN) { 1504 ret = 0; 1505 /* 1506 * for quota file, just redirty left pages to 1507 * avoid deadlock caused by cluster update race 1508 * from foreground operation. 1509 */ 1510 if (IS_NOQUOTA(cc->inode)) 1511 goto out; 1512 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 1513 goto retry_write; 1514 } 1515 goto out; 1516 } 1517 1518 *submitted_p += submitted; 1519 } 1520 1521out: 1522 if (compr_blocks > 0) 1523 f2fs_unlock_op(sbi); 1524 1525 f2fs_balance_fs(sbi, true); 1526 return ret; 1527} 1528 1529int f2fs_write_multi_pages(struct compress_ctx *cc, 1530 int *submitted, 1531 struct writeback_control *wbc, 1532 enum iostat_type io_type) 1533{ 1534 int err; 1535 1536 *submitted = 0; 1537 if (cluster_may_compress(cc)) { 1538 err = f2fs_compress_pages(cc); 1539 if (err == -EAGAIN) { 1540 add_compr_block_stat(cc->inode, cc->cluster_size); 1541 goto write; 1542 } else if (err) { 1543 f2fs_put_rpages_wbc(cc, wbc, true, 1); 1544 goto destroy_out; 1545 } 1546 1547 err = f2fs_write_compressed_pages(cc, submitted, 1548 wbc, io_type); 1549 if (!err) 1550 return 0; 1551 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1552 } 1553write: 1554 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1555 1556 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1557 f2fs_put_rpages_wbc(cc, wbc, false, 0); 1558destroy_out: 1559 f2fs_destroy_compress_ctx(cc, false); 1560 return err; 1561} 1562 1563static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, 1564 bool pre_alloc) 1565{ 1566 return pre_alloc ^ f2fs_low_mem_mode(sbi); 1567} 1568 1569static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 1570 bool pre_alloc) 1571{ 1572 const struct f2fs_compress_ops *cops = 1573 f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; 1574 int i; 1575 1576 if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) 1577 return 0; 1578 1579 dic->tpages = page_array_alloc(dic->inode, dic->cluster_size); 1580 if (!dic->tpages) 1581 return -ENOMEM; 1582 1583 for (i = 0; i < dic->cluster_size; i++) { 1584 if (dic->rpages[i]) { 1585 dic->tpages[i] = dic->rpages[i]; 1586 continue; 1587 } 1588 1589 dic->tpages[i] = f2fs_compress_alloc_page(); 1590 } 1591 1592 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); 1593 if (!dic->rbuf) 1594 return -ENOMEM; 1595 1596 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); 1597 if (!dic->cbuf) 1598 return -ENOMEM; 1599 1600 if (cops->init_decompress_ctx) 1601 return cops->init_decompress_ctx(dic); 1602 1603 return 0; 1604} 1605 1606static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 1607 bool bypass_destroy_callback, bool pre_alloc) 1608{ 1609 const struct f2fs_compress_ops *cops = 1610 f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; 1611 1612 if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) 1613 return; 1614 1615 if (!bypass_destroy_callback && cops->destroy_decompress_ctx) 1616 cops->destroy_decompress_ctx(dic); 1617 1618 if (dic->cbuf) 1619 vm_unmap_ram(dic->cbuf, dic->nr_cpages); 1620 1621 if (dic->rbuf) 1622 vm_unmap_ram(dic->rbuf, dic->cluster_size); 1623} 1624 1625static void f2fs_free_dic(struct decompress_io_ctx *dic, 1626 bool bypass_destroy_callback); 1627 1628struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1629{ 1630 struct decompress_io_ctx *dic; 1631 pgoff_t start_idx = start_idx_of_cluster(cc); 1632 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1633 int i, ret; 1634 1635 dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1636 if (!dic) 1637 return ERR_PTR(-ENOMEM); 1638 1639 dic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1640 if (!dic->rpages) { 1641 kmem_cache_free(dic_entry_slab, dic); 1642 return ERR_PTR(-ENOMEM); 1643 } 1644 1645 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1646 dic->inode = cc->inode; 1647 atomic_set(&dic->remaining_pages, cc->nr_cpages); 1648 dic->cluster_idx = cc->cluster_idx; 1649 dic->cluster_size = cc->cluster_size; 1650 dic->log_cluster_size = cc->log_cluster_size; 1651 dic->nr_cpages = cc->nr_cpages; 1652 refcount_set(&dic->refcnt, 1); 1653 dic->failed = false; 1654 dic->need_verity = f2fs_need_verity(cc->inode, start_idx); 1655 1656 for (i = 0; i < dic->cluster_size; i++) 1657 dic->rpages[i] = cc->rpages[i]; 1658 dic->nr_rpages = cc->cluster_size; 1659 1660 dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages); 1661 if (!dic->cpages) { 1662 ret = -ENOMEM; 1663 goto out_free; 1664 } 1665 1666 for (i = 0; i < dic->nr_cpages; i++) { 1667 struct page *page; 1668 1669 page = f2fs_compress_alloc_page(); 1670 f2fs_set_compressed_page(page, cc->inode, 1671 start_idx + i + 1, dic); 1672 dic->cpages[i] = page; 1673 } 1674 1675 ret = f2fs_prepare_decomp_mem(dic, true); 1676 if (ret) 1677 goto out_free; 1678 1679 return dic; 1680 1681out_free: 1682 f2fs_free_dic(dic, true); 1683 return ERR_PTR(ret); 1684} 1685 1686static void f2fs_free_dic(struct decompress_io_ctx *dic, 1687 bool bypass_destroy_callback) 1688{ 1689 int i; 1690 1691 f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); 1692 1693 if (dic->tpages) { 1694 for (i = 0; i < dic->cluster_size; i++) { 1695 if (dic->rpages[i]) 1696 continue; 1697 if (!dic->tpages[i]) 1698 continue; 1699 f2fs_compress_free_page(dic->tpages[i]); 1700 } 1701 page_array_free(dic->inode, dic->tpages, dic->cluster_size); 1702 } 1703 1704 if (dic->cpages) { 1705 for (i = 0; i < dic->nr_cpages; i++) { 1706 if (!dic->cpages[i]) 1707 continue; 1708 f2fs_compress_free_page(dic->cpages[i]); 1709 } 1710 page_array_free(dic->inode, dic->cpages, dic->nr_cpages); 1711 } 1712 1713 page_array_free(dic->inode, dic->rpages, dic->nr_rpages); 1714 kmem_cache_free(dic_entry_slab, dic); 1715} 1716 1717static void f2fs_late_free_dic(struct work_struct *work) 1718{ 1719 struct decompress_io_ctx *dic = 1720 container_of(work, struct decompress_io_ctx, free_work); 1721 1722 f2fs_free_dic(dic, false); 1723} 1724 1725static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) 1726{ 1727 if (refcount_dec_and_test(&dic->refcnt)) { 1728 if (in_task) { 1729 f2fs_free_dic(dic, false); 1730 } else { 1731 INIT_WORK(&dic->free_work, f2fs_late_free_dic); 1732 queue_work(F2FS_I_SB(dic->inode)->post_read_wq, 1733 &dic->free_work); 1734 } 1735 } 1736} 1737 1738static void f2fs_verify_cluster(struct work_struct *work) 1739{ 1740 struct decompress_io_ctx *dic = 1741 container_of(work, struct decompress_io_ctx, verity_work); 1742 int i; 1743 1744 /* Verify, update, and unlock the decompressed pages. */ 1745 for (i = 0; i < dic->cluster_size; i++) { 1746 struct page *rpage = dic->rpages[i]; 1747 1748 if (!rpage) 1749 continue; 1750 1751 if (fsverity_verify_page(rpage)) 1752 SetPageUptodate(rpage); 1753 else 1754 ClearPageUptodate(rpage); 1755 unlock_page(rpage); 1756 } 1757 1758 f2fs_put_dic(dic, true); 1759} 1760 1761/* 1762 * This is called when a compressed cluster has been decompressed 1763 * (or failed to be read and/or decompressed). 1764 */ 1765void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, 1766 bool in_task) 1767{ 1768 int i; 1769 1770 if (!failed && dic->need_verity) { 1771 /* 1772 * Note that to avoid deadlocks, the verity work can't be done 1773 * on the decompression workqueue. This is because verifying 1774 * the data pages can involve reading metadata pages from the 1775 * file, and these metadata pages may be compressed. 1776 */ 1777 INIT_WORK(&dic->verity_work, f2fs_verify_cluster); 1778 fsverity_enqueue_verify_work(&dic->verity_work); 1779 return; 1780 } 1781 1782 /* Update and unlock the cluster's pagecache pages. */ 1783 for (i = 0; i < dic->cluster_size; i++) { 1784 struct page *rpage = dic->rpages[i]; 1785 1786 if (!rpage) 1787 continue; 1788 1789 if (failed) 1790 ClearPageUptodate(rpage); 1791 else 1792 SetPageUptodate(rpage); 1793 unlock_page(rpage); 1794 } 1795 1796 /* 1797 * Release the reference to the decompress_io_ctx that was being held 1798 * for I/O completion. 1799 */ 1800 f2fs_put_dic(dic, in_task); 1801} 1802 1803/* 1804 * Put a reference to a compressed page's decompress_io_ctx. 1805 * 1806 * This is called when the page is no longer needed and can be freed. 1807 */ 1808void f2fs_put_page_dic(struct page *page, bool in_task) 1809{ 1810 struct decompress_io_ctx *dic = 1811 (struct decompress_io_ctx *)page_private(page); 1812 1813 f2fs_put_dic(dic, in_task); 1814} 1815 1816/* 1817 * check whether cluster blocks are contiguous, and add extent cache entry 1818 * only if cluster blocks are logically and physically contiguous. 1819 */ 1820unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, 1821 unsigned int ofs_in_node) 1822{ 1823 bool compressed = data_blkaddr(dn->inode, dn->node_page, 1824 ofs_in_node) == COMPRESS_ADDR; 1825 int i = compressed ? 1 : 0; 1826 block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_page, 1827 ofs_in_node + i); 1828 1829 for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { 1830 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 1831 ofs_in_node + i); 1832 1833 if (!__is_valid_data_blkaddr(blkaddr)) 1834 break; 1835 if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) 1836 return 0; 1837 } 1838 1839 return compressed ? i - 1 : i; 1840} 1841 1842const struct address_space_operations f2fs_compress_aops = { 1843 .release_folio = f2fs_release_folio, 1844 .invalidate_folio = f2fs_invalidate_folio, 1845 .migrate_folio = filemap_migrate_folio, 1846}; 1847 1848struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) 1849{ 1850 return sbi->compress_inode->i_mapping; 1851} 1852 1853void f2fs_invalidate_compress_page(struct f2fs_sb_info *sbi, block_t blkaddr) 1854{ 1855 if (!sbi->compress_inode) 1856 return; 1857 invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr); 1858} 1859 1860void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page, 1861 nid_t ino, block_t blkaddr) 1862{ 1863 struct page *cpage; 1864 int ret; 1865 1866 if (!test_opt(sbi, COMPRESS_CACHE)) 1867 return; 1868 1869 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) 1870 return; 1871 1872 if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) 1873 return; 1874 1875 cpage = find_get_page(COMPRESS_MAPPING(sbi), blkaddr); 1876 if (cpage) { 1877 f2fs_put_page(cpage, 0); 1878 return; 1879 } 1880 1881 cpage = alloc_page(__GFP_NOWARN | __GFP_IO); 1882 if (!cpage) 1883 return; 1884 1885 ret = add_to_page_cache_lru(cpage, COMPRESS_MAPPING(sbi), 1886 blkaddr, GFP_NOFS); 1887 if (ret) { 1888 f2fs_put_page(cpage, 0); 1889 return; 1890 } 1891 1892 set_page_private_data(cpage, ino); 1893 1894 memcpy(page_address(cpage), page_address(page), PAGE_SIZE); 1895 SetPageUptodate(cpage); 1896 f2fs_put_page(cpage, 1); 1897} 1898 1899bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page, 1900 block_t blkaddr) 1901{ 1902 struct page *cpage; 1903 bool hitted = false; 1904 1905 if (!test_opt(sbi, COMPRESS_CACHE)) 1906 return false; 1907 1908 cpage = f2fs_pagecache_get_page(COMPRESS_MAPPING(sbi), 1909 blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); 1910 if (cpage) { 1911 if (PageUptodate(cpage)) { 1912 atomic_inc(&sbi->compress_page_hit); 1913 memcpy(page_address(page), 1914 page_address(cpage), PAGE_SIZE); 1915 hitted = true; 1916 } 1917 f2fs_put_page(cpage, 1); 1918 } 1919 1920 return hitted; 1921} 1922 1923void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) 1924{ 1925 struct address_space *mapping = COMPRESS_MAPPING(sbi); 1926 struct folio_batch fbatch; 1927 pgoff_t index = 0; 1928 pgoff_t end = MAX_BLKADDR(sbi); 1929 1930 if (!mapping->nrpages) 1931 return; 1932 1933 folio_batch_init(&fbatch); 1934 1935 do { 1936 unsigned int nr, i; 1937 1938 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); 1939 if (!nr) 1940 break; 1941 1942 for (i = 0; i < nr; i++) { 1943 struct folio *folio = fbatch.folios[i]; 1944 1945 folio_lock(folio); 1946 if (folio->mapping != mapping) { 1947 folio_unlock(folio); 1948 continue; 1949 } 1950 1951 if (ino != get_page_private_data(&folio->page)) { 1952 folio_unlock(folio); 1953 continue; 1954 } 1955 1956 generic_error_remove_folio(mapping, folio); 1957 folio_unlock(folio); 1958 } 1959 folio_batch_release(&fbatch); 1960 cond_resched(); 1961 } while (index < end); 1962} 1963 1964int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) 1965{ 1966 struct inode *inode; 1967 1968 if (!test_opt(sbi, COMPRESS_CACHE)) 1969 return 0; 1970 1971 inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); 1972 if (IS_ERR(inode)) 1973 return PTR_ERR(inode); 1974 sbi->compress_inode = inode; 1975 1976 sbi->compress_percent = COMPRESS_PERCENT; 1977 sbi->compress_watermark = COMPRESS_WATERMARK; 1978 1979 atomic_set(&sbi->compress_page_hit, 0); 1980 1981 return 0; 1982} 1983 1984void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) 1985{ 1986 if (!sbi->compress_inode) 1987 return; 1988 iput(sbi->compress_inode); 1989 sbi->compress_inode = NULL; 1990} 1991 1992int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) 1993{ 1994 dev_t dev = sbi->sb->s_bdev->bd_dev; 1995 char slab_name[35]; 1996 1997 if (!f2fs_sb_has_compression(sbi)) 1998 return 0; 1999 2000 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); 2001 2002 sbi->page_array_slab_size = sizeof(struct page *) << 2003 F2FS_OPTION(sbi).compress_log_size; 2004 2005 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, 2006 sbi->page_array_slab_size); 2007 return sbi->page_array_slab ? 0 : -ENOMEM; 2008} 2009 2010void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) 2011{ 2012 kmem_cache_destroy(sbi->page_array_slab); 2013} 2014 2015int __init f2fs_init_compress_cache(void) 2016{ 2017 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", 2018 sizeof(struct compress_io_ctx)); 2019 if (!cic_entry_slab) 2020 return -ENOMEM; 2021 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", 2022 sizeof(struct decompress_io_ctx)); 2023 if (!dic_entry_slab) 2024 goto free_cic; 2025 return 0; 2026free_cic: 2027 kmem_cache_destroy(cic_entry_slab); 2028 return -ENOMEM; 2029} 2030 2031void f2fs_destroy_compress_cache(void) 2032{ 2033 kmem_cache_destroy(dic_entry_slab); 2034 kmem_cache_destroy(cic_entry_slab); 2035} 2036