1// SPDX-License-Identifier: GPL-2.0 2/* 3 * linux/mm/swap_state.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 * 8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie 9 */ 10#include <linux/mm.h> 11#include <linux/gfp.h> 12#include <linux/kernel_stat.h> 13#include <linux/mempolicy.h> 14#include <linux/swap.h> 15#include <linux/swapops.h> 16#include <linux/init.h> 17#include <linux/pagemap.h> 18#include <linux/pagevec.h> 19#include <linux/backing-dev.h> 20#include <linux/blkdev.h> 21#include <linux/migrate.h> 22#include <linux/vmalloc.h> 23#include <linux/swap_slots.h> 24#include <linux/huge_mm.h> 25#include <linux/shmem_fs.h> 26#include "internal.h" 27#include "swap.h" 28 29/* 30 * swapper_space is a fiction, retained to simplify the path through 31 * vmscan's shrink_page_list. 32 */ 33static const struct address_space_operations swap_aops = { 34 .writepage = swap_writepage, 35 .dirty_folio = noop_dirty_folio, 36#ifdef CONFIG_MIGRATION 37 .migrate_folio = migrate_folio, 38#endif 39}; 40 41struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly; 42static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly; 43static bool enable_vma_readahead __read_mostly = true; 44 45#define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) 46#define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) 47#define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK 48#define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK) 49 50#define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK) 51#define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT) 52#define SWAP_RA_ADDR(v) ((v) & PAGE_MASK) 53 54#define SWAP_RA_VAL(addr, win, hits) \ 55 (((addr) & PAGE_MASK) | \ 56 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \ 57 ((hits) & SWAP_RA_HITS_MASK)) 58 59/* Initial readahead hits is 4 to start up with a small window */ 60#define GET_SWAP_RA_VAL(vma) \ 61 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4) 62 63static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); 64 65void show_swap_cache_info(void) 66{ 67 printk("%lu pages in swap cache\n", total_swapcache_pages()); 68 printk("Free swap = %ldkB\n", K(get_nr_swap_pages())); 69 printk("Total swap = %lukB\n", K(total_swap_pages)); 70} 71 72void *get_shadow_from_swap_cache(swp_entry_t entry) 73{ 74 struct address_space *address_space = swap_address_space(entry); 75 pgoff_t idx = swp_offset(entry); 76 struct page *page; 77 78 page = xa_load(&address_space->i_pages, idx); 79 if (xa_is_value(page)) 80 return page; 81 return NULL; 82} 83 84/* 85 * add_to_swap_cache resembles filemap_add_folio on swapper_space, 86 * but sets SwapCache flag and private instead of mapping and index. 87 */ 88int add_to_swap_cache(struct folio *folio, swp_entry_t entry, 89 gfp_t gfp, void **shadowp) 90{ 91 struct address_space *address_space = swap_address_space(entry); 92 pgoff_t idx = swp_offset(entry); 93 XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio)); 94 unsigned long i, nr = folio_nr_pages(folio); 95 void *old; 96 97 xas_set_update(&xas, workingset_update_node); 98 99 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 100 VM_BUG_ON_FOLIO(folio_test_swapcache(folio), folio); 101 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio); 102 103 folio_ref_add(folio, nr); 104 folio_set_swapcache(folio); 105 folio->swap = entry; 106 107 do { 108 xas_lock_irq(&xas); 109 xas_create_range(&xas); 110 if (xas_error(&xas)) 111 goto unlock; 112 for (i = 0; i < nr; i++) { 113 VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio); 114 if (shadowp) { 115 old = xas_load(&xas); 116 if (xa_is_value(old)) 117 *shadowp = old; 118 } 119 xas_store(&xas, folio); 120 xas_next(&xas); 121 } 122 address_space->nrpages += nr; 123 __node_stat_mod_folio(folio, NR_FILE_PAGES, nr); 124 __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, nr); 125unlock: 126 xas_unlock_irq(&xas); 127 } while (xas_nomem(&xas, gfp)); 128 129 if (!xas_error(&xas)) 130 return 0; 131 132 folio_clear_swapcache(folio); 133 folio_ref_sub(folio, nr); 134 return xas_error(&xas); 135} 136 137/* 138 * This must be called only on folios that have 139 * been verified to be in the swap cache. 140 */ 141void __delete_from_swap_cache(struct folio *folio, 142 swp_entry_t entry, void *shadow) 143{ 144 struct address_space *address_space = swap_address_space(entry); 145 int i; 146 long nr = folio_nr_pages(folio); 147 pgoff_t idx = swp_offset(entry); 148 XA_STATE(xas, &address_space->i_pages, idx); 149 150 xas_set_update(&xas, workingset_update_node); 151 152 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 153 VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); 154 VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); 155 156 for (i = 0; i < nr; i++) { 157 void *entry = xas_store(&xas, shadow); 158 VM_BUG_ON_PAGE(entry != folio, entry); 159 xas_next(&xas); 160 } 161 folio->swap.val = 0; 162 folio_clear_swapcache(folio); 163 address_space->nrpages -= nr; 164 __node_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 165 __lruvec_stat_mod_folio(folio, NR_SWAPCACHE, -nr); 166} 167 168/** 169 * add_to_swap - allocate swap space for a folio 170 * @folio: folio we want to move to swap 171 * 172 * Allocate swap space for the folio and add the folio to the 173 * swap cache. 174 * 175 * Context: Caller needs to hold the folio lock. 176 * Return: Whether the folio was added to the swap cache. 177 */ 178bool add_to_swap(struct folio *folio) 179{ 180 swp_entry_t entry; 181 int err; 182 183 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 184 VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); 185 186 entry = folio_alloc_swap(folio); 187 if (!entry.val) 188 return false; 189 190 /* 191 * XArray node allocations from PF_MEMALLOC contexts could 192 * completely exhaust the page allocator. __GFP_NOMEMALLOC 193 * stops emergency reserves from being allocated. 194 * 195 * TODO: this could cause a theoretical memory reclaim 196 * deadlock in the swap out path. 197 */ 198 /* 199 * Add it to the swap cache. 200 */ 201 err = add_to_swap_cache(folio, entry, 202 __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN, NULL); 203 if (err) 204 /* 205 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 206 * clear SWAP_HAS_CACHE flag. 207 */ 208 goto fail; 209 /* 210 * Normally the folio will be dirtied in unmap because its 211 * pte should be dirty. A special case is MADV_FREE page. The 212 * page's pte could have dirty bit cleared but the folio's 213 * SwapBacked flag is still set because clearing the dirty bit 214 * and SwapBacked flag has no lock protected. For such folio, 215 * unmap will not set dirty bit for it, so folio reclaim will 216 * not write the folio out. This can cause data corruption when 217 * the folio is swapped in later. Always setting the dirty flag 218 * for the folio solves the problem. 219 */ 220 folio_mark_dirty(folio); 221 222 return true; 223 224fail: 225 put_swap_folio(folio, entry); 226 return false; 227} 228 229/* 230 * This must be called only on folios that have 231 * been verified to be in the swap cache and locked. 232 * It will never put the folio into the free list, 233 * the caller has a reference on the folio. 234 */ 235void delete_from_swap_cache(struct folio *folio) 236{ 237 swp_entry_t entry = folio->swap; 238 struct address_space *address_space = swap_address_space(entry); 239 240 xa_lock_irq(&address_space->i_pages); 241 __delete_from_swap_cache(folio, entry, NULL); 242 xa_unlock_irq(&address_space->i_pages); 243 244 put_swap_folio(folio, entry); 245 folio_ref_sub(folio, folio_nr_pages(folio)); 246} 247 248void clear_shadow_from_swap_cache(int type, unsigned long begin, 249 unsigned long end) 250{ 251 unsigned long curr = begin; 252 void *old; 253 254 for (;;) { 255 swp_entry_t entry = swp_entry(type, curr); 256 struct address_space *address_space = swap_address_space(entry); 257 XA_STATE(xas, &address_space->i_pages, curr); 258 259 xas_set_update(&xas, workingset_update_node); 260 261 xa_lock_irq(&address_space->i_pages); 262 xas_for_each(&xas, old, end) { 263 if (!xa_is_value(old)) 264 continue; 265 xas_store(&xas, NULL); 266 } 267 xa_unlock_irq(&address_space->i_pages); 268 269 /* search the next swapcache until we meet end */ 270 curr >>= SWAP_ADDRESS_SPACE_SHIFT; 271 curr++; 272 curr <<= SWAP_ADDRESS_SPACE_SHIFT; 273 if (curr > end) 274 break; 275 } 276} 277 278/* 279 * If we are the only user, then try to free up the swap cache. 280 * 281 * Its ok to check the swapcache flag without the folio lock 282 * here because we are going to recheck again inside 283 * folio_free_swap() _with_ the lock. 284 * - Marcelo 285 */ 286void free_swap_cache(struct folio *folio) 287{ 288 if (folio_test_swapcache(folio) && !folio_mapped(folio) && 289 folio_trylock(folio)) { 290 folio_free_swap(folio); 291 folio_unlock(folio); 292 } 293} 294 295/* 296 * Perform a free_page(), also freeing any swap cache associated with 297 * this page if it is the last user of the page. 298 */ 299void free_page_and_swap_cache(struct page *page) 300{ 301 struct folio *folio = page_folio(page); 302 303 free_swap_cache(folio); 304 if (!is_huge_zero_page(page)) 305 folio_put(folio); 306} 307 308/* 309 * Passed an array of pages, drop them all from swapcache and then release 310 * them. They are removed from the LRU and freed if this is their last use. 311 */ 312void free_pages_and_swap_cache(struct encoded_page **pages, int nr) 313{ 314 struct folio_batch folios; 315 unsigned int refs[PAGEVEC_SIZE]; 316 317 lru_add_drain(); 318 folio_batch_init(&folios); 319 for (int i = 0; i < nr; i++) { 320 struct folio *folio = page_folio(encoded_page_ptr(pages[i])); 321 322 free_swap_cache(folio); 323 refs[folios.nr] = 1; 324 if (unlikely(encoded_page_flags(pages[i]) & 325 ENCODED_PAGE_BIT_NR_PAGES_NEXT)) 326 refs[folios.nr] = encoded_nr_pages(pages[++i]); 327 328 if (folio_batch_add(&folios, folio) == 0) 329 folios_put_refs(&folios, refs); 330 } 331 if (folios.nr) 332 folios_put_refs(&folios, refs); 333} 334 335static inline bool swap_use_vma_readahead(void) 336{ 337 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 338} 339 340/* 341 * Lookup a swap entry in the swap cache. A found folio will be returned 342 * unlocked and with its refcount incremented - we rely on the kernel 343 * lock getting page table operations atomic even if we drop the folio 344 * lock before returning. 345 * 346 * Caller must lock the swap device or hold a reference to keep it valid. 347 */ 348struct folio *swap_cache_get_folio(swp_entry_t entry, 349 struct vm_area_struct *vma, unsigned long addr) 350{ 351 struct folio *folio; 352 353 folio = filemap_get_folio(swap_address_space(entry), swp_offset(entry)); 354 if (!IS_ERR(folio)) { 355 bool vma_ra = swap_use_vma_readahead(); 356 bool readahead; 357 358 /* 359 * At the moment, we don't support PG_readahead for anon THP 360 * so let's bail out rather than confusing the readahead stat. 361 */ 362 if (unlikely(folio_test_large(folio))) 363 return folio; 364 365 readahead = folio_test_clear_readahead(folio); 366 if (vma && vma_ra) { 367 unsigned long ra_val; 368 int win, hits; 369 370 ra_val = GET_SWAP_RA_VAL(vma); 371 win = SWAP_RA_WIN(ra_val); 372 hits = SWAP_RA_HITS(ra_val); 373 if (readahead) 374 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 375 atomic_long_set(&vma->swap_readahead_info, 376 SWAP_RA_VAL(addr, win, hits)); 377 } 378 379 if (readahead) { 380 count_vm_event(SWAP_RA_HIT); 381 if (!vma || !vma_ra) 382 atomic_inc(&swapin_readahead_hits); 383 } 384 } else { 385 folio = NULL; 386 } 387 388 return folio; 389} 390 391/** 392 * filemap_get_incore_folio - Find and get a folio from the page or swap caches. 393 * @mapping: The address_space to search. 394 * @index: The page cache index. 395 * 396 * This differs from filemap_get_folio() in that it will also look for the 397 * folio in the swap cache. 398 * 399 * Return: The found folio or %NULL. 400 */ 401struct folio *filemap_get_incore_folio(struct address_space *mapping, 402 pgoff_t index) 403{ 404 swp_entry_t swp; 405 struct swap_info_struct *si; 406 struct folio *folio = filemap_get_entry(mapping, index); 407 408 if (!folio) 409 return ERR_PTR(-ENOENT); 410 if (!xa_is_value(folio)) 411 return folio; 412 if (!shmem_mapping(mapping)) 413 return ERR_PTR(-ENOENT); 414 415 swp = radix_to_swp_entry(folio); 416 /* There might be swapin error entries in shmem mapping. */ 417 if (non_swap_entry(swp)) 418 return ERR_PTR(-ENOENT); 419 /* Prevent swapoff from happening to us */ 420 si = get_swap_device(swp); 421 if (!si) 422 return ERR_PTR(-ENOENT); 423 index = swp_offset(swp); 424 folio = filemap_get_folio(swap_address_space(swp), index); 425 put_swap_device(si); 426 return folio; 427} 428 429struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 430 struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated, 431 bool skip_if_exists) 432{ 433 struct swap_info_struct *si; 434 struct folio *folio; 435 void *shadow = NULL; 436 437 *new_page_allocated = false; 438 si = get_swap_device(entry); 439 if (!si) 440 return NULL; 441 442 for (;;) { 443 int err; 444 /* 445 * First check the swap cache. Since this is normally 446 * called after swap_cache_get_folio() failed, re-calling 447 * that would confuse statistics. 448 */ 449 folio = filemap_get_folio(swap_address_space(entry), 450 swp_offset(entry)); 451 if (!IS_ERR(folio)) 452 goto got_folio; 453 454 /* 455 * Just skip read ahead for unused swap slot. 456 * During swap_off when swap_slot_cache is disabled, 457 * we have to handle the race between putting 458 * swap entry in swap cache and marking swap slot 459 * as SWAP_HAS_CACHE. That's done in later part of code or 460 * else swap_off will be aborted if we return NULL. 461 */ 462 if (!swap_swapcount(si, entry) && swap_slot_cache_enabled) 463 goto fail_put_swap; 464 465 /* 466 * Get a new folio to read into from swap. Allocate it now, 467 * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will 468 * cause any racers to loop around until we add it to cache. 469 */ 470 folio = (struct folio *)alloc_pages_mpol(gfp_mask, 0, 471 mpol, ilx, numa_node_id()); 472 if (!folio) 473 goto fail_put_swap; 474 475 /* 476 * Swap entry may have been freed since our caller observed it. 477 */ 478 err = swapcache_prepare(entry); 479 if (!err) 480 break; 481 482 folio_put(folio); 483 if (err != -EEXIST) 484 goto fail_put_swap; 485 486 /* 487 * Protect against a recursive call to __read_swap_cache_async() 488 * on the same entry waiting forever here because SWAP_HAS_CACHE 489 * is set but the folio is not the swap cache yet. This can 490 * happen today if mem_cgroup_swapin_charge_folio() below 491 * triggers reclaim through zswap, which may call 492 * __read_swap_cache_async() in the writeback path. 493 */ 494 if (skip_if_exists) 495 goto fail_put_swap; 496 497 /* 498 * We might race against __delete_from_swap_cache(), and 499 * stumble across a swap_map entry whose SWAP_HAS_CACHE 500 * has not yet been cleared. Or race against another 501 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE 502 * in swap_map, but not yet added its folio to swap cache. 503 */ 504 schedule_timeout_uninterruptible(1); 505 } 506 507 /* 508 * The swap entry is ours to swap in. Prepare the new folio. 509 */ 510 511 __folio_set_locked(folio); 512 __folio_set_swapbacked(folio); 513 514 if (mem_cgroup_swapin_charge_folio(folio, NULL, gfp_mask, entry)) 515 goto fail_unlock; 516 517 /* May fail (-ENOMEM) if XArray node allocation failed. */ 518 if (add_to_swap_cache(folio, entry, gfp_mask & GFP_RECLAIM_MASK, &shadow)) 519 goto fail_unlock; 520 521 mem_cgroup_swapin_uncharge_swap(entry); 522 523 if (shadow) 524 workingset_refault(folio, shadow); 525 526 /* Caller will initiate read into locked folio */ 527 folio_add_lru(folio); 528 *new_page_allocated = true; 529got_folio: 530 put_swap_device(si); 531 return folio; 532 533fail_unlock: 534 put_swap_folio(folio, entry); 535 folio_unlock(folio); 536 folio_put(folio); 537fail_put_swap: 538 put_swap_device(si); 539 return NULL; 540} 541 542/* 543 * Locate a page of swap in physical memory, reserving swap cache space 544 * and reading the disk if it is not already cached. 545 * A failure return means that either the page allocation failed or that 546 * the swap entry is no longer in use. 547 * 548 * get/put_swap_device() aren't needed to call this function, because 549 * __read_swap_cache_async() call them and swap_read_folio() holds the 550 * swap cache folio lock. 551 */ 552struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 553 struct vm_area_struct *vma, unsigned long addr, 554 struct swap_iocb **plug) 555{ 556 bool page_allocated; 557 struct mempolicy *mpol; 558 pgoff_t ilx; 559 struct folio *folio; 560 561 mpol = get_vma_policy(vma, addr, 0, &ilx); 562 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 563 &page_allocated, false); 564 mpol_cond_put(mpol); 565 566 if (page_allocated) 567 swap_read_folio(folio, false, plug); 568 return folio; 569} 570 571static unsigned int __swapin_nr_pages(unsigned long prev_offset, 572 unsigned long offset, 573 int hits, 574 int max_pages, 575 int prev_win) 576{ 577 unsigned int pages, last_ra; 578 579 /* 580 * This heuristic has been found to work well on both sequential and 581 * random loads, swapping to hard disk or to SSD: please don't ask 582 * what the "+ 2" means, it just happens to work well, that's all. 583 */ 584 pages = hits + 2; 585 if (pages == 2) { 586 /* 587 * We can have no readahead hits to judge by: but must not get 588 * stuck here forever, so check for an adjacent offset instead 589 * (and don't even bother to check whether swap type is same). 590 */ 591 if (offset != prev_offset + 1 && offset != prev_offset - 1) 592 pages = 1; 593 } else { 594 unsigned int roundup = 4; 595 while (roundup < pages) 596 roundup <<= 1; 597 pages = roundup; 598 } 599 600 if (pages > max_pages) 601 pages = max_pages; 602 603 /* Don't shrink readahead too fast */ 604 last_ra = prev_win / 2; 605 if (pages < last_ra) 606 pages = last_ra; 607 608 return pages; 609} 610 611static unsigned long swapin_nr_pages(unsigned long offset) 612{ 613 static unsigned long prev_offset; 614 unsigned int hits, pages, max_pages; 615 static atomic_t last_readahead_pages; 616 617 max_pages = 1 << READ_ONCE(page_cluster); 618 if (max_pages <= 1) 619 return 1; 620 621 hits = atomic_xchg(&swapin_readahead_hits, 0); 622 pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits, 623 max_pages, 624 atomic_read(&last_readahead_pages)); 625 if (!hits) 626 WRITE_ONCE(prev_offset, offset); 627 atomic_set(&last_readahead_pages, pages); 628 629 return pages; 630} 631 632/** 633 * swap_cluster_readahead - swap in pages in hope we need them soon 634 * @entry: swap entry of this memory 635 * @gfp_mask: memory allocation flags 636 * @mpol: NUMA memory allocation policy to be applied 637 * @ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 638 * 639 * Returns the struct folio for entry and addr, after queueing swapin. 640 * 641 * Primitive swap readahead code. We simply read an aligned block of 642 * (1 << page_cluster) entries in the swap area. This method is chosen 643 * because it doesn't cost us any seek time. We also make sure to queue 644 * the 'original' request together with the readahead ones... 645 * 646 * Note: it is intentional that the same NUMA policy and interleave index 647 * are used for every page of the readahead: neighbouring pages on swap 648 * are fairly likely to have been swapped out from the same node. 649 */ 650struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 651 struct mempolicy *mpol, pgoff_t ilx) 652{ 653 struct folio *folio; 654 unsigned long entry_offset = swp_offset(entry); 655 unsigned long offset = entry_offset; 656 unsigned long start_offset, end_offset; 657 unsigned long mask; 658 struct swap_info_struct *si = swp_swap_info(entry); 659 struct blk_plug plug; 660 struct swap_iocb *splug = NULL; 661 bool page_allocated; 662 663 mask = swapin_nr_pages(offset) - 1; 664 if (!mask) 665 goto skip; 666 667 /* Read a page_cluster sized and aligned cluster around offset. */ 668 start_offset = offset & ~mask; 669 end_offset = offset | mask; 670 if (!start_offset) /* First page is swap header. */ 671 start_offset++; 672 if (end_offset >= si->max) 673 end_offset = si->max - 1; 674 675 blk_start_plug(&plug); 676 for (offset = start_offset; offset <= end_offset ; offset++) { 677 /* Ok, do the async read-ahead now */ 678 folio = __read_swap_cache_async( 679 swp_entry(swp_type(entry), offset), 680 gfp_mask, mpol, ilx, &page_allocated, false); 681 if (!folio) 682 continue; 683 if (page_allocated) { 684 swap_read_folio(folio, false, &splug); 685 if (offset != entry_offset) { 686 folio_set_readahead(folio); 687 count_vm_event(SWAP_RA); 688 } 689 } 690 folio_put(folio); 691 } 692 blk_finish_plug(&plug); 693 swap_read_unplug(splug); 694 lru_add_drain(); /* Push any new pages onto the LRU now */ 695skip: 696 /* The page was likely read above, so no need for plugging here */ 697 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 698 &page_allocated, false); 699 if (unlikely(page_allocated)) { 700 zswap_folio_swapin(folio); 701 swap_read_folio(folio, false, NULL); 702 } 703 return folio; 704} 705 706int init_swap_address_space(unsigned int type, unsigned long nr_pages) 707{ 708 struct address_space *spaces, *space; 709 unsigned int i, nr; 710 711 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); 712 spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL); 713 if (!spaces) 714 return -ENOMEM; 715 for (i = 0; i < nr; i++) { 716 space = spaces + i; 717 xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ); 718 atomic_set(&space->i_mmap_writable, 0); 719 space->a_ops = &swap_aops; 720 /* swap cache doesn't use writeback related tags */ 721 mapping_set_no_writeback_tags(space); 722 } 723 nr_swapper_spaces[type] = nr; 724 swapper_spaces[type] = spaces; 725 726 return 0; 727} 728 729void exit_swap_address_space(unsigned int type) 730{ 731 int i; 732 struct address_space *spaces = swapper_spaces[type]; 733 734 for (i = 0; i < nr_swapper_spaces[type]; i++) 735 VM_WARN_ON_ONCE(!mapping_empty(&spaces[i])); 736 kvfree(spaces); 737 nr_swapper_spaces[type] = 0; 738 swapper_spaces[type] = NULL; 739} 740 741#define SWAP_RA_ORDER_CEILING 5 742 743struct vma_swap_readahead { 744 unsigned short win; 745 unsigned short offset; 746 unsigned short nr_pte; 747}; 748 749static void swap_ra_info(struct vm_fault *vmf, 750 struct vma_swap_readahead *ra_info) 751{ 752 struct vm_area_struct *vma = vmf->vma; 753 unsigned long ra_val; 754 unsigned long faddr, pfn, fpfn, lpfn, rpfn; 755 unsigned long start, end; 756 unsigned int max_win, hits, prev_win, win; 757 758 max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster), 759 SWAP_RA_ORDER_CEILING); 760 if (max_win == 1) { 761 ra_info->win = 1; 762 return; 763 } 764 765 faddr = vmf->address; 766 fpfn = PFN_DOWN(faddr); 767 ra_val = GET_SWAP_RA_VAL(vma); 768 pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val)); 769 prev_win = SWAP_RA_WIN(ra_val); 770 hits = SWAP_RA_HITS(ra_val); 771 ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits, 772 max_win, prev_win); 773 atomic_long_set(&vma->swap_readahead_info, 774 SWAP_RA_VAL(faddr, win, 0)); 775 if (win == 1) 776 return; 777 778 if (fpfn == pfn + 1) { 779 lpfn = fpfn; 780 rpfn = fpfn + win; 781 } else if (pfn == fpfn + 1) { 782 lpfn = fpfn - win + 1; 783 rpfn = fpfn + 1; 784 } else { 785 unsigned int left = (win - 1) / 2; 786 787 lpfn = fpfn - left; 788 rpfn = fpfn + win - left; 789 } 790 start = max3(lpfn, PFN_DOWN(vma->vm_start), 791 PFN_DOWN(faddr & PMD_MASK)); 792 end = min3(rpfn, PFN_DOWN(vma->vm_end), 793 PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE)); 794 795 ra_info->nr_pte = end - start; 796 ra_info->offset = fpfn - start; 797} 798 799/** 800 * swap_vma_readahead - swap in pages in hope we need them soon 801 * @targ_entry: swap entry of the targeted memory 802 * @gfp_mask: memory allocation flags 803 * @mpol: NUMA memory allocation policy to be applied 804 * @targ_ilx: NUMA interleave index, for use only when MPOL_INTERLEAVE 805 * @vmf: fault information 806 * 807 * Returns the struct folio for entry and addr, after queueing swapin. 808 * 809 * Primitive swap readahead code. We simply read in a few pages whose 810 * virtual addresses are around the fault address in the same vma. 811 * 812 * Caller must hold read mmap_lock if vmf->vma is not NULL. 813 * 814 */ 815static struct folio *swap_vma_readahead(swp_entry_t targ_entry, gfp_t gfp_mask, 816 struct mempolicy *mpol, pgoff_t targ_ilx, struct vm_fault *vmf) 817{ 818 struct blk_plug plug; 819 struct swap_iocb *splug = NULL; 820 struct folio *folio; 821 pte_t *pte = NULL, pentry; 822 unsigned long addr; 823 swp_entry_t entry; 824 pgoff_t ilx; 825 unsigned int i; 826 bool page_allocated; 827 struct vma_swap_readahead ra_info = { 828 .win = 1, 829 }; 830 831 swap_ra_info(vmf, &ra_info); 832 if (ra_info.win == 1) 833 goto skip; 834 835 addr = vmf->address - (ra_info.offset * PAGE_SIZE); 836 ilx = targ_ilx - ra_info.offset; 837 838 blk_start_plug(&plug); 839 for (i = 0; i < ra_info.nr_pte; i++, ilx++, addr += PAGE_SIZE) { 840 if (!pte++) { 841 pte = pte_offset_map(vmf->pmd, addr); 842 if (!pte) 843 break; 844 } 845 pentry = ptep_get_lockless(pte); 846 if (!is_swap_pte(pentry)) 847 continue; 848 entry = pte_to_swp_entry(pentry); 849 if (unlikely(non_swap_entry(entry))) 850 continue; 851 pte_unmap(pte); 852 pte = NULL; 853 folio = __read_swap_cache_async(entry, gfp_mask, mpol, ilx, 854 &page_allocated, false); 855 if (!folio) 856 continue; 857 if (page_allocated) { 858 swap_read_folio(folio, false, &splug); 859 if (i != ra_info.offset) { 860 folio_set_readahead(folio); 861 count_vm_event(SWAP_RA); 862 } 863 } 864 folio_put(folio); 865 } 866 if (pte) 867 pte_unmap(pte); 868 blk_finish_plug(&plug); 869 swap_read_unplug(splug); 870 lru_add_drain(); 871skip: 872 /* The folio was likely read above, so no need for plugging here */ 873 folio = __read_swap_cache_async(targ_entry, gfp_mask, mpol, targ_ilx, 874 &page_allocated, false); 875 if (unlikely(page_allocated)) { 876 zswap_folio_swapin(folio); 877 swap_read_folio(folio, false, NULL); 878 } 879 return folio; 880} 881 882/** 883 * swapin_readahead - swap in pages in hope we need them soon 884 * @entry: swap entry of this memory 885 * @gfp_mask: memory allocation flags 886 * @vmf: fault information 887 * 888 * Returns the struct page for entry and addr, after queueing swapin. 889 * 890 * It's a main entry function for swap readahead. By the configuration, 891 * it will read ahead blocks by cluster-based(ie, physical disk based) 892 * or vma-based(ie, virtual address based on faulty address) readahead. 893 */ 894struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, 895 struct vm_fault *vmf) 896{ 897 struct mempolicy *mpol; 898 pgoff_t ilx; 899 struct folio *folio; 900 901 mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); 902 folio = swap_use_vma_readahead() ? 903 swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : 904 swap_cluster_readahead(entry, gfp_mask, mpol, ilx); 905 mpol_cond_put(mpol); 906 907 if (!folio) 908 return NULL; 909 return folio_file_page(folio, swp_offset(entry)); 910} 911 912#ifdef CONFIG_SYSFS 913static ssize_t vma_ra_enabled_show(struct kobject *kobj, 914 struct kobj_attribute *attr, char *buf) 915{ 916 return sysfs_emit(buf, "%s\n", 917 enable_vma_readahead ? "true" : "false"); 918} 919static ssize_t vma_ra_enabled_store(struct kobject *kobj, 920 struct kobj_attribute *attr, 921 const char *buf, size_t count) 922{ 923 ssize_t ret; 924 925 ret = kstrtobool(buf, &enable_vma_readahead); 926 if (ret) 927 return ret; 928 929 return count; 930} 931static struct kobj_attribute vma_ra_enabled_attr = __ATTR_RW(vma_ra_enabled); 932 933static struct attribute *swap_attrs[] = { 934 &vma_ra_enabled_attr.attr, 935 NULL, 936}; 937 938static const struct attribute_group swap_attr_group = { 939 .attrs = swap_attrs, 940}; 941 942static int __init swap_init_sysfs(void) 943{ 944 int err; 945 struct kobject *swap_kobj; 946 947 swap_kobj = kobject_create_and_add("swap", mm_kobj); 948 if (!swap_kobj) { 949 pr_err("failed to create swap kobject\n"); 950 return -ENOMEM; 951 } 952 err = sysfs_create_group(swap_kobj, &swap_attr_group); 953 if (err) { 954 pr_err("failed to register swap group\n"); 955 goto delete_obj; 956 } 957 return 0; 958 959delete_obj: 960 kobject_put(swap_kobj); 961 return err; 962} 963subsys_initcall(swap_init_sysfs); 964#endif 965