118316Swollman// SPDX-License-Identifier: GPL-2.0-only 218316Swollman/* 318316Swollman * linux/mm/swapfile.c 418316Swollman * 518316Swollman * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 618316Swollman * Swap reorganised 29.12.95, Stephen Tweedie 718316Swollman */ 818316Swollman 918316Swollman#include <linux/blkdev.h> 1018316Swollman#include <linux/mm.h> 1118316Swollman#include <linux/sched/mm.h> 1218316Swollman#include <linux/sched/task.h> 1318316Swollman#include <linux/hugetlb.h> 1418316Swollman#include <linux/mman.h> 1518316Swollman#include <linux/slab.h> 1618316Swollman#include <linux/kernel_stat.h> 1718316Swollman#include <linux/swap.h> 1818316Swollman#include <linux/vmalloc.h> 1918316Swollman#include <linux/pagemap.h> 2018316Swollman#include <linux/namei.h> 2118316Swollman#include <linux/shmem_fs.h> 2218316Swollman#include <linux/blk-cgroup.h> 2318316Swollman#include <linux/random.h> 2418316Swollman#include <linux/writeback.h> 2518316Swollman#include <linux/proc_fs.h> 2618316Swollman#include <linux/seq_file.h> 2718316Swollman#include <linux/init.h> 2846303Smarkm#include <linux/ksm.h> 2950476Speter#include <linux/rmap.h> 3018316Swollman#include <linux/security.h> 3118316Swollman#include <linux/backing-dev.h> 3218316Swollman#include <linux/mutex.h> 3318316Swollman#include <linux/capability.h> 3418316Swollman#include <linux/syscalls.h> 3518316Swollman#include <linux/memcontrol.h> 3618316Swollman#include <linux/poll.h> 37126250Sbms#include <linux/oom.h> 3864483Ssheldonh#include <linux/swapfile.h> 39126250Sbms#include <linux/export.h> 40126250Sbms#include <linux/swap_slots.h> 41126250Sbms#include <linux/sort.h> 42126250Sbms#include <linux/completion.h> 43126250Sbms#include <linux/suspend.h> 4446303Smarkm#include <linux/zswap.h> 4546303Smarkm#include <linux/plist.h> 4618316Swollman 4718316Swollman#include <asm/tlbflush.h> 4818316Swollman#include <linux/swapops.h> 4918316Swollman#include <linux/swap_cgroup.h> 5018316Swollman#include "internal.h" 5118316Swollman#include "swap.h" 5218316Swollman 5318316Swollmanstatic bool swap_count_continued(struct swap_info_struct *, pgoff_t, 5418316Swollman unsigned char); 5518316Swollmanstatic void free_swap_count_continuations(struct swap_info_struct *); 5618316Swollman 5718316Swollmanstatic DEFINE_SPINLOCK(swap_lock); 5818316Swollmanstatic unsigned int nr_swapfiles; 5918316Swollmanatomic_long_t nr_swap_pages; 6018316Swollman/* 6118316Swollman * Some modules use swappable objects and may try to swap them out under 6218316Swollman * memory pressure (via the shrinker). Before doing so, they may wish to 6318316Swollman * check to see if any swap space is available. 6418316Swollman */ 6518316SwollmanEXPORT_SYMBOL_GPL(nr_swap_pages); 6618316Swollman/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */ 6718316Swollmanlong total_swap_pages; 6818316Swollmanstatic int least_priority = -1; 6918316Swollmanunsigned long swapfile_maximum_size; 7018316Swollman#ifdef CONFIG_MIGRATION 7118316Swollmanbool swap_migration_ad_supported; 7218316Swollman#endif /* CONFIG_MIGRATION */ 7318316Swollman 7418316Swollmanstatic const char Bad_file[] = "Bad swap file entry "; 7518316Swollmanstatic const char Unused_file[] = "Unused swap file entry "; 76190716Sphkstatic const char Bad_offset[] = "Bad swap offset entry "; 7718316Swollmanstatic const char Unused_offset[] = "Unused swap offset entry "; 7818316Swollman 7918316Swollman/* 8018316Swollman * all active swap_info_structs 8118316Swollman * protected with swap_lock, and ordered by priority. 8246303Smarkm */ 83190716Sphkstatic PLIST_HEAD(swap_active_head); 8418316Swollman 8518316Swollman/* 8618316Swollman * all available (active, not full) swap_info_structs 8764131Ssheldonh * protected with swap_avail_lock, ordered by priority. 8818316Swollman * This is used by folio_alloc_swap() instead of swap_active_head 8918316Swollman * because swap_active_head includes all swap_info_structs, 90190716Sphk * but folio_alloc_swap() doesn't need to look at full ones. 91190716Sphk * This uses its own lock instead of swap_lock because when a 92190716Sphk * swap_info_struct changes between not-full/full, it needs to 9318316Swollman * add/remove itself to/from this list, but the swap_info_struct->lock 9446303Smarkm * is held and the locking order requires swap_lock to be taken 9546303Smarkm * before any swap_info_struct->lock. 9646303Smarkm */ 9746303Smarkmstatic struct plist_head *swap_avail_heads; 9846303Smarkmstatic DEFINE_SPINLOCK(swap_avail_lock); 9946303Smarkm 100126250Sbmsstatic struct swap_info_struct *swap_info[MAX_SWAPFILES]; 101126250Sbms 102126250Sbmsstatic DEFINE_MUTEX(swapon_mutex); 10318316Swollman 10418316Swollmanstatic DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait); 10518316Swollman/* Activity counter to indicate that a swapon or swapoff has occurred */ 10618316Swollmanstatic atomic_t proc_poll_event = ATOMIC_INIT(0); 10718316Swollman 10818316Swollmanatomic_t nr_rotate_swap = ATOMIC_INIT(0); 10918316Swollman 11046303Smarkmstatic struct swap_info_struct *swap_type_to_swap_info(int type) 11118316Swollman{ 11218316Swollman if (type >= MAX_SWAPFILES) 11318316Swollman return NULL; 11418316Swollman 11518316Swollman return READ_ONCE(swap_info[type]); /* rcu_dereference() */ 11618316Swollman} 11718316Swollman 11818316Swollmanstatic inline unsigned char swap_count(unsigned char ent) 11918316Swollman{ 12018316Swollman return ent & ~SWAP_HAS_CACHE; /* may include COUNT_CONTINUED flag */ 12118316Swollman} 12218316Swollman 12318316Swollman/* Reclaim the swap entry anyway if possible */ 12418316Swollman#define TTRS_ANYWAY 0x1 12518316Swollman/* 12618316Swollman * Reclaim the swap entry if there are no more mappings of the 12718316Swollman * corresponding page 12818316Swollman */ 12918316Swollman#define TTRS_UNMAPPED 0x2 13018316Swollman/* Reclaim the swap entry if swap is getting full*/ 13118316Swollman#define TTRS_FULL 0x4 13218316Swollman 13318316Swollman/* returns 1 if swap entry is freed */ 13418316Swollmanstatic int __try_to_reclaim_swap(struct swap_info_struct *si, 13518316Swollman unsigned long offset, unsigned long flags) 13618316Swollman{ 13718316Swollman swp_entry_t entry = swp_entry(si->type, offset); 13820339Swollman struct folio *folio; 13918316Swollman int ret = 0; 14018316Swollman 14118316Swollman folio = filemap_get_folio(swap_address_space(entry), offset); 14218316Swollman if (IS_ERR(folio)) 14318316Swollman return 0; 14418316Swollman /* 14519880Swollman * When this function is called from scan_swap_map_slots() and it's 14618316Swollman * called by vmscan.c at reclaiming folios. So we hold a folio lock 14718316Swollman * here. We have to use trylock for avoiding deadlock. This is a special 14846303Smarkm * case and you should use folio_free_swap() with explicit folio_lock() 14918316Swollman * in usual operations. 15018316Swollman */ 15118316Swollman if (folio_trylock(folio)) { 15218316Swollman if ((flags & TTRS_ANYWAY) || 15318316Swollman ((flags & TTRS_UNMAPPED) && !folio_mapped(folio)) || 15418316Swollman ((flags & TTRS_FULL) && mem_cgroup_swap_full(folio))) 15518316Swollman ret = folio_free_swap(folio); 15618316Swollman folio_unlock(folio); 15718316Swollman } 15818316Swollman folio_put(folio); 15918316Swollman return ret; 16018316Swollman} 16118316Swollman 16218316Swollmanstatic inline struct swap_extent *first_se(struct swap_info_struct *sis) 16318316Swollman{ 16418316Swollman struct rb_node *rb = rb_first(&sis->swap_extent_root); 16518316Swollman return rb_entry(rb, struct swap_extent, rb_node); 16618316Swollman} 16718316Swollman 16818316Swollmanstatic inline struct swap_extent *next_se(struct swap_extent *se) 16918316Swollman{ 17018316Swollman struct rb_node *rb = rb_next(&se->rb_node); 17119880Swollman return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL; 17219880Swollman} 173180993Sphk 174180993Sphk/* 17518316Swollman * swapon tell device that all the old swap contents can be discarded, 176190718Sphk * to allow the swap device to optimize its wear-levelling. 177190718Sphk */ 17818316Swollmanstatic int discard_swap(struct swap_info_struct *si) 17918316Swollman{ 18018316Swollman struct swap_extent *se; 18118316Swollman sector_t start_block; 18218316Swollman sector_t nr_blocks; 18318316Swollman int err = 0; 18418316Swollman 18518316Swollman /* Do not discard the swap header page! */ 18618316Swollman se = first_se(si); 18718316Swollman start_block = (se->start_block + 1) << (PAGE_SHIFT - 9); 18819880Swollman nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9); 18918316Swollman if (nr_blocks) { 19018316Swollman err = blkdev_issue_discard(si->bdev, start_block, 19118316Swollman nr_blocks, GFP_KERNEL); 19218316Swollman if (err) 193180993Sphk return err; 194180993Sphk cond_resched(); 195180993Sphk } 196180993Sphk 197180993Sphk for (se = next_se(se); se; se = next_se(se)) { 198180993Sphk start_block = se->start_block << (PAGE_SHIFT - 9); 19918316Swollman nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9); 200180993Sphk 20118316Swollman err = blkdev_issue_discard(si->bdev, start_block, 20218316Swollman nr_blocks, GFP_KERNEL); 20318316Swollman if (err) 20418316Swollman break; 20518316Swollman 20618316Swollman cond_resched(); 207180993Sphk } 20818316Swollman return err; /* That will often be -EOPNOTSUPP */ 209180993Sphk} 210180993Sphk 211180993Sphkstatic struct swap_extent * 21218316Swollmanoffset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) 21318316Swollman{ 21418316Swollman struct swap_extent *se; 21518316Swollman struct rb_node *rb; 21618316Swollman 21718316Swollman rb = sis->swap_extent_root.rb_node; 218180993Sphk while (rb) { 219180993Sphk se = rb_entry(rb, struct swap_extent, rb_node); 220180993Sphk if (offset < se->start_page) 221180993Sphk rb = rb->rb_left; 22218316Swollman else if (offset >= se->start_page + se->nr_pages) 22318316Swollman rb = rb->rb_right; 22418316Swollman else 22518316Swollman return se; 22618316Swollman } 22718316Swollman /* It *must* be present */ 22818316Swollman BUG(); 22918316Swollman} 23018316Swollman 23118316Swollmansector_t swap_folio_sector(struct folio *folio) 23218316Swollman{ 233180993Sphk struct swap_info_struct *sis = swp_swap_info(folio->swap); 23418316Swollman struct swap_extent *se; 235180993Sphk sector_t sector; 236180993Sphk pgoff_t offset; 237180993Sphk 23818316Swollman offset = swp_offset(folio->swap); 23918316Swollman se = offset_to_swap_extent(sis, offset); 24018316Swollman sector = se->start_block + (offset - se->start_page); 24118316Swollman return sector << (PAGE_SHIFT - 9); 24218316Swollman} 24318316Swollman 244180993Sphk/* 245180993Sphk * swap allocation tell device that a cluster of swap can now be discarded, 246180993Sphk * to allow the swap device to optimize its wear-levelling. 247180993Sphk */ 24818316Swollmanstatic void discard_swap_cluster(struct swap_info_struct *si, 24918316Swollman pgoff_t start_page, pgoff_t nr_pages) 25018316Swollman{ 25118316Swollman struct swap_extent *se = offset_to_swap_extent(si, start_page); 25218316Swollman 25318316Swollman while (nr_pages) { 25418316Swollman pgoff_t offset = start_page - se->start_page; 25518316Swollman sector_t start_block = se->start_block + offset; 25618316Swollman sector_t nr_blocks = se->nr_pages - offset; 25718316Swollman 25818316Swollman if (nr_blocks > nr_pages) 25918316Swollman nr_blocks = nr_pages; 26018316Swollman start_page += nr_blocks; 26118316Swollman nr_pages -= nr_blocks; 26218316Swollman 26318316Swollman start_block <<= PAGE_SHIFT - 9; 26418316Swollman nr_blocks <<= PAGE_SHIFT - 9; 26518316Swollman if (blkdev_issue_discard(si->bdev, start_block, 26637908Scharnier nr_blocks, GFP_NOIO)) 26718316Swollman break; 26818316Swollman 26918316Swollman se = next_se(se); 27018316Swollman } 27118316Swollman} 27218316Swollman 27318316Swollman#ifdef CONFIG_THP_SWAP 27418316Swollman#define SWAPFILE_CLUSTER HPAGE_PMD_NR 27518316Swollman 27618316Swollman#define swap_entry_size(size) (size) 27718316Swollman#else 27818316Swollman#define SWAPFILE_CLUSTER 256 27918316Swollman 28018316Swollman/* 28118316Swollman * Define swap_entry_size() as constant to let compiler to optimize 28218316Swollman * out some code if !CONFIG_THP_SWAP 28318316Swollman */ 28418316Swollman#define swap_entry_size(size) 1 285190711Sphk#endif 28618316Swollman#define LATENCY_LIMIT 256 28718316Swollman 28818316Swollmanstatic inline void cluster_set_flag(struct swap_cluster_info *info, 28918316Swollman unsigned int flag) 29018316Swollman{ 29118316Swollman info->flags = flag; 29218316Swollman} 29318316Swollman 29418316Swollmanstatic inline unsigned int cluster_count(struct swap_cluster_info *info) 29518316Swollman{ 29618316Swollman return info->data; 29718316Swollman} 29818316Swollman 29918316Swollmanstatic inline void cluster_set_count(struct swap_cluster_info *info, 30018316Swollman unsigned int c) 30118316Swollman{ 30218316Swollman info->data = c; 30318316Swollman} 30418316Swollman 30518316Swollmanstatic inline void cluster_set_count_flag(struct swap_cluster_info *info, 30618316Swollman unsigned int c, unsigned int f) 30718316Swollman{ 30837908Scharnier info->flags = f; 30918316Swollman info->data = c; 31046303Smarkm} 31118316Swollman 31218316Swollmanstatic inline unsigned int cluster_next(struct swap_cluster_info *info) 31318316Swollman{ 31418316Swollman return info->data; 31518316Swollman} 31618316Swollman 31718316Swollmanstatic inline void cluster_set_next(struct swap_cluster_info *info, 31818316Swollman unsigned int n) 31918316Swollman{ 32018316Swollman info->data = n; 32137908Scharnier} 32218316Swollman 32318316Swollmanstatic inline void cluster_set_next_flag(struct swap_cluster_info *info, 32418316Swollman unsigned int n, unsigned int f) 32518316Swollman{ 32618316Swollman info->flags = f; 32718316Swollman info->data = n; 32818316Swollman} 32918316Swollman 33018316Swollmanstatic inline bool cluster_is_free(struct swap_cluster_info *info) 33118316Swollman{ 33218316Swollman return info->flags & CLUSTER_FLAG_FREE; 33318316Swollman} 33418316Swollman 33518316Swollmanstatic inline bool cluster_is_null(struct swap_cluster_info *info) 33618316Swollman{ 33718316Swollman return info->flags & CLUSTER_FLAG_NEXT_NULL; 33818316Swollman} 33918316Swollman 34019880Swollmanstatic inline void cluster_set_null(struct swap_cluster_info *info) 34118316Swollman{ 34218316Swollman info->flags = CLUSTER_FLAG_NEXT_NULL; 34318316Swollman info->data = 0; 34418316Swollman} 34518316Swollman 34618316Swollmanstatic inline bool cluster_is_huge(struct swap_cluster_info *info) 34718316Swollman{ 34818316Swollman if (IS_ENABLED(CONFIG_THP_SWAP)) 34918316Swollman return info->flags & CLUSTER_FLAG_HUGE; 35018316Swollman return false; 35146303Smarkm} 35218316Swollman 35346303Smarkmstatic inline void cluster_clear_huge(struct swap_cluster_info *info) 35446303Smarkm{ 35546303Smarkm info->flags &= ~CLUSTER_FLAG_HUGE; 35646303Smarkm} 35746303Smarkm 35846303Smarkmstatic inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si, 35946303Smarkm unsigned long offset) 36046303Smarkm{ 36118316Swollman struct swap_cluster_info *ci; 36218316Swollman 36318316Swollman ci = si->cluster_info; 36418316Swollman if (ci) { 36518316Swollman ci += offset / SWAPFILE_CLUSTER; 36618316Swollman spin_lock(&ci->lock); 36718316Swollman } 36818316Swollman return ci; 36918316Swollman} 37018316Swollman 37118316Swollmanstatic inline void unlock_cluster(struct swap_cluster_info *ci) 37218316Swollman{ 37318316Swollman if (ci) 37418316Swollman spin_unlock(&ci->lock); 37518316Swollman} 37646303Smarkm 37718316Swollman/* 37818316Swollman * Determine the locking method in use for this device. Return 37918316Swollman * swap_cluster_info if SSD-style cluster-based locking is in place. 38046303Smarkm */ 38146303Smarkmstatic inline struct swap_cluster_info *lock_cluster_or_swap_info( 38218316Swollman struct swap_info_struct *si, unsigned long offset) 38318316Swollman{ 38418316Swollman struct swap_cluster_info *ci; 38518316Swollman 38618316Swollman /* Try to use fine-grained SSD-style locking if available: */ 38718316Swollman ci = lock_cluster(si, offset); 38818316Swollman /* Otherwise, fall back to traditional, coarse locking: */ 38918316Swollman if (!ci) 39018316Swollman spin_lock(&si->lock); 39118316Swollman 39218316Swollman return ci; 39318316Swollman} 39418316Swollman 39518316Swollmanstatic inline void unlock_cluster_or_swap_info(struct swap_info_struct *si, 39618316Swollman struct swap_cluster_info *ci) 39718316Swollman{ 39818316Swollman if (ci) 39918316Swollman unlock_cluster(ci); 40018316Swollman else 40118316Swollman spin_unlock(&si->lock); 40218316Swollman} 40318316Swollman 40418316Swollmanstatic inline bool cluster_list_empty(struct swap_cluster_list *list) 40518316Swollman{ 40618316Swollman return cluster_is_null(&list->head); 40746303Smarkm} 40818316Swollman 40918316Swollmanstatic inline unsigned int cluster_list_first(struct swap_cluster_list *list) 41018316Swollman{ 41146303Smarkm return cluster_next(&list->head); 41218316Swollman} 41318316Swollman 41418316Swollmanstatic void cluster_list_init(struct swap_cluster_list *list) 41518316Swollman{ 41618316Swollman cluster_set_null(&list->head); 41718316Swollman cluster_set_null(&list->tail); 41818316Swollman} 41918316Swollman 42018316Swollmanstatic void cluster_list_add_tail(struct swap_cluster_list *list, 42118316Swollman struct swap_cluster_info *ci, 42218316Swollman unsigned int idx) 42318316Swollman{ 42418316Swollman if (cluster_list_empty(list)) { 42518316Swollman cluster_set_next_flag(&list->head, idx, 0); 42618316Swollman cluster_set_next_flag(&list->tail, idx, 0); 42718316Swollman } else { 42818316Swollman struct swap_cluster_info *ci_tail; 42918316Swollman unsigned int tail = cluster_next(&list->tail); 43046303Smarkm 43146303Smarkm /* 43246303Smarkm * Nested cluster lock, but both cluster locks are 43346303Smarkm * only acquired when we held swap_info_struct->lock 43446303Smarkm */ 43546303Smarkm ci_tail = ci + tail; 43646303Smarkm spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING); 43746303Smarkm cluster_set_next(ci_tail, idx); 43818316Swollman spin_unlock(&ci_tail->lock); 43918316Swollman cluster_set_next_flag(&list->tail, idx, 0); 44018316Swollman } 44146303Smarkm} 44246303Smarkm 44346303Smarkmstatic unsigned int cluster_list_del_first(struct swap_cluster_list *list, 44446303Smarkm struct swap_cluster_info *ci) 44518316Swollman{ 44618316Swollman unsigned int idx; 44718316Swollman 44818316Swollman idx = cluster_next(&list->head); 44918316Swollman if (cluster_next(&list->tail) == idx) { 45018316Swollman cluster_set_null(&list->head); 45118316Swollman cluster_set_null(&list->tail); 45218316Swollman } else 45318316Swollman cluster_set_next_flag(&list->head, 45418316Swollman cluster_next(&ci[idx]), 0); 45518316Swollman 45618316Swollman return idx; 45746303Smarkm} 45818316Swollman 45946303Smarkm/* Add a cluster to discard list and schedule it to do discard */ 46046303Smarkmstatic void swap_cluster_schedule_discard(struct swap_info_struct *si, 46118316Swollman unsigned int idx) 46218316Swollman{ 46318316Swollman /* 46418316Swollman * If scan_swap_map_slots() can't find a free cluster, it will check 46518316Swollman * si->swap_map directly. To make sure the discarding cluster isn't 46618316Swollman * taken by scan_swap_map_slots(), mark the swap entries bad (occupied). 46718316Swollman * It will be cleared after discard 46818316Swollman */ 46918316Swollman memset(si->swap_map + idx * SWAPFILE_CLUSTER, 47018316Swollman SWAP_MAP_BAD, SWAPFILE_CLUSTER); 47118316Swollman 47218316Swollman cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx); 47318316Swollman 47418316Swollman schedule_work(&si->discard_work); 47518316Swollman} 47618316Swollman 47718316Swollmanstatic void __free_cluster(struct swap_info_struct *si, unsigned long idx) 47818316Swollman{ 47918316Swollman struct swap_cluster_info *ci = si->cluster_info; 48018316Swollman 48118316Swollman cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE); 48218316Swollman cluster_list_add_tail(&si->free_clusters, ci, idx); 48318316Swollman} 48418316Swollman 48518316Swollman/* 48618316Swollman * Doing discard actually. After a cluster discard is finished, the cluster 48718316Swollman * will be added to free cluster list. caller should hold si->lock. 48818316Swollman*/ 48918316Swollmanstatic void swap_do_scheduled_discard(struct swap_info_struct *si) 49018316Swollman{ 49118316Swollman struct swap_cluster_info *info, *ci; 49218316Swollman unsigned int idx; 49318316Swollman 49418316Swollman info = si->cluster_info; 49518316Swollman 49618316Swollman while (!cluster_list_empty(&si->discard_clusters)) { 49718316Swollman idx = cluster_list_del_first(&si->discard_clusters, info); 49818316Swollman spin_unlock(&si->lock); 49918316Swollman 50018316Swollman discard_swap_cluster(si, idx * SWAPFILE_CLUSTER, 50118316Swollman SWAPFILE_CLUSTER); 50218316Swollman 50318316Swollman spin_lock(&si->lock); 50418316Swollman ci = lock_cluster(si, idx * SWAPFILE_CLUSTER); 50518316Swollman __free_cluster(si, idx); 50618316Swollman memset(si->swap_map + idx * SWAPFILE_CLUSTER, 50718316Swollman 0, SWAPFILE_CLUSTER); 50818316Swollman unlock_cluster(ci); 50918316Swollman } 51018316Swollman} 51118316Swollman 51218316Swollmanstatic void swap_discard_work(struct work_struct *work) 51319880Swollman{ 51418316Swollman struct swap_info_struct *si; 51518316Swollman 51618316Swollman si = container_of(work, struct swap_info_struct, discard_work); 51718316Swollman 51846303Smarkm spin_lock(&si->lock); 51946303Smarkm swap_do_scheduled_discard(si); 52046303Smarkm spin_unlock(&si->lock); 52118316Swollman} 52246303Smarkm 52318316Swollmanstatic void swap_users_ref_free(struct percpu_ref *ref) 52418316Swollman{ 52518316Swollman struct swap_info_struct *si; 52618316Swollman 52718316Swollman si = container_of(ref, struct swap_info_struct, users); 52818316Swollman complete(&si->comp); 52919880Swollman} 53018316Swollman 53118316Swollmanstatic void alloc_cluster(struct swap_info_struct *si, unsigned long idx) 53218316Swollman{ 53318316Swollman struct swap_cluster_info *ci = si->cluster_info; 53418316Swollman 53518316Swollman VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx); 53619880Swollman cluster_list_del_first(&si->free_clusters, ci); 53718316Swollman cluster_set_count_flag(ci + idx, 0, 0); 53818316Swollman} 53918316Swollman 54018316Swollmanstatic void free_cluster(struct swap_info_struct *si, unsigned long idx) 54118316Swollman{ 54218316Swollman struct swap_cluster_info *ci = si->cluster_info + idx; 54346303Smarkm 54446303Smarkm VM_BUG_ON(cluster_count(ci) != 0); 54546303Smarkm /* 54646303Smarkm * If the swap is discardable, prepare discard the cluster 54746303Smarkm * instead of free it immediately. The cluster will be freed 54846303Smarkm * after discard. 54918316Swollman */ 55046303Smarkm if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) == 55118316Swollman (SWP_WRITEOK | SWP_PAGE_DISCARD)) { 55246303Smarkm swap_cluster_schedule_discard(si, idx); 55318316Swollman return; 55418316Swollman } 55518316Swollman 55618316Swollman __free_cluster(si, idx); 55718316Swollman} 55846303Smarkm 55946303Smarkm/* 56046303Smarkm * The cluster corresponding to page_nr will be used. The cluster will be 56146303Smarkm * removed from free cluster list and its usage counter will be increased. 56246303Smarkm */ 56346303Smarkmstatic void inc_cluster_info_page(struct swap_info_struct *p, 56446303Smarkm struct swap_cluster_info *cluster_info, unsigned long page_nr) 56518316Swollman{ 56618316Swollman unsigned long idx = page_nr / SWAPFILE_CLUSTER; 56718316Swollman 56818316Swollman if (!cluster_info) 56918316Swollman return; 57018316Swollman if (cluster_is_free(&cluster_info[idx])) 57118316Swollman alloc_cluster(p, idx); 57218316Swollman 57346303Smarkm VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER); 57464131Ssheldonh cluster_set_count(&cluster_info[idx], 57518316Swollman cluster_count(&cluster_info[idx]) + 1); 57618316Swollman} 57719880Swollman 57818316Swollman/* 57918316Swollman * The cluster corresponding to page_nr decreases one usage. If the usage 58018316Swollman * counter becomes 0, which means no page in the cluster is in using, we can 58118316Swollman * optionally discard the cluster and add it to free cluster list. 58218316Swollman */ 58319880Swollmanstatic void dec_cluster_info_page(struct swap_info_struct *p, 58419880Swollman struct swap_cluster_info *cluster_info, unsigned long page_nr) 58519880Swollman{ 58618316Swollman unsigned long idx = page_nr / SWAPFILE_CLUSTER; 58718316Swollman 58818316Swollman if (!cluster_info) 58918316Swollman return; 59018316Swollman 59118316Swollman VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0); 59219880Swollman cluster_set_count(&cluster_info[idx], 59318316Swollman cluster_count(&cluster_info[idx]) - 1); 59418316Swollman 59518316Swollman if (cluster_count(&cluster_info[idx]) == 0) 59619880Swollman free_cluster(p, idx); 59719880Swollman} 59818316Swollman 59918316Swollman/* 60018316Swollman * It's possible scan_swap_map_slots() uses a free cluster in the middle of free 60118316Swollman * cluster list. Avoiding such abuse to avoid list corruption. 60218316Swollman */ 60318316Swollmanstatic bool 60446303Smarkmscan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si, 60518316Swollman unsigned long offset) 60646303Smarkm{ 60718316Swollman struct percpu_cluster *percpu_cluster; 60818316Swollman bool conflict; 60918316Swollman 61018316Swollman offset /= SWAPFILE_CLUSTER; 61118316Swollman conflict = !cluster_list_empty(&si->free_clusters) && 61218316Swollman offset != cluster_list_first(&si->free_clusters) && 61318316Swollman cluster_is_free(&si->cluster_info[offset]); 61418316Swollman 61518316Swollman if (!conflict) 61618316Swollman return false; 61718316Swollman 61818316Swollman percpu_cluster = this_cpu_ptr(si->percpu_cluster); 61918316Swollman cluster_set_null(&percpu_cluster->index); 62018316Swollman return true; 62118316Swollman} 62218316Swollman 62318316Swollman/* 62418316Swollman * Try to get a swap entry from current cpu's swap entry pool (a cluster). This 62518316Swollman * might involve allocating a new cluster for current CPU too. 62618316Swollman */ 62718316Swollmanstatic bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si, 62818316Swollman unsigned long *offset, unsigned long *scan_base) 62918316Swollman{ 63018316Swollman struct percpu_cluster *cluster; 63118316Swollman struct swap_cluster_info *ci; 63218316Swollman unsigned long tmp, max; 63318316Swollman 63418316Swollmannew_cluster: 63518316Swollman cluster = this_cpu_ptr(si->percpu_cluster); 63618316Swollman if (cluster_is_null(&cluster->index)) { 63737908Scharnier if (!cluster_list_empty(&si->free_clusters)) { 63818316Swollman cluster->index = si->free_clusters.head; 63918316Swollman cluster->next = cluster_next(&cluster->index) * 64018316Swollman SWAPFILE_CLUSTER; 64118316Swollman } else if (!cluster_list_empty(&si->discard_clusters)) { 64218316Swollman /* 64318316Swollman * we don't have free cluster but have some clusters in 64418316Swollman * discarding, do discard now and reclaim them, then 64518316Swollman * reread cluster_next_cpu since we dropped si->lock 64618316Swollman */ 64718316Swollman swap_do_scheduled_discard(si); 64818316Swollman *scan_base = this_cpu_read(*si->cluster_next_cpu); 64918316Swollman *offset = *scan_base; 65018316Swollman goto new_cluster; 65118316Swollman } else 65218316Swollman return false; 65318316Swollman } 65418316Swollman 65518316Swollman /* 65664131Ssheldonh * Other CPUs can use our cluster if they can't find a free cluster, 65718316Swollman * check if there is still free entry in the cluster 65818316Swollman */ 65918316Swollman tmp = cluster->next; 66018316Swollman max = min_t(unsigned long, si->max, 66118316Swollman (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER); 66218316Swollman if (tmp < max) { 66318316Swollman ci = lock_cluster(si, tmp); 66418316Swollman while (tmp < max) { 66518316Swollman if (!si->swap_map[tmp]) 66618316Swollman break; 66718316Swollman tmp++; 66818316Swollman } 66918316Swollman unlock_cluster(ci); 67018316Swollman } 67118316Swollman if (tmp >= max) { 67218316Swollman cluster_set_null(&cluster->index); 67318316Swollman goto new_cluster; 67418316Swollman } 67518316Swollman cluster->next = tmp + 1; 67618316Swollman *offset = tmp; 67718316Swollman *scan_base = tmp; 67818316Swollman return true; 67918316Swollman} 68018316Swollman 68118316Swollmanstatic void __del_from_avail_list(struct swap_info_struct *p) 68218316Swollman{ 68318316Swollman int nid; 68418316Swollman 68518316Swollman assert_spin_locked(&p->lock); 68618316Swollman for_each_node(nid) 68718316Swollman plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]); 68818316Swollman} 68918316Swollman 69018316Swollmanstatic void del_from_avail_list(struct swap_info_struct *p) 69118316Swollman{ 69218316Swollman spin_lock(&swap_avail_lock); 69318316Swollman __del_from_avail_list(p); 69418316Swollman spin_unlock(&swap_avail_lock); 69518316Swollman} 69618316Swollman 69746303Smarkmstatic void swap_range_alloc(struct swap_info_struct *si, unsigned long offset, 69818316Swollman unsigned int nr_entries) 69918316Swollman{ 700126250Sbms unsigned int end = offset + nr_entries - 1; 70118316Swollman 70246303Smarkm if (offset == si->lowest_bit) 70318316Swollman si->lowest_bit += nr_entries; 70418316Swollman if (end == si->highest_bit) 705126250Sbms WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries); 706126250Sbms WRITE_ONCE(si->inuse_pages, si->inuse_pages + nr_entries); 707126250Sbms if (si->inuse_pages == si->pages) { 70818316Swollman si->lowest_bit = si->max; 709126250Sbms si->highest_bit = 0; 71018316Swollman del_from_avail_list(si); 71118316Swollman } 71218316Swollman} 71318316Swollman 71446303Smarkmstatic void add_to_avail_list(struct swap_info_struct *p) 71520339Swollman{ 71618316Swollman int nid; 71718316Swollman 71818316Swollman spin_lock(&swap_avail_lock); 71918316Swollman for_each_node(nid) 72018316Swollman plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]); 72118316Swollman spin_unlock(&swap_avail_lock); 722126250Sbms} 72318316Swollman 72418316Swollmanstatic void swap_range_free(struct swap_info_struct *si, unsigned long offset, 725126250Sbms unsigned int nr_entries) 72618316Swollman{ 72718316Swollman unsigned long begin = offset; 72818316Swollman unsigned long end = offset + nr_entries - 1; 72918316Swollman void (*swap_slot_free_notify)(struct block_device *, unsigned long); 73018316Swollman 73118316Swollman if (offset < si->lowest_bit) 73218316Swollman si->lowest_bit = offset; 73319880Swollman if (end > si->highest_bit) { 73418316Swollman bool was_full = !si->highest_bit; 73518316Swollman 73618316Swollman WRITE_ONCE(si->highest_bit, end); 73718316Swollman if (was_full && (si->flags & SWP_WRITEOK)) 73818316Swollman add_to_avail_list(si); 739180993Sphk } 740180993Sphk if (si->flags & SWP_BLKDEV) 741180993Sphk swap_slot_free_notify = 742180993Sphk si->bdev->bd_disk->fops->swap_slot_free_notify; 74318316Swollman else 74418316Swollman swap_slot_free_notify = NULL; 745180993Sphk while (offset <= end) { 746180993Sphk arch_swap_invalidate_page(si->type, offset); 74718316Swollman if (swap_slot_free_notify) 74818316Swollman swap_slot_free_notify(si->bdev, offset); 74918316Swollman offset++; 75018316Swollman } 75118316Swollman clear_shadow_from_swap_cache(si->type, begin, end); 75218316Swollman 75318316Swollman /* 75418316Swollman * Make sure that try_to_unuse() observes si->inuse_pages reaching 0 75518316Swollman * only after the above cleanups are done. 75618316Swollman */ 75718316Swollman smp_wmb(); 75818316Swollman atomic_long_add(nr_entries, &nr_swap_pages); 75918316Swollman WRITE_ONCE(si->inuse_pages, si->inuse_pages - nr_entries); 76018316Swollman} 761126250Sbms 76218316Swollmanstatic void set_cluster_next(struct swap_info_struct *si, unsigned long next) 76318316Swollman{ 76418316Swollman unsigned long prev; 765126250Sbms 76618316Swollman if (!(si->flags & SWP_SOLIDSTATE)) { 76718316Swollman si->cluster_next = next; 76818316Swollman return; 76918316Swollman } 770126250Sbms 77118316Swollman prev = this_cpu_read(*si->cluster_next_cpu); 77218316Swollman /* 77318316Swollman * Cross the swap address space size aligned trunk, choose 77418316Swollman * another trunk randomly to avoid lock contention on swap 77518316Swollman * address space if possible. 77618316Swollman */ 77718316Swollman if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) != 77818316Swollman (next >> SWAP_ADDRESS_SPACE_SHIFT)) { 77918316Swollman /* No free swap slots available */ 78018316Swollman if (si->highest_bit <= si->lowest_bit) 78118316Swollman return; 78246303Smarkm next = get_random_u32_inclusive(si->lowest_bit, si->highest_bit); 78318316Swollman next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES); 78418316Swollman next = max_t(unsigned int, next, si->lowest_bit); 78518316Swollman } 78618316Swollman this_cpu_write(*si->cluster_next_cpu, next); 78718316Swollman} 78818316Swollman 78946303Smarkmstatic bool swap_offset_available_and_locked(struct swap_info_struct *si, 79018316Swollman unsigned long offset) 79118316Swollman{ 79218316Swollman if (data_race(!si->swap_map[offset])) { 79318316Swollman spin_lock(&si->lock); 79418316Swollman return true; 79518316Swollman } 79618316Swollman 79746303Smarkm if (vm_swap_full() && READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) { 79846303Smarkm spin_lock(&si->lock); 79946303Smarkm return true; 80046303Smarkm } 80146303Smarkm 80246303Smarkm return false; 80346303Smarkm} 80418316Swollman 80518316Swollmanstatic int scan_swap_map_slots(struct swap_info_struct *si, 80618316Swollman unsigned char usage, int nr, 80718316Swollman swp_entry_t slots[]) 80818316Swollman{ 80918316Swollman struct swap_cluster_info *ci; 81018316Swollman unsigned long offset; 81118316Swollman unsigned long scan_base; 81218316Swollman unsigned long last_in_cluster = 0; 81318316Swollman int latency_ration = LATENCY_LIMIT; 81418316Swollman int n_ret = 0; 81518316Swollman bool scanned_many = false; 81618316Swollman 81718316Swollman /* 81818316Swollman * We try to cluster swap pages by allocating them sequentially 81919880Swollman * in swap. Once we've allocated SWAPFILE_CLUSTER pages this 82019880Swollman * way, however, we resort to first-free allocation, starting 82118316Swollman * a new cluster. This prevents us from scattering swap pages 82218316Swollman * all over the entire swap partition, so that we reduce 82318316Swollman * overall disk seek times between swap pages. -- sct 824190711Sphk * But we do now try to find an empty cluster. -Andrea 82519880Swollman * And we let swap pages go all over an SSD partition. Hugh 82618316Swollman */ 82718316Swollman 82818316Swollman si->flags += SWP_SCANNING; 82918316Swollman /* 83018316Swollman * Use percpu scan base for SSD to reduce lock contention on 83118316Swollman * cluster and swap cache. For HDD, sequential access is more 83218316Swollman * important. 83318316Swollman */ 83418316Swollman if (si->flags & SWP_SOLIDSTATE) 83518316Swollman scan_base = this_cpu_read(*si->cluster_next_cpu); 83618316Swollman else 83718316Swollman scan_base = si->cluster_next; 83818316Swollman offset = scan_base; 83918316Swollman 84018316Swollman /* SSD algorithm */ 84118316Swollman if (si->cluster_info) { 84218316Swollman if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 84318316Swollman goto scan; 84418316Swollman } else if (unlikely(!si->cluster_nr--)) { 84518316Swollman if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) { 84618316Swollman si->cluster_nr = SWAPFILE_CLUSTER - 1; 84718316Swollman goto checks; 84818316Swollman } 84918316Swollman 85018316Swollman spin_unlock(&si->lock); 85118316Swollman 85218316Swollman /* 85318316Swollman * If seek is expensive, start searching for new cluster from 85418316Swollman * start of partition, to minimize the span of allocated swap. 85518316Swollman * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info 85618316Swollman * case, just handled by scan_swap_map_try_ssd_cluster() above. 85718316Swollman */ 85818316Swollman scan_base = offset = si->lowest_bit; 85918316Swollman last_in_cluster = offset + SWAPFILE_CLUSTER - 1; 86018316Swollman 86119880Swollman /* Locate the first empty (unaligned) cluster */ 86219880Swollman for (; last_in_cluster <= si->highest_bit; offset++) { 86319880Swollman if (si->swap_map[offset]) 86418316Swollman last_in_cluster = offset + SWAPFILE_CLUSTER; 86518316Swollman else if (offset == last_in_cluster) { 866190711Sphk spin_lock(&si->lock); 86719880Swollman offset -= SWAPFILE_CLUSTER - 1; 86818316Swollman si->cluster_next = offset; 86918316Swollman si->cluster_nr = SWAPFILE_CLUSTER - 1; 87018316Swollman goto checks; 87118316Swollman } 87246303Smarkm if (unlikely(--latency_ration < 0)) { 87318316Swollman cond_resched(); 87418316Swollman latency_ration = LATENCY_LIMIT; 87518316Swollman } 87618316Swollman } 87718316Swollman 87818316Swollman offset = scan_base; 87918316Swollman spin_lock(&si->lock); 88018316Swollman si->cluster_nr = SWAPFILE_CLUSTER - 1; 88118316Swollman } 88218316Swollman 88318316Swollmanchecks: 88418316Swollman if (si->cluster_info) { 88518316Swollman while (scan_swap_map_ssd_cluster_conflict(si, offset)) { 88618316Swollman /* take a break if we already got some slots */ 88718316Swollman if (n_ret) 88818316Swollman goto done; 88918316Swollman if (!scan_swap_map_try_ssd_cluster(si, &offset, 89018316Swollman &scan_base)) 89118316Swollman goto scan; 89218316Swollman } 89318316Swollman } 89418316Swollman if (!(si->flags & SWP_WRITEOK)) 89518316Swollman goto no_page; 89646303Smarkm if (!si->highest_bit) 89718316Swollman goto no_page; 89819880Swollman if (offset > si->highest_bit) 89918316Swollman scan_base = offset = si->lowest_bit; 90018316Swollman 90118316Swollman ci = lock_cluster(si, offset); 90218316Swollman /* reuse swap entry of cache-only swap if not busy. */ 90346303Smarkm if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) { 90418316Swollman int swap_was_freed; 90518316Swollman unlock_cluster(ci); 90618316Swollman spin_unlock(&si->lock); 90718316Swollman swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY); 90818316Swollman spin_lock(&si->lock); 90918316Swollman /* entry was freed successfully, try to use this again */ 91018316Swollman if (swap_was_freed) 91118316Swollman goto checks; 91218316Swollman goto scan; /* check next one */ 91318316Swollman } 91418316Swollman 91519880Swollman if (si->swap_map[offset]) { 91618316Swollman unlock_cluster(ci); 91718316Swollman if (!n_ret) 91818316Swollman goto scan; 91918316Swollman else 92018316Swollman goto done; 92118316Swollman } 92218316Swollman WRITE_ONCE(si->swap_map[offset], usage); 92318316Swollman inc_cluster_info_page(si, si->cluster_info, offset); 92418316Swollman unlock_cluster(ci); 92518316Swollman 92618316Swollman swap_range_alloc(si, offset, 1); 92718316Swollman slots[n_ret++] = swp_entry(si->type, offset); 92818316Swollman 92918316Swollman /* got enough slots or reach max slots? */ 93018316Swollman if ((n_ret == nr) || (offset >= si->highest_bit)) 93118316Swollman goto done; 93218316Swollman 93318316Swollman /* search for next available slot */ 93418316Swollman 93518316Swollman /* time to take a break? */ 93618316Swollman if (unlikely(--latency_ration < 0)) { 93719880Swollman if (n_ret) 93820606Swollman goto done; 93920606Swollman spin_unlock(&si->lock); 94020606Swollman cond_resched(); 94118316Swollman spin_lock(&si->lock); 94218316Swollman latency_ration = LATENCY_LIMIT; 94319880Swollman } 94419880Swollman 94519880Swollman /* try to get more slots in cluster */ 94619880Swollman if (si->cluster_info) { 94719880Swollman if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base)) 94819880Swollman goto checks; 94919880Swollman } else if (si->cluster_nr && !si->swap_map[++offset]) { 95019880Swollman /* non-ssd case, still more slots in cluster? */ 95119880Swollman --si->cluster_nr; 95218316Swollman goto checks; 95318316Swollman } 95418316Swollman 95518316Swollman /* 95618316Swollman * Even if there's no free clusters available (fragmented), 95718316Swollman * try to scan a little more quickly with lock held unless we 95818316Swollman * have scanned too many slots already. 95919880Swollman */ 96018316Swollman if (!scanned_many) { 96118316Swollman unsigned long scan_limit; 96218316Swollman 96318316Swollman if (offset < scan_base) 96418316Swollman scan_limit = scan_base; 96518316Swollman else 96618316Swollman scan_limit = si->highest_bit; 96718316Swollman for (; offset <= scan_limit && --latency_ration > 0; 96818316Swollman offset++) { 96918316Swollman if (!si->swap_map[offset]) 97019880Swollman goto checks; 97119880Swollman } 97219880Swollman } 97319880Swollman 97419880Swollmandone: 97518316Swollman set_cluster_next(si, offset + 1); 97619880Swollman si->flags -= SWP_SCANNING; 97718316Swollman return n_ret; 97818316Swollman 97919880Swollmanscan: 98018316Swollman spin_unlock(&si->lock); 98118316Swollman while (++offset <= READ_ONCE(si->highest_bit)) { 98219880Swollman if (unlikely(--latency_ration < 0)) { 98319880Swollman cond_resched(); 98420606Swollman latency_ration = LATENCY_LIMIT; 98520606Swollman scanned_many = true; 98620606Swollman } 98720606Swollman if (swap_offset_available_and_locked(si, offset)) 98819880Swollman goto checks; 98919880Swollman } 99019880Swollman offset = si->lowest_bit; 99119880Swollman while (offset < scan_base) { 99219880Swollman if (unlikely(--latency_ration < 0)) { 99319880Swollman cond_resched(); 99419880Swollman latency_ration = LATENCY_LIMIT; 99520606Swollman scanned_many = true; 99620606Swollman } 99718316Swollman if (swap_offset_available_and_locked(si, offset)) 99818316Swollman goto checks; 99918316Swollman offset++; 100020606Swollman } 100120606Swollman spin_lock(&si->lock); 100218316Swollman 100318316Swollmanno_page: 100418316Swollman si->flags -= SWP_SCANNING; 100518316Swollman return n_ret; 100618316Swollman} 100718316Swollman 100846303Smarkmstatic int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot) 100919880Swollman{ 101019880Swollman unsigned long idx; 101119880Swollman struct swap_cluster_info *ci; 101218316Swollman unsigned long offset; 101318316Swollman 101418316Swollman /* 101519880Swollman * Should not even be attempting cluster allocations when huge 101618316Swollman * page swap is disabled. Warn and fail the allocation. 101718316Swollman */ 101846303Smarkm if (!IS_ENABLED(CONFIG_THP_SWAP)) { 101946303Smarkm VM_WARN_ON_ONCE(1); 102046303Smarkm return 0; 102146303Smarkm } 102219880Swollman 102319880Swollman if (cluster_list_empty(&si->free_clusters)) 102419880Swollman return 0; 102518316Swollman 102618316Swollman idx = cluster_list_first(&si->free_clusters); 102718316Swollman offset = idx * SWAPFILE_CLUSTER; 102818316Swollman ci = lock_cluster(si, offset); 102918316Swollman alloc_cluster(si, idx); 103018316Swollman cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE); 103118316Swollman 103218316Swollman memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER); 103318316Swollman unlock_cluster(ci); 103418316Swollman swap_range_alloc(si, offset, SWAPFILE_CLUSTER); 103518316Swollman *slot = swp_entry(si->type, offset); 103618316Swollman 103718316Swollman return 1; 103818316Swollman} 103918316Swollman 104018316Swollmanstatic void swap_free_cluster(struct swap_info_struct *si, unsigned long idx) 104118316Swollman{ 104218316Swollman unsigned long offset = idx * SWAPFILE_CLUSTER; 104318316Swollman struct swap_cluster_info *ci; 104418316Swollman 104518316Swollman ci = lock_cluster(si, offset); 104618316Swollman memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER); 104718316Swollman cluster_set_count_flag(ci, 0, 0); 104846303Smarkm free_cluster(si, idx); 104946303Smarkm unlock_cluster(ci); 105018316Swollman swap_range_free(si, offset, SWAPFILE_CLUSTER); 105118316Swollman} 105218316Swollman 105318316Swollmanint get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size) 105418316Swollman{ 105518316Swollman unsigned long size = swap_entry_size(entry_size); 105618316Swollman struct swap_info_struct *si, *next; 105718316Swollman long avail_pgs; 105818316Swollman int n_ret = 0; 105918316Swollman int node; 106018316Swollman 106118316Swollman /* Only single cluster request supported */ 106218316Swollman WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER); 1063 1064 spin_lock(&swap_avail_lock); 1065 1066 avail_pgs = atomic_long_read(&nr_swap_pages) / size; 1067 if (avail_pgs <= 0) { 1068 spin_unlock(&swap_avail_lock); 1069 goto noswap; 1070 } 1071 1072 n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs); 1073 1074 atomic_long_sub(n_goal * size, &nr_swap_pages); 1075 1076start_over: 1077 node = numa_node_id(); 1078 plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) { 1079 /* requeue si to after same-priority siblings */ 1080 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]); 1081 spin_unlock(&swap_avail_lock); 1082 spin_lock(&si->lock); 1083 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) { 1084 spin_lock(&swap_avail_lock); 1085 if (plist_node_empty(&si->avail_lists[node])) { 1086 spin_unlock(&si->lock); 1087 goto nextsi; 1088 } 1089 WARN(!si->highest_bit, 1090 "swap_info %d in list but !highest_bit\n", 1091 si->type); 1092 WARN(!(si->flags & SWP_WRITEOK), 1093 "swap_info %d in list but !SWP_WRITEOK\n", 1094 si->type); 1095 __del_from_avail_list(si); 1096 spin_unlock(&si->lock); 1097 goto nextsi; 1098 } 1099 if (size == SWAPFILE_CLUSTER) { 1100 if (si->flags & SWP_BLKDEV) 1101 n_ret = swap_alloc_cluster(si, swp_entries); 1102 } else 1103 n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE, 1104 n_goal, swp_entries); 1105 spin_unlock(&si->lock); 1106 if (n_ret || size == SWAPFILE_CLUSTER) 1107 goto check_out; 1108 cond_resched(); 1109 1110 spin_lock(&swap_avail_lock); 1111nextsi: 1112 /* 1113 * if we got here, it's likely that si was almost full before, 1114 * and since scan_swap_map_slots() can drop the si->lock, 1115 * multiple callers probably all tried to get a page from the 1116 * same si and it filled up before we could get one; or, the si 1117 * filled up between us dropping swap_avail_lock and taking 1118 * si->lock. Since we dropped the swap_avail_lock, the 1119 * swap_avail_head list may have been modified; so if next is 1120 * still in the swap_avail_head list then try it, otherwise 1121 * start over if we have not gotten any slots. 1122 */ 1123 if (plist_node_empty(&next->avail_lists[node])) 1124 goto start_over; 1125 } 1126 1127 spin_unlock(&swap_avail_lock); 1128 1129check_out: 1130 if (n_ret < n_goal) 1131 atomic_long_add((long)(n_goal - n_ret) * size, 1132 &nr_swap_pages); 1133noswap: 1134 return n_ret; 1135} 1136 1137static struct swap_info_struct *_swap_info_get(swp_entry_t entry) 1138{ 1139 struct swap_info_struct *p; 1140 unsigned long offset; 1141 1142 if (!entry.val) 1143 goto out; 1144 p = swp_swap_info(entry); 1145 if (!p) 1146 goto bad_nofile; 1147 if (data_race(!(p->flags & SWP_USED))) 1148 goto bad_device; 1149 offset = swp_offset(entry); 1150 if (offset >= p->max) 1151 goto bad_offset; 1152 if (data_race(!p->swap_map[swp_offset(entry)])) 1153 goto bad_free; 1154 return p; 1155 1156bad_free: 1157 pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val); 1158 goto out; 1159bad_offset: 1160 pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); 1161 goto out; 1162bad_device: 1163 pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val); 1164 goto out; 1165bad_nofile: 1166 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1167out: 1168 return NULL; 1169} 1170 1171static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry, 1172 struct swap_info_struct *q) 1173{ 1174 struct swap_info_struct *p; 1175 1176 p = _swap_info_get(entry); 1177 1178 if (p != q) { 1179 if (q != NULL) 1180 spin_unlock(&q->lock); 1181 if (p != NULL) 1182 spin_lock(&p->lock); 1183 } 1184 return p; 1185} 1186 1187static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, 1188 unsigned long offset, 1189 unsigned char usage) 1190{ 1191 unsigned char count; 1192 unsigned char has_cache; 1193 1194 count = p->swap_map[offset]; 1195 1196 has_cache = count & SWAP_HAS_CACHE; 1197 count &= ~SWAP_HAS_CACHE; 1198 1199 if (usage == SWAP_HAS_CACHE) { 1200 VM_BUG_ON(!has_cache); 1201 has_cache = 0; 1202 } else if (count == SWAP_MAP_SHMEM) { 1203 /* 1204 * Or we could insist on shmem.c using a special 1205 * swap_shmem_free() and free_shmem_swap_and_cache()... 1206 */ 1207 count = 0; 1208 } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) { 1209 if (count == COUNT_CONTINUED) { 1210 if (swap_count_continued(p, offset, count)) 1211 count = SWAP_MAP_MAX | COUNT_CONTINUED; 1212 else 1213 count = SWAP_MAP_MAX; 1214 } else 1215 count--; 1216 } 1217 1218 usage = count | has_cache; 1219 if (usage) 1220 WRITE_ONCE(p->swap_map[offset], usage); 1221 else 1222 WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE); 1223 1224 return usage; 1225} 1226 1227/* 1228 * When we get a swap entry, if there aren't some other ways to 1229 * prevent swapoff, such as the folio in swap cache is locked, page 1230 * table lock is held, etc., the swap entry may become invalid because 1231 * of swapoff. Then, we need to enclose all swap related functions 1232 * with get_swap_device() and put_swap_device(), unless the swap 1233 * functions call get/put_swap_device() by themselves. 1234 * 1235 * Note that when only holding the PTL, swapoff might succeed immediately 1236 * after freeing a swap entry. Therefore, immediately after 1237 * __swap_entry_free(), the swap info might become stale and should not 1238 * be touched without a prior get_swap_device(). 1239 * 1240 * Check whether swap entry is valid in the swap device. If so, 1241 * return pointer to swap_info_struct, and keep the swap entry valid 1242 * via preventing the swap device from being swapoff, until 1243 * put_swap_device() is called. Otherwise return NULL. 1244 * 1245 * Notice that swapoff or swapoff+swapon can still happen before the 1246 * percpu_ref_tryget_live() in get_swap_device() or after the 1247 * percpu_ref_put() in put_swap_device() if there isn't any other way 1248 * to prevent swapoff. The caller must be prepared for that. For 1249 * example, the following situation is possible. 1250 * 1251 * CPU1 CPU2 1252 * do_swap_page() 1253 * ... swapoff+swapon 1254 * __read_swap_cache_async() 1255 * swapcache_prepare() 1256 * __swap_duplicate() 1257 * // check swap_map 1258 * // verify PTE not changed 1259 * 1260 * In __swap_duplicate(), the swap_map need to be checked before 1261 * changing partly because the specified swap entry may be for another 1262 * swap device which has been swapoff. And in do_swap_page(), after 1263 * the page is read from the swap device, the PTE is verified not 1264 * changed with the page table locked to check whether the swap device 1265 * has been swapoff or swapoff+swapon. 1266 */ 1267struct swap_info_struct *get_swap_device(swp_entry_t entry) 1268{ 1269 struct swap_info_struct *si; 1270 unsigned long offset; 1271 1272 if (!entry.val) 1273 goto out; 1274 si = swp_swap_info(entry); 1275 if (!si) 1276 goto bad_nofile; 1277 if (!percpu_ref_tryget_live(&si->users)) 1278 goto out; 1279 /* 1280 * Guarantee the si->users are checked before accessing other 1281 * fields of swap_info_struct. 1282 * 1283 * Paired with the spin_unlock() after setup_swap_info() in 1284 * enable_swap_info(). 1285 */ 1286 smp_rmb(); 1287 offset = swp_offset(entry); 1288 if (offset >= si->max) 1289 goto put_out; 1290 1291 return si; 1292bad_nofile: 1293 pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); 1294out: 1295 return NULL; 1296put_out: 1297 pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); 1298 percpu_ref_put(&si->users); 1299 return NULL; 1300} 1301 1302static unsigned char __swap_entry_free(struct swap_info_struct *p, 1303 swp_entry_t entry) 1304{ 1305 struct swap_cluster_info *ci; 1306 unsigned long offset = swp_offset(entry); 1307 unsigned char usage; 1308 1309 ci = lock_cluster_or_swap_info(p, offset); 1310 usage = __swap_entry_free_locked(p, offset, 1); 1311 unlock_cluster_or_swap_info(p, ci); 1312 if (!usage) 1313 free_swap_slot(entry); 1314 1315 return usage; 1316} 1317 1318static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) 1319{ 1320 struct swap_cluster_info *ci; 1321 unsigned long offset = swp_offset(entry); 1322 unsigned char count; 1323 1324 ci = lock_cluster(p, offset); 1325 count = p->swap_map[offset]; 1326 VM_BUG_ON(count != SWAP_HAS_CACHE); 1327 p->swap_map[offset] = 0; 1328 dec_cluster_info_page(p, p->cluster_info, offset); 1329 unlock_cluster(ci); 1330 1331 mem_cgroup_uncharge_swap(entry, 1); 1332 swap_range_free(p, offset, 1); 1333} 1334 1335/* 1336 * Caller has made sure that the swap device corresponding to entry 1337 * is still around or has not been recycled. 1338 */ 1339void swap_free(swp_entry_t entry) 1340{ 1341 struct swap_info_struct *p; 1342 1343 p = _swap_info_get(entry); 1344 if (p) 1345 __swap_entry_free(p, entry); 1346} 1347 1348/* 1349 * Called after dropping swapcache to decrease refcnt to swap entries. 1350 */ 1351void put_swap_folio(struct folio *folio, swp_entry_t entry) 1352{ 1353 unsigned long offset = swp_offset(entry); 1354 unsigned long idx = offset / SWAPFILE_CLUSTER; 1355 struct swap_cluster_info *ci; 1356 struct swap_info_struct *si; 1357 unsigned char *map; 1358 unsigned int i, free_entries = 0; 1359 unsigned char val; 1360 int size = swap_entry_size(folio_nr_pages(folio)); 1361 1362 si = _swap_info_get(entry); 1363 if (!si) 1364 return; 1365 1366 ci = lock_cluster_or_swap_info(si, offset); 1367 if (size == SWAPFILE_CLUSTER) { 1368 VM_BUG_ON(!cluster_is_huge(ci)); 1369 map = si->swap_map + offset; 1370 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1371 val = map[i]; 1372 VM_BUG_ON(!(val & SWAP_HAS_CACHE)); 1373 if (val == SWAP_HAS_CACHE) 1374 free_entries++; 1375 } 1376 cluster_clear_huge(ci); 1377 if (free_entries == SWAPFILE_CLUSTER) { 1378 unlock_cluster_or_swap_info(si, ci); 1379 spin_lock(&si->lock); 1380 mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER); 1381 swap_free_cluster(si, idx); 1382 spin_unlock(&si->lock); 1383 return; 1384 } 1385 } 1386 for (i = 0; i < size; i++, entry.val++) { 1387 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) { 1388 unlock_cluster_or_swap_info(si, ci); 1389 free_swap_slot(entry); 1390 if (i == size - 1) 1391 return; 1392 lock_cluster_or_swap_info(si, offset); 1393 } 1394 } 1395 unlock_cluster_or_swap_info(si, ci); 1396} 1397 1398#ifdef CONFIG_THP_SWAP 1399int split_swap_cluster(swp_entry_t entry) 1400{ 1401 struct swap_info_struct *si; 1402 struct swap_cluster_info *ci; 1403 unsigned long offset = swp_offset(entry); 1404 1405 si = _swap_info_get(entry); 1406 if (!si) 1407 return -EBUSY; 1408 ci = lock_cluster(si, offset); 1409 cluster_clear_huge(ci); 1410 unlock_cluster(ci); 1411 return 0; 1412} 1413#endif 1414 1415static int swp_entry_cmp(const void *ent1, const void *ent2) 1416{ 1417 const swp_entry_t *e1 = ent1, *e2 = ent2; 1418 1419 return (int)swp_type(*e1) - (int)swp_type(*e2); 1420} 1421 1422void swapcache_free_entries(swp_entry_t *entries, int n) 1423{ 1424 struct swap_info_struct *p, *prev; 1425 int i; 1426 1427 if (n <= 0) 1428 return; 1429 1430 prev = NULL; 1431 p = NULL; 1432 1433 /* 1434 * Sort swap entries by swap device, so each lock is only taken once. 1435 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is 1436 * so low that it isn't necessary to optimize further. 1437 */ 1438 if (nr_swapfiles > 1) 1439 sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL); 1440 for (i = 0; i < n; ++i) { 1441 p = swap_info_get_cont(entries[i], prev); 1442 if (p) 1443 swap_entry_free(p, entries[i]); 1444 prev = p; 1445 } 1446 if (p) 1447 spin_unlock(&p->lock); 1448} 1449 1450int __swap_count(swp_entry_t entry) 1451{ 1452 struct swap_info_struct *si = swp_swap_info(entry); 1453 pgoff_t offset = swp_offset(entry); 1454 1455 return swap_count(si->swap_map[offset]); 1456} 1457 1458/* 1459 * How many references to @entry are currently swapped out? 1460 * This does not give an exact answer when swap count is continued, 1461 * but does include the high COUNT_CONTINUED flag to allow for that. 1462 */ 1463int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry) 1464{ 1465 pgoff_t offset = swp_offset(entry); 1466 struct swap_cluster_info *ci; 1467 int count; 1468 1469 ci = lock_cluster_or_swap_info(si, offset); 1470 count = swap_count(si->swap_map[offset]); 1471 unlock_cluster_or_swap_info(si, ci); 1472 return count; 1473} 1474 1475/* 1476 * How many references to @entry are currently swapped out? 1477 * This considers COUNT_CONTINUED so it returns exact answer. 1478 */ 1479int swp_swapcount(swp_entry_t entry) 1480{ 1481 int count, tmp_count, n; 1482 struct swap_info_struct *p; 1483 struct swap_cluster_info *ci; 1484 struct page *page; 1485 pgoff_t offset; 1486 unsigned char *map; 1487 1488 p = _swap_info_get(entry); 1489 if (!p) 1490 return 0; 1491 1492 offset = swp_offset(entry); 1493 1494 ci = lock_cluster_or_swap_info(p, offset); 1495 1496 count = swap_count(p->swap_map[offset]); 1497 if (!(count & COUNT_CONTINUED)) 1498 goto out; 1499 1500 count &= ~COUNT_CONTINUED; 1501 n = SWAP_MAP_MAX + 1; 1502 1503 page = vmalloc_to_page(p->swap_map + offset); 1504 offset &= ~PAGE_MASK; 1505 VM_BUG_ON(page_private(page) != SWP_CONTINUED); 1506 1507 do { 1508 page = list_next_entry(page, lru); 1509 map = kmap_local_page(page); 1510 tmp_count = map[offset]; 1511 kunmap_local(map); 1512 1513 count += (tmp_count & ~COUNT_CONTINUED) * n; 1514 n *= (SWAP_CONT_MAX + 1); 1515 } while (tmp_count & COUNT_CONTINUED); 1516out: 1517 unlock_cluster_or_swap_info(p, ci); 1518 return count; 1519} 1520 1521static bool swap_page_trans_huge_swapped(struct swap_info_struct *si, 1522 swp_entry_t entry) 1523{ 1524 struct swap_cluster_info *ci; 1525 unsigned char *map = si->swap_map; 1526 unsigned long roffset = swp_offset(entry); 1527 unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER); 1528 int i; 1529 bool ret = false; 1530 1531 ci = lock_cluster_or_swap_info(si, offset); 1532 if (!ci || !cluster_is_huge(ci)) { 1533 if (swap_count(map[roffset])) 1534 ret = true; 1535 goto unlock_out; 1536 } 1537 for (i = 0; i < SWAPFILE_CLUSTER; i++) { 1538 if (swap_count(map[offset + i])) { 1539 ret = true; 1540 break; 1541 } 1542 } 1543unlock_out: 1544 unlock_cluster_or_swap_info(si, ci); 1545 return ret; 1546} 1547 1548static bool folio_swapped(struct folio *folio) 1549{ 1550 swp_entry_t entry = folio->swap; 1551 struct swap_info_struct *si = _swap_info_get(entry); 1552 1553 if (!si) 1554 return false; 1555 1556 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio))) 1557 return swap_swapcount(si, entry) != 0; 1558 1559 return swap_page_trans_huge_swapped(si, entry); 1560} 1561 1562/** 1563 * folio_free_swap() - Free the swap space used for this folio. 1564 * @folio: The folio to remove. 1565 * 1566 * If swap is getting full, or if there are no more mappings of this folio, 1567 * then call folio_free_swap to free its swap space. 1568 * 1569 * Return: true if we were able to release the swap space. 1570 */ 1571bool folio_free_swap(struct folio *folio) 1572{ 1573 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 1574 1575 if (!folio_test_swapcache(folio)) 1576 return false; 1577 if (folio_test_writeback(folio)) 1578 return false; 1579 if (folio_swapped(folio)) 1580 return false; 1581 1582 /* 1583 * Once hibernation has begun to create its image of memory, 1584 * there's a danger that one of the calls to folio_free_swap() 1585 * - most probably a call from __try_to_reclaim_swap() while 1586 * hibernation is allocating its own swap pages for the image, 1587 * but conceivably even a call from memory reclaim - will free 1588 * the swap from a folio which has already been recorded in the 1589 * image as a clean swapcache folio, and then reuse its swap for 1590 * another page of the image. On waking from hibernation, the 1591 * original folio might be freed under memory pressure, then 1592 * later read back in from swap, now with the wrong data. 1593 * 1594 * Hibernation suspends storage while it is writing the image 1595 * to disk so check that here. 1596 */ 1597 if (pm_suspended_storage()) 1598 return false; 1599 1600 delete_from_swap_cache(folio); 1601 folio_set_dirty(folio); 1602 return true; 1603} 1604 1605/* 1606 * Free the swap entry like above, but also try to 1607 * free the page cache entry if it is the last user. 1608 */ 1609int free_swap_and_cache(swp_entry_t entry) 1610{ 1611 struct swap_info_struct *p; 1612 unsigned char count; 1613 1614 if (non_swap_entry(entry)) 1615 return 1; 1616 1617 p = get_swap_device(entry); 1618 if (p) { 1619 if (WARN_ON(data_race(!p->swap_map[swp_offset(entry)]))) { 1620 put_swap_device(p); 1621 return 0; 1622 } 1623 1624 count = __swap_entry_free(p, entry); 1625 if (count == SWAP_HAS_CACHE && 1626 !swap_page_trans_huge_swapped(p, entry)) 1627 __try_to_reclaim_swap(p, swp_offset(entry), 1628 TTRS_UNMAPPED | TTRS_FULL); 1629 put_swap_device(p); 1630 } 1631 return p != NULL; 1632} 1633 1634#ifdef CONFIG_HIBERNATION 1635 1636swp_entry_t get_swap_page_of_type(int type) 1637{ 1638 struct swap_info_struct *si = swap_type_to_swap_info(type); 1639 swp_entry_t entry = {0}; 1640 1641 if (!si) 1642 goto fail; 1643 1644 /* This is called for allocating swap entry, not cache */ 1645 spin_lock(&si->lock); 1646 if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry)) 1647 atomic_long_dec(&nr_swap_pages); 1648 spin_unlock(&si->lock); 1649fail: 1650 return entry; 1651} 1652 1653/* 1654 * Find the swap type that corresponds to given device (if any). 1655 * 1656 * @offset - number of the PAGE_SIZE-sized block of the device, starting 1657 * from 0, in which the swap header is expected to be located. 1658 * 1659 * This is needed for the suspend to disk (aka swsusp). 1660 */ 1661int swap_type_of(dev_t device, sector_t offset) 1662{ 1663 int type; 1664 1665 if (!device) 1666 return -1; 1667 1668 spin_lock(&swap_lock); 1669 for (type = 0; type < nr_swapfiles; type++) { 1670 struct swap_info_struct *sis = swap_info[type]; 1671 1672 if (!(sis->flags & SWP_WRITEOK)) 1673 continue; 1674 1675 if (device == sis->bdev->bd_dev) { 1676 struct swap_extent *se = first_se(sis); 1677 1678 if (se->start_block == offset) { 1679 spin_unlock(&swap_lock); 1680 return type; 1681 } 1682 } 1683 } 1684 spin_unlock(&swap_lock); 1685 return -ENODEV; 1686} 1687 1688int find_first_swap(dev_t *device) 1689{ 1690 int type; 1691 1692 spin_lock(&swap_lock); 1693 for (type = 0; type < nr_swapfiles; type++) { 1694 struct swap_info_struct *sis = swap_info[type]; 1695 1696 if (!(sis->flags & SWP_WRITEOK)) 1697 continue; 1698 *device = sis->bdev->bd_dev; 1699 spin_unlock(&swap_lock); 1700 return type; 1701 } 1702 spin_unlock(&swap_lock); 1703 return -ENODEV; 1704} 1705 1706/* 1707 * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev 1708 * corresponding to given index in swap_info (swap type). 1709 */ 1710sector_t swapdev_block(int type, pgoff_t offset) 1711{ 1712 struct swap_info_struct *si = swap_type_to_swap_info(type); 1713 struct swap_extent *se; 1714 1715 if (!si || !(si->flags & SWP_WRITEOK)) 1716 return 0; 1717 se = offset_to_swap_extent(si, offset); 1718 return se->start_block + (offset - se->start_page); 1719} 1720 1721/* 1722 * Return either the total number of swap pages of given type, or the number 1723 * of free pages of that type (depending on @free) 1724 * 1725 * This is needed for software suspend 1726 */ 1727unsigned int count_swap_pages(int type, int free) 1728{ 1729 unsigned int n = 0; 1730 1731 spin_lock(&swap_lock); 1732 if ((unsigned int)type < nr_swapfiles) { 1733 struct swap_info_struct *sis = swap_info[type]; 1734 1735 spin_lock(&sis->lock); 1736 if (sis->flags & SWP_WRITEOK) { 1737 n = sis->pages; 1738 if (free) 1739 n -= sis->inuse_pages; 1740 } 1741 spin_unlock(&sis->lock); 1742 } 1743 spin_unlock(&swap_lock); 1744 return n; 1745} 1746#endif /* CONFIG_HIBERNATION */ 1747 1748static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) 1749{ 1750 return pte_same(pte_swp_clear_flags(pte), swp_pte); 1751} 1752 1753/* 1754 * No need to decide whether this PTE shares the swap entry with others, 1755 * just let do_wp_page work it out if a write is requested later - to 1756 * force COW, vm_page_prot omits write permission from any private vma. 1757 */ 1758static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, 1759 unsigned long addr, swp_entry_t entry, struct folio *folio) 1760{ 1761 struct page *page; 1762 struct folio *swapcache; 1763 spinlock_t *ptl; 1764 pte_t *pte, new_pte, old_pte; 1765 bool hwpoisoned = false; 1766 int ret = 1; 1767 1768 swapcache = folio; 1769 folio = ksm_might_need_to_copy(folio, vma, addr); 1770 if (unlikely(!folio)) 1771 return -ENOMEM; 1772 else if (unlikely(folio == ERR_PTR(-EHWPOISON))) { 1773 hwpoisoned = true; 1774 folio = swapcache; 1775 } 1776 1777 page = folio_file_page(folio, swp_offset(entry)); 1778 if (PageHWPoison(page)) 1779 hwpoisoned = true; 1780 1781 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1782 if (unlikely(!pte || !pte_same_as_swp(ptep_get(pte), 1783 swp_entry_to_pte(entry)))) { 1784 ret = 0; 1785 goto out; 1786 } 1787 1788 old_pte = ptep_get(pte); 1789 1790 if (unlikely(hwpoisoned || !folio_test_uptodate(folio))) { 1791 swp_entry_t swp_entry; 1792 1793 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 1794 if (hwpoisoned) { 1795 swp_entry = make_hwpoison_entry(page); 1796 } else { 1797 swp_entry = make_poisoned_swp_entry(); 1798 } 1799 new_pte = swp_entry_to_pte(swp_entry); 1800 ret = 0; 1801 goto setpte; 1802 } 1803 1804 /* 1805 * Some architectures may have to restore extra metadata to the page 1806 * when reading from swap. This metadata may be indexed by swap entry 1807 * so this must be called before swap_free(). 1808 */ 1809 arch_swap_restore(entry, folio); 1810 1811 dec_mm_counter(vma->vm_mm, MM_SWAPENTS); 1812 inc_mm_counter(vma->vm_mm, MM_ANONPAGES); 1813 folio_get(folio); 1814 if (folio == swapcache) { 1815 rmap_t rmap_flags = RMAP_NONE; 1816 1817 /* 1818 * See do_swap_page(): writeback would be problematic. 1819 * However, we do a folio_wait_writeback() just before this 1820 * call and have the folio locked. 1821 */ 1822 VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); 1823 if (pte_swp_exclusive(old_pte)) 1824 rmap_flags |= RMAP_EXCLUSIVE; 1825 1826 folio_add_anon_rmap_pte(folio, page, vma, addr, rmap_flags); 1827 } else { /* ksm created a completely new copy */ 1828 folio_add_new_anon_rmap(folio, vma, addr); 1829 folio_add_lru_vma(folio, vma); 1830 } 1831 new_pte = pte_mkold(mk_pte(page, vma->vm_page_prot)); 1832 if (pte_swp_soft_dirty(old_pte)) 1833 new_pte = pte_mksoft_dirty(new_pte); 1834 if (pte_swp_uffd_wp(old_pte)) 1835 new_pte = pte_mkuffd_wp(new_pte); 1836setpte: 1837 set_pte_at(vma->vm_mm, addr, pte, new_pte); 1838 swap_free(entry); 1839out: 1840 if (pte) 1841 pte_unmap_unlock(pte, ptl); 1842 if (folio != swapcache) { 1843 folio_unlock(folio); 1844 folio_put(folio); 1845 } 1846 return ret; 1847} 1848 1849static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 1850 unsigned long addr, unsigned long end, 1851 unsigned int type) 1852{ 1853 pte_t *pte = NULL; 1854 struct swap_info_struct *si; 1855 1856 si = swap_info[type]; 1857 do { 1858 struct folio *folio; 1859 unsigned long offset; 1860 unsigned char swp_count; 1861 swp_entry_t entry; 1862 int ret; 1863 pte_t ptent; 1864 1865 if (!pte++) { 1866 pte = pte_offset_map(pmd, addr); 1867 if (!pte) 1868 break; 1869 } 1870 1871 ptent = ptep_get_lockless(pte); 1872 1873 if (!is_swap_pte(ptent)) 1874 continue; 1875 1876 entry = pte_to_swp_entry(ptent); 1877 if (swp_type(entry) != type) 1878 continue; 1879 1880 offset = swp_offset(entry); 1881 pte_unmap(pte); 1882 pte = NULL; 1883 1884 folio = swap_cache_get_folio(entry, vma, addr); 1885 if (!folio) { 1886 struct page *page; 1887 struct vm_fault vmf = { 1888 .vma = vma, 1889 .address = addr, 1890 .real_address = addr, 1891 .pmd = pmd, 1892 }; 1893 1894 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, 1895 &vmf); 1896 if (page) 1897 folio = page_folio(page); 1898 } 1899 if (!folio) { 1900 swp_count = READ_ONCE(si->swap_map[offset]); 1901 if (swp_count == 0 || swp_count == SWAP_MAP_BAD) 1902 continue; 1903 return -ENOMEM; 1904 } 1905 1906 folio_lock(folio); 1907 folio_wait_writeback(folio); 1908 ret = unuse_pte(vma, pmd, addr, entry, folio); 1909 if (ret < 0) { 1910 folio_unlock(folio); 1911 folio_put(folio); 1912 return ret; 1913 } 1914 1915 folio_free_swap(folio); 1916 folio_unlock(folio); 1917 folio_put(folio); 1918 } while (addr += PAGE_SIZE, addr != end); 1919 1920 if (pte) 1921 pte_unmap(pte); 1922 return 0; 1923} 1924 1925static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, 1926 unsigned long addr, unsigned long end, 1927 unsigned int type) 1928{ 1929 pmd_t *pmd; 1930 unsigned long next; 1931 int ret; 1932 1933 pmd = pmd_offset(pud, addr); 1934 do { 1935 cond_resched(); 1936 next = pmd_addr_end(addr, end); 1937 ret = unuse_pte_range(vma, pmd, addr, next, type); 1938 if (ret) 1939 return ret; 1940 } while (pmd++, addr = next, addr != end); 1941 return 0; 1942} 1943 1944static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, 1945 unsigned long addr, unsigned long end, 1946 unsigned int type) 1947{ 1948 pud_t *pud; 1949 unsigned long next; 1950 int ret; 1951 1952 pud = pud_offset(p4d, addr); 1953 do { 1954 next = pud_addr_end(addr, end); 1955 if (pud_none_or_clear_bad(pud)) 1956 continue; 1957 ret = unuse_pmd_range(vma, pud, addr, next, type); 1958 if (ret) 1959 return ret; 1960 } while (pud++, addr = next, addr != end); 1961 return 0; 1962} 1963 1964static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, 1965 unsigned long addr, unsigned long end, 1966 unsigned int type) 1967{ 1968 p4d_t *p4d; 1969 unsigned long next; 1970 int ret; 1971 1972 p4d = p4d_offset(pgd, addr); 1973 do { 1974 next = p4d_addr_end(addr, end); 1975 if (p4d_none_or_clear_bad(p4d)) 1976 continue; 1977 ret = unuse_pud_range(vma, p4d, addr, next, type); 1978 if (ret) 1979 return ret; 1980 } while (p4d++, addr = next, addr != end); 1981 return 0; 1982} 1983 1984static int unuse_vma(struct vm_area_struct *vma, unsigned int type) 1985{ 1986 pgd_t *pgd; 1987 unsigned long addr, end, next; 1988 int ret; 1989 1990 addr = vma->vm_start; 1991 end = vma->vm_end; 1992 1993 pgd = pgd_offset(vma->vm_mm, addr); 1994 do { 1995 next = pgd_addr_end(addr, end); 1996 if (pgd_none_or_clear_bad(pgd)) 1997 continue; 1998 ret = unuse_p4d_range(vma, pgd, addr, next, type); 1999 if (ret) 2000 return ret; 2001 } while (pgd++, addr = next, addr != end); 2002 return 0; 2003} 2004 2005static int unuse_mm(struct mm_struct *mm, unsigned int type) 2006{ 2007 struct vm_area_struct *vma; 2008 int ret = 0; 2009 VMA_ITERATOR(vmi, mm, 0); 2010 2011 mmap_read_lock(mm); 2012 for_each_vma(vmi, vma) { 2013 if (vma->anon_vma) { 2014 ret = unuse_vma(vma, type); 2015 if (ret) 2016 break; 2017 } 2018 2019 cond_resched(); 2020 } 2021 mmap_read_unlock(mm); 2022 return ret; 2023} 2024 2025/* 2026 * Scan swap_map from current position to next entry still in use. 2027 * Return 0 if there are no inuse entries after prev till end of 2028 * the map. 2029 */ 2030static unsigned int find_next_to_unuse(struct swap_info_struct *si, 2031 unsigned int prev) 2032{ 2033 unsigned int i; 2034 unsigned char count; 2035 2036 /* 2037 * No need for swap_lock here: we're just looking 2038 * for whether an entry is in use, not modifying it; false 2039 * hits are okay, and sys_swapoff() has already prevented new 2040 * allocations from this area (while holding swap_lock). 2041 */ 2042 for (i = prev + 1; i < si->max; i++) { 2043 count = READ_ONCE(si->swap_map[i]); 2044 if (count && swap_count(count) != SWAP_MAP_BAD) 2045 break; 2046 if ((i % LATENCY_LIMIT) == 0) 2047 cond_resched(); 2048 } 2049 2050 if (i == si->max) 2051 i = 0; 2052 2053 return i; 2054} 2055 2056static int try_to_unuse(unsigned int type) 2057{ 2058 struct mm_struct *prev_mm; 2059 struct mm_struct *mm; 2060 struct list_head *p; 2061 int retval = 0; 2062 struct swap_info_struct *si = swap_info[type]; 2063 struct folio *folio; 2064 swp_entry_t entry; 2065 unsigned int i; 2066 2067 if (!READ_ONCE(si->inuse_pages)) 2068 goto success; 2069 2070retry: 2071 retval = shmem_unuse(type); 2072 if (retval) 2073 return retval; 2074 2075 prev_mm = &init_mm; 2076 mmget(prev_mm); 2077 2078 spin_lock(&mmlist_lock); 2079 p = &init_mm.mmlist; 2080 while (READ_ONCE(si->inuse_pages) && 2081 !signal_pending(current) && 2082 (p = p->next) != &init_mm.mmlist) { 2083 2084 mm = list_entry(p, struct mm_struct, mmlist); 2085 if (!mmget_not_zero(mm)) 2086 continue; 2087 spin_unlock(&mmlist_lock); 2088 mmput(prev_mm); 2089 prev_mm = mm; 2090 retval = unuse_mm(mm, type); 2091 if (retval) { 2092 mmput(prev_mm); 2093 return retval; 2094 } 2095 2096 /* 2097 * Make sure that we aren't completely killing 2098 * interactive performance. 2099 */ 2100 cond_resched(); 2101 spin_lock(&mmlist_lock); 2102 } 2103 spin_unlock(&mmlist_lock); 2104 2105 mmput(prev_mm); 2106 2107 i = 0; 2108 while (READ_ONCE(si->inuse_pages) && 2109 !signal_pending(current) && 2110 (i = find_next_to_unuse(si, i)) != 0) { 2111 2112 entry = swp_entry(type, i); 2113 folio = filemap_get_folio(swap_address_space(entry), i); 2114 if (IS_ERR(folio)) 2115 continue; 2116 2117 /* 2118 * It is conceivable that a racing task removed this folio from 2119 * swap cache just before we acquired the page lock. The folio 2120 * might even be back in swap cache on another swap area. But 2121 * that is okay, folio_free_swap() only removes stale folios. 2122 */ 2123 folio_lock(folio); 2124 folio_wait_writeback(folio); 2125 folio_free_swap(folio); 2126 folio_unlock(folio); 2127 folio_put(folio); 2128 } 2129 2130 /* 2131 * Lets check again to see if there are still swap entries in the map. 2132 * If yes, we would need to do retry the unuse logic again. 2133 * Under global memory pressure, swap entries can be reinserted back 2134 * into process space after the mmlist loop above passes over them. 2135 * 2136 * Limit the number of retries? No: when mmget_not_zero() 2137 * above fails, that mm is likely to be freeing swap from 2138 * exit_mmap(), which proceeds at its own independent pace; 2139 * and even shmem_writepage() could have been preempted after 2140 * folio_alloc_swap(), temporarily hiding that swap. It's easy 2141 * and robust (though cpu-intensive) just to keep retrying. 2142 */ 2143 if (READ_ONCE(si->inuse_pages)) { 2144 if (!signal_pending(current)) 2145 goto retry; 2146 return -EINTR; 2147 } 2148 2149success: 2150 /* 2151 * Make sure that further cleanups after try_to_unuse() returns happen 2152 * after swap_range_free() reduces si->inuse_pages to 0. 2153 */ 2154 smp_mb(); 2155 return 0; 2156} 2157 2158/* 2159 * After a successful try_to_unuse, if no swap is now in use, we know 2160 * we can empty the mmlist. swap_lock must be held on entry and exit. 2161 * Note that mmlist_lock nests inside swap_lock, and an mm must be 2162 * added to the mmlist just after page_duplicate - before would be racy. 2163 */ 2164static void drain_mmlist(void) 2165{ 2166 struct list_head *p, *next; 2167 unsigned int type; 2168 2169 for (type = 0; type < nr_swapfiles; type++) 2170 if (swap_info[type]->inuse_pages) 2171 return; 2172 spin_lock(&mmlist_lock); 2173 list_for_each_safe(p, next, &init_mm.mmlist) 2174 list_del_init(p); 2175 spin_unlock(&mmlist_lock); 2176} 2177 2178/* 2179 * Free all of a swapdev's extent information 2180 */ 2181static void destroy_swap_extents(struct swap_info_struct *sis) 2182{ 2183 while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) { 2184 struct rb_node *rb = sis->swap_extent_root.rb_node; 2185 struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node); 2186 2187 rb_erase(rb, &sis->swap_extent_root); 2188 kfree(se); 2189 } 2190 2191 if (sis->flags & SWP_ACTIVATED) { 2192 struct file *swap_file = sis->swap_file; 2193 struct address_space *mapping = swap_file->f_mapping; 2194 2195 sis->flags &= ~SWP_ACTIVATED; 2196 if (mapping->a_ops->swap_deactivate) 2197 mapping->a_ops->swap_deactivate(swap_file); 2198 } 2199} 2200 2201/* 2202 * Add a block range (and the corresponding page range) into this swapdev's 2203 * extent tree. 2204 * 2205 * This function rather assumes that it is called in ascending page order. 2206 */ 2207int 2208add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, 2209 unsigned long nr_pages, sector_t start_block) 2210{ 2211 struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL; 2212 struct swap_extent *se; 2213 struct swap_extent *new_se; 2214 2215 /* 2216 * place the new node at the right most since the 2217 * function is called in ascending page order. 2218 */ 2219 while (*link) { 2220 parent = *link; 2221 link = &parent->rb_right; 2222 } 2223 2224 if (parent) { 2225 se = rb_entry(parent, struct swap_extent, rb_node); 2226 BUG_ON(se->start_page + se->nr_pages != start_page); 2227 if (se->start_block + se->nr_pages == start_block) { 2228 /* Merge it */ 2229 se->nr_pages += nr_pages; 2230 return 0; 2231 } 2232 } 2233 2234 /* No merge, insert a new extent. */ 2235 new_se = kmalloc(sizeof(*se), GFP_KERNEL); 2236 if (new_se == NULL) 2237 return -ENOMEM; 2238 new_se->start_page = start_page; 2239 new_se->nr_pages = nr_pages; 2240 new_se->start_block = start_block; 2241 2242 rb_link_node(&new_se->rb_node, parent, link); 2243 rb_insert_color(&new_se->rb_node, &sis->swap_extent_root); 2244 return 1; 2245} 2246EXPORT_SYMBOL_GPL(add_swap_extent); 2247 2248/* 2249 * A `swap extent' is a simple thing which maps a contiguous range of pages 2250 * onto a contiguous range of disk blocks. A rbtree of swap extents is 2251 * built at swapon time and is then used at swap_writepage/swap_read_folio 2252 * time for locating where on disk a page belongs. 2253 * 2254 * If the swapfile is an S_ISBLK block device, a single extent is installed. 2255 * This is done so that the main operating code can treat S_ISBLK and S_ISREG 2256 * swap files identically. 2257 * 2258 * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap 2259 * extent rbtree operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK 2260 * swapfiles are handled *identically* after swapon time. 2261 * 2262 * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks 2263 * and will parse them into a rbtree, in PAGE_SIZE chunks. If some stray 2264 * blocks are found which do not fall within the PAGE_SIZE alignment 2265 * requirements, they are simply tossed out - we will never use those blocks 2266 * for swapping. 2267 * 2268 * For all swap devices we set S_SWAPFILE across the life of the swapon. This 2269 * prevents users from writing to the swap device, which will corrupt memory. 2270 * 2271 * The amount of disk space which a single swap extent represents varies. 2272 * Typically it is in the 1-4 megabyte range. So we can have hundreds of 2273 * extents in the rbtree. - akpm. 2274 */ 2275static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span) 2276{ 2277 struct file *swap_file = sis->swap_file; 2278 struct address_space *mapping = swap_file->f_mapping; 2279 struct inode *inode = mapping->host; 2280 int ret; 2281 2282 if (S_ISBLK(inode->i_mode)) { 2283 ret = add_swap_extent(sis, 0, sis->max, 0); 2284 *span = sis->pages; 2285 return ret; 2286 } 2287 2288 if (mapping->a_ops->swap_activate) { 2289 ret = mapping->a_ops->swap_activate(sis, swap_file, span); 2290 if (ret < 0) 2291 return ret; 2292 sis->flags |= SWP_ACTIVATED; 2293 if ((sis->flags & SWP_FS_OPS) && 2294 sio_pool_init() != 0) { 2295 destroy_swap_extents(sis); 2296 return -ENOMEM; 2297 } 2298 return ret; 2299 } 2300 2301 return generic_swapfile_activate(sis, swap_file, span); 2302} 2303 2304static int swap_node(struct swap_info_struct *p) 2305{ 2306 struct block_device *bdev; 2307 2308 if (p->bdev) 2309 bdev = p->bdev; 2310 else 2311 bdev = p->swap_file->f_inode->i_sb->s_bdev; 2312 2313 return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE; 2314} 2315 2316static void setup_swap_info(struct swap_info_struct *p, int prio, 2317 unsigned char *swap_map, 2318 struct swap_cluster_info *cluster_info) 2319{ 2320 int i; 2321 2322 if (prio >= 0) 2323 p->prio = prio; 2324 else 2325 p->prio = --least_priority; 2326 /* 2327 * the plist prio is negated because plist ordering is 2328 * low-to-high, while swap ordering is high-to-low 2329 */ 2330 p->list.prio = -p->prio; 2331 for_each_node(i) { 2332 if (p->prio >= 0) 2333 p->avail_lists[i].prio = -p->prio; 2334 else { 2335 if (swap_node(p) == i) 2336 p->avail_lists[i].prio = 1; 2337 else 2338 p->avail_lists[i].prio = -p->prio; 2339 } 2340 } 2341 p->swap_map = swap_map; 2342 p->cluster_info = cluster_info; 2343} 2344 2345static void _enable_swap_info(struct swap_info_struct *p) 2346{ 2347 p->flags |= SWP_WRITEOK; 2348 atomic_long_add(p->pages, &nr_swap_pages); 2349 total_swap_pages += p->pages; 2350 2351 assert_spin_locked(&swap_lock); 2352 /* 2353 * both lists are plists, and thus priority ordered. 2354 * swap_active_head needs to be priority ordered for swapoff(), 2355 * which on removal of any swap_info_struct with an auto-assigned 2356 * (i.e. negative) priority increments the auto-assigned priority 2357 * of any lower-priority swap_info_structs. 2358 * swap_avail_head needs to be priority ordered for folio_alloc_swap(), 2359 * which allocates swap pages from the highest available priority 2360 * swap_info_struct. 2361 */ 2362 plist_add(&p->list, &swap_active_head); 2363 2364 /* add to available list iff swap device is not full */ 2365 if (p->highest_bit) 2366 add_to_avail_list(p); 2367} 2368 2369static void enable_swap_info(struct swap_info_struct *p, int prio, 2370 unsigned char *swap_map, 2371 struct swap_cluster_info *cluster_info) 2372{ 2373 spin_lock(&swap_lock); 2374 spin_lock(&p->lock); 2375 setup_swap_info(p, prio, swap_map, cluster_info); 2376 spin_unlock(&p->lock); 2377 spin_unlock(&swap_lock); 2378 /* 2379 * Finished initializing swap device, now it's safe to reference it. 2380 */ 2381 percpu_ref_resurrect(&p->users); 2382 spin_lock(&swap_lock); 2383 spin_lock(&p->lock); 2384 _enable_swap_info(p); 2385 spin_unlock(&p->lock); 2386 spin_unlock(&swap_lock); 2387} 2388 2389static void reinsert_swap_info(struct swap_info_struct *p) 2390{ 2391 spin_lock(&swap_lock); 2392 spin_lock(&p->lock); 2393 setup_swap_info(p, p->prio, p->swap_map, p->cluster_info); 2394 _enable_swap_info(p); 2395 spin_unlock(&p->lock); 2396 spin_unlock(&swap_lock); 2397} 2398 2399bool has_usable_swap(void) 2400{ 2401 bool ret = true; 2402 2403 spin_lock(&swap_lock); 2404 if (plist_head_empty(&swap_active_head)) 2405 ret = false; 2406 spin_unlock(&swap_lock); 2407 return ret; 2408} 2409 2410SYSCALL_DEFINE1(swapoff, const char __user *, specialfile) 2411{ 2412 struct swap_info_struct *p = NULL; 2413 unsigned char *swap_map; 2414 struct swap_cluster_info *cluster_info; 2415 struct file *swap_file, *victim; 2416 struct address_space *mapping; 2417 struct inode *inode; 2418 struct filename *pathname; 2419 int err, found = 0; 2420 unsigned int old_block_size; 2421 2422 if (!capable(CAP_SYS_ADMIN)) 2423 return -EPERM; 2424 2425 BUG_ON(!current->mm); 2426 2427 pathname = getname(specialfile); 2428 if (IS_ERR(pathname)) 2429 return PTR_ERR(pathname); 2430 2431 victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0); 2432 err = PTR_ERR(victim); 2433 if (IS_ERR(victim)) 2434 goto out; 2435 2436 mapping = victim->f_mapping; 2437 spin_lock(&swap_lock); 2438 plist_for_each_entry(p, &swap_active_head, list) { 2439 if (p->flags & SWP_WRITEOK) { 2440 if (p->swap_file->f_mapping == mapping) { 2441 found = 1; 2442 break; 2443 } 2444 } 2445 } 2446 if (!found) { 2447 err = -EINVAL; 2448 spin_unlock(&swap_lock); 2449 goto out_dput; 2450 } 2451 if (!security_vm_enough_memory_mm(current->mm, p->pages)) 2452 vm_unacct_memory(p->pages); 2453 else { 2454 err = -ENOMEM; 2455 spin_unlock(&swap_lock); 2456 goto out_dput; 2457 } 2458 spin_lock(&p->lock); 2459 del_from_avail_list(p); 2460 if (p->prio < 0) { 2461 struct swap_info_struct *si = p; 2462 int nid; 2463 2464 plist_for_each_entry_continue(si, &swap_active_head, list) { 2465 si->prio++; 2466 si->list.prio--; 2467 for_each_node(nid) { 2468 if (si->avail_lists[nid].prio != 1) 2469 si->avail_lists[nid].prio--; 2470 } 2471 } 2472 least_priority++; 2473 } 2474 plist_del(&p->list, &swap_active_head); 2475 atomic_long_sub(p->pages, &nr_swap_pages); 2476 total_swap_pages -= p->pages; 2477 p->flags &= ~SWP_WRITEOK; 2478 spin_unlock(&p->lock); 2479 spin_unlock(&swap_lock); 2480 2481 disable_swap_slots_cache_lock(); 2482 2483 set_current_oom_origin(); 2484 err = try_to_unuse(p->type); 2485 clear_current_oom_origin(); 2486 2487 if (err) { 2488 /* re-insert swap space back into swap_list */ 2489 reinsert_swap_info(p); 2490 reenable_swap_slots_cache_unlock(); 2491 goto out_dput; 2492 } 2493 2494 reenable_swap_slots_cache_unlock(); 2495 2496 /* 2497 * Wait for swap operations protected by get/put_swap_device() 2498 * to complete. 2499 * 2500 * We need synchronize_rcu() here to protect the accessing to 2501 * the swap cache data structure. 2502 */ 2503 percpu_ref_kill(&p->users); 2504 synchronize_rcu(); 2505 wait_for_completion(&p->comp); 2506 2507 flush_work(&p->discard_work); 2508 2509 destroy_swap_extents(p); 2510 if (p->flags & SWP_CONTINUED) 2511 free_swap_count_continuations(p); 2512 2513 if (!p->bdev || !bdev_nonrot(p->bdev)) 2514 atomic_dec(&nr_rotate_swap); 2515 2516 mutex_lock(&swapon_mutex); 2517 spin_lock(&swap_lock); 2518 spin_lock(&p->lock); 2519 drain_mmlist(); 2520 2521 /* wait for anyone still in scan_swap_map_slots */ 2522 p->highest_bit = 0; /* cuts scans short */ 2523 while (p->flags >= SWP_SCANNING) { 2524 spin_unlock(&p->lock); 2525 spin_unlock(&swap_lock); 2526 schedule_timeout_uninterruptible(1); 2527 spin_lock(&swap_lock); 2528 spin_lock(&p->lock); 2529 } 2530 2531 swap_file = p->swap_file; 2532 old_block_size = p->old_block_size; 2533 p->swap_file = NULL; 2534 p->max = 0; 2535 swap_map = p->swap_map; 2536 p->swap_map = NULL; 2537 cluster_info = p->cluster_info; 2538 p->cluster_info = NULL; 2539 spin_unlock(&p->lock); 2540 spin_unlock(&swap_lock); 2541 arch_swap_invalidate_area(p->type); 2542 zswap_swapoff(p->type); 2543 mutex_unlock(&swapon_mutex); 2544 free_percpu(p->percpu_cluster); 2545 p->percpu_cluster = NULL; 2546 free_percpu(p->cluster_next_cpu); 2547 p->cluster_next_cpu = NULL; 2548 vfree(swap_map); 2549 kvfree(cluster_info); 2550 /* Destroy swap account information */ 2551 swap_cgroup_swapoff(p->type); 2552 exit_swap_address_space(p->type); 2553 2554 inode = mapping->host; 2555 if (p->bdev_file) { 2556 set_blocksize(p->bdev, old_block_size); 2557 fput(p->bdev_file); 2558 p->bdev_file = NULL; 2559 } 2560 2561 inode_lock(inode); 2562 inode->i_flags &= ~S_SWAPFILE; 2563 inode_unlock(inode); 2564 filp_close(swap_file, NULL); 2565 2566 /* 2567 * Clear the SWP_USED flag after all resources are freed so that swapon 2568 * can reuse this swap_info in alloc_swap_info() safely. It is ok to 2569 * not hold p->lock after we cleared its SWP_WRITEOK. 2570 */ 2571 spin_lock(&swap_lock); 2572 p->flags = 0; 2573 spin_unlock(&swap_lock); 2574 2575 err = 0; 2576 atomic_inc(&proc_poll_event); 2577 wake_up_interruptible(&proc_poll_wait); 2578 2579out_dput: 2580 filp_close(victim, NULL); 2581out: 2582 putname(pathname); 2583 return err; 2584} 2585 2586#ifdef CONFIG_PROC_FS 2587static __poll_t swaps_poll(struct file *file, poll_table *wait) 2588{ 2589 struct seq_file *seq = file->private_data; 2590 2591 poll_wait(file, &proc_poll_wait, wait); 2592 2593 if (seq->poll_event != atomic_read(&proc_poll_event)) { 2594 seq->poll_event = atomic_read(&proc_poll_event); 2595 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI; 2596 } 2597 2598 return EPOLLIN | EPOLLRDNORM; 2599} 2600 2601/* iterator */ 2602static void *swap_start(struct seq_file *swap, loff_t *pos) 2603{ 2604 struct swap_info_struct *si; 2605 int type; 2606 loff_t l = *pos; 2607 2608 mutex_lock(&swapon_mutex); 2609 2610 if (!l) 2611 return SEQ_START_TOKEN; 2612 2613 for (type = 0; (si = swap_type_to_swap_info(type)); type++) { 2614 if (!(si->flags & SWP_USED) || !si->swap_map) 2615 continue; 2616 if (!--l) 2617 return si; 2618 } 2619 2620 return NULL; 2621} 2622 2623static void *swap_next(struct seq_file *swap, void *v, loff_t *pos) 2624{ 2625 struct swap_info_struct *si = v; 2626 int type; 2627 2628 if (v == SEQ_START_TOKEN) 2629 type = 0; 2630 else 2631 type = si->type + 1; 2632 2633 ++(*pos); 2634 for (; (si = swap_type_to_swap_info(type)); type++) { 2635 if (!(si->flags & SWP_USED) || !si->swap_map) 2636 continue; 2637 return si; 2638 } 2639 2640 return NULL; 2641} 2642 2643static void swap_stop(struct seq_file *swap, void *v) 2644{ 2645 mutex_unlock(&swapon_mutex); 2646} 2647 2648static int swap_show(struct seq_file *swap, void *v) 2649{ 2650 struct swap_info_struct *si = v; 2651 struct file *file; 2652 int len; 2653 unsigned long bytes, inuse; 2654 2655 if (si == SEQ_START_TOKEN) { 2656 seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"); 2657 return 0; 2658 } 2659 2660 bytes = K(si->pages); 2661 inuse = K(READ_ONCE(si->inuse_pages)); 2662 2663 file = si->swap_file; 2664 len = seq_file_path(swap, file, " \t\n\\"); 2665 seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n", 2666 len < 40 ? 40 - len : 1, " ", 2667 S_ISBLK(file_inode(file)->i_mode) ? 2668 "partition" : "file\t", 2669 bytes, bytes < 10000000 ? "\t" : "", 2670 inuse, inuse < 10000000 ? "\t" : "", 2671 si->prio); 2672 return 0; 2673} 2674 2675static const struct seq_operations swaps_op = { 2676 .start = swap_start, 2677 .next = swap_next, 2678 .stop = swap_stop, 2679 .show = swap_show 2680}; 2681 2682static int swaps_open(struct inode *inode, struct file *file) 2683{ 2684 struct seq_file *seq; 2685 int ret; 2686 2687 ret = seq_open(file, &swaps_op); 2688 if (ret) 2689 return ret; 2690 2691 seq = file->private_data; 2692 seq->poll_event = atomic_read(&proc_poll_event); 2693 return 0; 2694} 2695 2696static const struct proc_ops swaps_proc_ops = { 2697 .proc_flags = PROC_ENTRY_PERMANENT, 2698 .proc_open = swaps_open, 2699 .proc_read = seq_read, 2700 .proc_lseek = seq_lseek, 2701 .proc_release = seq_release, 2702 .proc_poll = swaps_poll, 2703}; 2704 2705static int __init procswaps_init(void) 2706{ 2707 proc_create("swaps", 0, NULL, &swaps_proc_ops); 2708 return 0; 2709} 2710__initcall(procswaps_init); 2711#endif /* CONFIG_PROC_FS */ 2712 2713#ifdef MAX_SWAPFILES_CHECK 2714static int __init max_swapfiles_check(void) 2715{ 2716 MAX_SWAPFILES_CHECK(); 2717 return 0; 2718} 2719late_initcall(max_swapfiles_check); 2720#endif 2721 2722static struct swap_info_struct *alloc_swap_info(void) 2723{ 2724 struct swap_info_struct *p; 2725 struct swap_info_struct *defer = NULL; 2726 unsigned int type; 2727 int i; 2728 2729 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL); 2730 if (!p) 2731 return ERR_PTR(-ENOMEM); 2732 2733 if (percpu_ref_init(&p->users, swap_users_ref_free, 2734 PERCPU_REF_INIT_DEAD, GFP_KERNEL)) { 2735 kvfree(p); 2736 return ERR_PTR(-ENOMEM); 2737 } 2738 2739 spin_lock(&swap_lock); 2740 for (type = 0; type < nr_swapfiles; type++) { 2741 if (!(swap_info[type]->flags & SWP_USED)) 2742 break; 2743 } 2744 if (type >= MAX_SWAPFILES) { 2745 spin_unlock(&swap_lock); 2746 percpu_ref_exit(&p->users); 2747 kvfree(p); 2748 return ERR_PTR(-EPERM); 2749 } 2750 if (type >= nr_swapfiles) { 2751 p->type = type; 2752 /* 2753 * Publish the swap_info_struct after initializing it. 2754 * Note that kvzalloc() above zeroes all its fields. 2755 */ 2756 smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */ 2757 nr_swapfiles++; 2758 } else { 2759 defer = p; 2760 p = swap_info[type]; 2761 /* 2762 * Do not memset this entry: a racing procfs swap_next() 2763 * would be relying on p->type to remain valid. 2764 */ 2765 } 2766 p->swap_extent_root = RB_ROOT; 2767 plist_node_init(&p->list, 0); 2768 for_each_node(i) 2769 plist_node_init(&p->avail_lists[i], 0); 2770 p->flags = SWP_USED; 2771 spin_unlock(&swap_lock); 2772 if (defer) { 2773 percpu_ref_exit(&defer->users); 2774 kvfree(defer); 2775 } 2776 spin_lock_init(&p->lock); 2777 spin_lock_init(&p->cont_lock); 2778 init_completion(&p->comp); 2779 2780 return p; 2781} 2782 2783static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) 2784{ 2785 int error; 2786 2787 if (S_ISBLK(inode->i_mode)) { 2788 p->bdev_file = bdev_file_open_by_dev(inode->i_rdev, 2789 BLK_OPEN_READ | BLK_OPEN_WRITE, p, NULL); 2790 if (IS_ERR(p->bdev_file)) { 2791 error = PTR_ERR(p->bdev_file); 2792 p->bdev_file = NULL; 2793 return error; 2794 } 2795 p->bdev = file_bdev(p->bdev_file); 2796 p->old_block_size = block_size(p->bdev); 2797 error = set_blocksize(p->bdev, PAGE_SIZE); 2798 if (error < 0) 2799 return error; 2800 /* 2801 * Zoned block devices contain zones that have a sequential 2802 * write only restriction. Hence zoned block devices are not 2803 * suitable for swapping. Disallow them here. 2804 */ 2805 if (bdev_is_zoned(p->bdev)) 2806 return -EINVAL; 2807 p->flags |= SWP_BLKDEV; 2808 } else if (S_ISREG(inode->i_mode)) { 2809 p->bdev = inode->i_sb->s_bdev; 2810 } 2811 2812 return 0; 2813} 2814 2815 2816/* 2817 * Find out how many pages are allowed for a single swap device. There 2818 * are two limiting factors: 2819 * 1) the number of bits for the swap offset in the swp_entry_t type, and 2820 * 2) the number of bits in the swap pte, as defined by the different 2821 * architectures. 2822 * 2823 * In order to find the largest possible bit mask, a swap entry with 2824 * swap type 0 and swap offset ~0UL is created, encoded to a swap pte, 2825 * decoded to a swp_entry_t again, and finally the swap offset is 2826 * extracted. 2827 * 2828 * This will mask all the bits from the initial ~0UL mask that can't 2829 * be encoded in either the swp_entry_t or the architecture definition 2830 * of a swap pte. 2831 */ 2832unsigned long generic_max_swapfile_size(void) 2833{ 2834 return swp_offset(pte_to_swp_entry( 2835 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1; 2836} 2837 2838/* Can be overridden by an architecture for additional checks. */ 2839__weak unsigned long arch_max_swapfile_size(void) 2840{ 2841 return generic_max_swapfile_size(); 2842} 2843 2844static unsigned long read_swap_header(struct swap_info_struct *p, 2845 union swap_header *swap_header, 2846 struct inode *inode) 2847{ 2848 int i; 2849 unsigned long maxpages; 2850 unsigned long swapfilepages; 2851 unsigned long last_page; 2852 2853 if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) { 2854 pr_err("Unable to find swap-space signature\n"); 2855 return 0; 2856 } 2857 2858 /* swap partition endianness hack... */ 2859 if (swab32(swap_header->info.version) == 1) { 2860 swab32s(&swap_header->info.version); 2861 swab32s(&swap_header->info.last_page); 2862 swab32s(&swap_header->info.nr_badpages); 2863 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2864 return 0; 2865 for (i = 0; i < swap_header->info.nr_badpages; i++) 2866 swab32s(&swap_header->info.badpages[i]); 2867 } 2868 /* Check the swap header's sub-version */ 2869 if (swap_header->info.version != 1) { 2870 pr_warn("Unable to handle swap header version %d\n", 2871 swap_header->info.version); 2872 return 0; 2873 } 2874 2875 p->lowest_bit = 1; 2876 p->cluster_next = 1; 2877 p->cluster_nr = 0; 2878 2879 maxpages = swapfile_maximum_size; 2880 last_page = swap_header->info.last_page; 2881 if (!last_page) { 2882 pr_warn("Empty swap-file\n"); 2883 return 0; 2884 } 2885 if (last_page > maxpages) { 2886 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n", 2887 K(maxpages), K(last_page)); 2888 } 2889 if (maxpages > last_page) { 2890 maxpages = last_page + 1; 2891 /* p->max is an unsigned int: don't overflow it */ 2892 if ((unsigned int)maxpages == 0) 2893 maxpages = UINT_MAX; 2894 } 2895 p->highest_bit = maxpages - 1; 2896 2897 if (!maxpages) 2898 return 0; 2899 swapfilepages = i_size_read(inode) >> PAGE_SHIFT; 2900 if (swapfilepages && maxpages > swapfilepages) { 2901 pr_warn("Swap area shorter than signature indicates\n"); 2902 return 0; 2903 } 2904 if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode)) 2905 return 0; 2906 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES) 2907 return 0; 2908 2909 return maxpages; 2910} 2911 2912#define SWAP_CLUSTER_INFO_COLS \ 2913 DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info)) 2914#define SWAP_CLUSTER_SPACE_COLS \ 2915 DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER) 2916#define SWAP_CLUSTER_COLS \ 2917 max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS) 2918 2919static int setup_swap_map_and_extents(struct swap_info_struct *p, 2920 union swap_header *swap_header, 2921 unsigned char *swap_map, 2922 struct swap_cluster_info *cluster_info, 2923 unsigned long maxpages, 2924 sector_t *span) 2925{ 2926 unsigned int j, k; 2927 unsigned int nr_good_pages; 2928 int nr_extents; 2929 unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 2930 unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS; 2931 unsigned long i, idx; 2932 2933 nr_good_pages = maxpages - 1; /* omit header page */ 2934 2935 cluster_list_init(&p->free_clusters); 2936 cluster_list_init(&p->discard_clusters); 2937 2938 for (i = 0; i < swap_header->info.nr_badpages; i++) { 2939 unsigned int page_nr = swap_header->info.badpages[i]; 2940 if (page_nr == 0 || page_nr > swap_header->info.last_page) 2941 return -EINVAL; 2942 if (page_nr < maxpages) { 2943 swap_map[page_nr] = SWAP_MAP_BAD; 2944 nr_good_pages--; 2945 /* 2946 * Haven't marked the cluster free yet, no list 2947 * operation involved 2948 */ 2949 inc_cluster_info_page(p, cluster_info, page_nr); 2950 } 2951 } 2952 2953 /* Haven't marked the cluster free yet, no list operation involved */ 2954 for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++) 2955 inc_cluster_info_page(p, cluster_info, i); 2956 2957 if (nr_good_pages) { 2958 swap_map[0] = SWAP_MAP_BAD; 2959 /* 2960 * Not mark the cluster free yet, no list 2961 * operation involved 2962 */ 2963 inc_cluster_info_page(p, cluster_info, 0); 2964 p->max = maxpages; 2965 p->pages = nr_good_pages; 2966 nr_extents = setup_swap_extents(p, span); 2967 if (nr_extents < 0) 2968 return nr_extents; 2969 nr_good_pages = p->pages; 2970 } 2971 if (!nr_good_pages) { 2972 pr_warn("Empty swap-file\n"); 2973 return -EINVAL; 2974 } 2975 2976 if (!cluster_info) 2977 return nr_extents; 2978 2979 2980 /* 2981 * Reduce false cache line sharing between cluster_info and 2982 * sharing same address space. 2983 */ 2984 for (k = 0; k < SWAP_CLUSTER_COLS; k++) { 2985 j = (k + col) % SWAP_CLUSTER_COLS; 2986 for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) { 2987 idx = i * SWAP_CLUSTER_COLS + j; 2988 if (idx >= nr_clusters) 2989 continue; 2990 if (cluster_count(&cluster_info[idx])) 2991 continue; 2992 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE); 2993 cluster_list_add_tail(&p->free_clusters, cluster_info, 2994 idx); 2995 } 2996 } 2997 return nr_extents; 2998} 2999 3000SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) 3001{ 3002 struct swap_info_struct *p; 3003 struct filename *name; 3004 struct file *swap_file = NULL; 3005 struct address_space *mapping; 3006 struct dentry *dentry; 3007 int prio; 3008 int error; 3009 union swap_header *swap_header; 3010 int nr_extents; 3011 sector_t span; 3012 unsigned long maxpages; 3013 unsigned char *swap_map = NULL; 3014 struct swap_cluster_info *cluster_info = NULL; 3015 struct page *page = NULL; 3016 struct inode *inode = NULL; 3017 bool inced_nr_rotate_swap = false; 3018 3019 if (swap_flags & ~SWAP_FLAGS_VALID) 3020 return -EINVAL; 3021 3022 if (!capable(CAP_SYS_ADMIN)) 3023 return -EPERM; 3024 3025 if (!swap_avail_heads) 3026 return -ENOMEM; 3027 3028 p = alloc_swap_info(); 3029 if (IS_ERR(p)) 3030 return PTR_ERR(p); 3031 3032 INIT_WORK(&p->discard_work, swap_discard_work); 3033 3034 name = getname(specialfile); 3035 if (IS_ERR(name)) { 3036 error = PTR_ERR(name); 3037 name = NULL; 3038 goto bad_swap; 3039 } 3040 swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0); 3041 if (IS_ERR(swap_file)) { 3042 error = PTR_ERR(swap_file); 3043 swap_file = NULL; 3044 goto bad_swap; 3045 } 3046 3047 p->swap_file = swap_file; 3048 mapping = swap_file->f_mapping; 3049 dentry = swap_file->f_path.dentry; 3050 inode = mapping->host; 3051 3052 error = claim_swapfile(p, inode); 3053 if (unlikely(error)) 3054 goto bad_swap; 3055 3056 inode_lock(inode); 3057 if (d_unlinked(dentry) || cant_mount(dentry)) { 3058 error = -ENOENT; 3059 goto bad_swap_unlock_inode; 3060 } 3061 if (IS_SWAPFILE(inode)) { 3062 error = -EBUSY; 3063 goto bad_swap_unlock_inode; 3064 } 3065 3066 /* 3067 * Read the swap header. 3068 */ 3069 if (!mapping->a_ops->read_folio) { 3070 error = -EINVAL; 3071 goto bad_swap_unlock_inode; 3072 } 3073 page = read_mapping_page(mapping, 0, swap_file); 3074 if (IS_ERR(page)) { 3075 error = PTR_ERR(page); 3076 goto bad_swap_unlock_inode; 3077 } 3078 swap_header = kmap(page); 3079 3080 maxpages = read_swap_header(p, swap_header, inode); 3081 if (unlikely(!maxpages)) { 3082 error = -EINVAL; 3083 goto bad_swap_unlock_inode; 3084 } 3085 3086 /* OK, set up the swap map and apply the bad block list */ 3087 swap_map = vzalloc(maxpages); 3088 if (!swap_map) { 3089 error = -ENOMEM; 3090 goto bad_swap_unlock_inode; 3091 } 3092 3093 if (p->bdev && bdev_stable_writes(p->bdev)) 3094 p->flags |= SWP_STABLE_WRITES; 3095 3096 if (p->bdev && bdev_synchronous(p->bdev)) 3097 p->flags |= SWP_SYNCHRONOUS_IO; 3098 3099 if (p->bdev && bdev_nonrot(p->bdev)) { 3100 int cpu; 3101 unsigned long ci, nr_cluster; 3102 3103 p->flags |= SWP_SOLIDSTATE; 3104 p->cluster_next_cpu = alloc_percpu(unsigned int); 3105 if (!p->cluster_next_cpu) { 3106 error = -ENOMEM; 3107 goto bad_swap_unlock_inode; 3108 } 3109 /* 3110 * select a random position to start with to help wear leveling 3111 * SSD 3112 */ 3113 for_each_possible_cpu(cpu) { 3114 per_cpu(*p->cluster_next_cpu, cpu) = 3115 get_random_u32_inclusive(1, p->highest_bit); 3116 } 3117 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER); 3118 3119 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info), 3120 GFP_KERNEL); 3121 if (!cluster_info) { 3122 error = -ENOMEM; 3123 goto bad_swap_unlock_inode; 3124 } 3125 3126 for (ci = 0; ci < nr_cluster; ci++) 3127 spin_lock_init(&((cluster_info + ci)->lock)); 3128 3129 p->percpu_cluster = alloc_percpu(struct percpu_cluster); 3130 if (!p->percpu_cluster) { 3131 error = -ENOMEM; 3132 goto bad_swap_unlock_inode; 3133 } 3134 for_each_possible_cpu(cpu) { 3135 struct percpu_cluster *cluster; 3136 cluster = per_cpu_ptr(p->percpu_cluster, cpu); 3137 cluster_set_null(&cluster->index); 3138 } 3139 } else { 3140 atomic_inc(&nr_rotate_swap); 3141 inced_nr_rotate_swap = true; 3142 } 3143 3144 error = swap_cgroup_swapon(p->type, maxpages); 3145 if (error) 3146 goto bad_swap_unlock_inode; 3147 3148 nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, 3149 cluster_info, maxpages, &span); 3150 if (unlikely(nr_extents < 0)) { 3151 error = nr_extents; 3152 goto bad_swap_unlock_inode; 3153 } 3154 3155 if ((swap_flags & SWAP_FLAG_DISCARD) && 3156 p->bdev && bdev_max_discard_sectors(p->bdev)) { 3157 /* 3158 * When discard is enabled for swap with no particular 3159 * policy flagged, we set all swap discard flags here in 3160 * order to sustain backward compatibility with older 3161 * swapon(8) releases. 3162 */ 3163 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD | 3164 SWP_PAGE_DISCARD); 3165 3166 /* 3167 * By flagging sys_swapon, a sysadmin can tell us to 3168 * either do single-time area discards only, or to just 3169 * perform discards for released swap page-clusters. 3170 * Now it's time to adjust the p->flags accordingly. 3171 */ 3172 if (swap_flags & SWAP_FLAG_DISCARD_ONCE) 3173 p->flags &= ~SWP_PAGE_DISCARD; 3174 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES) 3175 p->flags &= ~SWP_AREA_DISCARD; 3176 3177 /* issue a swapon-time discard if it's still required */ 3178 if (p->flags & SWP_AREA_DISCARD) { 3179 int err = discard_swap(p); 3180 if (unlikely(err)) 3181 pr_err("swapon: discard_swap(%p): %d\n", 3182 p, err); 3183 } 3184 } 3185 3186 error = init_swap_address_space(p->type, maxpages); 3187 if (error) 3188 goto bad_swap_unlock_inode; 3189 3190 error = zswap_swapon(p->type, maxpages); 3191 if (error) 3192 goto free_swap_address_space; 3193 3194 /* 3195 * Flush any pending IO and dirty mappings before we start using this 3196 * swap device. 3197 */ 3198 inode->i_flags |= S_SWAPFILE; 3199 error = inode_drain_writes(inode); 3200 if (error) { 3201 inode->i_flags &= ~S_SWAPFILE; 3202 goto free_swap_zswap; 3203 } 3204 3205 mutex_lock(&swapon_mutex); 3206 prio = -1; 3207 if (swap_flags & SWAP_FLAG_PREFER) 3208 prio = 3209 (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT; 3210 enable_swap_info(p, prio, swap_map, cluster_info); 3211 3212 pr_info("Adding %uk swap on %s. Priority:%d extents:%d across:%lluk %s%s%s%s\n", 3213 K(p->pages), name->name, p->prio, nr_extents, 3214 K((unsigned long long)span), 3215 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", 3216 (p->flags & SWP_DISCARDABLE) ? "D" : "", 3217 (p->flags & SWP_AREA_DISCARD) ? "s" : "", 3218 (p->flags & SWP_PAGE_DISCARD) ? "c" : ""); 3219 3220 mutex_unlock(&swapon_mutex); 3221 atomic_inc(&proc_poll_event); 3222 wake_up_interruptible(&proc_poll_wait); 3223 3224 error = 0; 3225 goto out; 3226free_swap_zswap: 3227 zswap_swapoff(p->type); 3228free_swap_address_space: 3229 exit_swap_address_space(p->type); 3230bad_swap_unlock_inode: 3231 inode_unlock(inode); 3232bad_swap: 3233 free_percpu(p->percpu_cluster); 3234 p->percpu_cluster = NULL; 3235 free_percpu(p->cluster_next_cpu); 3236 p->cluster_next_cpu = NULL; 3237 if (p->bdev_file) { 3238 set_blocksize(p->bdev, p->old_block_size); 3239 fput(p->bdev_file); 3240 p->bdev_file = NULL; 3241 } 3242 inode = NULL; 3243 destroy_swap_extents(p); 3244 swap_cgroup_swapoff(p->type); 3245 spin_lock(&swap_lock); 3246 p->swap_file = NULL; 3247 p->flags = 0; 3248 spin_unlock(&swap_lock); 3249 vfree(swap_map); 3250 kvfree(cluster_info); 3251 if (inced_nr_rotate_swap) 3252 atomic_dec(&nr_rotate_swap); 3253 if (swap_file) 3254 filp_close(swap_file, NULL); 3255out: 3256 if (page && !IS_ERR(page)) { 3257 kunmap(page); 3258 put_page(page); 3259 } 3260 if (name) 3261 putname(name); 3262 if (inode) 3263 inode_unlock(inode); 3264 if (!error) 3265 enable_swap_slots_cache(); 3266 return error; 3267} 3268 3269void si_swapinfo(struct sysinfo *val) 3270{ 3271 unsigned int type; 3272 unsigned long nr_to_be_unused = 0; 3273 3274 spin_lock(&swap_lock); 3275 for (type = 0; type < nr_swapfiles; type++) { 3276 struct swap_info_struct *si = swap_info[type]; 3277 3278 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK)) 3279 nr_to_be_unused += READ_ONCE(si->inuse_pages); 3280 } 3281 val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused; 3282 val->totalswap = total_swap_pages + nr_to_be_unused; 3283 spin_unlock(&swap_lock); 3284} 3285 3286/* 3287 * Verify that a swap entry is valid and increment its swap map count. 3288 * 3289 * Returns error code in following case. 3290 * - success -> 0 3291 * - swp_entry is invalid -> EINVAL 3292 * - swp_entry is migration entry -> EINVAL 3293 * - swap-cache reference is requested but there is already one. -> EEXIST 3294 * - swap-cache reference is requested but the entry is not used. -> ENOENT 3295 * - swap-mapped reference requested but needs continued swap count. -> ENOMEM 3296 */ 3297static int __swap_duplicate(swp_entry_t entry, unsigned char usage) 3298{ 3299 struct swap_info_struct *p; 3300 struct swap_cluster_info *ci; 3301 unsigned long offset; 3302 unsigned char count; 3303 unsigned char has_cache; 3304 int err; 3305 3306 p = swp_swap_info(entry); 3307 3308 offset = swp_offset(entry); 3309 ci = lock_cluster_or_swap_info(p, offset); 3310 3311 count = p->swap_map[offset]; 3312 3313 /* 3314 * swapin_readahead() doesn't check if a swap entry is valid, so the 3315 * swap entry could be SWAP_MAP_BAD. Check here with lock held. 3316 */ 3317 if (unlikely(swap_count(count) == SWAP_MAP_BAD)) { 3318 err = -ENOENT; 3319 goto unlock_out; 3320 } 3321 3322 has_cache = count & SWAP_HAS_CACHE; 3323 count &= ~SWAP_HAS_CACHE; 3324 err = 0; 3325 3326 if (usage == SWAP_HAS_CACHE) { 3327 3328 /* set SWAP_HAS_CACHE if there is no cache and entry is used */ 3329 if (!has_cache && count) 3330 has_cache = SWAP_HAS_CACHE; 3331 else if (has_cache) /* someone else added cache */ 3332 err = -EEXIST; 3333 else /* no users remaining */ 3334 err = -ENOENT; 3335 3336 } else if (count || has_cache) { 3337 3338 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX) 3339 count += usage; 3340 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX) 3341 err = -EINVAL; 3342 else if (swap_count_continued(p, offset, count)) 3343 count = COUNT_CONTINUED; 3344 else 3345 err = -ENOMEM; 3346 } else 3347 err = -ENOENT; /* unused swap entry */ 3348 3349 if (!err) 3350 WRITE_ONCE(p->swap_map[offset], count | has_cache); 3351 3352unlock_out: 3353 unlock_cluster_or_swap_info(p, ci); 3354 return err; 3355} 3356 3357/* 3358 * Help swapoff by noting that swap entry belongs to shmem/tmpfs 3359 * (in which case its reference count is never incremented). 3360 */ 3361void swap_shmem_alloc(swp_entry_t entry) 3362{ 3363 __swap_duplicate(entry, SWAP_MAP_SHMEM); 3364} 3365 3366/* 3367 * Increase reference count of swap entry by 1. 3368 * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required 3369 * but could not be atomically allocated. Returns 0, just as if it succeeded, 3370 * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which 3371 * might occur if a page table entry has got corrupted. 3372 */ 3373int swap_duplicate(swp_entry_t entry) 3374{ 3375 int err = 0; 3376 3377 while (!err && __swap_duplicate(entry, 1) == -ENOMEM) 3378 err = add_swap_count_continuation(entry, GFP_ATOMIC); 3379 return err; 3380} 3381 3382/* 3383 * @entry: swap entry for which we allocate swap cache. 3384 * 3385 * Called when allocating swap cache for existing swap entry, 3386 * This can return error codes. Returns 0 at success. 3387 * -EEXIST means there is a swap cache. 3388 * Note: return code is different from swap_duplicate(). 3389 */ 3390int swapcache_prepare(swp_entry_t entry) 3391{ 3392 return __swap_duplicate(entry, SWAP_HAS_CACHE); 3393} 3394 3395void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry) 3396{ 3397 struct swap_cluster_info *ci; 3398 unsigned long offset = swp_offset(entry); 3399 unsigned char usage; 3400 3401 ci = lock_cluster_or_swap_info(si, offset); 3402 usage = __swap_entry_free_locked(si, offset, SWAP_HAS_CACHE); 3403 unlock_cluster_or_swap_info(si, ci); 3404 if (!usage) 3405 free_swap_slot(entry); 3406} 3407 3408struct swap_info_struct *swp_swap_info(swp_entry_t entry) 3409{ 3410 return swap_type_to_swap_info(swp_type(entry)); 3411} 3412 3413/* 3414 * out-of-line methods to avoid include hell. 3415 */ 3416struct address_space *swapcache_mapping(struct folio *folio) 3417{ 3418 return swp_swap_info(folio->swap)->swap_file->f_mapping; 3419} 3420EXPORT_SYMBOL_GPL(swapcache_mapping); 3421 3422pgoff_t __page_file_index(struct page *page) 3423{ 3424 swp_entry_t swap = page_swap_entry(page); 3425 return swp_offset(swap); 3426} 3427EXPORT_SYMBOL_GPL(__page_file_index); 3428 3429/* 3430 * add_swap_count_continuation - called when a swap count is duplicated 3431 * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's 3432 * page of the original vmalloc'ed swap_map, to hold the continuation count 3433 * (for that entry and for its neighbouring PAGE_SIZE swap entries). Called 3434 * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc. 3435 * 3436 * These continuation pages are seldom referenced: the common paths all work 3437 * on the original swap_map, only referring to a continuation page when the 3438 * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX. 3439 * 3440 * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding 3441 * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL) 3442 * can be called after dropping locks. 3443 */ 3444int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask) 3445{ 3446 struct swap_info_struct *si; 3447 struct swap_cluster_info *ci; 3448 struct page *head; 3449 struct page *page; 3450 struct page *list_page; 3451 pgoff_t offset; 3452 unsigned char count; 3453 int ret = 0; 3454 3455 /* 3456 * When debugging, it's easier to use __GFP_ZERO here; but it's better 3457 * for latency not to zero a page while GFP_ATOMIC and holding locks. 3458 */ 3459 page = alloc_page(gfp_mask | __GFP_HIGHMEM); 3460 3461 si = get_swap_device(entry); 3462 if (!si) { 3463 /* 3464 * An acceptable race has occurred since the failing 3465 * __swap_duplicate(): the swap device may be swapoff 3466 */ 3467 goto outer; 3468 } 3469 spin_lock(&si->lock); 3470 3471 offset = swp_offset(entry); 3472 3473 ci = lock_cluster(si, offset); 3474 3475 count = swap_count(si->swap_map[offset]); 3476 3477 if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) { 3478 /* 3479 * The higher the swap count, the more likely it is that tasks 3480 * will race to add swap count continuation: we need to avoid 3481 * over-provisioning. 3482 */ 3483 goto out; 3484 } 3485 3486 if (!page) { 3487 ret = -ENOMEM; 3488 goto out; 3489 } 3490 3491 head = vmalloc_to_page(si->swap_map + offset); 3492 offset &= ~PAGE_MASK; 3493 3494 spin_lock(&si->cont_lock); 3495 /* 3496 * Page allocation does not initialize the page's lru field, 3497 * but it does always reset its private field. 3498 */ 3499 if (!page_private(head)) { 3500 BUG_ON(count & COUNT_CONTINUED); 3501 INIT_LIST_HEAD(&head->lru); 3502 set_page_private(head, SWP_CONTINUED); 3503 si->flags |= SWP_CONTINUED; 3504 } 3505 3506 list_for_each_entry(list_page, &head->lru, lru) { 3507 unsigned char *map; 3508 3509 /* 3510 * If the previous map said no continuation, but we've found 3511 * a continuation page, free our allocation and use this one. 3512 */ 3513 if (!(count & COUNT_CONTINUED)) 3514 goto out_unlock_cont; 3515 3516 map = kmap_local_page(list_page) + offset; 3517 count = *map; 3518 kunmap_local(map); 3519 3520 /* 3521 * If this continuation count now has some space in it, 3522 * free our allocation and use this one. 3523 */ 3524 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX) 3525 goto out_unlock_cont; 3526 } 3527 3528 list_add_tail(&page->lru, &head->lru); 3529 page = NULL; /* now it's attached, don't free it */ 3530out_unlock_cont: 3531 spin_unlock(&si->cont_lock); 3532out: 3533 unlock_cluster(ci); 3534 spin_unlock(&si->lock); 3535 put_swap_device(si); 3536outer: 3537 if (page) 3538 __free_page(page); 3539 return ret; 3540} 3541 3542/* 3543 * swap_count_continued - when the original swap_map count is incremented 3544 * from SWAP_MAP_MAX, check if there is already a continuation page to carry 3545 * into, carry if so, or else fail until a new continuation page is allocated; 3546 * when the original swap_map count is decremented from 0 with continuation, 3547 * borrow from the continuation and report whether it still holds more. 3548 * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster 3549 * lock. 3550 */ 3551static bool swap_count_continued(struct swap_info_struct *si, 3552 pgoff_t offset, unsigned char count) 3553{ 3554 struct page *head; 3555 struct page *page; 3556 unsigned char *map; 3557 bool ret; 3558 3559 head = vmalloc_to_page(si->swap_map + offset); 3560 if (page_private(head) != SWP_CONTINUED) { 3561 BUG_ON(count & COUNT_CONTINUED); 3562 return false; /* need to add count continuation */ 3563 } 3564 3565 spin_lock(&si->cont_lock); 3566 offset &= ~PAGE_MASK; 3567 page = list_next_entry(head, lru); 3568 map = kmap_local_page(page) + offset; 3569 3570 if (count == SWAP_MAP_MAX) /* initial increment from swap_map */ 3571 goto init_map; /* jump over SWAP_CONT_MAX checks */ 3572 3573 if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */ 3574 /* 3575 * Think of how you add 1 to 999 3576 */ 3577 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) { 3578 kunmap_local(map); 3579 page = list_next_entry(page, lru); 3580 BUG_ON(page == head); 3581 map = kmap_local_page(page) + offset; 3582 } 3583 if (*map == SWAP_CONT_MAX) { 3584 kunmap_local(map); 3585 page = list_next_entry(page, lru); 3586 if (page == head) { 3587 ret = false; /* add count continuation */ 3588 goto out; 3589 } 3590 map = kmap_local_page(page) + offset; 3591init_map: *map = 0; /* we didn't zero the page */ 3592 } 3593 *map += 1; 3594 kunmap_local(map); 3595 while ((page = list_prev_entry(page, lru)) != head) { 3596 map = kmap_local_page(page) + offset; 3597 *map = COUNT_CONTINUED; 3598 kunmap_local(map); 3599 } 3600 ret = true; /* incremented */ 3601 3602 } else { /* decrementing */ 3603 /* 3604 * Think of how you subtract 1 from 1000 3605 */ 3606 BUG_ON(count != COUNT_CONTINUED); 3607 while (*map == COUNT_CONTINUED) { 3608 kunmap_local(map); 3609 page = list_next_entry(page, lru); 3610 BUG_ON(page == head); 3611 map = kmap_local_page(page) + offset; 3612 } 3613 BUG_ON(*map == 0); 3614 *map -= 1; 3615 if (*map == 0) 3616 count = 0; 3617 kunmap_local(map); 3618 while ((page = list_prev_entry(page, lru)) != head) { 3619 map = kmap_local_page(page) + offset; 3620 *map = SWAP_CONT_MAX | count; 3621 count = COUNT_CONTINUED; 3622 kunmap_local(map); 3623 } 3624 ret = count == COUNT_CONTINUED; 3625 } 3626out: 3627 spin_unlock(&si->cont_lock); 3628 return ret; 3629} 3630 3631/* 3632 * free_swap_count_continuations - swapoff free all the continuation pages 3633 * appended to the swap_map, after swap_map is quiesced, before vfree'ing it. 3634 */ 3635static void free_swap_count_continuations(struct swap_info_struct *si) 3636{ 3637 pgoff_t offset; 3638 3639 for (offset = 0; offset < si->max; offset += PAGE_SIZE) { 3640 struct page *head; 3641 head = vmalloc_to_page(si->swap_map + offset); 3642 if (page_private(head)) { 3643 struct page *page, *next; 3644 3645 list_for_each_entry_safe(page, next, &head->lru, lru) { 3646 list_del(&page->lru); 3647 __free_page(page); 3648 } 3649 } 3650 } 3651} 3652 3653#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) 3654void __folio_throttle_swaprate(struct folio *folio, gfp_t gfp) 3655{ 3656 struct swap_info_struct *si, *next; 3657 int nid = folio_nid(folio); 3658 3659 if (!(gfp & __GFP_IO)) 3660 return; 3661 3662 if (!blk_cgroup_congested()) 3663 return; 3664 3665 /* 3666 * We've already scheduled a throttle, avoid taking the global swap 3667 * lock. 3668 */ 3669 if (current->throttle_disk) 3670 return; 3671 3672 spin_lock(&swap_avail_lock); 3673 plist_for_each_entry_safe(si, next, &swap_avail_heads[nid], 3674 avail_lists[nid]) { 3675 if (si->bdev) { 3676 blkcg_schedule_throttle(si->bdev->bd_disk, true); 3677 break; 3678 } 3679 } 3680 spin_unlock(&swap_avail_lock); 3681} 3682#endif 3683 3684static int __init swapfile_init(void) 3685{ 3686 int nid; 3687 3688 swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head), 3689 GFP_KERNEL); 3690 if (!swap_avail_heads) { 3691 pr_emerg("Not enough memory for swap heads, swap is disabled\n"); 3692 return -ENOMEM; 3693 } 3694 3695 for_each_node(nid) 3696 plist_head_init(&swap_avail_heads[nid]); 3697 3698 swapfile_maximum_size = arch_max_swapfile_size(); 3699 3700#ifdef CONFIG_MIGRATION 3701 if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS)) 3702 swap_migration_ad_supported = true; 3703#endif /* CONFIG_MIGRATION */ 3704 3705 return 0; 3706} 3707subsys_initcall(swapfile_init); 3708