1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _LINUX_SWAPOPS_H 3#define _LINUX_SWAPOPS_H 4 5#include <linux/radix-tree.h> 6#include <linux/bug.h> 7#include <linux/mm_types.h> 8 9#ifdef CONFIG_MMU 10 11#ifdef CONFIG_SWAP 12#include <linux/swapfile.h> 13#endif /* CONFIG_SWAP */ 14 15/* 16 * swapcache pages are stored in the swapper_space radix tree. We want to 17 * get good packing density in that tree, so the index should be dense in 18 * the low-order bits. 19 * 20 * We arrange the `type' and `offset' fields so that `type' is at the six 21 * high-order bits of the swp_entry_t and `offset' is right-aligned in the 22 * remaining bits. Although `type' itself needs only five bits, we allow for 23 * shmem/tmpfs to shift it all up a further one bit: see swp_to_radix_entry(). 24 * 25 * swp_entry_t's are *never* stored anywhere in their arch-dependent format. 26 */ 27#define SWP_TYPE_SHIFT (BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT) 28#define SWP_OFFSET_MASK ((1UL << SWP_TYPE_SHIFT) - 1) 29 30/* 31 * Definitions only for PFN swap entries (see is_pfn_swap_entry()). To 32 * store PFN, we only need SWP_PFN_BITS bits. Each of the pfn swap entries 33 * can use the extra bits to store other information besides PFN. 34 */ 35#ifdef MAX_PHYSMEM_BITS 36#define SWP_PFN_BITS (MAX_PHYSMEM_BITS - PAGE_SHIFT) 37#else /* MAX_PHYSMEM_BITS */ 38#define SWP_PFN_BITS min_t(int, \ 39 sizeof(phys_addr_t) * 8 - PAGE_SHIFT, \ 40 SWP_TYPE_SHIFT) 41#endif /* MAX_PHYSMEM_BITS */ 42#define SWP_PFN_MASK (BIT(SWP_PFN_BITS) - 1) 43 44/** 45 * Migration swap entry specific bitfield definitions. Layout: 46 * 47 * |----------+--------------------| 48 * | swp_type | swp_offset | 49 * |----------+--------+-+-+-------| 50 * | | resv |D|A| PFN | 51 * |----------+--------+-+-+-------| 52 * 53 * @SWP_MIG_YOUNG_BIT: Whether the page used to have young bit set (bit A) 54 * @SWP_MIG_DIRTY_BIT: Whether the page used to have dirty bit set (bit D) 55 * 56 * Note: A/D bits will be stored in migration entries iff there're enough 57 * free bits in arch specific swp offset. By default we'll ignore A/D bits 58 * when migrating a page. Please refer to migration_entry_supports_ad() 59 * for more information. If there're more bits besides PFN and A/D bits, 60 * they should be reserved and always be zeros. 61 */ 62#define SWP_MIG_YOUNG_BIT (SWP_PFN_BITS) 63#define SWP_MIG_DIRTY_BIT (SWP_PFN_BITS + 1) 64#define SWP_MIG_TOTAL_BITS (SWP_PFN_BITS + 2) 65 66#define SWP_MIG_YOUNG BIT(SWP_MIG_YOUNG_BIT) 67#define SWP_MIG_DIRTY BIT(SWP_MIG_DIRTY_BIT) 68 69static inline bool is_pfn_swap_entry(swp_entry_t entry); 70 71/* Clear all flags but only keep swp_entry_t related information */ 72static inline pte_t pte_swp_clear_flags(pte_t pte) 73{ 74 if (pte_swp_exclusive(pte)) 75 pte = pte_swp_clear_exclusive(pte); 76 if (pte_swp_soft_dirty(pte)) 77 pte = pte_swp_clear_soft_dirty(pte); 78 if (pte_swp_uffd_wp(pte)) 79 pte = pte_swp_clear_uffd_wp(pte); 80 return pte; 81} 82 83/* 84 * Store a type+offset into a swp_entry_t in an arch-independent format 85 */ 86static inline swp_entry_t swp_entry(unsigned long type, pgoff_t offset) 87{ 88 swp_entry_t ret; 89 90 ret.val = (type << SWP_TYPE_SHIFT) | (offset & SWP_OFFSET_MASK); 91 return ret; 92} 93 94/* 95 * Extract the `type' field from a swp_entry_t. The swp_entry_t is in 96 * arch-independent format 97 */ 98static inline unsigned swp_type(swp_entry_t entry) 99{ 100 return (entry.val >> SWP_TYPE_SHIFT); 101} 102 103/* 104 * Extract the `offset' field from a swp_entry_t. The swp_entry_t is in 105 * arch-independent format 106 */ 107static inline pgoff_t swp_offset(swp_entry_t entry) 108{ 109 return entry.val & SWP_OFFSET_MASK; 110} 111 112/* 113 * This should only be called upon a pfn swap entry to get the PFN stored 114 * in the swap entry. Please refers to is_pfn_swap_entry() for definition 115 * of pfn swap entry. 116 */ 117static inline unsigned long swp_offset_pfn(swp_entry_t entry) 118{ 119 VM_BUG_ON(!is_pfn_swap_entry(entry)); 120 return swp_offset(entry) & SWP_PFN_MASK; 121} 122 123/* check whether a pte points to a swap entry */ 124static inline int is_swap_pte(pte_t pte) 125{ 126 return !pte_none(pte) && !pte_present(pte); 127} 128 129/* 130 * Convert the arch-dependent pte representation of a swp_entry_t into an 131 * arch-independent swp_entry_t. 132 */ 133static inline swp_entry_t pte_to_swp_entry(pte_t pte) 134{ 135 swp_entry_t arch_entry; 136 137 pte = pte_swp_clear_flags(pte); 138 arch_entry = __pte_to_swp_entry(pte); 139 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 140} 141 142/* 143 * Convert the arch-independent representation of a swp_entry_t into the 144 * arch-dependent pte representation. 145 */ 146static inline pte_t swp_entry_to_pte(swp_entry_t entry) 147{ 148 swp_entry_t arch_entry; 149 150 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 151 return __swp_entry_to_pte(arch_entry); 152} 153 154static inline swp_entry_t radix_to_swp_entry(void *arg) 155{ 156 swp_entry_t entry; 157 158 entry.val = xa_to_value(arg); 159 return entry; 160} 161 162static inline void *swp_to_radix_entry(swp_entry_t entry) 163{ 164 return xa_mk_value(entry.val); 165} 166 167#if IS_ENABLED(CONFIG_DEVICE_PRIVATE) 168static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 169{ 170 return swp_entry(SWP_DEVICE_READ, offset); 171} 172 173static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 174{ 175 return swp_entry(SWP_DEVICE_WRITE, offset); 176} 177 178static inline bool is_device_private_entry(swp_entry_t entry) 179{ 180 int type = swp_type(entry); 181 return type == SWP_DEVICE_READ || type == SWP_DEVICE_WRITE; 182} 183 184static inline bool is_writable_device_private_entry(swp_entry_t entry) 185{ 186 return unlikely(swp_type(entry) == SWP_DEVICE_WRITE); 187} 188 189static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 190{ 191 return swp_entry(SWP_DEVICE_EXCLUSIVE_READ, offset); 192} 193 194static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 195{ 196 return swp_entry(SWP_DEVICE_EXCLUSIVE_WRITE, offset); 197} 198 199static inline bool is_device_exclusive_entry(swp_entry_t entry) 200{ 201 return swp_type(entry) == SWP_DEVICE_EXCLUSIVE_READ || 202 swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE; 203} 204 205static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 206{ 207 return unlikely(swp_type(entry) == SWP_DEVICE_EXCLUSIVE_WRITE); 208} 209#else /* CONFIG_DEVICE_PRIVATE */ 210static inline swp_entry_t make_readable_device_private_entry(pgoff_t offset) 211{ 212 return swp_entry(0, 0); 213} 214 215static inline swp_entry_t make_writable_device_private_entry(pgoff_t offset) 216{ 217 return swp_entry(0, 0); 218} 219 220static inline bool is_device_private_entry(swp_entry_t entry) 221{ 222 return false; 223} 224 225static inline bool is_writable_device_private_entry(swp_entry_t entry) 226{ 227 return false; 228} 229 230static inline swp_entry_t make_readable_device_exclusive_entry(pgoff_t offset) 231{ 232 return swp_entry(0, 0); 233} 234 235static inline swp_entry_t make_writable_device_exclusive_entry(pgoff_t offset) 236{ 237 return swp_entry(0, 0); 238} 239 240static inline bool is_device_exclusive_entry(swp_entry_t entry) 241{ 242 return false; 243} 244 245static inline bool is_writable_device_exclusive_entry(swp_entry_t entry) 246{ 247 return false; 248} 249#endif /* CONFIG_DEVICE_PRIVATE */ 250 251#ifdef CONFIG_MIGRATION 252static inline int is_migration_entry(swp_entry_t entry) 253{ 254 return unlikely(swp_type(entry) == SWP_MIGRATION_READ || 255 swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE || 256 swp_type(entry) == SWP_MIGRATION_WRITE); 257} 258 259static inline int is_writable_migration_entry(swp_entry_t entry) 260{ 261 return unlikely(swp_type(entry) == SWP_MIGRATION_WRITE); 262} 263 264static inline int is_readable_migration_entry(swp_entry_t entry) 265{ 266 return unlikely(swp_type(entry) == SWP_MIGRATION_READ); 267} 268 269static inline int is_readable_exclusive_migration_entry(swp_entry_t entry) 270{ 271 return unlikely(swp_type(entry) == SWP_MIGRATION_READ_EXCLUSIVE); 272} 273 274static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 275{ 276 return swp_entry(SWP_MIGRATION_READ, offset); 277} 278 279static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 280{ 281 return swp_entry(SWP_MIGRATION_READ_EXCLUSIVE, offset); 282} 283 284static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 285{ 286 return swp_entry(SWP_MIGRATION_WRITE, offset); 287} 288 289/* 290 * Returns whether the host has large enough swap offset field to support 291 * carrying over pgtable A/D bits for page migrations. The result is 292 * pretty much arch specific. 293 */ 294static inline bool migration_entry_supports_ad(void) 295{ 296#ifdef CONFIG_SWAP 297 return swap_migration_ad_supported; 298#else /* CONFIG_SWAP */ 299 return false; 300#endif /* CONFIG_SWAP */ 301} 302 303static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 304{ 305 if (migration_entry_supports_ad()) 306 return swp_entry(swp_type(entry), 307 swp_offset(entry) | SWP_MIG_YOUNG); 308 return entry; 309} 310 311static inline bool is_migration_entry_young(swp_entry_t entry) 312{ 313 if (migration_entry_supports_ad()) 314 return swp_offset(entry) & SWP_MIG_YOUNG; 315 /* Keep the old behavior of aging page after migration */ 316 return false; 317} 318 319static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 320{ 321 if (migration_entry_supports_ad()) 322 return swp_entry(swp_type(entry), 323 swp_offset(entry) | SWP_MIG_DIRTY); 324 return entry; 325} 326 327static inline bool is_migration_entry_dirty(swp_entry_t entry) 328{ 329 if (migration_entry_supports_ad()) 330 return swp_offset(entry) & SWP_MIG_DIRTY; 331 /* Keep the old behavior of clean page after migration */ 332 return false; 333} 334 335extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 336 unsigned long address); 337extern void migration_entry_wait_huge(struct vm_area_struct *vma, pte_t *pte); 338#else /* CONFIG_MIGRATION */ 339static inline swp_entry_t make_readable_migration_entry(pgoff_t offset) 340{ 341 return swp_entry(0, 0); 342} 343 344static inline swp_entry_t make_readable_exclusive_migration_entry(pgoff_t offset) 345{ 346 return swp_entry(0, 0); 347} 348 349static inline swp_entry_t make_writable_migration_entry(pgoff_t offset) 350{ 351 return swp_entry(0, 0); 352} 353 354static inline int is_migration_entry(swp_entry_t swp) 355{ 356 return 0; 357} 358 359static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 360 unsigned long address) { } 361static inline void migration_entry_wait_huge(struct vm_area_struct *vma, 362 pte_t *pte) { } 363static inline int is_writable_migration_entry(swp_entry_t entry) 364{ 365 return 0; 366} 367static inline int is_readable_migration_entry(swp_entry_t entry) 368{ 369 return 0; 370} 371 372static inline swp_entry_t make_migration_entry_young(swp_entry_t entry) 373{ 374 return entry; 375} 376 377static inline bool is_migration_entry_young(swp_entry_t entry) 378{ 379 return false; 380} 381 382static inline swp_entry_t make_migration_entry_dirty(swp_entry_t entry) 383{ 384 return entry; 385} 386 387static inline bool is_migration_entry_dirty(swp_entry_t entry) 388{ 389 return false; 390} 391#endif /* CONFIG_MIGRATION */ 392 393typedef unsigned long pte_marker; 394 395#define PTE_MARKER_UFFD_WP BIT(0) 396/* 397 * "Poisoned" here is meant in the very general sense of "future accesses are 398 * invalid", instead of referring very specifically to hardware memory errors. 399 * This marker is meant to represent any of various different causes of this. 400 */ 401#define PTE_MARKER_POISONED BIT(1) 402#define PTE_MARKER_MASK (BIT(2) - 1) 403 404static inline swp_entry_t make_pte_marker_entry(pte_marker marker) 405{ 406 return swp_entry(SWP_PTE_MARKER, marker); 407} 408 409static inline bool is_pte_marker_entry(swp_entry_t entry) 410{ 411 return swp_type(entry) == SWP_PTE_MARKER; 412} 413 414static inline pte_marker pte_marker_get(swp_entry_t entry) 415{ 416 return swp_offset(entry) & PTE_MARKER_MASK; 417} 418 419static inline bool is_pte_marker(pte_t pte) 420{ 421 return is_swap_pte(pte) && is_pte_marker_entry(pte_to_swp_entry(pte)); 422} 423 424static inline pte_t make_pte_marker(pte_marker marker) 425{ 426 return swp_entry_to_pte(make_pte_marker_entry(marker)); 427} 428 429static inline swp_entry_t make_poisoned_swp_entry(void) 430{ 431 return make_pte_marker_entry(PTE_MARKER_POISONED); 432} 433 434static inline int is_poisoned_swp_entry(swp_entry_t entry) 435{ 436 return is_pte_marker_entry(entry) && 437 (pte_marker_get(entry) & PTE_MARKER_POISONED); 438} 439 440/* 441 * This is a special version to check pte_none() just to cover the case when 442 * the pte is a pte marker. It existed because in many cases the pte marker 443 * should be seen as a none pte; it's just that we have stored some information 444 * onto the none pte so it becomes not-none any more. 445 * 446 * It should be used when the pte is file-backed, ram-based and backing 447 * userspace pages, like shmem. It is not needed upon pgtables that do not 448 * support pte markers at all. For example, it's not needed on anonymous 449 * memory, kernel-only memory (including when the system is during-boot), 450 * non-ram based generic file-system. It's fine to be used even there, but the 451 * extra pte marker check will be pure overhead. 452 */ 453static inline int pte_none_mostly(pte_t pte) 454{ 455 return pte_none(pte) || is_pte_marker(pte); 456} 457 458static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) 459{ 460 struct page *p = pfn_to_page(swp_offset_pfn(entry)); 461 462 /* 463 * Any use of migration entries may only occur while the 464 * corresponding page is locked 465 */ 466 BUG_ON(is_migration_entry(entry) && !PageLocked(p)); 467 468 return p; 469} 470 471static inline struct folio *pfn_swap_entry_folio(swp_entry_t entry) 472{ 473 struct folio *folio = pfn_folio(swp_offset_pfn(entry)); 474 475 /* 476 * Any use of migration entries may only occur while the 477 * corresponding folio is locked 478 */ 479 BUG_ON(is_migration_entry(entry) && !folio_test_locked(folio)); 480 481 return folio; 482} 483 484/* 485 * A pfn swap entry is a special type of swap entry that always has a pfn stored 486 * in the swap offset. They are used to represent unaddressable device memory 487 * and to restrict access to a page undergoing migration. 488 */ 489static inline bool is_pfn_swap_entry(swp_entry_t entry) 490{ 491 /* Make sure the swp offset can always store the needed fields */ 492 BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS); 493 494 return is_migration_entry(entry) || is_device_private_entry(entry) || 495 is_device_exclusive_entry(entry); 496} 497 498struct page_vma_mapped_walk; 499 500#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION 501extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 502 struct page *page); 503 504extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 505 struct page *new); 506 507extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); 508 509static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 510{ 511 swp_entry_t arch_entry; 512 513 if (pmd_swp_soft_dirty(pmd)) 514 pmd = pmd_swp_clear_soft_dirty(pmd); 515 if (pmd_swp_uffd_wp(pmd)) 516 pmd = pmd_swp_clear_uffd_wp(pmd); 517 arch_entry = __pmd_to_swp_entry(pmd); 518 return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); 519} 520 521static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 522{ 523 swp_entry_t arch_entry; 524 525 arch_entry = __swp_entry(swp_type(entry), swp_offset(entry)); 526 return __swp_entry_to_pmd(arch_entry); 527} 528 529static inline int is_pmd_migration_entry(pmd_t pmd) 530{ 531 return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); 532} 533#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 534static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, 535 struct page *page) 536{ 537 BUILD_BUG(); 538} 539 540static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, 541 struct page *new) 542{ 543 BUILD_BUG(); 544} 545 546static inline void pmd_migration_entry_wait(struct mm_struct *m, pmd_t *p) { } 547 548static inline swp_entry_t pmd_to_swp_entry(pmd_t pmd) 549{ 550 return swp_entry(0, 0); 551} 552 553static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) 554{ 555 return __pmd(0); 556} 557 558static inline int is_pmd_migration_entry(pmd_t pmd) 559{ 560 return 0; 561} 562#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ 563 564#ifdef CONFIG_MEMORY_FAILURE 565 566/* 567 * Support for hardware poisoned pages 568 */ 569static inline swp_entry_t make_hwpoison_entry(struct page *page) 570{ 571 BUG_ON(!PageLocked(page)); 572 return swp_entry(SWP_HWPOISON, page_to_pfn(page)); 573} 574 575static inline int is_hwpoison_entry(swp_entry_t entry) 576{ 577 return swp_type(entry) == SWP_HWPOISON; 578} 579 580#else 581 582static inline swp_entry_t make_hwpoison_entry(struct page *page) 583{ 584 return swp_entry(0, 0); 585} 586 587static inline int is_hwpoison_entry(swp_entry_t swp) 588{ 589 return 0; 590} 591#endif 592 593static inline int non_swap_entry(swp_entry_t entry) 594{ 595 return swp_type(entry) >= MAX_SWAPFILES; 596} 597 598#endif /* CONFIG_MMU */ 599#endif /* _LINUX_SWAPOPS_H */ 600