1/* 2 * Memory Migration functionality - linux/mm/migration.c 3 * 4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter 5 * 6 * Page migration was first developed in the context of the memory hotplug 7 * project. The main authors of the migration code are: 8 * 9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp> 10 * Hirokazu Takahashi <taka@valinux.co.jp> 11 * Dave Hansen <haveblue@us.ibm.com> 12 * Christoph Lameter 13 */ 14 15#include <linux/migrate.h> 16#include <linux/module.h> 17#include <linux/swap.h> 18#include <linux/swapops.h> 19#include <linux/pagemap.h> 20#include <linux/buffer_head.h> 21#include <linux/mm_inline.h> 22#include <linux/nsproxy.h> 23#include <linux/pagevec.h> 24#include <linux/ksm.h> 25#include <linux/rmap.h> 26#include <linux/topology.h> 27#include <linux/cpu.h> 28#include <linux/cpuset.h> 29#include <linux/writeback.h> 30#include <linux/mempolicy.h> 31#include <linux/vmalloc.h> 32#include <linux/security.h> 33#include <linux/memcontrol.h> 34#include <linux/syscalls.h> 35#include <linux/gfp.h> 36 37#include "internal.h" 38 39#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) 40 41/* 42 * migrate_prep() needs to be called before we start compiling a list of pages 43 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is 44 * undesirable, use migrate_prep_local() 45 */ 46int migrate_prep(void) 47{ 48 /* 49 * Clear the LRU lists so pages can be isolated. 50 * Note that pages may be moved off the LRU after we have 51 * drained them. Those pages will fail to migrate like other 52 * pages that may be busy. 53 */ 54 lru_add_drain_all(); 55 56 return 0; 57} 58 59/* Do the necessary work of migrate_prep but not if it involves other CPUs */ 60int migrate_prep_local(void) 61{ 62 lru_add_drain(); 63 64 return 0; 65} 66 67/* 68 * Add isolated pages on the list back to the LRU under page lock 69 * to avoid leaking evictable pages back onto unevictable list. 70 */ 71void putback_lru_pages(struct list_head *l) 72{ 73 struct page *page; 74 struct page *page2; 75 76 list_for_each_entry_safe(page, page2, l, lru) { 77 list_del(&page->lru); 78 dec_zone_page_state(page, NR_ISOLATED_ANON + 79 page_is_file_cache(page)); 80 putback_lru_page(page); 81 } 82} 83 84/* 85 * Restore a potential migration pte to a working pte entry 86 */ 87static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, 88 unsigned long addr, void *old) 89{ 90 struct mm_struct *mm = vma->vm_mm; 91 swp_entry_t entry; 92 pgd_t *pgd; 93 pud_t *pud; 94 pmd_t *pmd; 95 pte_t *ptep, pte; 96 spinlock_t *ptl; 97 98 pgd = pgd_offset(mm, addr); 99 if (!pgd_present(*pgd)) 100 goto out; 101 102 pud = pud_offset(pgd, addr); 103 if (!pud_present(*pud)) 104 goto out; 105 106 pmd = pmd_offset(pud, addr); 107 if (!pmd_present(*pmd)) 108 goto out; 109 110 ptep = pte_offset_map(pmd, addr); 111 112 if (!is_swap_pte(*ptep)) { 113 pte_unmap(ptep); 114 goto out; 115 } 116 117 ptl = pte_lockptr(mm, pmd); 118 spin_lock(ptl); 119 pte = *ptep; 120 if (!is_swap_pte(pte)) 121 goto unlock; 122 123 entry = pte_to_swp_entry(pte); 124 125 if (!is_migration_entry(entry) || 126 migration_entry_to_page(entry) != old) 127 goto unlock; 128 129 get_page(new); 130 pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); 131 if (is_write_migration_entry(entry)) 132 pte = pte_mkwrite(pte); 133 flush_cache_page(vma, addr, pte_pfn(pte)); 134 set_pte_at(mm, addr, ptep, pte); 135 136 if (PageAnon(new)) 137 page_add_anon_rmap(new, vma, addr); 138 else 139 page_add_file_rmap(new); 140 141 /* No need to invalidate - it was non-present before */ 142 update_mmu_cache(vma, addr, ptep); 143unlock: 144 pte_unmap_unlock(ptep, ptl); 145out: 146 return SWAP_AGAIN; 147} 148 149/* 150 * Get rid of all migration entries and replace them by 151 * references to the indicated page. 152 */ 153static void remove_migration_ptes(struct page *old, struct page *new) 154{ 155 rmap_walk(new, remove_migration_pte, old); 156} 157 158/* 159 * Something used the pte of a page under migration. We need to 160 * get to the page and wait until migration is finished. 161 * When we return from this function the fault will be retried. 162 * 163 * This function is called from do_swap_page(). 164 */ 165void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 166 unsigned long address) 167{ 168 pte_t *ptep, pte; 169 spinlock_t *ptl; 170 swp_entry_t entry; 171 struct page *page; 172 173 ptep = pte_offset_map_lock(mm, pmd, address, &ptl); 174 pte = *ptep; 175 if (!is_swap_pte(pte)) 176 goto out; 177 178 entry = pte_to_swp_entry(pte); 179 if (!is_migration_entry(entry)) 180 goto out; 181 182 page = migration_entry_to_page(entry); 183 184 /* 185 * Once radix-tree replacement of page migration started, page_count 186 * *must* be zero. And, we don't want to call wait_on_page_locked() 187 * against a page without get_page(). 188 * So, we use get_page_unless_zero(), here. Even failed, page fault 189 * will occur again. 190 */ 191 if (!get_page_unless_zero(page)) 192 goto out; 193 pte_unmap_unlock(ptep, ptl); 194 wait_on_page_locked(page); 195 put_page(page); 196 return; 197out: 198 pte_unmap_unlock(ptep, ptl); 199} 200 201/* 202 * Replace the page in the mapping. 203 * 204 * The number of remaining references must be: 205 * 1 for anonymous pages without a mapping 206 * 2 for pages with a mapping 207 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. 208 */ 209static int migrate_page_move_mapping(struct address_space *mapping, 210 struct page *newpage, struct page *page) 211{ 212 int expected_count; 213 void **pslot; 214 215 if (!mapping) { 216 /* Anonymous page without mapping */ 217 if (page_count(page) != 1) 218 return -EAGAIN; 219 return 0; 220 } 221 222 spin_lock_irq(&mapping->tree_lock); 223 224 pslot = radix_tree_lookup_slot(&mapping->page_tree, 225 page_index(page)); 226 227 expected_count = 2 + page_has_private(page); 228 if (page_count(page) != expected_count || 229 (struct page *)radix_tree_deref_slot(pslot) != page) { 230 spin_unlock_irq(&mapping->tree_lock); 231 return -EAGAIN; 232 } 233 234 if (!page_freeze_refs(page, expected_count)) { 235 spin_unlock_irq(&mapping->tree_lock); 236 return -EAGAIN; 237 } 238 239 /* 240 * Now we know that no one else is looking at the page. 241 */ 242 get_page(newpage); /* add cache reference */ 243 if (PageSwapCache(page)) { 244 SetPageSwapCache(newpage); 245 set_page_private(newpage, page_private(page)); 246 } 247 248 radix_tree_replace_slot(pslot, newpage); 249 250 page_unfreeze_refs(page, expected_count); 251 /* 252 * Drop cache reference from old page. 253 * We know this isn't the last reference. 254 */ 255 __put_page(page); 256 257 /* 258 * If moved to a different zone then also account 259 * the page for that zone. Other VM counters will be 260 * taken care of when we establish references to the 261 * new page and drop references to the old page. 262 * 263 * Note that anonymous pages are accounted for 264 * via NR_FILE_PAGES and NR_ANON_PAGES if they 265 * are mapped to swap space. 266 */ 267 __dec_zone_page_state(page, NR_FILE_PAGES); 268 __inc_zone_page_state(newpage, NR_FILE_PAGES); 269 if (PageSwapBacked(page)) { 270 __dec_zone_page_state(page, NR_SHMEM); 271 __inc_zone_page_state(newpage, NR_SHMEM); 272 } 273 spin_unlock_irq(&mapping->tree_lock); 274 275 return 0; 276} 277 278/* 279 * Copy the page to its new location 280 */ 281static void migrate_page_copy(struct page *newpage, struct page *page) 282{ 283 copy_highpage(newpage, page); 284 285 if (PageError(page)) 286 SetPageError(newpage); 287 if (PageReferenced(page)) 288 SetPageReferenced(newpage); 289 if (PageUptodate(page)) 290 SetPageUptodate(newpage); 291 if (TestClearPageActive(page)) { 292 VM_BUG_ON(PageUnevictable(page)); 293 SetPageActive(newpage); 294 } else if (TestClearPageUnevictable(page)) 295 SetPageUnevictable(newpage); 296 if (PageChecked(page)) 297 SetPageChecked(newpage); 298 if (PageMappedToDisk(page)) 299 SetPageMappedToDisk(newpage); 300 301 if (PageDirty(page)) { 302 clear_page_dirty_for_io(page); 303 /* 304 * Want to mark the page and the radix tree as dirty, and 305 * redo the accounting that clear_page_dirty_for_io undid, 306 * but we can't use set_page_dirty because that function 307 * is actually a signal that all of the page has become dirty. 308 * Wheras only part of our page may be dirty. 309 */ 310 __set_page_dirty_nobuffers(newpage); 311 } 312 313 mlock_migrate_page(newpage, page); 314 ksm_migrate_page(newpage, page); 315 316 ClearPageSwapCache(page); 317 ClearPagePrivate(page); 318 set_page_private(page, 0); 319 page->mapping = NULL; 320 321 /* 322 * If any waiters have accumulated on the new page then 323 * wake them up. 324 */ 325 if (PageWriteback(newpage)) 326 end_page_writeback(newpage); 327} 328 329/************************************************************ 330 * Migration functions 331 ***********************************************************/ 332 333/* Always fail migration. Used for mappings that are not movable */ 334int fail_migrate_page(struct address_space *mapping, 335 struct page *newpage, struct page *page) 336{ 337 return -EIO; 338} 339EXPORT_SYMBOL(fail_migrate_page); 340 341/* 342 * Common logic to directly migrate a single page suitable for 343 * pages that do not use PagePrivate/PagePrivate2. 344 * 345 * Pages are locked upon entry and exit. 346 */ 347int migrate_page(struct address_space *mapping, 348 struct page *newpage, struct page *page) 349{ 350 int rc; 351 352 BUG_ON(PageWriteback(page)); /* Writeback must be complete */ 353 354 rc = migrate_page_move_mapping(mapping, newpage, page); 355 356 if (rc) 357 return rc; 358 359 migrate_page_copy(newpage, page); 360 return 0; 361} 362EXPORT_SYMBOL(migrate_page); 363 364#ifdef CONFIG_BLOCK 365/* 366 * Migration function for pages with buffers. This function can only be used 367 * if the underlying filesystem guarantees that no other references to "page" 368 * exist. 369 */ 370int buffer_migrate_page(struct address_space *mapping, 371 struct page *newpage, struct page *page) 372{ 373 struct buffer_head *bh, *head; 374 int rc; 375 376 if (!page_has_buffers(page)) 377 return migrate_page(mapping, newpage, page); 378 379 head = page_buffers(page); 380 381 rc = migrate_page_move_mapping(mapping, newpage, page); 382 383 if (rc) 384 return rc; 385 386 bh = head; 387 do { 388 get_bh(bh); 389 lock_buffer(bh); 390 bh = bh->b_this_page; 391 392 } while (bh != head); 393 394 ClearPagePrivate(page); 395 set_page_private(newpage, page_private(page)); 396 set_page_private(page, 0); 397 put_page(page); 398 get_page(newpage); 399 400 bh = head; 401 do { 402 set_bh_page(bh, newpage, bh_offset(bh)); 403 bh = bh->b_this_page; 404 405 } while (bh != head); 406 407 SetPagePrivate(newpage); 408 409 migrate_page_copy(newpage, page); 410 411 bh = head; 412 do { 413 unlock_buffer(bh); 414 put_bh(bh); 415 bh = bh->b_this_page; 416 417 } while (bh != head); 418 419 return 0; 420} 421EXPORT_SYMBOL(buffer_migrate_page); 422#endif 423 424/* 425 * Writeback a page to clean the dirty state 426 */ 427static int writeout(struct address_space *mapping, struct page *page) 428{ 429 struct writeback_control wbc = { 430 .sync_mode = WB_SYNC_NONE, 431 .nr_to_write = 1, 432 .range_start = 0, 433 .range_end = LLONG_MAX, 434 .nonblocking = 1, 435 .for_reclaim = 1 436 }; 437 int rc; 438 439 if (!mapping->a_ops->writepage) 440 /* No write method for the address space */ 441 return -EINVAL; 442 443 if (!clear_page_dirty_for_io(page)) 444 /* Someone else already triggered a write */ 445 return -EAGAIN; 446 447 /* 448 * A dirty page may imply that the underlying filesystem has 449 * the page on some queue. So the page must be clean for 450 * migration. Writeout may mean we loose the lock and the 451 * page state is no longer what we checked for earlier. 452 * At this point we know that the migration attempt cannot 453 * be successful. 454 */ 455 remove_migration_ptes(page, page); 456 457 rc = mapping->a_ops->writepage(page, &wbc); 458 459 if (rc != AOP_WRITEPAGE_ACTIVATE) 460 /* unlocked. Relock */ 461 lock_page(page); 462 463 return (rc < 0) ? -EIO : -EAGAIN; 464} 465 466/* 467 * Default handling if a filesystem does not provide a migration function. 468 */ 469static int fallback_migrate_page(struct address_space *mapping, 470 struct page *newpage, struct page *page) 471{ 472 if (PageDirty(page)) 473 return writeout(mapping, page); 474 475 /* 476 * Buffers may be managed in a filesystem specific way. 477 * We must have no buffers or drop them. 478 */ 479 if (page_has_private(page) && 480 !try_to_release_page(page, GFP_KERNEL)) 481 return -EAGAIN; 482 483 return migrate_page(mapping, newpage, page); 484} 485 486/* 487 * Move a page to a newly allocated page 488 * The page is locked and all ptes have been successfully removed. 489 * 490 * The new page will have replaced the old page if this function 491 * is successful. 492 * 493 * Return value: 494 * < 0 - error code 495 * == 0 - success 496 */ 497static int move_to_new_page(struct page *newpage, struct page *page, 498 int remap_swapcache) 499{ 500 struct address_space *mapping; 501 int rc; 502 503 /* 504 * Block others from accessing the page when we get around to 505 * establishing additional references. We are the only one 506 * holding a reference to the new page at this point. 507 */ 508 if (!trylock_page(newpage)) 509 BUG(); 510 511 /* Prepare mapping for the new page.*/ 512 newpage->index = page->index; 513 newpage->mapping = page->mapping; 514 if (PageSwapBacked(page)) 515 SetPageSwapBacked(newpage); 516 517 mapping = page_mapping(page); 518 if (!mapping) 519 rc = migrate_page(mapping, newpage, page); 520 else if (mapping->a_ops->migratepage) 521 /* 522 * Most pages have a mapping and most filesystems 523 * should provide a migration function. Anonymous 524 * pages are part of swap space which also has its 525 * own migration function. This is the most common 526 * path for page migration. 527 */ 528 rc = mapping->a_ops->migratepage(mapping, 529 newpage, page); 530 else 531 rc = fallback_migrate_page(mapping, newpage, page); 532 533 if (rc) { 534 newpage->mapping = NULL; 535 } else { 536 if (remap_swapcache) 537 remove_migration_ptes(page, newpage); 538 } 539 540 unlock_page(newpage); 541 542 return rc; 543} 544 545/* 546 * Obtain the lock on page, remove all ptes and migrate the page 547 * to the newly allocated page in newpage. 548 */ 549static int unmap_and_move(new_page_t get_new_page, unsigned long private, 550 struct page *page, int force, int offlining) 551{ 552 int rc = 0; 553 int *result = NULL; 554 struct page *newpage = get_new_page(page, private, &result); 555 int remap_swapcache = 1; 556 int charge = 0; 557 struct mem_cgroup *mem = NULL; 558 struct anon_vma *anon_vma = NULL; 559 560 if (!newpage) 561 return -ENOMEM; 562 563 if (page_count(page) == 1) { 564 /* page was freed from under us. So we are done. */ 565 goto move_newpage; 566 } 567 568 /* prepare cgroup just returns 0 or -ENOMEM */ 569 rc = -EAGAIN; 570 571 if (!trylock_page(page)) { 572 if (!force) 573 goto move_newpage; 574 lock_page(page); 575 } 576 577 /* 578 * Only memory hotplug's offline_pages() caller has locked out KSM, 579 * and can safely migrate a KSM page. The other cases have skipped 580 * PageKsm along with PageReserved - but it is only now when we have 581 * the page lock that we can be certain it will not go KSM beneath us 582 * (KSM will not upgrade a page from PageAnon to PageKsm when it sees 583 * its pagecount raised, but only here do we take the page lock which 584 * serializes that). 585 */ 586 if (PageKsm(page) && !offlining) { 587 rc = -EBUSY; 588 goto unlock; 589 } 590 591 /* charge against new page */ 592 charge = mem_cgroup_prepare_migration(page, newpage, &mem); 593 if (charge == -ENOMEM) { 594 rc = -ENOMEM; 595 goto unlock; 596 } 597 BUG_ON(charge); 598 599 if (PageWriteback(page)) { 600 if (!force) 601 goto uncharge; 602 wait_on_page_writeback(page); 603 } 604 /* 605 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case, 606 * we cannot notice that anon_vma is freed while we migrates a page. 607 * This get_anon_vma() delays freeing anon_vma pointer until the end 608 * of migration. File cache pages are no problem because of page_lock() 609 * File Caches may use write_page() or lock_page() in migration, then, 610 * just care Anon page here. 611 */ 612 if (PageAnon(page)) { 613 /* 614 * Only page_lock_anon_vma() understands the subtleties of 615 * getting a hold on an anon_vma from outside one of its mms. 616 */ 617 anon_vma = page_lock_anon_vma(page); 618 if (anon_vma) { 619 /* 620 * Take a reference count on the anon_vma if the 621 * page is mapped so that it is guaranteed to 622 * exist when the page is remapped later 623 */ 624 get_anon_vma(anon_vma); 625 page_unlock_anon_vma(anon_vma); 626 } else if (PageSwapCache(page)) { 627 /* 628 * We cannot be sure that the anon_vma of an unmapped 629 * swapcache page is safe to use because we don't 630 * know in advance if the VMA that this page belonged 631 * to still exists. If the VMA and others sharing the 632 * data have been freed, then the anon_vma could 633 * already be invalid. 634 * 635 * To avoid this possibility, swapcache pages get 636 * migrated but are not remapped when migration 637 * completes 638 */ 639 remap_swapcache = 0; 640 } else { 641 goto uncharge; 642 } 643 } 644 645 /* 646 * Corner case handling: 647 * 1. When a new swap-cache page is read into, it is added to the LRU 648 * and treated as swapcache but it has no rmap yet. 649 * Calling try_to_unmap() against a page->mapping==NULL page will 650 * trigger a BUG. So handle it here. 651 * 2. An orphaned page (see truncate_complete_page) might have 652 * fs-private metadata. The page can be picked up due to memory 653 * offlining. Everywhere else except page reclaim, the page is 654 * invisible to the vm, so the page can not be migrated. So try to 655 * free the metadata, so the page can be freed. 656 */ 657 if (!page->mapping) { 658 VM_BUG_ON(PageAnon(page)); 659 if (page_has_private(page)) { 660 try_to_free_buffers(page); 661 goto uncharge; 662 } 663 goto skip_unmap; 664 } 665 666 /* Establish migration ptes or remove ptes */ 667 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS); 668 669skip_unmap: 670 if (!page_mapped(page)) 671 rc = move_to_new_page(newpage, page, remap_swapcache); 672 673 if (rc && remap_swapcache) 674 remove_migration_ptes(page, page); 675 676 /* Drop an anon_vma reference if we took one */ 677 if (anon_vma) 678 drop_anon_vma(anon_vma); 679 680uncharge: 681 if (!charge) 682 mem_cgroup_end_migration(mem, page, newpage); 683unlock: 684 unlock_page(page); 685 686 if (rc != -EAGAIN) { 687 /* 688 * A page that has been migrated has all references 689 * removed and will be freed. A page that has not been 690 * migrated will have kepts its references and be 691 * restored. 692 */ 693 list_del(&page->lru); 694 dec_zone_page_state(page, NR_ISOLATED_ANON + 695 page_is_file_cache(page)); 696 putback_lru_page(page); 697 } 698 699move_newpage: 700 701 /* 702 * Move the new page to the LRU. If migration was not successful 703 * then this will free the page. 704 */ 705 putback_lru_page(newpage); 706 707 if (result) { 708 if (rc) 709 *result = rc; 710 else 711 *result = page_to_nid(newpage); 712 } 713 return rc; 714} 715 716/* 717 * migrate_pages 718 * 719 * The function takes one list of pages to migrate and a function 720 * that determines from the page to be migrated and the private data 721 * the target of the move and allocates the page. 722 * 723 * The function returns after 10 attempts or if no pages 724 * are movable anymore because to has become empty 725 * or no retryable pages exist anymore. All pages will be 726 * returned to the LRU or freed. 727 * 728 * Return: Number of pages not migrated or error code. 729 */ 730int migrate_pages(struct list_head *from, 731 new_page_t get_new_page, unsigned long private, int offlining) 732{ 733 int retry = 1; 734 int nr_failed = 0; 735 int pass = 0; 736 struct page *page; 737 struct page *page2; 738 int swapwrite = current->flags & PF_SWAPWRITE; 739 int rc; 740 741 if (!swapwrite) 742 current->flags |= PF_SWAPWRITE; 743 744 for(pass = 0; pass < 10 && retry; pass++) { 745 retry = 0; 746 747 list_for_each_entry_safe(page, page2, from, lru) { 748 cond_resched(); 749 750 rc = unmap_and_move(get_new_page, private, 751 page, pass > 2, offlining); 752 753 switch(rc) { 754 case -ENOMEM: 755 goto out; 756 case -EAGAIN: 757 retry++; 758 break; 759 case 0: 760 break; 761 default: 762 /* Permanent failure */ 763 nr_failed++; 764 break; 765 } 766 } 767 } 768 rc = 0; 769out: 770 if (!swapwrite) 771 current->flags &= ~PF_SWAPWRITE; 772 773 putback_lru_pages(from); 774 775 if (rc) 776 return rc; 777 778 return nr_failed + retry; 779} 780 781#ifdef CONFIG_NUMA 782/* 783 * Move a list of individual pages 784 */ 785struct page_to_node { 786 unsigned long addr; 787 struct page *page; 788 int node; 789 int status; 790}; 791 792static struct page *new_page_node(struct page *p, unsigned long private, 793 int **result) 794{ 795 struct page_to_node *pm = (struct page_to_node *)private; 796 797 while (pm->node != MAX_NUMNODES && pm->page != p) 798 pm++; 799 800 if (pm->node == MAX_NUMNODES) 801 return NULL; 802 803 *result = &pm->status; 804 805 return alloc_pages_exact_node(pm->node, 806 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0); 807} 808 809/* 810 * Move a set of pages as indicated in the pm array. The addr 811 * field must be set to the virtual address of the page to be moved 812 * and the node number must contain a valid target node. 813 * The pm array ends with node = MAX_NUMNODES. 814 */ 815static int do_move_page_to_node_array(struct mm_struct *mm, 816 struct page_to_node *pm, 817 int migrate_all) 818{ 819 int err; 820 struct page_to_node *pp; 821 LIST_HEAD(pagelist); 822 823 down_read(&mm->mmap_sem); 824 825 /* 826 * Build a list of pages to migrate 827 */ 828 for (pp = pm; pp->node != MAX_NUMNODES; pp++) { 829 struct vm_area_struct *vma; 830 struct page *page; 831 832 err = -EFAULT; 833 vma = find_vma(mm, pp->addr); 834 if (!vma || !vma_migratable(vma)) 835 goto set_status; 836 837 page = follow_page(vma, pp->addr, FOLL_GET); 838 839 err = PTR_ERR(page); 840 if (IS_ERR(page)) 841 goto set_status; 842 843 err = -ENOENT; 844 if (!page) 845 goto set_status; 846 847 /* Use PageReserved to check for zero page */ 848 if (PageReserved(page) || PageKsm(page)) 849 goto put_and_set; 850 851 pp->page = page; 852 err = page_to_nid(page); 853 854 if (err == pp->node) 855 /* 856 * Node already in the right place 857 */ 858 goto put_and_set; 859 860 err = -EACCES; 861 if (page_mapcount(page) > 1 && 862 !migrate_all) 863 goto put_and_set; 864 865 err = isolate_lru_page(page); 866 if (!err) { 867 list_add_tail(&page->lru, &pagelist); 868 inc_zone_page_state(page, NR_ISOLATED_ANON + 869 page_is_file_cache(page)); 870 } 871put_and_set: 872 /* 873 * Either remove the duplicate refcount from 874 * isolate_lru_page() or drop the page ref if it was 875 * not isolated. 876 */ 877 put_page(page); 878set_status: 879 pp->status = err; 880 } 881 882 err = 0; 883 if (!list_empty(&pagelist)) 884 err = migrate_pages(&pagelist, new_page_node, 885 (unsigned long)pm, 0); 886 887 up_read(&mm->mmap_sem); 888 return err; 889} 890 891/* 892 * Migrate an array of page address onto an array of nodes and fill 893 * the corresponding array of status. 894 */ 895static int do_pages_move(struct mm_struct *mm, struct task_struct *task, 896 unsigned long nr_pages, 897 const void __user * __user *pages, 898 const int __user *nodes, 899 int __user *status, int flags) 900{ 901 struct page_to_node *pm; 902 nodemask_t task_nodes; 903 unsigned long chunk_nr_pages; 904 unsigned long chunk_start; 905 int err; 906 907 task_nodes = cpuset_mems_allowed(task); 908 909 err = -ENOMEM; 910 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL); 911 if (!pm) 912 goto out; 913 914 migrate_prep(); 915 916 /* 917 * Store a chunk of page_to_node array in a page, 918 * but keep the last one as a marker 919 */ 920 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1; 921 922 for (chunk_start = 0; 923 chunk_start < nr_pages; 924 chunk_start += chunk_nr_pages) { 925 int j; 926 927 if (chunk_start + chunk_nr_pages > nr_pages) 928 chunk_nr_pages = nr_pages - chunk_start; 929 930 /* fill the chunk pm with addrs and nodes from user-space */ 931 for (j = 0; j < chunk_nr_pages; j++) { 932 const void __user *p; 933 int node; 934 935 err = -EFAULT; 936 if (get_user(p, pages + j + chunk_start)) 937 goto out_pm; 938 pm[j].addr = (unsigned long) p; 939 940 if (get_user(node, nodes + j + chunk_start)) 941 goto out_pm; 942 943 err = -ENODEV; 944 if (node < 0 || node >= MAX_NUMNODES) 945 goto out_pm; 946 947 if (!node_state(node, N_HIGH_MEMORY)) 948 goto out_pm; 949 950 err = -EACCES; 951 if (!node_isset(node, task_nodes)) 952 goto out_pm; 953 954 pm[j].node = node; 955 } 956 957 /* End marker for this chunk */ 958 pm[chunk_nr_pages].node = MAX_NUMNODES; 959 960 /* Migrate this chunk */ 961 err = do_move_page_to_node_array(mm, pm, 962 flags & MPOL_MF_MOVE_ALL); 963 if (err < 0) 964 goto out_pm; 965 966 /* Return status information */ 967 for (j = 0; j < chunk_nr_pages; j++) 968 if (put_user(pm[j].status, status + j + chunk_start)) { 969 err = -EFAULT; 970 goto out_pm; 971 } 972 } 973 err = 0; 974 975out_pm: 976 free_page((unsigned long)pm); 977out: 978 return err; 979} 980 981/* 982 * Determine the nodes of an array of pages and store it in an array of status. 983 */ 984static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages, 985 const void __user **pages, int *status) 986{ 987 unsigned long i; 988 989 down_read(&mm->mmap_sem); 990 991 for (i = 0; i < nr_pages; i++) { 992 unsigned long addr = (unsigned long)(*pages); 993 struct vm_area_struct *vma; 994 struct page *page; 995 int err = -EFAULT; 996 997 vma = find_vma(mm, addr); 998 if (!vma) 999 goto set_status; 1000 1001 page = follow_page(vma, addr, 0); 1002 1003 err = PTR_ERR(page); 1004 if (IS_ERR(page)) 1005 goto set_status; 1006 1007 err = -ENOENT; 1008 /* Use PageReserved to check for zero page */ 1009 if (!page || PageReserved(page) || PageKsm(page)) 1010 goto set_status; 1011 1012 err = page_to_nid(page); 1013set_status: 1014 *status = err; 1015 1016 pages++; 1017 status++; 1018 } 1019 1020 up_read(&mm->mmap_sem); 1021} 1022 1023/* 1024 * Determine the nodes of a user array of pages and store it in 1025 * a user array of status. 1026 */ 1027static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages, 1028 const void __user * __user *pages, 1029 int __user *status) 1030{ 1031#define DO_PAGES_STAT_CHUNK_NR 16 1032 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR]; 1033 int chunk_status[DO_PAGES_STAT_CHUNK_NR]; 1034 1035 while (nr_pages) { 1036 unsigned long chunk_nr; 1037 1038 chunk_nr = nr_pages; 1039 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR) 1040 chunk_nr = DO_PAGES_STAT_CHUNK_NR; 1041 1042 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages))) 1043 break; 1044 1045 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status); 1046 1047 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status))) 1048 break; 1049 1050 pages += chunk_nr; 1051 status += chunk_nr; 1052 nr_pages -= chunk_nr; 1053 } 1054 return nr_pages ? -EFAULT : 0; 1055} 1056 1057/* 1058 * Move a list of pages in the address space of the currently executing 1059 * process. 1060 */ 1061SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages, 1062 const void __user * __user *, pages, 1063 const int __user *, nodes, 1064 int __user *, status, int, flags) 1065{ 1066 const struct cred *cred = current_cred(), *tcred; 1067 struct task_struct *task; 1068 struct mm_struct *mm; 1069 int err; 1070 1071 /* Check flags */ 1072 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL)) 1073 return -EINVAL; 1074 1075 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) 1076 return -EPERM; 1077 1078 /* Find the mm_struct */ 1079 read_lock(&tasklist_lock); 1080 task = pid ? find_task_by_vpid(pid) : current; 1081 if (!task) { 1082 read_unlock(&tasklist_lock); 1083 return -ESRCH; 1084 } 1085 mm = get_task_mm(task); 1086 read_unlock(&tasklist_lock); 1087 1088 if (!mm) 1089 return -EINVAL; 1090 1091 /* 1092 * Check if this process has the right to modify the specified 1093 * process. The right exists if the process has administrative 1094 * capabilities, superuser privileges or the same 1095 * userid as the target process. 1096 */ 1097 rcu_read_lock(); 1098 tcred = __task_cred(task); 1099 if (cred->euid != tcred->suid && cred->euid != tcred->uid && 1100 cred->uid != tcred->suid && cred->uid != tcred->uid && 1101 !capable(CAP_SYS_NICE)) { 1102 rcu_read_unlock(); 1103 err = -EPERM; 1104 goto out; 1105 } 1106 rcu_read_unlock(); 1107 1108 err = security_task_movememory(task); 1109 if (err) 1110 goto out; 1111 1112 if (nodes) { 1113 err = do_pages_move(mm, task, nr_pages, pages, nodes, status, 1114 flags); 1115 } else { 1116 err = do_pages_stat(mm, nr_pages, pages, status); 1117 } 1118 1119out: 1120 mmput(mm); 1121 return err; 1122} 1123 1124/* 1125 * Call migration functions in the vma_ops that may prepare 1126 * memory in a vm for migration. migration functions may perform 1127 * the migration for vmas that do not have an underlying page struct. 1128 */ 1129int migrate_vmas(struct mm_struct *mm, const nodemask_t *to, 1130 const nodemask_t *from, unsigned long flags) 1131{ 1132 struct vm_area_struct *vma; 1133 int err = 0; 1134 1135 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) { 1136 if (vma->vm_ops && vma->vm_ops->migrate) { 1137 err = vma->vm_ops->migrate(vma, to, from, flags); 1138 if (err) 1139 break; 1140 } 1141 } 1142 return err; 1143} 1144#endif 1145