1/* 2 * linux/kernel/power/snapshot.c 3 * 4 * This file provides system snapshot/restore functionality for swsusp. 5 * 6 * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz> 7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 8 * 9 * This file is released under the GPLv2. 10 * 11 */ 12 13#include <linux/version.h> 14#include <linux/module.h> 15#include <linux/mm.h> 16#include <linux/suspend.h> 17#include <linux/delay.h> 18#include <linux/bitops.h> 19#include <linux/spinlock.h> 20#include <linux/kernel.h> 21#include <linux/pm.h> 22#include <linux/device.h> 23#include <linux/init.h> 24#include <linux/bootmem.h> 25#include <linux/syscalls.h> 26#include <linux/console.h> 27#include <linux/highmem.h> 28#include <linux/list.h> 29#include <linux/slab.h> 30 31#include <asm/uaccess.h> 32#include <asm/mmu_context.h> 33#include <asm/pgtable.h> 34#include <asm/tlbflush.h> 35#include <asm/io.h> 36 37#include "power.h" 38 39static int swsusp_page_is_free(struct page *); 40static void swsusp_set_page_forbidden(struct page *); 41static void swsusp_unset_page_forbidden(struct page *); 42 43/* 44 * Preferred image size in bytes (tunable via /sys/power/image_size). 45 * When it is set to N, swsusp will do its best to ensure the image 46 * size will not exceed N bytes, but if that is impossible, it will 47 * try to create the smallest image possible. 48 */ 49unsigned long image_size = 500 * 1024 * 1024; 50 51/* List of PBEs needed for restoring the pages that were allocated before 52 * the suspend and included in the suspend image, but have also been 53 * allocated by the "resume" kernel, so their contents cannot be written 54 * directly to their "original" page frames. 55 */ 56struct pbe *restore_pblist; 57 58/* Pointer to an auxiliary buffer (1 page) */ 59static void *buffer; 60 61/** 62 * @safe_needed - on resume, for storing the PBE list and the image, 63 * we can only use memory pages that do not conflict with the pages 64 * used before suspend. The unsafe pages have PageNosaveFree set 65 * and we count them using unsafe_pages. 66 * 67 * Each allocated image page is marked as PageNosave and PageNosaveFree 68 * so that swsusp_free() can release it. 69 */ 70 71#define PG_ANY 0 72#define PG_SAFE 1 73#define PG_UNSAFE_CLEAR 1 74#define PG_UNSAFE_KEEP 0 75 76static unsigned int allocated_unsafe_pages; 77 78static void *get_image_page(gfp_t gfp_mask, int safe_needed) 79{ 80 void *res; 81 82 res = (void *)get_zeroed_page(gfp_mask); 83 if (safe_needed) 84 while (res && swsusp_page_is_free(virt_to_page(res))) { 85 /* The page is unsafe, mark it for swsusp_free() */ 86 swsusp_set_page_forbidden(virt_to_page(res)); 87 allocated_unsafe_pages++; 88 res = (void *)get_zeroed_page(gfp_mask); 89 } 90 if (res) { 91 swsusp_set_page_forbidden(virt_to_page(res)); 92 swsusp_set_page_free(virt_to_page(res)); 93 } 94 return res; 95} 96 97unsigned long get_safe_page(gfp_t gfp_mask) 98{ 99 return (unsigned long)get_image_page(gfp_mask, PG_SAFE); 100} 101 102static struct page *alloc_image_page(gfp_t gfp_mask) 103{ 104 struct page *page; 105 106 page = alloc_page(gfp_mask); 107 if (page) { 108 swsusp_set_page_forbidden(page); 109 swsusp_set_page_free(page); 110 } 111 return page; 112} 113 114/** 115 * free_image_page - free page represented by @addr, allocated with 116 * get_image_page (page flags set by it must be cleared) 117 */ 118 119static inline void free_image_page(void *addr, int clear_nosave_free) 120{ 121 struct page *page; 122 123 BUG_ON(!virt_addr_valid(addr)); 124 125 page = virt_to_page(addr); 126 127 swsusp_unset_page_forbidden(page); 128 if (clear_nosave_free) 129 swsusp_unset_page_free(page); 130 131 __free_page(page); 132} 133 134/* struct linked_page is used to build chains of pages */ 135 136#define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *)) 137 138struct linked_page { 139 struct linked_page *next; 140 char data[LINKED_PAGE_DATA_SIZE]; 141} __attribute__((packed)); 142 143static inline void 144free_list_of_pages(struct linked_page *list, int clear_page_nosave) 145{ 146 while (list) { 147 struct linked_page *lp = list->next; 148 149 free_image_page(list, clear_page_nosave); 150 list = lp; 151 } 152} 153 154/** 155 * struct chain_allocator is used for allocating small objects out of 156 * a linked list of pages called 'the chain'. 157 * 158 * The chain grows each time when there is no room for a new object in 159 * the current page. The allocated objects cannot be freed individually. 160 * It is only possible to free them all at once, by freeing the entire 161 * chain. 162 * 163 * NOTE: The chain allocator may be inefficient if the allocated objects 164 * are not much smaller than PAGE_SIZE. 165 */ 166 167struct chain_allocator { 168 struct linked_page *chain; /* the chain */ 169 unsigned int used_space; /* total size of objects allocated out 170 * of the current page 171 */ 172 gfp_t gfp_mask; /* mask for allocating pages */ 173 int safe_needed; /* if set, only "safe" pages are allocated */ 174}; 175 176static void 177chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed) 178{ 179 ca->chain = NULL; 180 ca->used_space = LINKED_PAGE_DATA_SIZE; 181 ca->gfp_mask = gfp_mask; 182 ca->safe_needed = safe_needed; 183} 184 185static void *chain_alloc(struct chain_allocator *ca, unsigned int size) 186{ 187 void *ret; 188 189 if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) { 190 struct linked_page *lp; 191 192 lp = get_image_page(ca->gfp_mask, ca->safe_needed); 193 if (!lp) 194 return NULL; 195 196 lp->next = ca->chain; 197 ca->chain = lp; 198 ca->used_space = 0; 199 } 200 ret = ca->chain->data + ca->used_space; 201 ca->used_space += size; 202 return ret; 203} 204 205/** 206 * Data types related to memory bitmaps. 207 * 208 * Memory bitmap is a structure consiting of many linked lists of 209 * objects. The main list's elements are of type struct zone_bitmap 210 * and each of them corresonds to one zone. For each zone bitmap 211 * object there is a list of objects of type struct bm_block that 212 * represent each blocks of bitmap in which information is stored. 213 * 214 * struct memory_bitmap contains a pointer to the main list of zone 215 * bitmap objects, a struct bm_position used for browsing the bitmap, 216 * and a pointer to the list of pages used for allocating all of the 217 * zone bitmap objects and bitmap block objects. 218 * 219 * NOTE: It has to be possible to lay out the bitmap in memory 220 * using only allocations of order 0. Additionally, the bitmap is 221 * designed to work with arbitrary number of zones (this is over the 222 * top for now, but let's avoid making unnecessary assumptions ;-). 223 * 224 * struct zone_bitmap contains a pointer to a list of bitmap block 225 * objects and a pointer to the bitmap block object that has been 226 * most recently used for setting bits. Additionally, it contains the 227 * pfns that correspond to the start and end of the represented zone. 228 * 229 * struct bm_block contains a pointer to the memory page in which 230 * information is stored (in the form of a block of bitmap) 231 * It also contains the pfns that correspond to the start and end of 232 * the represented memory area. 233 */ 234 235#define BM_END_OF_MAP (~0UL) 236 237#define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE) 238 239struct bm_block { 240 struct list_head hook; /* hook into a list of bitmap blocks */ 241 unsigned long start_pfn; /* pfn represented by the first bit */ 242 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */ 243 unsigned long *data; /* bitmap representing pages */ 244}; 245 246static inline unsigned long bm_block_bits(struct bm_block *bb) 247{ 248 return bb->end_pfn - bb->start_pfn; 249} 250 251/* strcut bm_position is used for browsing memory bitmaps */ 252 253struct bm_position { 254 struct bm_block *block; 255 int bit; 256}; 257 258struct memory_bitmap { 259 struct list_head blocks; /* list of bitmap blocks */ 260 struct linked_page *p_list; /* list of pages used to store zone 261 * bitmap objects and bitmap block 262 * objects 263 */ 264 struct bm_position cur; /* most recently used bit position */ 265}; 266 267/* Functions that operate on memory bitmaps */ 268 269static void memory_bm_position_reset(struct memory_bitmap *bm) 270{ 271 bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook); 272 bm->cur.bit = 0; 273} 274 275static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); 276 277/** 278 * create_bm_block_list - create a list of block bitmap objects 279 * @pages - number of pages to track 280 * @list - list to put the allocated blocks into 281 * @ca - chain allocator to be used for allocating memory 282 */ 283static int create_bm_block_list(unsigned long pages, 284 struct list_head *list, 285 struct chain_allocator *ca) 286{ 287 unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK); 288 289 while (nr_blocks-- > 0) { 290 struct bm_block *bb; 291 292 bb = chain_alloc(ca, sizeof(struct bm_block)); 293 if (!bb) 294 return -ENOMEM; 295 list_add(&bb->hook, list); 296 } 297 298 return 0; 299} 300 301struct mem_extent { 302 struct list_head hook; 303 unsigned long start; 304 unsigned long end; 305}; 306 307/** 308 * free_mem_extents - free a list of memory extents 309 * @list - list of extents to empty 310 */ 311static void free_mem_extents(struct list_head *list) 312{ 313 struct mem_extent *ext, *aux; 314 315 list_for_each_entry_safe(ext, aux, list, hook) { 316 list_del(&ext->hook); 317 kfree(ext); 318 } 319} 320 321/** 322 * create_mem_extents - create a list of memory extents representing 323 * contiguous ranges of PFNs 324 * @list - list to put the extents into 325 * @gfp_mask - mask to use for memory allocations 326 */ 327static int create_mem_extents(struct list_head *list, gfp_t gfp_mask) 328{ 329 struct zone *zone; 330 331 INIT_LIST_HEAD(list); 332 333 for_each_populated_zone(zone) { 334 unsigned long zone_start, zone_end; 335 struct mem_extent *ext, *cur, *aux; 336 337 zone_start = zone->zone_start_pfn; 338 zone_end = zone->zone_start_pfn + zone->spanned_pages; 339 340 list_for_each_entry(ext, list, hook) 341 if (zone_start <= ext->end) 342 break; 343 344 if (&ext->hook == list || zone_end < ext->start) { 345 /* New extent is necessary */ 346 struct mem_extent *new_ext; 347 348 new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask); 349 if (!new_ext) { 350 free_mem_extents(list); 351 return -ENOMEM; 352 } 353 new_ext->start = zone_start; 354 new_ext->end = zone_end; 355 list_add_tail(&new_ext->hook, &ext->hook); 356 continue; 357 } 358 359 /* Merge this zone's range of PFNs with the existing one */ 360 if (zone_start < ext->start) 361 ext->start = zone_start; 362 if (zone_end > ext->end) 363 ext->end = zone_end; 364 365 /* More merging may be possible */ 366 cur = ext; 367 list_for_each_entry_safe_continue(cur, aux, list, hook) { 368 if (zone_end < cur->start) 369 break; 370 if (zone_end < cur->end) 371 ext->end = cur->end; 372 list_del(&cur->hook); 373 kfree(cur); 374 } 375 } 376 377 return 0; 378} 379 380/** 381 * memory_bm_create - allocate memory for a memory bitmap 382 */ 383static int 384memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed) 385{ 386 struct chain_allocator ca; 387 struct list_head mem_extents; 388 struct mem_extent *ext; 389 int error; 390 391 chain_init(&ca, gfp_mask, safe_needed); 392 INIT_LIST_HEAD(&bm->blocks); 393 394 error = create_mem_extents(&mem_extents, gfp_mask); 395 if (error) 396 return error; 397 398 list_for_each_entry(ext, &mem_extents, hook) { 399 struct bm_block *bb; 400 unsigned long pfn = ext->start; 401 unsigned long pages = ext->end - ext->start; 402 403 bb = list_entry(bm->blocks.prev, struct bm_block, hook); 404 405 error = create_bm_block_list(pages, bm->blocks.prev, &ca); 406 if (error) 407 goto Error; 408 409 list_for_each_entry_continue(bb, &bm->blocks, hook) { 410 bb->data = get_image_page(gfp_mask, safe_needed); 411 if (!bb->data) { 412 error = -ENOMEM; 413 goto Error; 414 } 415 416 bb->start_pfn = pfn; 417 if (pages >= BM_BITS_PER_BLOCK) { 418 pfn += BM_BITS_PER_BLOCK; 419 pages -= BM_BITS_PER_BLOCK; 420 } else { 421 /* This is executed only once in the loop */ 422 pfn += pages; 423 } 424 bb->end_pfn = pfn; 425 } 426 } 427 428 bm->p_list = ca.chain; 429 memory_bm_position_reset(bm); 430 Exit: 431 free_mem_extents(&mem_extents); 432 return error; 433 434 Error: 435 bm->p_list = ca.chain; 436 memory_bm_free(bm, PG_UNSAFE_CLEAR); 437 goto Exit; 438} 439 440/** 441 * memory_bm_free - free memory occupied by the memory bitmap @bm 442 */ 443static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) 444{ 445 struct bm_block *bb; 446 447 list_for_each_entry(bb, &bm->blocks, hook) 448 if (bb->data) 449 free_image_page(bb->data, clear_nosave_free); 450 451 free_list_of_pages(bm->p_list, clear_nosave_free); 452 453 INIT_LIST_HEAD(&bm->blocks); 454} 455 456/** 457 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds 458 * to given pfn. The cur_zone_bm member of @bm and the cur_block member 459 * of @bm->cur_zone_bm are updated. 460 */ 461static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn, 462 void **addr, unsigned int *bit_nr) 463{ 464 struct bm_block *bb; 465 466 /* 467 * Check if the pfn corresponds to the current bitmap block and find 468 * the block where it fits if this is not the case. 469 */ 470 bb = bm->cur.block; 471 if (pfn < bb->start_pfn) 472 list_for_each_entry_continue_reverse(bb, &bm->blocks, hook) 473 if (pfn >= bb->start_pfn) 474 break; 475 476 if (pfn >= bb->end_pfn) 477 list_for_each_entry_continue(bb, &bm->blocks, hook) 478 if (pfn >= bb->start_pfn && pfn < bb->end_pfn) 479 break; 480 481 if (&bb->hook == &bm->blocks) 482 return -EFAULT; 483 484 /* The block has been found */ 485 bm->cur.block = bb; 486 pfn -= bb->start_pfn; 487 bm->cur.bit = pfn + 1; 488 *bit_nr = pfn; 489 *addr = bb->data; 490 return 0; 491} 492 493static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn) 494{ 495 void *addr; 496 unsigned int bit; 497 int error; 498 499 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 500 BUG_ON(error); 501 set_bit(bit, addr); 502} 503 504static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn) 505{ 506 void *addr; 507 unsigned int bit; 508 int error; 509 510 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 511 if (!error) 512 set_bit(bit, addr); 513 return error; 514} 515 516static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn) 517{ 518 void *addr; 519 unsigned int bit; 520 int error; 521 522 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 523 BUG_ON(error); 524 clear_bit(bit, addr); 525} 526 527static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn) 528{ 529 void *addr; 530 unsigned int bit; 531 int error; 532 533 error = memory_bm_find_bit(bm, pfn, &addr, &bit); 534 BUG_ON(error); 535 return test_bit(bit, addr); 536} 537 538static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn) 539{ 540 void *addr; 541 unsigned int bit; 542 543 return !memory_bm_find_bit(bm, pfn, &addr, &bit); 544} 545 546/** 547 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit 548 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is 549 * returned. 550 * 551 * It is required to run memory_bm_position_reset() before the first call to 552 * this function. 553 */ 554 555static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm) 556{ 557 struct bm_block *bb; 558 int bit; 559 560 bb = bm->cur.block; 561 do { 562 bit = bm->cur.bit; 563 bit = find_next_bit(bb->data, bm_block_bits(bb), bit); 564 if (bit < bm_block_bits(bb)) 565 goto Return_pfn; 566 567 bb = list_entry(bb->hook.next, struct bm_block, hook); 568 bm->cur.block = bb; 569 bm->cur.bit = 0; 570 } while (&bb->hook != &bm->blocks); 571 572 memory_bm_position_reset(bm); 573 return BM_END_OF_MAP; 574 575 Return_pfn: 576 bm->cur.bit = bit + 1; 577 return bb->start_pfn + bit; 578} 579 580/** 581 * This structure represents a range of page frames the contents of which 582 * should not be saved during the suspend. 583 */ 584 585struct nosave_region { 586 struct list_head list; 587 unsigned long start_pfn; 588 unsigned long end_pfn; 589}; 590 591static LIST_HEAD(nosave_regions); 592 593/** 594 * register_nosave_region - register a range of page frames the contents 595 * of which should not be saved during the suspend (to be used in the early 596 * initialization code) 597 */ 598 599void __init 600__register_nosave_region(unsigned long start_pfn, unsigned long end_pfn, 601 int use_kmalloc) 602{ 603 struct nosave_region *region; 604 605 if (start_pfn >= end_pfn) 606 return; 607 608 if (!list_empty(&nosave_regions)) { 609 /* Try to extend the previous region (they should be sorted) */ 610 region = list_entry(nosave_regions.prev, 611 struct nosave_region, list); 612 if (region->end_pfn == start_pfn) { 613 region->end_pfn = end_pfn; 614 goto Report; 615 } 616 } 617 if (use_kmalloc) { 618 /* during init, this shouldn't fail */ 619 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL); 620 BUG_ON(!region); 621 } else 622 /* This allocation cannot fail */ 623 region = alloc_bootmem(sizeof(struct nosave_region)); 624 region->start_pfn = start_pfn; 625 region->end_pfn = end_pfn; 626 list_add_tail(®ion->list, &nosave_regions); 627 Report: 628 printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n", 629 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT); 630} 631 632/* 633 * Set bits in this map correspond to the page frames the contents of which 634 * should not be saved during the suspend. 635 */ 636static struct memory_bitmap *forbidden_pages_map; 637 638/* Set bits in this map correspond to free page frames. */ 639static struct memory_bitmap *free_pages_map; 640 641/* 642 * Each page frame allocated for creating the image is marked by setting the 643 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously 644 */ 645 646void swsusp_set_page_free(struct page *page) 647{ 648 if (free_pages_map) 649 memory_bm_set_bit(free_pages_map, page_to_pfn(page)); 650} 651 652static int swsusp_page_is_free(struct page *page) 653{ 654 return free_pages_map ? 655 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0; 656} 657 658void swsusp_unset_page_free(struct page *page) 659{ 660 if (free_pages_map) 661 memory_bm_clear_bit(free_pages_map, page_to_pfn(page)); 662} 663 664static void swsusp_set_page_forbidden(struct page *page) 665{ 666 if (forbidden_pages_map) 667 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page)); 668} 669 670int swsusp_page_is_forbidden(struct page *page) 671{ 672 return forbidden_pages_map ? 673 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0; 674} 675 676static void swsusp_unset_page_forbidden(struct page *page) 677{ 678 if (forbidden_pages_map) 679 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page)); 680} 681 682/** 683 * mark_nosave_pages - set bits corresponding to the page frames the 684 * contents of which should not be saved in a given bitmap. 685 */ 686 687static void mark_nosave_pages(struct memory_bitmap *bm) 688{ 689 struct nosave_region *region; 690 691 if (list_empty(&nosave_regions)) 692 return; 693 694 list_for_each_entry(region, &nosave_regions, list) { 695 unsigned long pfn; 696 697 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n", 698 region->start_pfn << PAGE_SHIFT, 699 region->end_pfn << PAGE_SHIFT); 700 701 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++) 702 if (pfn_valid(pfn)) { 703 /* 704 * It is safe to ignore the result of 705 * mem_bm_set_bit_check() here, since we won't 706 * touch the PFNs for which the error is 707 * returned anyway. 708 */ 709 mem_bm_set_bit_check(bm, pfn); 710 } 711 } 712} 713 714/** 715 * create_basic_memory_bitmaps - create bitmaps needed for marking page 716 * frames that should not be saved and free page frames. The pointers 717 * forbidden_pages_map and free_pages_map are only modified if everything 718 * goes well, because we don't want the bits to be used before both bitmaps 719 * are set up. 720 */ 721 722int create_basic_memory_bitmaps(void) 723{ 724 struct memory_bitmap *bm1, *bm2; 725 int error = 0; 726 727 BUG_ON(forbidden_pages_map || free_pages_map); 728 729 bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 730 if (!bm1) 731 return -ENOMEM; 732 733 error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY); 734 if (error) 735 goto Free_first_object; 736 737 bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL); 738 if (!bm2) 739 goto Free_first_bitmap; 740 741 error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY); 742 if (error) 743 goto Free_second_object; 744 745 forbidden_pages_map = bm1; 746 free_pages_map = bm2; 747 mark_nosave_pages(forbidden_pages_map); 748 749 pr_debug("PM: Basic memory bitmaps created\n"); 750 751 return 0; 752 753 Free_second_object: 754 kfree(bm2); 755 Free_first_bitmap: 756 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 757 Free_first_object: 758 kfree(bm1); 759 return -ENOMEM; 760} 761 762/** 763 * free_basic_memory_bitmaps - free memory bitmaps allocated by 764 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary 765 * so that the bitmaps themselves are not referred to while they are being 766 * freed. 767 */ 768 769void free_basic_memory_bitmaps(void) 770{ 771 struct memory_bitmap *bm1, *bm2; 772 773 BUG_ON(!(forbidden_pages_map && free_pages_map)); 774 775 bm1 = forbidden_pages_map; 776 bm2 = free_pages_map; 777 forbidden_pages_map = NULL; 778 free_pages_map = NULL; 779 memory_bm_free(bm1, PG_UNSAFE_CLEAR); 780 kfree(bm1); 781 memory_bm_free(bm2, PG_UNSAFE_CLEAR); 782 kfree(bm2); 783 784 pr_debug("PM: Basic memory bitmaps freed\n"); 785} 786 787/** 788 * snapshot_additional_pages - estimate the number of additional pages 789 * be needed for setting up the suspend image data structures for given 790 * zone (usually the returned value is greater than the exact number) 791 */ 792 793unsigned int snapshot_additional_pages(struct zone *zone) 794{ 795 unsigned int res; 796 797 res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK); 798 res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE); 799 return 2 * res; 800} 801 802#ifdef CONFIG_HIGHMEM 803/** 804 * count_free_highmem_pages - compute the total number of free highmem 805 * pages, system-wide. 806 */ 807 808static unsigned int count_free_highmem_pages(void) 809{ 810 struct zone *zone; 811 unsigned int cnt = 0; 812 813 for_each_populated_zone(zone) 814 if (is_highmem(zone)) 815 cnt += zone_page_state(zone, NR_FREE_PAGES); 816 817 return cnt; 818} 819 820/** 821 * saveable_highmem_page - Determine whether a highmem page should be 822 * included in the suspend image. 823 * 824 * We should save the page if it isn't Nosave or NosaveFree, or Reserved, 825 * and it isn't a part of a free chunk of pages. 826 */ 827static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn) 828{ 829 struct page *page; 830 831 if (!pfn_valid(pfn)) 832 return NULL; 833 834 page = pfn_to_page(pfn); 835 if (page_zone(page) != zone) 836 return NULL; 837 838 BUG_ON(!PageHighMem(page)); 839 840 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page) || 841 PageReserved(page)) 842 return NULL; 843 844 return page; 845} 846 847/** 848 * count_highmem_pages - compute the total number of saveable highmem 849 * pages. 850 */ 851 852static unsigned int count_highmem_pages(void) 853{ 854 struct zone *zone; 855 unsigned int n = 0; 856 857 for_each_populated_zone(zone) { 858 unsigned long pfn, max_zone_pfn; 859 860 if (!is_highmem(zone)) 861 continue; 862 863 mark_free_pages(zone); 864 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 865 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 866 if (saveable_highmem_page(zone, pfn)) 867 n++; 868 } 869 return n; 870} 871#else 872static inline void *saveable_highmem_page(struct zone *z, unsigned long p) 873{ 874 return NULL; 875} 876#endif /* CONFIG_HIGHMEM */ 877 878/** 879 * saveable_page - Determine whether a non-highmem page should be included 880 * in the suspend image. 881 * 882 * We should save the page if it isn't Nosave, and is not in the range 883 * of pages statically defined as 'unsaveable', and it isn't a part of 884 * a free chunk of pages. 885 */ 886static struct page *saveable_page(struct zone *zone, unsigned long pfn) 887{ 888 struct page *page; 889 890 if (!pfn_valid(pfn)) 891 return NULL; 892 893 page = pfn_to_page(pfn); 894 if (page_zone(page) != zone) 895 return NULL; 896 897 BUG_ON(PageHighMem(page)); 898 899 if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page)) 900 return NULL; 901 902 if (PageReserved(page) 903 && (!kernel_page_present(page) || pfn_is_nosave(pfn))) 904 return NULL; 905 906 return page; 907} 908 909/** 910 * count_data_pages - compute the total number of saveable non-highmem 911 * pages. 912 */ 913 914static unsigned int count_data_pages(void) 915{ 916 struct zone *zone; 917 unsigned long pfn, max_zone_pfn; 918 unsigned int n = 0; 919 920 for_each_populated_zone(zone) { 921 if (is_highmem(zone)) 922 continue; 923 924 mark_free_pages(zone); 925 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 926 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 927 if (saveable_page(zone, pfn)) 928 n++; 929 } 930 return n; 931} 932 933/* This is needed, because copy_page and memcpy are not usable for copying 934 * task structs. 935 */ 936static inline void do_copy_page(long *dst, long *src) 937{ 938 int n; 939 940 for (n = PAGE_SIZE / sizeof(long); n; n--) 941 *dst++ = *src++; 942} 943 944 945/** 946 * safe_copy_page - check if the page we are going to copy is marked as 947 * present in the kernel page tables (this always is the case if 948 * CONFIG_DEBUG_PAGEALLOC is not set and in that case 949 * kernel_page_present() always returns 'true'). 950 */ 951static void safe_copy_page(void *dst, struct page *s_page) 952{ 953 if (kernel_page_present(s_page)) { 954 do_copy_page(dst, page_address(s_page)); 955 } else { 956 kernel_map_pages(s_page, 1, 1); 957 do_copy_page(dst, page_address(s_page)); 958 kernel_map_pages(s_page, 1, 0); 959 } 960} 961 962 963#ifdef CONFIG_HIGHMEM 964static inline struct page * 965page_is_saveable(struct zone *zone, unsigned long pfn) 966{ 967 return is_highmem(zone) ? 968 saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn); 969} 970 971static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 972{ 973 struct page *s_page, *d_page; 974 void *src, *dst; 975 976 s_page = pfn_to_page(src_pfn); 977 d_page = pfn_to_page(dst_pfn); 978 if (PageHighMem(s_page)) { 979 src = kmap_atomic(s_page, KM_USER0); 980 dst = kmap_atomic(d_page, KM_USER1); 981 do_copy_page(dst, src); 982 kunmap_atomic(src, KM_USER0); 983 kunmap_atomic(dst, KM_USER1); 984 } else { 985 if (PageHighMem(d_page)) { 986 /* Page pointed to by src may contain some kernel 987 * data modified by kmap_atomic() 988 */ 989 safe_copy_page(buffer, s_page); 990 dst = kmap_atomic(d_page, KM_USER0); 991 memcpy(dst, buffer, PAGE_SIZE); 992 kunmap_atomic(dst, KM_USER0); 993 } else { 994 safe_copy_page(page_address(d_page), s_page); 995 } 996 } 997} 998#else 999#define page_is_saveable(zone, pfn) saveable_page(zone, pfn) 1000 1001static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn) 1002{ 1003 safe_copy_page(page_address(pfn_to_page(dst_pfn)), 1004 pfn_to_page(src_pfn)); 1005} 1006#endif /* CONFIG_HIGHMEM */ 1007 1008static void 1009copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm) 1010{ 1011 struct zone *zone; 1012 unsigned long pfn; 1013 1014 for_each_populated_zone(zone) { 1015 unsigned long max_zone_pfn; 1016 1017 mark_free_pages(zone); 1018 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1019 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1020 if (page_is_saveable(zone, pfn)) 1021 memory_bm_set_bit(orig_bm, pfn); 1022 } 1023 memory_bm_position_reset(orig_bm); 1024 memory_bm_position_reset(copy_bm); 1025 for(;;) { 1026 pfn = memory_bm_next_pfn(orig_bm); 1027 if (unlikely(pfn == BM_END_OF_MAP)) 1028 break; 1029 copy_data_page(memory_bm_next_pfn(copy_bm), pfn); 1030 } 1031} 1032 1033/* Total number of image pages */ 1034static unsigned int nr_copy_pages; 1035/* Number of pages needed for saving the original pfns of the image pages */ 1036static unsigned int nr_meta_pages; 1037/* 1038 * Numbers of normal and highmem page frames allocated for hibernation image 1039 * before suspending devices. 1040 */ 1041unsigned int alloc_normal, alloc_highmem; 1042/* 1043 * Memory bitmap used for marking saveable pages (during hibernation) or 1044 * hibernation image pages (during restore) 1045 */ 1046static struct memory_bitmap orig_bm; 1047/* 1048 * Memory bitmap used during hibernation for marking allocated page frames that 1049 * will contain copies of saveable pages. During restore it is initially used 1050 * for marking hibernation image pages, but then the set bits from it are 1051 * duplicated in @orig_bm and it is released. On highmem systems it is next 1052 * used for marking "safe" highmem pages, but it has to be reinitialized for 1053 * this purpose. 1054 */ 1055static struct memory_bitmap copy_bm; 1056 1057/** 1058 * swsusp_free - free pages allocated for the suspend. 1059 * 1060 * Suspend pages are alocated before the atomic copy is made, so we 1061 * need to release them after the resume. 1062 */ 1063 1064void swsusp_free(void) 1065{ 1066 struct zone *zone; 1067 unsigned long pfn, max_zone_pfn; 1068 1069 for_each_populated_zone(zone) { 1070 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1071 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1072 if (pfn_valid(pfn)) { 1073 struct page *page = pfn_to_page(pfn); 1074 1075 if (swsusp_page_is_forbidden(page) && 1076 swsusp_page_is_free(page)) { 1077 swsusp_unset_page_forbidden(page); 1078 swsusp_unset_page_free(page); 1079 __free_page(page); 1080 } 1081 } 1082 } 1083 nr_copy_pages = 0; 1084 nr_meta_pages = 0; 1085 restore_pblist = NULL; 1086 buffer = NULL; 1087 alloc_normal = 0; 1088 alloc_highmem = 0; 1089} 1090 1091/* Helper functions used for the shrinking of memory. */ 1092 1093#define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN) 1094 1095/** 1096 * preallocate_image_pages - Allocate a number of pages for hibernation image 1097 * @nr_pages: Number of page frames to allocate. 1098 * @mask: GFP flags to use for the allocation. 1099 * 1100 * Return value: Number of page frames actually allocated 1101 */ 1102static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask) 1103{ 1104 unsigned long nr_alloc = 0; 1105 1106 while (nr_pages > 0) { 1107 struct page *page; 1108 1109 page = alloc_image_page(mask); 1110 if (!page) 1111 break; 1112 memory_bm_set_bit(©_bm, page_to_pfn(page)); 1113 if (PageHighMem(page)) 1114 alloc_highmem++; 1115 else 1116 alloc_normal++; 1117 nr_pages--; 1118 nr_alloc++; 1119 } 1120 1121 return nr_alloc; 1122} 1123 1124static unsigned long preallocate_image_memory(unsigned long nr_pages, 1125 unsigned long avail_normal) 1126{ 1127 unsigned long alloc; 1128 1129 if (avail_normal <= alloc_normal) 1130 return 0; 1131 1132 alloc = avail_normal - alloc_normal; 1133 if (nr_pages < alloc) 1134 alloc = nr_pages; 1135 1136 return preallocate_image_pages(alloc, GFP_IMAGE); 1137} 1138 1139#ifdef CONFIG_HIGHMEM 1140static unsigned long preallocate_image_highmem(unsigned long nr_pages) 1141{ 1142 return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM); 1143} 1144 1145/** 1146 * __fraction - Compute (an approximation of) x * (multiplier / base) 1147 */ 1148static unsigned long __fraction(u64 x, u64 multiplier, u64 base) 1149{ 1150 x *= multiplier; 1151 do_div(x, base); 1152 return (unsigned long)x; 1153} 1154 1155static unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1156 unsigned long highmem, 1157 unsigned long total) 1158{ 1159 unsigned long alloc = __fraction(nr_pages, highmem, total); 1160 1161 return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM); 1162} 1163#else /* CONFIG_HIGHMEM */ 1164static inline unsigned long preallocate_image_highmem(unsigned long nr_pages) 1165{ 1166 return 0; 1167} 1168 1169static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages, 1170 unsigned long highmem, 1171 unsigned long total) 1172{ 1173 return 0; 1174} 1175#endif /* CONFIG_HIGHMEM */ 1176 1177/** 1178 * free_unnecessary_pages - Release preallocated pages not needed for the image 1179 */ 1180static void free_unnecessary_pages(void) 1181{ 1182 unsigned long save, to_free_normal, to_free_highmem; 1183 1184 save = count_data_pages(); 1185 if (alloc_normal >= save) { 1186 to_free_normal = alloc_normal - save; 1187 save = 0; 1188 } else { 1189 to_free_normal = 0; 1190 save -= alloc_normal; 1191 } 1192 save += count_highmem_pages(); 1193 if (alloc_highmem >= save) { 1194 to_free_highmem = alloc_highmem - save; 1195 } else { 1196 to_free_highmem = 0; 1197 to_free_normal -= save - alloc_highmem; 1198 } 1199 1200 memory_bm_position_reset(©_bm); 1201 1202 while (to_free_normal > 0 || to_free_highmem > 0) { 1203 unsigned long pfn = memory_bm_next_pfn(©_bm); 1204 struct page *page = pfn_to_page(pfn); 1205 1206 if (PageHighMem(page)) { 1207 if (!to_free_highmem) 1208 continue; 1209 to_free_highmem--; 1210 alloc_highmem--; 1211 } else { 1212 if (!to_free_normal) 1213 continue; 1214 to_free_normal--; 1215 alloc_normal--; 1216 } 1217 memory_bm_clear_bit(©_bm, pfn); 1218 swsusp_unset_page_forbidden(page); 1219 swsusp_unset_page_free(page); 1220 __free_page(page); 1221 } 1222} 1223 1224/** 1225 * minimum_image_size - Estimate the minimum acceptable size of an image 1226 * @saveable: Number of saveable pages in the system. 1227 * 1228 * We want to avoid attempting to free too much memory too hard, so estimate the 1229 * minimum acceptable size of a hibernation image to use as the lower limit for 1230 * preallocating memory. 1231 * 1232 * We assume that the minimum image size should be proportional to 1233 * 1234 * [number of saveable pages] - [number of pages that can be freed in theory] 1235 * 1236 * where the second term is the sum of (1) reclaimable slab pages, (2) active 1237 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages, 1238 * minus mapped file pages. 1239 */ 1240static unsigned long minimum_image_size(unsigned long saveable) 1241{ 1242 unsigned long size; 1243 1244 size = global_page_state(NR_SLAB_RECLAIMABLE) 1245 + global_page_state(NR_ACTIVE_ANON) 1246 + global_page_state(NR_INACTIVE_ANON) 1247 + global_page_state(NR_ACTIVE_FILE) 1248 + global_page_state(NR_INACTIVE_FILE) 1249 - global_page_state(NR_FILE_MAPPED); 1250 1251 return saveable <= size ? 0 : saveable - size; 1252} 1253 1254/** 1255 * hibernate_preallocate_memory - Preallocate memory for hibernation image 1256 * 1257 * To create a hibernation image it is necessary to make a copy of every page 1258 * frame in use. We also need a number of page frames to be free during 1259 * hibernation for allocations made while saving the image and for device 1260 * drivers, in case they need to allocate memory from their hibernation 1261 * callbacks (these two numbers are given by PAGES_FOR_IO and SPARE_PAGES, 1262 * respectively, both of which are rough estimates). To make this happen, we 1263 * compute the total number of available page frames and allocate at least 1264 * 1265 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 + 2 * SPARE_PAGES 1266 * 1267 * of them, which corresponds to the maximum size of a hibernation image. 1268 * 1269 * If image_size is set below the number following from the above formula, 1270 * the preallocation of memory is continued until the total number of saveable 1271 * pages in the system is below the requested image size or the minimum 1272 * acceptable image size returned by minimum_image_size(), whichever is greater. 1273 */ 1274int hibernate_preallocate_memory(void) 1275{ 1276 struct zone *zone; 1277 unsigned long saveable, size, max_size, count, highmem, pages = 0; 1278 unsigned long alloc, save_highmem, pages_highmem, avail_normal; 1279 struct timeval start, stop; 1280 int error; 1281 1282 printk(KERN_INFO "PM: Preallocating image memory... "); 1283 do_gettimeofday(&start); 1284 1285 error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY); 1286 if (error) 1287 goto err_out; 1288 1289 error = memory_bm_create(©_bm, GFP_IMAGE, PG_ANY); 1290 if (error) 1291 goto err_out; 1292 1293 alloc_normal = 0; 1294 alloc_highmem = 0; 1295 1296 /* Count the number of saveable data pages. */ 1297 save_highmem = count_highmem_pages(); 1298 saveable = count_data_pages(); 1299 1300 /* 1301 * Compute the total number of page frames we can use (count) and the 1302 * number of pages needed for image metadata (size). 1303 */ 1304 count = saveable; 1305 saveable += save_highmem; 1306 highmem = save_highmem; 1307 size = 0; 1308 for_each_populated_zone(zone) { 1309 size += snapshot_additional_pages(zone); 1310 if (is_highmem(zone)) 1311 highmem += zone_page_state(zone, NR_FREE_PAGES); 1312 else 1313 count += zone_page_state(zone, NR_FREE_PAGES); 1314 } 1315 avail_normal = count; 1316 count += highmem; 1317 count -= totalreserve_pages; 1318 1319 /* Compute the maximum number of saveable pages to leave in memory. */ 1320 max_size = (count - (size + PAGES_FOR_IO)) / 2 - 2 * SPARE_PAGES; 1321 size = DIV_ROUND_UP(image_size, PAGE_SIZE); 1322 if (size > max_size) 1323 size = max_size; 1324 /* 1325 * If the maximum is not less than the current number of saveable pages 1326 * in memory, allocate page frames for the image and we're done. 1327 */ 1328 if (size >= saveable) { 1329 pages = preallocate_image_highmem(save_highmem); 1330 pages += preallocate_image_memory(saveable - pages, avail_normal); 1331 goto out; 1332 } 1333 1334 /* Estimate the minimum size of the image. */ 1335 pages = minimum_image_size(saveable); 1336 /* 1337 * To avoid excessive pressure on the normal zone, leave room in it to 1338 * accommodate an image of the minimum size (unless it's already too 1339 * small, in which case don't preallocate pages from it at all). 1340 */ 1341 if (avail_normal > pages) 1342 avail_normal -= pages; 1343 else 1344 avail_normal = 0; 1345 if (size < pages) 1346 size = min_t(unsigned long, pages, max_size); 1347 1348 /* 1349 * Let the memory management subsystem know that we're going to need a 1350 * large number of page frames to allocate and make it free some memory. 1351 * NOTE: If this is not done, performance will be hurt badly in some 1352 * test cases. 1353 */ 1354 shrink_all_memory(saveable - size); 1355 1356 /* 1357 * The number of saveable pages in memory was too high, so apply some 1358 * pressure to decrease it. First, make room for the largest possible 1359 * image and fail if that doesn't work. Next, try to decrease the size 1360 * of the image as much as indicated by 'size' using allocations from 1361 * highmem and non-highmem zones separately. 1362 */ 1363 pages_highmem = preallocate_image_highmem(highmem / 2); 1364 alloc = (count - max_size) - pages_highmem; 1365 pages = preallocate_image_memory(alloc, avail_normal); 1366 if (pages < alloc) { 1367 /* We have exhausted non-highmem pages, try highmem. */ 1368 alloc -= pages; 1369 pages += pages_highmem; 1370 pages_highmem = preallocate_image_highmem(alloc); 1371 if (pages_highmem < alloc) 1372 goto err_out; 1373 pages += pages_highmem; 1374 /* 1375 * size is the desired number of saveable pages to leave in 1376 * memory, so try to preallocate (all memory - size) pages. 1377 */ 1378 alloc = (count - pages) - size; 1379 pages += preallocate_image_highmem(alloc); 1380 } else { 1381 /* 1382 * There are approximately max_size saveable pages at this point 1383 * and we want to reduce this number down to size. 1384 */ 1385 alloc = max_size - size; 1386 size = preallocate_highmem_fraction(alloc, highmem, count); 1387 pages_highmem += size; 1388 alloc -= size; 1389 size = preallocate_image_memory(alloc, avail_normal); 1390 pages_highmem += preallocate_image_highmem(alloc - size); 1391 pages += pages_highmem + size; 1392 } 1393 1394 /* 1395 * We only need as many page frames for the image as there are saveable 1396 * pages in memory, but we have allocated more. Release the excessive 1397 * ones now. 1398 */ 1399 free_unnecessary_pages(); 1400 1401 out: 1402 do_gettimeofday(&stop); 1403 printk(KERN_CONT "done (allocated %lu pages)\n", pages); 1404 swsusp_show_speed(&start, &stop, pages, "Allocated"); 1405 1406 return 0; 1407 1408 err_out: 1409 printk(KERN_CONT "\n"); 1410 swsusp_free(); 1411 return -ENOMEM; 1412} 1413 1414#ifdef CONFIG_HIGHMEM 1415/** 1416 * count_pages_for_highmem - compute the number of non-highmem pages 1417 * that will be necessary for creating copies of highmem pages. 1418 */ 1419 1420static unsigned int count_pages_for_highmem(unsigned int nr_highmem) 1421{ 1422 unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem; 1423 1424 if (free_highmem >= nr_highmem) 1425 nr_highmem = 0; 1426 else 1427 nr_highmem -= free_highmem; 1428 1429 return nr_highmem; 1430} 1431#else 1432static unsigned int 1433count_pages_for_highmem(unsigned int nr_highmem) { return 0; } 1434#endif /* CONFIG_HIGHMEM */ 1435 1436/** 1437 * enough_free_mem - Make sure we have enough free memory for the 1438 * snapshot image. 1439 */ 1440 1441static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) 1442{ 1443 struct zone *zone; 1444 unsigned int free = alloc_normal; 1445 1446 for_each_populated_zone(zone) 1447 if (!is_highmem(zone)) 1448 free += zone_page_state(zone, NR_FREE_PAGES); 1449 1450 nr_pages += count_pages_for_highmem(nr_highmem); 1451 pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n", 1452 nr_pages, PAGES_FOR_IO, free); 1453 1454 return free > nr_pages + PAGES_FOR_IO; 1455} 1456 1457#ifdef CONFIG_HIGHMEM 1458/** 1459 * get_highmem_buffer - if there are some highmem pages in the suspend 1460 * image, we may need the buffer to copy them and/or load their data. 1461 */ 1462 1463static inline int get_highmem_buffer(int safe_needed) 1464{ 1465 buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed); 1466 return buffer ? 0 : -ENOMEM; 1467} 1468 1469/** 1470 * alloc_highmem_image_pages - allocate some highmem pages for the image. 1471 * Try to allocate as many pages as needed, but if the number of free 1472 * highmem pages is lesser than that, allocate them all. 1473 */ 1474 1475static inline unsigned int 1476alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem) 1477{ 1478 unsigned int to_alloc = count_free_highmem_pages(); 1479 1480 if (to_alloc > nr_highmem) 1481 to_alloc = nr_highmem; 1482 1483 nr_highmem -= to_alloc; 1484 while (to_alloc-- > 0) { 1485 struct page *page; 1486 1487 page = alloc_image_page(__GFP_HIGHMEM); 1488 memory_bm_set_bit(bm, page_to_pfn(page)); 1489 } 1490 return nr_highmem; 1491} 1492#else 1493static inline int get_highmem_buffer(int safe_needed) { return 0; } 1494 1495static inline unsigned int 1496alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; } 1497#endif /* CONFIG_HIGHMEM */ 1498 1499/** 1500 * swsusp_alloc - allocate memory for the suspend image 1501 * 1502 * We first try to allocate as many highmem pages as there are 1503 * saveable highmem pages in the system. If that fails, we allocate 1504 * non-highmem pages for the copies of the remaining highmem ones. 1505 * 1506 * In this approach it is likely that the copies of highmem pages will 1507 * also be located in the high memory, because of the way in which 1508 * copy_data_pages() works. 1509 */ 1510 1511static int 1512swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm, 1513 unsigned int nr_pages, unsigned int nr_highmem) 1514{ 1515 int error = 0; 1516 1517 if (nr_highmem > 0) { 1518 error = get_highmem_buffer(PG_ANY); 1519 if (error) 1520 goto err_out; 1521 if (nr_highmem > alloc_highmem) { 1522 nr_highmem -= alloc_highmem; 1523 nr_pages += alloc_highmem_pages(copy_bm, nr_highmem); 1524 } 1525 } 1526 if (nr_pages > alloc_normal) { 1527 nr_pages -= alloc_normal; 1528 while (nr_pages-- > 0) { 1529 struct page *page; 1530 1531 page = alloc_image_page(GFP_ATOMIC | __GFP_COLD); 1532 if (!page) 1533 goto err_out; 1534 memory_bm_set_bit(copy_bm, page_to_pfn(page)); 1535 } 1536 } 1537 1538 return 0; 1539 1540 err_out: 1541 swsusp_free(); 1542 return error; 1543} 1544 1545asmlinkage int swsusp_save(void) 1546{ 1547 unsigned int nr_pages, nr_highmem; 1548 1549 printk(KERN_INFO "PM: Creating hibernation image:\n"); 1550 1551 drain_local_pages(NULL); 1552 nr_pages = count_data_pages(); 1553 nr_highmem = count_highmem_pages(); 1554 printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem); 1555 1556 if (!enough_free_mem(nr_pages, nr_highmem)) { 1557 printk(KERN_ERR "PM: Not enough free memory\n"); 1558 return -ENOMEM; 1559 } 1560 1561 if (swsusp_alloc(&orig_bm, ©_bm, nr_pages, nr_highmem)) { 1562 printk(KERN_ERR "PM: Memory allocation failed\n"); 1563 return -ENOMEM; 1564 } 1565 1566 /* During allocating of suspend pagedir, new cold pages may appear. 1567 * Kill them. 1568 */ 1569 drain_local_pages(NULL); 1570 copy_data_pages(©_bm, &orig_bm); 1571 1572 /* 1573 * End of critical section. From now on, we can write to memory, 1574 * but we should not touch disk. This specially means we must _not_ 1575 * touch swap space! Except we must write out our image of course. 1576 */ 1577 1578 nr_pages += nr_highmem; 1579 nr_copy_pages = nr_pages; 1580 nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE); 1581 1582 printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n", 1583 nr_pages); 1584 1585 return 0; 1586} 1587 1588#ifndef CONFIG_ARCH_HIBERNATION_HEADER 1589static int init_header_complete(struct swsusp_info *info) 1590{ 1591 memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname)); 1592 info->version_code = LINUX_VERSION_CODE; 1593 return 0; 1594} 1595 1596static char *check_image_kernel(struct swsusp_info *info) 1597{ 1598 if (info->version_code != LINUX_VERSION_CODE) 1599 return "kernel version"; 1600 if (strcmp(info->uts.sysname,init_utsname()->sysname)) 1601 return "system type"; 1602 if (strcmp(info->uts.release,init_utsname()->release)) 1603 return "kernel release"; 1604 if (strcmp(info->uts.version,init_utsname()->version)) 1605 return "version"; 1606 if (strcmp(info->uts.machine,init_utsname()->machine)) 1607 return "machine"; 1608 return NULL; 1609} 1610#endif /* CONFIG_ARCH_HIBERNATION_HEADER */ 1611 1612unsigned long snapshot_get_image_size(void) 1613{ 1614 return nr_copy_pages + nr_meta_pages + 1; 1615} 1616 1617static int init_header(struct swsusp_info *info) 1618{ 1619 memset(info, 0, sizeof(struct swsusp_info)); 1620 info->num_physpages = num_physpages; 1621 info->image_pages = nr_copy_pages; 1622 info->pages = snapshot_get_image_size(); 1623 info->size = info->pages; 1624 info->size <<= PAGE_SHIFT; 1625 return init_header_complete(info); 1626} 1627 1628/** 1629 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm 1630 * are stored in the array @buf[] (1 page at a time) 1631 */ 1632 1633static inline void 1634pack_pfns(unsigned long *buf, struct memory_bitmap *bm) 1635{ 1636 int j; 1637 1638 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1639 buf[j] = memory_bm_next_pfn(bm); 1640 if (unlikely(buf[j] == BM_END_OF_MAP)) 1641 break; 1642 } 1643} 1644 1645/** 1646 * snapshot_read_next - used for reading the system memory snapshot. 1647 * 1648 * On the first call to it @handle should point to a zeroed 1649 * snapshot_handle structure. The structure gets updated and a pointer 1650 * to it should be passed to this function every next time. 1651 * 1652 * On success the function returns a positive number. Then, the caller 1653 * is allowed to read up to the returned number of bytes from the memory 1654 * location computed by the data_of() macro. 1655 * 1656 * The function returns 0 to indicate the end of data stream condition, 1657 * and a negative number is returned on error. In such cases the 1658 * structure pointed to by @handle is not updated and should not be used 1659 * any more. 1660 */ 1661 1662int snapshot_read_next(struct snapshot_handle *handle) 1663{ 1664 if (handle->cur > nr_meta_pages + nr_copy_pages) 1665 return 0; 1666 1667 if (!buffer) { 1668 /* This makes the buffer be freed by swsusp_free() */ 1669 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 1670 if (!buffer) 1671 return -ENOMEM; 1672 } 1673 if (!handle->cur) { 1674 int error; 1675 1676 error = init_header((struct swsusp_info *)buffer); 1677 if (error) 1678 return error; 1679 handle->buffer = buffer; 1680 memory_bm_position_reset(&orig_bm); 1681 memory_bm_position_reset(©_bm); 1682 } else if (handle->cur <= nr_meta_pages) { 1683 memset(buffer, 0, PAGE_SIZE); 1684 pack_pfns(buffer, &orig_bm); 1685 } else { 1686 struct page *page; 1687 1688 page = pfn_to_page(memory_bm_next_pfn(©_bm)); 1689 if (PageHighMem(page)) { 1690 /* Highmem pages are copied to the buffer, 1691 * because we can't return with a kmapped 1692 * highmem page (we may not be called again). 1693 */ 1694 void *kaddr; 1695 1696 kaddr = kmap_atomic(page, KM_USER0); 1697 memcpy(buffer, kaddr, PAGE_SIZE); 1698 kunmap_atomic(kaddr, KM_USER0); 1699 handle->buffer = buffer; 1700 } else { 1701 handle->buffer = page_address(page); 1702 } 1703 } 1704 handle->cur++; 1705 return PAGE_SIZE; 1706} 1707 1708/** 1709 * mark_unsafe_pages - mark the pages that cannot be used for storing 1710 * the image during resume, because they conflict with the pages that 1711 * had been used before suspend 1712 */ 1713 1714static int mark_unsafe_pages(struct memory_bitmap *bm) 1715{ 1716 struct zone *zone; 1717 unsigned long pfn, max_zone_pfn; 1718 1719 /* Clear page flags */ 1720 for_each_populated_zone(zone) { 1721 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; 1722 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) 1723 if (pfn_valid(pfn)) 1724 swsusp_unset_page_free(pfn_to_page(pfn)); 1725 } 1726 1727 /* Mark pages that correspond to the "original" pfns as "unsafe" */ 1728 memory_bm_position_reset(bm); 1729 do { 1730 pfn = memory_bm_next_pfn(bm); 1731 if (likely(pfn != BM_END_OF_MAP)) { 1732 if (likely(pfn_valid(pfn))) 1733 swsusp_set_page_free(pfn_to_page(pfn)); 1734 else 1735 return -EFAULT; 1736 } 1737 } while (pfn != BM_END_OF_MAP); 1738 1739 allocated_unsafe_pages = 0; 1740 1741 return 0; 1742} 1743 1744static void 1745duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src) 1746{ 1747 unsigned long pfn; 1748 1749 memory_bm_position_reset(src); 1750 pfn = memory_bm_next_pfn(src); 1751 while (pfn != BM_END_OF_MAP) { 1752 memory_bm_set_bit(dst, pfn); 1753 pfn = memory_bm_next_pfn(src); 1754 } 1755} 1756 1757static int check_header(struct swsusp_info *info) 1758{ 1759 char *reason; 1760 1761 reason = check_image_kernel(info); 1762 if (!reason && info->num_physpages != num_physpages) 1763 reason = "memory size"; 1764 if (reason) { 1765 printk(KERN_ERR "PM: Image mismatch: %s\n", reason); 1766 return -EPERM; 1767 } 1768 return 0; 1769} 1770 1771/** 1772 * load header - check the image header and copy data from it 1773 */ 1774 1775static int 1776load_header(struct swsusp_info *info) 1777{ 1778 int error; 1779 1780 restore_pblist = NULL; 1781 error = check_header(info); 1782 if (!error) { 1783 nr_copy_pages = info->image_pages; 1784 nr_meta_pages = info->pages - info->image_pages - 1; 1785 } 1786 return error; 1787} 1788 1789/** 1790 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set 1791 * the corresponding bit in the memory bitmap @bm 1792 */ 1793static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm) 1794{ 1795 int j; 1796 1797 for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { 1798 if (unlikely(buf[j] == BM_END_OF_MAP)) 1799 break; 1800 1801 if (memory_bm_pfn_present(bm, buf[j])) 1802 memory_bm_set_bit(bm, buf[j]); 1803 else 1804 return -EFAULT; 1805 } 1806 1807 return 0; 1808} 1809 1810/* List of "safe" pages that may be used to store data loaded from the suspend 1811 * image 1812 */ 1813static struct linked_page *safe_pages_list; 1814 1815#ifdef CONFIG_HIGHMEM 1816/* struct highmem_pbe is used for creating the list of highmem pages that 1817 * should be restored atomically during the resume from disk, because the page 1818 * frames they have occupied before the suspend are in use. 1819 */ 1820struct highmem_pbe { 1821 struct page *copy_page; /* data is here now */ 1822 struct page *orig_page; /* data was here before the suspend */ 1823 struct highmem_pbe *next; 1824}; 1825 1826/* List of highmem PBEs needed for restoring the highmem pages that were 1827 * allocated before the suspend and included in the suspend image, but have 1828 * also been allocated by the "resume" kernel, so their contents cannot be 1829 * written directly to their "original" page frames. 1830 */ 1831static struct highmem_pbe *highmem_pblist; 1832 1833/** 1834 * count_highmem_image_pages - compute the number of highmem pages in the 1835 * suspend image. The bits in the memory bitmap @bm that correspond to the 1836 * image pages are assumed to be set. 1837 */ 1838 1839static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) 1840{ 1841 unsigned long pfn; 1842 unsigned int cnt = 0; 1843 1844 memory_bm_position_reset(bm); 1845 pfn = memory_bm_next_pfn(bm); 1846 while (pfn != BM_END_OF_MAP) { 1847 if (PageHighMem(pfn_to_page(pfn))) 1848 cnt++; 1849 1850 pfn = memory_bm_next_pfn(bm); 1851 } 1852 return cnt; 1853} 1854 1855/** 1856 * prepare_highmem_image - try to allocate as many highmem pages as 1857 * there are highmem image pages (@nr_highmem_p points to the variable 1858 * containing the number of highmem image pages). The pages that are 1859 * "safe" (ie. will not be overwritten when the suspend image is 1860 * restored) have the corresponding bits set in @bm (it must be 1861 * unitialized). 1862 * 1863 * NOTE: This function should not be called if there are no highmem 1864 * image pages. 1865 */ 1866 1867static unsigned int safe_highmem_pages; 1868 1869static struct memory_bitmap *safe_highmem_bm; 1870 1871static int 1872prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 1873{ 1874 unsigned int to_alloc; 1875 1876 if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE)) 1877 return -ENOMEM; 1878 1879 if (get_highmem_buffer(PG_SAFE)) 1880 return -ENOMEM; 1881 1882 to_alloc = count_free_highmem_pages(); 1883 if (to_alloc > *nr_highmem_p) 1884 to_alloc = *nr_highmem_p; 1885 else 1886 *nr_highmem_p = to_alloc; 1887 1888 safe_highmem_pages = 0; 1889 while (to_alloc-- > 0) { 1890 struct page *page; 1891 1892 page = alloc_page(__GFP_HIGHMEM); 1893 if (!swsusp_page_is_free(page)) { 1894 /* The page is "safe", set its bit the bitmap */ 1895 memory_bm_set_bit(bm, page_to_pfn(page)); 1896 safe_highmem_pages++; 1897 } 1898 /* Mark the page as allocated */ 1899 swsusp_set_page_forbidden(page); 1900 swsusp_set_page_free(page); 1901 } 1902 memory_bm_position_reset(bm); 1903 safe_highmem_bm = bm; 1904 return 0; 1905} 1906 1907/** 1908 * get_highmem_page_buffer - for given highmem image page find the buffer 1909 * that suspend_write_next() should set for its caller to write to. 1910 * 1911 * If the page is to be saved to its "original" page frame or a copy of 1912 * the page is to be made in the highmem, @buffer is returned. Otherwise, 1913 * the copy of the page is to be made in normal memory, so the address of 1914 * the copy is returned. 1915 * 1916 * If @buffer is returned, the caller of suspend_write_next() will write 1917 * the page's contents to @buffer, so they will have to be copied to the 1918 * right location on the next call to suspend_write_next() and it is done 1919 * with the help of copy_last_highmem_page(). For this purpose, if 1920 * @buffer is returned, @last_highmem page is set to the page to which 1921 * the data will have to be copied from @buffer. 1922 */ 1923 1924static struct page *last_highmem_page; 1925 1926static void * 1927get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 1928{ 1929 struct highmem_pbe *pbe; 1930 void *kaddr; 1931 1932 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) { 1933 /* We have allocated the "original" page frame and we can 1934 * use it directly to store the loaded page. 1935 */ 1936 last_highmem_page = page; 1937 return buffer; 1938 } 1939 /* The "original" page frame has not been allocated and we have to 1940 * use a "safe" page frame to store the loaded page. 1941 */ 1942 pbe = chain_alloc(ca, sizeof(struct highmem_pbe)); 1943 if (!pbe) { 1944 swsusp_free(); 1945 return ERR_PTR(-ENOMEM); 1946 } 1947 pbe->orig_page = page; 1948 if (safe_highmem_pages > 0) { 1949 struct page *tmp; 1950 1951 /* Copy of the page will be stored in high memory */ 1952 kaddr = buffer; 1953 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm)); 1954 safe_highmem_pages--; 1955 last_highmem_page = tmp; 1956 pbe->copy_page = tmp; 1957 } else { 1958 /* Copy of the page will be stored in normal memory */ 1959 kaddr = safe_pages_list; 1960 safe_pages_list = safe_pages_list->next; 1961 pbe->copy_page = virt_to_page(kaddr); 1962 } 1963 pbe->next = highmem_pblist; 1964 highmem_pblist = pbe; 1965 return kaddr; 1966} 1967 1968/** 1969 * copy_last_highmem_page - copy the contents of a highmem image from 1970 * @buffer, where the caller of snapshot_write_next() has place them, 1971 * to the right location represented by @last_highmem_page . 1972 */ 1973 1974static void copy_last_highmem_page(void) 1975{ 1976 if (last_highmem_page) { 1977 void *dst; 1978 1979 dst = kmap_atomic(last_highmem_page, KM_USER0); 1980 memcpy(dst, buffer, PAGE_SIZE); 1981 kunmap_atomic(dst, KM_USER0); 1982 last_highmem_page = NULL; 1983 } 1984} 1985 1986static inline int last_highmem_page_copied(void) 1987{ 1988 return !last_highmem_page; 1989} 1990 1991static inline void free_highmem_data(void) 1992{ 1993 if (safe_highmem_bm) 1994 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR); 1995 1996 if (buffer) 1997 free_image_page(buffer, PG_UNSAFE_CLEAR); 1998} 1999#else 2000static inline int get_safe_write_buffer(void) { return 0; } 2001 2002static unsigned int 2003count_highmem_image_pages(struct memory_bitmap *bm) { return 0; } 2004 2005static inline int 2006prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p) 2007{ 2008 return 0; 2009} 2010 2011static inline void * 2012get_highmem_page_buffer(struct page *page, struct chain_allocator *ca) 2013{ 2014 return ERR_PTR(-EINVAL); 2015} 2016 2017static inline void copy_last_highmem_page(void) {} 2018static inline int last_highmem_page_copied(void) { return 1; } 2019static inline void free_highmem_data(void) {} 2020#endif /* CONFIG_HIGHMEM */ 2021 2022/** 2023 * prepare_image - use the memory bitmap @bm to mark the pages that will 2024 * be overwritten in the process of restoring the system memory state 2025 * from the suspend image ("unsafe" pages) and allocate memory for the 2026 * image. 2027 * 2028 * The idea is to allocate a new memory bitmap first and then allocate 2029 * as many pages as needed for the image data, but not to assign these 2030 * pages to specific tasks initially. Instead, we just mark them as 2031 * allocated and create a lists of "safe" pages that will be used 2032 * later. On systems with high memory a list of "safe" highmem pages is 2033 * also created. 2034 */ 2035 2036#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe)) 2037 2038static int 2039prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm) 2040{ 2041 unsigned int nr_pages, nr_highmem; 2042 struct linked_page *sp_list, *lp; 2043 int error; 2044 2045 /* If there is no highmem, the buffer will not be necessary */ 2046 free_image_page(buffer, PG_UNSAFE_CLEAR); 2047 buffer = NULL; 2048 2049 nr_highmem = count_highmem_image_pages(bm); 2050 error = mark_unsafe_pages(bm); 2051 if (error) 2052 goto Free; 2053 2054 error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE); 2055 if (error) 2056 goto Free; 2057 2058 duplicate_memory_bitmap(new_bm, bm); 2059 memory_bm_free(bm, PG_UNSAFE_KEEP); 2060 if (nr_highmem > 0) { 2061 error = prepare_highmem_image(bm, &nr_highmem); 2062 if (error) 2063 goto Free; 2064 } 2065 /* Reserve some safe pages for potential later use. 2066 * 2067 * NOTE: This way we make sure there will be enough safe pages for the 2068 * chain_alloc() in get_buffer(). It is a bit wasteful, but 2069 * nr_copy_pages cannot be greater than 50% of the memory anyway. 2070 */ 2071 sp_list = NULL; 2072 /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */ 2073 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2074 nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE); 2075 while (nr_pages > 0) { 2076 lp = get_image_page(GFP_ATOMIC, PG_SAFE); 2077 if (!lp) { 2078 error = -ENOMEM; 2079 goto Free; 2080 } 2081 lp->next = sp_list; 2082 sp_list = lp; 2083 nr_pages--; 2084 } 2085 /* Preallocate memory for the image */ 2086 safe_pages_list = NULL; 2087 nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages; 2088 while (nr_pages > 0) { 2089 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC); 2090 if (!lp) { 2091 error = -ENOMEM; 2092 goto Free; 2093 } 2094 if (!swsusp_page_is_free(virt_to_page(lp))) { 2095 /* The page is "safe", add it to the list */ 2096 lp->next = safe_pages_list; 2097 safe_pages_list = lp; 2098 } 2099 /* Mark the page as allocated */ 2100 swsusp_set_page_forbidden(virt_to_page(lp)); 2101 swsusp_set_page_free(virt_to_page(lp)); 2102 nr_pages--; 2103 } 2104 /* Free the reserved safe pages so that chain_alloc() can use them */ 2105 while (sp_list) { 2106 lp = sp_list->next; 2107 free_image_page(sp_list, PG_UNSAFE_CLEAR); 2108 sp_list = lp; 2109 } 2110 return 0; 2111 2112 Free: 2113 swsusp_free(); 2114 return error; 2115} 2116 2117/** 2118 * get_buffer - compute the address that snapshot_write_next() should 2119 * set for its caller to write to. 2120 */ 2121 2122static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) 2123{ 2124 struct pbe *pbe; 2125 struct page *page; 2126 unsigned long pfn = memory_bm_next_pfn(bm); 2127 2128 if (pfn == BM_END_OF_MAP) 2129 return ERR_PTR(-EFAULT); 2130 2131 page = pfn_to_page(pfn); 2132 if (PageHighMem(page)) 2133 return get_highmem_page_buffer(page, ca); 2134 2135 if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) 2136 /* We have allocated the "original" page frame and we can 2137 * use it directly to store the loaded page. 2138 */ 2139 return page_address(page); 2140 2141 /* The "original" page frame has not been allocated and we have to 2142 * use a "safe" page frame to store the loaded page. 2143 */ 2144 pbe = chain_alloc(ca, sizeof(struct pbe)); 2145 if (!pbe) { 2146 swsusp_free(); 2147 return ERR_PTR(-ENOMEM); 2148 } 2149 pbe->orig_address = page_address(page); 2150 pbe->address = safe_pages_list; 2151 safe_pages_list = safe_pages_list->next; 2152 pbe->next = restore_pblist; 2153 restore_pblist = pbe; 2154 return pbe->address; 2155} 2156 2157/** 2158 * snapshot_write_next - used for writing the system memory snapshot. 2159 * 2160 * On the first call to it @handle should point to a zeroed 2161 * snapshot_handle structure. The structure gets updated and a pointer 2162 * to it should be passed to this function every next time. 2163 * 2164 * On success the function returns a positive number. Then, the caller 2165 * is allowed to write up to the returned number of bytes to the memory 2166 * location computed by the data_of() macro. 2167 * 2168 * The function returns 0 to indicate the "end of file" condition, 2169 * and a negative number is returned on error. In such cases the 2170 * structure pointed to by @handle is not updated and should not be used 2171 * any more. 2172 */ 2173 2174int snapshot_write_next(struct snapshot_handle *handle) 2175{ 2176 static struct chain_allocator ca; 2177 int error = 0; 2178 2179 /* Check if we have already loaded the entire image */ 2180 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) 2181 return 0; 2182 2183 handle->sync_read = 1; 2184 2185 if (!handle->cur) { 2186 if (!buffer) 2187 /* This makes the buffer be freed by swsusp_free() */ 2188 buffer = get_image_page(GFP_ATOMIC, PG_ANY); 2189 2190 if (!buffer) 2191 return -ENOMEM; 2192 2193 handle->buffer = buffer; 2194 } else if (handle->cur == 1) { 2195 error = load_header(buffer); 2196 if (error) 2197 return error; 2198 2199 error = memory_bm_create(©_bm, GFP_ATOMIC, PG_ANY); 2200 if (error) 2201 return error; 2202 2203 } else if (handle->cur <= nr_meta_pages + 1) { 2204 error = unpack_orig_pfns(buffer, ©_bm); 2205 if (error) 2206 return error; 2207 2208 if (handle->cur == nr_meta_pages + 1) { 2209 error = prepare_image(&orig_bm, ©_bm); 2210 if (error) 2211 return error; 2212 2213 chain_init(&ca, GFP_ATOMIC, PG_SAFE); 2214 memory_bm_position_reset(&orig_bm); 2215 restore_pblist = NULL; 2216 handle->buffer = get_buffer(&orig_bm, &ca); 2217 handle->sync_read = 0; 2218 if (IS_ERR(handle->buffer)) 2219 return PTR_ERR(handle->buffer); 2220 } 2221 } else { 2222 copy_last_highmem_page(); 2223 handle->buffer = get_buffer(&orig_bm, &ca); 2224 if (IS_ERR(handle->buffer)) 2225 return PTR_ERR(handle->buffer); 2226 if (handle->buffer != buffer) 2227 handle->sync_read = 0; 2228 } 2229 handle->cur++; 2230 return PAGE_SIZE; 2231} 2232 2233/** 2234 * snapshot_write_finalize - must be called after the last call to 2235 * snapshot_write_next() in case the last page in the image happens 2236 * to be a highmem page and its contents should be stored in the 2237 * highmem. Additionally, it releases the memory that will not be 2238 * used any more. 2239 */ 2240 2241void snapshot_write_finalize(struct snapshot_handle *handle) 2242{ 2243 copy_last_highmem_page(); 2244 /* Free only if we have loaded the image entirely */ 2245 if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) { 2246 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR); 2247 free_highmem_data(); 2248 } 2249} 2250 2251int snapshot_image_loaded(struct snapshot_handle *handle) 2252{ 2253 return !(!nr_copy_pages || !last_highmem_page_copied() || 2254 handle->cur <= nr_meta_pages + nr_copy_pages); 2255} 2256 2257#ifdef CONFIG_HIGHMEM 2258/* Assumes that @buf is ready and points to a "safe" page */ 2259static inline void 2260swap_two_pages_data(struct page *p1, struct page *p2, void *buf) 2261{ 2262 void *kaddr1, *kaddr2; 2263 2264 kaddr1 = kmap_atomic(p1, KM_USER0); 2265 kaddr2 = kmap_atomic(p2, KM_USER1); 2266 memcpy(buf, kaddr1, PAGE_SIZE); 2267 memcpy(kaddr1, kaddr2, PAGE_SIZE); 2268 memcpy(kaddr2, buf, PAGE_SIZE); 2269 kunmap_atomic(kaddr1, KM_USER0); 2270 kunmap_atomic(kaddr2, KM_USER1); 2271} 2272 2273/** 2274 * restore_highmem - for each highmem page that was allocated before 2275 * the suspend and included in the suspend image, and also has been 2276 * allocated by the "resume" kernel swap its current (ie. "before 2277 * resume") contents with the previous (ie. "before suspend") one. 2278 * 2279 * If the resume eventually fails, we can call this function once 2280 * again and restore the "before resume" highmem state. 2281 */ 2282 2283int restore_highmem(void) 2284{ 2285 struct highmem_pbe *pbe = highmem_pblist; 2286 void *buf; 2287 2288 if (!pbe) 2289 return 0; 2290 2291 buf = get_image_page(GFP_ATOMIC, PG_SAFE); 2292 if (!buf) 2293 return -ENOMEM; 2294 2295 while (pbe) { 2296 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf); 2297 pbe = pbe->next; 2298 } 2299 free_image_page(buf, PG_UNSAFE_CLEAR); 2300 return 0; 2301} 2302#endif /* CONFIG_HIGHMEM */ 2303