linux-master/mm/page_isolation.c

7 #include <linux/page-isolation.h>
28  * Returns a page without holding a reference. If the caller wants to
29  * dereference that page (e.g., dumping), it has to make sure that it
33 static struct page *has_unmovable_pages(unsigned long start_pfn, unsigned long end_pfn,
36 	struct page *page = pfn_to_page(start_pfn);
37 	struct zone *zone = page_zone(page);
43 	if (is_migrate_cma_page(page)) {
52 		return page;
56 		page = pfn_to_page(pfn);
64 		if (PageReserved(page))
65 			return page;
79 		 * handle each tail page individually in migration.
81 		if (PageHuge(page) || PageTransCompound(page)) {
82 			struct folio *folio = page_folio(page);
85 			if (PageHuge(page)) {
87 					return page;
89 				return page;
92 			skip_pages = folio_nr_pages(folio) - folio_page_idx(folio, page);
98 		 * We can't use page_count without pin a page
99 		 * because another CPU can free compound page.
101 		 * because their page->_refcount is zero at all time.
103 		if (!page_ref_count(page)) {
104 			if (PageBuddy(page))
105 				pfn += (1 << buddy_order(page)) - 1;
110 		 * The HWPoisoned page may be not in buddy system, and
113 		if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
126 		if ((flags & MEMORY_OFFLINE) && PageOffline(page))
129 		if (__PageMovable(page) || PageLRU(page))
137 		return page;
143  * This function set pageblock migratetype to isolate if no unmovable page is
147 static int set_migratetype_isolate(struct page *page, int migratetype, int isol_flags,
150 	struct zone *zone = page_zone(page);
151 	struct page *unmovable;
162 	if (is_migrate_isolate_page(page)) {
171 	 * Pass the intersection of [start_pfn, end_pfn) and the page's pageblock
174 	check_unmovable_start = max(page_to_pfn(page), start_pfn);
175 	check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),
182 		int mt = get_pageblock_migratetype(page);
184 		set_pageblock_migratetype(page, MIGRATE_ISOLATE);
186 		nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE,
200 		dump_page(unmovable, "unmovable page");
206 static void unset_migratetype_isolate(struct page *page, int migratetype)
212 	struct page *buddy;
214 	zone = page_zone(page);
216 	if (!is_migrate_isolate_page(page))
222 	 * it is possible that there is free buddy page.
227 	if (PageBuddy(page)) {
228 		order = buddy_order(page);
230 			buddy = find_buddy_page_pfn(page, page_to_pfn(page),
233 				isolated_page = !!__isolate_free_page(page, order);
235 				 * Isolating a free page in an isolated pageblock
255 		nr_pages = move_freepages_block(zone, page, migratetype, NULL);
258 	set_pageblock_migratetype(page, migratetype);
260 		__putback_isolated_page(page, order, migratetype);
266 static inline struct page *
272 		struct page *page;
274 		page = pfn_to_online_page(pfn + i);
275 		if (!page)
277 		return page;
284  * within a free or in-use page.
285  * @boundary_pfn:		pageblock-aligned pfn that a page might cross
294  * pageblock. When not all pageblocks within a page are isolated at the same
295  * time, free page accounting can go wrong. For example, in the case of
296  * MAX_PAGE_ORDER = pageblock_order + 1, a MAX_PAGE_ORDER page has two
300  * When either pageblock is isolated, if it is a free page, the page is not
302  * in-use page and freed later, __free_one_page() does not split the free page
303  * either. The function handles this by splitting the free page or migrating
304  * the in-use page then splitting the free page.
326 	 * free or in-use page. Also make sure all to-be-isolated pageblocks
347 	 * a free or in-use page across boundary_pfn:
349 	 * 1. isolate before boundary_pfn: the page after is not online
350 	 * 2. isolate after boundary_pfn: the page before is not online
366 		struct page *page = __first_valid_page(pfn, boundary_pfn - pfn);
368 		VM_BUG_ON(!page);
369 		pfn = page_to_pfn(page);
372 		 * free pages in [start_pfn, boundary_pfn), its head page will
375 		if (PageBuddy(page)) {
376 			int order = buddy_order(page);
379 				/* free page changed before split, check it again */
380 				if (split_free_page(page, order, boundary_pfn - pfn))
388 		 * migrate compound pages then let the free page handling code
391 		if (PageCompound(page)) {
392 			struct page *head = compound_head(page);
405 			if (PageHuge(page) || PageLRU(page) || __PageMovable(page)) {
408 				int page_mt = get_pageblock_migratetype(page);
409 				bool isolate_page = !is_migrate_isolate_page(page);
423 				 * XXX: mark the page as MIGRATE_ISOLATE so that
424 				 * no one else can grab the freed page after migration.
425 				 * Ideally, the page should be freed as two separate
430 					ret = set_migratetype_isolate(page, page_mt,
440 				 * restore the page's migratetype so that it can
445 					unset_migratetype_isolate(page, page_mt);
450 				 * reset pfn to the head of the free page, so
451 				 * that the free page handling code above can split
452 				 * the free page to the right migratetype list.
454 				 * head_pfn is not used here as a hugetlb page order
456 				 * freed, the free page order is not. Use pfn within
457 				 * the range to find the head of the free page.
462 					/* stop if we cannot find the free page */
485  * start_isolate_page_range() - mark page range MIGRATE_ISOLATE
499  * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
518  * Please note that there is no strong synchronization with the page allocator
519  * either. Pages might be freed while their page blocks are marked ISOLATED.
533 	struct page *page;
534 	/* isolation is done at page block granularity */
561 		page = __first_valid_page(pfn, pageblock_nr_pages);
562 		if (page && set_migratetype_isolate(page, migratetype, flags,
580  * This finds every MIGRATE_ISOLATE page block in the given range
587 	struct page *page;
594 		page = __first_valid_page(pfn, pageblock_nr_pages);
595 		if (!page || !is_migrate_isolate_page(page))
597 		unset_migratetype_isolate(page, migratetype);
611 	struct page *page;
614 		page = pfn_to_page(pfn);
615 		if (PageBuddy(page))
617 			 * If the page is on a free list, it has to be on
621 			pfn += 1 << buddy_order(page);
622 		else if ((flags & MEMORY_OFFLINE) && PageHWPoison(page))
623 			/* A HWPoisoned page cannot be also PageBuddy */
625 		else if ((flags & MEMORY_OFFLINE) && PageOffline(page) &&
626 			 !page_count(page))
659 	struct page *page;
669 		page = __first_valid_page(pfn, pageblock_nr_pages);
670 		if (page && !is_migrate_isolate_page(page))
673 	page = __first_valid_page(start_pfn, end_pfn - start_pfn);
674 	if ((pfn < end_pfn) || !page) {
680 	zone = page_zone(page);