1// SPDX-License-Identifier: GPL-2.0 2/* 3 * sparse memory mappings. 4 */ 5#include <linux/mm.h> 6#include <linux/slab.h> 7#include <linux/mmzone.h> 8#include <linux/memblock.h> 9#include <linux/compiler.h> 10#include <linux/highmem.h> 11#include <linux/export.h> 12#include <linux/spinlock.h> 13#include <linux/vmalloc.h> 14#include <linux/swap.h> 15#include <linux/swapops.h> 16#include <linux/bootmem_info.h> 17 18#include "internal.h" 19#include <asm/dma.h> 20 21/* 22 * Permanent SPARSEMEM data: 23 * 24 * 1) mem_section - memory sections, mem_map's for valid memory 25 */ 26#ifdef CONFIG_SPARSEMEM_EXTREME 27struct mem_section **mem_section; 28#else 29struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 30 ____cacheline_internodealigned_in_smp; 31#endif 32EXPORT_SYMBOL(mem_section); 33 34#ifdef NODE_NOT_IN_PAGE_FLAGS 35/* 36 * If we did not store the node number in the page then we have to 37 * do a lookup in the section_to_node_table in order to find which 38 * node the page belongs to. 39 */ 40#if MAX_NUMNODES <= 256 41static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 42#else 43static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 44#endif 45 46int page_to_nid(const struct page *page) 47{ 48 return section_to_node_table[page_to_section(page)]; 49} 50EXPORT_SYMBOL(page_to_nid); 51 52static void set_section_nid(unsigned long section_nr, int nid) 53{ 54 section_to_node_table[section_nr] = nid; 55} 56#else /* !NODE_NOT_IN_PAGE_FLAGS */ 57static inline void set_section_nid(unsigned long section_nr, int nid) 58{ 59} 60#endif 61 62#ifdef CONFIG_SPARSEMEM_EXTREME 63static noinline struct mem_section __ref *sparse_index_alloc(int nid) 64{ 65 struct mem_section *section = NULL; 66 unsigned long array_size = SECTIONS_PER_ROOT * 67 sizeof(struct mem_section); 68 69 if (slab_is_available()) { 70 section = kzalloc_node(array_size, GFP_KERNEL, nid); 71 } else { 72 section = memblock_alloc_node(array_size, SMP_CACHE_BYTES, 73 nid); 74 if (!section) 75 panic("%s: Failed to allocate %lu bytes nid=%d\n", 76 __func__, array_size, nid); 77 } 78 79 return section; 80} 81 82static int __meminit sparse_index_init(unsigned long section_nr, int nid) 83{ 84 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 85 struct mem_section *section; 86 87 /* 88 * An existing section is possible in the sub-section hotplug 89 * case. First hot-add instantiates, follow-on hot-add reuses 90 * the existing section. 91 * 92 * The mem_hotplug_lock resolves the apparent race below. 93 */ 94 if (mem_section[root]) 95 return 0; 96 97 section = sparse_index_alloc(nid); 98 if (!section) 99 return -ENOMEM; 100 101 mem_section[root] = section; 102 103 return 0; 104} 105#else /* !SPARSEMEM_EXTREME */ 106static inline int sparse_index_init(unsigned long section_nr, int nid) 107{ 108 return 0; 109} 110#endif 111 112/* 113 * During early boot, before section_mem_map is used for an actual 114 * mem_map, we use section_mem_map to store the section's NUMA 115 * node. This keeps us from having to use another data structure. The 116 * node information is cleared just before we store the real mem_map. 117 */ 118static inline unsigned long sparse_encode_early_nid(int nid) 119{ 120 return ((unsigned long)nid << SECTION_NID_SHIFT); 121} 122 123static inline int sparse_early_nid(struct mem_section *section) 124{ 125 return (section->section_mem_map >> SECTION_NID_SHIFT); 126} 127 128/* Validate the physical addressing limitations of the model */ 129static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 130 unsigned long *end_pfn) 131{ 132 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 133 134 /* 135 * Sanity checks - do not allow an architecture to pass 136 * in larger pfns than the maximum scope of sparsemem: 137 */ 138 if (*start_pfn > max_sparsemem_pfn) { 139 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 140 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 141 *start_pfn, *end_pfn, max_sparsemem_pfn); 142 WARN_ON_ONCE(1); 143 *start_pfn = max_sparsemem_pfn; 144 *end_pfn = max_sparsemem_pfn; 145 } else if (*end_pfn > max_sparsemem_pfn) { 146 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 147 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 148 *start_pfn, *end_pfn, max_sparsemem_pfn); 149 WARN_ON_ONCE(1); 150 *end_pfn = max_sparsemem_pfn; 151 } 152} 153 154/* 155 * There are a number of times that we loop over NR_MEM_SECTIONS, 156 * looking for section_present() on each. But, when we have very 157 * large physical address spaces, NR_MEM_SECTIONS can also be 158 * very large which makes the loops quite long. 159 * 160 * Keeping track of this gives us an easy way to break out of 161 * those loops early. 162 */ 163unsigned long __highest_present_section_nr; 164static void __section_mark_present(struct mem_section *ms, 165 unsigned long section_nr) 166{ 167 if (section_nr > __highest_present_section_nr) 168 __highest_present_section_nr = section_nr; 169 170 ms->section_mem_map |= SECTION_MARKED_PRESENT; 171} 172 173#define for_each_present_section_nr(start, section_nr) \ 174 for (section_nr = next_present_section_nr(start-1); \ 175 section_nr != -1; \ 176 section_nr = next_present_section_nr(section_nr)) 177 178static inline unsigned long first_present_section_nr(void) 179{ 180 return next_present_section_nr(-1); 181} 182 183#ifdef CONFIG_SPARSEMEM_VMEMMAP 184static void subsection_mask_set(unsigned long *map, unsigned long pfn, 185 unsigned long nr_pages) 186{ 187 int idx = subsection_map_index(pfn); 188 int end = subsection_map_index(pfn + nr_pages - 1); 189 190 bitmap_set(map, idx, end - idx + 1); 191} 192 193void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 194{ 195 int end_sec = pfn_to_section_nr(pfn + nr_pages - 1); 196 unsigned long nr, start_sec = pfn_to_section_nr(pfn); 197 198 if (!nr_pages) 199 return; 200 201 for (nr = start_sec; nr <= end_sec; nr++) { 202 struct mem_section *ms; 203 unsigned long pfns; 204 205 pfns = min(nr_pages, PAGES_PER_SECTION 206 - (pfn & ~PAGE_SECTION_MASK)); 207 ms = __nr_to_section(nr); 208 subsection_mask_set(ms->usage->subsection_map, pfn, pfns); 209 210 pr_debug("%s: sec: %lu pfns: %lu set(%d, %d)\n", __func__, nr, 211 pfns, subsection_map_index(pfn), 212 subsection_map_index(pfn + pfns - 1)); 213 214 pfn += pfns; 215 nr_pages -= pfns; 216 } 217} 218#else 219void __init subsection_map_init(unsigned long pfn, unsigned long nr_pages) 220{ 221} 222#endif 223 224/* Record a memory area against a node. */ 225static void __init memory_present(int nid, unsigned long start, unsigned long end) 226{ 227 unsigned long pfn; 228 229#ifdef CONFIG_SPARSEMEM_EXTREME 230 if (unlikely(!mem_section)) { 231 unsigned long size, align; 232 233 size = sizeof(struct mem_section *) * NR_SECTION_ROOTS; 234 align = 1 << (INTERNODE_CACHE_SHIFT); 235 mem_section = memblock_alloc(size, align); 236 if (!mem_section) 237 panic("%s: Failed to allocate %lu bytes align=0x%lx\n", 238 __func__, size, align); 239 } 240#endif 241 242 start &= PAGE_SECTION_MASK; 243 mminit_validate_memmodel_limits(&start, &end); 244 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 245 unsigned long section = pfn_to_section_nr(pfn); 246 struct mem_section *ms; 247 248 sparse_index_init(section, nid); 249 set_section_nid(section, nid); 250 251 ms = __nr_to_section(section); 252 if (!ms->section_mem_map) { 253 ms->section_mem_map = sparse_encode_early_nid(nid) | 254 SECTION_IS_ONLINE; 255 __section_mark_present(ms, section); 256 } 257 } 258} 259 260/* 261 * Mark all memblocks as present using memory_present(). 262 * This is a convenience function that is useful to mark all of the systems 263 * memory as present during initialization. 264 */ 265static void __init memblocks_present(void) 266{ 267 unsigned long start, end; 268 int i, nid; 269 270 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) 271 memory_present(nid, start, end); 272} 273 274/* 275 * Subtle, we encode the real pfn into the mem_map such that 276 * the identity pfn - section_mem_map will return the actual 277 * physical page frame number. 278 */ 279static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 280{ 281 unsigned long coded_mem_map = 282 (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 283 BUILD_BUG_ON(SECTION_MAP_LAST_BIT > PFN_SECTION_SHIFT); 284 BUG_ON(coded_mem_map & ~SECTION_MAP_MASK); 285 return coded_mem_map; 286} 287 288#ifdef CONFIG_MEMORY_HOTPLUG 289/* 290 * Decode mem_map from the coded memmap 291 */ 292struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 293{ 294 /* mask off the extra low bits of information */ 295 coded_mem_map &= SECTION_MAP_MASK; 296 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 297} 298#endif /* CONFIG_MEMORY_HOTPLUG */ 299 300static void __meminit sparse_init_one_section(struct mem_section *ms, 301 unsigned long pnum, struct page *mem_map, 302 struct mem_section_usage *usage, unsigned long flags) 303{ 304 ms->section_mem_map &= ~SECTION_MAP_MASK; 305 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) 306 | SECTION_HAS_MEM_MAP | flags; 307 ms->usage = usage; 308} 309 310static unsigned long usemap_size(void) 311{ 312 return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long); 313} 314 315size_t mem_section_usage_size(void) 316{ 317 return sizeof(struct mem_section_usage) + usemap_size(); 318} 319 320#ifdef CONFIG_MEMORY_HOTREMOVE 321static inline phys_addr_t pgdat_to_phys(struct pglist_data *pgdat) 322{ 323#ifndef CONFIG_NUMA 324 VM_BUG_ON(pgdat != &contig_page_data); 325 return __pa_symbol(&contig_page_data); 326#else 327 return __pa(pgdat); 328#endif 329} 330 331static struct mem_section_usage * __init 332sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 333 unsigned long size) 334{ 335 struct mem_section_usage *usage; 336 unsigned long goal, limit; 337 int nid; 338 /* 339 * A page may contain usemaps for other sections preventing the 340 * page being freed and making a section unremovable while 341 * other sections referencing the usemap remain active. Similarly, 342 * a pgdat can prevent a section being removed. If section A 343 * contains a pgdat and section B contains the usemap, both 344 * sections become inter-dependent. This allocates usemaps 345 * from the same section as the pgdat where possible to avoid 346 * this problem. 347 */ 348 goal = pgdat_to_phys(pgdat) & (PAGE_SECTION_MASK << PAGE_SHIFT); 349 limit = goal + (1UL << PA_SECTION_SHIFT); 350 nid = early_pfn_to_nid(goal >> PAGE_SHIFT); 351again: 352 usage = memblock_alloc_try_nid(size, SMP_CACHE_BYTES, goal, limit, nid); 353 if (!usage && limit) { 354 limit = 0; 355 goto again; 356 } 357 return usage; 358} 359 360static void __init check_usemap_section_nr(int nid, 361 struct mem_section_usage *usage) 362{ 363 unsigned long usemap_snr, pgdat_snr; 364 static unsigned long old_usemap_snr; 365 static unsigned long old_pgdat_snr; 366 struct pglist_data *pgdat = NODE_DATA(nid); 367 int usemap_nid; 368 369 /* First call */ 370 if (!old_usemap_snr) { 371 old_usemap_snr = NR_MEM_SECTIONS; 372 old_pgdat_snr = NR_MEM_SECTIONS; 373 } 374 375 usemap_snr = pfn_to_section_nr(__pa(usage) >> PAGE_SHIFT); 376 pgdat_snr = pfn_to_section_nr(pgdat_to_phys(pgdat) >> PAGE_SHIFT); 377 if (usemap_snr == pgdat_snr) 378 return; 379 380 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 381 /* skip redundant message */ 382 return; 383 384 old_usemap_snr = usemap_snr; 385 old_pgdat_snr = pgdat_snr; 386 387 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 388 if (usemap_nid != nid) { 389 pr_info("node %d must be removed before remove section %ld\n", 390 nid, usemap_snr); 391 return; 392 } 393 /* 394 * There is a circular dependency. 395 * Some platforms allow un-removable section because they will just 396 * gather other removable sections for dynamic partitioning. 397 * Just notify un-removable section's number here. 398 */ 399 pr_info("Section %ld and %ld (node %d) have a circular dependency on usemap and pgdat allocations\n", 400 usemap_snr, pgdat_snr, nid); 401} 402#else 403static struct mem_section_usage * __init 404sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat, 405 unsigned long size) 406{ 407 return memblock_alloc_node(size, SMP_CACHE_BYTES, pgdat->node_id); 408} 409 410static void __init check_usemap_section_nr(int nid, 411 struct mem_section_usage *usage) 412{ 413} 414#endif /* CONFIG_MEMORY_HOTREMOVE */ 415 416#ifdef CONFIG_SPARSEMEM_VMEMMAP 417static unsigned long __init section_map_size(void) 418{ 419 return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE); 420} 421 422#else 423static unsigned long __init section_map_size(void) 424{ 425 return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION); 426} 427 428struct page __init *__populate_section_memmap(unsigned long pfn, 429 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 430 struct dev_pagemap *pgmap) 431{ 432 unsigned long size = section_map_size(); 433 struct page *map = sparse_buffer_alloc(size); 434 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 435 436 if (map) 437 return map; 438 439 map = memmap_alloc(size, size, addr, nid, false); 440 if (!map) 441 panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n", 442 __func__, size, PAGE_SIZE, nid, &addr); 443 444 return map; 445} 446#endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 447 448static void *sparsemap_buf __meminitdata; 449static void *sparsemap_buf_end __meminitdata; 450 451static inline void __meminit sparse_buffer_free(unsigned long size) 452{ 453 WARN_ON(!sparsemap_buf || size == 0); 454 memblock_free(sparsemap_buf, size); 455} 456 457static void __init sparse_buffer_init(unsigned long size, int nid) 458{ 459 phys_addr_t addr = __pa(MAX_DMA_ADDRESS); 460 WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */ 461 /* 462 * Pre-allocated buffer is mainly used by __populate_section_memmap 463 * and we want it to be properly aligned to the section size - this is 464 * especially the case for VMEMMAP which maps memmap to PMDs 465 */ 466 sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true); 467 sparsemap_buf_end = sparsemap_buf + size; 468} 469 470static void __init sparse_buffer_fini(void) 471{ 472 unsigned long size = sparsemap_buf_end - sparsemap_buf; 473 474 if (sparsemap_buf && size > 0) 475 sparse_buffer_free(size); 476 sparsemap_buf = NULL; 477} 478 479void * __meminit sparse_buffer_alloc(unsigned long size) 480{ 481 void *ptr = NULL; 482 483 if (sparsemap_buf) { 484 ptr = (void *) roundup((unsigned long)sparsemap_buf, size); 485 if (ptr + size > sparsemap_buf_end) 486 ptr = NULL; 487 else { 488 /* Free redundant aligned space */ 489 if ((unsigned long)(ptr - sparsemap_buf) > 0) 490 sparse_buffer_free((unsigned long)(ptr - sparsemap_buf)); 491 sparsemap_buf = ptr + size; 492 } 493 } 494 return ptr; 495} 496 497void __weak __meminit vmemmap_populate_print_last(void) 498{ 499} 500 501/* 502 * Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end) 503 * And number of present sections in this node is map_count. 504 */ 505static void __init sparse_init_nid(int nid, unsigned long pnum_begin, 506 unsigned long pnum_end, 507 unsigned long map_count) 508{ 509 struct mem_section_usage *usage; 510 unsigned long pnum; 511 struct page *map; 512 513 usage = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nid), 514 mem_section_usage_size() * map_count); 515 if (!usage) { 516 pr_err("%s: node[%d] usemap allocation failed", __func__, nid); 517 goto failed; 518 } 519 sparse_buffer_init(map_count * section_map_size(), nid); 520 for_each_present_section_nr(pnum_begin, pnum) { 521 unsigned long pfn = section_nr_to_pfn(pnum); 522 523 if (pnum >= pnum_end) 524 break; 525 526 map = __populate_section_memmap(pfn, PAGES_PER_SECTION, 527 nid, NULL, NULL); 528 if (!map) { 529 pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.", 530 __func__, nid); 531 pnum_begin = pnum; 532 sparse_buffer_fini(); 533 goto failed; 534 } 535 check_usemap_section_nr(nid, usage); 536 sparse_init_one_section(__nr_to_section(pnum), pnum, map, usage, 537 SECTION_IS_EARLY); 538 usage = (void *) usage + mem_section_usage_size(); 539 } 540 sparse_buffer_fini(); 541 return; 542failed: 543 /* We failed to allocate, mark all the following pnums as not present */ 544 for_each_present_section_nr(pnum_begin, pnum) { 545 struct mem_section *ms; 546 547 if (pnum >= pnum_end) 548 break; 549 ms = __nr_to_section(pnum); 550 ms->section_mem_map = 0; 551 } 552} 553 554/* 555 * Allocate the accumulated non-linear sections, allocate a mem_map 556 * for each and record the physical to section mapping. 557 */ 558void __init sparse_init(void) 559{ 560 unsigned long pnum_end, pnum_begin, map_count = 1; 561 int nid_begin; 562 563 memblocks_present(); 564 565 pnum_begin = first_present_section_nr(); 566 nid_begin = sparse_early_nid(__nr_to_section(pnum_begin)); 567 568 /* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */ 569 set_pageblock_order(); 570 571 for_each_present_section_nr(pnum_begin + 1, pnum_end) { 572 int nid = sparse_early_nid(__nr_to_section(pnum_end)); 573 574 if (nid == nid_begin) { 575 map_count++; 576 continue; 577 } 578 /* Init node with sections in range [pnum_begin, pnum_end) */ 579 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 580 nid_begin = nid; 581 pnum_begin = pnum_end; 582 map_count = 1; 583 } 584 /* cover the last node */ 585 sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count); 586 vmemmap_populate_print_last(); 587} 588 589#ifdef CONFIG_MEMORY_HOTPLUG 590 591/* Mark all memory sections within the pfn range as online */ 592void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 593{ 594 unsigned long pfn; 595 596 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 597 unsigned long section_nr = pfn_to_section_nr(pfn); 598 struct mem_section *ms; 599 600 /* onlining code should never touch invalid ranges */ 601 if (WARN_ON(!valid_section_nr(section_nr))) 602 continue; 603 604 ms = __nr_to_section(section_nr); 605 ms->section_mem_map |= SECTION_IS_ONLINE; 606 } 607} 608 609/* Mark all memory sections within the pfn range as offline */ 610void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn) 611{ 612 unsigned long pfn; 613 614 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 615 unsigned long section_nr = pfn_to_section_nr(pfn); 616 struct mem_section *ms; 617 618 /* 619 * TODO this needs some double checking. Offlining code makes 620 * sure to check pfn_valid but those checks might be just bogus 621 */ 622 if (WARN_ON(!valid_section_nr(section_nr))) 623 continue; 624 625 ms = __nr_to_section(section_nr); 626 ms->section_mem_map &= ~SECTION_IS_ONLINE; 627 } 628} 629 630#ifdef CONFIG_SPARSEMEM_VMEMMAP 631static struct page * __meminit populate_section_memmap(unsigned long pfn, 632 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 633 struct dev_pagemap *pgmap) 634{ 635 return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 636} 637 638static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 639 struct vmem_altmap *altmap) 640{ 641 unsigned long start = (unsigned long) pfn_to_page(pfn); 642 unsigned long end = start + nr_pages * sizeof(struct page); 643 644 vmemmap_free(start, end, altmap); 645} 646static void free_map_bootmem(struct page *memmap) 647{ 648 unsigned long start = (unsigned long)memmap; 649 unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); 650 651 vmemmap_free(start, end, NULL); 652} 653 654static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 655{ 656 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 657 DECLARE_BITMAP(tmp, SUBSECTIONS_PER_SECTION) = { 0 }; 658 struct mem_section *ms = __pfn_to_section(pfn); 659 unsigned long *subsection_map = ms->usage 660 ? &ms->usage->subsection_map[0] : NULL; 661 662 subsection_mask_set(map, pfn, nr_pages); 663 if (subsection_map) 664 bitmap_and(tmp, map, subsection_map, SUBSECTIONS_PER_SECTION); 665 666 if (WARN(!subsection_map || !bitmap_equal(tmp, map, SUBSECTIONS_PER_SECTION), 667 "section already deactivated (%#lx + %ld)\n", 668 pfn, nr_pages)) 669 return -EINVAL; 670 671 bitmap_xor(subsection_map, map, subsection_map, SUBSECTIONS_PER_SECTION); 672 return 0; 673} 674 675static bool is_subsection_map_empty(struct mem_section *ms) 676{ 677 return bitmap_empty(&ms->usage->subsection_map[0], 678 SUBSECTIONS_PER_SECTION); 679} 680 681static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 682{ 683 struct mem_section *ms = __pfn_to_section(pfn); 684 DECLARE_BITMAP(map, SUBSECTIONS_PER_SECTION) = { 0 }; 685 unsigned long *subsection_map; 686 int rc = 0; 687 688 subsection_mask_set(map, pfn, nr_pages); 689 690 subsection_map = &ms->usage->subsection_map[0]; 691 692 if (bitmap_empty(map, SUBSECTIONS_PER_SECTION)) 693 rc = -EINVAL; 694 else if (bitmap_intersects(map, subsection_map, SUBSECTIONS_PER_SECTION)) 695 rc = -EEXIST; 696 else 697 bitmap_or(subsection_map, map, subsection_map, 698 SUBSECTIONS_PER_SECTION); 699 700 return rc; 701} 702#else 703static struct page * __meminit populate_section_memmap(unsigned long pfn, 704 unsigned long nr_pages, int nid, struct vmem_altmap *altmap, 705 struct dev_pagemap *pgmap) 706{ 707 return kvmalloc_node(array_size(sizeof(struct page), 708 PAGES_PER_SECTION), GFP_KERNEL, nid); 709} 710 711static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, 712 struct vmem_altmap *altmap) 713{ 714 kvfree(pfn_to_page(pfn)); 715} 716 717static void free_map_bootmem(struct page *memmap) 718{ 719 unsigned long maps_section_nr, removing_section_nr, i; 720 unsigned long magic, nr_pages; 721 struct page *page = virt_to_page(memmap); 722 723 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 724 >> PAGE_SHIFT; 725 726 for (i = 0; i < nr_pages; i++, page++) { 727 magic = page->index; 728 729 BUG_ON(magic == NODE_INFO); 730 731 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 732 removing_section_nr = page_private(page); 733 734 /* 735 * When this function is called, the removing section is 736 * logical offlined state. This means all pages are isolated 737 * from page allocator. If removing section's memmap is placed 738 * on the same section, it must not be freed. 739 * If it is freed, page allocator may allocate it which will 740 * be removed physically soon. 741 */ 742 if (maps_section_nr != removing_section_nr) 743 put_page_bootmem(page); 744 } 745} 746 747static int clear_subsection_map(unsigned long pfn, unsigned long nr_pages) 748{ 749 return 0; 750} 751 752static bool is_subsection_map_empty(struct mem_section *ms) 753{ 754 return true; 755} 756 757static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages) 758{ 759 return 0; 760} 761#endif /* CONFIG_SPARSEMEM_VMEMMAP */ 762 763/* 764 * To deactivate a memory region, there are 3 cases to handle across 765 * two configurations (SPARSEMEM_VMEMMAP={y,n}): 766 * 767 * 1. deactivation of a partial hot-added section (only possible in 768 * the SPARSEMEM_VMEMMAP=y case). 769 * a) section was present at memory init. 770 * b) section was hot-added post memory init. 771 * 2. deactivation of a complete hot-added section. 772 * 3. deactivation of a complete section from memory init. 773 * 774 * For 1, when subsection_map does not empty we will not be freeing the 775 * usage map, but still need to free the vmemmap range. 776 * 777 * For 2 and 3, the SPARSEMEM_VMEMMAP={y,n} cases are unified 778 */ 779static void section_deactivate(unsigned long pfn, unsigned long nr_pages, 780 struct vmem_altmap *altmap) 781{ 782 struct mem_section *ms = __pfn_to_section(pfn); 783 bool section_is_early = early_section(ms); 784 struct page *memmap = NULL; 785 bool empty; 786 787 if (clear_subsection_map(pfn, nr_pages)) 788 return; 789 790 empty = is_subsection_map_empty(ms); 791 if (empty) { 792 unsigned long section_nr = pfn_to_section_nr(pfn); 793 794 /* 795 * Mark the section invalid so that valid_section() 796 * return false. This prevents code from dereferencing 797 * ms->usage array. 798 */ 799 ms->section_mem_map &= ~SECTION_HAS_MEM_MAP; 800 801 /* 802 * When removing an early section, the usage map is kept (as the 803 * usage maps of other sections fall into the same page). It 804 * will be re-used when re-adding the section - which is then no 805 * longer an early section. If the usage map is PageReserved, it 806 * was allocated during boot. 807 */ 808 if (!PageReserved(virt_to_page(ms->usage))) { 809 kfree_rcu(ms->usage, rcu); 810 WRITE_ONCE(ms->usage, NULL); 811 } 812 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 813 } 814 815 /* 816 * The memmap of early sections is always fully populated. See 817 * section_activate() and pfn_valid() . 818 */ 819 if (!section_is_early) 820 depopulate_section_memmap(pfn, nr_pages, altmap); 821 else if (memmap) 822 free_map_bootmem(memmap); 823 824 if (empty) 825 ms->section_mem_map = (unsigned long)NULL; 826} 827 828static struct page * __meminit section_activate(int nid, unsigned long pfn, 829 unsigned long nr_pages, struct vmem_altmap *altmap, 830 struct dev_pagemap *pgmap) 831{ 832 struct mem_section *ms = __pfn_to_section(pfn); 833 struct mem_section_usage *usage = NULL; 834 struct page *memmap; 835 int rc; 836 837 if (!ms->usage) { 838 usage = kzalloc(mem_section_usage_size(), GFP_KERNEL); 839 if (!usage) 840 return ERR_PTR(-ENOMEM); 841 ms->usage = usage; 842 } 843 844 rc = fill_subsection_map(pfn, nr_pages); 845 if (rc) { 846 if (usage) 847 ms->usage = NULL; 848 kfree(usage); 849 return ERR_PTR(rc); 850 } 851 852 /* 853 * The early init code does not consider partially populated 854 * initial sections, it simply assumes that memory will never be 855 * referenced. If we hot-add memory into such a section then we 856 * do not need to populate the memmap and can simply reuse what 857 * is already there. 858 */ 859 if (nr_pages < PAGES_PER_SECTION && early_section(ms)) 860 return pfn_to_page(pfn); 861 862 memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); 863 if (!memmap) { 864 section_deactivate(pfn, nr_pages, altmap); 865 return ERR_PTR(-ENOMEM); 866 } 867 868 return memmap; 869} 870 871/** 872 * sparse_add_section - add a memory section, or populate an existing one 873 * @nid: The node to add section on 874 * @start_pfn: start pfn of the memory range 875 * @nr_pages: number of pfns to add in the section 876 * @altmap: alternate pfns to allocate the memmap backing store 877 * @pgmap: alternate compound page geometry for devmap mappings 878 * 879 * This is only intended for hotplug. 880 * 881 * Note that only VMEMMAP supports sub-section aligned hotplug, 882 * the proper alignment and size are gated by check_pfn_span(). 883 * 884 * 885 * Return: 886 * * 0 - On success. 887 * * -EEXIST - Section has been present. 888 * * -ENOMEM - Out of memory. 889 */ 890int __meminit sparse_add_section(int nid, unsigned long start_pfn, 891 unsigned long nr_pages, struct vmem_altmap *altmap, 892 struct dev_pagemap *pgmap) 893{ 894 unsigned long section_nr = pfn_to_section_nr(start_pfn); 895 struct mem_section *ms; 896 struct page *memmap; 897 int ret; 898 899 ret = sparse_index_init(section_nr, nid); 900 if (ret < 0) 901 return ret; 902 903 memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap); 904 if (IS_ERR(memmap)) 905 return PTR_ERR(memmap); 906 907 /* 908 * Poison uninitialized struct pages in order to catch invalid flags 909 * combinations. 910 */ 911 if (!altmap || !altmap->inaccessible) 912 page_init_poison(memmap, sizeof(struct page) * nr_pages); 913 914 ms = __nr_to_section(section_nr); 915 set_section_nid(section_nr, nid); 916 __section_mark_present(ms, section_nr); 917 918 /* Align memmap to section boundary in the subsection case */ 919 if (section_nr_to_pfn(section_nr) != start_pfn) 920 memmap = pfn_to_page(section_nr_to_pfn(section_nr)); 921 sparse_init_one_section(ms, section_nr, memmap, ms->usage, 0); 922 923 return 0; 924} 925 926void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, 927 struct vmem_altmap *altmap) 928{ 929 struct mem_section *ms = __pfn_to_section(pfn); 930 931 if (WARN_ON_ONCE(!valid_section(ms))) 932 return; 933 934 section_deactivate(pfn, nr_pages, altmap); 935} 936#endif /* CONFIG_MEMORY_HOTPLUG */ 937