1/* 2 * linux/arch/x86_64/mm/init.c 3 * 4 * Copyright (C) 1995 Linus Torvalds 5 * Copyright (C) 2000 Pavel Machek <pavel@ucw.cz> 6 * Copyright (C) 2002,2003 Andi Kleen <ak@suse.de> 7 */ 8 9#include <linux/signal.h> 10#include <linux/sched.h> 11#include <linux/kernel.h> 12#include <linux/errno.h> 13#include <linux/string.h> 14#include <linux/types.h> 15#include <linux/ptrace.h> 16#include <linux/mman.h> 17#include <linux/mm.h> 18#include <linux/swap.h> 19#include <linux/smp.h> 20#include <linux/init.h> 21#include <linux/initrd.h> 22#include <linux/pagemap.h> 23#include <linux/bootmem.h> 24#include <linux/proc_fs.h> 25#include <linux/pci.h> 26#include <linux/pfn.h> 27#include <linux/poison.h> 28#include <linux/dma-mapping.h> 29#include <linux/module.h> 30#include <linux/memory_hotplug.h> 31#include <linux/nmi.h> 32#include <linux/gfp.h> 33 34#include <asm/processor.h> 35#include <asm/bios_ebda.h> 36#include <asm/system.h> 37#include <asm/uaccess.h> 38#include <asm/pgtable.h> 39#include <asm/pgalloc.h> 40#include <asm/dma.h> 41#include <asm/fixmap.h> 42#include <asm/e820.h> 43#include <asm/apic.h> 44#include <asm/tlb.h> 45#include <asm/mmu_context.h> 46#include <asm/proto.h> 47#include <asm/smp.h> 48#include <asm/sections.h> 49#include <asm/kdebug.h> 50#include <asm/numa.h> 51#include <asm/cacheflush.h> 52#include <asm/init.h> 53#include <linux/bootmem.h> 54 55static unsigned long dma_reserve __initdata; 56 57static int __init parse_direct_gbpages_off(char *arg) 58{ 59 direct_gbpages = 0; 60 return 0; 61} 62early_param("nogbpages", parse_direct_gbpages_off); 63 64static int __init parse_direct_gbpages_on(char *arg) 65{ 66 direct_gbpages = 1; 67 return 0; 68} 69early_param("gbpages", parse_direct_gbpages_on); 70 71/* 72 * NOTE: pagetable_init alloc all the fixmap pagetables contiguous on the 73 * physical space so we can cache the place of the first one and move 74 * around without checking the pgd every time. 75 */ 76 77pteval_t __supported_pte_mask __read_mostly = ~_PAGE_IOMAP; 78EXPORT_SYMBOL_GPL(__supported_pte_mask); 79 80int force_personality32; 81 82/* 83 * noexec32=on|off 84 * Control non executable heap for 32bit processes. 85 * To control the stack too use noexec=off 86 * 87 * on PROT_READ does not imply PROT_EXEC for 32-bit processes (default) 88 * off PROT_READ implies PROT_EXEC 89 */ 90static int __init nonx32_setup(char *str) 91{ 92 if (!strcmp(str, "on")) 93 force_personality32 &= ~READ_IMPLIES_EXEC; 94 else if (!strcmp(str, "off")) 95 force_personality32 |= READ_IMPLIES_EXEC; 96 return 1; 97} 98__setup("noexec32=", nonx32_setup); 99 100/* 101 * NOTE: This function is marked __ref because it calls __init function 102 * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. 103 */ 104static __ref void *spp_getpage(void) 105{ 106 void *ptr; 107 108 if (after_bootmem) 109 ptr = (void *) get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK); 110 else 111 ptr = alloc_bootmem_pages(PAGE_SIZE); 112 113 if (!ptr || ((unsigned long)ptr & ~PAGE_MASK)) { 114 panic("set_pte_phys: cannot allocate page data %s\n", 115 after_bootmem ? "after bootmem" : ""); 116 } 117 118 pr_debug("spp_getpage %p\n", ptr); 119 120 return ptr; 121} 122 123static pud_t *fill_pud(pgd_t *pgd, unsigned long vaddr) 124{ 125 if (pgd_none(*pgd)) { 126 pud_t *pud = (pud_t *)spp_getpage(); 127 pgd_populate(&init_mm, pgd, pud); 128 if (pud != pud_offset(pgd, 0)) 129 printk(KERN_ERR "PAGETABLE BUG #00! %p <-> %p\n", 130 pud, pud_offset(pgd, 0)); 131 } 132 return pud_offset(pgd, vaddr); 133} 134 135static pmd_t *fill_pmd(pud_t *pud, unsigned long vaddr) 136{ 137 if (pud_none(*pud)) { 138 pmd_t *pmd = (pmd_t *) spp_getpage(); 139 pud_populate(&init_mm, pud, pmd); 140 if (pmd != pmd_offset(pud, 0)) 141 printk(KERN_ERR "PAGETABLE BUG #01! %p <-> %p\n", 142 pmd, pmd_offset(pud, 0)); 143 } 144 return pmd_offset(pud, vaddr); 145} 146 147static pte_t *fill_pte(pmd_t *pmd, unsigned long vaddr) 148{ 149 if (pmd_none(*pmd)) { 150 pte_t *pte = (pte_t *) spp_getpage(); 151 pmd_populate_kernel(&init_mm, pmd, pte); 152 if (pte != pte_offset_kernel(pmd, 0)) 153 printk(KERN_ERR "PAGETABLE BUG #02!\n"); 154 } 155 return pte_offset_kernel(pmd, vaddr); 156} 157 158void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte) 159{ 160 pud_t *pud; 161 pmd_t *pmd; 162 pte_t *pte; 163 164 pud = pud_page + pud_index(vaddr); 165 pmd = fill_pmd(pud, vaddr); 166 pte = fill_pte(pmd, vaddr); 167 168 set_pte(pte, new_pte); 169 170 /* 171 * It's enough to flush this one mapping. 172 * (PGE mappings get flushed as well) 173 */ 174 __flush_tlb_one(vaddr); 175} 176 177void set_pte_vaddr(unsigned long vaddr, pte_t pteval) 178{ 179 pgd_t *pgd; 180 pud_t *pud_page; 181 182 pr_debug("set_pte_vaddr %lx to %lx\n", vaddr, native_pte_val(pteval)); 183 184 pgd = pgd_offset_k(vaddr); 185 if (pgd_none(*pgd)) { 186 printk(KERN_ERR 187 "PGD FIXMAP MISSING, it should be setup in head.S!\n"); 188 return; 189 } 190 pud_page = (pud_t*)pgd_page_vaddr(*pgd); 191 set_pte_vaddr_pud(pud_page, vaddr, pteval); 192} 193 194pmd_t * __init populate_extra_pmd(unsigned long vaddr) 195{ 196 pgd_t *pgd; 197 pud_t *pud; 198 199 pgd = pgd_offset_k(vaddr); 200 pud = fill_pud(pgd, vaddr); 201 return fill_pmd(pud, vaddr); 202} 203 204pte_t * __init populate_extra_pte(unsigned long vaddr) 205{ 206 pmd_t *pmd; 207 208 pmd = populate_extra_pmd(vaddr); 209 return fill_pte(pmd, vaddr); 210} 211 212/* 213 * Create large page table mappings for a range of physical addresses. 214 */ 215static void __init __init_extra_mapping(unsigned long phys, unsigned long size, 216 pgprot_t prot) 217{ 218 pgd_t *pgd; 219 pud_t *pud; 220 pmd_t *pmd; 221 222 BUG_ON((phys & ~PMD_MASK) || (size & ~PMD_MASK)); 223 for (; size; phys += PMD_SIZE, size -= PMD_SIZE) { 224 pgd = pgd_offset_k((unsigned long)__va(phys)); 225 if (pgd_none(*pgd)) { 226 pud = (pud_t *) spp_getpage(); 227 set_pgd(pgd, __pgd(__pa(pud) | _KERNPG_TABLE | 228 _PAGE_USER)); 229 } 230 pud = pud_offset(pgd, (unsigned long)__va(phys)); 231 if (pud_none(*pud)) { 232 pmd = (pmd_t *) spp_getpage(); 233 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE | 234 _PAGE_USER)); 235 } 236 pmd = pmd_offset(pud, phys); 237 BUG_ON(!pmd_none(*pmd)); 238 set_pmd(pmd, __pmd(phys | pgprot_val(prot))); 239 } 240} 241 242void __init init_extra_mapping_wb(unsigned long phys, unsigned long size) 243{ 244 __init_extra_mapping(phys, size, PAGE_KERNEL_LARGE); 245} 246 247void __init init_extra_mapping_uc(unsigned long phys, unsigned long size) 248{ 249 __init_extra_mapping(phys, size, PAGE_KERNEL_LARGE_NOCACHE); 250} 251 252/* 253 * The head.S code sets up the kernel high mapping: 254 * 255 * from __START_KERNEL_map to __START_KERNEL_map + size (== _end-_text) 256 * 257 * phys_addr holds the negative offset to the kernel, which is added 258 * to the compile time generated pmds. This results in invalid pmds up 259 * to the point where we hit the physaddr 0 mapping. 260 * 261 * We limit the mappings to the region from _text to _end. _end is 262 * rounded up to the 2MB boundary. This catches the invalid pmds as 263 * well, as they are located before _text: 264 */ 265void __init cleanup_highmap(void) 266{ 267 unsigned long vaddr = __START_KERNEL_map; 268 unsigned long end = roundup((unsigned long)_end, PMD_SIZE) - 1; 269 pmd_t *pmd = level2_kernel_pgt; 270 pmd_t *last_pmd = pmd + PTRS_PER_PMD; 271 272 for (; pmd < last_pmd; pmd++, vaddr += PMD_SIZE) { 273 if (pmd_none(*pmd)) 274 continue; 275 if (vaddr < (unsigned long) _text || vaddr > end) 276 set_pmd(pmd, __pmd(0)); 277 } 278} 279 280static __ref void *alloc_low_page(unsigned long *phys) 281{ 282 unsigned long pfn = e820_table_end++; 283 void *adr; 284 285 if (after_bootmem) { 286 adr = (void *)get_zeroed_page(GFP_ATOMIC | __GFP_NOTRACK); 287 *phys = __pa(adr); 288 289 return adr; 290 } 291 292 if (pfn >= e820_table_top) 293 panic("alloc_low_page: ran out of memory"); 294 295 adr = early_memremap(pfn * PAGE_SIZE, PAGE_SIZE); 296 memset(adr, 0, PAGE_SIZE); 297 *phys = pfn * PAGE_SIZE; 298 return adr; 299} 300 301static __ref void unmap_low_page(void *adr) 302{ 303 if (after_bootmem) 304 return; 305 306 early_iounmap(adr, PAGE_SIZE); 307} 308 309static unsigned long __meminit 310phys_pte_init(pte_t *pte_page, unsigned long addr, unsigned long end, 311 pgprot_t prot) 312{ 313 unsigned pages = 0; 314 unsigned long last_map_addr = end; 315 int i; 316 317 pte_t *pte = pte_page + pte_index(addr); 318 319 for(i = pte_index(addr); i < PTRS_PER_PTE; i++, addr += PAGE_SIZE, pte++) { 320 321 if (addr >= end) { 322 if (!after_bootmem) { 323 for(; i < PTRS_PER_PTE; i++, pte++) 324 set_pte(pte, __pte(0)); 325 } 326 break; 327 } 328 329 /* 330 * We will re-use the existing mapping. 331 * Xen for example has some special requirements, like mapping 332 * pagetable pages as RO. So assume someone who pre-setup 333 * these mappings are more intelligent. 334 */ 335 if (pte_val(*pte)) { 336 pages++; 337 continue; 338 } 339 340 if (0) 341 printk(" pte=%p addr=%lx pte=%016lx\n", 342 pte, addr, pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL).pte); 343 pages++; 344 set_pte(pte, pfn_pte(addr >> PAGE_SHIFT, prot)); 345 last_map_addr = (addr & PAGE_MASK) + PAGE_SIZE; 346 } 347 348 update_page_count(PG_LEVEL_4K, pages); 349 350 return last_map_addr; 351} 352 353static unsigned long __meminit 354phys_pte_update(pmd_t *pmd, unsigned long address, unsigned long end, 355 pgprot_t prot) 356{ 357 pte_t *pte = (pte_t *)pmd_page_vaddr(*pmd); 358 359 return phys_pte_init(pte, address, end, prot); 360} 361 362static unsigned long __meminit 363phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, 364 unsigned long page_size_mask, pgprot_t prot) 365{ 366 unsigned long pages = 0; 367 unsigned long last_map_addr = end; 368 369 int i = pmd_index(address); 370 371 for (; i < PTRS_PER_PMD; i++, address += PMD_SIZE) { 372 unsigned long pte_phys; 373 pmd_t *pmd = pmd_page + pmd_index(address); 374 pte_t *pte; 375 pgprot_t new_prot = prot; 376 377 if (address >= end) { 378 if (!after_bootmem) { 379 for (; i < PTRS_PER_PMD; i++, pmd++) 380 set_pmd(pmd, __pmd(0)); 381 } 382 break; 383 } 384 385 if (pmd_val(*pmd)) { 386 if (!pmd_large(*pmd)) { 387 spin_lock(&init_mm.page_table_lock); 388 last_map_addr = phys_pte_update(pmd, address, 389 end, prot); 390 spin_unlock(&init_mm.page_table_lock); 391 continue; 392 } 393 /* 394 * If we are ok with PG_LEVEL_2M mapping, then we will 395 * use the existing mapping, 396 * 397 * Otherwise, we will split the large page mapping but 398 * use the same existing protection bits except for 399 * large page, so that we don't violate Intel's TLB 400 * Application note (317080) which says, while changing 401 * the page sizes, new and old translations should 402 * not differ with respect to page frame and 403 * attributes. 404 */ 405 if (page_size_mask & (1 << PG_LEVEL_2M)) { 406 pages++; 407 continue; 408 } 409 new_prot = pte_pgprot(pte_clrhuge(*(pte_t *)pmd)); 410 } 411 412 if (page_size_mask & (1<<PG_LEVEL_2M)) { 413 pages++; 414 spin_lock(&init_mm.page_table_lock); 415 set_pte((pte_t *)pmd, 416 pfn_pte(address >> PAGE_SHIFT, 417 __pgprot(pgprot_val(prot) | _PAGE_PSE))); 418 spin_unlock(&init_mm.page_table_lock); 419 last_map_addr = (address & PMD_MASK) + PMD_SIZE; 420 continue; 421 } 422 423 pte = alloc_low_page(&pte_phys); 424 last_map_addr = phys_pte_init(pte, address, end, new_prot); 425 unmap_low_page(pte); 426 427 spin_lock(&init_mm.page_table_lock); 428 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys)); 429 spin_unlock(&init_mm.page_table_lock); 430 } 431 update_page_count(PG_LEVEL_2M, pages); 432 return last_map_addr; 433} 434 435static unsigned long __meminit 436phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end, 437 unsigned long page_size_mask, pgprot_t prot) 438{ 439 pmd_t *pmd = pmd_offset(pud, 0); 440 unsigned long last_map_addr; 441 442 last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask, prot); 443 __flush_tlb_all(); 444 return last_map_addr; 445} 446 447static unsigned long __meminit 448phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, 449 unsigned long page_size_mask) 450{ 451 unsigned long pages = 0; 452 unsigned long last_map_addr = end; 453 int i = pud_index(addr); 454 455 for (; i < PTRS_PER_PUD; i++, addr = (addr & PUD_MASK) + PUD_SIZE) { 456 unsigned long pmd_phys; 457 pud_t *pud = pud_page + pud_index(addr); 458 pmd_t *pmd; 459 pgprot_t prot = PAGE_KERNEL; 460 461 if (addr >= end) 462 break; 463 464 if (!after_bootmem && 465 !e820_any_mapped(addr, addr+PUD_SIZE, 0)) { 466 set_pud(pud, __pud(0)); 467 continue; 468 } 469 470 if (pud_val(*pud)) { 471 if (!pud_large(*pud)) { 472 last_map_addr = phys_pmd_update(pud, addr, end, 473 page_size_mask, prot); 474 continue; 475 } 476 /* 477 * If we are ok with PG_LEVEL_1G mapping, then we will 478 * use the existing mapping. 479 * 480 * Otherwise, we will split the gbpage mapping but use 481 * the same existing protection bits except for large 482 * page, so that we don't violate Intel's TLB 483 * Application note (317080) which says, while changing 484 * the page sizes, new and old translations should 485 * not differ with respect to page frame and 486 * attributes. 487 */ 488 if (page_size_mask & (1 << PG_LEVEL_1G)) { 489 pages++; 490 continue; 491 } 492 prot = pte_pgprot(pte_clrhuge(*(pte_t *)pud)); 493 } 494 495 if (page_size_mask & (1<<PG_LEVEL_1G)) { 496 pages++; 497 spin_lock(&init_mm.page_table_lock); 498 set_pte((pte_t *)pud, 499 pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); 500 spin_unlock(&init_mm.page_table_lock); 501 last_map_addr = (addr & PUD_MASK) + PUD_SIZE; 502 continue; 503 } 504 505 pmd = alloc_low_page(&pmd_phys); 506 last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask, 507 prot); 508 unmap_low_page(pmd); 509 510 spin_lock(&init_mm.page_table_lock); 511 pud_populate(&init_mm, pud, __va(pmd_phys)); 512 spin_unlock(&init_mm.page_table_lock); 513 } 514 __flush_tlb_all(); 515 516 update_page_count(PG_LEVEL_1G, pages); 517 518 return last_map_addr; 519} 520 521static unsigned long __meminit 522phys_pud_update(pgd_t *pgd, unsigned long addr, unsigned long end, 523 unsigned long page_size_mask) 524{ 525 pud_t *pud; 526 527 pud = (pud_t *)pgd_page_vaddr(*pgd); 528 529 return phys_pud_init(pud, addr, end, page_size_mask); 530} 531 532unsigned long __meminit 533kernel_physical_mapping_init(unsigned long start, 534 unsigned long end, 535 unsigned long page_size_mask) 536{ 537 538 unsigned long next, last_map_addr = end; 539 540 start = (unsigned long)__va(start); 541 end = (unsigned long)__va(end); 542 543 for (; start < end; start = next) { 544 pgd_t *pgd = pgd_offset_k(start); 545 unsigned long pud_phys; 546 pud_t *pud; 547 548 next = (start + PGDIR_SIZE) & PGDIR_MASK; 549 if (next > end) 550 next = end; 551 552 if (pgd_val(*pgd)) { 553 last_map_addr = phys_pud_update(pgd, __pa(start), 554 __pa(end), page_size_mask); 555 continue; 556 } 557 558 pud = alloc_low_page(&pud_phys); 559 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next), 560 page_size_mask); 561 unmap_low_page(pud); 562 563 spin_lock(&init_mm.page_table_lock); 564 pgd_populate(&init_mm, pgd, __va(pud_phys)); 565 spin_unlock(&init_mm.page_table_lock); 566 } 567 __flush_tlb_all(); 568 569 return last_map_addr; 570} 571 572#ifndef CONFIG_NUMA 573void __init initmem_init(unsigned long start_pfn, unsigned long end_pfn, 574 int acpi, int k8) 575{ 576#ifndef CONFIG_NO_BOOTMEM 577 unsigned long bootmap_size, bootmap; 578 579 bootmap_size = bootmem_bootmap_pages(end_pfn)<<PAGE_SHIFT; 580 bootmap = find_e820_area(0, end_pfn<<PAGE_SHIFT, bootmap_size, 581 PAGE_SIZE); 582 if (bootmap == -1L) 583 panic("Cannot find bootmem map of size %ld\n", bootmap_size); 584 reserve_early(bootmap, bootmap + bootmap_size, "BOOTMAP"); 585 /* don't touch min_low_pfn */ 586 bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap >> PAGE_SHIFT, 587 0, end_pfn); 588 e820_register_active_regions(0, start_pfn, end_pfn); 589 free_bootmem_with_active_regions(0, end_pfn); 590#else 591 e820_register_active_regions(0, start_pfn, end_pfn); 592#endif 593} 594#endif 595 596void __init paging_init(void) 597{ 598 unsigned long max_zone_pfns[MAX_NR_ZONES]; 599 600 memset(max_zone_pfns, 0, sizeof(max_zone_pfns)); 601 max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN; 602 max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN; 603 max_zone_pfns[ZONE_NORMAL] = max_pfn; 604 605 sparse_memory_present_with_active_regions(MAX_NUMNODES); 606 sparse_init(); 607 608 /* 609 * clear the default setting with node 0 610 * note: don't use nodes_clear here, that is really clearing when 611 * numa support is not compiled in, and later node_set_state 612 * will not set it back. 613 */ 614 node_clear_state(0, N_NORMAL_MEMORY); 615 616 free_area_init_nodes(max_zone_pfns); 617} 618 619/* 620 * Memory hotplug specific functions 621 */ 622#ifdef CONFIG_MEMORY_HOTPLUG 623/* 624 * After memory hotplug the variables max_pfn, max_low_pfn and high_memory need 625 * updating. 626 */ 627static void update_end_of_memory_vars(u64 start, u64 size) 628{ 629 unsigned long end_pfn = PFN_UP(start + size); 630 631 if (end_pfn > max_pfn) { 632 max_pfn = end_pfn; 633 max_low_pfn = end_pfn; 634 high_memory = (void *)__va(max_pfn * PAGE_SIZE - 1) + 1; 635 } 636} 637 638/* 639 * Memory is added always to NORMAL zone. This means you will never get 640 * additional DMA/DMA32 memory. 641 */ 642int arch_add_memory(int nid, u64 start, u64 size) 643{ 644 struct pglist_data *pgdat = NODE_DATA(nid); 645 struct zone *zone = pgdat->node_zones + ZONE_NORMAL; 646 unsigned long last_mapped_pfn, start_pfn = start >> PAGE_SHIFT; 647 unsigned long nr_pages = size >> PAGE_SHIFT; 648 int ret; 649 650 last_mapped_pfn = init_memory_mapping(start, start + size); 651 if (last_mapped_pfn > max_pfn_mapped) 652 max_pfn_mapped = last_mapped_pfn; 653 654 ret = __add_pages(nid, zone, start_pfn, nr_pages); 655 WARN_ON_ONCE(ret); 656 657 /* update max_pfn, max_low_pfn and high_memory */ 658 update_end_of_memory_vars(start, size); 659 660 return ret; 661} 662EXPORT_SYMBOL_GPL(arch_add_memory); 663 664#if !defined(CONFIG_ACPI_NUMA) && defined(CONFIG_NUMA) 665int memory_add_physaddr_to_nid(u64 start) 666{ 667 return 0; 668} 669EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); 670#endif 671 672#endif /* CONFIG_MEMORY_HOTPLUG */ 673 674static struct kcore_list kcore_vsyscall; 675 676void __init mem_init(void) 677{ 678 long codesize, reservedpages, datasize, initsize; 679 unsigned long absent_pages; 680 681 pci_iommu_alloc(); 682 683 /* clear_bss() already clear the empty_zero_page */ 684 685 reservedpages = 0; 686 687 /* this will put all low memory onto the freelists */ 688#ifdef CONFIG_NUMA 689 totalram_pages = numa_free_all_bootmem(); 690#else 691 totalram_pages = free_all_bootmem(); 692#endif 693 694 absent_pages = absent_pages_in_range(0, max_pfn); 695 reservedpages = max_pfn - totalram_pages - absent_pages; 696 after_bootmem = 1; 697 698 codesize = (unsigned long) &_etext - (unsigned long) &_text; 699 datasize = (unsigned long) &_edata - (unsigned long) &_etext; 700 initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; 701 702 /* Register memory areas for /proc/kcore */ 703 kclist_add(&kcore_vsyscall, (void *)VSYSCALL_START, 704 VSYSCALL_END - VSYSCALL_START, KCORE_OTHER); 705 706 printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, " 707 "%ldk absent, %ldk reserved, %ldk data, %ldk init)\n", 708 nr_free_pages() << (PAGE_SHIFT-10), 709 max_pfn << (PAGE_SHIFT-10), 710 codesize >> 10, 711 absent_pages << (PAGE_SHIFT-10), 712 reservedpages << (PAGE_SHIFT-10), 713 datasize >> 10, 714 initsize >> 10); 715} 716 717#ifdef CONFIG_DEBUG_RODATA 718const int rodata_test_data = 0xC3; 719EXPORT_SYMBOL_GPL(rodata_test_data); 720 721int kernel_set_to_readonly; 722 723void set_kernel_text_rw(void) 724{ 725 unsigned long start = PFN_ALIGN(_text); 726 unsigned long end = PFN_ALIGN(__stop___ex_table); 727 728 if (!kernel_set_to_readonly) 729 return; 730 731 pr_debug("Set kernel text: %lx - %lx for read write\n", 732 start, end); 733 734 /* 735 * Make the kernel identity mapping for text RW. Kernel text 736 * mapping will always be RO. Refer to the comment in 737 * static_protections() in pageattr.c 738 */ 739 set_memory_rw(start, (end - start) >> PAGE_SHIFT); 740} 741 742void set_kernel_text_ro(void) 743{ 744 unsigned long start = PFN_ALIGN(_text); 745 unsigned long end = PFN_ALIGN(__stop___ex_table); 746 747 if (!kernel_set_to_readonly) 748 return; 749 750 pr_debug("Set kernel text: %lx - %lx for read only\n", 751 start, end); 752 753 /* 754 * Set the kernel identity mapping for text RO. 755 */ 756 set_memory_ro(start, (end - start) >> PAGE_SHIFT); 757} 758 759void mark_rodata_ro(void) 760{ 761 unsigned long start = PFN_ALIGN(_text); 762 unsigned long rodata_start = 763 ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK; 764 unsigned long end = (unsigned long) &__end_rodata_hpage_align; 765 unsigned long text_end = PAGE_ALIGN((unsigned long) &__stop___ex_table); 766 unsigned long rodata_end = PAGE_ALIGN((unsigned long) &__end_rodata); 767 unsigned long data_start = (unsigned long) &_sdata; 768 769 printk(KERN_INFO "Write protecting the kernel read-only data: %luk\n", 770 (end - start) >> 10); 771 set_memory_ro(start, (end - start) >> PAGE_SHIFT); 772 773 kernel_set_to_readonly = 1; 774 775 /* 776 * The rodata section (but not the kernel text!) should also be 777 * not-executable. 778 */ 779 set_memory_nx(rodata_start, (end - rodata_start) >> PAGE_SHIFT); 780 781 rodata_test(); 782 783#ifdef CONFIG_CPA_DEBUG 784 printk(KERN_INFO "Testing CPA: undo %lx-%lx\n", start, end); 785 set_memory_rw(start, (end-start) >> PAGE_SHIFT); 786 787 printk(KERN_INFO "Testing CPA: again\n"); 788 set_memory_ro(start, (end-start) >> PAGE_SHIFT); 789#endif 790 791 free_init_pages("unused kernel memory", 792 (unsigned long) page_address(virt_to_page(text_end)), 793 (unsigned long) 794 page_address(virt_to_page(rodata_start))); 795 free_init_pages("unused kernel memory", 796 (unsigned long) page_address(virt_to_page(rodata_end)), 797 (unsigned long) page_address(virt_to_page(data_start))); 798} 799 800#endif 801 802int __init reserve_bootmem_generic(unsigned long phys, unsigned long len, 803 int flags) 804{ 805#ifdef CONFIG_NUMA 806 int nid, next_nid; 807 int ret; 808#endif 809 unsigned long pfn = phys >> PAGE_SHIFT; 810 811 if (pfn >= max_pfn) { 812 /* 813 * This can happen with kdump kernels when accessing 814 * firmware tables: 815 */ 816 if (pfn < max_pfn_mapped) 817 return -EFAULT; 818 819 printk(KERN_ERR "reserve_bootmem: illegal reserve %lx %lu\n", 820 phys, len); 821 return -EFAULT; 822 } 823 824 /* Should check here against the e820 map to avoid double free */ 825#ifdef CONFIG_NUMA 826 nid = phys_to_nid(phys); 827 next_nid = phys_to_nid(phys + len - 1); 828 if (nid == next_nid) 829 ret = reserve_bootmem_node(NODE_DATA(nid), phys, len, flags); 830 else 831 ret = reserve_bootmem(phys, len, flags); 832 833 if (ret != 0) 834 return ret; 835 836#else 837 reserve_bootmem(phys, len, flags); 838#endif 839 840 if (phys+len <= MAX_DMA_PFN*PAGE_SIZE) { 841 dma_reserve += len / PAGE_SIZE; 842 set_dma_reserve(dma_reserve); 843 } 844 845 return 0; 846} 847 848int kern_addr_valid(unsigned long addr) 849{ 850 unsigned long above = ((long)addr) >> __VIRTUAL_MASK_SHIFT; 851 pgd_t *pgd; 852 pud_t *pud; 853 pmd_t *pmd; 854 pte_t *pte; 855 856 if (above != 0 && above != -1UL) 857 return 0; 858 859 pgd = pgd_offset_k(addr); 860 if (pgd_none(*pgd)) 861 return 0; 862 863 pud = pud_offset(pgd, addr); 864 if (pud_none(*pud)) 865 return 0; 866 867 pmd = pmd_offset(pud, addr); 868 if (pmd_none(*pmd)) 869 return 0; 870 871 if (pmd_large(*pmd)) 872 return pfn_valid(pmd_pfn(*pmd)); 873 874 pte = pte_offset_kernel(pmd, addr); 875 if (pte_none(*pte)) 876 return 0; 877 878 return pfn_valid(pte_pfn(*pte)); 879} 880 881/* 882 * A pseudo VMA to allow ptrace access for the vsyscall page. This only 883 * covers the 64bit vsyscall page now. 32bit has a real VMA now and does 884 * not need special handling anymore: 885 */ 886static struct vm_area_struct gate_vma = { 887 .vm_start = VSYSCALL_START, 888 .vm_end = VSYSCALL_START + (VSYSCALL_MAPPED_PAGES * PAGE_SIZE), 889 .vm_page_prot = PAGE_READONLY_EXEC, 890 .vm_flags = VM_READ | VM_EXEC 891}; 892 893struct vm_area_struct *get_gate_vma(struct task_struct *tsk) 894{ 895#ifdef CONFIG_IA32_EMULATION 896 if (test_tsk_thread_flag(tsk, TIF_IA32)) 897 return NULL; 898#endif 899 return &gate_vma; 900} 901 902int in_gate_area(struct task_struct *task, unsigned long addr) 903{ 904 struct vm_area_struct *vma = get_gate_vma(task); 905 906 if (!vma) 907 return 0; 908 909 return (addr >= vma->vm_start) && (addr < vma->vm_end); 910} 911 912/* 913 * Use this when you have no reliable task/vma, typically from interrupt 914 * context. It is less reliable than using the task's vma and may give 915 * false positives: 916 */ 917int in_gate_area_no_task(unsigned long addr) 918{ 919 return (addr >= VSYSCALL_START) && (addr < VSYSCALL_END); 920} 921 922const char *arch_vma_name(struct vm_area_struct *vma) 923{ 924 if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso) 925 return "[vdso]"; 926 if (vma == &gate_vma) 927 return "[vsyscall]"; 928 return NULL; 929} 930 931#ifdef CONFIG_SPARSEMEM_VMEMMAP 932/* 933 * Initialise the sparsemem vmemmap using huge-pages at the PMD level. 934 */ 935static long __meminitdata addr_start, addr_end; 936static void __meminitdata *p_start, *p_end; 937static int __meminitdata node_start; 938 939int __meminit 940vmemmap_populate(struct page *start_page, unsigned long size, int node) 941{ 942 unsigned long addr = (unsigned long)start_page; 943 unsigned long end = (unsigned long)(start_page + size); 944 unsigned long next; 945 pgd_t *pgd; 946 pud_t *pud; 947 pmd_t *pmd; 948 949 for (; addr < end; addr = next) { 950 void *p = NULL; 951 952 pgd = vmemmap_pgd_populate(addr, node); 953 if (!pgd) 954 return -ENOMEM; 955 956 pud = vmemmap_pud_populate(pgd, addr, node); 957 if (!pud) 958 return -ENOMEM; 959 960 if (!cpu_has_pse) { 961 next = (addr + PAGE_SIZE) & PAGE_MASK; 962 pmd = vmemmap_pmd_populate(pud, addr, node); 963 964 if (!pmd) 965 return -ENOMEM; 966 967 p = vmemmap_pte_populate(pmd, addr, node); 968 969 if (!p) 970 return -ENOMEM; 971 972 addr_end = addr + PAGE_SIZE; 973 p_end = p + PAGE_SIZE; 974 } else { 975 next = pmd_addr_end(addr, end); 976 977 pmd = pmd_offset(pud, addr); 978 if (pmd_none(*pmd)) { 979 pte_t entry; 980 981 p = vmemmap_alloc_block_buf(PMD_SIZE, node); 982 if (!p) 983 return -ENOMEM; 984 985 entry = pfn_pte(__pa(p) >> PAGE_SHIFT, 986 PAGE_KERNEL_LARGE); 987 set_pmd(pmd, __pmd(pte_val(entry))); 988 989 /* check to see if we have contiguous blocks */ 990 if (p_end != p || node_start != node) { 991 if (p_start) 992 printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n", 993 addr_start, addr_end-1, p_start, p_end-1, node_start); 994 addr_start = addr; 995 node_start = node; 996 p_start = p; 997 } 998 999 addr_end = addr + PMD_SIZE; 1000 p_end = p + PMD_SIZE; 1001 } else 1002 vmemmap_verify((pte_t *)pmd, node, addr, next); 1003 } 1004 1005 } 1006 return 0; 1007} 1008 1009void __meminit vmemmap_populate_print_last(void) 1010{ 1011 if (p_start) { 1012 printk(KERN_DEBUG " [%lx-%lx] PMD -> [%p-%p] on node %d\n", 1013 addr_start, addr_end-1, p_start, p_end-1, node_start); 1014 p_start = NULL; 1015 p_end = NULL; 1016 node_start = 0; 1017 } 1018} 1019#endif 1020