1/*
2 *  PowerPC version
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 *  Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
6 *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
7 *    Copyright (C) 1996 Paul Mackerras
8 *  Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk).
9 *  PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
10 *
11 *  Derived from "arch/i386/mm/init.c"
12 *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
13 *
14 *  This program is free software; you can redistribute it and/or
15 *  modify it under the terms of the GNU General Public License
16 *  as published by the Free Software Foundation; either version
17 *  2 of the License, or (at your option) any later version.
18 *
19 */
20
21#include <linux/module.h>
22#include <linux/sched.h>
23#include <linux/kernel.h>
24#include <linux/errno.h>
25#include <linux/string.h>
26#include <linux/types.h>
27#include <linux/mm.h>
28#include <linux/stddef.h>
29#include <linux/init.h>
30#include <linux/bootmem.h>
31#include <linux/highmem.h>
32#include <linux/initrd.h>
33#include <linux/pagemap.h>
34
35#include <asm/pgalloc.h>
36#include <asm/prom.h>
37#include <asm/io.h>
38#include <asm/mmu_context.h>
39#include <asm/pgtable.h>
40#include <asm/mmu.h>
41#include <asm/smp.h>
42#include <asm/machdep.h>
43#include <asm/btext.h>
44#include <asm/tlb.h>
45#include <asm/bootinfo.h>
46
47#include "mem_pieces.h"
48#include "mmu_decl.h"
49
50#if defined(CONFIG_KERNEL_START_BOOL) || defined(CONFIG_LOWMEM_SIZE_BOOL)
51/* The amount of lowmem must be within 0xF0000000 - KERNELBASE. */
52#if (CONFIG_LOWMEM_SIZE > (0xF0000000 - KERNELBASE))
53#error "You must adjust CONFIG_LOWMEM_SIZE or CONFIG_START_KERNEL"
54#endif
55#endif
56#define MAX_LOW_MEM	CONFIG_LOWMEM_SIZE
57
58DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
59
60unsigned long total_memory;
61unsigned long total_lowmem;
62
63unsigned long ppc_memstart;
64unsigned long ppc_memoffset = PAGE_OFFSET;
65
66int mem_init_done;
67int init_bootmem_done;
68int boot_mapsize;
69
70extern char _end[];
71extern char etext[], _stext[];
72extern char __init_begin, __init_end;
73
74#ifdef CONFIG_HIGHMEM
75pte_t *kmap_pte;
76pgprot_t kmap_prot;
77
78EXPORT_SYMBOL(kmap_prot);
79EXPORT_SYMBOL(kmap_pte);
80#endif
81
82void MMU_init(void);
83void set_phys_avail(unsigned long total_ram);
84
85extern struct task_struct *current_set[NR_CPUS];
86
87char *klimit = _end;
88struct mem_pieces phys_avail;
89
90/*
91 * this tells the system to map all of ram with the segregs
92 * (i.e. page tables) instead of the bats.
93 * -- Cort
94 */
95int __map_without_bats;
96int __map_without_ltlbs;
97
98/* max amount of RAM to use */
99unsigned long __max_memory;
100/* max amount of low RAM to map in */
101unsigned long __max_low_memory = MAX_LOW_MEM;
102
103void show_mem(void)
104{
105	int i,free = 0,total = 0,reserved = 0;
106	int shared = 0, cached = 0;
107	int highmem = 0;
108
109	printk("Mem-info:\n");
110	show_free_areas();
111	printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
112	i = max_mapnr;
113	while (i-- > 0) {
114		total++;
115		if (PageHighMem(mem_map+i))
116			highmem++;
117		if (PageReserved(mem_map+i))
118			reserved++;
119		else if (PageSwapCache(mem_map+i))
120			cached++;
121		else if (!page_count(mem_map+i))
122			free++;
123		else
124			shared += page_count(mem_map+i) - 1;
125	}
126	printk("%d pages of RAM\n",total);
127	printk("%d pages of HIGHMEM\n", highmem);
128	printk("%d free pages\n",free);
129	printk("%d reserved pages\n",reserved);
130	printk("%d pages shared\n",shared);
131	printk("%d pages swap cached\n",cached);
132}
133
134/* Free up now-unused memory */
135static void free_sec(unsigned long start, unsigned long end, const char *name)
136{
137	unsigned long cnt = 0;
138
139	while (start < end) {
140		ClearPageReserved(virt_to_page(start));
141		init_page_count(virt_to_page(start));
142		free_page(start);
143		cnt++;
144		start += PAGE_SIZE;
145 	}
146	if (cnt) {
147		printk(" %ldk %s", cnt << (PAGE_SHIFT - 10), name);
148		totalram_pages += cnt;
149	}
150}
151
152void free_initmem(void)
153{
154#define FREESEC(TYPE) \
155	free_sec((unsigned long)(&__ ## TYPE ## _begin), \
156		 (unsigned long)(&__ ## TYPE ## _end), \
157		 #TYPE);
158
159	printk ("Freeing unused kernel memory:");
160	FREESEC(init);
161 	printk("\n");
162	ppc_md.progress = NULL;
163#undef FREESEC
164}
165
166#ifdef CONFIG_BLK_DEV_INITRD
167void free_initrd_mem(unsigned long start, unsigned long end)
168{
169	printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
170
171	for (; start < end; start += PAGE_SIZE) {
172		ClearPageReserved(virt_to_page(start));
173		init_page_count(virt_to_page(start));
174		free_page(start);
175		totalram_pages++;
176	}
177}
178#endif
179
180/*
181 * Check for command-line options that affect what MMU_init will do.
182 */
183void MMU_setup(void)
184{
185	/* Check for nobats option (used in mapin_ram). */
186	if (strstr(cmd_line, "nobats")) {
187		__map_without_bats = 1;
188	}
189
190	if (strstr(cmd_line, "noltlbs")) {
191		__map_without_ltlbs = 1;
192	}
193
194	/* Look for mem= option on command line */
195	if (strstr(cmd_line, "mem=")) {
196		char *p, *q;
197		unsigned long maxmem = 0;
198
199		for (q = cmd_line; (p = strstr(q, "mem=")) != 0; ) {
200			q = p + 4;
201			if (p > cmd_line && p[-1] != ' ')
202				continue;
203			maxmem = simple_strtoul(q, &q, 0);
204			if (*q == 'k' || *q == 'K') {
205				maxmem <<= 10;
206				++q;
207			} else if (*q == 'm' || *q == 'M') {
208				maxmem <<= 20;
209				++q;
210			}
211		}
212		__max_memory = maxmem;
213	}
214}
215
216/*
217 * MMU_init sets up the basic memory mappings for the kernel,
218 * including both RAM and possibly some I/O regions,
219 * and sets up the page tables and the MMU hardware ready to go.
220 */
221void __init MMU_init(void)
222{
223	if (ppc_md.progress)
224		ppc_md.progress("MMU:enter", 0x111);
225
226	/* parse args from command line */
227	MMU_setup();
228
229	/*
230	 * Figure out how much memory we have, how much
231	 * is lowmem, and how much is highmem.  If we were
232	 * passed the total memory size from the bootloader,
233	 * just use it.
234	 */
235	if (boot_mem_size)
236		total_memory = boot_mem_size;
237	else
238		total_memory = ppc_md.find_end_of_memory();
239
240	if (__max_memory && total_memory > __max_memory)
241		total_memory = __max_memory;
242	total_lowmem = total_memory;
243#ifdef CONFIG_FSL_BOOKE
244	/* Freescale Book-E parts expect lowmem to be mapped by fixed TLB
245	 * entries, so we need to adjust lowmem to match the amount we can map
246	 * in the fixed entries */
247	adjust_total_lowmem();
248#endif /* CONFIG_FSL_BOOKE */
249	if (total_lowmem > __max_low_memory) {
250		total_lowmem = __max_low_memory;
251#ifndef CONFIG_HIGHMEM
252		total_memory = total_lowmem;
253#endif /* CONFIG_HIGHMEM */
254	}
255	set_phys_avail(total_lowmem);
256
257	/* Initialize the MMU hardware */
258	if (ppc_md.progress)
259		ppc_md.progress("MMU:hw init", 0x300);
260	MMU_init_hw();
261
262	/* Map in all of RAM starting at KERNELBASE */
263	if (ppc_md.progress)
264		ppc_md.progress("MMU:mapin", 0x301);
265	mapin_ram();
266
267#ifdef CONFIG_HIGHMEM
268	ioremap_base = PKMAP_BASE;
269#else
270	ioremap_base = 0xfe000000UL;	/* for now, could be 0xfffff000 */
271#endif /* CONFIG_HIGHMEM */
272	ioremap_bot = ioremap_base;
273
274	/* Map in I/O resources */
275	if (ppc_md.progress)
276		ppc_md.progress("MMU:setio", 0x302);
277	if (ppc_md.setup_io_mappings)
278		ppc_md.setup_io_mappings();
279
280	/* Initialize the context management stuff */
281	mmu_context_init();
282
283	if (ppc_md.progress)
284		ppc_md.progress("MMU:exit", 0x211);
285
286#ifdef CONFIG_BOOTX_TEXT
287	/* By default, we are no longer mapped */
288       	boot_text_mapped = 0;
289	/* Must be done last, or ppc_md.progress will die. */
290	map_boot_text();
291#endif
292}
293
294/* This is only called until mem_init is done. */
295void __init *early_get_page(void)
296{
297	void *p;
298
299	if (init_bootmem_done) {
300		p = alloc_bootmem_pages(PAGE_SIZE);
301	} else {
302		p = mem_pieces_find(PAGE_SIZE, PAGE_SIZE);
303	}
304	return p;
305}
306
307/*
308 * Initialize the bootmem system and give it all the memory we
309 * have available.
310 */
311void __init do_init_bootmem(void)
312{
313	unsigned long start, size;
314	int i;
315
316	/*
317	 * Find an area to use for the bootmem bitmap.
318	 * We look for the first area which is at least
319	 * 128kB in length (128kB is enough for a bitmap
320	 * for 4GB of memory, using 4kB pages), plus 1 page
321	 * (in case the address isn't page-aligned).
322	 */
323	start = 0;
324	size = 0;
325	for (i = 0; i < phys_avail.n_regions; ++i) {
326		unsigned long a = phys_avail.regions[i].address;
327		unsigned long s = phys_avail.regions[i].size;
328		if (s <= size)
329			continue;
330		start = a;
331		size = s;
332		if (s >= 33 * PAGE_SIZE)
333			break;
334	}
335	start = PAGE_ALIGN(start);
336
337	min_low_pfn = start >> PAGE_SHIFT;
338	max_low_pfn = (PPC_MEMSTART + total_lowmem) >> PAGE_SHIFT;
339	max_pfn = (PPC_MEMSTART + total_memory) >> PAGE_SHIFT;
340	boot_mapsize = init_bootmem_node(&contig_page_data, min_low_pfn,
341					 PPC_MEMSTART >> PAGE_SHIFT,
342					 max_low_pfn);
343
344	/* remove the bootmem bitmap from the available memory */
345	mem_pieces_remove(&phys_avail, start, boot_mapsize, 1);
346
347	/* add everything in phys_avail into the bootmem map */
348	for (i = 0; i < phys_avail.n_regions; ++i)
349		free_bootmem(phys_avail.regions[i].address,
350			     phys_avail.regions[i].size);
351
352	init_bootmem_done = 1;
353}
354
355/*
356 * paging_init() sets up the page tables - in fact we've already done this.
357 */
358void __init paging_init(void)
359{
360	unsigned long start_pfn, end_pfn;
361	unsigned long max_zone_pfns[MAX_NR_ZONES];
362#ifdef CONFIG_HIGHMEM
363	map_page(PKMAP_BASE, 0, 0);
364	pkmap_page_table = pte_offset_kernel(pmd_offset(pgd_offset_k
365			(PKMAP_BASE), PKMAP_BASE), PKMAP_BASE);
366	map_page(KMAP_FIX_BEGIN, 0, 0);
367	kmap_pte = pte_offset_kernel(pmd_offset(pgd_offset_k
368			(KMAP_FIX_BEGIN), KMAP_FIX_BEGIN), KMAP_FIX_BEGIN);
369	kmap_prot = PAGE_KERNEL;
370#endif /* CONFIG_HIGHMEM */
371	/* All pages are DMA-able so we put them all in the DMA zone. */
372	start_pfn = __pa(PAGE_OFFSET) >> PAGE_SHIFT;
373	end_pfn = start_pfn + (total_memory >> PAGE_SHIFT);
374	add_active_range(0, start_pfn, end_pfn);
375
376	memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
377#ifdef CONFIG_HIGHMEM
378	max_zone_pfns[ZONE_DMA] = total_lowmem >> PAGE_SHIFT;
379	max_zone_pfns[ZONE_HIGHMEM] = total_memory >> PAGE_SHIFT;
380#else
381	max_zone_pfns[ZONE_DMA] = total_memory >> PAGE_SHIFT;
382#endif /* CONFIG_HIGHMEM */
383	free_area_init_nodes(max_zone_pfns);
384}
385
386void __init mem_init(void)
387{
388	unsigned long addr;
389	int codepages = 0;
390	int datapages = 0;
391	int initpages = 0;
392#ifdef CONFIG_HIGHMEM
393	unsigned long highmem_mapnr;
394
395	highmem_mapnr = total_lowmem >> PAGE_SHIFT;
396#endif /* CONFIG_HIGHMEM */
397	max_mapnr = total_memory >> PAGE_SHIFT;
398
399	high_memory = (void *) __va(PPC_MEMSTART + total_lowmem);
400	num_physpages = max_mapnr;	/* RAM is assumed contiguous */
401
402	totalram_pages += free_all_bootmem();
403
404#ifdef CONFIG_BLK_DEV_INITRD
405	/* if we are booted from BootX with an initial ramdisk,
406	   make sure the ramdisk pages aren't reserved. */
407	if (initrd_start) {
408		for (addr = initrd_start; addr < initrd_end; addr += PAGE_SIZE)
409			ClearPageReserved(virt_to_page(addr));
410	}
411#endif /* CONFIG_BLK_DEV_INITRD */
412
413	for (addr = PAGE_OFFSET; addr < (unsigned long)high_memory;
414	     addr += PAGE_SIZE) {
415		if (!PageReserved(virt_to_page(addr)))
416			continue;
417		if (addr < (ulong) etext)
418			codepages++;
419		else if (addr >= (unsigned long)&__init_begin
420			 && addr < (unsigned long)&__init_end)
421			initpages++;
422		else if (addr < (ulong) klimit)
423			datapages++;
424	}
425
426#ifdef CONFIG_HIGHMEM
427	{
428		unsigned long pfn;
429
430		for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
431			struct page *page = mem_map + pfn;
432
433			ClearPageReserved(page);
434			init_page_count(page);
435			__free_page(page);
436			totalhigh_pages++;
437		}
438		totalram_pages += totalhigh_pages;
439	}
440#endif /* CONFIG_HIGHMEM */
441
442        printk("Memory: %luk available (%dk kernel code, %dk data, %dk init, %ldk highmem)\n",
443	       (unsigned long)nr_free_pages()<< (PAGE_SHIFT-10),
444	       codepages<< (PAGE_SHIFT-10), datapages<< (PAGE_SHIFT-10),
445	       initpages<< (PAGE_SHIFT-10),
446	       (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10)));
447
448	mem_init_done = 1;
449}
450
451/*
452 * Set phys_avail to the amount of physical memory,
453 * less the kernel text/data/bss.
454 */
455void __init
456set_phys_avail(unsigned long total_memory)
457{
458	unsigned long kstart, ksize;
459
460	/*
461	 * Initially, available physical memory is equivalent to all
462	 * physical memory.
463	 */
464
465	phys_avail.regions[0].address = PPC_MEMSTART;
466	phys_avail.regions[0].size = total_memory;
467	phys_avail.n_regions = 1;
468
469	/*
470	 * Map out the kernel text/data/bss from the available physical
471	 * memory.
472	 */
473
474	kstart = __pa(_stext);	/* should be 0 */
475	ksize = PAGE_ALIGN(klimit - _stext);
476
477	mem_pieces_remove(&phys_avail, kstart, ksize, 0);
478	mem_pieces_remove(&phys_avail, 0, 0x4000, 0);
479
480#if defined(CONFIG_BLK_DEV_INITRD)
481	/* Remove the init RAM disk from the available memory. */
482	if (initrd_start) {
483		mem_pieces_remove(&phys_avail, __pa(initrd_start),
484				  initrd_end - initrd_start, 1);
485	}
486#endif /* CONFIG_BLK_DEV_INITRD */
487}
488
489/* Mark some memory as reserved by removing it from phys_avail. */
490void __init reserve_phys_mem(unsigned long start, unsigned long size)
491{
492	mem_pieces_remove(&phys_avail, start, size, 1);
493}
494
495/*
496 * This is called when a page has been modified by the kernel.
497 * It just marks the page as not i-cache clean.  We do the i-cache
498 * flush later when the page is given to a user process, if necessary.
499 */
500void flush_dcache_page(struct page *page)
501{
502	clear_bit(PG_arch_1, &page->flags);
503}
504
505void flush_dcache_icache_page(struct page *page)
506{
507#ifdef CONFIG_BOOKE
508	void *start = kmap_atomic(page, KM_PPC_SYNC_ICACHE);
509	__flush_dcache_icache(start);
510	kunmap_atomic(start, KM_PPC_SYNC_ICACHE);
511#elif defined(CONFIG_8xx)
512	/* On 8xx there is no need to kmap since highmem is not supported */
513	__flush_dcache_icache(page_address(page));
514#else
515	__flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
516#endif
517
518}
519void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
520{
521	clear_page(page);
522	clear_bit(PG_arch_1, &pg->flags);
523}
524
525void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
526		    struct page *pg)
527{
528	copy_page(vto, vfrom);
529	clear_bit(PG_arch_1, &pg->flags);
530}
531
532void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
533			     unsigned long addr, int len)
534{
535	unsigned long maddr;
536
537	maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
538	flush_icache_range(maddr, maddr + len);
539	kunmap(page);
540}
541
542/*
543 * This is called at the end of handling a user page fault, when the
544 * fault has been handled by updating a PTE in the linux page tables.
545 * We use it to preload an HPTE into the hash table corresponding to
546 * the updated linux PTE.
547 */
548void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
549		      pte_t pte)
550{
551	/* handle i-cache coherency */
552	unsigned long pfn = pte_pfn(pte);
553
554	if (pfn_valid(pfn)) {
555		struct page *page = pfn_to_page(pfn);
556#ifdef CONFIG_8xx
557		/* On 8xx, the TLB handlers work in 2 stages:
558	 	 * First, a zeroed entry is loaded by TLBMiss handler,
559		 * which causes the TLBError handler to be triggered.
560		 * That means the zeroed TLB has to be invalidated
561		 * whenever a page miss occurs.
562		 */
563		_tlbie(address);
564#endif
565		if (!PageReserved(page)
566		    && !test_bit(PG_arch_1, &page->flags)) {
567			if (vma->vm_mm == current->active_mm)
568				__flush_dcache_icache((void *) address);
569			else
570				flush_dcache_icache_page(page);
571			set_bit(PG_arch_1, &page->flags);
572		}
573	}
574
575#ifdef CONFIG_PPC_STD_MMU
576	/* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
577	if (Hash != 0 && pte_young(pte)) {
578		struct mm_struct *mm;
579		pmd_t *pmd;
580
581		mm = (address < TASK_SIZE)? vma->vm_mm: &init_mm;
582		pmd = pmd_offset(pgd_offset(mm, address), address);
583		if (!pmd_none(*pmd))
584			add_hash_page(mm->context.id, address, pmd_val(*pmd));
585	}
586#endif
587}
588
589/*
590 * This is called by /dev/mem to know if a given address has to
591 * be mapped non-cacheable or not
592 */
593int page_is_ram(unsigned long pfn)
594{
595	return pfn < max_pfn;
596}
597
598pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
599			      unsigned long size, pgprot_t vma_prot)
600{
601	if (ppc_md.phys_mem_access_prot)
602		return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
603
604	if (!page_is_ram(pfn))
605		vma_prot = __pgprot(pgprot_val(vma_prot)
606				    | _PAGE_GUARDED | _PAGE_NO_CACHE);
607	return vma_prot;
608}
609EXPORT_SYMBOL(phys_mem_access_prot);
610