1/*
2 *  arch/s390/mm/vmem.c
3 *
4 *    Copyright IBM Corp. 2006
5 *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
6 */
7
8#include <linux/bootmem.h>
9#include <linux/pfn.h>
10#include <linux/mm.h>
11#include <linux/module.h>
12#include <linux/list.h>
13#include <asm/pgalloc.h>
14#include <asm/pgtable.h>
15#include <asm/setup.h>
16#include <asm/tlbflush.h>
17
18unsigned long vmalloc_end;
19EXPORT_SYMBOL(vmalloc_end);
20
21static struct page *vmem_map;
22static DEFINE_MUTEX(vmem_mutex);
23
24struct memory_segment {
25	struct list_head list;
26	unsigned long start;
27	unsigned long size;
28};
29
30static LIST_HEAD(mem_segs);
31
32void memmap_init(unsigned long size, int nid, unsigned long zone,
33		 unsigned long start_pfn)
34{
35	struct page *start, *end;
36	struct page *map_start, *map_end;
37	int i;
38
39	start = pfn_to_page(start_pfn);
40	end = start + size;
41
42	for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
43		unsigned long cstart, cend;
44
45		cstart = PFN_DOWN(memory_chunk[i].addr);
46		cend = cstart + PFN_DOWN(memory_chunk[i].size);
47
48		map_start = mem_map + cstart;
49		map_end = mem_map + cend;
50
51		if (map_start < start)
52			map_start = start;
53		if (map_end > end)
54			map_end = end;
55
56		map_start -= ((unsigned long) map_start & (PAGE_SIZE - 1))
57			/ sizeof(struct page);
58		map_end += ((PFN_ALIGN((unsigned long) map_end)
59			     - (unsigned long) map_end)
60			    / sizeof(struct page));
61
62		if (map_start < map_end)
63			memmap_init_zone((unsigned long)(map_end - map_start),
64					 nid, zone, page_to_pfn(map_start),
65					 MEMMAP_EARLY);
66	}
67}
68
69static inline void *vmem_alloc_pages(unsigned int order)
70{
71	if (slab_is_available())
72		return (void *)__get_free_pages(GFP_KERNEL, order);
73	return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
74}
75
76static inline pmd_t *vmem_pmd_alloc(void)
77{
78	pmd_t *pmd;
79	int i;
80
81	pmd = vmem_alloc_pages(PMD_ALLOC_ORDER);
82	if (!pmd)
83		return NULL;
84	for (i = 0; i < PTRS_PER_PMD; i++)
85		pmd_clear_kernel(pmd + i);
86	return pmd;
87}
88
89static inline pte_t *vmem_pte_alloc(void)
90{
91	pte_t *pte;
92	pte_t empty_pte;
93	int i;
94
95	pte = vmem_alloc_pages(PTE_ALLOC_ORDER);
96	if (!pte)
97		return NULL;
98	pte_val(empty_pte) = _PAGE_TYPE_EMPTY;
99	for (i = 0; i < PTRS_PER_PTE; i++)
100		pte[i] = empty_pte;
101	return pte;
102}
103
104/*
105 * Add a physical memory range to the 1:1 mapping.
106 */
107static int vmem_add_range(unsigned long start, unsigned long size)
108{
109	unsigned long address;
110	pgd_t *pg_dir;
111	pmd_t *pm_dir;
112	pte_t *pt_dir;
113	pte_t  pte;
114	int ret = -ENOMEM;
115
116	for (address = start; address < start + size; address += PAGE_SIZE) {
117		pg_dir = pgd_offset_k(address);
118		if (pgd_none(*pg_dir)) {
119			pm_dir = vmem_pmd_alloc();
120			if (!pm_dir)
121				goto out;
122			pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
123		}
124
125		pm_dir = pmd_offset(pg_dir, address);
126		if (pmd_none(*pm_dir)) {
127			pt_dir = vmem_pte_alloc();
128			if (!pt_dir)
129				goto out;
130			pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
131		}
132
133		pt_dir = pte_offset_kernel(pm_dir, address);
134		pte = pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL);
135		*pt_dir = pte;
136	}
137	ret = 0;
138out:
139	flush_tlb_kernel_range(start, start + size);
140	return ret;
141}
142
143/*
144 * Remove a physical memory range from the 1:1 mapping.
145 * Currently only invalidates page table entries.
146 */
147static void vmem_remove_range(unsigned long start, unsigned long size)
148{
149	unsigned long address;
150	pgd_t *pg_dir;
151	pmd_t *pm_dir;
152	pte_t *pt_dir;
153	pte_t  pte;
154
155	pte_val(pte) = _PAGE_TYPE_EMPTY;
156	for (address = start; address < start + size; address += PAGE_SIZE) {
157		pg_dir = pgd_offset_k(address);
158		if (pgd_none(*pg_dir))
159			continue;
160		pm_dir = pmd_offset(pg_dir, address);
161		if (pmd_none(*pm_dir))
162			continue;
163		pt_dir = pte_offset_kernel(pm_dir, address);
164		*pt_dir = pte;
165	}
166	flush_tlb_kernel_range(start, start + size);
167}
168
169/*
170 * Add a backed mem_map array to the virtual mem_map array.
171 */
172static int vmem_add_mem_map(unsigned long start, unsigned long size)
173{
174	unsigned long address, start_addr, end_addr;
175	struct page *map_start, *map_end;
176	pgd_t *pg_dir;
177	pmd_t *pm_dir;
178	pte_t *pt_dir;
179	pte_t  pte;
180	int ret = -ENOMEM;
181
182	map_start = vmem_map + PFN_DOWN(start);
183	map_end	= vmem_map + PFN_DOWN(start + size);
184
185	start_addr = (unsigned long) map_start & PAGE_MASK;
186	end_addr = PFN_ALIGN((unsigned long) map_end);
187
188	for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
189		pg_dir = pgd_offset_k(address);
190		if (pgd_none(*pg_dir)) {
191			pm_dir = vmem_pmd_alloc();
192			if (!pm_dir)
193				goto out;
194			pgd_populate_kernel(&init_mm, pg_dir, pm_dir);
195		}
196
197		pm_dir = pmd_offset(pg_dir, address);
198		if (pmd_none(*pm_dir)) {
199			pt_dir = vmem_pte_alloc();
200			if (!pt_dir)
201				goto out;
202			pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
203		}
204
205		pt_dir = pte_offset_kernel(pm_dir, address);
206		if (pte_none(*pt_dir)) {
207			unsigned long new_page;
208
209			new_page =__pa(vmem_alloc_pages(0));
210			if (!new_page)
211				goto out;
212			pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL);
213			*pt_dir = pte;
214		}
215	}
216	ret = 0;
217out:
218	flush_tlb_kernel_range(start_addr, end_addr);
219	return ret;
220}
221
222static int vmem_add_mem(unsigned long start, unsigned long size)
223{
224	int ret;
225
226	ret = vmem_add_range(start, size);
227	if (ret)
228		return ret;
229	return vmem_add_mem_map(start, size);
230}
231
232/*
233 * Add memory segment to the segment list if it doesn't overlap with
234 * an already present segment.
235 */
236static int insert_memory_segment(struct memory_segment *seg)
237{
238	struct memory_segment *tmp;
239
240	if (PFN_DOWN(seg->start + seg->size) > max_pfn ||
241	    seg->start + seg->size < seg->start)
242		return -ERANGE;
243
244	list_for_each_entry(tmp, &mem_segs, list) {
245		if (seg->start >= tmp->start + tmp->size)
246			continue;
247		if (seg->start + seg->size <= tmp->start)
248			continue;
249		return -ENOSPC;
250	}
251	list_add(&seg->list, &mem_segs);
252	return 0;
253}
254
255/*
256 * Remove memory segment from the segment list.
257 */
258static void remove_memory_segment(struct memory_segment *seg)
259{
260	list_del(&seg->list);
261}
262
263static void __remove_shared_memory(struct memory_segment *seg)
264{
265	remove_memory_segment(seg);
266	vmem_remove_range(seg->start, seg->size);
267}
268
269int remove_shared_memory(unsigned long start, unsigned long size)
270{
271	struct memory_segment *seg;
272	int ret;
273
274	mutex_lock(&vmem_mutex);
275
276	ret = -ENOENT;
277	list_for_each_entry(seg, &mem_segs, list) {
278		if (seg->start == start && seg->size == size)
279			break;
280	}
281
282	if (seg->start != start || seg->size != size)
283		goto out;
284
285	ret = 0;
286	__remove_shared_memory(seg);
287	kfree(seg);
288out:
289	mutex_unlock(&vmem_mutex);
290	return ret;
291}
292
293int add_shared_memory(unsigned long start, unsigned long size)
294{
295	struct memory_segment *seg;
296	struct page *page;
297	unsigned long pfn, num_pfn, end_pfn;
298	int ret;
299
300	mutex_lock(&vmem_mutex);
301	ret = -ENOMEM;
302	seg = kzalloc(sizeof(*seg), GFP_KERNEL);
303	if (!seg)
304		goto out;
305	seg->start = start;
306	seg->size = size;
307
308	ret = insert_memory_segment(seg);
309	if (ret)
310		goto out_free;
311
312	ret = vmem_add_mem(start, size);
313	if (ret)
314		goto out_remove;
315
316	pfn = PFN_DOWN(start);
317	num_pfn = PFN_DOWN(size);
318	end_pfn = pfn + num_pfn;
319
320	page = pfn_to_page(pfn);
321	memset(page, 0, num_pfn * sizeof(struct page));
322
323	for (; pfn < end_pfn; pfn++) {
324		page = pfn_to_page(pfn);
325		init_page_count(page);
326		reset_page_mapcount(page);
327		SetPageReserved(page);
328		INIT_LIST_HEAD(&page->lru);
329	}
330	goto out;
331
332out_remove:
333	__remove_shared_memory(seg);
334out_free:
335	kfree(seg);
336out:
337	mutex_unlock(&vmem_mutex);
338	return ret;
339}
340
341/*
342 * map whole physical memory to virtual memory (identity mapping)
343 */
344void __init vmem_map_init(void)
345{
346	unsigned long map_size;
347	int i;
348
349	map_size = ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) * sizeof(struct page);
350	vmalloc_end = PFN_ALIGN(VMALLOC_END_INIT) - PFN_ALIGN(map_size);
351	vmem_map = (struct page *) vmalloc_end;
352	NODE_DATA(0)->node_mem_map = vmem_map;
353
354	for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++)
355		vmem_add_mem(memory_chunk[i].addr, memory_chunk[i].size);
356}
357
358/*
359 * Convert memory chunk array to a memory segment list so there is a single
360 * list that contains both r/w memory and shared memory segments.
361 */
362static int __init vmem_convert_memory_chunk(void)
363{
364	struct memory_segment *seg;
365	int i;
366
367	mutex_lock(&vmem_mutex);
368	for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
369		if (!memory_chunk[i].size)
370			continue;
371		seg = kzalloc(sizeof(*seg), GFP_KERNEL);
372		if (!seg)
373			panic("Out of memory...\n");
374		seg->start = memory_chunk[i].addr;
375		seg->size = memory_chunk[i].size;
376		insert_memory_segment(seg);
377	}
378	mutex_unlock(&vmem_mutex);
379	return 0;
380}
381
382core_initcall(vmem_convert_memory_chunk);
383