1/* 2 * arch/s390/mm/vmem.c 3 * 4 * Copyright IBM Corp. 2006 5 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 6 */ 7 8#include <linux/bootmem.h> 9#include <linux/pfn.h> 10#include <linux/mm.h> 11#include <linux/module.h> 12#include <linux/list.h> 13#include <linux/hugetlb.h> 14#include <linux/slab.h> 15#include <asm/pgalloc.h> 16#include <asm/pgtable.h> 17#include <asm/setup.h> 18#include <asm/tlbflush.h> 19#include <asm/sections.h> 20 21static DEFINE_MUTEX(vmem_mutex); 22 23struct memory_segment { 24 struct list_head list; 25 unsigned long start; 26 unsigned long size; 27}; 28 29static LIST_HEAD(mem_segs); 30 31static void __ref *vmem_alloc_pages(unsigned int order) 32{ 33 if (slab_is_available()) 34 return (void *)__get_free_pages(GFP_KERNEL, order); 35 return alloc_bootmem_pages((1 << order) * PAGE_SIZE); 36} 37 38static inline pud_t *vmem_pud_alloc(void) 39{ 40 pud_t *pud = NULL; 41 42#ifdef CONFIG_64BIT 43 pud = vmem_alloc_pages(2); 44 if (!pud) 45 return NULL; 46 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 47#endif 48 return pud; 49} 50 51static inline pmd_t *vmem_pmd_alloc(void) 52{ 53 pmd_t *pmd = NULL; 54 55#ifdef CONFIG_64BIT 56 pmd = vmem_alloc_pages(2); 57 if (!pmd) 58 return NULL; 59 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 60#endif 61 return pmd; 62} 63 64static pte_t __ref *vmem_pte_alloc(void) 65{ 66 pte_t *pte; 67 68 if (slab_is_available()) 69 pte = (pte_t *) page_table_alloc(&init_mm); 70 else 71 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); 72 if (!pte) 73 return NULL; 74 clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, 75 PTRS_PER_PTE * sizeof(pte_t)); 76 return pte; 77} 78 79/* 80 * Add a physical memory range to the 1:1 mapping. 81 */ 82static int vmem_add_mem(unsigned long start, unsigned long size, int ro) 83{ 84 unsigned long address; 85 pgd_t *pg_dir; 86 pud_t *pu_dir; 87 pmd_t *pm_dir; 88 pte_t *pt_dir; 89 pte_t pte; 90 int ret = -ENOMEM; 91 92 for (address = start; address < start + size; address += PAGE_SIZE) { 93 pg_dir = pgd_offset_k(address); 94 if (pgd_none(*pg_dir)) { 95 pu_dir = vmem_pud_alloc(); 96 if (!pu_dir) 97 goto out; 98 pgd_populate_kernel(&init_mm, pg_dir, pu_dir); 99 } 100 101 pu_dir = pud_offset(pg_dir, address); 102 if (pud_none(*pu_dir)) { 103 pm_dir = vmem_pmd_alloc(); 104 if (!pm_dir) 105 goto out; 106 pud_populate_kernel(&init_mm, pu_dir, pm_dir); 107 } 108 109 pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0)); 110 pm_dir = pmd_offset(pu_dir, address); 111 112#ifdef __s390x__ 113 if (MACHINE_HAS_HPAGE && !(address & ~HPAGE_MASK) && 114 (address + HPAGE_SIZE <= start + size) && 115 (address >= HPAGE_SIZE)) { 116 pte_val(pte) |= _SEGMENT_ENTRY_LARGE; 117 pmd_val(*pm_dir) = pte_val(pte); 118 address += HPAGE_SIZE - PAGE_SIZE; 119 continue; 120 } 121#endif 122 if (pmd_none(*pm_dir)) { 123 pt_dir = vmem_pte_alloc(); 124 if (!pt_dir) 125 goto out; 126 pmd_populate_kernel(&init_mm, pm_dir, pt_dir); 127 } 128 129 pt_dir = pte_offset_kernel(pm_dir, address); 130 *pt_dir = pte; 131 } 132 ret = 0; 133out: 134 flush_tlb_kernel_range(start, start + size); 135 return ret; 136} 137 138/* 139 * Remove a physical memory range from the 1:1 mapping. 140 * Currently only invalidates page table entries. 141 */ 142static void vmem_remove_range(unsigned long start, unsigned long size) 143{ 144 unsigned long address; 145 pgd_t *pg_dir; 146 pud_t *pu_dir; 147 pmd_t *pm_dir; 148 pte_t *pt_dir; 149 pte_t pte; 150 151 pte_val(pte) = _PAGE_TYPE_EMPTY; 152 for (address = start; address < start + size; address += PAGE_SIZE) { 153 pg_dir = pgd_offset_k(address); 154 pu_dir = pud_offset(pg_dir, address); 155 if (pud_none(*pu_dir)) 156 continue; 157 pm_dir = pmd_offset(pu_dir, address); 158 if (pmd_none(*pm_dir)) 159 continue; 160 161 if (pmd_huge(*pm_dir)) { 162 pmd_clear_kernel(pm_dir); 163 address += HPAGE_SIZE - PAGE_SIZE; 164 continue; 165 } 166 167 pt_dir = pte_offset_kernel(pm_dir, address); 168 *pt_dir = pte; 169 } 170 flush_tlb_kernel_range(start, start + size); 171} 172 173/* 174 * Add a backed mem_map array to the virtual mem_map array. 175 */ 176int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) 177{ 178 unsigned long address, start_addr, end_addr; 179 pgd_t *pg_dir; 180 pud_t *pu_dir; 181 pmd_t *pm_dir; 182 pte_t *pt_dir; 183 pte_t pte; 184 int ret = -ENOMEM; 185 186 start_addr = (unsigned long) start; 187 end_addr = (unsigned long) (start + nr); 188 189 for (address = start_addr; address < end_addr; address += PAGE_SIZE) { 190 pg_dir = pgd_offset_k(address); 191 if (pgd_none(*pg_dir)) { 192 pu_dir = vmem_pud_alloc(); 193 if (!pu_dir) 194 goto out; 195 pgd_populate_kernel(&init_mm, pg_dir, pu_dir); 196 } 197 198 pu_dir = pud_offset(pg_dir, address); 199 if (pud_none(*pu_dir)) { 200 pm_dir = vmem_pmd_alloc(); 201 if (!pm_dir) 202 goto out; 203 pud_populate_kernel(&init_mm, pu_dir, pm_dir); 204 } 205 206 pm_dir = pmd_offset(pu_dir, address); 207 if (pmd_none(*pm_dir)) { 208 pt_dir = vmem_pte_alloc(); 209 if (!pt_dir) 210 goto out; 211 pmd_populate_kernel(&init_mm, pm_dir, pt_dir); 212 } 213 214 pt_dir = pte_offset_kernel(pm_dir, address); 215 if (pte_none(*pt_dir)) { 216 unsigned long new_page; 217 218 new_page =__pa(vmem_alloc_pages(0)); 219 if (!new_page) 220 goto out; 221 pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL); 222 *pt_dir = pte; 223 } 224 } 225 memset(start, 0, nr * sizeof(struct page)); 226 ret = 0; 227out: 228 flush_tlb_kernel_range(start_addr, end_addr); 229 return ret; 230} 231 232/* 233 * Add memory segment to the segment list if it doesn't overlap with 234 * an already present segment. 235 */ 236static int insert_memory_segment(struct memory_segment *seg) 237{ 238 struct memory_segment *tmp; 239 240 if (seg->start + seg->size > VMEM_MAX_PHYS || 241 seg->start + seg->size < seg->start) 242 return -ERANGE; 243 244 list_for_each_entry(tmp, &mem_segs, list) { 245 if (seg->start >= tmp->start + tmp->size) 246 continue; 247 if (seg->start + seg->size <= tmp->start) 248 continue; 249 return -ENOSPC; 250 } 251 list_add(&seg->list, &mem_segs); 252 return 0; 253} 254 255/* 256 * Remove memory segment from the segment list. 257 */ 258static void remove_memory_segment(struct memory_segment *seg) 259{ 260 list_del(&seg->list); 261} 262 263static void __remove_shared_memory(struct memory_segment *seg) 264{ 265 remove_memory_segment(seg); 266 vmem_remove_range(seg->start, seg->size); 267} 268 269int vmem_remove_mapping(unsigned long start, unsigned long size) 270{ 271 struct memory_segment *seg; 272 int ret; 273 274 mutex_lock(&vmem_mutex); 275 276 ret = -ENOENT; 277 list_for_each_entry(seg, &mem_segs, list) { 278 if (seg->start == start && seg->size == size) 279 break; 280 } 281 282 if (seg->start != start || seg->size != size) 283 goto out; 284 285 ret = 0; 286 __remove_shared_memory(seg); 287 kfree(seg); 288out: 289 mutex_unlock(&vmem_mutex); 290 return ret; 291} 292 293int vmem_add_mapping(unsigned long start, unsigned long size) 294{ 295 struct memory_segment *seg; 296 int ret; 297 298 mutex_lock(&vmem_mutex); 299 ret = -ENOMEM; 300 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 301 if (!seg) 302 goto out; 303 seg->start = start; 304 seg->size = size; 305 306 ret = insert_memory_segment(seg); 307 if (ret) 308 goto out_free; 309 310 ret = vmem_add_mem(start, size, 0); 311 if (ret) 312 goto out_remove; 313 goto out; 314 315out_remove: 316 __remove_shared_memory(seg); 317out_free: 318 kfree(seg); 319out: 320 mutex_unlock(&vmem_mutex); 321 return ret; 322} 323 324/* 325 * map whole physical memory to virtual memory (identity mapping) 326 * we reserve enough space in the vmalloc area for vmemmap to hotplug 327 * additional memory segments. 328 */ 329void __init vmem_map_init(void) 330{ 331 unsigned long ro_start, ro_end; 332 unsigned long start, end; 333 int i; 334 335 ro_start = ((unsigned long)&_stext) & PAGE_MASK; 336 ro_end = PFN_ALIGN((unsigned long)&_eshared); 337 for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) { 338 start = memory_chunk[i].addr; 339 end = memory_chunk[i].addr + memory_chunk[i].size; 340 if (start >= ro_end || end <= ro_start) 341 vmem_add_mem(start, end - start, 0); 342 else if (start >= ro_start && end <= ro_end) 343 vmem_add_mem(start, end - start, 1); 344 else if (start >= ro_start) { 345 vmem_add_mem(start, ro_end - start, 1); 346 vmem_add_mem(ro_end, end - ro_end, 0); 347 } else if (end < ro_end) { 348 vmem_add_mem(start, ro_start - start, 0); 349 vmem_add_mem(ro_start, end - ro_start, 1); 350 } else { 351 vmem_add_mem(start, ro_start - start, 0); 352 vmem_add_mem(ro_start, ro_end - ro_start, 1); 353 vmem_add_mem(ro_end, end - ro_end, 0); 354 } 355 } 356} 357 358/* 359 * Convert memory chunk array to a memory segment list so there is a single 360 * list that contains both r/w memory and shared memory segments. 361 */ 362static int __init vmem_convert_memory_chunk(void) 363{ 364 struct memory_segment *seg; 365 int i; 366 367 mutex_lock(&vmem_mutex); 368 for (i = 0; i < MEMORY_CHUNKS; i++) { 369 if (!memory_chunk[i].size) 370 continue; 371 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 372 if (!seg) 373 panic("Out of memory...\n"); 374 seg->start = memory_chunk[i].addr; 375 seg->size = memory_chunk[i].size; 376 insert_memory_segment(seg); 377 } 378 mutex_unlock(&vmem_mutex); 379 return 0; 380} 381 382core_initcall(vmem_convert_memory_chunk); 383