vm_kern.c (252330) | vm_kern.c (254025) |
---|---|
1/*- 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 49 unchanged lines hidden (view full) --- 58 * rights to redistribute these changes. 59 */ 60 61/* 62 * Kernel memory management. 63 */ 64 65#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without --- 49 unchanged lines hidden (view full) --- 58 * rights to redistribute these changes. 59 */ 60 61/* 62 * Kernel memory management. 63 */ 64 65#include <sys/cdefs.h> |
66__FBSDID("$FreeBSD: head/sys/vm/vm_kern.c 252330 2013-06-28 03:51:20Z jeff $"); | 66__FBSDID("$FreeBSD: head/sys/vm/vm_kern.c 254025 2013-08-07 06:21:20Z jeff $"); |
67 68#include <sys/param.h> 69#include <sys/systm.h> 70#include <sys/kernel.h> /* for ticks and hz */ 71#include <sys/eventhandler.h> 72#include <sys/lock.h> 73#include <sys/proc.h> 74#include <sys/malloc.h> 75#include <sys/rwlock.h> 76#include <sys/sysctl.h> | 67 68#include <sys/param.h> 69#include <sys/systm.h> 70#include <sys/kernel.h> /* for ticks and hz */ 71#include <sys/eventhandler.h> 72#include <sys/lock.h> 73#include <sys/proc.h> 74#include <sys/malloc.h> 75#include <sys/rwlock.h> 76#include <sys/sysctl.h> |
77#include <sys/vmem.h> |
|
77 78#include <vm/vm.h> 79#include <vm/vm_param.h> | 78 79#include <vm/vm.h> 80#include <vm/vm_param.h> |
81#include <vm/vm_kern.h> |
|
80#include <vm/pmap.h> 81#include <vm/vm_map.h> 82#include <vm/vm_object.h> 83#include <vm/vm_page.h> 84#include <vm/vm_pageout.h> 85#include <vm/vm_extern.h> 86#include <vm/uma.h> 87 88vm_map_t kernel_map; | 82#include <vm/pmap.h> 83#include <vm/vm_map.h> 84#include <vm/vm_object.h> 85#include <vm/vm_page.h> 86#include <vm/vm_pageout.h> 87#include <vm/vm_extern.h> 88#include <vm/uma.h> 89 90vm_map_t kernel_map; |
89vm_map_t kmem_map; | |
90vm_map_t exec_map; 91vm_map_t pipe_map; 92 93const void *zero_region; 94CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0); 95 96SYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CTLFLAG_RD, 97 NULL, VM_MIN_KERNEL_ADDRESS, "Min kernel address"); 98 99SYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD, 100#if defined(__arm__) || defined(__sparc64__) 101 &vm_max_kernel_address, 0, 102#else 103 NULL, VM_MAX_KERNEL_ADDRESS, 104#endif 105 "Max kernel address"); 106 107/* | 91vm_map_t exec_map; 92vm_map_t pipe_map; 93 94const void *zero_region; 95CTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0); 96 97SYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CTLFLAG_RD, 98 NULL, VM_MIN_KERNEL_ADDRESS, "Min kernel address"); 99 100SYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD, 101#if defined(__arm__) || defined(__sparc64__) 102 &vm_max_kernel_address, 0, 103#else 104 NULL, VM_MAX_KERNEL_ADDRESS, 105#endif 106 "Max kernel address"); 107 108/* |
108 * kmem_alloc_nofault: | 109 * kva_alloc: |
109 * 110 * Allocate a virtual address range with no underlying object and 111 * no initial mapping to physical memory. Any mapping from this 112 * range to physical memory must be explicitly created prior to 113 * its use, typically with pmap_qenter(). Any attempt to create 114 * a mapping on demand through vm_fault() will result in a panic. 115 */ 116vm_offset_t | 110 * 111 * Allocate a virtual address range with no underlying object and 112 * no initial mapping to physical memory. Any mapping from this 113 * range to physical memory must be explicitly created prior to 114 * its use, typically with pmap_qenter(). Any attempt to create 115 * a mapping on demand through vm_fault() will result in a panic. 116 */ 117vm_offset_t |
117kmem_alloc_nofault(map, size) 118 vm_map_t map; | 118kva_alloc(size) |
119 vm_size_t size; 120{ 121 vm_offset_t addr; | 119 vm_size_t size; 120{ 121 vm_offset_t addr; |
122 int result; | |
123 124 size = round_page(size); | 122 123 size = round_page(size); |
125 addr = vm_map_min(map); 126 result = vm_map_find(map, NULL, 0, &addr, size, VMFS_ANY_SPACE, 127 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 128 if (result != KERN_SUCCESS) { | 124 if (vmem_alloc(kernel_arena, size, M_BESTFIT | M_NOWAIT, &addr)) |
129 return (0); | 125 return (0); |
130 } | 126 |
131 return (addr); 132} 133 134/* | 127 return (addr); 128} 129 130/* |
135 * kmem_alloc_nofault_space: | 131 * kva_free: |
136 * | 132 * |
137 * Allocate a virtual address range with no underlying object and 138 * no initial mapping to physical memory within the specified 139 * address space. Any mapping from this range to physical memory 140 * must be explicitly created prior to its use, typically with 141 * pmap_qenter(). Any attempt to create a mapping on demand 142 * through vm_fault() will result in a panic. | 133 * Release a region of kernel virtual memory allocated 134 * with kva_alloc, and return the physical pages 135 * associated with that region. 136 * 137 * This routine may not block on kernel maps. |
143 */ | 138 */ |
144vm_offset_t 145kmem_alloc_nofault_space(map, size, find_space) 146 vm_map_t map; 147 vm_size_t size; 148 int find_space; 149{ | 139void 140kva_free(addr, size) |
150 vm_offset_t addr; | 141 vm_offset_t addr; |
151 int result; 152 153 size = round_page(size); 154 addr = vm_map_min(map); 155 result = vm_map_find(map, NULL, 0, &addr, size, find_space, 156 VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 157 if (result != KERN_SUCCESS) { 158 return (0); 159 } 160 return (addr); 161} 162 163/* 164 * Allocate wired-down memory in the kernel's address map 165 * or a submap. 166 */ 167vm_offset_t 168kmem_alloc(map, size) 169 vm_map_t map; | |
170 vm_size_t size; 171{ | 142 vm_size_t size; 143{ |
172 vm_offset_t addr; 173 vm_offset_t offset; | |
174 175 size = round_page(size); | 144 145 size = round_page(size); |
176 177 /* 178 * Use the kernel object for wired-down kernel pages. Assume that no 179 * region of the kernel object is referenced more than once. 180 */ 181 182 /* 183 * Locate sufficient space in the map. This will give us the final 184 * virtual address for the new memory, and thus will tell us the 185 * offset within the kernel map. 186 */ 187 vm_map_lock(map); 188 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 189 vm_map_unlock(map); 190 return (0); 191 } 192 offset = addr - VM_MIN_KERNEL_ADDRESS; 193 vm_object_reference(kernel_object); 194 vm_map_insert(map, kernel_object, offset, addr, addr + size, 195 VM_PROT_ALL, VM_PROT_ALL, 0); 196 vm_map_unlock(map); 197 198 /* 199 * And finally, mark the data as non-pageable. 200 */ 201 (void) vm_map_wire(map, addr, addr + size, 202 VM_MAP_WIRE_SYSTEM|VM_MAP_WIRE_NOHOLES); 203 204 return (addr); | 146 vmem_free(kernel_arena, addr, size); |
205} 206 207/* 208 * Allocates a region from the kernel address map and physical pages 209 * within the specified address range to the kernel object. Creates a 210 * wired mapping from this region to these pages, and returns the 211 * region's starting virtual address. The allocated pages are not 212 * necessarily physically contiguous. If M_ZERO is specified through the 213 * given flags, then the pages are zeroed before they are mapped. 214 */ 215vm_offset_t | 147} 148 149/* 150 * Allocates a region from the kernel address map and physical pages 151 * within the specified address range to the kernel object. Creates a 152 * wired mapping from this region to these pages, and returns the 153 * region's starting virtual address. The allocated pages are not 154 * necessarily physically contiguous. If M_ZERO is specified through the 155 * given flags, then the pages are zeroed before they are mapped. 156 */ 157vm_offset_t |
216kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, | 158kmem_alloc_attr(vmem_t *vmem, vm_size_t size, int flags, vm_paddr_t low, |
217 vm_paddr_t high, vm_memattr_t memattr) 218{ | 159 vm_paddr_t high, vm_memattr_t memattr) 160{ |
219 vm_object_t object = kernel_object; | 161 vm_object_t object = vmem == kmem_arena ? kmem_object : kernel_object; |
220 vm_offset_t addr; | 162 vm_offset_t addr; |
221 vm_ooffset_t end_offset, offset; | 163 vm_ooffset_t offset; |
222 vm_page_t m; 223 int pflags, tries; | 164 vm_page_t m; 165 int pflags, tries; |
166 int i; |
|
224 225 size = round_page(size); | 167 168 size = round_page(size); |
226 vm_map_lock(map); 227 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 228 vm_map_unlock(map); | 169 if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr)) |
229 return (0); | 170 return (0); |
230 } | |
231 offset = addr - VM_MIN_KERNEL_ADDRESS; | 171 offset = addr - VM_MIN_KERNEL_ADDRESS; |
232 vm_object_reference(object); 233 vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, 234 VM_PROT_ALL, 0); 235 pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY; | 172 pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; |
236 VM_OBJECT_WLOCK(object); | 173 VM_OBJECT_WLOCK(object); |
237 end_offset = offset + size; 238 for (; offset < end_offset; offset += PAGE_SIZE) { | 174 for (i = 0; i < size; i += PAGE_SIZE) { |
239 tries = 0; 240retry: | 175 tries = 0; 176retry: |
241 m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 1, 242 low, high, PAGE_SIZE, 0, memattr); | 177 m = vm_page_alloc_contig(object, OFF_TO_IDX(offset + i), 178 pflags, 1, low, high, PAGE_SIZE, 0, memattr); |
243 if (m == NULL) { 244 VM_OBJECT_WUNLOCK(object); 245 if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { | 179 if (m == NULL) { 180 VM_OBJECT_WUNLOCK(object); 181 if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { |
246 vm_map_unlock(map); | |
247 vm_pageout_grow_cache(tries, low, high); | 182 vm_pageout_grow_cache(tries, low, high); |
248 vm_map_lock(map); | |
249 VM_OBJECT_WLOCK(object); 250 tries++; 251 goto retry; 252 } | 183 VM_OBJECT_WLOCK(object); 184 tries++; 185 goto retry; 186 } |
253 254 /* 255 * Since the pages that were allocated by any previous 256 * iterations of this loop are not busy, they can be 257 * freed by vm_object_page_remove(), which is called 258 * by vm_map_delete(). | 187 /* 188 * Unmap and free the pages. |
259 */ | 189 */ |
260 vm_map_delete(map, addr, addr + size); 261 vm_map_unlock(map); | 190 if (i != 0) 191 pmap_remove(kernel_pmap, addr, addr + i); 192 while (i != 0) { 193 i -= PAGE_SIZE; 194 m = vm_page_lookup(object, 195 OFF_TO_IDX(offset + i)); 196 vm_page_unwire(m, 0); 197 vm_page_free(m); 198 } 199 vmem_free(vmem, addr, size); |
262 return (0); 263 } 264 if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 265 pmap_zero_page(m); 266 m->valid = VM_PAGE_BITS_ALL; | 200 return (0); 201 } 202 if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 203 pmap_zero_page(m); 204 m->valid = VM_PAGE_BITS_ALL; |
205 pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL, 206 TRUE); |
|
267 } 268 VM_OBJECT_WUNLOCK(object); | 207 } 208 VM_OBJECT_WUNLOCK(object); |
269 vm_map_unlock(map); 270 vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM | 271 VM_MAP_WIRE_NOHOLES); | |
272 return (addr); 273} 274 275/* 276 * Allocates a region from the kernel address map and physically 277 * contiguous pages within the specified address range to the kernel 278 * object. Creates a wired mapping from this region to these pages, and 279 * returns the region's starting virtual address. If M_ZERO is specified 280 * through the given flags, then the pages are zeroed before they are 281 * mapped. 282 */ 283vm_offset_t | 209 return (addr); 210} 211 212/* 213 * Allocates a region from the kernel address map and physically 214 * contiguous pages within the specified address range to the kernel 215 * object. Creates a wired mapping from this region to these pages, and 216 * returns the region's starting virtual address. If M_ZERO is specified 217 * through the given flags, then the pages are zeroed before they are 218 * mapped. 219 */ 220vm_offset_t |
284kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, | 221kmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low, |
285 vm_paddr_t high, u_long alignment, vm_paddr_t boundary, 286 vm_memattr_t memattr) 287{ | 222 vm_paddr_t high, u_long alignment, vm_paddr_t boundary, 223 vm_memattr_t memattr) 224{ |
288 vm_object_t object = kernel_object; 289 vm_offset_t addr; | 225 vm_object_t object = vmem == kmem_arena ? kmem_object : kernel_object; 226 vm_offset_t addr, tmp; |
290 vm_ooffset_t offset; 291 vm_page_t end_m, m; 292 int pflags, tries; 293 294 size = round_page(size); | 227 vm_ooffset_t offset; 228 vm_page_t end_m, m; 229 int pflags, tries; 230 231 size = round_page(size); |
295 vm_map_lock(map); 296 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 297 vm_map_unlock(map); | 232 if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr)) |
298 return (0); | 233 return (0); |
299 } | |
300 offset = addr - VM_MIN_KERNEL_ADDRESS; | 234 offset = addr - VM_MIN_KERNEL_ADDRESS; |
301 vm_object_reference(object); 302 vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, 303 VM_PROT_ALL, 0); 304 pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY; | 235 pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; |
305 VM_OBJECT_WLOCK(object); 306 tries = 0; 307retry: 308 m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 309 atop(size), low, high, alignment, boundary, memattr); 310 if (m == NULL) { 311 VM_OBJECT_WUNLOCK(object); 312 if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { | 236 VM_OBJECT_WLOCK(object); 237 tries = 0; 238retry: 239 m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 240 atop(size), low, high, alignment, boundary, memattr); 241 if (m == NULL) { 242 VM_OBJECT_WUNLOCK(object); 243 if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { |
313 vm_map_unlock(map); | |
314 vm_pageout_grow_cache(tries, low, high); | 244 vm_pageout_grow_cache(tries, low, high); |
315 vm_map_lock(map); | |
316 VM_OBJECT_WLOCK(object); 317 tries++; 318 goto retry; 319 } | 245 VM_OBJECT_WLOCK(object); 246 tries++; 247 goto retry; 248 } |
320 vm_map_delete(map, addr, addr + size); 321 vm_map_unlock(map); | 249 vmem_free(vmem, addr, size); |
322 return (0); 323 } 324 end_m = m + atop(size); | 250 return (0); 251 } 252 end_m = m + atop(size); |
253 tmp = addr; |
|
325 for (; m < end_m; m++) { 326 if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 327 pmap_zero_page(m); 328 m->valid = VM_PAGE_BITS_ALL; | 254 for (; m < end_m; m++) { 255 if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 256 pmap_zero_page(m); 257 m->valid = VM_PAGE_BITS_ALL; |
258 pmap_enter(kernel_pmap, tmp, VM_PROT_ALL, m, VM_PROT_ALL, true); 259 tmp += PAGE_SIZE; |
|
329 } 330 VM_OBJECT_WUNLOCK(object); | 260 } 261 VM_OBJECT_WUNLOCK(object); |
331 vm_map_unlock(map); 332 vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM | 333 VM_MAP_WIRE_NOHOLES); | |
334 return (addr); 335} 336 337/* | 262 return (addr); 263} 264 265/* |
338 * kmem_free: 339 * 340 * Release a region of kernel virtual memory allocated 341 * with kmem_alloc, and return the physical pages 342 * associated with that region. 343 * 344 * This routine may not block on kernel maps. 345 */ 346void 347kmem_free(map, addr, size) 348 vm_map_t map; 349 vm_offset_t addr; 350 vm_size_t size; 351{ 352 353 (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); 354} 355 356/* | |
357 * kmem_suballoc: 358 * 359 * Allocates a map to manage a subrange 360 * of the kernel virtual address space. 361 * 362 * Arguments are as follows: 363 * 364 * parent Map to take range from --- 23 unchanged lines hidden (view full) --- 388 if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS) 389 panic("kmem_suballoc: unable to change range to submap"); 390 return (result); 391} 392 393/* 394 * kmem_malloc: 395 * | 266 * kmem_suballoc: 267 * 268 * Allocates a map to manage a subrange 269 * of the kernel virtual address space. 270 * 271 * Arguments are as follows: 272 * 273 * parent Map to take range from --- 23 unchanged lines hidden (view full) --- 297 if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS) 298 panic("kmem_suballoc: unable to change range to submap"); 299 return (result); 300} 301 302/* 303 * kmem_malloc: 304 * |
396 * Allocate wired-down memory in the kernel's address map for the higher 397 * level kernel memory allocator (kern/kern_malloc.c). We cannot use 398 * kmem_alloc() because we may need to allocate memory at interrupt 399 * level where we cannot block (canwait == FALSE). 400 * 401 * This routine has its own private kernel submap (kmem_map) and object 402 * (kmem_object). This, combined with the fact that only malloc uses 403 * this routine, ensures that we will never block in map or object waits. 404 * 405 * We don't worry about expanding the map (adding entries) since entries 406 * for wired maps are statically allocated. 407 * 408 * `map' is ONLY allowed to be kmem_map or one of the mbuf submaps to 409 * which we never free. | 305 * Allocate wired-down pages in the kernel's address space. |
410 */ 411vm_offset_t | 306 */ 307vm_offset_t |
412kmem_malloc(map, size, flags) 413 vm_map_t map; 414 vm_size_t size; 415 int flags; | 308kmem_malloc(struct vmem *vmem, vm_size_t size, int flags) |
416{ 417 vm_offset_t addr; | 309{ 310 vm_offset_t addr; |
418 int i, rv; | 311 int rv; |
419 420 size = round_page(size); | 312 313 size = round_page(size); |
421 addr = vm_map_min(map); | 314 if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr)) 315 return (0); |
422 | 316 |
423 /* 424 * Locate sufficient space in the map. This will give us the final 425 * virtual address for the new memory, and thus will tell us the 426 * offset within the kernel map. 427 */ 428 vm_map_lock(map); 429 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 430 vm_map_unlock(map); 431 if ((flags & M_NOWAIT) == 0) { 432 for (i = 0; i < 8; i++) { 433 EVENTHANDLER_INVOKE(vm_lowmem, 0); 434 uma_reclaim(); 435 vm_map_lock(map); 436 if (vm_map_findspace(map, vm_map_min(map), 437 size, &addr) == 0) { 438 break; 439 } 440 vm_map_unlock(map); 441 tsleep(&i, 0, "nokva", (hz / 4) * (i + 1)); 442 } 443 if (i == 8) { 444 panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated", 445 (long)size, (long)map->size); 446 } 447 } else { 448 return (0); 449 } | 317 rv = kmem_back((vmem == kmem_arena) ? kmem_object : kernel_object, 318 addr, size, flags); 319 if (rv != KERN_SUCCESS) { 320 vmem_free(vmem, addr, size); 321 return (0); |
450 } | 322 } |
451 452 rv = kmem_back(map, addr, size, flags); 453 vm_map_unlock(map); 454 return (rv == KERN_SUCCESS ? addr : 0); | 323 return (addr); |
455} 456 457/* 458 * kmem_back: 459 * 460 * Allocate physical pages for the specified virtual address range. 461 */ 462int | 324} 325 326/* 327 * kmem_back: 328 * 329 * Allocate physical pages for the specified virtual address range. 330 */ 331int |
463kmem_back(vm_map_t map, vm_offset_t addr, vm_size_t size, int flags) | 332kmem_back(vm_object_t object, vm_offset_t addr, vm_size_t size, int flags) |
464{ 465 vm_offset_t offset, i; | 333{ 334 vm_offset_t offset, i; |
466 vm_map_entry_t entry; | |
467 vm_page_t m; 468 int pflags; | 335 vm_page_t m; 336 int pflags; |
469 boolean_t found; | |
470 | 337 |
471 KASSERT(vm_map_locked(map), ("kmem_back: map %p is not locked", map)); | 338 KASSERT(object == kmem_object || object == kernel_object, 339 ("kmem_back: only supports kernel objects.")); 340 |
472 offset = addr - VM_MIN_KERNEL_ADDRESS; | 341 offset = addr - VM_MIN_KERNEL_ADDRESS; |
473 vm_object_reference(kmem_object); 474 vm_map_insert(map, kmem_object, offset, addr, addr + size, 475 VM_PROT_ALL, VM_PROT_ALL, 0); | 342 pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; |
476 | 343 |
477 /* 478 * Assert: vm_map_insert() will never be able to extend the 479 * previous entry so vm_map_lookup_entry() will find a new 480 * entry exactly corresponding to this address range and it 481 * will have wired_count == 0. 482 */ 483 found = vm_map_lookup_entry(map, addr, &entry); 484 KASSERT(found && entry->start == addr && entry->end == addr + size && 485 entry->wired_count == 0 && (entry->eflags & MAP_ENTRY_IN_TRANSITION) 486 == 0, ("kmem_back: entry not found or misaligned")); 487 488 pflags = malloc2vm_flags(flags) | VM_ALLOC_WIRED; 489 490 VM_OBJECT_WLOCK(kmem_object); | 344 VM_OBJECT_WLOCK(object); |
491 for (i = 0; i < size; i += PAGE_SIZE) { 492retry: | 345 for (i = 0; i < size; i += PAGE_SIZE) { 346retry: |
493 m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), pflags); | 347 m = vm_page_alloc(object, OFF_TO_IDX(offset + i), pflags); |
494 495 /* 496 * Ran out of space, free everything up and return. Don't need 497 * to lock page queues here as we know that the pages we got 498 * aren't on any queues. 499 */ 500 if (m == NULL) { 501 if ((flags & M_NOWAIT) == 0) { | 348 349 /* 350 * Ran out of space, free everything up and return. Don't need 351 * to lock page queues here as we know that the pages we got 352 * aren't on any queues. 353 */ 354 if (m == NULL) { 355 if ((flags & M_NOWAIT) == 0) { |
502 VM_OBJECT_WUNLOCK(kmem_object); 503 entry->eflags |= MAP_ENTRY_IN_TRANSITION; 504 vm_map_unlock(map); | 356 VM_OBJECT_WUNLOCK(object); |
505 VM_WAIT; | 357 VM_WAIT; |
506 vm_map_lock(map); 507 KASSERT( 508(entry->eflags & (MAP_ENTRY_IN_TRANSITION | MAP_ENTRY_NEEDS_WAKEUP)) == 509 MAP_ENTRY_IN_TRANSITION, 510 ("kmem_back: volatile entry")); 511 entry->eflags &= ~MAP_ENTRY_IN_TRANSITION; 512 VM_OBJECT_WLOCK(kmem_object); | 358 VM_OBJECT_WLOCK(object); |
513 goto retry; 514 } 515 /* | 359 goto retry; 360 } 361 /* |
516 * Free the pages before removing the map entry. 517 * They are already marked busy. Calling 518 * vm_map_delete before the pages has been freed or 519 * unbusied will cause a deadlock. | 362 * Unmap and free the pages. |
520 */ | 363 */ |
364 if (i != 0) 365 pmap_remove(kernel_pmap, addr, addr + i); |
|
521 while (i != 0) { 522 i -= PAGE_SIZE; | 366 while (i != 0) { 367 i -= PAGE_SIZE; |
523 m = vm_page_lookup(kmem_object, | 368 m = vm_page_lookup(object, |
524 OFF_TO_IDX(offset + i)); 525 vm_page_unwire(m, 0); 526 vm_page_free(m); 527 } | 369 OFF_TO_IDX(offset + i)); 370 vm_page_unwire(m, 0); 371 vm_page_free(m); 372 } |
528 VM_OBJECT_WUNLOCK(kmem_object); 529 vm_map_delete(map, addr, addr + size); | 373 VM_OBJECT_WUNLOCK(object); |
530 return (KERN_NO_SPACE); 531 } 532 if (flags & M_ZERO && (m->flags & PG_ZERO) == 0) 533 pmap_zero_page(m); | 374 return (KERN_NO_SPACE); 375 } 376 if (flags & M_ZERO && (m->flags & PG_ZERO) == 0) 377 pmap_zero_page(m); |
534 m->valid = VM_PAGE_BITS_ALL; | |
535 KASSERT((m->oflags & VPO_UNMANAGED) != 0, 536 ("kmem_malloc: page %p is managed", m)); | 378 KASSERT((m->oflags & VPO_UNMANAGED) != 0, 379 ("kmem_malloc: page %p is managed", m)); |
380 m->valid = VM_PAGE_BITS_ALL; 381 pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL, 382 TRUE); |
|
537 } | 383 } |
538 VM_OBJECT_WUNLOCK(kmem_object); | 384 VM_OBJECT_WUNLOCK(object); |
539 | 385 |
540 /* 541 * Mark map entry as non-pageable. Repeat the assert. 542 */ 543 KASSERT(entry->start == addr && entry->end == addr + size && 544 entry->wired_count == 0, 545 ("kmem_back: entry not found or misaligned after allocation")); 546 entry->wired_count = 1; | 386 return (KERN_SUCCESS); 387} |
547 | 388 |
548 /* 549 * At this point, the kmem_object must be unlocked because 550 * vm_map_simplify_entry() calls vm_object_deallocate(), which 551 * locks the kmem_object. 552 */ 553 vm_map_simplify_entry(map, entry); | 389void 390kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size) 391{ 392 vm_page_t m; 393 vm_offset_t offset; 394 int i; |
554 | 395 |
555 /* 556 * Loop thru pages, entering them in the pmap. 557 */ 558 VM_OBJECT_WLOCK(kmem_object); | 396 KASSERT(object == kmem_object || object == kernel_object, 397 ("kmem_unback: only supports kernel objects.")); 398 399 offset = addr - VM_MIN_KERNEL_ADDRESS; 400 VM_OBJECT_WLOCK(object); 401 pmap_remove(kernel_pmap, addr, addr + size); |
559 for (i = 0; i < size; i += PAGE_SIZE) { | 402 for (i = 0; i < size; i += PAGE_SIZE) { |
560 m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i)); 561 /* 562 * Because this is kernel_pmap, this call will not block. 563 */ 564 pmap_enter(kernel_pmap, addr + i, VM_PROT_ALL, m, VM_PROT_ALL, 565 TRUE); 566 vm_page_wakeup(m); | 403 m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); 404 vm_page_unwire(m, 0); 405 vm_page_free(m); |
567 } | 406 } |
568 VM_OBJECT_WUNLOCK(kmem_object); | 407 VM_OBJECT_WUNLOCK(object); 408} |
569 | 409 |
570 return (KERN_SUCCESS); | 410/* 411 * kmem_free: 412 * 413 * Free memory allocated with kmem_malloc. The size must match the 414 * original allocation. 415 */ 416void 417kmem_free(struct vmem *vmem, vm_offset_t addr, vm_size_t size) 418{ 419 420 size = round_page(size); 421 kmem_unback((vmem == kmem_arena) ? kmem_object : kernel_object, 422 addr, size); 423 vmem_free(vmem, addr, size); |
571} 572 573/* | 424} 425 426/* |
574 * kmem_alloc_wait: | 427 * kmap_alloc_wait: |
575 * 576 * Allocates pageable memory from a sub-map of the kernel. If the submap 577 * has no room, the caller sleeps waiting for more memory in the submap. 578 * 579 * This routine may block. 580 */ 581vm_offset_t | 428 * 429 * Allocates pageable memory from a sub-map of the kernel. If the submap 430 * has no room, the caller sleeps waiting for more memory in the submap. 431 * 432 * This routine may block. 433 */ 434vm_offset_t |
582kmem_alloc_wait(map, size) | 435kmap_alloc_wait(map, size) |
583 vm_map_t map; 584 vm_size_t size; 585{ 586 vm_offset_t addr; 587 588 size = round_page(size); 589 if (!swap_reserve(size)) 590 return (0); --- 17 unchanged lines hidden (view full) --- 608 } 609 vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL, 610 VM_PROT_ALL, MAP_ACC_CHARGED); 611 vm_map_unlock(map); 612 return (addr); 613} 614 615/* | 436 vm_map_t map; 437 vm_size_t size; 438{ 439 vm_offset_t addr; 440 441 size = round_page(size); 442 if (!swap_reserve(size)) 443 return (0); --- 17 unchanged lines hidden (view full) --- 461 } 462 vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL, 463 VM_PROT_ALL, MAP_ACC_CHARGED); 464 vm_map_unlock(map); 465 return (addr); 466} 467 468/* |
616 * kmem_free_wakeup: | 469 * kmap_free_wakeup: |
617 * 618 * Returns memory to a submap of the kernel, and wakes up any processes 619 * waiting for memory in that map. 620 */ 621void | 470 * 471 * Returns memory to a submap of the kernel, and wakes up any processes 472 * waiting for memory in that map. 473 */ 474void |
622kmem_free_wakeup(map, addr, size) | 475kmap_free_wakeup(map, addr, size) |
623 vm_map_t map; 624 vm_offset_t addr; 625 vm_size_t size; 626{ 627 628 vm_map_lock(map); 629 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 630 if (map->needs_wakeup) { 631 map->needs_wakeup = FALSE; 632 vm_map_wakeup(map); 633 } 634 vm_map_unlock(map); 635} 636 | 476 vm_map_t map; 477 vm_offset_t addr; 478 vm_size_t size; 479{ 480 481 vm_map_lock(map); 482 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 483 if (map->needs_wakeup) { 484 map->needs_wakeup = FALSE; 485 vm_map_wakeup(map); 486 } 487 vm_map_unlock(map); 488} 489 |
637static void | 490void |
638kmem_init_zero_region(void) 639{ 640 vm_offset_t addr, i; 641 vm_page_t m; | 491kmem_init_zero_region(void) 492{ 493 vm_offset_t addr, i; 494 vm_page_t m; |
642 int error; | |
643 644 /* 645 * Map a single physical page of zeros to a larger virtual range. 646 * This requires less looping in places that want large amounts of 647 * zeros, while not using much more physical resources. 648 */ | 495 496 /* 497 * Map a single physical page of zeros to a larger virtual range. 498 * This requires less looping in places that want large amounts of 499 * zeros, while not using much more physical resources. 500 */ |
649 addr = kmem_alloc_nofault(kernel_map, ZERO_REGION_SIZE); | 501 addr = kva_alloc(ZERO_REGION_SIZE); |
650 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 651 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 652 if ((m->flags & PG_ZERO) == 0) 653 pmap_zero_page(m); 654 for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE) 655 pmap_qenter(addr + i, &m, 1); | 502 m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 503 VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 504 if ((m->flags & PG_ZERO) == 0) 505 pmap_zero_page(m); 506 for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE) 507 pmap_qenter(addr + i, &m, 1); |
656 error = vm_map_protect(kernel_map, addr, addr + ZERO_REGION_SIZE, 657 VM_PROT_READ, TRUE); 658 KASSERT(error == 0, ("error=%d", error)); | 508 pmap_protect(kernel_pmap, addr, addr + ZERO_REGION_SIZE, VM_PROT_READ); |
659 660 zero_region = (const void *)addr; 661} 662 663/* 664 * kmem_init: 665 * 666 * Create the kernel map; insert a mapping covering kernel text, --- 16 unchanged lines hidden (view full) --- 683#ifdef __amd64__ 684 KERNBASE, 685#else 686 VM_MIN_KERNEL_ADDRESS, 687#endif 688 start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 689 /* ... and ending with the completion of the above `insert' */ 690 vm_map_unlock(m); | 509 510 zero_region = (const void *)addr; 511} 512 513/* 514 * kmem_init: 515 * 516 * Create the kernel map; insert a mapping covering kernel text, --- 16 unchanged lines hidden (view full) --- 533#ifdef __amd64__ 534 KERNBASE, 535#else 536 VM_MIN_KERNEL_ADDRESS, 537#endif 538 start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 539 /* ... and ending with the completion of the above `insert' */ 540 vm_map_unlock(m); |
691 692 kmem_init_zero_region(); | |
693} 694 695#ifdef DIAGNOSTIC 696/* 697 * Allow userspace to directly trigger the VM drain routine for testing 698 * purposes. 699 */ 700static int --- 16 unchanged lines hidden --- | 541} 542 543#ifdef DIAGNOSTIC 544/* 545 * Allow userspace to directly trigger the VM drain routine for testing 546 * purposes. 547 */ 548static int --- 16 unchanged lines hidden --- |