1/* 2 * Copyright (c) 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 37 * 38 * 39 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 40 * All rights reserved. 41 * 42 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 43 * 44 * Permission to use, copy, modify and distribute this software and 45 * its documentation is hereby granted, provided that both the copyright 46 * notice and this permission notice appear in all copies of the 47 * software, derivative works or modified versions, and any portions 48 * thereof, and that both notices appear in supporting documentation. 49 * 50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 53 * 54 * Carnegie Mellon requests users of this software to return to 55 * 56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 57 * School of Computer Science 58 * Carnegie Mellon University 59 * Pittsburgh PA 15213-3890 60 * 61 * any improvements or extensions that they make and grant Carnegie the 62 * rights to redistribute these changes. 63 */ 64 65/* 66 * Kernel memory management. 67 */ 68 69#include <sys/cdefs.h>
|
71 72#include <sys/param.h> 73#include <sys/systm.h> 74#include <sys/kernel.h> /* for ticks and hz */ 75#include <sys/lock.h> 76#include <sys/mutex.h> 77#include <sys/proc.h> 78#include <sys/malloc.h> 79 80#include <vm/vm.h> 81#include <vm/vm_param.h> 82#include <vm/pmap.h> 83#include <vm/vm_map.h> 84#include <vm/vm_object.h> 85#include <vm/vm_page.h> 86#include <vm/vm_pageout.h> 87#include <vm/vm_extern.h> 88 89vm_map_t kernel_map=0; 90vm_map_t kmem_map=0; 91vm_map_t exec_map=0; 92vm_map_t pipe_map; 93vm_map_t clean_map=0; 94vm_map_t buffer_map=0; 95 96/* 97 * kmem_alloc_pageable: 98 * 99 * Allocate pageable memory to the kernel's address map. 100 * "map" must be kernel_map or a submap of kernel_map. 101 */ 102vm_offset_t 103kmem_alloc_pageable(map, size) 104 vm_map_t map; 105 vm_size_t size; 106{ 107 vm_offset_t addr; 108 int result; 109 110 size = round_page(size); 111 addr = vm_map_min(map); 112 result = vm_map_find(map, NULL, 0, 113 &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0); 114 if (result != KERN_SUCCESS) { 115 return (0); 116 } 117 return (addr); 118} 119 120/* 121 * kmem_alloc_nofault: 122 * 123 * Allocate a virtual address range with no underlying object and 124 * no initial mapping to physical memory. Any mapping from this 125 * range to physical memory must be explicitly created prior to 126 * its use, typically with pmap_qenter(). Any attempt to create 127 * a mapping on demand through vm_fault() will result in a panic. 128 */ 129vm_offset_t 130kmem_alloc_nofault(map, size) 131 vm_map_t map; 132 vm_size_t size; 133{ 134 vm_offset_t addr; 135 int result; 136 137 size = round_page(size); 138 addr = vm_map_min(map); 139 result = vm_map_find(map, NULL, 0, 140 &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 141 if (result != KERN_SUCCESS) { 142 return (0); 143 } 144 return (addr); 145} 146 147/* 148 * Allocate wired-down memory in the kernel's address map 149 * or a submap. 150 */ 151vm_offset_t 152kmem_alloc(map, size) 153 vm_map_t map; 154 vm_size_t size; 155{ 156 vm_offset_t addr; 157 vm_offset_t offset; 158 vm_offset_t i; 159 160 size = round_page(size); 161 162 /* 163 * Use the kernel object for wired-down kernel pages. Assume that no 164 * region of the kernel object is referenced more than once. 165 */ 166 167 /* 168 * Locate sufficient space in the map. This will give us the final 169 * virtual address for the new memory, and thus will tell us the 170 * offset within the kernel map. 171 */ 172 vm_map_lock(map); 173 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 174 vm_map_unlock(map); 175 return (0); 176 } 177 offset = addr - VM_MIN_KERNEL_ADDRESS; 178 vm_object_reference(kernel_object); 179 vm_map_insert(map, kernel_object, offset, addr, addr + size, 180 VM_PROT_ALL, VM_PROT_ALL, 0); 181 vm_map_unlock(map); 182 183 /* 184 * Guarantee that there are pages already in this object before 185 * calling vm_map_pageable. This is to prevent the following 186 * scenario: 187 * 188 * 1) Threads have swapped out, so that there is a pager for the 189 * kernel_object. 2) The kmsg zone is empty, and so we are 190 * kmem_allocing a new page for it. 3) vm_map_pageable calls vm_fault; 191 * there is no page, but there is a pager, so we call 192 * pager_data_request. But the kmsg zone is empty, so we must 193 * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when 194 * we get the data back from the pager, it will be (very stale) 195 * non-zero data. kmem_alloc is defined to return zero-filled memory. 196 * 197 * We're intentionally not activating the pages we allocate to prevent a 198 * race with page-out. vm_map_pageable will wire the pages. 199 */ 200 for (i = 0; i < size; i += PAGE_SIZE) { 201 vm_page_t mem; 202 203 VM_OBJECT_LOCK(kernel_object); 204 mem = vm_page_grab(kernel_object, OFF_TO_IDX(offset + i), 205 VM_ALLOC_ZERO | VM_ALLOC_RETRY); 206 VM_OBJECT_UNLOCK(kernel_object); 207 if ((mem->flags & PG_ZERO) == 0) 208 pmap_zero_page(mem); 209 vm_page_lock_queues(); 210 mem->valid = VM_PAGE_BITS_ALL; 211 vm_page_flag_clear(mem, PG_ZERO); 212 vm_page_wakeup(mem); 213 vm_page_unlock_queues(); 214 } 215 216 /* 217 * And finally, mark the data as non-pageable. 218 */
|
221 222 return (addr); 223} 224 225/* 226 * kmem_free: 227 * 228 * Release a region of kernel virtual memory allocated 229 * with kmem_alloc, and return the physical pages 230 * associated with that region. 231 * 232 * This routine may not block on kernel maps. 233 */ 234void 235kmem_free(map, addr, size) 236 vm_map_t map; 237 vm_offset_t addr; 238 vm_size_t size; 239{ 240 241 (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); 242} 243 244/* 245 * kmem_suballoc: 246 * 247 * Allocates a map to manage a subrange 248 * of the kernel virtual address space. 249 * 250 * Arguments are as follows: 251 * 252 * parent Map to take range from 253 * min, max Returned endpoints of map 254 * size Size of range to find 255 */ 256vm_map_t 257kmem_suballoc(parent, min, max, size) 258 vm_map_t parent; 259 vm_offset_t *min, *max; 260 vm_size_t size; 261{ 262 int ret; 263 vm_map_t result; 264 265 GIANT_REQUIRED; 266 267 size = round_page(size); 268 269 *min = (vm_offset_t) vm_map_min(parent); 270 ret = vm_map_find(parent, NULL, (vm_offset_t) 0, 271 min, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0); 272 if (ret != KERN_SUCCESS) { 273 printf("kmem_suballoc: bad status return of %d.\n", ret); 274 panic("kmem_suballoc"); 275 } 276 *max = *min + size; 277 result = vm_map_create(vm_map_pmap(parent), *min, *max); 278 if (result == NULL) 279 panic("kmem_suballoc: cannot create submap"); 280 if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS) 281 panic("kmem_suballoc: unable to change range to submap"); 282 return (result); 283} 284 285/* 286 * kmem_malloc: 287 * 288 * Allocate wired-down memory in the kernel's address map for the higher 289 * level kernel memory allocator (kern/kern_malloc.c). We cannot use 290 * kmem_alloc() because we may need to allocate memory at interrupt 291 * level where we cannot block (canwait == FALSE). 292 * 293 * This routine has its own private kernel submap (kmem_map) and object 294 * (kmem_object). This, combined with the fact that only malloc uses 295 * this routine, ensures that we will never block in map or object waits. 296 * 297 * Note that this still only works in a uni-processor environment and 298 * when called at splhigh(). 299 * 300 * We don't worry about expanding the map (adding entries) since entries 301 * for wired maps are statically allocated. 302 * 303 * NOTE: This routine is not supposed to block if M_NOWAIT is set, but 304 * I have not verified that it actually does not block. 305 * 306 * `map' is ONLY allowed to be kmem_map or one of the mbuf submaps to 307 * which we never free. 308 */ 309vm_offset_t 310kmem_malloc(map, size, flags) 311 vm_map_t map; 312 vm_size_t size; 313 int flags; 314{ 315 vm_offset_t offset, i; 316 vm_map_entry_t entry; 317 vm_offset_t addr; 318 vm_page_t m; 319 int pflags; 320 321 size = round_page(size); 322 addr = vm_map_min(map); 323 324 /* 325 * Locate sufficient space in the map. This will give us the final 326 * virtual address for the new memory, and thus will tell us the 327 * offset within the kernel map. 328 */ 329 vm_map_lock(map); 330 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 331 vm_map_unlock(map); 332 if (map != kmem_map) { 333 static int last_report; /* when we did it (in ticks) */ 334 if (ticks < last_report || 335 (ticks - last_report) >= hz) { 336 last_report = ticks; 337 printf("Out of mbuf address space!\n"); 338 printf("Consider increasing NMBCLUSTERS\n"); 339 } 340 return (0); 341 } 342 if ((flags & M_NOWAIT) == 0) 343 panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated", 344 (long)size, (long)map->size); 345 return (0); 346 } 347 offset = addr - VM_MIN_KERNEL_ADDRESS; 348 vm_object_reference(kmem_object); 349 vm_map_insert(map, kmem_object, offset, addr, addr + size, 350 VM_PROT_ALL, VM_PROT_ALL, 0); 351 352 /* 353 * Note: if M_NOWAIT specified alone, allocate from 354 * interrupt-safe queues only (just the free list). If 355 * M_USE_RESERVE is also specified, we can also 356 * allocate from the cache. Neither of the latter two 357 * flags may be specified from an interrupt since interrupts 358 * are not allowed to mess with the cache queue. 359 */ 360 361 if ((flags & (M_NOWAIT|M_USE_RESERVE)) == M_NOWAIT) 362 pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; 363 else 364 pflags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED; 365 366 if (flags & M_ZERO) 367 pflags |= VM_ALLOC_ZERO; 368 369 VM_OBJECT_LOCK(kmem_object); 370 for (i = 0; i < size; i += PAGE_SIZE) { 371retry: 372 m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), pflags); 373 374 /* 375 * Ran out of space, free everything up and return. Don't need 376 * to lock page queues here as we know that the pages we got 377 * aren't on any queues. 378 */ 379 if (m == NULL) { 380 if ((flags & M_NOWAIT) == 0) { 381 VM_OBJECT_UNLOCK(kmem_object); 382 vm_map_unlock(map); 383 VM_WAIT; 384 vm_map_lock(map); 385 VM_OBJECT_LOCK(kmem_object); 386 goto retry; 387 } 388 /* 389 * Free the pages before removing the map entry. 390 * They are already marked busy. Calling 391 * vm_map_delete before the pages has been freed or 392 * unbusied will cause a deadlock. 393 */ 394 while (i != 0) { 395 i -= PAGE_SIZE; 396 m = vm_page_lookup(kmem_object, 397 OFF_TO_IDX(offset + i)); 398 vm_page_lock_queues(); 399 vm_page_unwire(m, 0); 400 vm_page_free(m); 401 vm_page_unlock_queues(); 402 } 403 VM_OBJECT_UNLOCK(kmem_object); 404 vm_map_delete(map, addr, addr + size); 405 vm_map_unlock(map); 406 return (0); 407 } 408 if (flags & M_ZERO && (m->flags & PG_ZERO) == 0) 409 pmap_zero_page(m); 410 vm_page_lock_queues(); 411 vm_page_flag_clear(m, PG_ZERO); 412 m->valid = VM_PAGE_BITS_ALL; 413 vm_page_unlock_queues(); 414 } 415 VM_OBJECT_UNLOCK(kmem_object); 416 417 /* 418 * Mark map entry as non-pageable. Assert: vm_map_insert() will never 419 * be able to extend the previous entry so there will be a new entry 420 * exactly corresponding to this address range and it will have 421 * wired_count == 0. 422 */ 423 if (!vm_map_lookup_entry(map, addr, &entry) || 424 entry->start != addr || entry->end != addr + size || 425 entry->wired_count != 0) 426 panic("kmem_malloc: entry not found or misaligned"); 427 entry->wired_count = 1; 428 429 vm_map_simplify_entry(map, entry); 430 431 /* 432 * Loop thru pages, entering them in the pmap. (We cannot add them to 433 * the wired count without wrapping the vm_page_queue_lock in 434 * splimp...) 435 */ 436 for (i = 0; i < size; i += PAGE_SIZE) { 437 VM_OBJECT_LOCK(kmem_object); 438 m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i)); 439 VM_OBJECT_UNLOCK(kmem_object); 440 /* 441 * Because this is kernel_pmap, this call will not block. 442 */ 443 pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL, 1); 444 vm_page_lock_queues(); 445 vm_page_flag_set(m, PG_WRITEABLE | PG_REFERENCED); 446 vm_page_wakeup(m); 447 vm_page_unlock_queues(); 448 } 449 vm_map_unlock(map); 450 451 return (addr); 452} 453 454/* 455 * kmem_alloc_wait: 456 * 457 * Allocates pageable memory from a sub-map of the kernel. If the submap 458 * has no room, the caller sleeps waiting for more memory in the submap. 459 * 460 * This routine may block. 461 */ 462vm_offset_t 463kmem_alloc_wait(map, size) 464 vm_map_t map; 465 vm_size_t size; 466{ 467 vm_offset_t addr; 468 469 size = round_page(size); 470 471 for (;;) { 472 /* 473 * To make this work for more than one map, use the map's lock 474 * to lock out sleepers/wakers. 475 */ 476 vm_map_lock(map); 477 if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0) 478 break; 479 /* no space now; see if we can ever get space */ 480 if (vm_map_max(map) - vm_map_min(map) < size) { 481 vm_map_unlock(map); 482 return (0); 483 } 484 map->needs_wakeup = TRUE; 485 vm_map_unlock_and_wait(map, FALSE); 486 } 487 vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0); 488 vm_map_unlock(map); 489 return (addr); 490} 491 492/* 493 * kmem_free_wakeup: 494 * 495 * Returns memory to a submap of the kernel, and wakes up any processes 496 * waiting for memory in that map. 497 */ 498void 499kmem_free_wakeup(map, addr, size) 500 vm_map_t map; 501 vm_offset_t addr; 502 vm_size_t size; 503{ 504 505 vm_map_lock(map); 506 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 507 if (map->needs_wakeup) { 508 map->needs_wakeup = FALSE; 509 vm_map_wakeup(map); 510 } 511 vm_map_unlock(map); 512} 513 514/* 515 * kmem_init: 516 * 517 * Create the kernel map; insert a mapping covering kernel text, 518 * data, bss, and all space allocated thus far (`boostrap' data). The 519 * new map will thus map the range between VM_MIN_KERNEL_ADDRESS and 520 * `start' as allocated, and the range between `start' and `end' as free. 521 */ 522void 523kmem_init(start, end) 524 vm_offset_t start, end; 525{ 526 vm_map_t m; 527 528 m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); 529 m->system_map = 1; 530 vm_map_lock(m); 531 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ 532 kernel_map = m; 533 (void) vm_map_insert(m, NULL, (vm_ooffset_t) 0, 534 VM_MIN_KERNEL_ADDRESS, start, VM_PROT_ALL, VM_PROT_ALL, 0); 535 /* ... and ending with the completion of the above `insert' */ 536 vm_map_unlock(m); 537}
|