vm_kern.c revision 52635
119304Speter/* 219304Speter * Copyright (c) 1991, 1993 319304Speter * The Regents of the University of California. All rights reserved. 419304Speter * 519304Speter * This code is derived from software contributed to Berkeley by 619304Speter * The Mach Operating System project at Carnegie-Mellon University. 719304Speter * 819304Speter * Redistribution and use in source and binary forms, with or without 919304Speter * modification, are permitted provided that the following conditions 1019304Speter * are met: 1119304Speter * 1. Redistributions of source code must retain the above copyright 1219304Speter * notice, this list of conditions and the following disclaimer. 13254225Speter * 2. Redistributions in binary form must reproduce the above copyright 1419304Speter * notice, this list of conditions and the following disclaimer in the 1519304Speter * documentation and/or other materials provided with the distribution. 1619304Speter * 3. All advertising materials mentioning features or use of this software 1719304Speter * must display the following acknowledgement: 1819304Speter * This product includes software developed by the University of 1919304Speter * California, Berkeley and its contributors. 2019304Speter * 4. Neither the name of the University nor the names of its contributors 2119304Speter * may be used to endorse or promote products derived from this software 2219304Speter * without specific prior written permission. 2319304Speter * 2419304Speter * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2519304Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2619304Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2719304Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2819304Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2919304Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 3019304Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 3119304Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 3219304Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3319304Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3419304Speter * SUCH DAMAGE. 3519304Speter * 3619304Speter * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 3719304Speter * 38254225Speter * 3919304Speter * Copyright (c) 1987, 1990 Carnegie-Mellon University. 4019304Speter * All rights reserved. 4119304Speter * 4219304Speter * Authors: Avadis Tevanian, Jr., Michael Wayne Young 4319304Speter * 4419304Speter * Permission to use, copy, modify and distribute this software and 4519304Speter * its documentation is hereby granted, provided that both the copyright 4619304Speter * notice and this permission notice appear in all copies of the 4719304Speter * software, derivative works or modified versions, and any portions 4819304Speter * thereof, and that both notices appear in supporting documentation. 4919304Speter * 5019304Speter * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 5119304Speter * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 5219304Speter * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 5319304Speter * 5419304Speter * Carnegie Mellon requests users of this software to return to 5519304Speter * 5619304Speter * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 5719304Speter * School of Computer Science 5819304Speter * Carnegie Mellon University 5919304Speter * Pittsburgh PA 15213-3890 6019304Speter * 6119304Speter * any improvements or extensions that they make and grant Carnegie the 6219304Speter * rights to redistribute these changes. 6319304Speter * 6419304Speter * $FreeBSD: head/sys/vm/vm_kern.c 52635 1999-10-29 18:09:36Z phk $ 6519304Speter */ 6619304Speter 67254225Speter/* 6819304Speter * Kernel memory management. 6919304Speter */ 7019304Speter 7119304Speter#include <sys/param.h> 7219304Speter#include <sys/systm.h> 7319304Speter#include <sys/proc.h> 7419304Speter#include <sys/malloc.h> 7519304Speter 7619304Speter#include <vm/vm.h> 7719304Speter#include <vm/vm_param.h> 7819304Speter#include <sys/lock.h> 7919304Speter#include <vm/pmap.h> 8019304Speter#include <vm/vm_map.h> 8119304Speter#include <vm/vm_object.h> 8219304Speter#include <vm/vm_page.h> 8319304Speter#include <vm/vm_pageout.h> 8419304Speter#include <vm/vm_extern.h> 8519304Speter 8619304Spetervm_map_t kernel_map=0; 8719304Spetervm_map_t kmem_map=0; 8819304Spetervm_map_t exec_map=0; 8919304Spetervm_map_t clean_map=0; 9019304Spetervm_map_t buffer_map=0; 9119304Spetervm_map_t mb_map=0; 9219304Speterint mb_map_full=0; 9319304Speter 9419304Speter/* 9519304Speter * kmem_alloc_pageable: 9619304Speter * 9719304Speter * Allocate pageable memory to the kernel's address map. 9819304Speter * "map" must be kernel_map or a submap of kernel_map. 9919304Speter */ 10019304Speter 10119304Spetervm_offset_t 10219304Speterkmem_alloc_pageable(map, size) 10319304Speter vm_map_t map; 10419304Speter register vm_size_t size; 10519304Speter{ 106254225Speter vm_offset_t addr; 10719304Speter register int result; 10819304Speter 10919304Speter size = round_page(size); 11019304Speter addr = vm_map_min(map); 11119304Speter result = vm_map_find(map, NULL, (vm_offset_t) 0, 11219304Speter &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0); 11319304Speter if (result != KERN_SUCCESS) { 11419304Speter return (0); 11519304Speter } 11619304Speter return (addr); 11719304Speter} 11819304Speter 11919304Speter/* 12019304Speter * kmem_alloc_nofault: 12119304Speter * 12219304Speter * Same as kmem_alloc_pageable, except that it create a nofault entry. 12319304Speter */ 12419304Speter 12519304Spetervm_offset_t 12619304Speterkmem_alloc_nofault(map, size) 12719304Speter vm_map_t map; 12819304Speter register vm_size_t size; 12919304Speter{ 13019304Speter vm_offset_t addr; 13119304Speter register int result; 13219304Speter 13319304Speter size = round_page(size); 13419304Speter addr = vm_map_min(map); 13519304Speter result = vm_map_find(map, NULL, (vm_offset_t) 0, 13619304Speter &addr, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 137254225Speter if (result != KERN_SUCCESS) { 13819304Speter return (0); 13919304Speter } 14019304Speter return (addr); 141254225Speter} 142254225Speter 143254225Speter/* 14419304Speter * Allocate wired-down memory in the kernel's address map 14519304Speter * or a submap. 14619304Speter */ 14719304Spetervm_offset_t 14819304Speterkmem_alloc(map, size) 14919304Speter register vm_map_t map; 15019304Speter register vm_size_t size; 15119304Speter{ 15219304Speter vm_offset_t addr; 15319304Speter register vm_offset_t offset; 15419304Speter vm_offset_t i; 15519304Speter 15619304Speter size = round_page(size); 15719304Speter 15819304Speter /* 15919304Speter * Use the kernel object for wired-down kernel pages. Assume that no 16019304Speter * region of the kernel object is referenced more than once. 16119304Speter */ 16219304Speter 16319304Speter /* 16419304Speter * Locate sufficient space in the map. This will give us the final 16519304Speter * virtual address for the new memory, and thus will tell us the 16619304Speter * offset within the kernel map. 16719304Speter */ 16819304Speter vm_map_lock(map); 16919304Speter if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 17019304Speter vm_map_unlock(map); 17119304Speter return (0); 17219304Speter } 17319304Speter offset = addr - VM_MIN_KERNEL_ADDRESS; 17419304Speter vm_object_reference(kernel_object); 17519304Speter vm_map_insert(map, kernel_object, offset, addr, addr + size, 17619304Speter VM_PROT_ALL, VM_PROT_ALL, 0); 17719304Speter vm_map_unlock(map); 17819304Speter 17919304Speter /* 18019304Speter * Guarantee that there are pages already in this object before 18119304Speter * calling vm_map_pageable. This is to prevent the following 18219304Speter * scenario: 18319304Speter * 18419304Speter * 1) Threads have swapped out, so that there is a pager for the 18519304Speter * kernel_object. 2) The kmsg zone is empty, and so we are 18619304Speter * kmem_allocing a new page for it. 3) vm_map_pageable calls vm_fault; 18719304Speter * there is no page, but there is a pager, so we call 18819304Speter * pager_data_request. But the kmsg zone is empty, so we must 18919304Speter * kmem_alloc. 4) goto 1 5) Even if the kmsg zone is not empty: when 19019304Speter * we get the data back from the pager, it will be (very stale) 19119304Speter * non-zero data. kmem_alloc is defined to return zero-filled memory. 19219304Speter * 19319304Speter * We're intentionally not activating the pages we allocate to prevent a 194254225Speter * race with page-out. vm_map_pageable will wire the pages. 19519304Speter */ 19619304Speter 19719304Speter for (i = 0; i < size; i += PAGE_SIZE) { 19819304Speter vm_page_t mem; 19919304Speter 20019304Speter mem = vm_page_grab(kernel_object, OFF_TO_IDX(offset + i), 20119304Speter VM_ALLOC_ZERO | VM_ALLOC_RETRY); 20219304Speter if ((mem->flags & PG_ZERO) == 0) 20319304Speter vm_page_zero_fill(mem); 20419304Speter mem->valid = VM_PAGE_BITS_ALL; 20519304Speter vm_page_flag_clear(mem, PG_ZERO); 20619304Speter vm_page_wakeup(mem); 20719304Speter } 20819304Speter 20919304Speter /* 21019304Speter * And finally, mark the data as non-pageable. 21119304Speter */ 21219304Speter 21319304Speter (void) vm_map_pageable(map, (vm_offset_t) addr, addr + size, FALSE); 21419304Speter 21519304Speter return (addr); 21619304Speter} 21719304Speter 21819304Speter/* 219254225Speter * kmem_free: 22019304Speter * 22119304Speter * Release a region of kernel virtual memory allocated 22219304Speter * with kmem_alloc, and return the physical pages 223254225Speter * associated with that region. 224254225Speter * 225254225Speter * This routine may not block on kernel maps. 22619304Speter */ 22719304Spetervoid 22819304Speterkmem_free(map, addr, size) 22919304Speter vm_map_t map; 23019304Speter register vm_offset_t addr; 23119304Speter vm_size_t size; 23219304Speter{ 23319304Speter (void) vm_map_remove(map, trunc_page(addr), round_page(addr + size)); 23419304Speter} 23519304Speter 23619304Speter/* 23719304Speter * kmem_suballoc: 23819304Speter * 23919304Speter * Allocates a map to manage a subrange 24019304Speter * of the kernel virtual address space. 24119304Speter * 24219304Speter * Arguments are as follows: 24319304Speter * 24419304Speter * parent Map to take range from 24519304Speter * size Size of range to find 24619304Speter * min, max Returned endpoints of map 24719304Speter * pageable Can the region be paged 24819304Speter */ 24919304Spetervm_map_t 25019304Speterkmem_suballoc(parent, min, max, size) 25119304Speter register vm_map_t parent; 25219304Speter vm_offset_t *min, *max; 25319304Speter register vm_size_t size; 25419304Speter{ 25519304Speter register int ret; 25619304Speter vm_map_t result; 25719304Speter 25819304Speter size = round_page(size); 25919304Speter 26019304Speter *min = (vm_offset_t) vm_map_min(parent); 26119304Speter ret = vm_map_find(parent, NULL, (vm_offset_t) 0, 26219304Speter min, size, TRUE, VM_PROT_ALL, VM_PROT_ALL, 0); 26319304Speter if (ret != KERN_SUCCESS) { 26419304Speter printf("kmem_suballoc: bad status return of %d.\n", ret); 26519304Speter panic("kmem_suballoc"); 26619304Speter } 26719304Speter *max = *min + size; 26819304Speter pmap_reference(vm_map_pmap(parent)); 26919304Speter result = vm_map_create(vm_map_pmap(parent), *min, *max); 27019304Speter if (result == NULL) 27119304Speter panic("kmem_suballoc: cannot create submap"); 27219304Speter if ((ret = vm_map_submap(parent, *min, *max, result)) != KERN_SUCCESS) 273254225Speter panic("kmem_suballoc: unable to change range to submap"); 27419304Speter return (result); 27519304Speter} 27619304Speter 27719304Speter/* 27819304Speter * kmem_malloc: 279254225Speter * 28019304Speter * Allocate wired-down memory in the kernel's address map for the higher 28119304Speter * level kernel memory allocator (kern/kern_malloc.c). We cannot use 28219304Speter * kmem_alloc() because we may need to allocate memory at interrupt 283 * level where we cannot block (canwait == FALSE). 284 * 285 * This routine has its own private kernel submap (kmem_map) and object 286 * (kmem_object). This, combined with the fact that only malloc uses 287 * this routine, ensures that we will never block in map or object waits. 288 * 289 * Note that this still only works in a uni-processor environment and 290 * when called at splhigh(). 291 * 292 * We don't worry about expanding the map (adding entries) since entries 293 * for wired maps are statically allocated. 294 * 295 * NOTE: This routine is not supposed to block if M_NOWAIT is set, but 296 * I have not verified that it actually does not block. 297 */ 298vm_offset_t 299kmem_malloc(map, size, flags) 300 register vm_map_t map; 301 register vm_size_t size; 302 int flags; 303{ 304 register vm_offset_t offset, i; 305 vm_map_entry_t entry; 306 vm_offset_t addr; 307 vm_page_t m; 308 309 if (map != kmem_map && map != mb_map) 310 panic("kmem_malloc: map != {kmem,mb}_map"); 311 312 size = round_page(size); 313 addr = vm_map_min(map); 314 315 /* 316 * Locate sufficient space in the map. This will give us the final 317 * virtual address for the new memory, and thus will tell us the 318 * offset within the kernel map. 319 */ 320 vm_map_lock(map); 321 if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { 322 vm_map_unlock(map); 323 if (map == mb_map) { 324 mb_map_full = TRUE; 325 printf("Out of mbuf clusters - adjust NMBCLUSTERS or increase maxusers!\n"); 326 return (0); 327 } 328 if ((flags & M_NOWAIT) == 0) 329 panic("kmem_malloc(%ld): kmem_map too small: %ld total allocated", 330 (long)size, (long)map->size); 331 return (0); 332 } 333 offset = addr - VM_MIN_KERNEL_ADDRESS; 334 vm_object_reference(kmem_object); 335 vm_map_insert(map, kmem_object, offset, addr, addr + size, 336 VM_PROT_ALL, VM_PROT_ALL, 0); 337 338 for (i = 0; i < size; i += PAGE_SIZE) { 339 /* 340 * Note: if M_NOWAIT specified alone, allocate from 341 * interrupt-safe queues only (just the free list). If 342 * M_ASLEEP or M_USE_RESERVE is also specified, we can also 343 * allocate from the cache. Neither of the latter two 344 * flags may be specified from an interrupt since interrupts 345 * are not allowed to mess with the cache queue. 346 */ 347retry: 348 m = vm_page_alloc(kmem_object, OFF_TO_IDX(offset + i), 349 ((flags & (M_NOWAIT|M_ASLEEP|M_USE_RESERVE)) == M_NOWAIT) ? 350 VM_ALLOC_INTERRUPT : 351 VM_ALLOC_SYSTEM); 352 353 /* 354 * Ran out of space, free everything up and return. Don't need 355 * to lock page queues here as we know that the pages we got 356 * aren't on any queues. 357 */ 358 if (m == NULL) { 359 if ((flags & M_NOWAIT) == 0) { 360 vm_map_unlock(map); 361 VM_WAIT; 362 vm_map_lock(map); 363 goto retry; 364 } 365 vm_map_delete(map, addr, addr + size); 366 vm_map_unlock(map); 367 if (flags & M_ASLEEP) { 368 VM_AWAIT; 369 } 370 return (0); 371 } 372 vm_page_flag_clear(m, PG_ZERO); 373 m->valid = VM_PAGE_BITS_ALL; 374 } 375 376 /* 377 * Mark map entry as non-pageable. Assert: vm_map_insert() will never 378 * be able to extend the previous entry so there will be a new entry 379 * exactly corresponding to this address range and it will have 380 * wired_count == 0. 381 */ 382 if (!vm_map_lookup_entry(map, addr, &entry) || 383 entry->start != addr || entry->end != addr + size || 384 entry->wired_count != 0) 385 panic("kmem_malloc: entry not found or misaligned"); 386 entry->wired_count = 1; 387 388 vm_map_simplify_entry(map, entry); 389 390 /* 391 * Loop thru pages, entering them in the pmap. (We cannot add them to 392 * the wired count without wrapping the vm_page_queue_lock in 393 * splimp...) 394 */ 395 for (i = 0; i < size; i += PAGE_SIZE) { 396 m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i)); 397 vm_page_wire(m); 398 vm_page_wakeup(m); 399 /* 400 * Because this is kernel_pmap, this call will not block. 401 */ 402 pmap_enter(kernel_pmap, addr + i, VM_PAGE_TO_PHYS(m), 403 VM_PROT_ALL, 1); 404 vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE | PG_REFERENCED); 405 } 406 vm_map_unlock(map); 407 408 return (addr); 409} 410 411/* 412 * kmem_alloc_wait: 413 * 414 * Allocates pageable memory from a sub-map of the kernel. If the submap 415 * has no room, the caller sleeps waiting for more memory in the submap. 416 * 417 * This routine may block. 418 */ 419 420vm_offset_t 421kmem_alloc_wait(map, size) 422 vm_map_t map; 423 vm_size_t size; 424{ 425 vm_offset_t addr; 426 427 size = round_page(size); 428 429 for (;;) { 430 /* 431 * To make this work for more than one map, use the map's lock 432 * to lock out sleepers/wakers. 433 */ 434 vm_map_lock(map); 435 if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0) 436 break; 437 /* no space now; see if we can ever get space */ 438 if (vm_map_max(map) - vm_map_min(map) < size) { 439 vm_map_unlock(map); 440 return (0); 441 } 442 vm_map_unlock(map); 443 tsleep(map, PVM, "kmaw", 0); 444 } 445 vm_map_insert(map, NULL, (vm_offset_t) 0, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0); 446 vm_map_unlock(map); 447 return (addr); 448} 449 450/* 451 * kmem_free_wakeup: 452 * 453 * Returns memory to a submap of the kernel, and wakes up any processes 454 * waiting for memory in that map. 455 */ 456void 457kmem_free_wakeup(map, addr, size) 458 vm_map_t map; 459 vm_offset_t addr; 460 vm_size_t size; 461{ 462 vm_map_lock(map); 463 (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 464 wakeup(map); 465 vm_map_unlock(map); 466} 467 468/* 469 * kmem_init: 470 * 471 * Create the kernel map; insert a mapping covering kernel text, 472 * data, bss, and all space allocated thus far (`boostrap' data). The 473 * new map will thus map the range between VM_MIN_KERNEL_ADDRESS and 474 * `start' as allocated, and the range between `start' and `end' as free. 475 */ 476 477void 478kmem_init(start, end) 479 vm_offset_t start, end; 480{ 481 register vm_map_t m; 482 483 m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); 484 vm_map_lock(m); 485 /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ 486 kernel_map = m; 487 kernel_map->system_map = 1; 488 (void) vm_map_insert(m, NULL, (vm_offset_t) 0, 489 VM_MIN_KERNEL_ADDRESS, start, VM_PROT_ALL, VM_PROT_ALL, 0); 490 /* ... and ending with the completion of the above `insert' */ 491 vm_map_unlock(m); 492} 493 494