1139825Simp/*- 21541Srgrimes * Copyright (c) 1991, 1993 31541Srgrimes * The Regents of the University of California. All rights reserved. 41541Srgrimes * 51541Srgrimes * This code is derived from software contributed to Berkeley by 61541Srgrimes * The Mach Operating System project at Carnegie-Mellon University. 71541Srgrimes * 81541Srgrimes * Redistribution and use in source and binary forms, with or without 91541Srgrimes * modification, are permitted provided that the following conditions 101541Srgrimes * are met: 111541Srgrimes * 1. Redistributions of source code must retain the above copyright 121541Srgrimes * notice, this list of conditions and the following disclaimer. 131541Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 141541Srgrimes * notice, this list of conditions and the following disclaimer in the 151541Srgrimes * documentation and/or other materials provided with the distribution. 161541Srgrimes * 4. Neither the name of the University nor the names of its contributors 171541Srgrimes * may be used to endorse or promote products derived from this software 181541Srgrimes * without specific prior written permission. 191541Srgrimes * 201541Srgrimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 211541Srgrimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 221541Srgrimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 231541Srgrimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 241541Srgrimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 251541Srgrimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 261541Srgrimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 271541Srgrimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 281541Srgrimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 291541Srgrimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 301541Srgrimes * SUCH DAMAGE. 311541Srgrimes * 321817Sdg * from: @(#)vm_kern.c 8.3 (Berkeley) 1/12/94 331541Srgrimes * 341541Srgrimes * 351541Srgrimes * Copyright (c) 1987, 1990 Carnegie-Mellon University. 361541Srgrimes * All rights reserved. 371541Srgrimes * 381541Srgrimes * Authors: Avadis Tevanian, Jr., Michael Wayne Young 395455Sdg * 401541Srgrimes * Permission to use, copy, modify and distribute this software and 411541Srgrimes * its documentation is hereby granted, provided that both the copyright 421541Srgrimes * notice and this permission notice appear in all copies of the 431541Srgrimes * software, derivative works or modified versions, and any portions 441541Srgrimes * thereof, and that both notices appear in supporting documentation. 455455Sdg * 465455Sdg * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 475455Sdg * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 481541Srgrimes * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 495455Sdg * 501541Srgrimes * Carnegie Mellon requests users of this software to return to 511541Srgrimes * 521541Srgrimes * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 531541Srgrimes * School of Computer Science 541541Srgrimes * Carnegie Mellon University 551541Srgrimes * Pittsburgh PA 15213-3890 561541Srgrimes * 571541Srgrimes * any improvements or extensions that they make and grant Carnegie the 581541Srgrimes * rights to redistribute these changes. 591541Srgrimes */ 601541Srgrimes 611541Srgrimes/* 621541Srgrimes * Kernel memory management. 631541Srgrimes */ 641541Srgrimes 65116226Sobrien#include <sys/cdefs.h> 66116226Sobrien__FBSDID("$FreeBSD: stable/11/sys/vm/vm_kern.c 340660 2018-11-20 01:12:21Z markj $"); 67116226Sobrien 681541Srgrimes#include <sys/param.h> 691541Srgrimes#include <sys/systm.h> 7087157Sluigi#include <sys/kernel.h> /* for ticks and hz */ 71168395Spjd#include <sys/eventhandler.h> 7276166Smarkm#include <sys/lock.h> 732112Swollman#include <sys/proc.h> 746129Sdg#include <sys/malloc.h> 75248084Sattilio#include <sys/rwlock.h> 76188964Srwatson#include <sys/sysctl.h> 77254025Sjeff#include <sys/vmem.h> 781541Srgrimes 791541Srgrimes#include <vm/vm.h> 8012662Sdg#include <vm/vm_param.h> 81254025Sjeff#include <vm/vm_kern.h> 8212662Sdg#include <vm/pmap.h> 8312662Sdg#include <vm/vm_map.h> 8412662Sdg#include <vm/vm_object.h> 851541Srgrimes#include <vm/vm_page.h> 861541Srgrimes#include <vm/vm_pageout.h> 87337262Smarkj#include <vm/vm_phys.h> 88327701Smarkj#include <vm/vm_radix.h> 8912726Sbde#include <vm/vm_extern.h> 90168395Spjd#include <vm/uma.h> 911541Srgrimes 92248277Skibvm_map_t kernel_map; 93248277Skibvm_map_t exec_map; 94118764Ssilbyvm_map_t pipe_map; 952112Swollman 96221853Smdfconst void *zero_region; 97221853SmdfCTASSERT((ZERO_REGION_SIZE & PAGE_MASK) == 0); 98221853Smdf 99290728Sjhb/* NB: Used by kernel debuggers. */ 100290728Sjhbconst u_long vm_maxuser_address = VM_MAXUSER_ADDRESS; 101290728Sjhb 102320797Smarkju_int exec_map_entry_size; 103320797Smarkju_int exec_map_entries; 104320797Smarkj 105246316SmariusSYSCTL_ULONG(_vm, OID_AUTO, min_kernel_address, CTLFLAG_RD, 106273377Shselasky SYSCTL_NULL_ULONG_PTR, VM_MIN_KERNEL_ADDRESS, "Min kernel address"); 107246316Smarius 108246316SmariusSYSCTL_ULONG(_vm, OID_AUTO, max_kernel_address, CTLFLAG_RD, 109246926Salc#if defined(__arm__) || defined(__sparc64__) 110246316Smarius &vm_max_kernel_address, 0, 111246316Smarius#else 112273377Shselasky SYSCTL_NULL_ULONG_PTR, VM_MAX_KERNEL_ADDRESS, 113246316Smarius#endif 114246316Smarius "Max kernel address"); 115246316Smarius 1161541Srgrimes/* 117254025Sjeff * kva_alloc: 11847841Sdt * 119118317Salc * Allocate a virtual address range with no underlying object and 120118317Salc * no initial mapping to physical memory. Any mapping from this 121118317Salc * range to physical memory must be explicitly created prior to 122118317Salc * its use, typically with pmap_qenter(). Any attempt to create 123118317Salc * a mapping on demand through vm_fault() will result in a panic. 12447841Sdt */ 12547841Sdtvm_offset_t 126324781Semastekva_alloc(vm_size_t size) 12747841Sdt{ 12847841Sdt vm_offset_t addr; 12947841Sdt 13047841Sdt size = round_page(size); 131254025Sjeff if (vmem_alloc(kernel_arena, size, M_BESTFIT | M_NOWAIT, &addr)) 13247841Sdt return (0); 133254025Sjeff 13447841Sdt return (addr); 13547841Sdt} 13647841Sdt 13747841Sdt/* 138254025Sjeff * kva_free: 139206819Sjmallett * 140254025Sjeff * Release a region of kernel virtual memory allocated 141254025Sjeff * with kva_alloc, and return the physical pages 142254025Sjeff * associated with that region. 143254025Sjeff * 144254025Sjeff * This routine may not block on kernel maps. 145206819Sjmallett */ 146254025Sjeffvoid 147324781Semastekva_free(vm_offset_t addr, vm_size_t size) 1481541Srgrimes{ 1491541Srgrimes 1501541Srgrimes size = round_page(size); 151254025Sjeff vmem_free(kernel_arena, addr, size); 1521541Srgrimes} 1531541Srgrimes 1541541Srgrimes/* 155238452Salc * Allocates a region from the kernel address map and physical pages 156238452Salc * within the specified address range to the kernel object. Creates a 157238452Salc * wired mapping from this region to these pages, and returns the 158238452Salc * region's starting virtual address. The allocated pages are not 159238452Salc * necessarily physically contiguous. If M_ZERO is specified through the 160238452Salc * given flags, then the pages are zeroed before they are mapped. 161238452Salc */ 162238452Salcvm_offset_t 163254025Sjeffkmem_alloc_attr(vmem_t *vmem, vm_size_t size, int flags, vm_paddr_t low, 164238452Salc vm_paddr_t high, vm_memattr_t memattr) 165238452Salc{ 166254025Sjeff vm_object_t object = vmem == kmem_arena ? kmem_object : kernel_object; 167316073Skib vm_offset_t addr, i, offset; 168238452Salc vm_page_t m; 169238452Salc int pflags, tries; 170238452Salc 171238452Salc size = round_page(size); 172254025Sjeff if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr)) 173238452Salc return (0); 174238452Salc offset = addr - VM_MIN_KERNEL_ADDRESS; 175254025Sjeff pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; 176327785Smarkj pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); 177327785Smarkj pflags |= VM_ALLOC_NOWAIT; 178248084Sattilio VM_OBJECT_WLOCK(object); 179254025Sjeff for (i = 0; i < size; i += PAGE_SIZE) { 180238452Salc tries = 0; 181238452Salcretry: 182316073Skib m = vm_page_alloc_contig(object, atop(offset + i), 183254025Sjeff pflags, 1, low, high, PAGE_SIZE, 0, memattr); 184238452Salc if (m == NULL) { 185248084Sattilio VM_OBJECT_WUNLOCK(object); 186238452Salc if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { 187292469Salc if (!vm_page_reclaim_contig(pflags, 1, 188292469Salc low, high, PAGE_SIZE, 0) && 189292469Salc (flags & M_WAITOK) != 0) 190292469Salc VM_WAIT; 191248084Sattilio VM_OBJECT_WLOCK(object); 192238452Salc tries++; 193238452Salc goto retry; 194238452Salc } 195288281Salc kmem_unback(object, addr, i); 196254025Sjeff vmem_free(vmem, addr, size); 197238452Salc return (0); 198238452Salc } 199238452Salc if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 200238452Salc pmap_zero_page(m); 201238452Salc m->valid = VM_PAGE_BITS_ALL; 202269728Skib pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL, 203269728Skib VM_PROT_ALL | PMAP_ENTER_WIRED, 0); 204238452Salc } 205248084Sattilio VM_OBJECT_WUNLOCK(object); 206238452Salc return (addr); 207238452Salc} 208238452Salc 209238452Salc/* 210238452Salc * Allocates a region from the kernel address map and physically 211238452Salc * contiguous pages within the specified address range to the kernel 212238452Salc * object. Creates a wired mapping from this region to these pages, and 213238452Salc * returns the region's starting virtual address. If M_ZERO is specified 214238452Salc * through the given flags, then the pages are zeroed before they are 215238452Salc * mapped. 216238452Salc */ 217238452Salcvm_offset_t 218254025Sjeffkmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low, 219238452Salc vm_paddr_t high, u_long alignment, vm_paddr_t boundary, 220238452Salc vm_memattr_t memattr) 221238452Salc{ 222254025Sjeff vm_object_t object = vmem == kmem_arena ? kmem_object : kernel_object; 223316073Skib vm_offset_t addr, offset, tmp; 224238452Salc vm_page_t end_m, m; 225292469Salc u_long npages; 226238452Salc int pflags, tries; 227238452Salc 228238452Salc size = round_page(size); 229254025Sjeff if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr)) 230238452Salc return (0); 231238452Salc offset = addr - VM_MIN_KERNEL_ADDRESS; 232254025Sjeff pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; 233327785Smarkj pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); 234327785Smarkj pflags |= VM_ALLOC_NOWAIT; 235292469Salc npages = atop(size); 236248084Sattilio VM_OBJECT_WLOCK(object); 237238452Salc tries = 0; 238238452Salcretry: 239316073Skib m = vm_page_alloc_contig(object, atop(offset), pflags, 240292469Salc npages, low, high, alignment, boundary, memattr); 241238452Salc if (m == NULL) { 242248084Sattilio VM_OBJECT_WUNLOCK(object); 243238452Salc if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { 244292469Salc if (!vm_page_reclaim_contig(pflags, npages, low, high, 245292469Salc alignment, boundary) && (flags & M_WAITOK) != 0) 246292469Salc VM_WAIT; 247248084Sattilio VM_OBJECT_WLOCK(object); 248238452Salc tries++; 249238452Salc goto retry; 250238452Salc } 251254025Sjeff vmem_free(vmem, addr, size); 252238452Salc return (0); 253238452Salc } 254292469Salc end_m = m + npages; 255254025Sjeff tmp = addr; 256238452Salc for (; m < end_m; m++) { 257238452Salc if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) 258238452Salc pmap_zero_page(m); 259238452Salc m->valid = VM_PAGE_BITS_ALL; 260269728Skib pmap_enter(kernel_pmap, tmp, m, VM_PROT_ALL, 261269728Skib VM_PROT_ALL | PMAP_ENTER_WIRED, 0); 262254025Sjeff tmp += PAGE_SIZE; 263238452Salc } 264248084Sattilio VM_OBJECT_WUNLOCK(object); 265238452Salc return (addr); 266238452Salc} 267238452Salc 268238452Salc/* 2691541Srgrimes * kmem_suballoc: 2701541Srgrimes * 2711541Srgrimes * Allocates a map to manage a subrange 2721541Srgrimes * of the kernel virtual address space. 2731541Srgrimes * 2741541Srgrimes * Arguments are as follows: 2751541Srgrimes * 2761541Srgrimes * parent Map to take range from 27770480Salfred * min, max Returned endpoints of map 2781541Srgrimes * size Size of range to find 279178933Salc * superpage_align Request that min is superpage aligned 2801541Srgrimes */ 2818876Srgrimesvm_map_t 282178933Salckmem_suballoc(vm_map_t parent, vm_offset_t *min, vm_offset_t *max, 283178933Salc vm_size_t size, boolean_t superpage_align) 2841541Srgrimes{ 28570478Salfred int ret; 2865455Sdg vm_map_t result; 2871541Srgrimes 2881541Srgrimes size = round_page(size); 2891541Srgrimes 290178637Salc *min = vm_map_min(parent); 291255426Sjhb ret = vm_map_find(parent, NULL, 0, min, size, 0, superpage_align ? 292254430Sjhb VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, 293194766Skib MAP_ACC_NO_CHARGE); 294177762Salc if (ret != KERN_SUCCESS) 295177762Salc panic("kmem_suballoc: bad status return of %d", ret); 2961541Srgrimes *max = *min + size; 29732702Sdyson result = vm_map_create(vm_map_pmap(parent), *min, *max); 2981541Srgrimes if (result == NULL) 2991541Srgrimes panic("kmem_suballoc: cannot create submap"); 30070478Salfred if (vm_map_submap(parent, *min, *max, result) != KERN_SUCCESS) 3011541Srgrimes panic("kmem_suballoc: unable to change range to submap"); 3025455Sdg return (result); 3031541Srgrimes} 3041541Srgrimes 3051541Srgrimes/* 30642957Sdillon * kmem_malloc: 3071541Srgrimes * 308254025Sjeff * Allocate wired-down pages in the kernel's address space. 3091541Srgrimes */ 3101541Srgrimesvm_offset_t 311254025Sjeffkmem_malloc(struct vmem *vmem, vm_size_t size, int flags) 3121541Srgrimes{ 3135455Sdg vm_offset_t addr; 314254025Sjeff int rv; 3151541Srgrimes 3161541Srgrimes size = round_page(size); 317254025Sjeff if (vmem_alloc(vmem, size, flags | M_BESTFIT, &addr)) 318254025Sjeff return (0); 3191541Srgrimes 320254025Sjeff rv = kmem_back((vmem == kmem_arena) ? kmem_object : kernel_object, 321254025Sjeff addr, size, flags); 322254025Sjeff if (rv != KERN_SUCCESS) { 323254025Sjeff vmem_free(vmem, addr, size); 324254025Sjeff return (0); 3251541Srgrimes } 326254025Sjeff return (addr); 327211194Smdf} 328211194Smdf 329211194Smdf/* 330211194Smdf * kmem_back: 331211194Smdf * 332211194Smdf * Allocate physical pages for the specified virtual address range. 333211194Smdf */ 334211194Smdfint 335254025Sjeffkmem_back(vm_object_t object, vm_offset_t addr, vm_size_t size, int flags) 336211194Smdf{ 337211194Smdf vm_offset_t offset, i; 338327701Smarkj vm_page_t m, mpred; 339211194Smdf int pflags; 340211194Smdf 341254025Sjeff KASSERT(object == kmem_object || object == kernel_object, 342254025Sjeff ("kmem_back: only supports kernel objects.")); 343254025Sjeff 34415367Sdyson offset = addr - VM_MIN_KERNEL_ADDRESS; 345254025Sjeff pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; 346327785Smarkj pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); 347327785Smarkj if (flags & M_WAITOK) 348327785Smarkj pflags |= VM_ALLOC_WAITFAIL; 3491541Srgrimes 350327701Smarkj i = 0; 351327785Smarkj VM_OBJECT_WLOCK(object); 352327701Smarkjretry: 353327701Smarkj mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i)); 354327701Smarkj for (; i < size; i += PAGE_SIZE, mpred = m) { 355327701Smarkj m = vm_page_alloc_after(object, atop(offset + i), pflags, 356327701Smarkj mpred); 35798450Sjeff 3581541Srgrimes /* 3595455Sdg * Ran out of space, free everything up and return. Don't need 3605455Sdg * to lock page queues here as we know that the pages we got 3615455Sdg * aren't on any queues. 3621541Srgrimes */ 3631541Srgrimes if (m == NULL) { 364327785Smarkj if ((flags & M_NOWAIT) == 0) 365327785Smarkj goto retry; 366288281Salc VM_OBJECT_WUNLOCK(object); 367288281Salc kmem_unback(object, addr, i); 368211194Smdf return (KERN_NO_SPACE); 3691541Srgrimes } 37098455Sjeff if (flags & M_ZERO && (m->flags & PG_ZERO) == 0) 371102382Salc pmap_zero_page(m); 372224746Skib KASSERT((m->oflags & VPO_UNMANAGED) != 0, 373166964Salc ("kmem_malloc: page %p is managed", m)); 374254025Sjeff m->valid = VM_PAGE_BITS_ALL; 375269728Skib pmap_enter(kernel_pmap, addr + i, m, VM_PROT_ALL, 376269728Skib VM_PROT_ALL | PMAP_ENTER_WIRED, 0); 3771541Srgrimes } 378254025Sjeff VM_OBJECT_WUNLOCK(object); 3791541Srgrimes 380254025Sjeff return (KERN_SUCCESS); 381254025Sjeff} 3821541Srgrimes 383288281Salc/* 384288281Salc * kmem_unback: 385288281Salc * 386288281Salc * Unmap and free the physical pages underlying the specified virtual 387288281Salc * address range. 388288281Salc * 389288281Salc * A physical page must exist within the specified object at each index 390288281Salc * that is being unmapped. 391288281Salc */ 392254025Sjeffvoid 393254025Sjeffkmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size) 394254025Sjeff{ 395322673Smarkj vm_page_t m, next; 396322673Smarkj vm_offset_t end, offset; 39720993Sdyson 398254025Sjeff KASSERT(object == kmem_object || object == kernel_object, 399254025Sjeff ("kmem_unback: only supports kernel objects.")); 400254025Sjeff 401266588Salc pmap_remove(kernel_pmap, addr, addr + size); 402254025Sjeff offset = addr - VM_MIN_KERNEL_ADDRESS; 403322673Smarkj end = offset + size; 404254025Sjeff VM_OBJECT_WLOCK(object); 405322673Smarkj for (m = vm_page_lookup(object, atop(offset)); offset < end; 406322673Smarkj offset += PAGE_SIZE, m = next) { 407322673Smarkj next = vm_page_next(m); 408288912Salc vm_page_unwire(m, PQ_NONE); 409254025Sjeff vm_page_free(m); 4101541Srgrimes } 411254025Sjeff VM_OBJECT_WUNLOCK(object); 412254025Sjeff} 4131541Srgrimes 414254025Sjeff/* 415254025Sjeff * kmem_free: 416254025Sjeff * 417254025Sjeff * Free memory allocated with kmem_malloc. The size must match the 418254025Sjeff * original allocation. 419254025Sjeff */ 420254025Sjeffvoid 421254025Sjeffkmem_free(struct vmem *vmem, vm_offset_t addr, vm_size_t size) 422254025Sjeff{ 423254025Sjeff 424254025Sjeff size = round_page(size); 425254025Sjeff kmem_unback((vmem == kmem_arena) ? kmem_object : kernel_object, 426254025Sjeff addr, size); 427254025Sjeff vmem_free(vmem, addr, size); 4281541Srgrimes} 4291541Srgrimes 4301541Srgrimes/* 431254025Sjeff * kmap_alloc_wait: 4321541Srgrimes * 4331541Srgrimes * Allocates pageable memory from a sub-map of the kernel. If the submap 4341541Srgrimes * has no room, the caller sleeps waiting for more memory in the submap. 4351541Srgrimes * 43642957Sdillon * This routine may block. 4371541Srgrimes */ 4388876Srgrimesvm_offset_t 439324781Semastekmap_alloc_wait(vm_map_t map, vm_size_t size) 4401541Srgrimes{ 4415455Sdg vm_offset_t addr; 4421541Srgrimes 4431541Srgrimes size = round_page(size); 444194766Skib if (!swap_reserve(size)) 445194766Skib return (0); 4461541Srgrimes 4471541Srgrimes for (;;) { 4481541Srgrimes /* 4495455Sdg * To make this work for more than one map, use the map's lock 4505455Sdg * to lock out sleepers/wakers. 4511541Srgrimes */ 4521541Srgrimes vm_map_lock(map); 45333758Sdyson if (vm_map_findspace(map, vm_map_min(map), size, &addr) == 0) 4541541Srgrimes break; 4551541Srgrimes /* no space now; see if we can ever get space */ 4561541Srgrimes if (vm_map_max(map) - vm_map_min(map) < size) { 4571541Srgrimes vm_map_unlock(map); 458194766Skib swap_release(size); 4591541Srgrimes return (0); 4601541Srgrimes } 46199754Salc map->needs_wakeup = TRUE; 462173429Spjd vm_map_unlock_and_wait(map, 0); 4631541Srgrimes } 464340660Smarkj vm_map_insert(map, NULL, 0, addr, addr + size, VM_PROT_RW, VM_PROT_RW, 465340660Smarkj MAP_ACC_CHARGED); 4661541Srgrimes vm_map_unlock(map); 4671541Srgrimes return (addr); 4681541Srgrimes} 4691541Srgrimes 4701541Srgrimes/* 471254025Sjeff * kmap_free_wakeup: 4721541Srgrimes * 4739507Sdg * Returns memory to a submap of the kernel, and wakes up any processes 4741541Srgrimes * waiting for memory in that map. 4751541Srgrimes */ 4768876Srgrimesvoid 477324781Semastekmap_free_wakeup(vm_map_t map, vm_offset_t addr, vm_size_t size) 4781541Srgrimes{ 47976827Salfred 4801541Srgrimes vm_map_lock(map); 481189015Skib (void) vm_map_delete(map, trunc_page(addr), round_page(addr + size)); 48299754Salc if (map->needs_wakeup) { 48399754Salc map->needs_wakeup = FALSE; 48499754Salc vm_map_wakeup(map); 48599754Salc } 4861541Srgrimes vm_map_unlock(map); 4871541Srgrimes} 4881541Srgrimes 489254025Sjeffvoid 490221853Smdfkmem_init_zero_region(void) 491221853Smdf{ 492221855Smdf vm_offset_t addr, i; 493221853Smdf vm_page_t m; 494221853Smdf 495221855Smdf /* 496221855Smdf * Map a single physical page of zeros to a larger virtual range. 497221855Smdf * This requires less looping in places that want large amounts of 498221855Smdf * zeros, while not using much more physical resources. 499221855Smdf */ 500254025Sjeff addr = kva_alloc(ZERO_REGION_SIZE); 501226843Salc m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | 502221853Smdf VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); 503221853Smdf if ((m->flags & PG_ZERO) == 0) 504221853Smdf pmap_zero_page(m); 505221853Smdf for (i = 0; i < ZERO_REGION_SIZE; i += PAGE_SIZE) 506221853Smdf pmap_qenter(addr + i, &m, 1); 507254025Sjeff pmap_protect(kernel_pmap, addr, addr + ZERO_REGION_SIZE, VM_PROT_READ); 508221853Smdf 509221853Smdf zero_region = (const void *)addr; 510221853Smdf} 511221853Smdf 5121541Srgrimes/* 51342957Sdillon * kmem_init: 51442957Sdillon * 51542957Sdillon * Create the kernel map; insert a mapping covering kernel text, 51642957Sdillon * data, bss, and all space allocated thus far (`boostrap' data). The 51742957Sdillon * new map will thus map the range between VM_MIN_KERNEL_ADDRESS and 51842957Sdillon * `start' as allocated, and the range between `start' and `end' as free. 5191541Srgrimes */ 5208876Srgrimesvoid 521324781Semastekmem_init(vm_offset_t start, vm_offset_t end) 5221541Srgrimes{ 52370480Salfred vm_map_t m; 5241541Srgrimes 52532702Sdyson m = vm_map_create(kernel_pmap, VM_MIN_KERNEL_ADDRESS, end); 526108426Salc m->system_map = 1; 5271541Srgrimes vm_map_lock(m); 5281541Srgrimes /* N.B.: cannot use kgdb to debug, starting with this assignment ... */ 5291541Srgrimes kernel_map = m; 530108426Salc (void) vm_map_insert(m, NULL, (vm_ooffset_t) 0, 531179923Salc#ifdef __amd64__ 532179923Salc KERNBASE, 533179923Salc#else 534179923Salc VM_MIN_KERNEL_ADDRESS, 535179923Salc#endif 536179923Salc start, VM_PROT_ALL, VM_PROT_ALL, MAP_NOFAULT); 5371541Srgrimes /* ... and ending with the completion of the above `insert' */ 5381541Srgrimes vm_map_unlock(m); 5391541Srgrimes} 540188964Srwatson 541337262Smarkj/* 542337262Smarkj * kmem_bootstrap_free: 543337262Smarkj * 544337262Smarkj * Free pages backing preloaded data (e.g., kernel modules) to the 545337262Smarkj * system. Currently only supported on platforms that create a 546337262Smarkj * vm_phys segment for preloaded data. 547337262Smarkj */ 548337262Smarkjvoid 549337262Smarkjkmem_bootstrap_free(vm_offset_t start, vm_size_t size) 550337262Smarkj{ 551337262Smarkj#if defined(__i386__) || defined(__amd64__) 552337467Smarkj struct vm_domain *vmd; 553337262Smarkj vm_offset_t end, va; 554337262Smarkj vm_paddr_t pa; 555337262Smarkj vm_page_t m; 556337262Smarkj 557337262Smarkj end = trunc_page(start + size); 558337262Smarkj start = round_page(start); 559337262Smarkj 560337262Smarkj for (va = start; va < end; va += PAGE_SIZE) { 561337262Smarkj pa = pmap_kextract(va); 562337262Smarkj m = PHYS_TO_VM_PAGE(pa); 563337262Smarkj 564337467Smarkj vmd = vm_phys_domain(m); 565337262Smarkj mtx_lock(&vm_page_queue_free_mtx); 566337262Smarkj vm_phys_free_pages(m, 0); 567337467Smarkj vmd->vmd_page_count++; 568337467Smarkj vm_phys_freecnt_adj(m, 1); 569337262Smarkj mtx_unlock(&vm_page_queue_free_mtx); 570337467Smarkj 571337467Smarkj vm_cnt.v_page_count++; 572337262Smarkj } 573337262Smarkj pmap_remove(kernel_pmap, start, end); 574337262Smarkj (void)vmem_add(kernel_arena, start, end - start, M_WAITOK); 575337262Smarkj#endif 576337262Smarkj} 577337262Smarkj 578188967Srwatson#ifdef DIAGNOSTIC 579188964Srwatson/* 580188964Srwatson * Allow userspace to directly trigger the VM drain routine for testing 581188964Srwatson * purposes. 582188964Srwatson */ 583188964Srwatsonstatic int 584188964Srwatsondebug_vm_lowmem(SYSCTL_HANDLER_ARGS) 585188964Srwatson{ 586188964Srwatson int error, i; 587188964Srwatson 588188964Srwatson i = 0; 589188964Srwatson error = sysctl_handle_int(oidp, &i, 0, req); 590188964Srwatson if (error) 591188964Srwatson return (error); 592314663Savg if ((i & ~(VM_LOW_KMEM | VM_LOW_PAGES)) != 0) 593314663Savg return (EINVAL); 594314663Savg if (i != 0) 595314663Savg EVENTHANDLER_INVOKE(vm_lowmem, i); 596188964Srwatson return (0); 597188964Srwatson} 598188964Srwatson 599188964SrwatsonSYSCTL_PROC(_debug, OID_AUTO, vm_lowmem, CTLTYPE_INT | CTLFLAG_RW, 0, 0, 600314663Savg debug_vm_lowmem, "I", "set to trigger vm_lowmem event with given flags"); 601188967Srwatson#endif 602