vm_page.c revision 162750
1118611Snjl/*- 2118611Snjl * Copyright (c) 1991 Regents of the University of California. 3207344Sjkim * All rights reserved. 4118611Snjl * 5118611Snjl * This code is derived from software contributed to Berkeley by 6118611Snjl * The Mach Operating System project at Carnegie-Mellon University. 7217365Sjkim * 8298714Sjkim * Redistribution and use in source and binary forms, with or without 9118611Snjl * modification, are permitted provided that the following conditions 10118611Snjl * are met: 11217365Sjkim * 1. Redistributions of source code must retain the above copyright 12217365Sjkim * notice, this list of conditions and the following disclaimer. 13217365Sjkim * 2. Redistributions in binary form must reproduce the above copyright 14217365Sjkim * notice, this list of conditions and the following disclaimer in the 15217365Sjkim * documentation and/or other materials provided with the distribution. 16217365Sjkim * 4. Neither the name of the University nor the names of its contributors 17217365Sjkim * may be used to endorse or promote products derived from this software 18217365Sjkim * without specific prior written permission. 19217365Sjkim * 20217365Sjkim * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21217365Sjkim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22217365Sjkim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23217365Sjkim * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24217365Sjkim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25118611Snjl * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26217365Sjkim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27217365Sjkim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28217365Sjkim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29118611Snjl * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30217365Sjkim * SUCH DAMAGE. 31217365Sjkim * 32217365Sjkim * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 33217365Sjkim */ 34217365Sjkim 35217365Sjkim/*- 36217365Sjkim * Copyright (c) 1987, 1990 Carnegie-Mellon University. 37217365Sjkim * All rights reserved. 38217365Sjkim * 39217365Sjkim * Authors: Avadis Tevanian, Jr., Michael Wayne Young 40217365Sjkim * 41217365Sjkim * Permission to use, copy, modify and distribute this software and 42217365Sjkim * its documentation is hereby granted, provided that both the copyright 43118611Snjl * notice and this permission notice appear in all copies of the 44151937Sjkim * software, derivative works or modified versions, and any portions 45118611Snjl * thereof, and that both notices appear in supporting documentation. 46193529Sjkim * 47118611Snjl * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 48118611Snjl * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 49118611Snjl * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 50118611Snjl * 51118611Snjl * Carnegie Mellon requests users of this software to return to 52118611Snjl * 53118611Snjl * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 54118611Snjl * School of Computer Science 55207344Sjkim * Carnegie Mellon University 56207344Sjkim * Pittsburgh PA 15213-3890 57207344Sjkim * 58207344Sjkim * any improvements or extensions that they make and grant Carnegie the 59207344Sjkim * rights to redistribute these changes. 60207344Sjkim */ 61207344Sjkim 62207344Sjkim/* 63207344Sjkim * GENERAL RULES ON VM_PAGE MANIPULATION 64207344Sjkim * 65207344Sjkim * - a pageq mutex is required when adding or removing a page from a 66207344Sjkim * page queue (vm_page_queue[]), regardless of other mutexes or the 67207344Sjkim * busy state of a page. 68213806Sjkim * 69207344Sjkim * - a hash chain mutex is required when associating or disassociating 70207344Sjkim * a page from the VM PAGE CACHE hash table (vm_page_buckets), 71207344Sjkim * regardless of other mutexes or the busy state of a page. 72207344Sjkim * 73207344Sjkim * - either a hash chain mutex OR a busied page is required in order 74207344Sjkim * to modify the page flags. A hash chain mutex must be obtained in 75207344Sjkim * order to busy a page. A page's flags cannot be modified by a 76207344Sjkim * hash chain mutex if the page is marked busy. 77207344Sjkim * 78207344Sjkim * - The object memq mutex is held when inserting or removing 79207344Sjkim * pages from an object (vm_page_insert() or vm_page_remove()). This 80207344Sjkim * is different from the object's main mutex. 81207344Sjkim * 82207344Sjkim * Generally speaking, you have to be aware of side effects when running 83207344Sjkim * vm_page ops. A vm_page_lookup() will return with the hash chain 84207344Sjkim * locked, whether it was able to lookup the page or not. vm_page_free(), 85207344Sjkim * vm_page_cache(), vm_page_activate(), and a number of other routines 86207344Sjkim * will release the hash chain mutex for you. Intermediate manipulation 87207344Sjkim * routines such as vm_page_flag_set() expect the hash chain to be held 88213806Sjkim * on entry and the hash chain will remain held on return. 89213806Sjkim * 90207344Sjkim * pageq scanning can only occur with the pageq in question locked. 91207344Sjkim * We have a known bottleneck with the active queue, but the cache 92207344Sjkim * and free queues are actually arrays already. 93207344Sjkim */ 94207344Sjkim 95207344Sjkim/* 96207344Sjkim * Resident memory management module. 97213806Sjkim */ 98213806Sjkim 99213806Sjkim#include <sys/cdefs.h> 100213806Sjkim__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 162750 2006-09-29 00:20:56Z kensmith $"); 101213806Sjkim 102213806Sjkim#include <sys/param.h> 103213806Sjkim#include <sys/systm.h> 104213806Sjkim#include <sys/lock.h> 105213806Sjkim#include <sys/kernel.h> 106213806Sjkim#include <sys/malloc.h> 107213806Sjkim#include <sys/mutex.h> 108213806Sjkim#include <sys/proc.h> 109213806Sjkim#include <sys/sysctl.h> 110213806Sjkim#include <sys/vmmeter.h> 111213806Sjkim#include <sys/vnode.h> 112213806Sjkim 113213806Sjkim#include <vm/vm.h> 114213806Sjkim#include <vm/vm_param.h> 115213806Sjkim#include <vm/vm_kern.h> 116213806Sjkim#include <vm/vm_object.h> 117213806Sjkim#include <vm/vm_page.h> 118213806Sjkim#include <vm/vm_pageout.h> 119213806Sjkim#include <vm/vm_pager.h> 120213806Sjkim#include <vm/vm_extern.h> 121213806Sjkim#include <vm/uma.h> 122213806Sjkim#include <vm/uma_int.h> 123213806Sjkim 124213806Sjkim#include <machine/md_var.h> 125272444Sjkim 126272444Sjkim/* 127272444Sjkim * Associated with page of user-allocatable memory is a 128272444Sjkim * page structure. 129207344Sjkim */ 130207344Sjkim 131207344Sjkimstruct mtx vm_page_queue_mtx; 132207344Sjkimstruct mtx vm_page_queue_free_mtx; 133207344Sjkim 134207344Sjkimvm_page_t vm_page_array = 0; 135207344Sjkimint vm_page_array_size = 0; 136207344Sjkimlong first_page = 0; 137207344Sjkimint vm_page_zero_count = 0; 138207344Sjkim 139207344Sjkimstatic int boot_pages = UMA_BOOT_PAGES; 140207344SjkimTUNABLE_INT("vm.boot_pages", &boot_pages); 141272444SjkimSYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0, 142272444Sjkim "number of pages allocated for bootstrapping the VM system"); 143272444Sjkim 144272444Sjkim/* 145272444Sjkim * vm_set_page_size: 146272444Sjkim * 147272444Sjkim * Sets the page size, perhaps based upon the memory 148272444Sjkim * size. Must be called before any use of page-size 149272444Sjkim * dependent functions. 150272444Sjkim */ 151272444Sjkimvoid 152272444Sjkimvm_set_page_size(void) 153272444Sjkim{ 154207344Sjkim if (cnt.v_page_size == 0) 155207344Sjkim cnt.v_page_size = PAGE_SIZE; 156207344Sjkim if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0) 157207344Sjkim panic("vm_set_page_size: page size not a power of two"); 158207344Sjkim} 159207344Sjkim 160207344Sjkim/* 161207344Sjkim * vm_page_blacklist_lookup: 162207344Sjkim * 163207344Sjkim * See if a physical address in this page has been listed 164207344Sjkim * in the blacklist tunable. Entries in the tunable are 165207344Sjkim * separated by spaces or commas. If an invalid integer is 166207344Sjkim * encountered then the rest of the string is skipped. 167207344Sjkim */ 168207344Sjkimstatic int 169207344Sjkimvm_page_blacklist_lookup(char *list, vm_paddr_t pa) 170207344Sjkim{ 171207344Sjkim vm_paddr_t bad; 172207344Sjkim char *cp, *pos; 173207344Sjkim 174207344Sjkim for (pos = list; *pos != '\0'; pos = cp) { 175207344Sjkim bad = strtoq(pos, &cp, 0); 176207344Sjkim if (*cp != '\0') { 177207344Sjkim if (*cp == ' ' || *cp == ',') { 178207344Sjkim cp++; 179207344Sjkim if (cp == pos) 180207344Sjkim continue; 181207344Sjkim } else 182207344Sjkim break; 183207344Sjkim } 184207344Sjkim if (pa == trunc_page(bad)) 185207344Sjkim return (1); 186207344Sjkim } 187207344Sjkim return (0); 188207344Sjkim} 189207344Sjkim 190213806Sjkim/* 191207344Sjkim * vm_page_startup: 192207344Sjkim * 193207344Sjkim * Initializes the resident memory module. 194207344Sjkim * 195207344Sjkim * Allocates memory for the page cells, and 196207344Sjkim * for the object/offset-to-page hash table headers. 197207344Sjkim * Each page cell is initialized and placed on the free list. 198207344Sjkim */ 199207344Sjkimvm_offset_t 200207344Sjkimvm_page_startup(vm_offset_t vaddr) 201207344Sjkim{ 202207344Sjkim vm_offset_t mapped; 203207344Sjkim vm_size_t npages; 204207344Sjkim vm_paddr_t page_range; 205207344Sjkim vm_paddr_t new_end; 206207344Sjkim int i; 207207344Sjkim vm_paddr_t pa; 208207344Sjkim int nblocks; 209207344Sjkim vm_paddr_t last_pa; 210207344Sjkim char *list; 211207344Sjkim 212207344Sjkim /* the biggest memory array is the second group of pages */ 213207344Sjkim vm_paddr_t end; 214207344Sjkim vm_paddr_t biggestsize; 215207344Sjkim int biggestone; 216207344Sjkim 217207344Sjkim vm_paddr_t total; 218207344Sjkim 219207344Sjkim total = 0; 220213806Sjkim biggestsize = 0; 221213806Sjkim biggestone = 0; 222207344Sjkim nblocks = 0; 223207344Sjkim vaddr = round_page(vaddr); 224207344Sjkim 225207344Sjkim for (i = 0; phys_avail[i + 1]; i += 2) { 226207344Sjkim phys_avail[i] = round_page(phys_avail[i]); 227207344Sjkim phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 228207344Sjkim } 229213806Sjkim 230213806Sjkim for (i = 0; phys_avail[i + 1]; i += 2) { 231213806Sjkim vm_paddr_t size = phys_avail[i + 1] - phys_avail[i]; 232213806Sjkim 233213806Sjkim if (size > biggestsize) { 234213806Sjkim biggestone = i; 235213806Sjkim biggestsize = size; 236213806Sjkim } 237213806Sjkim ++nblocks; 238213806Sjkim total += size; 239213806Sjkim } 240213806Sjkim 241213806Sjkim end = phys_avail[biggestone+1]; 242213806Sjkim 243213806Sjkim /* 244213806Sjkim * Initialize the locks. 245213806Sjkim */ 246213806Sjkim mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF | 247213806Sjkim MTX_RECURSE); 248213806Sjkim mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL, 249213806Sjkim MTX_SPIN); 250213806Sjkim 251213806Sjkim /* 252213806Sjkim * Initialize the queue headers for the free queue, the active queue 253213806Sjkim * and the inactive queue. 254213806Sjkim */ 255207344Sjkim vm_pageq_init(); 256207344Sjkim 257207344Sjkim /* 258207344Sjkim * Allocate memory for use when boot strapping the kernel memory 259207344Sjkim * allocator. 260207344Sjkim */ 261207344Sjkim new_end = end - (boot_pages * UMA_SLAB_SIZE); 262207344Sjkim new_end = trunc_page(new_end); 263207344Sjkim mapped = pmap_map(&vaddr, new_end, end, 264207344Sjkim VM_PROT_READ | VM_PROT_WRITE); 265207344Sjkim bzero((void *)mapped, end - new_end); 266207344Sjkim uma_startup((void *)mapped, boot_pages); 267207344Sjkim 268207344Sjkim#if defined(__amd64__) || defined(__i386__) 269207344Sjkim /* 270207344Sjkim * Allocate a bitmap to indicate that a random physical page 271207344Sjkim * needs to be included in a minidump. 272207344Sjkim * 273207344Sjkim * The amd64 port needs this to indicate which direct map pages 274207344Sjkim * need to be dumped, via calls to dump_add_page()/dump_drop_page(). 275207344Sjkim * 276207344Sjkim * However, i386 still needs this workspace internally within the 277207344Sjkim * minidump code. In theory, they are not needed on i386, but are 278207344Sjkim * included should the sf_buf code decide to use them. 279207344Sjkim */ 280207344Sjkim page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE; 281207344Sjkim vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY); 282207344Sjkim new_end -= vm_page_dump_size; 283207344Sjkim vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end, 284207344Sjkim new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE); 285207344Sjkim bzero((void *)vm_page_dump, vm_page_dump_size); 286207344Sjkim#endif 287207344Sjkim /* 288207344Sjkim * Compute the number of pages of memory that will be available for 289207344Sjkim * use (taking into account the overhead of a page structure per 290207344Sjkim * page). 291207344Sjkim */ 292207344Sjkim first_page = phys_avail[0] / PAGE_SIZE; 293207344Sjkim page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page; 294207344Sjkim npages = (total - (page_range * sizeof(struct vm_page)) - 295207344Sjkim (end - new_end)) / PAGE_SIZE; 296207344Sjkim end = new_end; 297207344Sjkim 298207344Sjkim /* 299207344Sjkim * Reserve an unmapped guard page to trap access to vm_page_array[-1]. 300207344Sjkim */ 301207344Sjkim vaddr += PAGE_SIZE; 302207344Sjkim 303207344Sjkim /* 304207344Sjkim * Initialize the mem entry structures now, and put them in the free 305207344Sjkim * queue. 306207344Sjkim */ 307207344Sjkim new_end = trunc_page(end - page_range * sizeof(struct vm_page)); 308207344Sjkim mapped = pmap_map(&vaddr, new_end, end, 309207344Sjkim VM_PROT_READ | VM_PROT_WRITE); 310207344Sjkim vm_page_array = (vm_page_t) mapped; 311207344Sjkim#ifdef __amd64__ 312207344Sjkim /* 313207344Sjkim * pmap_map on amd64 comes out of the direct-map, not kvm like i386, 314207344Sjkim * so the pages must be tracked for a crashdump to include this data. 315207344Sjkim * This includes the vm_page_array and the early UMA bootstrap pages. 316207344Sjkim */ 317207344Sjkim for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE) 318207344Sjkim dump_add_page(pa); 319207344Sjkim#endif 320207344Sjkim phys_avail[biggestone + 1] = new_end; 321207344Sjkim 322250838Sjkim /* 323207344Sjkim * Clear all of the page structures 324207344Sjkim */ 325207344Sjkim bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page)); 326207344Sjkim vm_page_array_size = page_range; 327207344Sjkim 328207344Sjkim /* 329207344Sjkim * Construct the free queue(s) in descending order (by physical 330207344Sjkim * address) so that the first 16MB of physical memory is allocated 331207344Sjkim * last rather than first. On large-memory machines, this avoids 332207344Sjkim * the exhaustion of low physical memory before isa_dma_init has run. 333207344Sjkim */ 334207344Sjkim cnt.v_page_count = 0; 335207344Sjkim cnt.v_free_count = 0; 336207344Sjkim list = getenv("vm.blacklist"); 337207344Sjkim for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) { 338207344Sjkim pa = phys_avail[i]; 339207344Sjkim last_pa = phys_avail[i + 1]; 340207344Sjkim while (pa < last_pa && npages-- > 0) { 341207344Sjkim if (list != NULL && 342207344Sjkim vm_page_blacklist_lookup(list, pa)) 343207344Sjkim printf("Skipping page with pa 0x%jx\n", 344207344Sjkim (uintmax_t)pa); 345207344Sjkim else 346207344Sjkim vm_pageq_add_new_page(pa); 347207344Sjkim pa += PAGE_SIZE; 348207344Sjkim } 349207344Sjkim } 350207344Sjkim freeenv(list); 351207344Sjkim return (vaddr); 352207344Sjkim} 353207344Sjkim 354207344Sjkimvoid 355207344Sjkimvm_page_flag_set(vm_page_t m, unsigned short bits) 356207344Sjkim{ 357207344Sjkim 358207344Sjkim mtx_assert(&vm_page_queue_mtx, MA_OWNED); 359207344Sjkim m->flags |= bits; 360207344Sjkim} 361207344Sjkim 362207344Sjkimvoid 363207344Sjkimvm_page_flag_clear(vm_page_t m, unsigned short bits) 364207344Sjkim{ 365207344Sjkim 366207344Sjkim mtx_assert(&vm_page_queue_mtx, MA_OWNED); 367207344Sjkim m->flags &= ~bits; 368250838Sjkim} 369207344Sjkim 370207344Sjkimvoid 371207344Sjkimvm_page_busy(vm_page_t m) 372207344Sjkim{ 373207344Sjkim 374207344Sjkim VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 375207344Sjkim KASSERT((m->flags & PG_BUSY) == 0, 376207344Sjkim ("vm_page_busy: page already busy!!!")); 377207344Sjkim vm_page_flag_set(m, PG_BUSY); 378207344Sjkim} 379207344Sjkim 380207344Sjkim/* 381207344Sjkim * vm_page_flash: 382207344Sjkim * 383207344Sjkim * wakeup anyone waiting for the page. 384207344Sjkim */ 385207344Sjkimvoid 386207344Sjkimvm_page_flash(vm_page_t m) 387207344Sjkim{ 388207344Sjkim 389207344Sjkim VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 390207344Sjkim if (m->oflags & VPO_WANTED) { 391207344Sjkim m->oflags &= ~VPO_WANTED; 392207344Sjkim wakeup(m); 393207344Sjkim } 394207344Sjkim} 395207344Sjkim 396207344Sjkim/* 397207344Sjkim * vm_page_wakeup: 398207344Sjkim * 399298714Sjkim * clear the PG_BUSY flag and wakeup anyone waiting for the 400207344Sjkim * page. 401207344Sjkim * 402207344Sjkim */ 403241973Sjkimvoid 404207344Sjkimvm_page_wakeup(vm_page_t m) 405207344Sjkim{ 406207344Sjkim 407207344Sjkim VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 408207344Sjkim KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!")); 409118611Snjl vm_page_flag_clear(m, PG_BUSY); 410118611Snjl vm_page_flash(m); 411118611Snjl} 412118611Snjl 413118611Snjlvoid 414118611Snjlvm_page_io_start(vm_page_t m) 415118611Snjl{ 416118611Snjl 417118611Snjl VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 418118611Snjl m->busy++; 419118611Snjl} 420118611Snjl 421118611Snjlvoid 422118611Snjlvm_page_io_finish(vm_page_t m) 423118611Snjl{ 424118611Snjl 425118611Snjl VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 426118611Snjl m->busy--; 427118611Snjl if (m->busy == 0) 428118611Snjl vm_page_flash(m); 429118611Snjl} 430118611Snjl 431118611Snjl/* 432118611Snjl * Keep page from being freed by the page daemon 433118611Snjl * much of the same effect as wiring, except much lower 434118611Snjl * overhead and should be used only for *very* temporary 435118611Snjl * holding ("wiring"). 436118611Snjl */ 437118611Snjlvoid 438118611Snjlvm_page_hold(vm_page_t mem) 439118611Snjl{ 440118611Snjl 441228110Sjkim mtx_assert(&vm_page_queue_mtx, MA_OWNED); 442118611Snjl mem->hold_count++; 443151937Sjkim} 444118611Snjl 445118611Snjlvoid 446118611Snjlvm_page_unhold(vm_page_t mem) 447118611Snjl{ 448228110Sjkim 449118611Snjl mtx_assert(&vm_page_queue_mtx, MA_OWNED); 450118611Snjl --mem->hold_count; 451118611Snjl KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!")); 452118611Snjl if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD)) 453241973Sjkim vm_page_free_toq(mem); 454118611Snjl} 455118611Snjl 456118611Snjl/* 457118611Snjl * vm_page_free: 458118611Snjl * 459228110Sjkim * Free a page 460118611Snjl * 461118611Snjl * The clearing of PG_ZERO is a temporary safety until the code can be 462118611Snjl * reviewed to determine that PG_ZERO is being properly cleared on 463228110Sjkim * write faults or maps. PG_ZERO was previously cleared in 464228110Sjkim * vm_page_alloc(). 465118611Snjl */ 466118611Snjlvoid 467228110Sjkimvm_page_free(vm_page_t m) 468228110Sjkim{ 469118611Snjl vm_page_flag_clear(m, PG_ZERO); 470228110Sjkim vm_page_free_toq(m); 471228110Sjkim vm_page_zero_idle_wakeup(); 472118611Snjl} 473118611Snjl 474118611Snjl/* 475118611Snjl * vm_page_free_zero: 476118611Snjl * 477118611Snjl * Free a page to the zerod-pages queue 478118611Snjl */ 479118611Snjlvoid 480151937Sjkimvm_page_free_zero(vm_page_t m) 481118611Snjl{ 482118611Snjl vm_page_flag_set(m, PG_ZERO); 483118611Snjl vm_page_free_toq(m); 484118611Snjl} 485118611Snjl 486118611Snjl/* 487241973Sjkim * vm_page_sleep: 488241973Sjkim * 489118611Snjl * Sleep and release the page queues lock. 490118611Snjl * 491118611Snjl * The object containing the given page must be locked. 492118611Snjl */ 493118611Snjlvoid 494118611Snjlvm_page_sleep(vm_page_t m, const char *msg) 495118611Snjl{ 496118611Snjl 497118611Snjl VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 498118611Snjl if (!mtx_owned(&vm_page_queue_mtx)) 499118611Snjl vm_page_lock_queues(); 500118611Snjl vm_page_flag_set(m, PG_REFERENCED); 501118611Snjl vm_page_unlock_queues(); 502118611Snjl 503118611Snjl /* 504118611Snjl * It's possible that while we sleep, the page will get 505118611Snjl * unbusied and freed. If we are holding the object 506118611Snjl * lock, we will assume we hold a reference to the object 507118611Snjl * such that even if m->object changes, we can re-lock 508118611Snjl * it. 509118611Snjl */ 510118611Snjl m->oflags |= VPO_WANTED; 511118611Snjl msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0); 512118611Snjl} 513118611Snjl 514118611Snjl/* 515118611Snjl * vm_page_dirty: 516228110Sjkim * 517228110Sjkim * make page all dirty 518228110Sjkim */ 519228110Sjkimvoid 520228110Sjkimvm_page_dirty(vm_page_t m) 521228110Sjkim{ 522228110Sjkim KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_CACHE, 523228110Sjkim ("vm_page_dirty: page in cache!")); 524228110Sjkim KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_FREE, 525228110Sjkim ("vm_page_dirty: page is free!")); 526228110Sjkim m->dirty = VM_PAGE_BITS_ALL; 527228110Sjkim} 528228110Sjkim 529228110Sjkim/* 530228110Sjkim * vm_page_splay: 531228110Sjkim * 532228110Sjkim * Implements Sleator and Tarjan's top-down splay algorithm. Returns 533228110Sjkim * the vm_page containing the given pindex. If, however, that 534228110Sjkim * pindex is not found in the vm_object, returns a vm_page that is 535228110Sjkim * adjacent to the pindex, coming before or after it. 536228110Sjkim */ 537228110Sjkimvm_page_t 538228110Sjkimvm_page_splay(vm_pindex_t pindex, vm_page_t root) 539118611Snjl{ 540118611Snjl struct vm_page dummy; 541118611Snjl vm_page_t lefttreemax, righttreemin, y; 542118611Snjl 543118611Snjl if (root == NULL) 544118611Snjl return (root); 545118611Snjl lefttreemax = righttreemin = &dummy; 546118611Snjl for (;; root = y) { 547118611Snjl if (pindex < root->pindex) { 548118611Snjl if ((y = root->left) == NULL) 549118611Snjl break; 550118611Snjl if (pindex < y->pindex) { 551118611Snjl /* Rotate right. */ 552118611Snjl root->left = y->right; 553118611Snjl y->right = root; 554118611Snjl root = y; 555118611Snjl if ((y = root->left) == NULL) 556118611Snjl break; 557118611Snjl } 558118611Snjl /* Link into the new root's right tree. */ 559118611Snjl righttreemin->left = root; 560118611Snjl righttreemin = root; 561118611Snjl } else if (pindex > root->pindex) { 562118611Snjl if ((y = root->right) == NULL) 563118611Snjl break; 564118611Snjl if (pindex > y->pindex) { 565118611Snjl /* Rotate left. */ 566118611Snjl root->right = y->left; 567118611Snjl y->left = root; 568118611Snjl root = y; 569118611Snjl if ((y = root->right) == NULL) 570151937Sjkim break; 571151937Sjkim } 572151937Sjkim /* Link into the new root's left tree. */ 573151937Sjkim lefttreemax->right = root; 574151937Sjkim lefttreemax = root; 575151937Sjkim } else 576151937Sjkim break; 577151937Sjkim } 578151937Sjkim /* Assemble the new root. */ 579151937Sjkim lefttreemax->right = root->left; 580151937Sjkim righttreemin->left = root->right; 581151937Sjkim root->left = dummy.right; 582151937Sjkim root->right = dummy.left; 583151937Sjkim return (root); 584151937Sjkim} 585151937Sjkim 586151937Sjkim/* 587151937Sjkim * vm_page_insert: [ internal use only ] 588151937Sjkim * 589151937Sjkim * Inserts the given mem entry into the object and object list. 590151937Sjkim * 591151937Sjkim * The pagetables are not updated but will presumably fault the page 592151937Sjkim * in if necessary, or if a kernel page the caller will at some point 593151937Sjkim * enter the page into the kernel's pmap. We are not allowed to block 594151937Sjkim * here so we *can't* do this anyway. 595151937Sjkim * 596151937Sjkim * The object and page must be locked. 597151937Sjkim * This routine may not block. 598151937Sjkim */ 599151937Sjkimvoid 600151937Sjkimvm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) 601151937Sjkim{ 602151937Sjkim vm_page_t root; 603151937Sjkim 604151937Sjkim VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 605151937Sjkim if (m->object != NULL) 606151937Sjkim panic("vm_page_insert: page already inserted"); 607151937Sjkim 608151937Sjkim /* 609151937Sjkim * Record the object/offset pair in this page 610151937Sjkim */ 611151937Sjkim m->object = object; 612151937Sjkim m->pindex = pindex; 613151937Sjkim 614151937Sjkim /* 615151937Sjkim * Now link into the object's ordered list of backed pages. 616151937Sjkim */ 617151937Sjkim root = object->root; 618151937Sjkim if (root == NULL) { 619151937Sjkim m->left = NULL; 620151937Sjkim m->right = NULL; 621151937Sjkim TAILQ_INSERT_TAIL(&object->memq, m, listq); 622151937Sjkim } else { 623151937Sjkim root = vm_page_splay(pindex, root); 624151937Sjkim if (pindex < root->pindex) { 625151937Sjkim m->left = root->left; 626151937Sjkim m->right = root; 627151937Sjkim root->left = NULL; 628151937Sjkim TAILQ_INSERT_BEFORE(root, m, listq); 629151937Sjkim } else if (pindex == root->pindex) 630151937Sjkim panic("vm_page_insert: offset already allocated"); 631151937Sjkim else { 632151937Sjkim m->right = root->right; 633118611Snjl m->left = root; 634118611Snjl root->right = NULL; 635118611Snjl TAILQ_INSERT_AFTER(&object->memq, root, m, listq); 636118611Snjl } 637118611Snjl } 638118611Snjl object->root = m; 639118611Snjl object->generation++; 640118611Snjl 641118611Snjl /* 642118611Snjl * show that the object has one more resident page. 643118611Snjl */ 644118611Snjl object->resident_page_count++; 645118611Snjl /* 646118611Snjl * Hold the vnode until the last page is released. 647272444Sjkim */ 648118611Snjl if (object->resident_page_count == 1 && object->type == OBJT_VNODE) 649118611Snjl vhold((struct vnode *)object->handle); 650118611Snjl 651118611Snjl /* 652118611Snjl * Since we are inserting a new and possibly dirty page, 653167802Sjkim * update the object's OBJ_MIGHTBEDIRTY flag. 654118611Snjl */ 655272444Sjkim if (m->flags & PG_WRITEABLE) 656118611Snjl vm_object_set_writeable_dirty(object); 657118611Snjl} 658250838Sjkim 659272444Sjkim/* 660118611Snjl * vm_page_remove: 661118611Snjl * NOTE: used by device pager as well -wfj 662228110Sjkim * 663250838Sjkim * Removes the given mem entry from the object/offset-page 664272444Sjkim * table and the object page list, but do not invalidate/terminate 665228110Sjkim * the backing store. 666228110Sjkim * 667118611Snjl * The object and page must be locked. 668250838Sjkim * The underlying pmap entry (if any) is NOT removed here. 669272444Sjkim * This routine may not block. 670118611Snjl */ 671118611Snjlvoid 672118611Snjlvm_page_remove(vm_page_t m) 673250838Sjkim{ 674272444Sjkim vm_object_t object; 675118611Snjl vm_page_t root; 676118611Snjl 677151937Sjkim mtx_assert(&vm_page_queue_mtx, MA_OWNED); 678250838Sjkim if ((object = m->object) == NULL) 679272444Sjkim return; 680151937Sjkim VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 681151937Sjkim if (m->flags & PG_BUSY) { 682118611Snjl vm_page_flag_clear(m, PG_BUSY); 683250838Sjkim vm_page_flash(m); 684118611Snjl } 685118611Snjl 686118611Snjl /* 687250838Sjkim * Now remove from the object's list of backed pages. 688151937Sjkim */ 689272444Sjkim if (m != object->root) 690118611Snjl vm_page_splay(m->pindex, object->root); 691118611Snjl if (m->left == NULL) 692118611Snjl root = m->right; 693250838Sjkim else { 694151937Sjkim root = vm_page_splay(m->pindex, m->left); 695272444Sjkim root->right = m->right; 696118611Snjl } 697118611Snjl object->root = root; 698118611Snjl TAILQ_REMOVE(&object->memq, m, listq); 699118611Snjl 700250838Sjkim /* 701118611Snjl * And show that the object has one fewer resident page. 702118611Snjl */ 703118611Snjl object->resident_page_count--; 704118611Snjl object->generation++; 705272444Sjkim /* 706118611Snjl * The vnode may now be recycled. 707118611Snjl */ 708167802Sjkim if (object->resident_page_count == 0 && object->type == OBJT_VNODE) 709250838Sjkim vdrop((struct vnode *)object->handle); 710272444Sjkim 711167802Sjkim m->object = NULL; 712167802Sjkim} 713151937Sjkim 714250838Sjkim/* 715272444Sjkim * vm_page_lookup: 716151937Sjkim * 717151937Sjkim * Returns the page associated with the object/offset 718151937Sjkim * pair specified; if none is found, NULL is returned. 719250838Sjkim * 720272444Sjkim * The object must be locked. 721151937Sjkim * This routine may not block. 722151937Sjkim * This is a critical path routine 723151937Sjkim */ 724250838Sjkimvm_page_t 725272444Sjkimvm_page_lookup(vm_object_t object, vm_pindex_t pindex) 726151937Sjkim{ 727151937Sjkim vm_page_t m; 728118611Snjl 729250838Sjkim VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 730272444Sjkim if ((m = object->root) != NULL && m->pindex != pindex) { 731118611Snjl m = vm_page_splay(pindex, m); 732118611Snjl if ((object->root = m)->pindex != pindex) 733118611Snjl m = NULL; 734250838Sjkim } 735272444Sjkim return (m); 736118611Snjl} 737118611Snjl 738118611Snjl/* 739250838Sjkim * vm_page_rename: 740272444Sjkim * 741118611Snjl * Move the given memory entry from its 742118611Snjl * current object to the specified target object/offset. 743118611Snjl * 744250838Sjkim * The object must be locked. 745272444Sjkim * This routine may not block. 746118611Snjl * 747118611Snjl * Note: swap associated with the page must be invalidated by the move. We 748118611Snjl * have to do this for several reasons: (1) we aren't freeing the 749250838Sjkim * page, (2) we are dirtying the page, (3) the VM system is probably 750272444Sjkim * moving the page from object A to B, and will then later move 751118611Snjl * the backing store from A to B and we can't have a conflict. 752118611Snjl * 753118611Snjl * Note: we *always* dirty the page. It is necessary both for the 754250838Sjkim * fact that we moved it, and because we may be invalidating 755272444Sjkim * swap. If the page is on the cache, we have to deactivate it 756118611Snjl * or vm_page_dirty() will panic. Dirty pages are not allowed 757118611Snjl * on the cache. 758118611Snjl */ 759250838Sjkimvoid 760272444Sjkimvm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) 761118611Snjl{ 762118611Snjl 763118611Snjl vm_page_remove(m); 764250838Sjkim vm_page_insert(m, new_object, new_pindex); 765272444Sjkim if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) 766118611Snjl vm_page_deactivate(m); 767118611Snjl vm_page_dirty(m); 768118611Snjl} 769250838Sjkim 770272444Sjkim/* 771118611Snjl * vm_page_select_cache: 772118611Snjl * 773118611Snjl * Move a page of the given color from the cache queue to the free 774250838Sjkim * queue. As pages might be found, but are not applicable, they are 775272444Sjkim * deactivated. 776118611Snjl * 777118611Snjl * This routine may not block. 778151937Sjkim */ 779250838Sjkimvm_page_t 780272444Sjkimvm_page_select_cache(int color) 781151937Sjkim{ 782151937Sjkim vm_object_t object; 783118611Snjl vm_page_t m; 784250838Sjkim boolean_t was_trylocked; 785272444Sjkim 786118611Snjl mtx_assert(&vm_page_queue_mtx, MA_OWNED); 787118611Snjl while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) { 788118611Snjl KASSERT(m->dirty == 0, ("Found dirty cache page %p", m)); 789250838Sjkim KASSERT(!pmap_page_is_mapped(m), 790118611Snjl ("Found mapped cache page %p", m)); 791118611Snjl KASSERT((m->flags & PG_UNMANAGED) == 0, 792118611Snjl ("Found unmanaged cache page %p", m)); 793250838Sjkim KASSERT(m->wire_count == 0, ("Found wired cache page %p", m)); 794151937Sjkim if (m->hold_count == 0 && (object = m->object, 795272444Sjkim (was_trylocked = VM_OBJECT_TRYLOCK(object)) || 796118611Snjl VM_OBJECT_LOCKED(object))) { 797118611Snjl KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0, 798118611Snjl ("Found busy cache page %p", m)); 799118611Snjl vm_page_free(m); 800118611Snjl if (was_trylocked) 801250838Sjkim VM_OBJECT_UNLOCK(object); 802118611Snjl break; 803118611Snjl } 804118611Snjl vm_page_deactivate(m); 805118611Snjl } 806272444Sjkim return (m); 807118611Snjl} 808118611Snjl 809118611Snjl/* 810118611Snjl * vm_page_alloc: 811250838Sjkim * 812118611Snjl * Allocate and return a memory cell associated 813118611Snjl * with this VM object/offset pair. 814118611Snjl * 815250838Sjkim * page_req classes: 816151937Sjkim * VM_ALLOC_NORMAL normal process request 817272444Sjkim * VM_ALLOC_SYSTEM system *really* needs a page 818118611Snjl * VM_ALLOC_INTERRUPT interrupt time request 819118611Snjl * VM_ALLOC_ZERO zero page 820118611Snjl * 821118611Snjl * This routine may not block. 822118611Snjl * 823250838Sjkim * Additional special handling is required when called from an 824118611Snjl * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with 825118611Snjl * the page cache in this case. 826118611Snjl */ 827118611Snjlvm_page_t 828272444Sjkimvm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) 829118611Snjl{ 830118611Snjl vm_page_t m = NULL; 831118611Snjl int color, flags, page_req; 832118611Snjl 833250838Sjkim page_req = req & VM_ALLOC_CLASS_MASK; 834272444Sjkim KASSERT(curthread->td_intr_nesting_level == 0 || 835118611Snjl page_req == VM_ALLOC_INTERRUPT, 836118611Snjl ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context")); 837118611Snjl 838250838Sjkim if ((req & VM_ALLOC_NOOBJ) == 0) { 839272444Sjkim KASSERT(object != NULL, 840118611Snjl ("vm_page_alloc: NULL object.")); 841118611Snjl VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 842118611Snjl color = (pindex + object->pg_color) & PQ_COLORMASK; 843250838Sjkim } else 844272444Sjkim color = pindex & PQ_COLORMASK; 845118611Snjl 846118611Snjl /* 847118611Snjl * The pager is allowed to eat deeper into the free page list. 848250838Sjkim */ 849272444Sjkim if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) { 850118611Snjl page_req = VM_ALLOC_SYSTEM; 851118611Snjl }; 852151937Sjkim 853250838Sjkimloop: 854272444Sjkim mtx_lock_spin(&vm_page_queue_free_mtx); 855151937Sjkim if (cnt.v_free_count > cnt.v_free_reserved || 856151937Sjkim (page_req == VM_ALLOC_SYSTEM && 857228110Sjkim cnt.v_cache_count == 0 && 858250838Sjkim cnt.v_free_count > cnt.v_interrupt_free_min) || 859272444Sjkim (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) { 860228110Sjkim /* 861228110Sjkim * Allocate from the free queue if the number of free pages 862228110Sjkim * exceeds the minimum for the request class. 863250838Sjkim */ 864272444Sjkim m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0); 865228110Sjkim } else if (page_req != VM_ALLOC_INTERRUPT) { 866228110Sjkim mtx_unlock_spin(&vm_page_queue_free_mtx); 867228110Sjkim /* 868298714Sjkim * Allocatable from cache (non-interrupt only). On success, 869250838Sjkim * we must free the page and try again, thus ensuring that 870272444Sjkim * cnt.v_*_free_min counters are replenished. 871228110Sjkim */ 872228110Sjkim vm_page_lock_queues(); 873228110Sjkim if ((m = vm_page_select_cache(color)) == NULL) { 874298714Sjkim KASSERT(cnt.v_cache_count == 0, 875250838Sjkim ("vm_page_alloc: cache queue is missing %d pages", 876272444Sjkim cnt.v_cache_count)); 877228110Sjkim vm_page_unlock_queues(); 878228110Sjkim atomic_add_int(&vm_pageout_deficit, 1); 879228110Sjkim pagedaemon_wakeup(); 880298714Sjkim 881250838Sjkim if (page_req != VM_ALLOC_SYSTEM) 882272444Sjkim return (NULL); 883228110Sjkim 884228110Sjkim mtx_lock_spin(&vm_page_queue_free_mtx); 885118611Snjl if (cnt.v_free_count <= cnt.v_interrupt_free_min) { 886250838Sjkim mtx_unlock_spin(&vm_page_queue_free_mtx); 887118611Snjl return (NULL); 888118611Snjl } 889118611Snjl m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0); 890118611Snjl } else { 891250838Sjkim vm_page_unlock_queues(); 892118611Snjl goto loop; 893298714Sjkim } 894118611Snjl } else { 895118611Snjl /* 896118611Snjl * Not allocatable from cache from interrupt, give up. 897118611Snjl */ 898118611Snjl mtx_unlock_spin(&vm_page_queue_free_mtx); 899118611Snjl atomic_add_int(&vm_pageout_deficit, 1); 900118611Snjl pagedaemon_wakeup(); 901118611Snjl return (NULL); 902272444Sjkim } 903272444Sjkim 904272444Sjkim /* 905118611Snjl * At this point we had better have found a good page. 906167802Sjkim */ 907167802Sjkim 908272444Sjkim KASSERT( 909298714Sjkim m != NULL, 910298714Sjkim ("vm_page_alloc(): missing page on free queue") 911167802Sjkim ); 912167802Sjkim 913118611Snjl /* 914118611Snjl * Remove from free queue 915118611Snjl */ 916118611Snjl vm_pageq_remove_nowakeup(m); 917118611Snjl 918118611Snjl /* 919118611Snjl * Initialize structure. Only the PG_ZERO flag is inherited. 920118611Snjl */ 921118611Snjl flags = PG_BUSY; 922118611Snjl if (m->flags & PG_ZERO) { 923118611Snjl vm_page_zero_count--; 924118611Snjl if (req & VM_ALLOC_ZERO) 925118611Snjl flags = PG_ZERO | PG_BUSY; 926118611Snjl } 927118611Snjl if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ)) 928118611Snjl flags &= ~PG_BUSY; 929118611Snjl m->flags = flags; 930118611Snjl m->oflags = 0; 931118611Snjl if (req & VM_ALLOC_WIRED) { 932118611Snjl atomic_add_int(&cnt.v_wire_count, 1); 933118611Snjl m->wire_count = 1; 934118611Snjl } else 935118611Snjl m->wire_count = 0; 936118611Snjl m->hold_count = 0; 937118611Snjl m->act_count = 0; 938118611Snjl m->busy = 0; 939118611Snjl m->valid = 0; 940118611Snjl KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m)); 941118611Snjl mtx_unlock_spin(&vm_page_queue_free_mtx); 942118611Snjl 943118611Snjl if ((req & VM_ALLOC_NOOBJ) == 0) 944118611Snjl vm_page_insert(m, object, pindex); 945241973Sjkim else 946118611Snjl m->pindex = pindex; 947118611Snjl 948118611Snjl /* 949118611Snjl * Don't wakeup too often - wakeup the pageout daemon when 950118611Snjl * we would be nearly out of memory. 951118611Snjl */ 952118611Snjl if (vm_paging_needed()) 953118611Snjl pagedaemon_wakeup(); 954118611Snjl 955118611Snjl return (m); 956118611Snjl} 957118611Snjl 958118611Snjl/* 959118611Snjl * vm_wait: (also see VM_WAIT macro) 960118611Snjl * 961118611Snjl * Block until free pages are available for allocation 962118611Snjl * - Called in various places before memory allocations. 963118611Snjl */ 964118611Snjlvoid 965241973Sjkimvm_wait(void) 966118611Snjl{ 967118611Snjl 968118611Snjl vm_page_lock_queues(); 969118611Snjl if (curproc == pageproc) { 970118611Snjl vm_pageout_pages_needed = 1; 971118611Snjl msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx, 972118611Snjl PDROP | PSWP, "VMWait", 0); 973118611Snjl } else { 974118611Snjl if (!vm_pages_needed) { 975241973Sjkim vm_pages_needed = 1; 976118611Snjl wakeup(&vm_pages_needed); 977118611Snjl } 978118611Snjl msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM, 979118611Snjl "vmwait", 0); 980118611Snjl } 981118611Snjl} 982118611Snjl 983118611Snjl/* 984118611Snjl * vm_waitpfault: (also see VM_WAITPFAULT macro) 985118611Snjl * 986118611Snjl * Block until free pages are available for allocation 987118611Snjl * - Called only in vm_fault so that processes page faulting 988118611Snjl * can be easily tracked. 989118611Snjl * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing 990118611Snjl * processes will be able to grab memory first. Do not change 991118611Snjl * this balance without careful testing first. 992118611Snjl */ 993118611Snjlvoid 994272444Sjkimvm_waitpfault(void) 995118611Snjl{ 996118611Snjl 997118611Snjl vm_page_lock_queues(); 998167802Sjkim if (!vm_pages_needed) { 999167802Sjkim vm_pages_needed = 1; 1000167802Sjkim wakeup(&vm_pages_needed); 1001167802Sjkim } 1002167802Sjkim msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER, 1003167802Sjkim "pfault", 0); 1004167802Sjkim} 1005118611Snjl 1006118611Snjl/* 1007118611Snjl * vm_page_activate: 1008118611Snjl * 1009118611Snjl * Put the specified page on the active list (if appropriate). 1010118611Snjl * Ensure that act_count is at least ACT_INIT but do not otherwise 1011118611Snjl * mess with it. 1012118611Snjl * 1013118611Snjl * The page queues must be locked. 1014118611Snjl * This routine may not block. 1015118611Snjl */ 1016118611Snjlvoid 1017118611Snjlvm_page_activate(vm_page_t m) 1018167802Sjkim{ 1019167802Sjkim 1020167802Sjkim mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1021167802Sjkim if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) { 1022167802Sjkim if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) 1023118611Snjl cnt.v_reactivated++; 1024118611Snjl vm_pageq_remove(m); 1025118611Snjl if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { 1026118611Snjl if (m->act_count < ACT_INIT) 1027272444Sjkim m->act_count = ACT_INIT; 1028272444Sjkim vm_pageq_enqueue(PQ_ACTIVE, m); 1029272444Sjkim } 1030272444Sjkim } else { 1031272444Sjkim if (m->act_count < ACT_INIT) 1032272444Sjkim m->act_count = ACT_INIT; 1033272444Sjkim } 1034272444Sjkim} 1035272444Sjkim 1036272444Sjkim/* 1037272444Sjkim * vm_page_free_wakeup: 1038272444Sjkim * 1039272444Sjkim * Helper routine for vm_page_free_toq() and vm_page_cache(). This 1040272444Sjkim * routine is called when a page has been added to the cache or free 1041167802Sjkim * queues. 1042272444Sjkim * 1043118611Snjl * The page queues must be locked. 1044118611Snjl * This routine may not block. 1045118611Snjl */ 1046241973Sjkimstatic inline void 1047118611Snjlvm_page_free_wakeup(void) 1048118611Snjl{ 1049118611Snjl 1050118611Snjl mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1051118611Snjl /* 1052118611Snjl * if pageout daemon needs pages, then tell it that there are 1053118611Snjl * some free. 1054118611Snjl */ 1055118611Snjl if (vm_pageout_pages_needed && 1056118611Snjl cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) { 1057118611Snjl wakeup(&vm_pageout_pages_needed); 1058118611Snjl vm_pageout_pages_needed = 0; 1059118611Snjl } 1060118611Snjl /* 1061118611Snjl * wakeup processes that are waiting on memory if we hit a 1062118611Snjl * high water mark. And wakeup scheduler process if we have 1063118611Snjl * lots of memory. this process will swapin processes. 1064118611Snjl */ 1065118611Snjl if (vm_pages_needed && !vm_page_count_min()) { 1066118611Snjl vm_pages_needed = 0; 1067118611Snjl wakeup(&cnt.v_free_count); 1068118611Snjl } 1069118611Snjl} 1070118611Snjl 1071118611Snjl/* 1072118611Snjl * vm_page_free_toq: 1073118611Snjl * 1074118611Snjl * Returns the given page to the PQ_FREE list, 1075118611Snjl * disassociating it with any VM object. 1076167802Sjkim * 1077228110Sjkim * Object and page must be locked prior to entry. 1078118611Snjl * This routine may not block. 1079118611Snjl */ 1080118611Snjl 1081118611Snjlvoid 1082228110Sjkimvm_page_free_toq(vm_page_t m) 1083118611Snjl{ 1084118611Snjl struct vpgqueues *pq; 1085118611Snjl 1086118611Snjl mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1087118611Snjl KASSERT(!pmap_page_is_mapped(m), 1088118611Snjl ("vm_page_free_toq: freeing mapped page %p", m)); 1089167802Sjkim cnt.v_tfree++; 1090228110Sjkim 1091118611Snjl if (m->busy || VM_PAGE_INQUEUE1(m, PQ_FREE)) { 1092118611Snjl printf( 1093118611Snjl "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n", 1094 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0, 1095 m->hold_count); 1096 if (VM_PAGE_INQUEUE1(m, PQ_FREE)) 1097 panic("vm_page_free: freeing free page"); 1098 else 1099 panic("vm_page_free: freeing busy page"); 1100 } 1101 1102 /* 1103 * unqueue, then remove page. Note that we cannot destroy 1104 * the page here because we do not want to call the pager's 1105 * callback routine until after we've put the page on the 1106 * appropriate free queue. 1107 */ 1108 vm_pageq_remove_nowakeup(m); 1109 vm_page_remove(m); 1110 1111 /* 1112 * If fictitious remove object association and 1113 * return, otherwise delay object association removal. 1114 */ 1115 if ((m->flags & PG_FICTITIOUS) != 0) { 1116 return; 1117 } 1118 1119 m->valid = 0; 1120 vm_page_undirty(m); 1121 1122 if (m->wire_count != 0) { 1123 if (m->wire_count > 1) { 1124 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx", 1125 m->wire_count, (long)m->pindex); 1126 } 1127 panic("vm_page_free: freeing wired page"); 1128 } 1129 if (m->hold_count != 0) { 1130 m->flags &= ~PG_ZERO; 1131 VM_PAGE_SETQUEUE2(m, PQ_HOLD); 1132 } else 1133 VM_PAGE_SETQUEUE1(m, PQ_FREE); 1134 pq = &vm_page_queues[VM_PAGE_GETQUEUE(m)]; 1135 mtx_lock_spin(&vm_page_queue_free_mtx); 1136 pq->lcnt++; 1137 ++(*pq->cnt); 1138 1139 /* 1140 * Put zero'd pages on the end ( where we look for zero'd pages 1141 * first ) and non-zerod pages at the head. 1142 */ 1143 if (m->flags & PG_ZERO) { 1144 TAILQ_INSERT_TAIL(&pq->pl, m, pageq); 1145 ++vm_page_zero_count; 1146 } else { 1147 TAILQ_INSERT_HEAD(&pq->pl, m, pageq); 1148 } 1149 mtx_unlock_spin(&vm_page_queue_free_mtx); 1150 vm_page_free_wakeup(); 1151} 1152 1153/* 1154 * vm_page_unmanage: 1155 * 1156 * Prevent PV management from being done on the page. The page is 1157 * removed from the paging queues as if it were wired, and as a 1158 * consequence of no longer being managed the pageout daemon will not 1159 * touch it (since there is no way to locate the pte mappings for the 1160 * page). madvise() calls that mess with the pmap will also no longer 1161 * operate on the page. 1162 * 1163 * Beyond that the page is still reasonably 'normal'. Freeing the page 1164 * will clear the flag. 1165 * 1166 * This routine is used by OBJT_PHYS objects - objects using unswappable 1167 * physical memory as backing store rather then swap-backed memory and 1168 * will eventually be extended to support 4MB unmanaged physical 1169 * mappings. 1170 */ 1171void 1172vm_page_unmanage(vm_page_t m) 1173{ 1174 1175 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1176 if ((m->flags & PG_UNMANAGED) == 0) { 1177 if (m->wire_count == 0) 1178 vm_pageq_remove(m); 1179 } 1180 vm_page_flag_set(m, PG_UNMANAGED); 1181} 1182 1183/* 1184 * vm_page_wire: 1185 * 1186 * Mark this page as wired down by yet 1187 * another map, removing it from paging queues 1188 * as necessary. 1189 * 1190 * The page queues must be locked. 1191 * This routine may not block. 1192 */ 1193void 1194vm_page_wire(vm_page_t m) 1195{ 1196 1197 /* 1198 * Only bump the wire statistics if the page is not already wired, 1199 * and only unqueue the page if it is on some queue (if it is unmanaged 1200 * it is already off the queues). 1201 */ 1202 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1203 if (m->flags & PG_FICTITIOUS) 1204 return; 1205 if (m->wire_count == 0) { 1206 if ((m->flags & PG_UNMANAGED) == 0) 1207 vm_pageq_remove(m); 1208 atomic_add_int(&cnt.v_wire_count, 1); 1209 } 1210 m->wire_count++; 1211 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m)); 1212} 1213 1214/* 1215 * vm_page_unwire: 1216 * 1217 * Release one wiring of this page, potentially 1218 * enabling it to be paged again. 1219 * 1220 * Many pages placed on the inactive queue should actually go 1221 * into the cache, but it is difficult to figure out which. What 1222 * we do instead, if the inactive target is well met, is to put 1223 * clean pages at the head of the inactive queue instead of the tail. 1224 * This will cause them to be moved to the cache more quickly and 1225 * if not actively re-referenced, freed more quickly. If we just 1226 * stick these pages at the end of the inactive queue, heavy filesystem 1227 * meta-data accesses can cause an unnecessary paging load on memory bound 1228 * processes. This optimization causes one-time-use metadata to be 1229 * reused more quickly. 1230 * 1231 * BUT, if we are in a low-memory situation we have no choice but to 1232 * put clean pages on the cache queue. 1233 * 1234 * A number of routines use vm_page_unwire() to guarantee that the page 1235 * will go into either the inactive or active queues, and will NEVER 1236 * be placed in the cache - for example, just after dirtying a page. 1237 * dirty pages in the cache are not allowed. 1238 * 1239 * The page queues must be locked. 1240 * This routine may not block. 1241 */ 1242void 1243vm_page_unwire(vm_page_t m, int activate) 1244{ 1245 1246 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1247 if (m->flags & PG_FICTITIOUS) 1248 return; 1249 if (m->wire_count > 0) { 1250 m->wire_count--; 1251 if (m->wire_count == 0) { 1252 atomic_subtract_int(&cnt.v_wire_count, 1); 1253 if (m->flags & PG_UNMANAGED) { 1254 ; 1255 } else if (activate) 1256 vm_pageq_enqueue(PQ_ACTIVE, m); 1257 else { 1258 vm_page_flag_clear(m, PG_WINATCFLS); 1259 vm_pageq_enqueue(PQ_INACTIVE, m); 1260 } 1261 } 1262 } else { 1263 panic("vm_page_unwire: invalid wire count: %d", m->wire_count); 1264 } 1265} 1266 1267 1268/* 1269 * Move the specified page to the inactive queue. If the page has 1270 * any associated swap, the swap is deallocated. 1271 * 1272 * Normally athead is 0 resulting in LRU operation. athead is set 1273 * to 1 if we want this page to be 'as if it were placed in the cache', 1274 * except without unmapping it from the process address space. 1275 * 1276 * This routine may not block. 1277 */ 1278static inline void 1279_vm_page_deactivate(vm_page_t m, int athead) 1280{ 1281 1282 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1283 1284 /* 1285 * Ignore if already inactive. 1286 */ 1287 if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE)) 1288 return; 1289 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { 1290 if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) 1291 cnt.v_reactivated++; 1292 vm_page_flag_clear(m, PG_WINATCFLS); 1293 vm_pageq_remove(m); 1294 if (athead) 1295 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); 1296 else 1297 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); 1298 VM_PAGE_SETQUEUE2(m, PQ_INACTIVE); 1299 vm_page_queues[PQ_INACTIVE].lcnt++; 1300 cnt.v_inactive_count++; 1301 } 1302} 1303 1304void 1305vm_page_deactivate(vm_page_t m) 1306{ 1307 _vm_page_deactivate(m, 0); 1308} 1309 1310/* 1311 * vm_page_try_to_cache: 1312 * 1313 * Returns 0 on failure, 1 on success 1314 */ 1315int 1316vm_page_try_to_cache(vm_page_t m) 1317{ 1318 1319 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1320 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1321 if (m->dirty || m->hold_count || m->busy || m->wire_count || 1322 (m->flags & (PG_BUSY|PG_UNMANAGED))) { 1323 return (0); 1324 } 1325 pmap_remove_all(m); 1326 if (m->dirty) 1327 return (0); 1328 vm_page_cache(m); 1329 return (1); 1330} 1331 1332/* 1333 * vm_page_try_to_free() 1334 * 1335 * Attempt to free the page. If we cannot free it, we do nothing. 1336 * 1 is returned on success, 0 on failure. 1337 */ 1338int 1339vm_page_try_to_free(vm_page_t m) 1340{ 1341 1342 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1343 if (m->object != NULL) 1344 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1345 if (m->dirty || m->hold_count || m->busy || m->wire_count || 1346 (m->flags & (PG_BUSY|PG_UNMANAGED))) { 1347 return (0); 1348 } 1349 pmap_remove_all(m); 1350 if (m->dirty) 1351 return (0); 1352 vm_page_free(m); 1353 return (1); 1354} 1355 1356/* 1357 * vm_page_cache 1358 * 1359 * Put the specified page onto the page cache queue (if appropriate). 1360 * 1361 * This routine may not block. 1362 */ 1363void 1364vm_page_cache(vm_page_t m) 1365{ 1366 1367 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1368 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1369 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy || 1370 m->hold_count || m->wire_count) { 1371 printf("vm_page_cache: attempting to cache busy page\n"); 1372 return; 1373 } 1374 if (VM_PAGE_INQUEUE1(m, PQ_CACHE)) 1375 return; 1376 1377 /* 1378 * Remove all pmaps and indicate that the page is not 1379 * writeable or mapped. 1380 */ 1381 pmap_remove_all(m); 1382 if (m->dirty != 0) { 1383 panic("vm_page_cache: caching a dirty page, pindex: %ld", 1384 (long)m->pindex); 1385 } 1386 vm_pageq_remove_nowakeup(m); 1387 vm_pageq_enqueue(PQ_CACHE + m->pc, m); 1388 vm_page_free_wakeup(); 1389} 1390 1391/* 1392 * vm_page_dontneed 1393 * 1394 * Cache, deactivate, or do nothing as appropriate. This routine 1395 * is typically used by madvise() MADV_DONTNEED. 1396 * 1397 * Generally speaking we want to move the page into the cache so 1398 * it gets reused quickly. However, this can result in a silly syndrome 1399 * due to the page recycling too quickly. Small objects will not be 1400 * fully cached. On the otherhand, if we move the page to the inactive 1401 * queue we wind up with a problem whereby very large objects 1402 * unnecessarily blow away our inactive and cache queues. 1403 * 1404 * The solution is to move the pages based on a fixed weighting. We 1405 * either leave them alone, deactivate them, or move them to the cache, 1406 * where moving them to the cache has the highest weighting. 1407 * By forcing some pages into other queues we eventually force the 1408 * system to balance the queues, potentially recovering other unrelated 1409 * space from active. The idea is to not force this to happen too 1410 * often. 1411 */ 1412void 1413vm_page_dontneed(vm_page_t m) 1414{ 1415 static int dnweight; 1416 int dnw; 1417 int head; 1418 1419 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1420 dnw = ++dnweight; 1421 1422 /* 1423 * occassionally leave the page alone 1424 */ 1425 if ((dnw & 0x01F0) == 0 || 1426 VM_PAGE_INQUEUE2(m, PQ_INACTIVE) || 1427 VM_PAGE_INQUEUE1(m, PQ_CACHE) 1428 ) { 1429 if (m->act_count >= ACT_INIT) 1430 --m->act_count; 1431 return; 1432 } 1433 1434 if (m->dirty == 0 && pmap_is_modified(m)) 1435 vm_page_dirty(m); 1436 1437 if (m->dirty || (dnw & 0x0070) == 0) { 1438 /* 1439 * Deactivate the page 3 times out of 32. 1440 */ 1441 head = 0; 1442 } else { 1443 /* 1444 * Cache the page 28 times out of every 32. Note that 1445 * the page is deactivated instead of cached, but placed 1446 * at the head of the queue instead of the tail. 1447 */ 1448 head = 1; 1449 } 1450 _vm_page_deactivate(m, head); 1451} 1452 1453/* 1454 * Grab a page, waiting until we are waken up due to the page 1455 * changing state. We keep on waiting, if the page continues 1456 * to be in the object. If the page doesn't exist, first allocate it 1457 * and then conditionally zero it. 1458 * 1459 * This routine may block. 1460 */ 1461vm_page_t 1462vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) 1463{ 1464 vm_page_t m; 1465 1466 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 1467retrylookup: 1468 if ((m = vm_page_lookup(object, pindex)) != NULL) { 1469 if (vm_page_sleep_if_busy(m, TRUE, "pgrbwt")) { 1470 if ((allocflags & VM_ALLOC_RETRY) == 0) 1471 return (NULL); 1472 goto retrylookup; 1473 } else { 1474 vm_page_lock_queues(); 1475 if (allocflags & VM_ALLOC_WIRED) 1476 vm_page_wire(m); 1477 if ((allocflags & VM_ALLOC_NOBUSY) == 0) 1478 vm_page_busy(m); 1479 vm_page_unlock_queues(); 1480 return (m); 1481 } 1482 } 1483 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY); 1484 if (m == NULL) { 1485 VM_OBJECT_UNLOCK(object); 1486 VM_WAIT; 1487 VM_OBJECT_LOCK(object); 1488 if ((allocflags & VM_ALLOC_RETRY) == 0) 1489 return (NULL); 1490 goto retrylookup; 1491 } 1492 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) 1493 pmap_zero_page(m); 1494 return (m); 1495} 1496 1497/* 1498 * Mapping function for valid bits or for dirty bits in 1499 * a page. May not block. 1500 * 1501 * Inputs are required to range within a page. 1502 */ 1503inline int 1504vm_page_bits(int base, int size) 1505{ 1506 int first_bit; 1507 int last_bit; 1508 1509 KASSERT( 1510 base + size <= PAGE_SIZE, 1511 ("vm_page_bits: illegal base/size %d/%d", base, size) 1512 ); 1513 1514 if (size == 0) /* handle degenerate case */ 1515 return (0); 1516 1517 first_bit = base >> DEV_BSHIFT; 1518 last_bit = (base + size - 1) >> DEV_BSHIFT; 1519 1520 return ((2 << last_bit) - (1 << first_bit)); 1521} 1522 1523/* 1524 * vm_page_set_validclean: 1525 * 1526 * Sets portions of a page valid and clean. The arguments are expected 1527 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive 1528 * of any partial chunks touched by the range. The invalid portion of 1529 * such chunks will be zero'd. 1530 * 1531 * This routine may not block. 1532 * 1533 * (base + size) must be less then or equal to PAGE_SIZE. 1534 */ 1535void 1536vm_page_set_validclean(vm_page_t m, int base, int size) 1537{ 1538 int pagebits; 1539 int frag; 1540 int endoff; 1541 1542 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1543 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1544 if (size == 0) /* handle degenerate case */ 1545 return; 1546 1547 /* 1548 * If the base is not DEV_BSIZE aligned and the valid 1549 * bit is clear, we have to zero out a portion of the 1550 * first block. 1551 */ 1552 if ((frag = base & ~(DEV_BSIZE - 1)) != base && 1553 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0) 1554 pmap_zero_page_area(m, frag, base - frag); 1555 1556 /* 1557 * If the ending offset is not DEV_BSIZE aligned and the 1558 * valid bit is clear, we have to zero out a portion of 1559 * the last block. 1560 */ 1561 endoff = base + size; 1562 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff && 1563 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0) 1564 pmap_zero_page_area(m, endoff, 1565 DEV_BSIZE - (endoff & (DEV_BSIZE - 1))); 1566 1567 /* 1568 * Set valid, clear dirty bits. If validating the entire 1569 * page we can safely clear the pmap modify bit. We also 1570 * use this opportunity to clear the VPO_NOSYNC flag. If a process 1571 * takes a write fault on a MAP_NOSYNC memory area the flag will 1572 * be set again. 1573 * 1574 * We set valid bits inclusive of any overlap, but we can only 1575 * clear dirty bits for DEV_BSIZE chunks that are fully within 1576 * the range. 1577 */ 1578 pagebits = vm_page_bits(base, size); 1579 m->valid |= pagebits; 1580#if 0 /* NOT YET */ 1581 if ((frag = base & (DEV_BSIZE - 1)) != 0) { 1582 frag = DEV_BSIZE - frag; 1583 base += frag; 1584 size -= frag; 1585 if (size < 0) 1586 size = 0; 1587 } 1588 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1)); 1589#endif 1590 m->dirty &= ~pagebits; 1591 if (base == 0 && size == PAGE_SIZE) { 1592 pmap_clear_modify(m); 1593 m->oflags &= ~VPO_NOSYNC; 1594 } 1595} 1596 1597void 1598vm_page_clear_dirty(vm_page_t m, int base, int size) 1599{ 1600 1601 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1602 m->dirty &= ~vm_page_bits(base, size); 1603} 1604 1605/* 1606 * vm_page_set_invalid: 1607 * 1608 * Invalidates DEV_BSIZE'd chunks within a page. Both the 1609 * valid and dirty bits for the effected areas are cleared. 1610 * 1611 * May not block. 1612 */ 1613void 1614vm_page_set_invalid(vm_page_t m, int base, int size) 1615{ 1616 int bits; 1617 1618 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1619 bits = vm_page_bits(base, size); 1620 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1621 if (m->valid == VM_PAGE_BITS_ALL && bits != 0) 1622 pmap_remove_all(m); 1623 m->valid &= ~bits; 1624 m->dirty &= ~bits; 1625 m->object->generation++; 1626} 1627 1628/* 1629 * vm_page_zero_invalid() 1630 * 1631 * The kernel assumes that the invalid portions of a page contain 1632 * garbage, but such pages can be mapped into memory by user code. 1633 * When this occurs, we must zero out the non-valid portions of the 1634 * page so user code sees what it expects. 1635 * 1636 * Pages are most often semi-valid when the end of a file is mapped 1637 * into memory and the file's size is not page aligned. 1638 */ 1639void 1640vm_page_zero_invalid(vm_page_t m, boolean_t setvalid) 1641{ 1642 int b; 1643 int i; 1644 1645 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1646 /* 1647 * Scan the valid bits looking for invalid sections that 1648 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the 1649 * valid bit may be set ) have already been zerod by 1650 * vm_page_set_validclean(). 1651 */ 1652 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) { 1653 if (i == (PAGE_SIZE / DEV_BSIZE) || 1654 (m->valid & (1 << i)) 1655 ) { 1656 if (i > b) { 1657 pmap_zero_page_area(m, 1658 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT); 1659 } 1660 b = i + 1; 1661 } 1662 } 1663 1664 /* 1665 * setvalid is TRUE when we can safely set the zero'd areas 1666 * as being valid. We can do this if there are no cache consistancy 1667 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS. 1668 */ 1669 if (setvalid) 1670 m->valid = VM_PAGE_BITS_ALL; 1671} 1672 1673/* 1674 * vm_page_is_valid: 1675 * 1676 * Is (partial) page valid? Note that the case where size == 0 1677 * will return FALSE in the degenerate case where the page is 1678 * entirely invalid, and TRUE otherwise. 1679 * 1680 * May not block. 1681 */ 1682int 1683vm_page_is_valid(vm_page_t m, int base, int size) 1684{ 1685 int bits = vm_page_bits(base, size); 1686 1687 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); 1688 if (m->valid && ((m->valid & bits) == bits)) 1689 return 1; 1690 else 1691 return 0; 1692} 1693 1694/* 1695 * update dirty bits from pmap/mmu. May not block. 1696 */ 1697void 1698vm_page_test_dirty(vm_page_t m) 1699{ 1700 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) { 1701 vm_page_dirty(m); 1702 } 1703} 1704 1705int so_zerocp_fullpage = 0; 1706 1707void 1708vm_page_cowfault(vm_page_t m) 1709{ 1710 vm_page_t mnew; 1711 vm_object_t object; 1712 vm_pindex_t pindex; 1713 1714 object = m->object; 1715 pindex = m->pindex; 1716 1717 retry_alloc: 1718 pmap_remove_all(m); 1719 vm_page_remove(m); 1720 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY); 1721 if (mnew == NULL) { 1722 vm_page_insert(m, object, pindex); 1723 vm_page_unlock_queues(); 1724 VM_OBJECT_UNLOCK(object); 1725 VM_WAIT; 1726 VM_OBJECT_LOCK(object); 1727 vm_page_lock_queues(); 1728 goto retry_alloc; 1729 } 1730 1731 if (m->cow == 0) { 1732 /* 1733 * check to see if we raced with an xmit complete when 1734 * waiting to allocate a page. If so, put things back 1735 * the way they were 1736 */ 1737 vm_page_free(mnew); 1738 vm_page_insert(m, object, pindex); 1739 } else { /* clear COW & copy page */ 1740 if (!so_zerocp_fullpage) 1741 pmap_copy_page(m, mnew); 1742 mnew->valid = VM_PAGE_BITS_ALL; 1743 vm_page_dirty(mnew); 1744 mnew->wire_count = m->wire_count - m->cow; 1745 m->wire_count = m->cow; 1746 } 1747} 1748 1749void 1750vm_page_cowclear(vm_page_t m) 1751{ 1752 1753 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1754 if (m->cow) { 1755 m->cow--; 1756 /* 1757 * let vm_fault add back write permission lazily 1758 */ 1759 } 1760 /* 1761 * sf_buf_free() will free the page, so we needn't do it here 1762 */ 1763} 1764 1765void 1766vm_page_cowsetup(vm_page_t m) 1767{ 1768 1769 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1770 m->cow++; 1771 pmap_remove_write(m); 1772} 1773 1774#include "opt_ddb.h" 1775#ifdef DDB 1776#include <sys/kernel.h> 1777 1778#include <ddb/ddb.h> 1779 1780DB_SHOW_COMMAND(page, vm_page_print_page_info) 1781{ 1782 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count); 1783 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count); 1784 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count); 1785 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count); 1786 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count); 1787 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved); 1788 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min); 1789 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target); 1790 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min); 1791 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target); 1792} 1793 1794DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) 1795{ 1796 int i; 1797 db_printf("PQ_FREE:"); 1798 for (i = 0; i < PQ_NUMCOLORS; i++) { 1799 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt); 1800 } 1801 db_printf("\n"); 1802 1803 db_printf("PQ_CACHE:"); 1804 for (i = 0; i < PQ_NUMCOLORS; i++) { 1805 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt); 1806 } 1807 db_printf("\n"); 1808 1809 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n", 1810 vm_page_queues[PQ_ACTIVE].lcnt, 1811 vm_page_queues[PQ_INACTIVE].lcnt); 1812} 1813#endif /* DDB */ 1814