vm_page.c revision 101174
1/* 2 * Copyright (c) 1991 Regents of the University of California. 3 * All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * The Mach Operating System project at Carnegie-Mellon University. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by the University of 19 * California, Berkeley and its contributors. 20 * 4. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 37 * $FreeBSD: head/sys/vm/vm_page.c 101174 2002-08-01 17:57:42Z alc $ 38 */ 39 40/* 41 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 42 * All rights reserved. 43 * 44 * Authors: Avadis Tevanian, Jr., Michael Wayne Young 45 * 46 * Permission to use, copy, modify and distribute this software and 47 * its documentation is hereby granted, provided that both the copyright 48 * notice and this permission notice appear in all copies of the 49 * software, derivative works or modified versions, and any portions 50 * thereof, and that both notices appear in supporting documentation. 51 * 52 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 53 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 54 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 55 * 56 * Carnegie Mellon requests users of this software to return to 57 * 58 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 59 * School of Computer Science 60 * Carnegie Mellon University 61 * Pittsburgh PA 15213-3890 62 * 63 * any improvements or extensions that they make and grant Carnegie the 64 * rights to redistribute these changes. 65 */ 66 67/* 68 * GENERAL RULES ON VM_PAGE MANIPULATION 69 * 70 * - a pageq mutex is required when adding or removing a page from a 71 * page queue (vm_page_queue[]), regardless of other mutexes or the 72 * busy state of a page. 73 * 74 * - a hash chain mutex is required when associating or disassociating 75 * a page from the VM PAGE CACHE hash table (vm_page_buckets), 76 * regardless of other mutexes or the busy state of a page. 77 * 78 * - either a hash chain mutex OR a busied page is required in order 79 * to modify the page flags. A hash chain mutex must be obtained in 80 * order to busy a page. A page's flags cannot be modified by a 81 * hash chain mutex if the page is marked busy. 82 * 83 * - The object memq mutex is held when inserting or removing 84 * pages from an object (vm_page_insert() or vm_page_remove()). This 85 * is different from the object's main mutex. 86 * 87 * Generally speaking, you have to be aware of side effects when running 88 * vm_page ops. A vm_page_lookup() will return with the hash chain 89 * locked, whether it was able to lookup the page or not. vm_page_free(), 90 * vm_page_cache(), vm_page_activate(), and a number of other routines 91 * will release the hash chain mutex for you. Intermediate manipulation 92 * routines such as vm_page_flag_set() expect the hash chain to be held 93 * on entry and the hash chain will remain held on return. 94 * 95 * pageq scanning can only occur with the pageq in question locked. 96 * We have a known bottleneck with the active queue, but the cache 97 * and free queues are actually arrays already. 98 */ 99 100/* 101 * Resident memory management module. 102 */ 103 104#include <sys/param.h> 105#include <sys/systm.h> 106#include <sys/lock.h> 107#include <sys/malloc.h> 108#include <sys/mutex.h> 109#include <sys/proc.h> 110#include <sys/vmmeter.h> 111#include <sys/vnode.h> 112 113#include <vm/vm.h> 114#include <vm/vm_param.h> 115#include <vm/vm_kern.h> 116#include <vm/vm_object.h> 117#include <vm/vm_page.h> 118#include <vm/vm_pageout.h> 119#include <vm/vm_pager.h> 120#include <vm/vm_extern.h> 121#include <vm/uma.h> 122#include <vm/uma_int.h> 123 124/* 125 * Associated with page of user-allocatable memory is a 126 * page structure. 127 */ 128static struct mtx vm_page_buckets_mtx; 129static struct vm_page **vm_page_buckets; /* Array of buckets */ 130static int vm_page_bucket_count; /* How big is array? */ 131static int vm_page_hash_mask; /* Mask for hash function */ 132 133struct mtx vm_page_queue_mtx; 134struct mtx vm_page_queue_free_mtx; 135 136vm_page_t vm_page_array = 0; 137int vm_page_array_size = 0; 138long first_page = 0; 139int vm_page_zero_count = 0; 140 141/* 142 * vm_set_page_size: 143 * 144 * Sets the page size, perhaps based upon the memory 145 * size. Must be called before any use of page-size 146 * dependent functions. 147 */ 148void 149vm_set_page_size(void) 150{ 151 if (cnt.v_page_size == 0) 152 cnt.v_page_size = PAGE_SIZE; 153 if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0) 154 panic("vm_set_page_size: page size not a power of two"); 155} 156 157/* 158 * vm_page_startup: 159 * 160 * Initializes the resident memory module. 161 * 162 * Allocates memory for the page cells, and 163 * for the object/offset-to-page hash table headers. 164 * Each page cell is initialized and placed on the free list. 165 */ 166vm_offset_t 167vm_page_startup(vm_offset_t starta, vm_offset_t enda, vm_offset_t vaddr) 168{ 169 vm_offset_t mapped; 170 struct vm_page **bucket; 171 vm_size_t npages, page_range; 172 vm_offset_t new_end; 173 int i; 174 vm_offset_t pa; 175 int nblocks; 176 vm_offset_t last_pa; 177 178 /* the biggest memory array is the second group of pages */ 179 vm_offset_t end; 180 vm_offset_t biggestone, biggestsize; 181 182 vm_offset_t total; 183 vm_size_t bootpages; 184 185 total = 0; 186 biggestsize = 0; 187 biggestone = 0; 188 nblocks = 0; 189 vaddr = round_page(vaddr); 190 191 for (i = 0; phys_avail[i + 1]; i += 2) { 192 phys_avail[i] = round_page(phys_avail[i]); 193 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]); 194 } 195 196 for (i = 0; phys_avail[i + 1]; i += 2) { 197 vm_size_t size = phys_avail[i + 1] - phys_avail[i]; 198 199 if (size > biggestsize) { 200 biggestone = i; 201 biggestsize = size; 202 } 203 ++nblocks; 204 total += size; 205 } 206 207 end = phys_avail[biggestone+1]; 208 209 /* 210 * Initialize the locks. 211 */ 212 mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF); 213 mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL, 214 MTX_SPIN); 215 216 /* 217 * Initialize the queue headers for the free queue, the active queue 218 * and the inactive queue. 219 */ 220 vm_pageq_init(); 221 222 /* 223 * Allocate memory for use when boot strapping the kernel memory allocator 224 */ 225 bootpages = UMA_BOOT_PAGES * UMA_SLAB_SIZE; 226 new_end = end - bootpages; 227 new_end = trunc_page(new_end); 228 mapped = pmap_map(&vaddr, new_end, end, 229 VM_PROT_READ | VM_PROT_WRITE); 230 bzero((caddr_t) mapped, end - new_end); 231 uma_startup((caddr_t)mapped); 232 233 end = new_end; 234 235 /* 236 * Allocate (and initialize) the hash table buckets. 237 * 238 * The number of buckets MUST BE a power of 2, and the actual value is 239 * the next power of 2 greater than the number of physical pages in 240 * the system. 241 * 242 * We make the hash table approximately 2x the number of pages to 243 * reduce the chain length. This is about the same size using the 244 * singly-linked list as the 1x hash table we were using before 245 * using TAILQ but the chain length will be smaller. 246 * 247 * Note: This computation can be tweaked if desired. 248 */ 249 if (vm_page_bucket_count == 0) { 250 vm_page_bucket_count = 1; 251 while (vm_page_bucket_count < atop(total)) 252 vm_page_bucket_count <<= 1; 253 } 254 vm_page_bucket_count <<= 1; 255 vm_page_hash_mask = vm_page_bucket_count - 1; 256 257 /* 258 * Validate these addresses. 259 */ 260 new_end = end - vm_page_bucket_count * sizeof(struct vm_page *); 261 new_end = trunc_page(new_end); 262 mapped = pmap_map(&vaddr, new_end, end, 263 VM_PROT_READ | VM_PROT_WRITE); 264 bzero((caddr_t) mapped, end - new_end); 265 266 mtx_init(&vm_page_buckets_mtx, "vm page buckets mutex", NULL, MTX_SPIN); 267 vm_page_buckets = (struct vm_page **)mapped; 268 bucket = vm_page_buckets; 269 for (i = 0; i < vm_page_bucket_count; i++) { 270 *bucket = NULL; 271 bucket++; 272 } 273 274 /* 275 * Compute the number of pages of memory that will be available for 276 * use (taking into account the overhead of a page structure per 277 * page). 278 */ 279 first_page = phys_avail[0] / PAGE_SIZE; 280 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page; 281 npages = (total - (page_range * sizeof(struct vm_page)) - 282 (end - new_end)) / PAGE_SIZE; 283 end = new_end; 284 285 /* 286 * Initialize the mem entry structures now, and put them in the free 287 * queue. 288 */ 289 new_end = trunc_page(end - page_range * sizeof(struct vm_page)); 290 mapped = pmap_map(&vaddr, new_end, end, 291 VM_PROT_READ | VM_PROT_WRITE); 292 vm_page_array = (vm_page_t) mapped; 293 294 /* 295 * Clear all of the page structures 296 */ 297 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page)); 298 vm_page_array_size = page_range; 299 300 /* 301 * Construct the free queue(s) in descending order (by physical 302 * address) so that the first 16MB of physical memory is allocated 303 * last rather than first. On large-memory machines, this avoids 304 * the exhaustion of low physical memory before isa_dmainit has run. 305 */ 306 cnt.v_page_count = 0; 307 cnt.v_free_count = 0; 308 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) { 309 pa = phys_avail[i]; 310 if (i == biggestone) 311 last_pa = new_end; 312 else 313 last_pa = phys_avail[i + 1]; 314 while (pa < last_pa && npages-- > 0) { 315 vm_pageq_add_new_page(pa); 316 pa += PAGE_SIZE; 317 } 318 } 319 return (vaddr); 320} 321 322/* 323 * vm_page_hash: 324 * 325 * Distributes the object/offset key pair among hash buckets. 326 * 327 * NOTE: This macro depends on vm_page_bucket_count being a power of 2. 328 * This routine may not block. 329 * 330 * We try to randomize the hash based on the object to spread the pages 331 * out in the hash table without it costing us too much. 332 */ 333static __inline int 334vm_page_hash(vm_object_t object, vm_pindex_t pindex) 335{ 336 int i = ((uintptr_t)object + pindex) ^ object->hash_rand; 337 338 return (i & vm_page_hash_mask); 339} 340 341void 342vm_page_flag_set(vm_page_t m, unsigned short bits) 343{ 344 GIANT_REQUIRED; 345 m->flags |= bits; 346} 347 348void 349vm_page_flag_clear(vm_page_t m, unsigned short bits) 350{ 351 GIANT_REQUIRED; 352 m->flags &= ~bits; 353} 354 355void 356vm_page_busy(vm_page_t m) 357{ 358 KASSERT((m->flags & PG_BUSY) == 0, 359 ("vm_page_busy: page already busy!!!")); 360 vm_page_flag_set(m, PG_BUSY); 361} 362 363/* 364 * vm_page_flash: 365 * 366 * wakeup anyone waiting for the page. 367 */ 368void 369vm_page_flash(vm_page_t m) 370{ 371 if (m->flags & PG_WANTED) { 372 vm_page_flag_clear(m, PG_WANTED); 373 wakeup(m); 374 } 375} 376 377/* 378 * vm_page_wakeup: 379 * 380 * clear the PG_BUSY flag and wakeup anyone waiting for the 381 * page. 382 * 383 */ 384void 385vm_page_wakeup(vm_page_t m) 386{ 387 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!")); 388 vm_page_flag_clear(m, PG_BUSY); 389 vm_page_flash(m); 390} 391 392/* 393 * 394 * 395 */ 396void 397vm_page_io_start(vm_page_t m) 398{ 399 400 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 401 m->busy++; 402} 403 404void 405vm_page_io_finish(vm_page_t m) 406{ 407 408 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 409 m->busy--; 410 if (m->busy == 0) 411 vm_page_flash(m); 412} 413 414/* 415 * Keep page from being freed by the page daemon 416 * much of the same effect as wiring, except much lower 417 * overhead and should be used only for *very* temporary 418 * holding ("wiring"). 419 */ 420void 421vm_page_hold(vm_page_t mem) 422{ 423 GIANT_REQUIRED; 424 mem->hold_count++; 425} 426 427void 428vm_page_unhold(vm_page_t mem) 429{ 430 GIANT_REQUIRED; 431 --mem->hold_count; 432 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!")); 433 if (mem->hold_count == 0 && mem->queue == PQ_HOLD) 434 vm_page_free_toq(mem); 435} 436 437/* 438 * vm_page_protect: 439 * 440 * Reduce the protection of a page. This routine never raises the 441 * protection and therefore can be safely called if the page is already 442 * at VM_PROT_NONE (it will be a NOP effectively ). 443 */ 444void 445vm_page_protect(vm_page_t mem, int prot) 446{ 447 if (prot == VM_PROT_NONE) { 448 if (mem->flags & (PG_WRITEABLE|PG_MAPPED)) { 449 pmap_page_protect(mem, VM_PROT_NONE); 450 vm_page_flag_clear(mem, PG_WRITEABLE|PG_MAPPED); 451 } 452 } else if ((prot == VM_PROT_READ) && (mem->flags & PG_WRITEABLE)) { 453 pmap_page_protect(mem, VM_PROT_READ); 454 vm_page_flag_clear(mem, PG_WRITEABLE); 455 } 456} 457/* 458 * vm_page_zero_fill: 459 * 460 * Zero-fill the specified page. 461 * Written as a standard pagein routine, to 462 * be used by the zero-fill object. 463 */ 464boolean_t 465vm_page_zero_fill(vm_page_t m) 466{ 467 pmap_zero_page(m); 468 return (TRUE); 469} 470 471/* 472 * vm_page_zero_fill_area: 473 * 474 * Like vm_page_zero_fill but only fill the specified area. 475 */ 476boolean_t 477vm_page_zero_fill_area(vm_page_t m, int off, int size) 478{ 479 pmap_zero_page_area(m, off, size); 480 return (TRUE); 481} 482 483/* 484 * vm_page_copy: 485 * 486 * Copy one page to another 487 */ 488void 489vm_page_copy(vm_page_t src_m, vm_page_t dest_m) 490{ 491 pmap_copy_page(src_m, dest_m); 492 dest_m->valid = VM_PAGE_BITS_ALL; 493} 494 495/* 496 * vm_page_free: 497 * 498 * Free a page 499 * 500 * The clearing of PG_ZERO is a temporary safety until the code can be 501 * reviewed to determine that PG_ZERO is being properly cleared on 502 * write faults or maps. PG_ZERO was previously cleared in 503 * vm_page_alloc(). 504 */ 505void 506vm_page_free(vm_page_t m) 507{ 508 vm_page_flag_clear(m, PG_ZERO); 509 vm_page_free_toq(m); 510 vm_page_zero_idle_wakeup(); 511} 512 513/* 514 * vm_page_free_zero: 515 * 516 * Free a page to the zerod-pages queue 517 */ 518void 519vm_page_free_zero(vm_page_t m) 520{ 521 vm_page_flag_set(m, PG_ZERO); 522 vm_page_free_toq(m); 523} 524 525/* 526 * vm_page_sleep_busy: 527 * 528 * Wait until page is no longer PG_BUSY or (if also_m_busy is TRUE) 529 * m->busy is zero. Returns TRUE if it had to sleep ( including if 530 * it almost had to sleep and made temporary spl*() mods), FALSE 531 * otherwise. 532 * 533 * This routine assumes that interrupts can only remove the busy 534 * status from a page, not set the busy status or change it from 535 * PG_BUSY to m->busy or vise versa (which would create a timing 536 * window). 537 */ 538int 539vm_page_sleep_busy(vm_page_t m, int also_m_busy, const char *msg) 540{ 541 GIANT_REQUIRED; 542 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) { 543 int s = splvm(); 544 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) { 545 /* 546 * Page is busy. Wait and retry. 547 */ 548 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED); 549 tsleep(m, PVM, msg, 0); 550 } 551 splx(s); 552 return (TRUE); 553 /* not reached */ 554 } 555 return (FALSE); 556} 557 558/* 559 * vm_page_sleep_if_busy: 560 * 561 * Sleep and release the page queues lock if PG_BUSY is set or, 562 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the 563 * thread slept and the page queues lock was released. 564 * Otherwise, retains the page queues lock and returns FALSE. 565 */ 566int 567vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg) 568{ 569 570 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 571 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) { 572 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED); 573 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0); 574 return (TRUE); 575 } 576 return (FALSE); 577} 578 579/* 580 * vm_page_dirty: 581 * 582 * make page all dirty 583 */ 584void 585vm_page_dirty(vm_page_t m) 586{ 587 KASSERT(m->queue - m->pc != PQ_CACHE, 588 ("vm_page_dirty: page in cache!")); 589 m->dirty = VM_PAGE_BITS_ALL; 590} 591 592/* 593 * vm_page_undirty: 594 * 595 * Set page to not be dirty. Note: does not clear pmap modify bits 596 */ 597void 598vm_page_undirty(vm_page_t m) 599{ 600 m->dirty = 0; 601} 602 603/* 604 * vm_page_insert: [ internal use only ] 605 * 606 * Inserts the given mem entry into the object and object list. 607 * 608 * The pagetables are not updated but will presumably fault the page 609 * in if necessary, or if a kernel page the caller will at some point 610 * enter the page into the kernel's pmap. We are not allowed to block 611 * here so we *can't* do this anyway. 612 * 613 * The object and page must be locked, and must be splhigh. 614 * This routine may not block. 615 */ 616void 617vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) 618{ 619 struct vm_page **bucket; 620 621 GIANT_REQUIRED; 622 623 if (m->object != NULL) 624 panic("vm_page_insert: already inserted"); 625 626 /* 627 * Record the object/offset pair in this page 628 */ 629 m->object = object; 630 m->pindex = pindex; 631 632 /* 633 * Insert it into the object_object/offset hash table 634 */ 635 bucket = &vm_page_buckets[vm_page_hash(object, pindex)]; 636 mtx_lock_spin(&vm_page_buckets_mtx); 637 m->hnext = *bucket; 638 *bucket = m; 639 mtx_unlock_spin(&vm_page_buckets_mtx); 640 641 /* 642 * Now link into the object's list of backed pages. 643 */ 644 TAILQ_INSERT_TAIL(&object->memq, m, listq); 645 object->generation++; 646 647 /* 648 * show that the object has one more resident page. 649 */ 650 object->resident_page_count++; 651 652 /* 653 * Since we are inserting a new and possibly dirty page, 654 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags. 655 */ 656 if (m->flags & PG_WRITEABLE) 657 vm_object_set_writeable_dirty(object); 658} 659 660/* 661 * vm_page_remove: 662 * NOTE: used by device pager as well -wfj 663 * 664 * Removes the given mem entry from the object/offset-page 665 * table and the object page list, but do not invalidate/terminate 666 * the backing store. 667 * 668 * The object and page must be locked, and at splhigh. 669 * The underlying pmap entry (if any) is NOT removed here. 670 * This routine may not block. 671 */ 672void 673vm_page_remove(vm_page_t m) 674{ 675 vm_object_t object; 676 vm_page_t *bucket; 677 678 GIANT_REQUIRED; 679 680 if (m->object == NULL) 681 return; 682 683 if ((m->flags & PG_BUSY) == 0) { 684 panic("vm_page_remove: page not busy"); 685 } 686 687 /* 688 * Basically destroy the page. 689 */ 690 vm_page_wakeup(m); 691 692 object = m->object; 693 694 /* 695 * Remove from the object_object/offset hash table. The object 696 * must be on the hash queue, we will panic if it isn't 697 */ 698 bucket = &vm_page_buckets[vm_page_hash(m->object, m->pindex)]; 699 mtx_lock_spin(&vm_page_buckets_mtx); 700 while (*bucket != m) { 701 if (*bucket == NULL) 702 panic("vm_page_remove(): page not found in hash"); 703 bucket = &(*bucket)->hnext; 704 } 705 *bucket = m->hnext; 706 m->hnext = NULL; 707 mtx_unlock_spin(&vm_page_buckets_mtx); 708 709 /* 710 * Now remove from the object's list of backed pages. 711 */ 712 TAILQ_REMOVE(&object->memq, m, listq); 713 714 /* 715 * And show that the object has one fewer resident page. 716 */ 717 object->resident_page_count--; 718 object->generation++; 719 720 m->object = NULL; 721} 722 723/* 724 * vm_page_lookup: 725 * 726 * Returns the page associated with the object/offset 727 * pair specified; if none is found, NULL is returned. 728 * 729 * The object must be locked. No side effects. 730 * This routine may not block. 731 * This is a critical path routine 732 */ 733vm_page_t 734vm_page_lookup(vm_object_t object, vm_pindex_t pindex) 735{ 736 vm_page_t m; 737 struct vm_page **bucket; 738 739 /* 740 * Search the hash table for this object/offset pair 741 */ 742 bucket = &vm_page_buckets[vm_page_hash(object, pindex)]; 743 mtx_lock_spin(&vm_page_buckets_mtx); 744 for (m = *bucket; m != NULL; m = m->hnext) 745 if (m->object == object && m->pindex == pindex) 746 break; 747 mtx_unlock_spin(&vm_page_buckets_mtx); 748 return (m); 749} 750 751/* 752 * vm_page_rename: 753 * 754 * Move the given memory entry from its 755 * current object to the specified target object/offset. 756 * 757 * The object must be locked. 758 * This routine may not block. 759 * 760 * Note: this routine will raise itself to splvm(), the caller need not. 761 * 762 * Note: swap associated with the page must be invalidated by the move. We 763 * have to do this for several reasons: (1) we aren't freeing the 764 * page, (2) we are dirtying the page, (3) the VM system is probably 765 * moving the page from object A to B, and will then later move 766 * the backing store from A to B and we can't have a conflict. 767 * 768 * Note: we *always* dirty the page. It is necessary both for the 769 * fact that we moved it, and because we may be invalidating 770 * swap. If the page is on the cache, we have to deactivate it 771 * or vm_page_dirty() will panic. Dirty pages are not allowed 772 * on the cache. 773 */ 774void 775vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex) 776{ 777 int s; 778 779 s = splvm(); 780 vm_page_lock_queues(); 781 vm_page_remove(m); 782 vm_page_insert(m, new_object, new_pindex); 783 if (m->queue - m->pc == PQ_CACHE) 784 vm_page_deactivate(m); 785 vm_page_dirty(m); 786 vm_page_unlock_queues(); 787 splx(s); 788} 789 790/* 791 * vm_page_select_cache: 792 * 793 * Find a page on the cache queue with color optimization. As pages 794 * might be found, but not applicable, they are deactivated. This 795 * keeps us from using potentially busy cached pages. 796 * 797 * This routine must be called at splvm(). 798 * This routine may not block. 799 */ 800static vm_page_t 801vm_page_select_cache(vm_object_t object, vm_pindex_t pindex) 802{ 803 vm_page_t m; 804 805 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 806 while (TRUE) { 807 m = vm_pageq_find( 808 PQ_CACHE, 809 (pindex + object->pg_color) & PQ_L2_MASK, 810 FALSE 811 ); 812 if (m && ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy || 813 m->hold_count || m->wire_count)) { 814 vm_page_deactivate(m); 815 continue; 816 } 817 return m; 818 } 819} 820 821/* 822 * vm_page_select_free: 823 * 824 * Find a free or zero page, with specified preference. 825 * 826 * This routine must be called at splvm(). 827 * This routine may not block. 828 */ 829static __inline vm_page_t 830vm_page_select_free(vm_object_t object, vm_pindex_t pindex, boolean_t prefer_zero) 831{ 832 vm_page_t m; 833 834 m = vm_pageq_find( 835 PQ_FREE, 836 (pindex + object->pg_color) & PQ_L2_MASK, 837 prefer_zero 838 ); 839 return (m); 840} 841 842/* 843 * vm_page_alloc: 844 * 845 * Allocate and return a memory cell associated 846 * with this VM object/offset pair. 847 * 848 * page_req classes: 849 * VM_ALLOC_NORMAL normal process request 850 * VM_ALLOC_SYSTEM system *really* needs a page 851 * VM_ALLOC_INTERRUPT interrupt time request 852 * VM_ALLOC_ZERO zero page 853 * 854 * This routine may not block. 855 * 856 * Additional special handling is required when called from an 857 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with 858 * the page cache in this case. 859 */ 860vm_page_t 861vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) 862{ 863 vm_page_t m = NULL; 864 int page_req, s; 865 866 GIANT_REQUIRED; 867 868 KASSERT(!vm_page_lookup(object, pindex), 869 ("vm_page_alloc: page already allocated")); 870 871 page_req = req & VM_ALLOC_CLASS_MASK; 872 873 /* 874 * The pager is allowed to eat deeper into the free page list. 875 */ 876 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) { 877 page_req = VM_ALLOC_SYSTEM; 878 }; 879 880 s = splvm(); 881loop: 882 mtx_lock_spin(&vm_page_queue_free_mtx); 883 if (cnt.v_free_count > cnt.v_free_reserved) { 884 /* 885 * Allocate from the free queue if there are plenty of pages 886 * in it. 887 */ 888 m = vm_page_select_free(object, pindex, 889 (req & VM_ALLOC_ZERO) != 0); 890 } else if ( 891 (page_req == VM_ALLOC_SYSTEM && 892 cnt.v_cache_count == 0 && 893 cnt.v_free_count > cnt.v_interrupt_free_min) || 894 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0) 895 ) { 896 /* 897 * Interrupt or system, dig deeper into the free list. 898 */ 899 m = vm_page_select_free(object, pindex, FALSE); 900 } else if (page_req != VM_ALLOC_INTERRUPT) { 901 mtx_unlock_spin(&vm_page_queue_free_mtx); 902 /* 903 * Allocatable from cache (non-interrupt only). On success, 904 * we must free the page and try again, thus ensuring that 905 * cnt.v_*_free_min counters are replenished. 906 */ 907 vm_page_lock_queues(); 908 if ((m = vm_page_select_cache(object, pindex)) == NULL) { 909 vm_page_unlock_queues(); 910 splx(s); 911#if defined(DIAGNOSTIC) 912 if (cnt.v_cache_count > 0) 913 printf("vm_page_alloc(NORMAL): missing pages on cache queue: %d\n", cnt.v_cache_count); 914#endif 915 vm_pageout_deficit++; 916 pagedaemon_wakeup(); 917 return (NULL); 918 } 919 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m)); 920 vm_page_busy(m); 921 vm_page_protect(m, VM_PROT_NONE); 922 vm_page_free(m); 923 vm_page_unlock_queues(); 924 goto loop; 925 } else { 926 /* 927 * Not allocatable from cache from interrupt, give up. 928 */ 929 mtx_unlock_spin(&vm_page_queue_free_mtx); 930 splx(s); 931 vm_pageout_deficit++; 932 pagedaemon_wakeup(); 933 return (NULL); 934 } 935 936 /* 937 * At this point we had better have found a good page. 938 */ 939 940 KASSERT( 941 m != NULL, 942 ("vm_page_alloc(): missing page on free queue\n") 943 ); 944 945 /* 946 * Remove from free queue 947 */ 948 949 vm_pageq_remove_nowakeup(m); 950 951 /* 952 * Initialize structure. Only the PG_ZERO flag is inherited. 953 */ 954 if (m->flags & PG_ZERO) { 955 vm_page_zero_count--; 956 m->flags = PG_ZERO | PG_BUSY; 957 } else { 958 m->flags = PG_BUSY; 959 } 960 if (req & VM_ALLOC_WIRED) { 961 cnt.v_wire_count++; 962 m->flags |= PG_MAPPED; /* XXX this does not belong here */ 963 m->wire_count = 1; 964 } else 965 m->wire_count = 0; 966 m->hold_count = 0; 967 m->act_count = 0; 968 m->busy = 0; 969 m->valid = 0; 970 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m)); 971 mtx_unlock_spin(&vm_page_queue_free_mtx); 972 973 /* 974 * vm_page_insert() is safe prior to the splx(). Note also that 975 * inserting a page here does not insert it into the pmap (which 976 * could cause us to block allocating memory). We cannot block 977 * anywhere. 978 */ 979 vm_page_insert(m, object, pindex); 980 981 /* 982 * Don't wakeup too often - wakeup the pageout daemon when 983 * we would be nearly out of memory. 984 */ 985 if (vm_paging_needed()) 986 pagedaemon_wakeup(); 987 988 splx(s); 989 return (m); 990} 991 992/* 993 * vm_wait: (also see VM_WAIT macro) 994 * 995 * Block until free pages are available for allocation 996 * - Called in various places before memory allocations. 997 */ 998void 999vm_wait(void) 1000{ 1001 int s; 1002 1003 s = splvm(); 1004 if (curproc == pageproc) { 1005 vm_pageout_pages_needed = 1; 1006 tsleep(&vm_pageout_pages_needed, PSWP, "VMWait", 0); 1007 } else { 1008 if (!vm_pages_needed) { 1009 vm_pages_needed = 1; 1010 wakeup(&vm_pages_needed); 1011 } 1012 tsleep(&cnt.v_free_count, PVM, "vmwait", 0); 1013 } 1014 splx(s); 1015} 1016 1017/* 1018 * vm_waitpfault: (also see VM_WAITPFAULT macro) 1019 * 1020 * Block until free pages are available for allocation 1021 * - Called only in vm_fault so that processes page faulting 1022 * can be easily tracked. 1023 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing 1024 * processes will be able to grab memory first. Do not change 1025 * this balance without careful testing first. 1026 */ 1027void 1028vm_waitpfault(void) 1029{ 1030 int s; 1031 1032 s = splvm(); 1033 if (!vm_pages_needed) { 1034 vm_pages_needed = 1; 1035 wakeup(&vm_pages_needed); 1036 } 1037 tsleep(&cnt.v_free_count, PUSER, "pfault", 0); 1038 splx(s); 1039} 1040 1041/* 1042 * vm_page_activate: 1043 * 1044 * Put the specified page on the active list (if appropriate). 1045 * Ensure that act_count is at least ACT_INIT but do not otherwise 1046 * mess with it. 1047 * 1048 * The page queues must be locked. 1049 * This routine may not block. 1050 */ 1051void 1052vm_page_activate(vm_page_t m) 1053{ 1054 int s; 1055 1056 GIANT_REQUIRED; 1057 s = splvm(); 1058 if (m->queue != PQ_ACTIVE) { 1059 if ((m->queue - m->pc) == PQ_CACHE) 1060 cnt.v_reactivated++; 1061 vm_pageq_remove(m); 1062 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { 1063 if (m->act_count < ACT_INIT) 1064 m->act_count = ACT_INIT; 1065 vm_pageq_enqueue(PQ_ACTIVE, m); 1066 } 1067 } else { 1068 if (m->act_count < ACT_INIT) 1069 m->act_count = ACT_INIT; 1070 } 1071 splx(s); 1072} 1073 1074/* 1075 * vm_page_free_wakeup: 1076 * 1077 * Helper routine for vm_page_free_toq() and vm_page_cache(). This 1078 * routine is called when a page has been added to the cache or free 1079 * queues. 1080 * 1081 * This routine may not block. 1082 * This routine must be called at splvm() 1083 */ 1084static __inline void 1085vm_page_free_wakeup(void) 1086{ 1087 /* 1088 * if pageout daemon needs pages, then tell it that there are 1089 * some free. 1090 */ 1091 if (vm_pageout_pages_needed && 1092 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) { 1093 wakeup(&vm_pageout_pages_needed); 1094 vm_pageout_pages_needed = 0; 1095 } 1096 /* 1097 * wakeup processes that are waiting on memory if we hit a 1098 * high water mark. And wakeup scheduler process if we have 1099 * lots of memory. this process will swapin processes. 1100 */ 1101 if (vm_pages_needed && !vm_page_count_min()) { 1102 vm_pages_needed = 0; 1103 wakeup(&cnt.v_free_count); 1104 } 1105} 1106 1107/* 1108 * vm_page_free_toq: 1109 * 1110 * Returns the given page to the PQ_FREE list, 1111 * disassociating it with any VM object. 1112 * 1113 * Object and page must be locked prior to entry. 1114 * This routine may not block. 1115 */ 1116 1117void 1118vm_page_free_toq(vm_page_t m) 1119{ 1120 int s; 1121 struct vpgqueues *pq; 1122 vm_object_t object = m->object; 1123 1124 GIANT_REQUIRED; 1125 s = splvm(); 1126 cnt.v_tfree++; 1127 1128 if (m->busy || ((m->queue - m->pc) == PQ_FREE)) { 1129 printf( 1130 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n", 1131 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0, 1132 m->hold_count); 1133 if ((m->queue - m->pc) == PQ_FREE) 1134 panic("vm_page_free: freeing free page"); 1135 else 1136 panic("vm_page_free: freeing busy page"); 1137 } 1138 1139 /* 1140 * unqueue, then remove page. Note that we cannot destroy 1141 * the page here because we do not want to call the pager's 1142 * callback routine until after we've put the page on the 1143 * appropriate free queue. 1144 */ 1145 vm_pageq_remove_nowakeup(m); 1146 vm_page_remove(m); 1147 1148 /* 1149 * If fictitious remove object association and 1150 * return, otherwise delay object association removal. 1151 */ 1152 if ((m->flags & PG_FICTITIOUS) != 0) { 1153 splx(s); 1154 return; 1155 } 1156 1157 m->valid = 0; 1158 vm_page_undirty(m); 1159 1160 if (m->wire_count != 0) { 1161 if (m->wire_count > 1) { 1162 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx", 1163 m->wire_count, (long)m->pindex); 1164 } 1165 panic("vm_page_free: freeing wired page\n"); 1166 } 1167 1168 /* 1169 * If we've exhausted the object's resident pages we want to free 1170 * it up. 1171 */ 1172 if (object && 1173 (object->type == OBJT_VNODE) && 1174 ((object->flags & OBJ_DEAD) == 0) 1175 ) { 1176 struct vnode *vp = (struct vnode *)object->handle; 1177 1178 if (vp && VSHOULDFREE(vp)) 1179 vfree(vp); 1180 } 1181 1182 /* 1183 * Clear the UNMANAGED flag when freeing an unmanaged page. 1184 */ 1185 if (m->flags & PG_UNMANAGED) { 1186 m->flags &= ~PG_UNMANAGED; 1187 } else { 1188#ifdef __alpha__ 1189 pmap_page_is_free(m); 1190#endif 1191 } 1192 1193 if (m->hold_count != 0) { 1194 m->flags &= ~PG_ZERO; 1195 m->queue = PQ_HOLD; 1196 } else 1197 m->queue = PQ_FREE + m->pc; 1198 pq = &vm_page_queues[m->queue]; 1199 mtx_lock_spin(&vm_page_queue_free_mtx); 1200 pq->lcnt++; 1201 ++(*pq->cnt); 1202 1203 /* 1204 * Put zero'd pages on the end ( where we look for zero'd pages 1205 * first ) and non-zerod pages at the head. 1206 */ 1207 if (m->flags & PG_ZERO) { 1208 TAILQ_INSERT_TAIL(&pq->pl, m, pageq); 1209 ++vm_page_zero_count; 1210 } else { 1211 TAILQ_INSERT_HEAD(&pq->pl, m, pageq); 1212 } 1213 mtx_unlock_spin(&vm_page_queue_free_mtx); 1214 vm_page_free_wakeup(); 1215 splx(s); 1216} 1217 1218/* 1219 * vm_page_unmanage: 1220 * 1221 * Prevent PV management from being done on the page. The page is 1222 * removed from the paging queues as if it were wired, and as a 1223 * consequence of no longer being managed the pageout daemon will not 1224 * touch it (since there is no way to locate the pte mappings for the 1225 * page). madvise() calls that mess with the pmap will also no longer 1226 * operate on the page. 1227 * 1228 * Beyond that the page is still reasonably 'normal'. Freeing the page 1229 * will clear the flag. 1230 * 1231 * This routine is used by OBJT_PHYS objects - objects using unswappable 1232 * physical memory as backing store rather then swap-backed memory and 1233 * will eventually be extended to support 4MB unmanaged physical 1234 * mappings. 1235 */ 1236void 1237vm_page_unmanage(vm_page_t m) 1238{ 1239 int s; 1240 1241 s = splvm(); 1242 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1243 if ((m->flags & PG_UNMANAGED) == 0) { 1244 if (m->wire_count == 0) 1245 vm_pageq_remove(m); 1246 } 1247 vm_page_flag_set(m, PG_UNMANAGED); 1248 splx(s); 1249} 1250 1251/* 1252 * vm_page_wire: 1253 * 1254 * Mark this page as wired down by yet 1255 * another map, removing it from paging queues 1256 * as necessary. 1257 * 1258 * The page queues must be locked. 1259 * This routine may not block. 1260 */ 1261void 1262vm_page_wire(vm_page_t m) 1263{ 1264 int s; 1265 1266 /* 1267 * Only bump the wire statistics if the page is not already wired, 1268 * and only unqueue the page if it is on some queue (if it is unmanaged 1269 * it is already off the queues). 1270 */ 1271 s = splvm(); 1272 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1273 if (m->wire_count == 0) { 1274 if ((m->flags & PG_UNMANAGED) == 0) 1275 vm_pageq_remove(m); 1276 cnt.v_wire_count++; 1277 } 1278 m->wire_count++; 1279 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m)); 1280 splx(s); 1281 vm_page_flag_set(m, PG_MAPPED); /* XXX this does not belong here */ 1282} 1283 1284/* 1285 * vm_page_unwire: 1286 * 1287 * Release one wiring of this page, potentially 1288 * enabling it to be paged again. 1289 * 1290 * Many pages placed on the inactive queue should actually go 1291 * into the cache, but it is difficult to figure out which. What 1292 * we do instead, if the inactive target is well met, is to put 1293 * clean pages at the head of the inactive queue instead of the tail. 1294 * This will cause them to be moved to the cache more quickly and 1295 * if not actively re-referenced, freed more quickly. If we just 1296 * stick these pages at the end of the inactive queue, heavy filesystem 1297 * meta-data accesses can cause an unnecessary paging load on memory bound 1298 * processes. This optimization causes one-time-use metadata to be 1299 * reused more quickly. 1300 * 1301 * BUT, if we are in a low-memory situation we have no choice but to 1302 * put clean pages on the cache queue. 1303 * 1304 * A number of routines use vm_page_unwire() to guarantee that the page 1305 * will go into either the inactive or active queues, and will NEVER 1306 * be placed in the cache - for example, just after dirtying a page. 1307 * dirty pages in the cache are not allowed. 1308 * 1309 * The page queues must be locked. 1310 * This routine may not block. 1311 */ 1312void 1313vm_page_unwire(vm_page_t m, int activate) 1314{ 1315 int s; 1316 1317 s = splvm(); 1318 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1319 if (m->wire_count > 0) { 1320 m->wire_count--; 1321 if (m->wire_count == 0) { 1322 cnt.v_wire_count--; 1323 if (m->flags & PG_UNMANAGED) { 1324 ; 1325 } else if (activate) 1326 vm_pageq_enqueue(PQ_ACTIVE, m); 1327 else { 1328 vm_page_flag_clear(m, PG_WINATCFLS); 1329 vm_pageq_enqueue(PQ_INACTIVE, m); 1330 } 1331 } 1332 } else { 1333 panic("vm_page_unwire: invalid wire count: %d\n", m->wire_count); 1334 } 1335 splx(s); 1336} 1337 1338 1339/* 1340 * Move the specified page to the inactive queue. If the page has 1341 * any associated swap, the swap is deallocated. 1342 * 1343 * Normally athead is 0 resulting in LRU operation. athead is set 1344 * to 1 if we want this page to be 'as if it were placed in the cache', 1345 * except without unmapping it from the process address space. 1346 * 1347 * This routine may not block. 1348 */ 1349static __inline void 1350_vm_page_deactivate(vm_page_t m, int athead) 1351{ 1352 int s; 1353 1354 GIANT_REQUIRED; 1355 /* 1356 * Ignore if already inactive. 1357 */ 1358 if (m->queue == PQ_INACTIVE) 1359 return; 1360 1361 s = splvm(); 1362 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) { 1363 if ((m->queue - m->pc) == PQ_CACHE) 1364 cnt.v_reactivated++; 1365 vm_page_flag_clear(m, PG_WINATCFLS); 1366 vm_pageq_remove(m); 1367 if (athead) 1368 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); 1369 else 1370 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq); 1371 m->queue = PQ_INACTIVE; 1372 vm_page_queues[PQ_INACTIVE].lcnt++; 1373 cnt.v_inactive_count++; 1374 } 1375 splx(s); 1376} 1377 1378void 1379vm_page_deactivate(vm_page_t m) 1380{ 1381 _vm_page_deactivate(m, 0); 1382} 1383 1384/* 1385 * vm_page_try_to_cache: 1386 * 1387 * Returns 0 on failure, 1 on success 1388 */ 1389int 1390vm_page_try_to_cache(vm_page_t m) 1391{ 1392 1393 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1394 if (m->dirty || m->hold_count || m->busy || m->wire_count || 1395 (m->flags & (PG_BUSY|PG_UNMANAGED))) { 1396 return (0); 1397 } 1398 vm_page_test_dirty(m); 1399 if (m->dirty) 1400 return (0); 1401 vm_page_cache(m); 1402 return (1); 1403} 1404 1405/* 1406 * vm_page_try_to_free() 1407 * 1408 * Attempt to free the page. If we cannot free it, we do nothing. 1409 * 1 is returned on success, 0 on failure. 1410 */ 1411int 1412vm_page_try_to_free(vm_page_t m) 1413{ 1414 1415 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1416 if (m->dirty || m->hold_count || m->busy || m->wire_count || 1417 (m->flags & (PG_BUSY|PG_UNMANAGED))) { 1418 return (0); 1419 } 1420 vm_page_test_dirty(m); 1421 if (m->dirty) 1422 return (0); 1423 vm_page_busy(m); 1424 vm_page_protect(m, VM_PROT_NONE); 1425 vm_page_free(m); 1426 return (1); 1427} 1428 1429/* 1430 * vm_page_cache 1431 * 1432 * Put the specified page onto the page cache queue (if appropriate). 1433 * 1434 * This routine may not block. 1435 */ 1436void 1437vm_page_cache(vm_page_t m) 1438{ 1439 int s; 1440 1441 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1442 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy || m->wire_count) { 1443 printf("vm_page_cache: attempting to cache busy page\n"); 1444 return; 1445 } 1446 if ((m->queue - m->pc) == PQ_CACHE) 1447 return; 1448 1449 /* 1450 * Remove all pmaps and indicate that the page is not 1451 * writeable or mapped. 1452 */ 1453 vm_page_protect(m, VM_PROT_NONE); 1454 if (m->dirty != 0) { 1455 panic("vm_page_cache: caching a dirty page, pindex: %ld", 1456 (long)m->pindex); 1457 } 1458 s = splvm(); 1459 vm_pageq_remove_nowakeup(m); 1460 vm_pageq_enqueue(PQ_CACHE + m->pc, m); 1461 vm_page_free_wakeup(); 1462 splx(s); 1463} 1464 1465/* 1466 * vm_page_dontneed 1467 * 1468 * Cache, deactivate, or do nothing as appropriate. This routine 1469 * is typically used by madvise() MADV_DONTNEED. 1470 * 1471 * Generally speaking we want to move the page into the cache so 1472 * it gets reused quickly. However, this can result in a silly syndrome 1473 * due to the page recycling too quickly. Small objects will not be 1474 * fully cached. On the otherhand, if we move the page to the inactive 1475 * queue we wind up with a problem whereby very large objects 1476 * unnecessarily blow away our inactive and cache queues. 1477 * 1478 * The solution is to move the pages based on a fixed weighting. We 1479 * either leave them alone, deactivate them, or move them to the cache, 1480 * where moving them to the cache has the highest weighting. 1481 * By forcing some pages into other queues we eventually force the 1482 * system to balance the queues, potentially recovering other unrelated 1483 * space from active. The idea is to not force this to happen too 1484 * often. 1485 */ 1486void 1487vm_page_dontneed(vm_page_t m) 1488{ 1489 static int dnweight; 1490 int dnw; 1491 int head; 1492 1493 mtx_assert(&vm_page_queue_mtx, MA_OWNED); 1494 dnw = ++dnweight; 1495 1496 /* 1497 * occassionally leave the page alone 1498 */ 1499 if ((dnw & 0x01F0) == 0 || 1500 m->queue == PQ_INACTIVE || 1501 m->queue - m->pc == PQ_CACHE 1502 ) { 1503 if (m->act_count >= ACT_INIT) 1504 --m->act_count; 1505 return; 1506 } 1507 1508 if (m->dirty == 0) 1509 vm_page_test_dirty(m); 1510 1511 if (m->dirty || (dnw & 0x0070) == 0) { 1512 /* 1513 * Deactivate the page 3 times out of 32. 1514 */ 1515 head = 0; 1516 } else { 1517 /* 1518 * Cache the page 28 times out of every 32. Note that 1519 * the page is deactivated instead of cached, but placed 1520 * at the head of the queue instead of the tail. 1521 */ 1522 head = 1; 1523 } 1524 _vm_page_deactivate(m, head); 1525} 1526 1527/* 1528 * Grab a page, waiting until we are waken up due to the page 1529 * changing state. We keep on waiting, if the page continues 1530 * to be in the object. If the page doesn't exist, allocate it. 1531 * 1532 * This routine may block. 1533 */ 1534vm_page_t 1535vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) 1536{ 1537 vm_page_t m; 1538 int s, generation; 1539 1540 GIANT_REQUIRED; 1541retrylookup: 1542 if ((m = vm_page_lookup(object, pindex)) != NULL) { 1543 if (m->busy || (m->flags & PG_BUSY)) { 1544 generation = object->generation; 1545 1546 s = splvm(); 1547 while ((object->generation == generation) && 1548 (m->busy || (m->flags & PG_BUSY))) { 1549 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED); 1550 tsleep(m, PVM, "pgrbwt", 0); 1551 if ((allocflags & VM_ALLOC_RETRY) == 0) { 1552 splx(s); 1553 return NULL; 1554 } 1555 } 1556 splx(s); 1557 goto retrylookup; 1558 } else { 1559 vm_page_lock_queues(); 1560 if (allocflags & VM_ALLOC_WIRED) 1561 vm_page_wire(m); 1562 vm_page_busy(m); 1563 vm_page_unlock_queues(); 1564 return m; 1565 } 1566 } 1567 1568 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY); 1569 if (m == NULL) { 1570 VM_WAIT; 1571 if ((allocflags & VM_ALLOC_RETRY) == 0) 1572 return NULL; 1573 goto retrylookup; 1574 } 1575 1576 return m; 1577} 1578 1579/* 1580 * Mapping function for valid bits or for dirty bits in 1581 * a page. May not block. 1582 * 1583 * Inputs are required to range within a page. 1584 */ 1585__inline int 1586vm_page_bits(int base, int size) 1587{ 1588 int first_bit; 1589 int last_bit; 1590 1591 KASSERT( 1592 base + size <= PAGE_SIZE, 1593 ("vm_page_bits: illegal base/size %d/%d", base, size) 1594 ); 1595 1596 if (size == 0) /* handle degenerate case */ 1597 return (0); 1598 1599 first_bit = base >> DEV_BSHIFT; 1600 last_bit = (base + size - 1) >> DEV_BSHIFT; 1601 1602 return ((2 << last_bit) - (1 << first_bit)); 1603} 1604 1605/* 1606 * vm_page_set_validclean: 1607 * 1608 * Sets portions of a page valid and clean. The arguments are expected 1609 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive 1610 * of any partial chunks touched by the range. The invalid portion of 1611 * such chunks will be zero'd. 1612 * 1613 * This routine may not block. 1614 * 1615 * (base + size) must be less then or equal to PAGE_SIZE. 1616 */ 1617void 1618vm_page_set_validclean(vm_page_t m, int base, int size) 1619{ 1620 int pagebits; 1621 int frag; 1622 int endoff; 1623 1624 GIANT_REQUIRED; 1625 if (size == 0) /* handle degenerate case */ 1626 return; 1627 1628 /* 1629 * If the base is not DEV_BSIZE aligned and the valid 1630 * bit is clear, we have to zero out a portion of the 1631 * first block. 1632 */ 1633 if ((frag = base & ~(DEV_BSIZE - 1)) != base && 1634 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0) 1635 pmap_zero_page_area(m, frag, base - frag); 1636 1637 /* 1638 * If the ending offset is not DEV_BSIZE aligned and the 1639 * valid bit is clear, we have to zero out a portion of 1640 * the last block. 1641 */ 1642 endoff = base + size; 1643 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff && 1644 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0) 1645 pmap_zero_page_area(m, endoff, 1646 DEV_BSIZE - (endoff & (DEV_BSIZE - 1))); 1647 1648 /* 1649 * Set valid, clear dirty bits. If validating the entire 1650 * page we can safely clear the pmap modify bit. We also 1651 * use this opportunity to clear the PG_NOSYNC flag. If a process 1652 * takes a write fault on a MAP_NOSYNC memory area the flag will 1653 * be set again. 1654 * 1655 * We set valid bits inclusive of any overlap, but we can only 1656 * clear dirty bits for DEV_BSIZE chunks that are fully within 1657 * the range. 1658 */ 1659 pagebits = vm_page_bits(base, size); 1660 m->valid |= pagebits; 1661#if 0 /* NOT YET */ 1662 if ((frag = base & (DEV_BSIZE - 1)) != 0) { 1663 frag = DEV_BSIZE - frag; 1664 base += frag; 1665 size -= frag; 1666 if (size < 0) 1667 size = 0; 1668 } 1669 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1)); 1670#endif 1671 m->dirty &= ~pagebits; 1672 if (base == 0 && size == PAGE_SIZE) { 1673 pmap_clear_modify(m); 1674 vm_page_flag_clear(m, PG_NOSYNC); 1675 } 1676} 1677 1678#if 0 1679 1680void 1681vm_page_set_dirty(vm_page_t m, int base, int size) 1682{ 1683 m->dirty |= vm_page_bits(base, size); 1684} 1685 1686#endif 1687 1688void 1689vm_page_clear_dirty(vm_page_t m, int base, int size) 1690{ 1691 GIANT_REQUIRED; 1692 m->dirty &= ~vm_page_bits(base, size); 1693} 1694 1695/* 1696 * vm_page_set_invalid: 1697 * 1698 * Invalidates DEV_BSIZE'd chunks within a page. Both the 1699 * valid and dirty bits for the effected areas are cleared. 1700 * 1701 * May not block. 1702 */ 1703void 1704vm_page_set_invalid(vm_page_t m, int base, int size) 1705{ 1706 int bits; 1707 1708 GIANT_REQUIRED; 1709 bits = vm_page_bits(base, size); 1710 m->valid &= ~bits; 1711 m->dirty &= ~bits; 1712 m->object->generation++; 1713} 1714 1715/* 1716 * vm_page_zero_invalid() 1717 * 1718 * The kernel assumes that the invalid portions of a page contain 1719 * garbage, but such pages can be mapped into memory by user code. 1720 * When this occurs, we must zero out the non-valid portions of the 1721 * page so user code sees what it expects. 1722 * 1723 * Pages are most often semi-valid when the end of a file is mapped 1724 * into memory and the file's size is not page aligned. 1725 */ 1726void 1727vm_page_zero_invalid(vm_page_t m, boolean_t setvalid) 1728{ 1729 int b; 1730 int i; 1731 1732 /* 1733 * Scan the valid bits looking for invalid sections that 1734 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the 1735 * valid bit may be set ) have already been zerod by 1736 * vm_page_set_validclean(). 1737 */ 1738 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) { 1739 if (i == (PAGE_SIZE / DEV_BSIZE) || 1740 (m->valid & (1 << i)) 1741 ) { 1742 if (i > b) { 1743 pmap_zero_page_area(m, 1744 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT); 1745 } 1746 b = i + 1; 1747 } 1748 } 1749 1750 /* 1751 * setvalid is TRUE when we can safely set the zero'd areas 1752 * as being valid. We can do this if there are no cache consistancy 1753 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS. 1754 */ 1755 if (setvalid) 1756 m->valid = VM_PAGE_BITS_ALL; 1757} 1758 1759/* 1760 * vm_page_is_valid: 1761 * 1762 * Is (partial) page valid? Note that the case where size == 0 1763 * will return FALSE in the degenerate case where the page is 1764 * entirely invalid, and TRUE otherwise. 1765 * 1766 * May not block. 1767 */ 1768int 1769vm_page_is_valid(vm_page_t m, int base, int size) 1770{ 1771 int bits = vm_page_bits(base, size); 1772 1773 if (m->valid && ((m->valid & bits) == bits)) 1774 return 1; 1775 else 1776 return 0; 1777} 1778 1779/* 1780 * update dirty bits from pmap/mmu. May not block. 1781 */ 1782void 1783vm_page_test_dirty(vm_page_t m) 1784{ 1785 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) { 1786 vm_page_dirty(m); 1787 } 1788} 1789 1790int so_zerocp_fullpage = 0; 1791 1792void 1793vm_page_cowfault(vm_page_t m) 1794{ 1795 vm_page_t mnew; 1796 vm_object_t object; 1797 vm_pindex_t pindex; 1798 1799 object = m->object; 1800 pindex = m->pindex; 1801 vm_page_busy(m); 1802 1803 retry_alloc: 1804 vm_page_remove(m); 1805 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL); 1806 if (mnew == NULL) { 1807 vm_page_insert(m, object, pindex); 1808 VM_WAIT; 1809 goto retry_alloc; 1810 } 1811 1812 if (m->cow == 0) { 1813 /* 1814 * check to see if we raced with an xmit complete when 1815 * waiting to allocate a page. If so, put things back 1816 * the way they were 1817 */ 1818 vm_page_busy(mnew); 1819 vm_page_free(mnew); 1820 vm_page_insert(m, object, pindex); 1821 } else { /* clear COW & copy page */ 1822 if (so_zerocp_fullpage) { 1823 mnew->valid = VM_PAGE_BITS_ALL; 1824 } else { 1825 vm_page_copy(m, mnew); 1826 } 1827 vm_page_dirty(mnew); 1828 vm_page_flag_clear(mnew, PG_BUSY); 1829 } 1830} 1831 1832void 1833vm_page_cowclear(vm_page_t m) 1834{ 1835 1836 /* XXX KDM find out if giant is required here. */ 1837 GIANT_REQUIRED; 1838 if (m->cow) { 1839 atomic_subtract_int(&m->cow, 1); 1840 /* 1841 * let vm_fault add back write permission lazily 1842 */ 1843 } 1844 /* 1845 * sf_buf_free() will free the page, so we needn't do it here 1846 */ 1847} 1848 1849void 1850vm_page_cowsetup(vm_page_t m) 1851{ 1852 /* XXX KDM find out if giant is required here */ 1853 GIANT_REQUIRED; 1854 atomic_add_int(&m->cow, 1); 1855 vm_page_protect(m, VM_PROT_READ); 1856} 1857 1858#include "opt_ddb.h" 1859#ifdef DDB 1860#include <sys/kernel.h> 1861 1862#include <ddb/ddb.h> 1863 1864DB_SHOW_COMMAND(page, vm_page_print_page_info) 1865{ 1866 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count); 1867 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count); 1868 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count); 1869 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count); 1870 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count); 1871 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved); 1872 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min); 1873 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target); 1874 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min); 1875 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target); 1876} 1877 1878DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info) 1879{ 1880 int i; 1881 db_printf("PQ_FREE:"); 1882 for (i = 0; i < PQ_L2_SIZE; i++) { 1883 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt); 1884 } 1885 db_printf("\n"); 1886 1887 db_printf("PQ_CACHE:"); 1888 for (i = 0; i < PQ_L2_SIZE; i++) { 1889 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt); 1890 } 1891 db_printf("\n"); 1892 1893 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n", 1894 vm_page_queues[PQ_ACTIVE].lcnt, 1895 vm_page_queues[PQ_INACTIVE].lcnt); 1896} 1897#endif /* DDB */ 1898