Deleted Added
sdiff udiff text old ( 156420 ) new ( 157908 )
full compact
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 *
5 * This code is derived from software contributed to Berkeley by
6 * The Mach Operating System project at Carnegie-Mellon University.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 4. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91
33 */
34
35/*-
36 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
37 * All rights reserved.
38 *
39 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
40 *
41 * Permission to use, copy, modify and distribute this software and
42 * its documentation is hereby granted, provided that both the copyright
43 * notice and this permission notice appear in all copies of the
44 * software, derivative works or modified versions, and any portions
45 * thereof, and that both notices appear in supporting documentation.
46 *
47 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
48 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
49 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
50 *
51 * Carnegie Mellon requests users of this software to return to
52 *
53 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
54 * School of Computer Science
55 * Carnegie Mellon University
56 * Pittsburgh PA 15213-3890
57 *
58 * any improvements or extensions that they make and grant Carnegie the
59 * rights to redistribute these changes.
60 */
61
62/*
63 * GENERAL RULES ON VM_PAGE MANIPULATION
64 *
65 * - a pageq mutex is required when adding or removing a page from a
66 * page queue (vm_page_queue[]), regardless of other mutexes or the
67 * busy state of a page.
68 *
69 * - a hash chain mutex is required when associating or disassociating
70 * a page from the VM PAGE CACHE hash table (vm_page_buckets),
71 * regardless of other mutexes or the busy state of a page.
72 *
73 * - either a hash chain mutex OR a busied page is required in order
74 * to modify the page flags. A hash chain mutex must be obtained in
75 * order to busy a page. A page's flags cannot be modified by a
76 * hash chain mutex if the page is marked busy.
77 *
78 * - The object memq mutex is held when inserting or removing
79 * pages from an object (vm_page_insert() or vm_page_remove()). This
80 * is different from the object's main mutex.
81 *
82 * Generally speaking, you have to be aware of side effects when running
83 * vm_page ops. A vm_page_lookup() will return with the hash chain
84 * locked, whether it was able to lookup the page or not. vm_page_free(),
85 * vm_page_cache(), vm_page_activate(), and a number of other routines
86 * will release the hash chain mutex for you. Intermediate manipulation
87 * routines such as vm_page_flag_set() expect the hash chain to be held
88 * on entry and the hash chain will remain held on return.
89 *
90 * pageq scanning can only occur with the pageq in question locked.
91 * We have a known bottleneck with the active queue, but the cache
92 * and free queues are actually arrays already.
93 */
94
95/*
96 * Resident memory management module.
97 */
98
99#include <sys/cdefs.h>
100__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 156420 2006-03-08 06:31:46Z imp $");
100__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 157908 2006-04-21 04:24:50Z peter $");
101
102#include <sys/param.h>
103#include <sys/systm.h>
104#include <sys/lock.h>
105#include <sys/kernel.h>
106#include <sys/malloc.h>
107#include <sys/mutex.h>
108#include <sys/proc.h>
109#include <sys/sysctl.h>
110#include <sys/vmmeter.h>
111#include <sys/vnode.h>
112
113#include <vm/vm.h>
114#include <vm/vm_param.h>
115#include <vm/vm_kern.h>
116#include <vm/vm_object.h>
117#include <vm/vm_page.h>
118#include <vm/vm_pageout.h>
119#include <vm/vm_pager.h>
120#include <vm/vm_extern.h>
121#include <vm/uma.h>
122#include <vm/uma_int.h>
123
124#include <machine/md_var.h>
125
126/*
127 * Associated with page of user-allocatable memory is a
128 * page structure.
129 */
130
131struct mtx vm_page_queue_mtx;
132struct mtx vm_page_queue_free_mtx;
133
134vm_page_t vm_page_array = 0;
135int vm_page_array_size = 0;
136long first_page = 0;
137int vm_page_zero_count = 0;
138
139static int boot_pages = UMA_BOOT_PAGES;
140TUNABLE_INT("vm.boot_pages", &boot_pages);
141SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
142 "number of pages allocated for bootstrapping the VM system");
143
144/*
145 * vm_set_page_size:
146 *
147 * Sets the page size, perhaps based upon the memory
148 * size. Must be called before any use of page-size
149 * dependent functions.
150 */
151void
152vm_set_page_size(void)
153{
154 if (cnt.v_page_size == 0)
155 cnt.v_page_size = PAGE_SIZE;
156 if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
157 panic("vm_set_page_size: page size not a power of two");
158}
159
160/*
161 * vm_page_startup:
162 *
163 * Initializes the resident memory module.
164 *
165 * Allocates memory for the page cells, and
166 * for the object/offset-to-page hash table headers.
167 * Each page cell is initialized and placed on the free list.
168 */
169vm_offset_t
170vm_page_startup(vm_offset_t vaddr)
171{
172 vm_offset_t mapped;
173 vm_size_t npages;
174 vm_paddr_t page_range;
175 vm_paddr_t new_end;
176 int i;
177 vm_paddr_t pa;
178 int nblocks;
179 vm_paddr_t last_pa;
180
181 /* the biggest memory array is the second group of pages */
182 vm_paddr_t end;
183 vm_paddr_t biggestsize;
184 int biggestone;
185
186 vm_paddr_t total;
187
188 total = 0;
189 biggestsize = 0;
190 biggestone = 0;
191 nblocks = 0;
192 vaddr = round_page(vaddr);
193
194 for (i = 0; phys_avail[i + 1]; i += 2) {
195 phys_avail[i] = round_page(phys_avail[i]);
196 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
197 }
198
199 for (i = 0; phys_avail[i + 1]; i += 2) {
200 vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
201
202 if (size > biggestsize) {
203 biggestone = i;
204 biggestsize = size;
205 }
206 ++nblocks;
207 total += size;
208 }
209
210 end = phys_avail[biggestone+1];
211
212 /*
213 * Initialize the locks.
214 */
215 mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
216 MTX_RECURSE);
217 mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
218 MTX_SPIN);
219
220 /*
221 * Initialize the queue headers for the free queue, the active queue
222 * and the inactive queue.
223 */
224 vm_pageq_init();
225
226 /*
227 * Allocate memory for use when boot strapping the kernel memory
228 * allocator.
229 */
230 new_end = end - (boot_pages * UMA_SLAB_SIZE);
231 new_end = trunc_page(new_end);
232 mapped = pmap_map(&vaddr, new_end, end,
233 VM_PROT_READ | VM_PROT_WRITE);
234 bzero((void *)mapped, end - new_end);
235 uma_startup((void *)mapped, boot_pages);
236
237#if defined(__amd64__) || defined(__i386__)
238 /*
239 * Allocate a bitmap to indicate that a random physical page
240 * needs to be included in a minidump.
241 *
242 * The amd64 port needs this to indicate which direct map pages
243 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
244 *
245 * However, i386 still needs this workspace internally within the
246 * minidump code. In theory, they are not needed on i386, but are
247 * included should the sf_buf code decide to use them.
248 */
249 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE;
250 vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
251 new_end -= vm_page_dump_size;
252 vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
253 new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
254 bzero((void *)vm_page_dump, vm_page_dump_size);
255#endif
256 /*
257 * Compute the number of pages of memory that will be available for
258 * use (taking into account the overhead of a page structure per
259 * page).
260 */
261 first_page = phys_avail[0] / PAGE_SIZE;
262 page_range = phys_avail[(nblocks - 1) * 2 + 1] / PAGE_SIZE - first_page;
263 npages = (total - (page_range * sizeof(struct vm_page)) -
264 (end - new_end)) / PAGE_SIZE;
265 end = new_end;
266
267 /*
268 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
269 */
270 vaddr += PAGE_SIZE;
271
272 /*
273 * Initialize the mem entry structures now, and put them in the free
274 * queue.
275 */
276 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
277 mapped = pmap_map(&vaddr, new_end, end,
278 VM_PROT_READ | VM_PROT_WRITE);
279 vm_page_array = (vm_page_t) mapped;
280 phys_avail[biggestone + 1] = new_end;
281
282 /*
283 * Clear all of the page structures
284 */
285 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
286 vm_page_array_size = page_range;
287
288 /*
289 * Construct the free queue(s) in descending order (by physical
290 * address) so that the first 16MB of physical memory is allocated
291 * last rather than first. On large-memory machines, this avoids
292 * the exhaustion of low physical memory before isa_dma_init has run.
293 */
294 cnt.v_page_count = 0;
295 cnt.v_free_count = 0;
296 for (i = 0; phys_avail[i + 1] && npages > 0; i += 2) {
297 pa = phys_avail[i];
298 last_pa = phys_avail[i + 1];
299 while (pa < last_pa && npages-- > 0) {
300 vm_pageq_add_new_page(pa);
301 pa += PAGE_SIZE;
302 }
303 }
304 return (vaddr);
305}
306
307void
308vm_page_flag_set(vm_page_t m, unsigned short bits)
309{
310
311 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
312 m->flags |= bits;
313}
314
315void
316vm_page_flag_clear(vm_page_t m, unsigned short bits)
317{
318
319 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
320 m->flags &= ~bits;
321}
322
323void
324vm_page_busy(vm_page_t m)
325{
326
327 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
328 KASSERT((m->flags & PG_BUSY) == 0,
329 ("vm_page_busy: page already busy!!!"));
330 vm_page_flag_set(m, PG_BUSY);
331}
332
333/*
334 * vm_page_flash:
335 *
336 * wakeup anyone waiting for the page.
337 */
338void
339vm_page_flash(vm_page_t m)
340{
341
342 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
343 if (m->flags & PG_WANTED) {
344 vm_page_flag_clear(m, PG_WANTED);
345 wakeup(m);
346 }
347}
348
349/*
350 * vm_page_wakeup:
351 *
352 * clear the PG_BUSY flag and wakeup anyone waiting for the
353 * page.
354 *
355 */
356void
357vm_page_wakeup(vm_page_t m)
358{
359
360 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
361 KASSERT(m->flags & PG_BUSY, ("vm_page_wakeup: page not busy!!!"));
362 vm_page_flag_clear(m, PG_BUSY);
363 vm_page_flash(m);
364}
365
366void
367vm_page_io_start(vm_page_t m)
368{
369
370 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
371 m->busy++;
372}
373
374void
375vm_page_io_finish(vm_page_t m)
376{
377
378 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
379 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
380 m->busy--;
381 if (m->busy == 0)
382 vm_page_flash(m);
383}
384
385/*
386 * Keep page from being freed by the page daemon
387 * much of the same effect as wiring, except much lower
388 * overhead and should be used only for *very* temporary
389 * holding ("wiring").
390 */
391void
392vm_page_hold(vm_page_t mem)
393{
394
395 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
396 mem->hold_count++;
397}
398
399void
400vm_page_unhold(vm_page_t mem)
401{
402
403 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
404 --mem->hold_count;
405 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
406 if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD))
407 vm_page_free_toq(mem);
408}
409
410/*
411 * vm_page_free:
412 *
413 * Free a page
414 *
415 * The clearing of PG_ZERO is a temporary safety until the code can be
416 * reviewed to determine that PG_ZERO is being properly cleared on
417 * write faults or maps. PG_ZERO was previously cleared in
418 * vm_page_alloc().
419 */
420void
421vm_page_free(vm_page_t m)
422{
423 vm_page_flag_clear(m, PG_ZERO);
424 vm_page_free_toq(m);
425 vm_page_zero_idle_wakeup();
426}
427
428/*
429 * vm_page_free_zero:
430 *
431 * Free a page to the zerod-pages queue
432 */
433void
434vm_page_free_zero(vm_page_t m)
435{
436 vm_page_flag_set(m, PG_ZERO);
437 vm_page_free_toq(m);
438}
439
440/*
441 * vm_page_sleep_if_busy:
442 *
443 * Sleep and release the page queues lock if PG_BUSY is set or,
444 * if also_m_busy is TRUE, busy is non-zero. Returns TRUE if the
445 * thread slept and the page queues lock was released.
446 * Otherwise, retains the page queues lock and returns FALSE.
447 */
448int
449vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
450{
451 vm_object_t object;
452
453 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
454 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
455 if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
456 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
457 /*
458 * It's possible that while we sleep, the page will get
459 * unbusied and freed. If we are holding the object
460 * lock, we will assume we hold a reference to the object
461 * such that even if m->object changes, we can re-lock
462 * it.
463 */
464 object = m->object;
465 VM_OBJECT_UNLOCK(object);
466 msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
467 VM_OBJECT_LOCK(object);
468 return (TRUE);
469 }
470 return (FALSE);
471}
472
473/*
474 * vm_page_dirty:
475 *
476 * make page all dirty
477 */
478void
479vm_page_dirty(vm_page_t m)
480{
481 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_CACHE,
482 ("vm_page_dirty: page in cache!"));
483 KASSERT(VM_PAGE_GETKNOWNQUEUE1(m) != PQ_FREE,
484 ("vm_page_dirty: page is free!"));
485 m->dirty = VM_PAGE_BITS_ALL;
486}
487
488/*
489 * vm_page_splay:
490 *
491 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
492 * the vm_page containing the given pindex. If, however, that
493 * pindex is not found in the vm_object, returns a vm_page that is
494 * adjacent to the pindex, coming before or after it.
495 */
496vm_page_t
497vm_page_splay(vm_pindex_t pindex, vm_page_t root)
498{
499 struct vm_page dummy;
500 vm_page_t lefttreemax, righttreemin, y;
501
502 if (root == NULL)
503 return (root);
504 lefttreemax = righttreemin = &dummy;
505 for (;; root = y) {
506 if (pindex < root->pindex) {
507 if ((y = root->left) == NULL)
508 break;
509 if (pindex < y->pindex) {
510 /* Rotate right. */
511 root->left = y->right;
512 y->right = root;
513 root = y;
514 if ((y = root->left) == NULL)
515 break;
516 }
517 /* Link into the new root's right tree. */
518 righttreemin->left = root;
519 righttreemin = root;
520 } else if (pindex > root->pindex) {
521 if ((y = root->right) == NULL)
522 break;
523 if (pindex > y->pindex) {
524 /* Rotate left. */
525 root->right = y->left;
526 y->left = root;
527 root = y;
528 if ((y = root->right) == NULL)
529 break;
530 }
531 /* Link into the new root's left tree. */
532 lefttreemax->right = root;
533 lefttreemax = root;
534 } else
535 break;
536 }
537 /* Assemble the new root. */
538 lefttreemax->right = root->left;
539 righttreemin->left = root->right;
540 root->left = dummy.right;
541 root->right = dummy.left;
542 return (root);
543}
544
545/*
546 * vm_page_insert: [ internal use only ]
547 *
548 * Inserts the given mem entry into the object and object list.
549 *
550 * The pagetables are not updated but will presumably fault the page
551 * in if necessary, or if a kernel page the caller will at some point
552 * enter the page into the kernel's pmap. We are not allowed to block
553 * here so we *can't* do this anyway.
554 *
555 * The object and page must be locked.
556 * This routine may not block.
557 */
558void
559vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
560{
561 vm_page_t root;
562
563 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
564 if (m->object != NULL)
565 panic("vm_page_insert: page already inserted");
566
567 /*
568 * Record the object/offset pair in this page
569 */
570 m->object = object;
571 m->pindex = pindex;
572
573 /*
574 * Now link into the object's ordered list of backed pages.
575 */
576 root = object->root;
577 if (root == NULL) {
578 m->left = NULL;
579 m->right = NULL;
580 TAILQ_INSERT_TAIL(&object->memq, m, listq);
581 } else {
582 root = vm_page_splay(pindex, root);
583 if (pindex < root->pindex) {
584 m->left = root->left;
585 m->right = root;
586 root->left = NULL;
587 TAILQ_INSERT_BEFORE(root, m, listq);
588 } else if (pindex == root->pindex)
589 panic("vm_page_insert: offset already allocated");
590 else {
591 m->right = root->right;
592 m->left = root;
593 root->right = NULL;
594 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
595 }
596 }
597 object->root = m;
598 object->generation++;
599
600 /*
601 * show that the object has one more resident page.
602 */
603 object->resident_page_count++;
604 /*
605 * Hold the vnode until the last page is released.
606 */
607 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
608 vhold((struct vnode *)object->handle);
609
610 /*
611 * Since we are inserting a new and possibly dirty page,
612 * update the object's OBJ_WRITEABLE and OBJ_MIGHTBEDIRTY flags.
613 */
614 if (m->flags & PG_WRITEABLE)
615 vm_object_set_writeable_dirty(object);
616}
617
618/*
619 * vm_page_remove:
620 * NOTE: used by device pager as well -wfj
621 *
622 * Removes the given mem entry from the object/offset-page
623 * table and the object page list, but do not invalidate/terminate
624 * the backing store.
625 *
626 * The object and page must be locked.
627 * The underlying pmap entry (if any) is NOT removed here.
628 * This routine may not block.
629 */
630void
631vm_page_remove(vm_page_t m)
632{
633 vm_object_t object;
634 vm_page_t root;
635
636 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
637 if ((object = m->object) == NULL)
638 return;
639 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
640 if (m->flags & PG_BUSY) {
641 vm_page_flag_clear(m, PG_BUSY);
642 vm_page_flash(m);
643 }
644
645 /*
646 * Now remove from the object's list of backed pages.
647 */
648 if (m != object->root)
649 vm_page_splay(m->pindex, object->root);
650 if (m->left == NULL)
651 root = m->right;
652 else {
653 root = vm_page_splay(m->pindex, m->left);
654 root->right = m->right;
655 }
656 object->root = root;
657 TAILQ_REMOVE(&object->memq, m, listq);
658
659 /*
660 * And show that the object has one fewer resident page.
661 */
662 object->resident_page_count--;
663 object->generation++;
664 /*
665 * The vnode may now be recycled.
666 */
667 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
668 vdrop((struct vnode *)object->handle);
669
670 m->object = NULL;
671}
672
673/*
674 * vm_page_lookup:
675 *
676 * Returns the page associated with the object/offset
677 * pair specified; if none is found, NULL is returned.
678 *
679 * The object must be locked.
680 * This routine may not block.
681 * This is a critical path routine
682 */
683vm_page_t
684vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
685{
686 vm_page_t m;
687
688 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
689 if ((m = object->root) != NULL && m->pindex != pindex) {
690 m = vm_page_splay(pindex, m);
691 if ((object->root = m)->pindex != pindex)
692 m = NULL;
693 }
694 return (m);
695}
696
697/*
698 * vm_page_rename:
699 *
700 * Move the given memory entry from its
701 * current object to the specified target object/offset.
702 *
703 * The object must be locked.
704 * This routine may not block.
705 *
706 * Note: swap associated with the page must be invalidated by the move. We
707 * have to do this for several reasons: (1) we aren't freeing the
708 * page, (2) we are dirtying the page, (3) the VM system is probably
709 * moving the page from object A to B, and will then later move
710 * the backing store from A to B and we can't have a conflict.
711 *
712 * Note: we *always* dirty the page. It is necessary both for the
713 * fact that we moved it, and because we may be invalidating
714 * swap. If the page is on the cache, we have to deactivate it
715 * or vm_page_dirty() will panic. Dirty pages are not allowed
716 * on the cache.
717 */
718void
719vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
720{
721
722 vm_page_remove(m);
723 vm_page_insert(m, new_object, new_pindex);
724 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
725 vm_page_deactivate(m);
726 vm_page_dirty(m);
727}
728
729/*
730 * vm_page_select_cache:
731 *
732 * Move a page of the given color from the cache queue to the free
733 * queue. As pages might be found, but are not applicable, they are
734 * deactivated.
735 *
736 * This routine may not block.
737 */
738vm_page_t
739vm_page_select_cache(int color)
740{
741 vm_object_t object;
742 vm_page_t m;
743 boolean_t was_trylocked;
744
745 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
746 while ((m = vm_pageq_find(PQ_CACHE, color, FALSE)) != NULL) {
747 KASSERT(m->dirty == 0, ("Found dirty cache page %p", m));
748 KASSERT(!pmap_page_is_mapped(m),
749 ("Found mapped cache page %p", m));
750 KASSERT((m->flags & PG_UNMANAGED) == 0,
751 ("Found unmanaged cache page %p", m));
752 KASSERT(m->wire_count == 0, ("Found wired cache page %p", m));
753 if (m->hold_count == 0 && (object = m->object,
754 (was_trylocked = VM_OBJECT_TRYLOCK(object)) ||
755 VM_OBJECT_LOCKED(object))) {
756 KASSERT((m->flags & PG_BUSY) == 0 && m->busy == 0,
757 ("Found busy cache page %p", m));
758 vm_page_free(m);
759 if (was_trylocked)
760 VM_OBJECT_UNLOCK(object);
761 break;
762 }
763 vm_page_deactivate(m);
764 }
765 return (m);
766}
767
768/*
769 * vm_page_alloc:
770 *
771 * Allocate and return a memory cell associated
772 * with this VM object/offset pair.
773 *
774 * page_req classes:
775 * VM_ALLOC_NORMAL normal process request
776 * VM_ALLOC_SYSTEM system *really* needs a page
777 * VM_ALLOC_INTERRUPT interrupt time request
778 * VM_ALLOC_ZERO zero page
779 *
780 * This routine may not block.
781 *
782 * Additional special handling is required when called from an
783 * interrupt (VM_ALLOC_INTERRUPT). We are not allowed to mess with
784 * the page cache in this case.
785 */
786vm_page_t
787vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
788{
789 vm_page_t m = NULL;
790 int color, flags, page_req;
791
792 page_req = req & VM_ALLOC_CLASS_MASK;
793 KASSERT(curthread->td_intr_nesting_level == 0 ||
794 page_req == VM_ALLOC_INTERRUPT,
795 ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
796
797 if ((req & VM_ALLOC_NOOBJ) == 0) {
798 KASSERT(object != NULL,
799 ("vm_page_alloc: NULL object."));
800 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
801 color = (pindex + object->pg_color) & PQ_COLORMASK;
802 } else
803 color = pindex & PQ_COLORMASK;
804
805 /*
806 * The pager is allowed to eat deeper into the free page list.
807 */
808 if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
809 page_req = VM_ALLOC_SYSTEM;
810 };
811
812loop:
813 mtx_lock_spin(&vm_page_queue_free_mtx);
814 if (cnt.v_free_count > cnt.v_free_reserved ||
815 (page_req == VM_ALLOC_SYSTEM &&
816 cnt.v_cache_count == 0 &&
817 cnt.v_free_count > cnt.v_interrupt_free_min) ||
818 (page_req == VM_ALLOC_INTERRUPT && cnt.v_free_count > 0)) {
819 /*
820 * Allocate from the free queue if the number of free pages
821 * exceeds the minimum for the request class.
822 */
823 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
824 } else if (page_req != VM_ALLOC_INTERRUPT) {
825 mtx_unlock_spin(&vm_page_queue_free_mtx);
826 /*
827 * Allocatable from cache (non-interrupt only). On success,
828 * we must free the page and try again, thus ensuring that
829 * cnt.v_*_free_min counters are replenished.
830 */
831 vm_page_lock_queues();
832 if ((m = vm_page_select_cache(color)) == NULL) {
833 KASSERT(cnt.v_cache_count == 0,
834 ("vm_page_alloc: cache queue is missing %d pages",
835 cnt.v_cache_count));
836 vm_page_unlock_queues();
837 atomic_add_int(&vm_pageout_deficit, 1);
838 pagedaemon_wakeup();
839
840 if (page_req != VM_ALLOC_SYSTEM)
841 return NULL;
842
843 mtx_lock_spin(&vm_page_queue_free_mtx);
844 if (cnt.v_free_count <= cnt.v_interrupt_free_min) {
845 mtx_unlock_spin(&vm_page_queue_free_mtx);
846 return (NULL);
847 }
848 m = vm_pageq_find(PQ_FREE, color, (req & VM_ALLOC_ZERO) != 0);
849 } else {
850 vm_page_unlock_queues();
851 goto loop;
852 }
853 } else {
854 /*
855 * Not allocatable from cache from interrupt, give up.
856 */
857 mtx_unlock_spin(&vm_page_queue_free_mtx);
858 atomic_add_int(&vm_pageout_deficit, 1);
859 pagedaemon_wakeup();
860 return (NULL);
861 }
862
863 /*
864 * At this point we had better have found a good page.
865 */
866
867 KASSERT(
868 m != NULL,
869 ("vm_page_alloc(): missing page on free queue")
870 );
871
872 /*
873 * Remove from free queue
874 */
875 vm_pageq_remove_nowakeup(m);
876
877 /*
878 * Initialize structure. Only the PG_ZERO flag is inherited.
879 */
880 flags = PG_BUSY;
881 if (m->flags & PG_ZERO) {
882 vm_page_zero_count--;
883 if (req & VM_ALLOC_ZERO)
884 flags = PG_ZERO | PG_BUSY;
885 }
886 if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
887 flags &= ~PG_BUSY;
888 m->flags = flags;
889 if (req & VM_ALLOC_WIRED) {
890 atomic_add_int(&cnt.v_wire_count, 1);
891 m->wire_count = 1;
892 } else
893 m->wire_count = 0;
894 m->hold_count = 0;
895 m->act_count = 0;
896 m->busy = 0;
897 m->valid = 0;
898 KASSERT(m->dirty == 0, ("vm_page_alloc: free/cache page %p was dirty", m));
899 mtx_unlock_spin(&vm_page_queue_free_mtx);
900
901 if ((req & VM_ALLOC_NOOBJ) == 0)
902 vm_page_insert(m, object, pindex);
903 else
904 m->pindex = pindex;
905
906 /*
907 * Don't wakeup too often - wakeup the pageout daemon when
908 * we would be nearly out of memory.
909 */
910 if (vm_paging_needed())
911 pagedaemon_wakeup();
912
913 return (m);
914}
915
916/*
917 * vm_wait: (also see VM_WAIT macro)
918 *
919 * Block until free pages are available for allocation
920 * - Called in various places before memory allocations.
921 */
922void
923vm_wait(void)
924{
925
926 vm_page_lock_queues();
927 if (curproc == pageproc) {
928 vm_pageout_pages_needed = 1;
929 msleep(&vm_pageout_pages_needed, &vm_page_queue_mtx,
930 PDROP | PSWP, "VMWait", 0);
931 } else {
932 if (!vm_pages_needed) {
933 vm_pages_needed = 1;
934 wakeup(&vm_pages_needed);
935 }
936 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PVM,
937 "vmwait", 0);
938 }
939}
940
941/*
942 * vm_waitpfault: (also see VM_WAITPFAULT macro)
943 *
944 * Block until free pages are available for allocation
945 * - Called only in vm_fault so that processes page faulting
946 * can be easily tracked.
947 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
948 * processes will be able to grab memory first. Do not change
949 * this balance without careful testing first.
950 */
951void
952vm_waitpfault(void)
953{
954
955 vm_page_lock_queues();
956 if (!vm_pages_needed) {
957 vm_pages_needed = 1;
958 wakeup(&vm_pages_needed);
959 }
960 msleep(&cnt.v_free_count, &vm_page_queue_mtx, PDROP | PUSER,
961 "pfault", 0);
962}
963
964/*
965 * vm_page_activate:
966 *
967 * Put the specified page on the active list (if appropriate).
968 * Ensure that act_count is at least ACT_INIT but do not otherwise
969 * mess with it.
970 *
971 * The page queues must be locked.
972 * This routine may not block.
973 */
974void
975vm_page_activate(vm_page_t m)
976{
977
978 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
979 if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
980 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
981 cnt.v_reactivated++;
982 vm_pageq_remove(m);
983 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
984 if (m->act_count < ACT_INIT)
985 m->act_count = ACT_INIT;
986 vm_pageq_enqueue(PQ_ACTIVE, m);
987 }
988 } else {
989 if (m->act_count < ACT_INIT)
990 m->act_count = ACT_INIT;
991 }
992}
993
994/*
995 * vm_page_free_wakeup:
996 *
997 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
998 * routine is called when a page has been added to the cache or free
999 * queues.
1000 *
1001 * The page queues must be locked.
1002 * This routine may not block.
1003 */
1004static inline void
1005vm_page_free_wakeup(void)
1006{
1007
1008 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1009 /*
1010 * if pageout daemon needs pages, then tell it that there are
1011 * some free.
1012 */
1013 if (vm_pageout_pages_needed &&
1014 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1015 wakeup(&vm_pageout_pages_needed);
1016 vm_pageout_pages_needed = 0;
1017 }
1018 /*
1019 * wakeup processes that are waiting on memory if we hit a
1020 * high water mark. And wakeup scheduler process if we have
1021 * lots of memory. this process will swapin processes.
1022 */
1023 if (vm_pages_needed && !vm_page_count_min()) {
1024 vm_pages_needed = 0;
1025 wakeup(&cnt.v_free_count);
1026 }
1027}
1028
1029/*
1030 * vm_page_free_toq:
1031 *
1032 * Returns the given page to the PQ_FREE list,
1033 * disassociating it with any VM object.
1034 *
1035 * Object and page must be locked prior to entry.
1036 * This routine may not block.
1037 */
1038
1039void
1040vm_page_free_toq(vm_page_t m)
1041{
1042 struct vpgqueues *pq;
1043
1044 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1045 KASSERT(!pmap_page_is_mapped(m),
1046 ("vm_page_free_toq: freeing mapped page %p", m));
1047 cnt.v_tfree++;
1048
1049 if (m->busy || VM_PAGE_INQUEUE1(m, PQ_FREE)) {
1050 printf(
1051 "vm_page_free: pindex(%lu), busy(%d), PG_BUSY(%d), hold(%d)\n",
1052 (u_long)m->pindex, m->busy, (m->flags & PG_BUSY) ? 1 : 0,
1053 m->hold_count);
1054 if (VM_PAGE_INQUEUE1(m, PQ_FREE))
1055 panic("vm_page_free: freeing free page");
1056 else
1057 panic("vm_page_free: freeing busy page");
1058 }
1059
1060 /*
1061 * unqueue, then remove page. Note that we cannot destroy
1062 * the page here because we do not want to call the pager's
1063 * callback routine until after we've put the page on the
1064 * appropriate free queue.
1065 */
1066 vm_pageq_remove_nowakeup(m);
1067 vm_page_remove(m);
1068
1069 /*
1070 * If fictitious remove object association and
1071 * return, otherwise delay object association removal.
1072 */
1073 if ((m->flags & PG_FICTITIOUS) != 0) {
1074 return;
1075 }
1076
1077 m->valid = 0;
1078 vm_page_undirty(m);
1079
1080 if (m->wire_count != 0) {
1081 if (m->wire_count > 1) {
1082 panic("vm_page_free: invalid wire count (%d), pindex: 0x%lx",
1083 m->wire_count, (long)m->pindex);
1084 }
1085 panic("vm_page_free: freeing wired page");
1086 }
1087
1088 /*
1089 * Clear the UNMANAGED flag when freeing an unmanaged page.
1090 */
1091 if (m->flags & PG_UNMANAGED) {
1092 m->flags &= ~PG_UNMANAGED;
1093 }
1094
1095 if (m->hold_count != 0) {
1096 m->flags &= ~PG_ZERO;
1097 VM_PAGE_SETQUEUE2(m, PQ_HOLD);
1098 } else
1099 VM_PAGE_SETQUEUE1(m, PQ_FREE);
1100 pq = &vm_page_queues[VM_PAGE_GETQUEUE(m)];
1101 mtx_lock_spin(&vm_page_queue_free_mtx);
1102 pq->lcnt++;
1103 ++(*pq->cnt);
1104
1105 /*
1106 * Put zero'd pages on the end ( where we look for zero'd pages
1107 * first ) and non-zerod pages at the head.
1108 */
1109 if (m->flags & PG_ZERO) {
1110 TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
1111 ++vm_page_zero_count;
1112 } else {
1113 TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
1114 }
1115 mtx_unlock_spin(&vm_page_queue_free_mtx);
1116 vm_page_free_wakeup();
1117}
1118
1119/*
1120 * vm_page_unmanage:
1121 *
1122 * Prevent PV management from being done on the page. The page is
1123 * removed from the paging queues as if it were wired, and as a
1124 * consequence of no longer being managed the pageout daemon will not
1125 * touch it (since there is no way to locate the pte mappings for the
1126 * page). madvise() calls that mess with the pmap will also no longer
1127 * operate on the page.
1128 *
1129 * Beyond that the page is still reasonably 'normal'. Freeing the page
1130 * will clear the flag.
1131 *
1132 * This routine is used by OBJT_PHYS objects - objects using unswappable
1133 * physical memory as backing store rather then swap-backed memory and
1134 * will eventually be extended to support 4MB unmanaged physical
1135 * mappings.
1136 */
1137void
1138vm_page_unmanage(vm_page_t m)
1139{
1140
1141 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1142 if ((m->flags & PG_UNMANAGED) == 0) {
1143 if (m->wire_count == 0)
1144 vm_pageq_remove(m);
1145 }
1146 vm_page_flag_set(m, PG_UNMANAGED);
1147}
1148
1149/*
1150 * vm_page_wire:
1151 *
1152 * Mark this page as wired down by yet
1153 * another map, removing it from paging queues
1154 * as necessary.
1155 *
1156 * The page queues must be locked.
1157 * This routine may not block.
1158 */
1159void
1160vm_page_wire(vm_page_t m)
1161{
1162
1163 /*
1164 * Only bump the wire statistics if the page is not already wired,
1165 * and only unqueue the page if it is on some queue (if it is unmanaged
1166 * it is already off the queues).
1167 */
1168 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1169 if (m->flags & PG_FICTITIOUS)
1170 return;
1171 if (m->wire_count == 0) {
1172 if ((m->flags & PG_UNMANAGED) == 0)
1173 vm_pageq_remove(m);
1174 atomic_add_int(&cnt.v_wire_count, 1);
1175 }
1176 m->wire_count++;
1177 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1178}
1179
1180/*
1181 * vm_page_unwire:
1182 *
1183 * Release one wiring of this page, potentially
1184 * enabling it to be paged again.
1185 *
1186 * Many pages placed on the inactive queue should actually go
1187 * into the cache, but it is difficult to figure out which. What
1188 * we do instead, if the inactive target is well met, is to put
1189 * clean pages at the head of the inactive queue instead of the tail.
1190 * This will cause them to be moved to the cache more quickly and
1191 * if not actively re-referenced, freed more quickly. If we just
1192 * stick these pages at the end of the inactive queue, heavy filesystem
1193 * meta-data accesses can cause an unnecessary paging load on memory bound
1194 * processes. This optimization causes one-time-use metadata to be
1195 * reused more quickly.
1196 *
1197 * BUT, if we are in a low-memory situation we have no choice but to
1198 * put clean pages on the cache queue.
1199 *
1200 * A number of routines use vm_page_unwire() to guarantee that the page
1201 * will go into either the inactive or active queues, and will NEVER
1202 * be placed in the cache - for example, just after dirtying a page.
1203 * dirty pages in the cache are not allowed.
1204 *
1205 * The page queues must be locked.
1206 * This routine may not block.
1207 */
1208void
1209vm_page_unwire(vm_page_t m, int activate)
1210{
1211
1212 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1213 if (m->flags & PG_FICTITIOUS)
1214 return;
1215 if (m->wire_count > 0) {
1216 m->wire_count--;
1217 if (m->wire_count == 0) {
1218 atomic_subtract_int(&cnt.v_wire_count, 1);
1219 if (m->flags & PG_UNMANAGED) {
1220 ;
1221 } else if (activate)
1222 vm_pageq_enqueue(PQ_ACTIVE, m);
1223 else {
1224 vm_page_flag_clear(m, PG_WINATCFLS);
1225 vm_pageq_enqueue(PQ_INACTIVE, m);
1226 }
1227 }
1228 } else {
1229 panic("vm_page_unwire: invalid wire count: %d", m->wire_count);
1230 }
1231}
1232
1233
1234/*
1235 * Move the specified page to the inactive queue. If the page has
1236 * any associated swap, the swap is deallocated.
1237 *
1238 * Normally athead is 0 resulting in LRU operation. athead is set
1239 * to 1 if we want this page to be 'as if it were placed in the cache',
1240 * except without unmapping it from the process address space.
1241 *
1242 * This routine may not block.
1243 */
1244static inline void
1245_vm_page_deactivate(vm_page_t m, int athead)
1246{
1247
1248 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1249
1250 /*
1251 * Ignore if already inactive.
1252 */
1253 if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE))
1254 return;
1255 if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1256 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1257 cnt.v_reactivated++;
1258 vm_page_flag_clear(m, PG_WINATCFLS);
1259 vm_pageq_remove(m);
1260 if (athead)
1261 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1262 else
1263 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
1264 VM_PAGE_SETQUEUE2(m, PQ_INACTIVE);
1265 vm_page_queues[PQ_INACTIVE].lcnt++;
1266 cnt.v_inactive_count++;
1267 }
1268}
1269
1270void
1271vm_page_deactivate(vm_page_t m)
1272{
1273 _vm_page_deactivate(m, 0);
1274}
1275
1276/*
1277 * vm_page_try_to_cache:
1278 *
1279 * Returns 0 on failure, 1 on success
1280 */
1281int
1282vm_page_try_to_cache(vm_page_t m)
1283{
1284
1285 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1286 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1287 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1288 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1289 return (0);
1290 }
1291 pmap_remove_all(m);
1292 if (m->dirty)
1293 return (0);
1294 vm_page_cache(m);
1295 return (1);
1296}
1297
1298/*
1299 * vm_page_try_to_free()
1300 *
1301 * Attempt to free the page. If we cannot free it, we do nothing.
1302 * 1 is returned on success, 0 on failure.
1303 */
1304int
1305vm_page_try_to_free(vm_page_t m)
1306{
1307
1308 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1309 if (m->object != NULL)
1310 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1311 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1312 (m->flags & (PG_BUSY|PG_UNMANAGED))) {
1313 return (0);
1314 }
1315 pmap_remove_all(m);
1316 if (m->dirty)
1317 return (0);
1318 vm_page_free(m);
1319 return (1);
1320}
1321
1322/*
1323 * vm_page_cache
1324 *
1325 * Put the specified page onto the page cache queue (if appropriate).
1326 *
1327 * This routine may not block.
1328 */
1329void
1330vm_page_cache(vm_page_t m)
1331{
1332
1333 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1334 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1335 if ((m->flags & (PG_BUSY|PG_UNMANAGED)) || m->busy ||
1336 m->hold_count || m->wire_count) {
1337 printf("vm_page_cache: attempting to cache busy page\n");
1338 return;
1339 }
1340 if (VM_PAGE_INQUEUE1(m, PQ_CACHE))
1341 return;
1342
1343 /*
1344 * Remove all pmaps and indicate that the page is not
1345 * writeable or mapped.
1346 */
1347 pmap_remove_all(m);
1348 if (m->dirty != 0) {
1349 panic("vm_page_cache: caching a dirty page, pindex: %ld",
1350 (long)m->pindex);
1351 }
1352 vm_pageq_remove_nowakeup(m);
1353 vm_pageq_enqueue(PQ_CACHE + m->pc, m);
1354 vm_page_free_wakeup();
1355}
1356
1357/*
1358 * vm_page_dontneed
1359 *
1360 * Cache, deactivate, or do nothing as appropriate. This routine
1361 * is typically used by madvise() MADV_DONTNEED.
1362 *
1363 * Generally speaking we want to move the page into the cache so
1364 * it gets reused quickly. However, this can result in a silly syndrome
1365 * due to the page recycling too quickly. Small objects will not be
1366 * fully cached. On the otherhand, if we move the page to the inactive
1367 * queue we wind up with a problem whereby very large objects
1368 * unnecessarily blow away our inactive and cache queues.
1369 *
1370 * The solution is to move the pages based on a fixed weighting. We
1371 * either leave them alone, deactivate them, or move them to the cache,
1372 * where moving them to the cache has the highest weighting.
1373 * By forcing some pages into other queues we eventually force the
1374 * system to balance the queues, potentially recovering other unrelated
1375 * space from active. The idea is to not force this to happen too
1376 * often.
1377 */
1378void
1379vm_page_dontneed(vm_page_t m)
1380{
1381 static int dnweight;
1382 int dnw;
1383 int head;
1384
1385 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1386 dnw = ++dnweight;
1387
1388 /*
1389 * occassionally leave the page alone
1390 */
1391 if ((dnw & 0x01F0) == 0 ||
1392 VM_PAGE_INQUEUE2(m, PQ_INACTIVE) ||
1393 VM_PAGE_INQUEUE1(m, PQ_CACHE)
1394 ) {
1395 if (m->act_count >= ACT_INIT)
1396 --m->act_count;
1397 return;
1398 }
1399
1400 if (m->dirty == 0 && pmap_is_modified(m))
1401 vm_page_dirty(m);
1402
1403 if (m->dirty || (dnw & 0x0070) == 0) {
1404 /*
1405 * Deactivate the page 3 times out of 32.
1406 */
1407 head = 0;
1408 } else {
1409 /*
1410 * Cache the page 28 times out of every 32. Note that
1411 * the page is deactivated instead of cached, but placed
1412 * at the head of the queue instead of the tail.
1413 */
1414 head = 1;
1415 }
1416 _vm_page_deactivate(m, head);
1417}
1418
1419/*
1420 * Grab a page, waiting until we are waken up due to the page
1421 * changing state. We keep on waiting, if the page continues
1422 * to be in the object. If the page doesn't exist, first allocate it
1423 * and then conditionally zero it.
1424 *
1425 * This routine may block.
1426 */
1427vm_page_t
1428vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
1429{
1430 vm_page_t m;
1431
1432 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1433retrylookup:
1434 if ((m = vm_page_lookup(object, pindex)) != NULL) {
1435 vm_page_lock_queues();
1436 if (m->busy || (m->flags & PG_BUSY)) {
1437 vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
1438 VM_OBJECT_UNLOCK(object);
1439 msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
1440 VM_OBJECT_LOCK(object);
1441 if ((allocflags & VM_ALLOC_RETRY) == 0)
1442 return (NULL);
1443 goto retrylookup;
1444 } else {
1445 if (allocflags & VM_ALLOC_WIRED)
1446 vm_page_wire(m);
1447 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
1448 vm_page_busy(m);
1449 vm_page_unlock_queues();
1450 return (m);
1451 }
1452 }
1453 m = vm_page_alloc(object, pindex, allocflags & ~VM_ALLOC_RETRY);
1454 if (m == NULL) {
1455 VM_OBJECT_UNLOCK(object);
1456 VM_WAIT;
1457 VM_OBJECT_LOCK(object);
1458 if ((allocflags & VM_ALLOC_RETRY) == 0)
1459 return (NULL);
1460 goto retrylookup;
1461 }
1462 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
1463 pmap_zero_page(m);
1464 return (m);
1465}
1466
1467/*
1468 * Mapping function for valid bits or for dirty bits in
1469 * a page. May not block.
1470 *
1471 * Inputs are required to range within a page.
1472 */
1473inline int
1474vm_page_bits(int base, int size)
1475{
1476 int first_bit;
1477 int last_bit;
1478
1479 KASSERT(
1480 base + size <= PAGE_SIZE,
1481 ("vm_page_bits: illegal base/size %d/%d", base, size)
1482 );
1483
1484 if (size == 0) /* handle degenerate case */
1485 return (0);
1486
1487 first_bit = base >> DEV_BSHIFT;
1488 last_bit = (base + size - 1) >> DEV_BSHIFT;
1489
1490 return ((2 << last_bit) - (1 << first_bit));
1491}
1492
1493/*
1494 * vm_page_set_validclean:
1495 *
1496 * Sets portions of a page valid and clean. The arguments are expected
1497 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
1498 * of any partial chunks touched by the range. The invalid portion of
1499 * such chunks will be zero'd.
1500 *
1501 * This routine may not block.
1502 *
1503 * (base + size) must be less then or equal to PAGE_SIZE.
1504 */
1505void
1506vm_page_set_validclean(vm_page_t m, int base, int size)
1507{
1508 int pagebits;
1509 int frag;
1510 int endoff;
1511
1512 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1513 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1514 if (size == 0) /* handle degenerate case */
1515 return;
1516
1517 /*
1518 * If the base is not DEV_BSIZE aligned and the valid
1519 * bit is clear, we have to zero out a portion of the
1520 * first block.
1521 */
1522 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
1523 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
1524 pmap_zero_page_area(m, frag, base - frag);
1525
1526 /*
1527 * If the ending offset is not DEV_BSIZE aligned and the
1528 * valid bit is clear, we have to zero out a portion of
1529 * the last block.
1530 */
1531 endoff = base + size;
1532 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
1533 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
1534 pmap_zero_page_area(m, endoff,
1535 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
1536
1537 /*
1538 * Set valid, clear dirty bits. If validating the entire
1539 * page we can safely clear the pmap modify bit. We also
1540 * use this opportunity to clear the PG_NOSYNC flag. If a process
1541 * takes a write fault on a MAP_NOSYNC memory area the flag will
1542 * be set again.
1543 *
1544 * We set valid bits inclusive of any overlap, but we can only
1545 * clear dirty bits for DEV_BSIZE chunks that are fully within
1546 * the range.
1547 */
1548 pagebits = vm_page_bits(base, size);
1549 m->valid |= pagebits;
1550#if 0 /* NOT YET */
1551 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
1552 frag = DEV_BSIZE - frag;
1553 base += frag;
1554 size -= frag;
1555 if (size < 0)
1556 size = 0;
1557 }
1558 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
1559#endif
1560 m->dirty &= ~pagebits;
1561 if (base == 0 && size == PAGE_SIZE) {
1562 pmap_clear_modify(m);
1563 vm_page_flag_clear(m, PG_NOSYNC);
1564 }
1565}
1566
1567void
1568vm_page_clear_dirty(vm_page_t m, int base, int size)
1569{
1570
1571 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1572 m->dirty &= ~vm_page_bits(base, size);
1573}
1574
1575/*
1576 * vm_page_set_invalid:
1577 *
1578 * Invalidates DEV_BSIZE'd chunks within a page. Both the
1579 * valid and dirty bits for the effected areas are cleared.
1580 *
1581 * May not block.
1582 */
1583void
1584vm_page_set_invalid(vm_page_t m, int base, int size)
1585{
1586 int bits;
1587
1588 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1589 bits = vm_page_bits(base, size);
1590 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1591 if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
1592 pmap_remove_all(m);
1593 m->valid &= ~bits;
1594 m->dirty &= ~bits;
1595 m->object->generation++;
1596}
1597
1598/*
1599 * vm_page_zero_invalid()
1600 *
1601 * The kernel assumes that the invalid portions of a page contain
1602 * garbage, but such pages can be mapped into memory by user code.
1603 * When this occurs, we must zero out the non-valid portions of the
1604 * page so user code sees what it expects.
1605 *
1606 * Pages are most often semi-valid when the end of a file is mapped
1607 * into memory and the file's size is not page aligned.
1608 */
1609void
1610vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
1611{
1612 int b;
1613 int i;
1614
1615 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1616 /*
1617 * Scan the valid bits looking for invalid sections that
1618 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
1619 * valid bit may be set ) have already been zerod by
1620 * vm_page_set_validclean().
1621 */
1622 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
1623 if (i == (PAGE_SIZE / DEV_BSIZE) ||
1624 (m->valid & (1 << i))
1625 ) {
1626 if (i > b) {
1627 pmap_zero_page_area(m,
1628 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
1629 }
1630 b = i + 1;
1631 }
1632 }
1633
1634 /*
1635 * setvalid is TRUE when we can safely set the zero'd areas
1636 * as being valid. We can do this if there are no cache consistancy
1637 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
1638 */
1639 if (setvalid)
1640 m->valid = VM_PAGE_BITS_ALL;
1641}
1642
1643/*
1644 * vm_page_is_valid:
1645 *
1646 * Is (partial) page valid? Note that the case where size == 0
1647 * will return FALSE in the degenerate case where the page is
1648 * entirely invalid, and TRUE otherwise.
1649 *
1650 * May not block.
1651 */
1652int
1653vm_page_is_valid(vm_page_t m, int base, int size)
1654{
1655 int bits = vm_page_bits(base, size);
1656
1657 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1658 if (m->valid && ((m->valid & bits) == bits))
1659 return 1;
1660 else
1661 return 0;
1662}
1663
1664/*
1665 * update dirty bits from pmap/mmu. May not block.
1666 */
1667void
1668vm_page_test_dirty(vm_page_t m)
1669{
1670 if ((m->dirty != VM_PAGE_BITS_ALL) && pmap_is_modified(m)) {
1671 vm_page_dirty(m);
1672 }
1673}
1674
1675int so_zerocp_fullpage = 0;
1676
1677void
1678vm_page_cowfault(vm_page_t m)
1679{
1680 vm_page_t mnew;
1681 vm_object_t object;
1682 vm_pindex_t pindex;
1683
1684 object = m->object;
1685 pindex = m->pindex;
1686
1687 retry_alloc:
1688 pmap_remove_all(m);
1689 vm_page_remove(m);
1690 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL);
1691 if (mnew == NULL) {
1692 vm_page_insert(m, object, pindex);
1693 vm_page_unlock_queues();
1694 VM_OBJECT_UNLOCK(object);
1695 VM_WAIT;
1696 VM_OBJECT_LOCK(object);
1697 vm_page_lock_queues();
1698 goto retry_alloc;
1699 }
1700
1701 if (m->cow == 0) {
1702 /*
1703 * check to see if we raced with an xmit complete when
1704 * waiting to allocate a page. If so, put things back
1705 * the way they were
1706 */
1707 vm_page_free(mnew);
1708 vm_page_insert(m, object, pindex);
1709 } else { /* clear COW & copy page */
1710 if (!so_zerocp_fullpage)
1711 pmap_copy_page(m, mnew);
1712 mnew->valid = VM_PAGE_BITS_ALL;
1713 vm_page_dirty(mnew);
1714 vm_page_flag_clear(mnew, PG_BUSY);
1715 mnew->wire_count = m->wire_count - m->cow;
1716 m->wire_count = m->cow;
1717 }
1718}
1719
1720void
1721vm_page_cowclear(vm_page_t m)
1722{
1723
1724 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1725 if (m->cow) {
1726 m->cow--;
1727 /*
1728 * let vm_fault add back write permission lazily
1729 */
1730 }
1731 /*
1732 * sf_buf_free() will free the page, so we needn't do it here
1733 */
1734}
1735
1736void
1737vm_page_cowsetup(vm_page_t m)
1738{
1739
1740 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1741 m->cow++;
1742 pmap_page_protect(m, VM_PROT_READ);
1743}
1744
1745#include "opt_ddb.h"
1746#ifdef DDB
1747#include <sys/kernel.h>
1748
1749#include <ddb/ddb.h>
1750
1751DB_SHOW_COMMAND(page, vm_page_print_page_info)
1752{
1753 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
1754 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
1755 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
1756 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
1757 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
1758 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
1759 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
1760 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
1761 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
1762 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
1763}
1764
1765DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
1766{
1767 int i;
1768 db_printf("PQ_FREE:");
1769 for (i = 0; i < PQ_NUMCOLORS; i++) {
1770 db_printf(" %d", vm_page_queues[PQ_FREE + i].lcnt);
1771 }
1772 db_printf("\n");
1773
1774 db_printf("PQ_CACHE:");
1775 for (i = 0; i < PQ_NUMCOLORS; i++) {
1776 db_printf(" %d", vm_page_queues[PQ_CACHE + i].lcnt);
1777 }
1778 db_printf("\n");
1779
1780 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
1781 vm_page_queues[PQ_ACTIVE].lcnt,
1782 vm_page_queues[PQ_INACTIVE].lcnt);
1783}
1784#endif /* DDB */