vm_page.c revision 216899
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1998 Matthew Dillon.  All Rights Reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * The Mach Operating System project at Carnegie-Mellon University.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 4. Neither the name of the University nor the names of its contributors
18 *    may be used to endorse or promote products derived from this software
19 *    without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 *	from: @(#)vm_page.c	7.4 (Berkeley) 5/7/91
34 */
35
36/*-
37 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38 * All rights reserved.
39 *
40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41 *
42 * Permission to use, copy, modify and distribute this software and
43 * its documentation is hereby granted, provided that both the copyright
44 * notice and this permission notice appear in all copies of the
45 * software, derivative works or modified versions, and any portions
46 * thereof, and that both notices appear in supporting documentation.
47 *
48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51 *
52 * Carnegie Mellon requests users of this software to return to
53 *
54 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
55 *  School of Computer Science
56 *  Carnegie Mellon University
57 *  Pittsburgh PA 15213-3890
58 *
59 * any improvements or extensions that they make and grant Carnegie the
60 * rights to redistribute these changes.
61 */
62
63/*
64 *			GENERAL RULES ON VM_PAGE MANIPULATION
65 *
66 *	- a pageq mutex is required when adding or removing a page from a
67 *	  page queue (vm_page_queue[]), regardless of other mutexes or the
68 *	  busy state of a page.
69 *
70 *	- a hash chain mutex is required when associating or disassociating
71 *	  a page from the VM PAGE CACHE hash table (vm_page_buckets),
72 *	  regardless of other mutexes or the busy state of a page.
73 *
74 *	- either a hash chain mutex OR a busied page is required in order
75 *	  to modify the page flags.  A hash chain mutex must be obtained in
76 *	  order to busy a page.  A page's flags cannot be modified by a
77 *	  hash chain mutex if the page is marked busy.
78 *
79 *	- The object memq mutex is held when inserting or removing
80 *	  pages from an object (vm_page_insert() or vm_page_remove()).  This
81 *	  is different from the object's main mutex.
82 *
83 *	Generally speaking, you have to be aware of side effects when running
84 *	vm_page ops.  A vm_page_lookup() will return with the hash chain
85 *	locked, whether it was able to lookup the page or not.  vm_page_free(),
86 *	vm_page_cache(), vm_page_activate(), and a number of other routines
87 *	will release the hash chain mutex for you.  Intermediate manipulation
88 *	routines such as vm_page_flag_set() expect the hash chain to be held
89 *	on entry and the hash chain will remain held on return.
90 *
91 *	pageq scanning can only occur with the pageq in question locked.
92 *	We have a known bottleneck with the active queue, but the cache
93 *	and free queues are actually arrays already.
94 */
95
96/*
97 *	Resident memory management module.
98 */
99
100#include <sys/cdefs.h>
101__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 216899 2011-01-03 00:41:56Z alc $");
102
103#include "opt_msgbuf.h"
104#include "opt_vm.h"
105
106#include <sys/param.h>
107#include <sys/systm.h>
108#include <sys/lock.h>
109#include <sys/kernel.h>
110#include <sys/limits.h>
111#include <sys/malloc.h>
112#include <sys/msgbuf.h>
113#include <sys/mutex.h>
114#include <sys/proc.h>
115#include <sys/sysctl.h>
116#include <sys/vmmeter.h>
117#include <sys/vnode.h>
118
119#include <vm/vm.h>
120#include <vm/pmap.h>
121#include <vm/vm_param.h>
122#include <vm/vm_kern.h>
123#include <vm/vm_object.h>
124#include <vm/vm_page.h>
125#include <vm/vm_pageout.h>
126#include <vm/vm_pager.h>
127#include <vm/vm_phys.h>
128#include <vm/vm_reserv.h>
129#include <vm/vm_extern.h>
130#include <vm/uma.h>
131#include <vm/uma_int.h>
132
133#include <machine/md_var.h>
134
135#if defined(__amd64__) || defined (__i386__)
136extern struct sysctl_oid_list sysctl__vm_pmap_children;
137#else
138SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters");
139#endif
140
141static uint64_t pmap_tryrelock_calls;
142SYSCTL_QUAD(_vm_pmap, OID_AUTO, tryrelock_calls, CTLFLAG_RD,
143    &pmap_tryrelock_calls, 0, "Number of tryrelock calls");
144
145static int pmap_tryrelock_restart;
146SYSCTL_INT(_vm_pmap, OID_AUTO, tryrelock_restart, CTLFLAG_RD,
147    &pmap_tryrelock_restart, 0, "Number of tryrelock restarts");
148
149static int pmap_tryrelock_race;
150SYSCTL_INT(_vm_pmap, OID_AUTO, tryrelock_race, CTLFLAG_RD,
151    &pmap_tryrelock_race, 0, "Number of tryrelock pmap race cases");
152
153/*
154 *	Associated with page of user-allocatable memory is a
155 *	page structure.
156 */
157
158struct vpgqueues vm_page_queues[PQ_COUNT];
159struct vpglocks vm_page_queue_lock;
160struct vpglocks vm_page_queue_free_lock;
161
162struct vpglocks	pa_lock[PA_LOCK_COUNT] __aligned(CACHE_LINE_SIZE);
163
164vm_page_t vm_page_array = 0;
165int vm_page_array_size = 0;
166long first_page = 0;
167int vm_page_zero_count = 0;
168
169static int boot_pages = UMA_BOOT_PAGES;
170TUNABLE_INT("vm.boot_pages", &boot_pages);
171SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
172	"number of pages allocated for bootstrapping the VM system");
173
174static void vm_page_clear_dirty_mask(vm_page_t m, int pagebits);
175static void vm_page_queue_remove(int queue, vm_page_t m);
176static void vm_page_enqueue(int queue, vm_page_t m);
177
178/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
179#if PAGE_SIZE == 32768
180#ifdef CTASSERT
181CTASSERT(sizeof(u_long) >= 8);
182#endif
183#endif
184
185/*
186 * Try to acquire a physical address lock while a pmap is locked.  If we
187 * fail to trylock we unlock and lock the pmap directly and cache the
188 * locked pa in *locked.  The caller should then restart their loop in case
189 * the virtual to physical mapping has changed.
190 */
191int
192vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked)
193{
194	vm_paddr_t lockpa;
195	uint32_t gen_count;
196
197	gen_count = pmap->pm_gen_count;
198	atomic_add_long((volatile long *)&pmap_tryrelock_calls, 1);
199	lockpa = *locked;
200	*locked = pa;
201	if (lockpa) {
202		PA_LOCK_ASSERT(lockpa, MA_OWNED);
203		if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa))
204			return (0);
205		PA_UNLOCK(lockpa);
206	}
207	if (PA_TRYLOCK(pa))
208		return (0);
209	PMAP_UNLOCK(pmap);
210	atomic_add_int((volatile int *)&pmap_tryrelock_restart, 1);
211	PA_LOCK(pa);
212	PMAP_LOCK(pmap);
213
214	if (pmap->pm_gen_count != gen_count + 1) {
215		pmap->pm_retries++;
216		atomic_add_int((volatile int *)&pmap_tryrelock_race, 1);
217		return (EAGAIN);
218	}
219	return (0);
220}
221
222/*
223 *	vm_set_page_size:
224 *
225 *	Sets the page size, perhaps based upon the memory
226 *	size.  Must be called before any use of page-size
227 *	dependent functions.
228 */
229void
230vm_set_page_size(void)
231{
232	if (cnt.v_page_size == 0)
233		cnt.v_page_size = PAGE_SIZE;
234	if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
235		panic("vm_set_page_size: page size not a power of two");
236}
237
238/*
239 *	vm_page_blacklist_lookup:
240 *
241 *	See if a physical address in this page has been listed
242 *	in the blacklist tunable.  Entries in the tunable are
243 *	separated by spaces or commas.  If an invalid integer is
244 *	encountered then the rest of the string is skipped.
245 */
246static int
247vm_page_blacklist_lookup(char *list, vm_paddr_t pa)
248{
249	vm_paddr_t bad;
250	char *cp, *pos;
251
252	for (pos = list; *pos != '\0'; pos = cp) {
253		bad = strtoq(pos, &cp, 0);
254		if (*cp != '\0') {
255			if (*cp == ' ' || *cp == ',') {
256				cp++;
257				if (cp == pos)
258					continue;
259			} else
260				break;
261		}
262		if (pa == trunc_page(bad))
263			return (1);
264	}
265	return (0);
266}
267
268/*
269 *	vm_page_startup:
270 *
271 *	Initializes the resident memory module.
272 *
273 *	Allocates memory for the page cells, and
274 *	for the object/offset-to-page hash table headers.
275 *	Each page cell is initialized and placed on the free list.
276 */
277vm_offset_t
278vm_page_startup(vm_offset_t vaddr)
279{
280	vm_offset_t mapped;
281	vm_paddr_t page_range;
282	vm_paddr_t new_end;
283	int i;
284	vm_paddr_t pa;
285	vm_paddr_t last_pa;
286	char *list;
287
288	/* the biggest memory array is the second group of pages */
289	vm_paddr_t end;
290	vm_paddr_t biggestsize;
291	vm_paddr_t low_water, high_water;
292	int biggestone;
293
294	biggestsize = 0;
295	biggestone = 0;
296	vaddr = round_page(vaddr);
297
298	for (i = 0; phys_avail[i + 1]; i += 2) {
299		phys_avail[i] = round_page(phys_avail[i]);
300		phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
301	}
302
303	low_water = phys_avail[0];
304	high_water = phys_avail[1];
305
306	for (i = 0; phys_avail[i + 1]; i += 2) {
307		vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
308
309		if (size > biggestsize) {
310			biggestone = i;
311			biggestsize = size;
312		}
313		if (phys_avail[i] < low_water)
314			low_water = phys_avail[i];
315		if (phys_avail[i + 1] > high_water)
316			high_water = phys_avail[i + 1];
317	}
318
319#ifdef XEN
320	low_water = 0;
321#endif
322
323	end = phys_avail[biggestone+1];
324
325	/*
326	 * Initialize the locks.
327	 */
328	mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
329	    MTX_RECURSE);
330	mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
331	    MTX_DEF);
332
333	/* Setup page locks. */
334	for (i = 0; i < PA_LOCK_COUNT; i++)
335		mtx_init(&pa_lock[i].data, "page lock", NULL, MTX_DEF);
336
337	/*
338	 * Initialize the queue headers for the hold queue, the active queue,
339	 * and the inactive queue.
340	 */
341	for (i = 0; i < PQ_COUNT; i++)
342		TAILQ_INIT(&vm_page_queues[i].pl);
343	vm_page_queues[PQ_INACTIVE].cnt = &cnt.v_inactive_count;
344	vm_page_queues[PQ_ACTIVE].cnt = &cnt.v_active_count;
345	vm_page_queues[PQ_HOLD].cnt = &cnt.v_active_count;
346
347	/*
348	 * Allocate memory for use when boot strapping the kernel memory
349	 * allocator.
350	 */
351	new_end = end - (boot_pages * UMA_SLAB_SIZE);
352	new_end = trunc_page(new_end);
353	mapped = pmap_map(&vaddr, new_end, end,
354	    VM_PROT_READ | VM_PROT_WRITE);
355	bzero((void *)mapped, end - new_end);
356	uma_startup((void *)mapped, boot_pages);
357
358#if defined(__amd64__) || defined(__i386__) || defined(__arm__) || \
359    defined(__mips__)
360	/*
361	 * Allocate a bitmap to indicate that a random physical page
362	 * needs to be included in a minidump.
363	 *
364	 * The amd64 port needs this to indicate which direct map pages
365	 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
366	 *
367	 * However, i386 still needs this workspace internally within the
368	 * minidump code.  In theory, they are not needed on i386, but are
369	 * included should the sf_buf code decide to use them.
370	 */
371	last_pa = 0;
372	for (i = 0; dump_avail[i + 1] != 0; i += 2)
373		if (dump_avail[i + 1] > last_pa)
374			last_pa = dump_avail[i + 1];
375	page_range = last_pa / PAGE_SIZE;
376	vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
377	new_end -= vm_page_dump_size;
378	vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
379	    new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
380	bzero((void *)vm_page_dump, vm_page_dump_size);
381#endif
382#ifdef __amd64__
383	/*
384	 * Request that the physical pages underlying the message buffer be
385	 * included in a crash dump.  Since the message buffer is accessed
386	 * through the direct map, they are not automatically included.
387	 */
388	pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr);
389	last_pa = pa + round_page(MSGBUF_SIZE);
390	while (pa < last_pa) {
391		dump_add_page(pa);
392		pa += PAGE_SIZE;
393	}
394#endif
395	/*
396	 * Compute the number of pages of memory that will be available for
397	 * use (taking into account the overhead of a page structure per
398	 * page).
399	 */
400	first_page = low_water / PAGE_SIZE;
401#ifdef VM_PHYSSEG_SPARSE
402	page_range = 0;
403	for (i = 0; phys_avail[i + 1] != 0; i += 2)
404		page_range += atop(phys_avail[i + 1] - phys_avail[i]);
405#elif defined(VM_PHYSSEG_DENSE)
406	page_range = high_water / PAGE_SIZE - first_page;
407#else
408#error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
409#endif
410	end = new_end;
411
412	/*
413	 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
414	 */
415	vaddr += PAGE_SIZE;
416
417	/*
418	 * Initialize the mem entry structures now, and put them in the free
419	 * queue.
420	 */
421	new_end = trunc_page(end - page_range * sizeof(struct vm_page));
422	mapped = pmap_map(&vaddr, new_end, end,
423	    VM_PROT_READ | VM_PROT_WRITE);
424	vm_page_array = (vm_page_t) mapped;
425#if VM_NRESERVLEVEL > 0
426	/*
427	 * Allocate memory for the reservation management system's data
428	 * structures.
429	 */
430	new_end = vm_reserv_startup(&vaddr, new_end, high_water);
431#endif
432#if defined(__amd64__) || defined(__mips__)
433	/*
434	 * pmap_map on amd64 and mips can come out of the direct-map, not kvm
435	 * like i386, so the pages must be tracked for a crashdump to include
436	 * this data.  This includes the vm_page_array and the early UMA
437	 * bootstrap pages.
438	 */
439	for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE)
440		dump_add_page(pa);
441#endif
442	phys_avail[biggestone + 1] = new_end;
443
444	/*
445	 * Clear all of the page structures
446	 */
447	bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
448	for (i = 0; i < page_range; i++)
449		vm_page_array[i].order = VM_NFREEORDER;
450	vm_page_array_size = page_range;
451
452	/*
453	 * Initialize the physical memory allocator.
454	 */
455	vm_phys_init();
456
457	/*
458	 * Add every available physical page that is not blacklisted to
459	 * the free lists.
460	 */
461	cnt.v_page_count = 0;
462	cnt.v_free_count = 0;
463	list = getenv("vm.blacklist");
464	for (i = 0; phys_avail[i + 1] != 0; i += 2) {
465		pa = phys_avail[i];
466		last_pa = phys_avail[i + 1];
467		while (pa < last_pa) {
468			if (list != NULL &&
469			    vm_page_blacklist_lookup(list, pa))
470				printf("Skipping page with pa 0x%jx\n",
471				    (uintmax_t)pa);
472			else
473				vm_phys_add_page(pa);
474			pa += PAGE_SIZE;
475		}
476	}
477	freeenv(list);
478#if VM_NRESERVLEVEL > 0
479	/*
480	 * Initialize the reservation management system.
481	 */
482	vm_reserv_init();
483#endif
484	return (vaddr);
485}
486
487void
488vm_page_flag_set(vm_page_t m, unsigned short bits)
489{
490
491	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
492	/*
493	 * The PG_WRITEABLE flag can only be set if the page is managed and
494	 * VPO_BUSY.  Currently, this flag is only set by pmap_enter().
495	 */
496	KASSERT((bits & PG_WRITEABLE) == 0 ||
497	    ((m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) == 0 &&
498	    (m->oflags & VPO_BUSY) != 0), ("PG_WRITEABLE and !VPO_BUSY"));
499	m->flags |= bits;
500}
501
502void
503vm_page_flag_clear(vm_page_t m, unsigned short bits)
504{
505
506	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
507	/*
508	 * The PG_REFERENCED flag can only be cleared if the object
509	 * containing the page is locked.
510	 */
511	KASSERT((bits & PG_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object),
512	    ("PG_REFERENCED and !VM_OBJECT_LOCKED"));
513	m->flags &= ~bits;
514}
515
516void
517vm_page_busy(vm_page_t m)
518{
519
520	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
521	KASSERT((m->oflags & VPO_BUSY) == 0,
522	    ("vm_page_busy: page already busy!!!"));
523	m->oflags |= VPO_BUSY;
524}
525
526/*
527 *      vm_page_flash:
528 *
529 *      wakeup anyone waiting for the page.
530 */
531void
532vm_page_flash(vm_page_t m)
533{
534
535	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
536	if (m->oflags & VPO_WANTED) {
537		m->oflags &= ~VPO_WANTED;
538		wakeup(m);
539	}
540}
541
542/*
543 *      vm_page_wakeup:
544 *
545 *      clear the VPO_BUSY flag and wakeup anyone waiting for the
546 *      page.
547 *
548 */
549void
550vm_page_wakeup(vm_page_t m)
551{
552
553	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
554	KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!"));
555	m->oflags &= ~VPO_BUSY;
556	vm_page_flash(m);
557}
558
559void
560vm_page_io_start(vm_page_t m)
561{
562
563	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
564	m->busy++;
565}
566
567void
568vm_page_io_finish(vm_page_t m)
569{
570
571	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
572	m->busy--;
573	if (m->busy == 0)
574		vm_page_flash(m);
575}
576
577/*
578 * Keep page from being freed by the page daemon
579 * much of the same effect as wiring, except much lower
580 * overhead and should be used only for *very* temporary
581 * holding ("wiring").
582 */
583void
584vm_page_hold(vm_page_t mem)
585{
586
587	vm_page_lock_assert(mem, MA_OWNED);
588        mem->hold_count++;
589}
590
591void
592vm_page_unhold(vm_page_t mem)
593{
594
595	vm_page_lock_assert(mem, MA_OWNED);
596	--mem->hold_count;
597	KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
598	if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
599		vm_page_free_toq(mem);
600}
601
602/*
603 *	vm_page_unhold_pages:
604 *
605 *	Unhold each of the pages that is referenced by the given array.
606 */
607void
608vm_page_unhold_pages(vm_page_t *ma, int count)
609{
610	struct mtx *mtx, *new_mtx;
611
612	mtx = NULL;
613	for (; count != 0; count--) {
614		/*
615		 * Avoid releasing and reacquiring the same page lock.
616		 */
617		new_mtx = vm_page_lockptr(*ma);
618		if (mtx != new_mtx) {
619			if (mtx != NULL)
620				mtx_unlock(mtx);
621			mtx = new_mtx;
622			mtx_lock(mtx);
623		}
624		vm_page_unhold(*ma);
625		ma++;
626	}
627	if (mtx != NULL)
628		mtx_unlock(mtx);
629}
630
631/*
632 *	vm_page_free:
633 *
634 *	Free a page.
635 */
636void
637vm_page_free(vm_page_t m)
638{
639
640	m->flags &= ~PG_ZERO;
641	vm_page_free_toq(m);
642}
643
644/*
645 *	vm_page_free_zero:
646 *
647 *	Free a page to the zerod-pages queue
648 */
649void
650vm_page_free_zero(vm_page_t m)
651{
652
653	m->flags |= PG_ZERO;
654	vm_page_free_toq(m);
655}
656
657/*
658 *	vm_page_sleep:
659 *
660 *	Sleep and release the page and page queues locks.
661 *
662 *	The object containing the given page must be locked.
663 */
664void
665vm_page_sleep(vm_page_t m, const char *msg)
666{
667
668	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
669	if (mtx_owned(&vm_page_queue_mtx))
670		vm_page_unlock_queues();
671	if (mtx_owned(vm_page_lockptr(m)))
672		vm_page_unlock(m);
673
674	/*
675	 * It's possible that while we sleep, the page will get
676	 * unbusied and freed.  If we are holding the object
677	 * lock, we will assume we hold a reference to the object
678	 * such that even if m->object changes, we can re-lock
679	 * it.
680	 */
681	m->oflags |= VPO_WANTED;
682	msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0);
683}
684
685/*
686 *	vm_page_dirty:
687 *
688 *	make page all dirty
689 */
690void
691vm_page_dirty(vm_page_t m)
692{
693
694	KASSERT((m->flags & PG_CACHED) == 0,
695	    ("vm_page_dirty: page in cache!"));
696	KASSERT(!VM_PAGE_IS_FREE(m),
697	    ("vm_page_dirty: page is free!"));
698	KASSERT(m->valid == VM_PAGE_BITS_ALL,
699	    ("vm_page_dirty: page is invalid!"));
700	m->dirty = VM_PAGE_BITS_ALL;
701}
702
703/*
704 *	vm_page_splay:
705 *
706 *	Implements Sleator and Tarjan's top-down splay algorithm.  Returns
707 *	the vm_page containing the given pindex.  If, however, that
708 *	pindex is not found in the vm_object, returns a vm_page that is
709 *	adjacent to the pindex, coming before or after it.
710 */
711vm_page_t
712vm_page_splay(vm_pindex_t pindex, vm_page_t root)
713{
714	struct vm_page dummy;
715	vm_page_t lefttreemax, righttreemin, y;
716
717	if (root == NULL)
718		return (root);
719	lefttreemax = righttreemin = &dummy;
720	for (;; root = y) {
721		if (pindex < root->pindex) {
722			if ((y = root->left) == NULL)
723				break;
724			if (pindex < y->pindex) {
725				/* Rotate right. */
726				root->left = y->right;
727				y->right = root;
728				root = y;
729				if ((y = root->left) == NULL)
730					break;
731			}
732			/* Link into the new root's right tree. */
733			righttreemin->left = root;
734			righttreemin = root;
735		} else if (pindex > root->pindex) {
736			if ((y = root->right) == NULL)
737				break;
738			if (pindex > y->pindex) {
739				/* Rotate left. */
740				root->right = y->left;
741				y->left = root;
742				root = y;
743				if ((y = root->right) == NULL)
744					break;
745			}
746			/* Link into the new root's left tree. */
747			lefttreemax->right = root;
748			lefttreemax = root;
749		} else
750			break;
751	}
752	/* Assemble the new root. */
753	lefttreemax->right = root->left;
754	righttreemin->left = root->right;
755	root->left = dummy.right;
756	root->right = dummy.left;
757	return (root);
758}
759
760/*
761 *	vm_page_insert:		[ internal use only ]
762 *
763 *	Inserts the given mem entry into the object and object list.
764 *
765 *	The pagetables are not updated but will presumably fault the page
766 *	in if necessary, or if a kernel page the caller will at some point
767 *	enter the page into the kernel's pmap.  We are not allowed to block
768 *	here so we *can't* do this anyway.
769 *
770 *	The object and page must be locked.
771 *	This routine may not block.
772 */
773void
774vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
775{
776	vm_page_t root;
777
778	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
779	if (m->object != NULL)
780		panic("vm_page_insert: page already inserted");
781
782	/*
783	 * Record the object/offset pair in this page
784	 */
785	m->object = object;
786	m->pindex = pindex;
787
788	/*
789	 * Now link into the object's ordered list of backed pages.
790	 */
791	root = object->root;
792	if (root == NULL) {
793		m->left = NULL;
794		m->right = NULL;
795		TAILQ_INSERT_TAIL(&object->memq, m, listq);
796	} else {
797		root = vm_page_splay(pindex, root);
798		if (pindex < root->pindex) {
799			m->left = root->left;
800			m->right = root;
801			root->left = NULL;
802			TAILQ_INSERT_BEFORE(root, m, listq);
803		} else if (pindex == root->pindex)
804			panic("vm_page_insert: offset already allocated");
805		else {
806			m->right = root->right;
807			m->left = root;
808			root->right = NULL;
809			TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
810		}
811	}
812	object->root = m;
813
814	/*
815	 * show that the object has one more resident page.
816	 */
817	object->resident_page_count++;
818	/*
819	 * Hold the vnode until the last page is released.
820	 */
821	if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
822		vhold((struct vnode *)object->handle);
823
824	/*
825	 * Since we are inserting a new and possibly dirty page,
826	 * update the object's OBJ_MIGHTBEDIRTY flag.
827	 */
828	if (m->flags & PG_WRITEABLE)
829		vm_object_set_writeable_dirty(object);
830}
831
832/*
833 *	vm_page_remove:
834 *				NOTE: used by device pager as well -wfj
835 *
836 *	Removes the given mem entry from the object/offset-page
837 *	table and the object page list, but do not invalidate/terminate
838 *	the backing store.
839 *
840 *	The object and page must be locked.
841 *	The underlying pmap entry (if any) is NOT removed here.
842 *	This routine may not block.
843 */
844void
845vm_page_remove(vm_page_t m)
846{
847	vm_object_t object;
848	vm_page_t root;
849
850	if ((m->flags & PG_UNMANAGED) == 0)
851		vm_page_lock_assert(m, MA_OWNED);
852	if ((object = m->object) == NULL)
853		return;
854	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
855	if (m->oflags & VPO_BUSY) {
856		m->oflags &= ~VPO_BUSY;
857		vm_page_flash(m);
858	}
859
860	/*
861	 * Now remove from the object's list of backed pages.
862	 */
863	if (m != object->root)
864		vm_page_splay(m->pindex, object->root);
865	if (m->left == NULL)
866		root = m->right;
867	else {
868		root = vm_page_splay(m->pindex, m->left);
869		root->right = m->right;
870	}
871	object->root = root;
872	TAILQ_REMOVE(&object->memq, m, listq);
873
874	/*
875	 * And show that the object has one fewer resident page.
876	 */
877	object->resident_page_count--;
878	/*
879	 * The vnode may now be recycled.
880	 */
881	if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
882		vdrop((struct vnode *)object->handle);
883
884	m->object = NULL;
885}
886
887/*
888 *	vm_page_lookup:
889 *
890 *	Returns the page associated with the object/offset
891 *	pair specified; if none is found, NULL is returned.
892 *
893 *	The object must be locked.
894 *	This routine may not block.
895 *	This is a critical path routine
896 */
897vm_page_t
898vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
899{
900	vm_page_t m;
901
902	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
903	if ((m = object->root) != NULL && m->pindex != pindex) {
904		m = vm_page_splay(pindex, m);
905		if ((object->root = m)->pindex != pindex)
906			m = NULL;
907	}
908	return (m);
909}
910
911/*
912 *	vm_page_find_least:
913 *
914 *	Returns the page associated with the object with least pindex
915 *	greater than or equal to the parameter pindex, or NULL.
916 *
917 *	The object must be locked.
918 *	The routine may not block.
919 */
920vm_page_t
921vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
922{
923	vm_page_t m;
924
925	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
926	if ((m = TAILQ_FIRST(&object->memq)) != NULL) {
927		if (m->pindex < pindex) {
928			m = vm_page_splay(pindex, object->root);
929			if ((object->root = m)->pindex < pindex)
930				m = TAILQ_NEXT(m, listq);
931		}
932	}
933	return (m);
934}
935
936/*
937 * Returns the given page's successor (by pindex) within the object if it is
938 * resident; if none is found, NULL is returned.
939 *
940 * The object must be locked.
941 */
942vm_page_t
943vm_page_next(vm_page_t m)
944{
945	vm_page_t next;
946
947	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
948	if ((next = TAILQ_NEXT(m, listq)) != NULL &&
949	    next->pindex != m->pindex + 1)
950		next = NULL;
951	return (next);
952}
953
954/*
955 * Returns the given page's predecessor (by pindex) within the object if it is
956 * resident; if none is found, NULL is returned.
957 *
958 * The object must be locked.
959 */
960vm_page_t
961vm_page_prev(vm_page_t m)
962{
963	vm_page_t prev;
964
965	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
966	if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
967	    prev->pindex != m->pindex - 1)
968		prev = NULL;
969	return (prev);
970}
971
972/*
973 *	vm_page_rename:
974 *
975 *	Move the given memory entry from its
976 *	current object to the specified target object/offset.
977 *
978 *	The object must be locked.
979 *	This routine may not block.
980 *
981 *	Note: swap associated with the page must be invalidated by the move.  We
982 *	      have to do this for several reasons:  (1) we aren't freeing the
983 *	      page, (2) we are dirtying the page, (3) the VM system is probably
984 *	      moving the page from object A to B, and will then later move
985 *	      the backing store from A to B and we can't have a conflict.
986 *
987 *	Note: we *always* dirty the page.  It is necessary both for the
988 *	      fact that we moved it, and because we may be invalidating
989 *	      swap.  If the page is on the cache, we have to deactivate it
990 *	      or vm_page_dirty() will panic.  Dirty pages are not allowed
991 *	      on the cache.
992 */
993void
994vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
995{
996
997	vm_page_remove(m);
998	vm_page_insert(m, new_object, new_pindex);
999	vm_page_dirty(m);
1000}
1001
1002/*
1003 *	Convert all of the given object's cached pages that have a
1004 *	pindex within the given range into free pages.  If the value
1005 *	zero is given for "end", then the range's upper bound is
1006 *	infinity.  If the given object is backed by a vnode and it
1007 *	transitions from having one or more cached pages to none, the
1008 *	vnode's hold count is reduced.
1009 */
1010void
1011vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1012{
1013	vm_page_t m, m_next;
1014	boolean_t empty;
1015
1016	mtx_lock(&vm_page_queue_free_mtx);
1017	if (__predict_false(object->cache == NULL)) {
1018		mtx_unlock(&vm_page_queue_free_mtx);
1019		return;
1020	}
1021	m = object->cache = vm_page_splay(start, object->cache);
1022	if (m->pindex < start) {
1023		if (m->right == NULL)
1024			m = NULL;
1025		else {
1026			m_next = vm_page_splay(start, m->right);
1027			m_next->left = m;
1028			m->right = NULL;
1029			m = object->cache = m_next;
1030		}
1031	}
1032
1033	/*
1034	 * At this point, "m" is either (1) a reference to the page
1035	 * with the least pindex that is greater than or equal to
1036	 * "start" or (2) NULL.
1037	 */
1038	for (; m != NULL && (m->pindex < end || end == 0); m = m_next) {
1039		/*
1040		 * Find "m"'s successor and remove "m" from the
1041		 * object's cache.
1042		 */
1043		if (m->right == NULL) {
1044			object->cache = m->left;
1045			m_next = NULL;
1046		} else {
1047			m_next = vm_page_splay(start, m->right);
1048			m_next->left = m->left;
1049			object->cache = m_next;
1050		}
1051		/* Convert "m" to a free page. */
1052		m->object = NULL;
1053		m->valid = 0;
1054		/* Clear PG_CACHED and set PG_FREE. */
1055		m->flags ^= PG_CACHED | PG_FREE;
1056		KASSERT((m->flags & (PG_CACHED | PG_FREE)) == PG_FREE,
1057		    ("vm_page_cache_free: page %p has inconsistent flags", m));
1058		cnt.v_cache_count--;
1059		cnt.v_free_count++;
1060	}
1061	empty = object->cache == NULL;
1062	mtx_unlock(&vm_page_queue_free_mtx);
1063	if (object->type == OBJT_VNODE && empty)
1064		vdrop(object->handle);
1065}
1066
1067/*
1068 *	Returns the cached page that is associated with the given
1069 *	object and offset.  If, however, none exists, returns NULL.
1070 *
1071 *	The free page queue must be locked.
1072 */
1073static inline vm_page_t
1074vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex)
1075{
1076	vm_page_t m;
1077
1078	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1079	if ((m = object->cache) != NULL && m->pindex != pindex) {
1080		m = vm_page_splay(pindex, m);
1081		if ((object->cache = m)->pindex != pindex)
1082			m = NULL;
1083	}
1084	return (m);
1085}
1086
1087/*
1088 *	Remove the given cached page from its containing object's
1089 *	collection of cached pages.
1090 *
1091 *	The free page queue must be locked.
1092 */
1093void
1094vm_page_cache_remove(vm_page_t m)
1095{
1096	vm_object_t object;
1097	vm_page_t root;
1098
1099	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1100	KASSERT((m->flags & PG_CACHED) != 0,
1101	    ("vm_page_cache_remove: page %p is not cached", m));
1102	object = m->object;
1103	if (m != object->cache) {
1104		root = vm_page_splay(m->pindex, object->cache);
1105		KASSERT(root == m,
1106		    ("vm_page_cache_remove: page %p is not cached in object %p",
1107		    m, object));
1108	}
1109	if (m->left == NULL)
1110		root = m->right;
1111	else if (m->right == NULL)
1112		root = m->left;
1113	else {
1114		root = vm_page_splay(m->pindex, m->left);
1115		root->right = m->right;
1116	}
1117	object->cache = root;
1118	m->object = NULL;
1119	cnt.v_cache_count--;
1120}
1121
1122/*
1123 *	Transfer all of the cached pages with offset greater than or
1124 *	equal to 'offidxstart' from the original object's cache to the
1125 *	new object's cache.  However, any cached pages with offset
1126 *	greater than or equal to the new object's size are kept in the
1127 *	original object.  Initially, the new object's cache must be
1128 *	empty.  Offset 'offidxstart' in the original object must
1129 *	correspond to offset zero in the new object.
1130 *
1131 *	The new object must be locked.
1132 */
1133void
1134vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
1135    vm_object_t new_object)
1136{
1137	vm_page_t m, m_next;
1138
1139	/*
1140	 * Insertion into an object's collection of cached pages
1141	 * requires the object to be locked.  In contrast, removal does
1142	 * not.
1143	 */
1144	VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED);
1145	KASSERT(new_object->cache == NULL,
1146	    ("vm_page_cache_transfer: object %p has cached pages",
1147	    new_object));
1148	mtx_lock(&vm_page_queue_free_mtx);
1149	if ((m = orig_object->cache) != NULL) {
1150		/*
1151		 * Transfer all of the pages with offset greater than or
1152		 * equal to 'offidxstart' from the original object's
1153		 * cache to the new object's cache.
1154		 */
1155		m = vm_page_splay(offidxstart, m);
1156		if (m->pindex < offidxstart) {
1157			orig_object->cache = m;
1158			new_object->cache = m->right;
1159			m->right = NULL;
1160		} else {
1161			orig_object->cache = m->left;
1162			new_object->cache = m;
1163			m->left = NULL;
1164		}
1165		while ((m = new_object->cache) != NULL) {
1166			if ((m->pindex - offidxstart) >= new_object->size) {
1167				/*
1168				 * Return all of the cached pages with
1169				 * offset greater than or equal to the
1170				 * new object's size to the original
1171				 * object's cache.
1172				 */
1173				new_object->cache = m->left;
1174				m->left = orig_object->cache;
1175				orig_object->cache = m;
1176				break;
1177			}
1178			m_next = vm_page_splay(m->pindex, m->right);
1179			/* Update the page's object and offset. */
1180			m->object = new_object;
1181			m->pindex -= offidxstart;
1182			if (m_next == NULL)
1183				break;
1184			m->right = NULL;
1185			m_next->left = m;
1186			new_object->cache = m_next;
1187		}
1188		KASSERT(new_object->cache == NULL ||
1189		    new_object->type == OBJT_SWAP,
1190		    ("vm_page_cache_transfer: object %p's type is incompatible"
1191		    " with cached pages", new_object));
1192	}
1193	mtx_unlock(&vm_page_queue_free_mtx);
1194}
1195
1196/*
1197 *	vm_page_alloc:
1198 *
1199 *	Allocate and return a memory cell associated
1200 *	with this VM object/offset pair.
1201 *
1202 *	The caller must always specify an allocation class.
1203 *
1204 *	allocation classes:
1205 *	VM_ALLOC_NORMAL		normal process request
1206 *	VM_ALLOC_SYSTEM		system *really* needs a page
1207 *	VM_ALLOC_INTERRUPT	interrupt time request
1208 *
1209 *	optional allocation flags:
1210 *	VM_ALLOC_ZERO		prefer a zeroed page
1211 *	VM_ALLOC_WIRED		wire the allocated page
1212 *	VM_ALLOC_NOOBJ		page is not associated with a vm object
1213 *	VM_ALLOC_NOBUSY		do not set the page busy
1214 *	VM_ALLOC_IFCACHED	return page only if it is cached
1215 *	VM_ALLOC_IFNOTCACHED	return NULL, do not reactivate if the page
1216 *				is cached
1217 *
1218 *	This routine may not sleep.
1219 */
1220vm_page_t
1221vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
1222{
1223	struct vnode *vp = NULL;
1224	vm_object_t m_object;
1225	vm_page_t m;
1226	int flags, page_req;
1227
1228	page_req = req & VM_ALLOC_CLASS_MASK;
1229	KASSERT(curthread->td_intr_nesting_level == 0 ||
1230	    page_req == VM_ALLOC_INTERRUPT,
1231	    ("vm_page_alloc(NORMAL|SYSTEM) in interrupt context"));
1232
1233	if ((req & VM_ALLOC_NOOBJ) == 0) {
1234		KASSERT(object != NULL,
1235		    ("vm_page_alloc: NULL object."));
1236		VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1237	}
1238
1239	/*
1240	 * The pager is allowed to eat deeper into the free page list.
1241	 */
1242	if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) {
1243		page_req = VM_ALLOC_SYSTEM;
1244	};
1245
1246	mtx_lock(&vm_page_queue_free_mtx);
1247	if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1248	    (page_req == VM_ALLOC_SYSTEM &&
1249	    cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1250	    (page_req == VM_ALLOC_INTERRUPT &&
1251	    cnt.v_free_count + cnt.v_cache_count > 0)) {
1252		/*
1253		 * Allocate from the free queue if the number of free pages
1254		 * exceeds the minimum for the request class.
1255		 */
1256		if (object != NULL &&
1257		    (m = vm_page_cache_lookup(object, pindex)) != NULL) {
1258			if ((req & VM_ALLOC_IFNOTCACHED) != 0) {
1259				mtx_unlock(&vm_page_queue_free_mtx);
1260				return (NULL);
1261			}
1262			if (vm_phys_unfree_page(m))
1263				vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, 0);
1264#if VM_NRESERVLEVEL > 0
1265			else if (!vm_reserv_reactivate_page(m))
1266#else
1267			else
1268#endif
1269				panic("vm_page_alloc: cache page %p is missing"
1270				    " from the free queue", m);
1271		} else if ((req & VM_ALLOC_IFCACHED) != 0) {
1272			mtx_unlock(&vm_page_queue_free_mtx);
1273			return (NULL);
1274#if VM_NRESERVLEVEL > 0
1275		} else if (object == NULL || object->type == OBJT_DEVICE ||
1276		    object->type == OBJT_SG ||
1277		    (object->flags & OBJ_COLORED) == 0 ||
1278		    (m = vm_reserv_alloc_page(object, pindex)) == NULL) {
1279#else
1280		} else {
1281#endif
1282			m = vm_phys_alloc_pages(object != NULL ?
1283			    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
1284#if VM_NRESERVLEVEL > 0
1285			if (m == NULL && vm_reserv_reclaim_inactive()) {
1286				m = vm_phys_alloc_pages(object != NULL ?
1287				    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
1288				    0);
1289			}
1290#endif
1291		}
1292	} else {
1293		/*
1294		 * Not allocatable, give up.
1295		 */
1296		mtx_unlock(&vm_page_queue_free_mtx);
1297		atomic_add_int(&vm_pageout_deficit,
1298		    MAX((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
1299		pagedaemon_wakeup();
1300		return (NULL);
1301	}
1302
1303	/*
1304	 *  At this point we had better have found a good page.
1305	 */
1306
1307	KASSERT(m != NULL, ("vm_page_alloc: missing page"));
1308	KASSERT(m->queue == PQ_NONE,
1309	    ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue));
1310	KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m));
1311	KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m));
1312	KASSERT(m->busy == 0, ("vm_page_alloc: page %p is busy", m));
1313	KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m));
1314	KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1315	    ("vm_page_alloc: page %p has unexpected memattr %d", m,
1316	    pmap_page_get_memattr(m)));
1317	if ((m->flags & PG_CACHED) != 0) {
1318		KASSERT(m->valid != 0,
1319		    ("vm_page_alloc: cached page %p is invalid", m));
1320		if (m->object == object && m->pindex == pindex)
1321	  		cnt.v_reactivated++;
1322		else
1323			m->valid = 0;
1324		m_object = m->object;
1325		vm_page_cache_remove(m);
1326		if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1327			vp = m_object->handle;
1328	} else {
1329		KASSERT(VM_PAGE_IS_FREE(m),
1330		    ("vm_page_alloc: page %p is not free", m));
1331		KASSERT(m->valid == 0,
1332		    ("vm_page_alloc: free page %p is valid", m));
1333		cnt.v_free_count--;
1334	}
1335
1336	/*
1337	 * Initialize structure.  Only the PG_ZERO flag is inherited.
1338	 */
1339	flags = 0;
1340	if (m->flags & PG_ZERO) {
1341		vm_page_zero_count--;
1342		if (req & VM_ALLOC_ZERO)
1343			flags = PG_ZERO;
1344	}
1345	if (object == NULL || object->type == OBJT_PHYS)
1346		flags |= PG_UNMANAGED;
1347	m->flags = flags;
1348	if (req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ))
1349		m->oflags = 0;
1350	else
1351		m->oflags = VPO_BUSY;
1352	if (req & VM_ALLOC_WIRED) {
1353		atomic_add_int(&cnt.v_wire_count, 1);
1354		m->wire_count = 1;
1355	}
1356	m->act_count = 0;
1357	mtx_unlock(&vm_page_queue_free_mtx);
1358
1359	if (object != NULL) {
1360		/* Ignore device objects; the pager sets "memattr" for them. */
1361		if (object->memattr != VM_MEMATTR_DEFAULT &&
1362		    object->type != OBJT_DEVICE && object->type != OBJT_SG)
1363			pmap_page_set_memattr(m, object->memattr);
1364		vm_page_insert(m, object, pindex);
1365	} else
1366		m->pindex = pindex;
1367
1368	/*
1369	 * The following call to vdrop() must come after the above call
1370	 * to vm_page_insert() in case both affect the same object and
1371	 * vnode.  Otherwise, the affected vnode's hold count could
1372	 * temporarily become zero.
1373	 */
1374	if (vp != NULL)
1375		vdrop(vp);
1376
1377	/*
1378	 * Don't wakeup too often - wakeup the pageout daemon when
1379	 * we would be nearly out of memory.
1380	 */
1381	if (vm_paging_needed())
1382		pagedaemon_wakeup();
1383
1384	return (m);
1385}
1386
1387/*
1388 * Initialize a page that has been freshly dequeued from a freelist.
1389 * The caller has to drop the vnode returned, if it is not NULL.
1390 *
1391 * To be called with vm_page_queue_free_mtx held.
1392 */
1393struct vnode *
1394vm_page_alloc_init(vm_page_t m)
1395{
1396	struct vnode *drop;
1397	vm_object_t m_object;
1398
1399	KASSERT(m->queue == PQ_NONE,
1400	    ("vm_page_alloc_init: page %p has unexpected queue %d",
1401	    m, m->queue));
1402	KASSERT(m->wire_count == 0,
1403	    ("vm_page_alloc_init: page %p is wired", m));
1404	KASSERT(m->hold_count == 0,
1405	    ("vm_page_alloc_init: page %p is held", m));
1406	KASSERT(m->busy == 0,
1407	    ("vm_page_alloc_init: page %p is busy", m));
1408	KASSERT(m->dirty == 0,
1409	    ("vm_page_alloc_init: page %p is dirty", m));
1410	KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1411	    ("vm_page_alloc_init: page %p has unexpected memattr %d",
1412	    m, pmap_page_get_memattr(m)));
1413	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1414	drop = NULL;
1415	if ((m->flags & PG_CACHED) != 0) {
1416		m->valid = 0;
1417		m_object = m->object;
1418		vm_page_cache_remove(m);
1419		if (m_object->type == OBJT_VNODE &&
1420		    m_object->cache == NULL)
1421			drop = m_object->handle;
1422	} else {
1423		KASSERT(VM_PAGE_IS_FREE(m),
1424		    ("vm_page_alloc_init: page %p is not free", m));
1425		KASSERT(m->valid == 0,
1426		    ("vm_page_alloc_init: free page %p is valid", m));
1427		cnt.v_free_count--;
1428	}
1429	if (m->flags & PG_ZERO)
1430		vm_page_zero_count--;
1431	/* Don't clear the PG_ZERO flag; we'll need it later. */
1432	m->flags = PG_UNMANAGED | (m->flags & PG_ZERO);
1433	m->oflags = 0;
1434	/* Unmanaged pages don't use "act_count". */
1435	return (drop);
1436}
1437
1438/*
1439 * 	vm_page_alloc_freelist:
1440 *
1441 *	Allocate a page from the specified freelist.
1442 *	Only the ALLOC_CLASS values in req are honored, other request flags
1443 *	are ignored.
1444 */
1445vm_page_t
1446vm_page_alloc_freelist(int flind, int req)
1447{
1448	struct vnode *drop;
1449	vm_page_t m;
1450	int page_req;
1451
1452	m = NULL;
1453	page_req = req & VM_ALLOC_CLASS_MASK;
1454	mtx_lock(&vm_page_queue_free_mtx);
1455	/*
1456	 * Do not allocate reserved pages unless the req has asked for it.
1457	 */
1458	if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1459	    (page_req == VM_ALLOC_SYSTEM &&
1460	    cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1461	    (page_req == VM_ALLOC_INTERRUPT &&
1462	    cnt.v_free_count + cnt.v_cache_count > 0)) {
1463		m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0);
1464	}
1465	if (m == NULL) {
1466		mtx_unlock(&vm_page_queue_free_mtx);
1467		return (NULL);
1468	}
1469	drop = vm_page_alloc_init(m);
1470	mtx_unlock(&vm_page_queue_free_mtx);
1471	if (drop)
1472		vdrop(drop);
1473	return (m);
1474}
1475
1476/*
1477 *	vm_wait:	(also see VM_WAIT macro)
1478 *
1479 *	Block until free pages are available for allocation
1480 *	- Called in various places before memory allocations.
1481 */
1482void
1483vm_wait(void)
1484{
1485
1486	mtx_lock(&vm_page_queue_free_mtx);
1487	if (curproc == pageproc) {
1488		vm_pageout_pages_needed = 1;
1489		msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
1490		    PDROP | PSWP, "VMWait", 0);
1491	} else {
1492		if (!vm_pages_needed) {
1493			vm_pages_needed = 1;
1494			wakeup(&vm_pages_needed);
1495		}
1496		msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
1497		    "vmwait", 0);
1498	}
1499}
1500
1501/*
1502 *	vm_waitpfault:	(also see VM_WAITPFAULT macro)
1503 *
1504 *	Block until free pages are available for allocation
1505 *	- Called only in vm_fault so that processes page faulting
1506 *	  can be easily tracked.
1507 *	- Sleeps at a lower priority than vm_wait() so that vm_wait()ing
1508 *	  processes will be able to grab memory first.  Do not change
1509 *	  this balance without careful testing first.
1510 */
1511void
1512vm_waitpfault(void)
1513{
1514
1515	mtx_lock(&vm_page_queue_free_mtx);
1516	if (!vm_pages_needed) {
1517		vm_pages_needed = 1;
1518		wakeup(&vm_pages_needed);
1519	}
1520	msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
1521	    "pfault", 0);
1522}
1523
1524/*
1525 *	vm_page_requeue:
1526 *
1527 *	Move the given page to the tail of its present page queue.
1528 *
1529 *	The page queues must be locked.
1530 */
1531void
1532vm_page_requeue(vm_page_t m)
1533{
1534	struct vpgqueues *vpq;
1535	int queue;
1536
1537	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1538	queue = m->queue;
1539	KASSERT(queue != PQ_NONE,
1540	    ("vm_page_requeue: page %p is not queued", m));
1541	vpq = &vm_page_queues[queue];
1542	TAILQ_REMOVE(&vpq->pl, m, pageq);
1543	TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1544}
1545
1546/*
1547 *	vm_page_queue_remove:
1548 *
1549 *	Remove the given page from the specified queue.
1550 *
1551 *	The page and page queues must be locked.
1552 */
1553static __inline void
1554vm_page_queue_remove(int queue, vm_page_t m)
1555{
1556	struct vpgqueues *pq;
1557
1558	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1559	vm_page_lock_assert(m, MA_OWNED);
1560	pq = &vm_page_queues[queue];
1561	TAILQ_REMOVE(&pq->pl, m, pageq);
1562	(*pq->cnt)--;
1563}
1564
1565/*
1566 *	vm_pageq_remove:
1567 *
1568 *	Remove a page from its queue.
1569 *
1570 *	The given page must be locked.
1571 *	This routine may not block.
1572 */
1573void
1574vm_pageq_remove(vm_page_t m)
1575{
1576	int queue;
1577
1578	vm_page_lock_assert(m, MA_OWNED);
1579	if ((queue = m->queue) != PQ_NONE) {
1580		vm_page_lock_queues();
1581		m->queue = PQ_NONE;
1582		vm_page_queue_remove(queue, m);
1583		vm_page_unlock_queues();
1584	}
1585}
1586
1587/*
1588 *	vm_page_enqueue:
1589 *
1590 *	Add the given page to the specified queue.
1591 *
1592 *	The page queues must be locked.
1593 */
1594static void
1595vm_page_enqueue(int queue, vm_page_t m)
1596{
1597	struct vpgqueues *vpq;
1598
1599	vpq = &vm_page_queues[queue];
1600	m->queue = queue;
1601	TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1602	++*vpq->cnt;
1603}
1604
1605/*
1606 *	vm_page_activate:
1607 *
1608 *	Put the specified page on the active list (if appropriate).
1609 *	Ensure that act_count is at least ACT_INIT but do not otherwise
1610 *	mess with it.
1611 *
1612 *	The page must be locked.
1613 *	This routine may not block.
1614 */
1615void
1616vm_page_activate(vm_page_t m)
1617{
1618	int queue;
1619
1620	vm_page_lock_assert(m, MA_OWNED);
1621	if ((queue = m->queue) != PQ_ACTIVE) {
1622		if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1623			if (m->act_count < ACT_INIT)
1624				m->act_count = ACT_INIT;
1625			vm_page_lock_queues();
1626			if (queue != PQ_NONE)
1627				vm_page_queue_remove(queue, m);
1628			vm_page_enqueue(PQ_ACTIVE, m);
1629			vm_page_unlock_queues();
1630		} else
1631			KASSERT(queue == PQ_NONE,
1632			    ("vm_page_activate: wired page %p is queued", m));
1633	} else {
1634		if (m->act_count < ACT_INIT)
1635			m->act_count = ACT_INIT;
1636	}
1637}
1638
1639/*
1640 *	vm_page_free_wakeup:
1641 *
1642 *	Helper routine for vm_page_free_toq() and vm_page_cache().  This
1643 *	routine is called when a page has been added to the cache or free
1644 *	queues.
1645 *
1646 *	The page queues must be locked.
1647 *	This routine may not block.
1648 */
1649static inline void
1650vm_page_free_wakeup(void)
1651{
1652
1653	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1654	/*
1655	 * if pageout daemon needs pages, then tell it that there are
1656	 * some free.
1657	 */
1658	if (vm_pageout_pages_needed &&
1659	    cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1660		wakeup(&vm_pageout_pages_needed);
1661		vm_pageout_pages_needed = 0;
1662	}
1663	/*
1664	 * wakeup processes that are waiting on memory if we hit a
1665	 * high water mark. And wakeup scheduler process if we have
1666	 * lots of memory. this process will swapin processes.
1667	 */
1668	if (vm_pages_needed && !vm_page_count_min()) {
1669		vm_pages_needed = 0;
1670		wakeup(&cnt.v_free_count);
1671	}
1672}
1673
1674/*
1675 *	vm_page_free_toq:
1676 *
1677 *	Returns the given page to the free list,
1678 *	disassociating it with any VM object.
1679 *
1680 *	Object and page must be locked prior to entry.
1681 *	This routine may not block.
1682 */
1683
1684void
1685vm_page_free_toq(vm_page_t m)
1686{
1687
1688	if ((m->flags & PG_UNMANAGED) == 0) {
1689		vm_page_lock_assert(m, MA_OWNED);
1690		KASSERT(!pmap_page_is_mapped(m),
1691		    ("vm_page_free_toq: freeing mapped page %p", m));
1692	}
1693	PCPU_INC(cnt.v_tfree);
1694
1695	if (VM_PAGE_IS_FREE(m))
1696		panic("vm_page_free: freeing free page %p", m);
1697	else if (m->busy != 0)
1698		panic("vm_page_free: freeing busy page %p", m);
1699
1700	/*
1701	 * unqueue, then remove page.  Note that we cannot destroy
1702	 * the page here because we do not want to call the pager's
1703	 * callback routine until after we've put the page on the
1704	 * appropriate free queue.
1705	 */
1706	if ((m->flags & PG_UNMANAGED) == 0)
1707		vm_pageq_remove(m);
1708	vm_page_remove(m);
1709
1710	/*
1711	 * If fictitious remove object association and
1712	 * return, otherwise delay object association removal.
1713	 */
1714	if ((m->flags & PG_FICTITIOUS) != 0) {
1715		return;
1716	}
1717
1718	m->valid = 0;
1719	vm_page_undirty(m);
1720
1721	if (m->wire_count != 0)
1722		panic("vm_page_free: freeing wired page %p", m);
1723	if (m->hold_count != 0) {
1724		m->flags &= ~PG_ZERO;
1725		vm_page_lock_queues();
1726		vm_page_enqueue(PQ_HOLD, m);
1727		vm_page_unlock_queues();
1728	} else {
1729		/*
1730		 * Restore the default memory attribute to the page.
1731		 */
1732		if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
1733			pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
1734
1735		/*
1736		 * Insert the page into the physical memory allocator's
1737		 * cache/free page queues.
1738		 */
1739		mtx_lock(&vm_page_queue_free_mtx);
1740		m->flags |= PG_FREE;
1741		cnt.v_free_count++;
1742#if VM_NRESERVLEVEL > 0
1743		if (!vm_reserv_free_page(m))
1744#else
1745		if (TRUE)
1746#endif
1747			vm_phys_free_pages(m, 0);
1748		if ((m->flags & PG_ZERO) != 0)
1749			++vm_page_zero_count;
1750		else
1751			vm_page_zero_idle_wakeup();
1752		vm_page_free_wakeup();
1753		mtx_unlock(&vm_page_queue_free_mtx);
1754	}
1755}
1756
1757/*
1758 *	vm_page_wire:
1759 *
1760 *	Mark this page as wired down by yet
1761 *	another map, removing it from paging queues
1762 *	as necessary.
1763 *
1764 *	If the page is fictitious, then its wire count must remain one.
1765 *
1766 *	The page must be locked.
1767 *	This routine may not block.
1768 */
1769void
1770vm_page_wire(vm_page_t m)
1771{
1772
1773	/*
1774	 * Only bump the wire statistics if the page is not already wired,
1775	 * and only unqueue the page if it is on some queue (if it is unmanaged
1776	 * it is already off the queues).
1777	 */
1778	vm_page_lock_assert(m, MA_OWNED);
1779	if ((m->flags & PG_FICTITIOUS) != 0) {
1780		KASSERT(m->wire_count == 1,
1781		    ("vm_page_wire: fictitious page %p's wire count isn't one",
1782		    m));
1783		return;
1784	}
1785	if (m->wire_count == 0) {
1786		if ((m->flags & PG_UNMANAGED) == 0)
1787			vm_pageq_remove(m);
1788		atomic_add_int(&cnt.v_wire_count, 1);
1789	}
1790	m->wire_count++;
1791	KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
1792}
1793
1794/*
1795 * vm_page_unwire:
1796 *
1797 * Release one wiring of the specified page, potentially enabling it to be
1798 * paged again.  If paging is enabled, then the value of the parameter
1799 * "activate" determines to which queue the page is added.  If "activate" is
1800 * non-zero, then the page is added to the active queue.  Otherwise, it is
1801 * added to the inactive queue.
1802 *
1803 * However, unless the page belongs to an object, it is not enqueued because
1804 * it cannot be paged out.
1805 *
1806 * If a page is fictitious, then its wire count must alway be one.
1807 *
1808 * A managed page must be locked.
1809 */
1810void
1811vm_page_unwire(vm_page_t m, int activate)
1812{
1813
1814	if ((m->flags & PG_UNMANAGED) == 0)
1815		vm_page_lock_assert(m, MA_OWNED);
1816	if ((m->flags & PG_FICTITIOUS) != 0) {
1817		KASSERT(m->wire_count == 1,
1818	    ("vm_page_unwire: fictitious page %p's wire count isn't one", m));
1819		return;
1820	}
1821	if (m->wire_count > 0) {
1822		m->wire_count--;
1823		if (m->wire_count == 0) {
1824			atomic_subtract_int(&cnt.v_wire_count, 1);
1825			if ((m->flags & PG_UNMANAGED) != 0 ||
1826			    m->object == NULL)
1827				return;
1828			vm_page_lock_queues();
1829			if (activate)
1830				vm_page_enqueue(PQ_ACTIVE, m);
1831			else {
1832				vm_page_flag_clear(m, PG_WINATCFLS);
1833				vm_page_enqueue(PQ_INACTIVE, m);
1834			}
1835			vm_page_unlock_queues();
1836		}
1837	} else
1838		panic("vm_page_unwire: page %p's wire count is zero", m);
1839}
1840
1841/*
1842 * Move the specified page to the inactive queue.
1843 *
1844 * Many pages placed on the inactive queue should actually go
1845 * into the cache, but it is difficult to figure out which.  What
1846 * we do instead, if the inactive target is well met, is to put
1847 * clean pages at the head of the inactive queue instead of the tail.
1848 * This will cause them to be moved to the cache more quickly and
1849 * if not actively re-referenced, reclaimed more quickly.  If we just
1850 * stick these pages at the end of the inactive queue, heavy filesystem
1851 * meta-data accesses can cause an unnecessary paging load on memory bound
1852 * processes.  This optimization causes one-time-use metadata to be
1853 * reused more quickly.
1854 *
1855 * Normally athead is 0 resulting in LRU operation.  athead is set
1856 * to 1 if we want this page to be 'as if it were placed in the cache',
1857 * except without unmapping it from the process address space.
1858 *
1859 * This routine may not block.
1860 */
1861static inline void
1862_vm_page_deactivate(vm_page_t m, int athead)
1863{
1864	int queue;
1865
1866	vm_page_lock_assert(m, MA_OWNED);
1867
1868	/*
1869	 * Ignore if already inactive.
1870	 */
1871	if ((queue = m->queue) == PQ_INACTIVE)
1872		return;
1873	if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
1874		vm_page_lock_queues();
1875		vm_page_flag_clear(m, PG_WINATCFLS);
1876		if (queue != PQ_NONE)
1877			vm_page_queue_remove(queue, m);
1878		if (athead)
1879			TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m,
1880			    pageq);
1881		else
1882			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
1883			    pageq);
1884		m->queue = PQ_INACTIVE;
1885		cnt.v_inactive_count++;
1886		vm_page_unlock_queues();
1887	}
1888}
1889
1890/*
1891 * Move the specified page to the inactive queue.
1892 *
1893 * The page must be locked.
1894 */
1895void
1896vm_page_deactivate(vm_page_t m)
1897{
1898
1899	_vm_page_deactivate(m, 0);
1900}
1901
1902/*
1903 * vm_page_try_to_cache:
1904 *
1905 * Returns 0 on failure, 1 on success
1906 */
1907int
1908vm_page_try_to_cache(vm_page_t m)
1909{
1910
1911	vm_page_lock_assert(m, MA_OWNED);
1912	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1913	if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1914	    (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED))
1915		return (0);
1916	pmap_remove_all(m);
1917	if (m->dirty)
1918		return (0);
1919	vm_page_cache(m);
1920	return (1);
1921}
1922
1923/*
1924 * vm_page_try_to_free()
1925 *
1926 *	Attempt to free the page.  If we cannot free it, we do nothing.
1927 *	1 is returned on success, 0 on failure.
1928 */
1929int
1930vm_page_try_to_free(vm_page_t m)
1931{
1932
1933	vm_page_lock_assert(m, MA_OWNED);
1934	if (m->object != NULL)
1935		VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1936	if (m->dirty || m->hold_count || m->busy || m->wire_count ||
1937	    (m->oflags & VPO_BUSY) || (m->flags & PG_UNMANAGED))
1938		return (0);
1939	pmap_remove_all(m);
1940	if (m->dirty)
1941		return (0);
1942	vm_page_free(m);
1943	return (1);
1944}
1945
1946/*
1947 * vm_page_cache
1948 *
1949 * Put the specified page onto the page cache queue (if appropriate).
1950 *
1951 * This routine may not block.
1952 */
1953void
1954vm_page_cache(vm_page_t m)
1955{
1956	vm_object_t object;
1957	vm_page_t root;
1958
1959	vm_page_lock_assert(m, MA_OWNED);
1960	object = m->object;
1961	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1962	if ((m->flags & PG_UNMANAGED) || (m->oflags & VPO_BUSY) || m->busy ||
1963	    m->hold_count || m->wire_count)
1964		panic("vm_page_cache: attempting to cache busy page");
1965	pmap_remove_all(m);
1966	if (m->dirty != 0)
1967		panic("vm_page_cache: page %p is dirty", m);
1968	if (m->valid == 0 || object->type == OBJT_DEFAULT ||
1969	    (object->type == OBJT_SWAP &&
1970	    !vm_pager_has_page(object, m->pindex, NULL, NULL))) {
1971		/*
1972		 * Hypothesis: A cache-elgible page belonging to a
1973		 * default object or swap object but without a backing
1974		 * store must be zero filled.
1975		 */
1976		vm_page_free(m);
1977		return;
1978	}
1979	KASSERT((m->flags & PG_CACHED) == 0,
1980	    ("vm_page_cache: page %p is already cached", m));
1981	PCPU_INC(cnt.v_tcached);
1982
1983	/*
1984	 * Remove the page from the paging queues.
1985	 */
1986	vm_pageq_remove(m);
1987
1988	/*
1989	 * Remove the page from the object's collection of resident
1990	 * pages.
1991	 */
1992	if (m != object->root)
1993		vm_page_splay(m->pindex, object->root);
1994	if (m->left == NULL)
1995		root = m->right;
1996	else {
1997		root = vm_page_splay(m->pindex, m->left);
1998		root->right = m->right;
1999	}
2000	object->root = root;
2001	TAILQ_REMOVE(&object->memq, m, listq);
2002	object->resident_page_count--;
2003
2004	/*
2005	 * Restore the default memory attribute to the page.
2006	 */
2007	if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
2008		pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
2009
2010	/*
2011	 * Insert the page into the object's collection of cached pages
2012	 * and the physical memory allocator's cache/free page queues.
2013	 */
2014	m->flags &= ~PG_ZERO;
2015	mtx_lock(&vm_page_queue_free_mtx);
2016	m->flags |= PG_CACHED;
2017	cnt.v_cache_count++;
2018	root = object->cache;
2019	if (root == NULL) {
2020		m->left = NULL;
2021		m->right = NULL;
2022	} else {
2023		root = vm_page_splay(m->pindex, root);
2024		if (m->pindex < root->pindex) {
2025			m->left = root->left;
2026			m->right = root;
2027			root->left = NULL;
2028		} else if (__predict_false(m->pindex == root->pindex))
2029			panic("vm_page_cache: offset already cached");
2030		else {
2031			m->right = root->right;
2032			m->left = root;
2033			root->right = NULL;
2034		}
2035	}
2036	object->cache = m;
2037#if VM_NRESERVLEVEL > 0
2038	if (!vm_reserv_free_page(m)) {
2039#else
2040	if (TRUE) {
2041#endif
2042		vm_phys_set_pool(VM_FREEPOOL_CACHE, m, 0);
2043		vm_phys_free_pages(m, 0);
2044	}
2045	vm_page_free_wakeup();
2046	mtx_unlock(&vm_page_queue_free_mtx);
2047
2048	/*
2049	 * Increment the vnode's hold count if this is the object's only
2050	 * cached page.  Decrement the vnode's hold count if this was
2051	 * the object's only resident page.
2052	 */
2053	if (object->type == OBJT_VNODE) {
2054		if (root == NULL && object->resident_page_count != 0)
2055			vhold(object->handle);
2056		else if (root != NULL && object->resident_page_count == 0)
2057			vdrop(object->handle);
2058	}
2059}
2060
2061/*
2062 * vm_page_dontneed
2063 *
2064 *	Cache, deactivate, or do nothing as appropriate.  This routine
2065 *	is typically used by madvise() MADV_DONTNEED.
2066 *
2067 *	Generally speaking we want to move the page into the cache so
2068 *	it gets reused quickly.  However, this can result in a silly syndrome
2069 *	due to the page recycling too quickly.  Small objects will not be
2070 *	fully cached.  On the otherhand, if we move the page to the inactive
2071 *	queue we wind up with a problem whereby very large objects
2072 *	unnecessarily blow away our inactive and cache queues.
2073 *
2074 *	The solution is to move the pages based on a fixed weighting.  We
2075 *	either leave them alone, deactivate them, or move them to the cache,
2076 *	where moving them to the cache has the highest weighting.
2077 *	By forcing some pages into other queues we eventually force the
2078 *	system to balance the queues, potentially recovering other unrelated
2079 *	space from active.  The idea is to not force this to happen too
2080 *	often.
2081 */
2082void
2083vm_page_dontneed(vm_page_t m)
2084{
2085	int dnw;
2086	int head;
2087
2088	vm_page_lock_assert(m, MA_OWNED);
2089	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2090	dnw = PCPU_GET(dnweight);
2091	PCPU_INC(dnweight);
2092
2093	/*
2094	 * Occasionally leave the page alone.
2095	 */
2096	if ((dnw & 0x01F0) == 0 || m->queue == PQ_INACTIVE) {
2097		if (m->act_count >= ACT_INIT)
2098			--m->act_count;
2099		return;
2100	}
2101
2102	/*
2103	 * Clear any references to the page.  Otherwise, the page daemon will
2104	 * immediately reactivate the page.
2105	 *
2106	 * Perform the pmap_clear_reference() first.  Otherwise, a concurrent
2107	 * pmap operation, such as pmap_remove(), could clear a reference in
2108	 * the pmap and set PG_REFERENCED on the page before the
2109	 * pmap_clear_reference() had completed.  Consequently, the page would
2110	 * appear referenced based upon an old reference that occurred before
2111	 * this function ran.
2112	 */
2113	pmap_clear_reference(m);
2114	vm_page_lock_queues();
2115	vm_page_flag_clear(m, PG_REFERENCED);
2116	vm_page_unlock_queues();
2117
2118	if (m->dirty == 0 && pmap_is_modified(m))
2119		vm_page_dirty(m);
2120
2121	if (m->dirty || (dnw & 0x0070) == 0) {
2122		/*
2123		 * Deactivate the page 3 times out of 32.
2124		 */
2125		head = 0;
2126	} else {
2127		/*
2128		 * Cache the page 28 times out of every 32.  Note that
2129		 * the page is deactivated instead of cached, but placed
2130		 * at the head of the queue instead of the tail.
2131		 */
2132		head = 1;
2133	}
2134	_vm_page_deactivate(m, head);
2135}
2136
2137/*
2138 * Grab a page, waiting until we are waken up due to the page
2139 * changing state.  We keep on waiting, if the page continues
2140 * to be in the object.  If the page doesn't exist, first allocate it
2141 * and then conditionally zero it.
2142 *
2143 * The caller must always specify the VM_ALLOC_RETRY flag.  This is intended
2144 * to facilitate its eventual removal.
2145 *
2146 * This routine may block.
2147 */
2148vm_page_t
2149vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
2150{
2151	vm_page_t m;
2152
2153	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2154	KASSERT((allocflags & VM_ALLOC_RETRY) != 0,
2155	    ("vm_page_grab: VM_ALLOC_RETRY is required"));
2156retrylookup:
2157	if ((m = vm_page_lookup(object, pindex)) != NULL) {
2158		if ((m->oflags & VPO_BUSY) != 0 ||
2159		    ((allocflags & VM_ALLOC_IGN_SBUSY) == 0 && m->busy != 0)) {
2160			/*
2161			 * Reference the page before unlocking and
2162			 * sleeping so that the page daemon is less
2163			 * likely to reclaim it.
2164			 */
2165			vm_page_lock_queues();
2166			vm_page_flag_set(m, PG_REFERENCED);
2167			vm_page_sleep(m, "pgrbwt");
2168			goto retrylookup;
2169		} else {
2170			if ((allocflags & VM_ALLOC_WIRED) != 0) {
2171				vm_page_lock(m);
2172				vm_page_wire(m);
2173				vm_page_unlock(m);
2174			}
2175			if ((allocflags & VM_ALLOC_NOBUSY) == 0)
2176				vm_page_busy(m);
2177			return (m);
2178		}
2179	}
2180	m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY |
2181	    VM_ALLOC_IGN_SBUSY));
2182	if (m == NULL) {
2183		VM_OBJECT_UNLOCK(object);
2184		VM_WAIT;
2185		VM_OBJECT_LOCK(object);
2186		goto retrylookup;
2187	} else if (m->valid != 0)
2188		return (m);
2189	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
2190		pmap_zero_page(m);
2191	return (m);
2192}
2193
2194/*
2195 * Mapping function for valid bits or for dirty bits in
2196 * a page.  May not block.
2197 *
2198 * Inputs are required to range within a page.
2199 */
2200int
2201vm_page_bits(int base, int size)
2202{
2203	int first_bit;
2204	int last_bit;
2205
2206	KASSERT(
2207	    base + size <= PAGE_SIZE,
2208	    ("vm_page_bits: illegal base/size %d/%d", base, size)
2209	);
2210
2211	if (size == 0)		/* handle degenerate case */
2212		return (0);
2213
2214	first_bit = base >> DEV_BSHIFT;
2215	last_bit = (base + size - 1) >> DEV_BSHIFT;
2216
2217	return ((2 << last_bit) - (1 << first_bit));
2218}
2219
2220/*
2221 *	vm_page_set_valid:
2222 *
2223 *	Sets portions of a page valid.  The arguments are expected
2224 *	to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2225 *	of any partial chunks touched by the range.  The invalid portion of
2226 *	such chunks will be zeroed.
2227 *
2228 *	(base + size) must be less then or equal to PAGE_SIZE.
2229 */
2230void
2231vm_page_set_valid(vm_page_t m, int base, int size)
2232{
2233	int endoff, frag;
2234
2235	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2236	if (size == 0)	/* handle degenerate case */
2237		return;
2238
2239	/*
2240	 * If the base is not DEV_BSIZE aligned and the valid
2241	 * bit is clear, we have to zero out a portion of the
2242	 * first block.
2243	 */
2244	if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2245	    (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
2246		pmap_zero_page_area(m, frag, base - frag);
2247
2248	/*
2249	 * If the ending offset is not DEV_BSIZE aligned and the
2250	 * valid bit is clear, we have to zero out a portion of
2251	 * the last block.
2252	 */
2253	endoff = base + size;
2254	if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2255	    (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
2256		pmap_zero_page_area(m, endoff,
2257		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2258
2259	/*
2260	 * Assert that no previously invalid block that is now being validated
2261	 * is already dirty.
2262	 */
2263	KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0,
2264	    ("vm_page_set_valid: page %p is dirty", m));
2265
2266	/*
2267	 * Set valid bits inclusive of any overlap.
2268	 */
2269	m->valid |= vm_page_bits(base, size);
2270}
2271
2272/*
2273 * Clear the given bits from the specified page's dirty field.
2274 */
2275static __inline void
2276vm_page_clear_dirty_mask(vm_page_t m, int pagebits)
2277{
2278
2279	/*
2280	 * If the object is locked and the page is neither VPO_BUSY nor
2281	 * PG_WRITEABLE, then the page's dirty field cannot possibly be
2282	 * modified by a concurrent pmap operation.
2283	 */
2284	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2285	if ((m->oflags & VPO_BUSY) == 0 && (m->flags & PG_WRITEABLE) == 0)
2286		m->dirty &= ~pagebits;
2287	else {
2288		vm_page_lock_queues();
2289		m->dirty &= ~pagebits;
2290		vm_page_unlock_queues();
2291	}
2292}
2293
2294/*
2295 *	vm_page_set_validclean:
2296 *
2297 *	Sets portions of a page valid and clean.  The arguments are expected
2298 *	to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2299 *	of any partial chunks touched by the range.  The invalid portion of
2300 *	such chunks will be zero'd.
2301 *
2302 *	This routine may not block.
2303 *
2304 *	(base + size) must be less then or equal to PAGE_SIZE.
2305 */
2306void
2307vm_page_set_validclean(vm_page_t m, int base, int size)
2308{
2309	u_long oldvalid;
2310	int endoff, frag, pagebits;
2311
2312	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2313	if (size == 0)	/* handle degenerate case */
2314		return;
2315
2316	/*
2317	 * If the base is not DEV_BSIZE aligned and the valid
2318	 * bit is clear, we have to zero out a portion of the
2319	 * first block.
2320	 */
2321	if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2322	    (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
2323		pmap_zero_page_area(m, frag, base - frag);
2324
2325	/*
2326	 * If the ending offset is not DEV_BSIZE aligned and the
2327	 * valid bit is clear, we have to zero out a portion of
2328	 * the last block.
2329	 */
2330	endoff = base + size;
2331	if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2332	    (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
2333		pmap_zero_page_area(m, endoff,
2334		    DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2335
2336	/*
2337	 * Set valid, clear dirty bits.  If validating the entire
2338	 * page we can safely clear the pmap modify bit.  We also
2339	 * use this opportunity to clear the VPO_NOSYNC flag.  If a process
2340	 * takes a write fault on a MAP_NOSYNC memory area the flag will
2341	 * be set again.
2342	 *
2343	 * We set valid bits inclusive of any overlap, but we can only
2344	 * clear dirty bits for DEV_BSIZE chunks that are fully within
2345	 * the range.
2346	 */
2347	oldvalid = m->valid;
2348	pagebits = vm_page_bits(base, size);
2349	m->valid |= pagebits;
2350#if 0	/* NOT YET */
2351	if ((frag = base & (DEV_BSIZE - 1)) != 0) {
2352		frag = DEV_BSIZE - frag;
2353		base += frag;
2354		size -= frag;
2355		if (size < 0)
2356			size = 0;
2357	}
2358	pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
2359#endif
2360	if (base == 0 && size == PAGE_SIZE) {
2361		/*
2362		 * The page can only be modified within the pmap if it is
2363		 * mapped, and it can only be mapped if it was previously
2364		 * fully valid.
2365		 */
2366		if (oldvalid == VM_PAGE_BITS_ALL)
2367			/*
2368			 * Perform the pmap_clear_modify() first.  Otherwise,
2369			 * a concurrent pmap operation, such as
2370			 * pmap_protect(), could clear a modification in the
2371			 * pmap and set the dirty field on the page before
2372			 * pmap_clear_modify() had begun and after the dirty
2373			 * field was cleared here.
2374			 */
2375			pmap_clear_modify(m);
2376		m->dirty = 0;
2377		m->oflags &= ~VPO_NOSYNC;
2378	} else if (oldvalid != VM_PAGE_BITS_ALL)
2379		m->dirty &= ~pagebits;
2380	else
2381		vm_page_clear_dirty_mask(m, pagebits);
2382}
2383
2384void
2385vm_page_clear_dirty(vm_page_t m, int base, int size)
2386{
2387
2388	vm_page_clear_dirty_mask(m, vm_page_bits(base, size));
2389}
2390
2391/*
2392 *	vm_page_set_invalid:
2393 *
2394 *	Invalidates DEV_BSIZE'd chunks within a page.  Both the
2395 *	valid and dirty bits for the effected areas are cleared.
2396 *
2397 *	May not block.
2398 */
2399void
2400vm_page_set_invalid(vm_page_t m, int base, int size)
2401{
2402	int bits;
2403
2404	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2405	KASSERT((m->oflags & VPO_BUSY) == 0,
2406	    ("vm_page_set_invalid: page %p is busy", m));
2407	bits = vm_page_bits(base, size);
2408	if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
2409		pmap_remove_all(m);
2410	KASSERT(!pmap_page_is_mapped(m),
2411	    ("vm_page_set_invalid: page %p is mapped", m));
2412	m->valid &= ~bits;
2413	m->dirty &= ~bits;
2414}
2415
2416/*
2417 * vm_page_zero_invalid()
2418 *
2419 *	The kernel assumes that the invalid portions of a page contain
2420 *	garbage, but such pages can be mapped into memory by user code.
2421 *	When this occurs, we must zero out the non-valid portions of the
2422 *	page so user code sees what it expects.
2423 *
2424 *	Pages are most often semi-valid when the end of a file is mapped
2425 *	into memory and the file's size is not page aligned.
2426 */
2427void
2428vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
2429{
2430	int b;
2431	int i;
2432
2433	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2434	/*
2435	 * Scan the valid bits looking for invalid sections that
2436	 * must be zerod.  Invalid sub-DEV_BSIZE'd areas ( where the
2437	 * valid bit may be set ) have already been zerod by
2438	 * vm_page_set_validclean().
2439	 */
2440	for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
2441		if (i == (PAGE_SIZE / DEV_BSIZE) ||
2442		    (m->valid & (1 << i))
2443		) {
2444			if (i > b) {
2445				pmap_zero_page_area(m,
2446				    b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
2447			}
2448			b = i + 1;
2449		}
2450	}
2451
2452	/*
2453	 * setvalid is TRUE when we can safely set the zero'd areas
2454	 * as being valid.  We can do this if there are no cache consistancy
2455	 * issues.  e.g. it is ok to do with UFS, but not ok to do with NFS.
2456	 */
2457	if (setvalid)
2458		m->valid = VM_PAGE_BITS_ALL;
2459}
2460
2461/*
2462 *	vm_page_is_valid:
2463 *
2464 *	Is (partial) page valid?  Note that the case where size == 0
2465 *	will return FALSE in the degenerate case where the page is
2466 *	entirely invalid, and TRUE otherwise.
2467 *
2468 *	May not block.
2469 */
2470int
2471vm_page_is_valid(vm_page_t m, int base, int size)
2472{
2473	int bits = vm_page_bits(base, size);
2474
2475	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2476	if (m->valid && ((m->valid & bits) == bits))
2477		return 1;
2478	else
2479		return 0;
2480}
2481
2482/*
2483 * update dirty bits from pmap/mmu.  May not block.
2484 */
2485void
2486vm_page_test_dirty(vm_page_t m)
2487{
2488
2489	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2490	if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m))
2491		vm_page_dirty(m);
2492}
2493
2494int so_zerocp_fullpage = 0;
2495
2496/*
2497 *	Replace the given page with a copy.  The copied page assumes
2498 *	the portion of the given page's "wire_count" that is not the
2499 *	responsibility of this copy-on-write mechanism.
2500 *
2501 *	The object containing the given page must have a non-zero
2502 *	paging-in-progress count and be locked.
2503 */
2504void
2505vm_page_cowfault(vm_page_t m)
2506{
2507	vm_page_t mnew;
2508	vm_object_t object;
2509	vm_pindex_t pindex;
2510
2511	mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
2512	vm_page_lock_assert(m, MA_OWNED);
2513	object = m->object;
2514	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2515	KASSERT(object->paging_in_progress != 0,
2516	    ("vm_page_cowfault: object %p's paging-in-progress count is zero.",
2517	    object));
2518	pindex = m->pindex;
2519
2520 retry_alloc:
2521	pmap_remove_all(m);
2522	vm_page_remove(m);
2523	mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
2524	if (mnew == NULL) {
2525		vm_page_insert(m, object, pindex);
2526		vm_page_unlock(m);
2527		VM_OBJECT_UNLOCK(object);
2528		VM_WAIT;
2529		VM_OBJECT_LOCK(object);
2530		if (m == vm_page_lookup(object, pindex)) {
2531			vm_page_lock(m);
2532			goto retry_alloc;
2533		} else {
2534			/*
2535			 * Page disappeared during the wait.
2536			 */
2537			return;
2538		}
2539	}
2540
2541	if (m->cow == 0) {
2542		/*
2543		 * check to see if we raced with an xmit complete when
2544		 * waiting to allocate a page.  If so, put things back
2545		 * the way they were
2546		 */
2547		vm_page_unlock(m);
2548		vm_page_lock(mnew);
2549		vm_page_free(mnew);
2550		vm_page_unlock(mnew);
2551		vm_page_insert(m, object, pindex);
2552	} else { /* clear COW & copy page */
2553		if (!so_zerocp_fullpage)
2554			pmap_copy_page(m, mnew);
2555		mnew->valid = VM_PAGE_BITS_ALL;
2556		vm_page_dirty(mnew);
2557		mnew->wire_count = m->wire_count - m->cow;
2558		m->wire_count = m->cow;
2559		vm_page_unlock(m);
2560	}
2561}
2562
2563void
2564vm_page_cowclear(vm_page_t m)
2565{
2566
2567	vm_page_lock_assert(m, MA_OWNED);
2568	if (m->cow) {
2569		m->cow--;
2570		/*
2571		 * let vm_fault add back write permission  lazily
2572		 */
2573	}
2574	/*
2575	 *  sf_buf_free() will free the page, so we needn't do it here
2576	 */
2577}
2578
2579int
2580vm_page_cowsetup(vm_page_t m)
2581{
2582
2583	vm_page_lock_assert(m, MA_OWNED);
2584	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0 ||
2585	    m->cow == USHRT_MAX - 1 || !VM_OBJECT_TRYLOCK(m->object))
2586		return (EBUSY);
2587	m->cow++;
2588	pmap_remove_write(m);
2589	VM_OBJECT_UNLOCK(m->object);
2590	return (0);
2591}
2592
2593#include "opt_ddb.h"
2594#ifdef DDB
2595#include <sys/kernel.h>
2596
2597#include <ddb/ddb.h>
2598
2599DB_SHOW_COMMAND(page, vm_page_print_page_info)
2600{
2601	db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
2602	db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
2603	db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
2604	db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
2605	db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
2606	db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
2607	db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
2608	db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
2609	db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
2610	db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
2611}
2612
2613DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
2614{
2615
2616	db_printf("PQ_FREE:");
2617	db_printf(" %d", cnt.v_free_count);
2618	db_printf("\n");
2619
2620	db_printf("PQ_CACHE:");
2621	db_printf(" %d", cnt.v_cache_count);
2622	db_printf("\n");
2623
2624	db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
2625		*vm_page_queues[PQ_ACTIVE].cnt,
2626		*vm_page_queues[PQ_INACTIVE].cnt);
2627}
2628#endif /* DDB */
2629