1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1998 Matthew Dillon. All Rights Reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * The Mach Operating System project at Carnegie-Mellon University.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91
34 */
35
36/*-
37 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38 * All rights reserved.
39 *
40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41 *
42 * Permission to use, copy, modify and distribute this software and
43 * its documentation is hereby granted, provided that both the copyright
44 * notice and this permission notice appear in all copies of the
45 * software, derivative works or modified versions, and any portions
46 * thereof, and that both notices appear in supporting documentation.
47 *
48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51 *
52 * Carnegie Mellon requests users of this software to return to
53 *
54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
55 * School of Computer Science
56 * Carnegie Mellon University
57 * Pittsburgh PA 15213-3890
58 *
59 * any improvements or extensions that they make and grant Carnegie the
60 * rights to redistribute these changes.
61 */
62
63/*
64 * GENERAL RULES ON VM_PAGE MANIPULATION
65 *
66 * - a pageq mutex is required when adding or removing a page from a
67 * page queue (vm_page_queue[]), regardless of other mutexes or the
68 * busy state of a page.
69 *
70 * - The object mutex is held when inserting or removing
71 * pages from an object (vm_page_insert() or vm_page_remove()).
72 *
73 */
74
75/*
76 * Resident memory management module.
77 */
78
79#include <sys/cdefs.h>
| 1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1998 Matthew Dillon. All Rights Reserved.
5 *
6 * This code is derived from software contributed to Berkeley by
7 * The Mach Operating System project at Carnegie-Mellon University.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 4. Neither the name of the University nor the names of its contributors
18 * may be used to endorse or promote products derived from this software
19 * without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * SUCH DAMAGE.
32 *
33 * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91
34 */
35
36/*-
37 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38 * All rights reserved.
39 *
40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41 *
42 * Permission to use, copy, modify and distribute this software and
43 * its documentation is hereby granted, provided that both the copyright
44 * notice and this permission notice appear in all copies of the
45 * software, derivative works or modified versions, and any portions
46 * thereof, and that both notices appear in supporting documentation.
47 *
48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51 *
52 * Carnegie Mellon requests users of this software to return to
53 *
54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
55 * School of Computer Science
56 * Carnegie Mellon University
57 * Pittsburgh PA 15213-3890
58 *
59 * any improvements or extensions that they make and grant Carnegie the
60 * rights to redistribute these changes.
61 */
62
63/*
64 * GENERAL RULES ON VM_PAGE MANIPULATION
65 *
66 * - a pageq mutex is required when adding or removing a page from a
67 * page queue (vm_page_queue[]), regardless of other mutexes or the
68 * busy state of a page.
69 *
70 * - The object mutex is held when inserting or removing
71 * pages from an object (vm_page_insert() or vm_page_remove()).
72 *
73 */
74
75/*
76 * Resident memory management module.
77 */
78
79#include <sys/cdefs.h>
|
80__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 228287 2011-12-05 18:29:25Z alc $");
| 80__FBSDID("$FreeBSD: head/sys/vm/vm_page.c 230623 2012-01-27 20:18:31Z kmacy $");
|
81
82#include "opt_vm.h"
83
84#include <sys/param.h>
85#include <sys/systm.h>
86#include <sys/lock.h>
87#include <sys/kernel.h>
88#include <sys/limits.h>
89#include <sys/malloc.h>
90#include <sys/msgbuf.h>
91#include <sys/mutex.h>
92#include <sys/proc.h>
93#include <sys/sysctl.h>
94#include <sys/vmmeter.h>
95#include <sys/vnode.h>
96
97#include <vm/vm.h>
98#include <vm/pmap.h>
99#include <vm/vm_param.h>
100#include <vm/vm_kern.h>
101#include <vm/vm_object.h>
102#include <vm/vm_page.h>
103#include <vm/vm_pageout.h>
104#include <vm/vm_pager.h>
105#include <vm/vm_phys.h>
106#include <vm/vm_reserv.h>
107#include <vm/vm_extern.h>
108#include <vm/uma.h>
109#include <vm/uma_int.h>
110
111#include <machine/md_var.h>
112
113/*
114 * Associated with page of user-allocatable memory is a
115 * page structure.
116 */
117
118struct vpgqueues vm_page_queues[PQ_COUNT];
119struct vpglocks vm_page_queue_lock;
120struct vpglocks vm_page_queue_free_lock;
121
122struct vpglocks pa_lock[PA_LOCK_COUNT];
123
124vm_page_t vm_page_array = 0;
125int vm_page_array_size = 0;
126long first_page = 0;
127int vm_page_zero_count = 0;
128
129static int boot_pages = UMA_BOOT_PAGES;
130TUNABLE_INT("vm.boot_pages", &boot_pages);
131SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
132 "number of pages allocated for bootstrapping the VM system");
133
134static int pa_tryrelock_restart;
135SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD,
136 &pa_tryrelock_restart, 0, "Number of tryrelock restarts");
137
138static uma_zone_t fakepg_zone;
139
140static struct vnode *vm_page_alloc_init(vm_page_t m);
141static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
142static void vm_page_queue_remove(int queue, vm_page_t m);
143static void vm_page_enqueue(int queue, vm_page_t m);
144static void vm_page_init_fakepg(void *dummy);
145
146SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL);
147
148static void
149vm_page_init_fakepg(void *dummy)
150{
151
152 fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL,
153 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
154}
155
156/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
157#if PAGE_SIZE == 32768
158#ifdef CTASSERT
159CTASSERT(sizeof(u_long) >= 8);
160#endif
161#endif
162
163/*
164 * Try to acquire a physical address lock while a pmap is locked. If we
165 * fail to trylock we unlock and lock the pmap directly and cache the
166 * locked pa in *locked. The caller should then restart their loop in case
167 * the virtual to physical mapping has changed.
168 */
169int
170vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked)
171{
172 vm_paddr_t lockpa;
173
174 lockpa = *locked;
175 *locked = pa;
176 if (lockpa) {
177 PA_LOCK_ASSERT(lockpa, MA_OWNED);
178 if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa))
179 return (0);
180 PA_UNLOCK(lockpa);
181 }
182 if (PA_TRYLOCK(pa))
183 return (0);
184 PMAP_UNLOCK(pmap);
185 atomic_add_int(&pa_tryrelock_restart, 1);
186 PA_LOCK(pa);
187 PMAP_LOCK(pmap);
188 return (EAGAIN);
189}
190
191/*
192 * vm_set_page_size:
193 *
194 * Sets the page size, perhaps based upon the memory
195 * size. Must be called before any use of page-size
196 * dependent functions.
197 */
198void
199vm_set_page_size(void)
200{
201 if (cnt.v_page_size == 0)
202 cnt.v_page_size = PAGE_SIZE;
203 if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
204 panic("vm_set_page_size: page size not a power of two");
205}
206
207/*
208 * vm_page_blacklist_lookup:
209 *
210 * See if a physical address in this page has been listed
211 * in the blacklist tunable. Entries in the tunable are
212 * separated by spaces or commas. If an invalid integer is
213 * encountered then the rest of the string is skipped.
214 */
215static int
216vm_page_blacklist_lookup(char *list, vm_paddr_t pa)
217{
218 vm_paddr_t bad;
219 char *cp, *pos;
220
221 for (pos = list; *pos != '\0'; pos = cp) {
222 bad = strtoq(pos, &cp, 0);
223 if (*cp != '\0') {
224 if (*cp == ' ' || *cp == ',') {
225 cp++;
226 if (cp == pos)
227 continue;
228 } else
229 break;
230 }
231 if (pa == trunc_page(bad))
232 return (1);
233 }
234 return (0);
235}
236
237/*
238 * vm_page_startup:
239 *
240 * Initializes the resident memory module.
241 *
242 * Allocates memory for the page cells, and
243 * for the object/offset-to-page hash table headers.
244 * Each page cell is initialized and placed on the free list.
245 */
246vm_offset_t
247vm_page_startup(vm_offset_t vaddr)
248{
249 vm_offset_t mapped;
250 vm_paddr_t page_range;
251 vm_paddr_t new_end;
252 int i;
253 vm_paddr_t pa;
254 vm_paddr_t last_pa;
255 char *list;
256
257 /* the biggest memory array is the second group of pages */
258 vm_paddr_t end;
259 vm_paddr_t biggestsize;
260 vm_paddr_t low_water, high_water;
261 int biggestone;
262
263 biggestsize = 0;
264 biggestone = 0;
265 vaddr = round_page(vaddr);
266
267 for (i = 0; phys_avail[i + 1]; i += 2) {
268 phys_avail[i] = round_page(phys_avail[i]);
269 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
270 }
271
272 low_water = phys_avail[0];
273 high_water = phys_avail[1];
274
275 for (i = 0; phys_avail[i + 1]; i += 2) {
276 vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
277
278 if (size > biggestsize) {
279 biggestone = i;
280 biggestsize = size;
281 }
282 if (phys_avail[i] < low_water)
283 low_water = phys_avail[i];
284 if (phys_avail[i + 1] > high_water)
285 high_water = phys_avail[i + 1];
286 }
287
288#ifdef XEN
289 low_water = 0;
290#endif
291
292 end = phys_avail[biggestone+1];
293
294 /*
295 * Initialize the locks.
296 */
297 mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
298 MTX_RECURSE);
299 mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
300 MTX_DEF);
301
302 /* Setup page locks. */
303 for (i = 0; i < PA_LOCK_COUNT; i++)
304 mtx_init(&pa_lock[i].data, "page lock", NULL, MTX_DEF);
305
306 /*
307 * Initialize the queue headers for the hold queue, the active queue,
308 * and the inactive queue.
309 */
310 for (i = 0; i < PQ_COUNT; i++)
311 TAILQ_INIT(&vm_page_queues[i].pl);
312 vm_page_queues[PQ_INACTIVE].cnt = &cnt.v_inactive_count;
313 vm_page_queues[PQ_ACTIVE].cnt = &cnt.v_active_count;
314 vm_page_queues[PQ_HOLD].cnt = &cnt.v_active_count;
315
316 /*
317 * Allocate memory for use when boot strapping the kernel memory
318 * allocator.
319 */
320 new_end = end - (boot_pages * UMA_SLAB_SIZE);
321 new_end = trunc_page(new_end);
322 mapped = pmap_map(&vaddr, new_end, end,
323 VM_PROT_READ | VM_PROT_WRITE);
324 bzero((void *)mapped, end - new_end);
325 uma_startup((void *)mapped, boot_pages);
326
327#if defined(__amd64__) || defined(__i386__) || defined(__arm__) || \
328 defined(__mips__)
329 /*
330 * Allocate a bitmap to indicate that a random physical page
331 * needs to be included in a minidump.
332 *
333 * The amd64 port needs this to indicate which direct map pages
334 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
335 *
336 * However, i386 still needs this workspace internally within the
337 * minidump code. In theory, they are not needed on i386, but are
338 * included should the sf_buf code decide to use them.
339 */
340 last_pa = 0;
341 for (i = 0; dump_avail[i + 1] != 0; i += 2)
342 if (dump_avail[i + 1] > last_pa)
343 last_pa = dump_avail[i + 1];
344 page_range = last_pa / PAGE_SIZE;
345 vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
346 new_end -= vm_page_dump_size;
347 vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
348 new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
349 bzero((void *)vm_page_dump, vm_page_dump_size);
350#endif
351#ifdef __amd64__
352 /*
353 * Request that the physical pages underlying the message buffer be
354 * included in a crash dump. Since the message buffer is accessed
355 * through the direct map, they are not automatically included.
356 */
357 pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr);
358 last_pa = pa + round_page(msgbufsize);
359 while (pa < last_pa) {
360 dump_add_page(pa);
361 pa += PAGE_SIZE;
362 }
363#endif
364 /*
365 * Compute the number of pages of memory that will be available for
366 * use (taking into account the overhead of a page structure per
367 * page).
368 */
369 first_page = low_water / PAGE_SIZE;
370#ifdef VM_PHYSSEG_SPARSE
371 page_range = 0;
372 for (i = 0; phys_avail[i + 1] != 0; i += 2)
373 page_range += atop(phys_avail[i + 1] - phys_avail[i]);
374#elif defined(VM_PHYSSEG_DENSE)
375 page_range = high_water / PAGE_SIZE - first_page;
376#else
377#error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
378#endif
379 end = new_end;
380
381 /*
382 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
383 */
384 vaddr += PAGE_SIZE;
385
386 /*
387 * Initialize the mem entry structures now, and put them in the free
388 * queue.
389 */
390 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
391 mapped = pmap_map(&vaddr, new_end, end,
392 VM_PROT_READ | VM_PROT_WRITE);
393 vm_page_array = (vm_page_t) mapped;
394#if VM_NRESERVLEVEL > 0
395 /*
396 * Allocate memory for the reservation management system's data
397 * structures.
398 */
399 new_end = vm_reserv_startup(&vaddr, new_end, high_water);
400#endif
401#if defined(__amd64__) || defined(__mips__)
402 /*
403 * pmap_map on amd64 and mips can come out of the direct-map, not kvm
404 * like i386, so the pages must be tracked for a crashdump to include
405 * this data. This includes the vm_page_array and the early UMA
406 * bootstrap pages.
407 */
408 for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE)
409 dump_add_page(pa);
410#endif
411 phys_avail[biggestone + 1] = new_end;
412
413 /*
414 * Clear all of the page structures
415 */
416 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
417 for (i = 0; i < page_range; i++)
418 vm_page_array[i].order = VM_NFREEORDER;
419 vm_page_array_size = page_range;
420
421 /*
422 * Initialize the physical memory allocator.
423 */
424 vm_phys_init();
425
426 /*
427 * Add every available physical page that is not blacklisted to
428 * the free lists.
429 */
430 cnt.v_page_count = 0;
431 cnt.v_free_count = 0;
432 list = getenv("vm.blacklist");
433 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
434 pa = phys_avail[i];
435 last_pa = phys_avail[i + 1];
436 while (pa < last_pa) {
437 if (list != NULL &&
438 vm_page_blacklist_lookup(list, pa))
439 printf("Skipping page with pa 0x%jx\n",
440 (uintmax_t)pa);
441 else
442 vm_phys_add_page(pa);
443 pa += PAGE_SIZE;
444 }
445 }
446 freeenv(list);
447#if VM_NRESERVLEVEL > 0
448 /*
449 * Initialize the reservation management system.
450 */
451 vm_reserv_init();
452#endif
453 return (vaddr);
454}
455
456
457CTASSERT(offsetof(struct vm_page, aflags) % sizeof(uint32_t) == 0);
458
459void
460vm_page_aflag_set(vm_page_t m, uint8_t bits)
461{
462 uint32_t *addr, val;
463
464 /*
465 * The PGA_WRITEABLE flag can only be set if the page is managed and
466 * VPO_BUSY. Currently, this flag is only set by pmap_enter().
467 */
468 KASSERT((bits & PGA_WRITEABLE) == 0 ||
469 (m->oflags & (VPO_UNMANAGED | VPO_BUSY)) == VPO_BUSY,
470 ("PGA_WRITEABLE and !VPO_BUSY"));
471
472 /*
473 * We want to use atomic updates for m->aflags, which is a
474 * byte wide. Not all architectures provide atomic operations
475 * on the single-byte destination. Punt and access the whole
476 * 4-byte word with an atomic update. Parallel non-atomic
477 * updates to the fields included in the update by proximity
478 * are handled properly by atomics.
479 */
480 addr = (void *)&m->aflags;
481 MPASS(((uintptr_t)addr & (sizeof(uint32_t) - 1)) == 0);
482 val = bits;
483#if BYTE_ORDER == BIG_ENDIAN
484 val <<= 24;
485#endif
486 atomic_set_32(addr, val);
487}
488
489void
490vm_page_aflag_clear(vm_page_t m, uint8_t bits)
491{
492 uint32_t *addr, val;
493
494 /*
495 * The PGA_REFERENCED flag can only be cleared if the object
496 * containing the page is locked.
497 */
498 KASSERT((bits & PGA_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object),
499 ("PGA_REFERENCED and !VM_OBJECT_LOCKED"));
500
501 /*
502 * See the comment in vm_page_aflag_set().
503 */
504 addr = (void *)&m->aflags;
505 MPASS(((uintptr_t)addr & (sizeof(uint32_t) - 1)) == 0);
506 val = bits;
507#if BYTE_ORDER == BIG_ENDIAN
508 val <<= 24;
509#endif
510 atomic_clear_32(addr, val);
511}
512
513void
514vm_page_reference(vm_page_t m)
515{
516
517 vm_page_aflag_set(m, PGA_REFERENCED);
518}
519
520void
521vm_page_busy(vm_page_t m)
522{
523
524 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
525 KASSERT((m->oflags & VPO_BUSY) == 0,
526 ("vm_page_busy: page already busy!!!"));
527 m->oflags |= VPO_BUSY;
528}
529
530/*
531 * vm_page_flash:
532 *
533 * wakeup anyone waiting for the page.
534 */
535void
536vm_page_flash(vm_page_t m)
537{
538
539 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
540 if (m->oflags & VPO_WANTED) {
541 m->oflags &= ~VPO_WANTED;
542 wakeup(m);
543 }
544}
545
546/*
547 * vm_page_wakeup:
548 *
549 * clear the VPO_BUSY flag and wakeup anyone waiting for the
550 * page.
551 *
552 */
553void
554vm_page_wakeup(vm_page_t m)
555{
556
557 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
558 KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!"));
559 m->oflags &= ~VPO_BUSY;
560 vm_page_flash(m);
561}
562
563void
564vm_page_io_start(vm_page_t m)
565{
566
567 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
568 m->busy++;
569}
570
571void
572vm_page_io_finish(vm_page_t m)
573{
574
575 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
576 KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m));
577 m->busy--;
578 if (m->busy == 0)
579 vm_page_flash(m);
580}
581
582/*
583 * Keep page from being freed by the page daemon
584 * much of the same effect as wiring, except much lower
585 * overhead and should be used only for *very* temporary
586 * holding ("wiring").
587 */
588void
589vm_page_hold(vm_page_t mem)
590{
591
592 vm_page_lock_assert(mem, MA_OWNED);
593 mem->hold_count++;
594}
595
596void
597vm_page_unhold(vm_page_t mem)
598{
599
600 vm_page_lock_assert(mem, MA_OWNED);
601 --mem->hold_count;
602 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
603 if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
604 vm_page_free_toq(mem);
605}
606
607/*
608 * vm_page_unhold_pages:
609 *
610 * Unhold each of the pages that is referenced by the given array.
611 */
612void
613vm_page_unhold_pages(vm_page_t *ma, int count)
614{
615 struct mtx *mtx, *new_mtx;
616
617 mtx = NULL;
618 for (; count != 0; count--) {
619 /*
620 * Avoid releasing and reacquiring the same page lock.
621 */
622 new_mtx = vm_page_lockptr(*ma);
623 if (mtx != new_mtx) {
624 if (mtx != NULL)
625 mtx_unlock(mtx);
626 mtx = new_mtx;
627 mtx_lock(mtx);
628 }
629 vm_page_unhold(*ma);
630 ma++;
631 }
632 if (mtx != NULL)
633 mtx_unlock(mtx);
634}
635
636/*
637 * vm_page_getfake:
638 *
639 * Create a fictitious page with the specified physical address and
640 * memory attribute. The memory attribute is the only the machine-
641 * dependent aspect of a fictitious page that must be initialized.
642 */
643vm_page_t
644vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
645{
646 vm_page_t m;
647
648 m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO);
649 m->phys_addr = paddr;
650 m->queue = PQ_NONE;
651 /* Fictitious pages don't use "segind". */
652 m->flags = PG_FICTITIOUS;
653 /* Fictitious pages don't use "order" or "pool". */
654 m->oflags = VPO_BUSY | VPO_UNMANAGED;
655 m->wire_count = 1;
656 pmap_page_set_memattr(m, memattr);
657 return (m);
658}
659
660/*
661 * vm_page_putfake:
662 *
663 * Release a fictitious page.
664 */
665void
666vm_page_putfake(vm_page_t m)
667{
668
669 KASSERT((m->flags & PG_FICTITIOUS) != 0,
670 ("vm_page_putfake: bad page %p", m));
671 uma_zfree(fakepg_zone, m);
672}
673
674/*
675 * vm_page_updatefake:
676 *
677 * Update the given fictitious page to the specified physical address and
678 * memory attribute.
679 */
680void
681vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
682{
683
684 KASSERT((m->flags & PG_FICTITIOUS) != 0,
685 ("vm_page_updatefake: bad page %p", m));
686 m->phys_addr = paddr;
687 pmap_page_set_memattr(m, memattr);
688}
689
690/*
691 * vm_page_free:
692 *
693 * Free a page.
694 */
695void
696vm_page_free(vm_page_t m)
697{
698
699 m->flags &= ~PG_ZERO;
700 vm_page_free_toq(m);
701}
702
703/*
704 * vm_page_free_zero:
705 *
706 * Free a page to the zerod-pages queue
707 */
708void
709vm_page_free_zero(vm_page_t m)
710{
711
712 m->flags |= PG_ZERO;
713 vm_page_free_toq(m);
714}
715
716/*
717 * vm_page_sleep:
718 *
719 * Sleep and release the page and page queues locks.
720 *
721 * The object containing the given page must be locked.
722 */
723void
724vm_page_sleep(vm_page_t m, const char *msg)
725{
726
727 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
728 if (mtx_owned(&vm_page_queue_mtx))
729 vm_page_unlock_queues();
730 if (mtx_owned(vm_page_lockptr(m)))
731 vm_page_unlock(m);
732
733 /*
734 * It's possible that while we sleep, the page will get
735 * unbusied and freed. If we are holding the object
736 * lock, we will assume we hold a reference to the object
737 * such that even if m->object changes, we can re-lock
738 * it.
739 */
740 m->oflags |= VPO_WANTED;
741 msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0);
742}
743
744/*
745 * vm_page_dirty:
746 *
747 * Set all bits in the page's dirty field.
748 *
749 * The object containing the specified page must be locked if the
750 * call is made from the machine-independent layer.
751 *
752 * See vm_page_clear_dirty_mask().
753 */
754void
755vm_page_dirty(vm_page_t m)
756{
757
758 KASSERT((m->flags & PG_CACHED) == 0,
759 ("vm_page_dirty: page in cache!"));
760 KASSERT(!VM_PAGE_IS_FREE(m),
761 ("vm_page_dirty: page is free!"));
762 KASSERT(m->valid == VM_PAGE_BITS_ALL,
763 ("vm_page_dirty: page is invalid!"));
764 m->dirty = VM_PAGE_BITS_ALL;
765}
766
767/*
768 * vm_page_splay:
769 *
770 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
771 * the vm_page containing the given pindex. If, however, that
772 * pindex is not found in the vm_object, returns a vm_page that is
773 * adjacent to the pindex, coming before or after it.
774 */
775vm_page_t
776vm_page_splay(vm_pindex_t pindex, vm_page_t root)
777{
778 struct vm_page dummy;
779 vm_page_t lefttreemax, righttreemin, y;
780
781 if (root == NULL)
782 return (root);
783 lefttreemax = righttreemin = &dummy;
784 for (;; root = y) {
785 if (pindex < root->pindex) {
786 if ((y = root->left) == NULL)
787 break;
788 if (pindex < y->pindex) {
789 /* Rotate right. */
790 root->left = y->right;
791 y->right = root;
792 root = y;
793 if ((y = root->left) == NULL)
794 break;
795 }
796 /* Link into the new root's right tree. */
797 righttreemin->left = root;
798 righttreemin = root;
799 } else if (pindex > root->pindex) {
800 if ((y = root->right) == NULL)
801 break;
802 if (pindex > y->pindex) {
803 /* Rotate left. */
804 root->right = y->left;
805 y->left = root;
806 root = y;
807 if ((y = root->right) == NULL)
808 break;
809 }
810 /* Link into the new root's left tree. */
811 lefttreemax->right = root;
812 lefttreemax = root;
813 } else
814 break;
815 }
816 /* Assemble the new root. */
817 lefttreemax->right = root->left;
818 righttreemin->left = root->right;
819 root->left = dummy.right;
820 root->right = dummy.left;
821 return (root);
822}
823
824/*
825 * vm_page_insert: [ internal use only ]
826 *
827 * Inserts the given mem entry into the object and object list.
828 *
829 * The pagetables are not updated but will presumably fault the page
830 * in if necessary, or if a kernel page the caller will at some point
831 * enter the page into the kernel's pmap. We are not allowed to block
832 * here so we *can't* do this anyway.
833 *
834 * The object and page must be locked.
835 * This routine may not block.
836 */
837void
838vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
839{
840 vm_page_t root;
841
842 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
843 if (m->object != NULL)
844 panic("vm_page_insert: page already inserted");
845
846 /*
847 * Record the object/offset pair in this page
848 */
849 m->object = object;
850 m->pindex = pindex;
851
852 /*
853 * Now link into the object's ordered list of backed pages.
854 */
855 root = object->root;
856 if (root == NULL) {
857 m->left = NULL;
858 m->right = NULL;
859 TAILQ_INSERT_TAIL(&object->memq, m, listq);
860 } else {
861 root = vm_page_splay(pindex, root);
862 if (pindex < root->pindex) {
863 m->left = root->left;
864 m->right = root;
865 root->left = NULL;
866 TAILQ_INSERT_BEFORE(root, m, listq);
867 } else if (pindex == root->pindex)
868 panic("vm_page_insert: offset already allocated");
869 else {
870 m->right = root->right;
871 m->left = root;
872 root->right = NULL;
873 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
874 }
875 }
876 object->root = m;
877
878 /*
879 * show that the object has one more resident page.
880 */
881 object->resident_page_count++;
882 /*
883 * Hold the vnode until the last page is released.
884 */
885 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
886 vhold((struct vnode *)object->handle);
887
888 /*
889 * Since we are inserting a new and possibly dirty page,
890 * update the object's OBJ_MIGHTBEDIRTY flag.
891 */
892 if (m->aflags & PGA_WRITEABLE)
893 vm_object_set_writeable_dirty(object);
894}
895
896/*
897 * vm_page_remove:
898 * NOTE: used by device pager as well -wfj
899 *
900 * Removes the given mem entry from the object/offset-page
901 * table and the object page list, but do not invalidate/terminate
902 * the backing store.
903 *
904 * The object and page must be locked.
905 * The underlying pmap entry (if any) is NOT removed here.
906 * This routine may not block.
907 */
908void
909vm_page_remove(vm_page_t m)
910{
911 vm_object_t object;
912 vm_page_t next, prev, root;
913
914 if ((m->oflags & VPO_UNMANAGED) == 0)
915 vm_page_lock_assert(m, MA_OWNED);
916 if ((object = m->object) == NULL)
917 return;
918 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
919 if (m->oflags & VPO_BUSY) {
920 m->oflags &= ~VPO_BUSY;
921 vm_page_flash(m);
922 }
923
924 /*
925 * Now remove from the object's list of backed pages.
926 */
927 if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
928 /*
929 * Since the page's successor in the list is also its parent
930 * in the tree, its right subtree must be empty.
931 */
932 next->left = m->left;
933 KASSERT(m->right == NULL,
934 ("vm_page_remove: page %p has right child", m));
935 } else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
936 prev->right == m) {
937 /*
938 * Since the page's predecessor in the list is also its parent
939 * in the tree, its left subtree must be empty.
940 */
941 KASSERT(m->left == NULL,
942 ("vm_page_remove: page %p has left child", m));
943 prev->right = m->right;
944 } else {
945 if (m != object->root)
946 vm_page_splay(m->pindex, object->root);
947 if (m->left == NULL)
948 root = m->right;
949 else if (m->right == NULL)
950 root = m->left;
951 else {
952 /*
953 * Move the page's successor to the root, because
954 * pages are usually removed in ascending order.
955 */
956 if (m->right != next)
957 vm_page_splay(m->pindex, m->right);
958 next->left = m->left;
959 root = next;
960 }
961 object->root = root;
962 }
963 TAILQ_REMOVE(&object->memq, m, listq);
964
965 /*
966 * And show that the object has one fewer resident page.
967 */
968 object->resident_page_count--;
969 /*
970 * The vnode may now be recycled.
971 */
972 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
973 vdrop((struct vnode *)object->handle);
974
975 m->object = NULL;
976}
977
978/*
979 * vm_page_lookup:
980 *
981 * Returns the page associated with the object/offset
982 * pair specified; if none is found, NULL is returned.
983 *
984 * The object must be locked.
985 * This routine may not block.
986 * This is a critical path routine
987 */
988vm_page_t
989vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
990{
991 vm_page_t m;
992
993 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
994 if ((m = object->root) != NULL && m->pindex != pindex) {
995 m = vm_page_splay(pindex, m);
996 if ((object->root = m)->pindex != pindex)
997 m = NULL;
998 }
999 return (m);
1000}
1001
1002/*
1003 * vm_page_find_least:
1004 *
1005 * Returns the page associated with the object with least pindex
1006 * greater than or equal to the parameter pindex, or NULL.
1007 *
1008 * The object must be locked.
1009 * The routine may not block.
1010 */
1011vm_page_t
1012vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
1013{
1014 vm_page_t m;
1015
1016 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1017 if ((m = TAILQ_FIRST(&object->memq)) != NULL) {
1018 if (m->pindex < pindex) {
1019 m = vm_page_splay(pindex, object->root);
1020 if ((object->root = m)->pindex < pindex)
1021 m = TAILQ_NEXT(m, listq);
1022 }
1023 }
1024 return (m);
1025}
1026
1027/*
1028 * Returns the given page's successor (by pindex) within the object if it is
1029 * resident; if none is found, NULL is returned.
1030 *
1031 * The object must be locked.
1032 */
1033vm_page_t
1034vm_page_next(vm_page_t m)
1035{
1036 vm_page_t next;
1037
1038 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1039 if ((next = TAILQ_NEXT(m, listq)) != NULL &&
1040 next->pindex != m->pindex + 1)
1041 next = NULL;
1042 return (next);
1043}
1044
1045/*
1046 * Returns the given page's predecessor (by pindex) within the object if it is
1047 * resident; if none is found, NULL is returned.
1048 *
1049 * The object must be locked.
1050 */
1051vm_page_t
1052vm_page_prev(vm_page_t m)
1053{
1054 vm_page_t prev;
1055
1056 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1057 if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
1058 prev->pindex != m->pindex - 1)
1059 prev = NULL;
1060 return (prev);
1061}
1062
1063/*
1064 * vm_page_rename:
1065 *
1066 * Move the given memory entry from its
1067 * current object to the specified target object/offset.
1068 *
1069 * The object must be locked.
1070 * This routine may not block.
1071 *
1072 * Note: swap associated with the page must be invalidated by the move. We
1073 * have to do this for several reasons: (1) we aren't freeing the
1074 * page, (2) we are dirtying the page, (3) the VM system is probably
1075 * moving the page from object A to B, and will then later move
1076 * the backing store from A to B and we can't have a conflict.
1077 *
1078 * Note: we *always* dirty the page. It is necessary both for the
1079 * fact that we moved it, and because we may be invalidating
1080 * swap. If the page is on the cache, we have to deactivate it
1081 * or vm_page_dirty() will panic. Dirty pages are not allowed
1082 * on the cache.
1083 */
1084void
1085vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
1086{
1087
1088 vm_page_remove(m);
1089 vm_page_insert(m, new_object, new_pindex);
1090 vm_page_dirty(m);
1091}
1092
1093/*
1094 * Convert all of the given object's cached pages that have a
1095 * pindex within the given range into free pages. If the value
1096 * zero is given for "end", then the range's upper bound is
1097 * infinity. If the given object is backed by a vnode and it
1098 * transitions from having one or more cached pages to none, the
1099 * vnode's hold count is reduced.
1100 */
1101void
1102vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1103{
1104 vm_page_t m, m_next;
1105 boolean_t empty;
1106
1107 mtx_lock(&vm_page_queue_free_mtx);
1108 if (__predict_false(object->cache == NULL)) {
1109 mtx_unlock(&vm_page_queue_free_mtx);
1110 return;
1111 }
1112 m = object->cache = vm_page_splay(start, object->cache);
1113 if (m->pindex < start) {
1114 if (m->right == NULL)
1115 m = NULL;
1116 else {
1117 m_next = vm_page_splay(start, m->right);
1118 m_next->left = m;
1119 m->right = NULL;
1120 m = object->cache = m_next;
1121 }
1122 }
1123
1124 /*
1125 * At this point, "m" is either (1) a reference to the page
1126 * with the least pindex that is greater than or equal to
1127 * "start" or (2) NULL.
1128 */
1129 for (; m != NULL && (m->pindex < end || end == 0); m = m_next) {
1130 /*
1131 * Find "m"'s successor and remove "m" from the
1132 * object's cache.
1133 */
1134 if (m->right == NULL) {
1135 object->cache = m->left;
1136 m_next = NULL;
1137 } else {
1138 m_next = vm_page_splay(start, m->right);
1139 m_next->left = m->left;
1140 object->cache = m_next;
1141 }
1142 /* Convert "m" to a free page. */
1143 m->object = NULL;
1144 m->valid = 0;
1145 /* Clear PG_CACHED and set PG_FREE. */
1146 m->flags ^= PG_CACHED | PG_FREE;
1147 KASSERT((m->flags & (PG_CACHED | PG_FREE)) == PG_FREE,
1148 ("vm_page_cache_free: page %p has inconsistent flags", m));
1149 cnt.v_cache_count--;
1150 cnt.v_free_count++;
1151 }
1152 empty = object->cache == NULL;
1153 mtx_unlock(&vm_page_queue_free_mtx);
1154 if (object->type == OBJT_VNODE && empty)
1155 vdrop(object->handle);
1156}
1157
1158/*
1159 * Returns the cached page that is associated with the given
1160 * object and offset. If, however, none exists, returns NULL.
1161 *
1162 * The free page queue must be locked.
1163 */
1164static inline vm_page_t
1165vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex)
1166{
1167 vm_page_t m;
1168
1169 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1170 if ((m = object->cache) != NULL && m->pindex != pindex) {
1171 m = vm_page_splay(pindex, m);
1172 if ((object->cache = m)->pindex != pindex)
1173 m = NULL;
1174 }
1175 return (m);
1176}
1177
1178/*
1179 * Remove the given cached page from its containing object's
1180 * collection of cached pages.
1181 *
1182 * The free page queue must be locked.
1183 */
1184void
1185vm_page_cache_remove(vm_page_t m)
1186{
1187 vm_object_t object;
1188 vm_page_t root;
1189
1190 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1191 KASSERT((m->flags & PG_CACHED) != 0,
1192 ("vm_page_cache_remove: page %p is not cached", m));
1193 object = m->object;
1194 if (m != object->cache) {
1195 root = vm_page_splay(m->pindex, object->cache);
1196 KASSERT(root == m,
1197 ("vm_page_cache_remove: page %p is not cached in object %p",
1198 m, object));
1199 }
1200 if (m->left == NULL)
1201 root = m->right;
1202 else if (m->right == NULL)
1203 root = m->left;
1204 else {
1205 root = vm_page_splay(m->pindex, m->left);
1206 root->right = m->right;
1207 }
1208 object->cache = root;
1209 m->object = NULL;
1210 cnt.v_cache_count--;
1211}
1212
1213/*
1214 * Transfer all of the cached pages with offset greater than or
1215 * equal to 'offidxstart' from the original object's cache to the
1216 * new object's cache. However, any cached pages with offset
1217 * greater than or equal to the new object's size are kept in the
1218 * original object. Initially, the new object's cache must be
1219 * empty. Offset 'offidxstart' in the original object must
1220 * correspond to offset zero in the new object.
1221 *
1222 * The new object must be locked.
1223 */
1224void
1225vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
1226 vm_object_t new_object)
1227{
1228 vm_page_t m, m_next;
1229
1230 /*
1231 * Insertion into an object's collection of cached pages
1232 * requires the object to be locked. In contrast, removal does
1233 * not.
1234 */
1235 VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED);
1236 KASSERT(new_object->cache == NULL,
1237 ("vm_page_cache_transfer: object %p has cached pages",
1238 new_object));
1239 mtx_lock(&vm_page_queue_free_mtx);
1240 if ((m = orig_object->cache) != NULL) {
1241 /*
1242 * Transfer all of the pages with offset greater than or
1243 * equal to 'offidxstart' from the original object's
1244 * cache to the new object's cache.
1245 */
1246 m = vm_page_splay(offidxstart, m);
1247 if (m->pindex < offidxstart) {
1248 orig_object->cache = m;
1249 new_object->cache = m->right;
1250 m->right = NULL;
1251 } else {
1252 orig_object->cache = m->left;
1253 new_object->cache = m;
1254 m->left = NULL;
1255 }
1256 while ((m = new_object->cache) != NULL) {
1257 if ((m->pindex - offidxstart) >= new_object->size) {
1258 /*
1259 * Return all of the cached pages with
1260 * offset greater than or equal to the
1261 * new object's size to the original
1262 * object's cache.
1263 */
1264 new_object->cache = m->left;
1265 m->left = orig_object->cache;
1266 orig_object->cache = m;
1267 break;
1268 }
1269 m_next = vm_page_splay(m->pindex, m->right);
1270 /* Update the page's object and offset. */
1271 m->object = new_object;
1272 m->pindex -= offidxstart;
1273 if (m_next == NULL)
1274 break;
1275 m->right = NULL;
1276 m_next->left = m;
1277 new_object->cache = m_next;
1278 }
1279 KASSERT(new_object->cache == NULL ||
1280 new_object->type == OBJT_SWAP,
1281 ("vm_page_cache_transfer: object %p's type is incompatible"
1282 " with cached pages", new_object));
1283 }
1284 mtx_unlock(&vm_page_queue_free_mtx);
1285}
1286
1287/*
1288 * vm_page_alloc:
1289 *
1290 * Allocate and return a page that is associated with the specified
1291 * object and offset pair. By default, this page has the flag VPO_BUSY
1292 * set.
1293 *
1294 * The caller must always specify an allocation class.
1295 *
1296 * allocation classes:
1297 * VM_ALLOC_NORMAL normal process request
1298 * VM_ALLOC_SYSTEM system *really* needs a page
1299 * VM_ALLOC_INTERRUPT interrupt time request
1300 *
1301 * optional allocation flags:
1302 * VM_ALLOC_COUNT(number) the number of additional pages that the caller
1303 * intends to allocate
1304 * VM_ALLOC_IFCACHED return page only if it is cached
1305 * VM_ALLOC_IFNOTCACHED return NULL, do not reactivate if the page
1306 * is cached
1307 * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page
| 81
82#include "opt_vm.h"
83
84#include <sys/param.h>
85#include <sys/systm.h>
86#include <sys/lock.h>
87#include <sys/kernel.h>
88#include <sys/limits.h>
89#include <sys/malloc.h>
90#include <sys/msgbuf.h>
91#include <sys/mutex.h>
92#include <sys/proc.h>
93#include <sys/sysctl.h>
94#include <sys/vmmeter.h>
95#include <sys/vnode.h>
96
97#include <vm/vm.h>
98#include <vm/pmap.h>
99#include <vm/vm_param.h>
100#include <vm/vm_kern.h>
101#include <vm/vm_object.h>
102#include <vm/vm_page.h>
103#include <vm/vm_pageout.h>
104#include <vm/vm_pager.h>
105#include <vm/vm_phys.h>
106#include <vm/vm_reserv.h>
107#include <vm/vm_extern.h>
108#include <vm/uma.h>
109#include <vm/uma_int.h>
110
111#include <machine/md_var.h>
112
113/*
114 * Associated with page of user-allocatable memory is a
115 * page structure.
116 */
117
118struct vpgqueues vm_page_queues[PQ_COUNT];
119struct vpglocks vm_page_queue_lock;
120struct vpglocks vm_page_queue_free_lock;
121
122struct vpglocks pa_lock[PA_LOCK_COUNT];
123
124vm_page_t vm_page_array = 0;
125int vm_page_array_size = 0;
126long first_page = 0;
127int vm_page_zero_count = 0;
128
129static int boot_pages = UMA_BOOT_PAGES;
130TUNABLE_INT("vm.boot_pages", &boot_pages);
131SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
132 "number of pages allocated for bootstrapping the VM system");
133
134static int pa_tryrelock_restart;
135SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFLAG_RD,
136 &pa_tryrelock_restart, 0, "Number of tryrelock restarts");
137
138static uma_zone_t fakepg_zone;
139
140static struct vnode *vm_page_alloc_init(vm_page_t m);
141static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits);
142static void vm_page_queue_remove(int queue, vm_page_t m);
143static void vm_page_enqueue(int queue, vm_page_t m);
144static void vm_page_init_fakepg(void *dummy);
145
146SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL);
147
148static void
149vm_page_init_fakepg(void *dummy)
150{
151
152 fakepg_zone = uma_zcreate("fakepg", sizeof(struct vm_page), NULL, NULL,
153 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE | UMA_ZONE_VM);
154}
155
156/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
157#if PAGE_SIZE == 32768
158#ifdef CTASSERT
159CTASSERT(sizeof(u_long) >= 8);
160#endif
161#endif
162
163/*
164 * Try to acquire a physical address lock while a pmap is locked. If we
165 * fail to trylock we unlock and lock the pmap directly and cache the
166 * locked pa in *locked. The caller should then restart their loop in case
167 * the virtual to physical mapping has changed.
168 */
169int
170vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked)
171{
172 vm_paddr_t lockpa;
173
174 lockpa = *locked;
175 *locked = pa;
176 if (lockpa) {
177 PA_LOCK_ASSERT(lockpa, MA_OWNED);
178 if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa))
179 return (0);
180 PA_UNLOCK(lockpa);
181 }
182 if (PA_TRYLOCK(pa))
183 return (0);
184 PMAP_UNLOCK(pmap);
185 atomic_add_int(&pa_tryrelock_restart, 1);
186 PA_LOCK(pa);
187 PMAP_LOCK(pmap);
188 return (EAGAIN);
189}
190
191/*
192 * vm_set_page_size:
193 *
194 * Sets the page size, perhaps based upon the memory
195 * size. Must be called before any use of page-size
196 * dependent functions.
197 */
198void
199vm_set_page_size(void)
200{
201 if (cnt.v_page_size == 0)
202 cnt.v_page_size = PAGE_SIZE;
203 if (((cnt.v_page_size - 1) & cnt.v_page_size) != 0)
204 panic("vm_set_page_size: page size not a power of two");
205}
206
207/*
208 * vm_page_blacklist_lookup:
209 *
210 * See if a physical address in this page has been listed
211 * in the blacklist tunable. Entries in the tunable are
212 * separated by spaces or commas. If an invalid integer is
213 * encountered then the rest of the string is skipped.
214 */
215static int
216vm_page_blacklist_lookup(char *list, vm_paddr_t pa)
217{
218 vm_paddr_t bad;
219 char *cp, *pos;
220
221 for (pos = list; *pos != '\0'; pos = cp) {
222 bad = strtoq(pos, &cp, 0);
223 if (*cp != '\0') {
224 if (*cp == ' ' || *cp == ',') {
225 cp++;
226 if (cp == pos)
227 continue;
228 } else
229 break;
230 }
231 if (pa == trunc_page(bad))
232 return (1);
233 }
234 return (0);
235}
236
237/*
238 * vm_page_startup:
239 *
240 * Initializes the resident memory module.
241 *
242 * Allocates memory for the page cells, and
243 * for the object/offset-to-page hash table headers.
244 * Each page cell is initialized and placed on the free list.
245 */
246vm_offset_t
247vm_page_startup(vm_offset_t vaddr)
248{
249 vm_offset_t mapped;
250 vm_paddr_t page_range;
251 vm_paddr_t new_end;
252 int i;
253 vm_paddr_t pa;
254 vm_paddr_t last_pa;
255 char *list;
256
257 /* the biggest memory array is the second group of pages */
258 vm_paddr_t end;
259 vm_paddr_t biggestsize;
260 vm_paddr_t low_water, high_water;
261 int biggestone;
262
263 biggestsize = 0;
264 biggestone = 0;
265 vaddr = round_page(vaddr);
266
267 for (i = 0; phys_avail[i + 1]; i += 2) {
268 phys_avail[i] = round_page(phys_avail[i]);
269 phys_avail[i + 1] = trunc_page(phys_avail[i + 1]);
270 }
271
272 low_water = phys_avail[0];
273 high_water = phys_avail[1];
274
275 for (i = 0; phys_avail[i + 1]; i += 2) {
276 vm_paddr_t size = phys_avail[i + 1] - phys_avail[i];
277
278 if (size > biggestsize) {
279 biggestone = i;
280 biggestsize = size;
281 }
282 if (phys_avail[i] < low_water)
283 low_water = phys_avail[i];
284 if (phys_avail[i + 1] > high_water)
285 high_water = phys_avail[i + 1];
286 }
287
288#ifdef XEN
289 low_water = 0;
290#endif
291
292 end = phys_avail[biggestone+1];
293
294 /*
295 * Initialize the locks.
296 */
297 mtx_init(&vm_page_queue_mtx, "vm page queue mutex", NULL, MTX_DEF |
298 MTX_RECURSE);
299 mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL,
300 MTX_DEF);
301
302 /* Setup page locks. */
303 for (i = 0; i < PA_LOCK_COUNT; i++)
304 mtx_init(&pa_lock[i].data, "page lock", NULL, MTX_DEF);
305
306 /*
307 * Initialize the queue headers for the hold queue, the active queue,
308 * and the inactive queue.
309 */
310 for (i = 0; i < PQ_COUNT; i++)
311 TAILQ_INIT(&vm_page_queues[i].pl);
312 vm_page_queues[PQ_INACTIVE].cnt = &cnt.v_inactive_count;
313 vm_page_queues[PQ_ACTIVE].cnt = &cnt.v_active_count;
314 vm_page_queues[PQ_HOLD].cnt = &cnt.v_active_count;
315
316 /*
317 * Allocate memory for use when boot strapping the kernel memory
318 * allocator.
319 */
320 new_end = end - (boot_pages * UMA_SLAB_SIZE);
321 new_end = trunc_page(new_end);
322 mapped = pmap_map(&vaddr, new_end, end,
323 VM_PROT_READ | VM_PROT_WRITE);
324 bzero((void *)mapped, end - new_end);
325 uma_startup((void *)mapped, boot_pages);
326
327#if defined(__amd64__) || defined(__i386__) || defined(__arm__) || \
328 defined(__mips__)
329 /*
330 * Allocate a bitmap to indicate that a random physical page
331 * needs to be included in a minidump.
332 *
333 * The amd64 port needs this to indicate which direct map pages
334 * need to be dumped, via calls to dump_add_page()/dump_drop_page().
335 *
336 * However, i386 still needs this workspace internally within the
337 * minidump code. In theory, they are not needed on i386, but are
338 * included should the sf_buf code decide to use them.
339 */
340 last_pa = 0;
341 for (i = 0; dump_avail[i + 1] != 0; i += 2)
342 if (dump_avail[i + 1] > last_pa)
343 last_pa = dump_avail[i + 1];
344 page_range = last_pa / PAGE_SIZE;
345 vm_page_dump_size = round_page(roundup2(page_range, NBBY) / NBBY);
346 new_end -= vm_page_dump_size;
347 vm_page_dump = (void *)(uintptr_t)pmap_map(&vaddr, new_end,
348 new_end + vm_page_dump_size, VM_PROT_READ | VM_PROT_WRITE);
349 bzero((void *)vm_page_dump, vm_page_dump_size);
350#endif
351#ifdef __amd64__
352 /*
353 * Request that the physical pages underlying the message buffer be
354 * included in a crash dump. Since the message buffer is accessed
355 * through the direct map, they are not automatically included.
356 */
357 pa = DMAP_TO_PHYS((vm_offset_t)msgbufp->msg_ptr);
358 last_pa = pa + round_page(msgbufsize);
359 while (pa < last_pa) {
360 dump_add_page(pa);
361 pa += PAGE_SIZE;
362 }
363#endif
364 /*
365 * Compute the number of pages of memory that will be available for
366 * use (taking into account the overhead of a page structure per
367 * page).
368 */
369 first_page = low_water / PAGE_SIZE;
370#ifdef VM_PHYSSEG_SPARSE
371 page_range = 0;
372 for (i = 0; phys_avail[i + 1] != 0; i += 2)
373 page_range += atop(phys_avail[i + 1] - phys_avail[i]);
374#elif defined(VM_PHYSSEG_DENSE)
375 page_range = high_water / PAGE_SIZE - first_page;
376#else
377#error "Either VM_PHYSSEG_DENSE or VM_PHYSSEG_SPARSE must be defined."
378#endif
379 end = new_end;
380
381 /*
382 * Reserve an unmapped guard page to trap access to vm_page_array[-1].
383 */
384 vaddr += PAGE_SIZE;
385
386 /*
387 * Initialize the mem entry structures now, and put them in the free
388 * queue.
389 */
390 new_end = trunc_page(end - page_range * sizeof(struct vm_page));
391 mapped = pmap_map(&vaddr, new_end, end,
392 VM_PROT_READ | VM_PROT_WRITE);
393 vm_page_array = (vm_page_t) mapped;
394#if VM_NRESERVLEVEL > 0
395 /*
396 * Allocate memory for the reservation management system's data
397 * structures.
398 */
399 new_end = vm_reserv_startup(&vaddr, new_end, high_water);
400#endif
401#if defined(__amd64__) || defined(__mips__)
402 /*
403 * pmap_map on amd64 and mips can come out of the direct-map, not kvm
404 * like i386, so the pages must be tracked for a crashdump to include
405 * this data. This includes the vm_page_array and the early UMA
406 * bootstrap pages.
407 */
408 for (pa = new_end; pa < phys_avail[biggestone + 1]; pa += PAGE_SIZE)
409 dump_add_page(pa);
410#endif
411 phys_avail[biggestone + 1] = new_end;
412
413 /*
414 * Clear all of the page structures
415 */
416 bzero((caddr_t) vm_page_array, page_range * sizeof(struct vm_page));
417 for (i = 0; i < page_range; i++)
418 vm_page_array[i].order = VM_NFREEORDER;
419 vm_page_array_size = page_range;
420
421 /*
422 * Initialize the physical memory allocator.
423 */
424 vm_phys_init();
425
426 /*
427 * Add every available physical page that is not blacklisted to
428 * the free lists.
429 */
430 cnt.v_page_count = 0;
431 cnt.v_free_count = 0;
432 list = getenv("vm.blacklist");
433 for (i = 0; phys_avail[i + 1] != 0; i += 2) {
434 pa = phys_avail[i];
435 last_pa = phys_avail[i + 1];
436 while (pa < last_pa) {
437 if (list != NULL &&
438 vm_page_blacklist_lookup(list, pa))
439 printf("Skipping page with pa 0x%jx\n",
440 (uintmax_t)pa);
441 else
442 vm_phys_add_page(pa);
443 pa += PAGE_SIZE;
444 }
445 }
446 freeenv(list);
447#if VM_NRESERVLEVEL > 0
448 /*
449 * Initialize the reservation management system.
450 */
451 vm_reserv_init();
452#endif
453 return (vaddr);
454}
455
456
457CTASSERT(offsetof(struct vm_page, aflags) % sizeof(uint32_t) == 0);
458
459void
460vm_page_aflag_set(vm_page_t m, uint8_t bits)
461{
462 uint32_t *addr, val;
463
464 /*
465 * The PGA_WRITEABLE flag can only be set if the page is managed and
466 * VPO_BUSY. Currently, this flag is only set by pmap_enter().
467 */
468 KASSERT((bits & PGA_WRITEABLE) == 0 ||
469 (m->oflags & (VPO_UNMANAGED | VPO_BUSY)) == VPO_BUSY,
470 ("PGA_WRITEABLE and !VPO_BUSY"));
471
472 /*
473 * We want to use atomic updates for m->aflags, which is a
474 * byte wide. Not all architectures provide atomic operations
475 * on the single-byte destination. Punt and access the whole
476 * 4-byte word with an atomic update. Parallel non-atomic
477 * updates to the fields included in the update by proximity
478 * are handled properly by atomics.
479 */
480 addr = (void *)&m->aflags;
481 MPASS(((uintptr_t)addr & (sizeof(uint32_t) - 1)) == 0);
482 val = bits;
483#if BYTE_ORDER == BIG_ENDIAN
484 val <<= 24;
485#endif
486 atomic_set_32(addr, val);
487}
488
489void
490vm_page_aflag_clear(vm_page_t m, uint8_t bits)
491{
492 uint32_t *addr, val;
493
494 /*
495 * The PGA_REFERENCED flag can only be cleared if the object
496 * containing the page is locked.
497 */
498 KASSERT((bits & PGA_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object),
499 ("PGA_REFERENCED and !VM_OBJECT_LOCKED"));
500
501 /*
502 * See the comment in vm_page_aflag_set().
503 */
504 addr = (void *)&m->aflags;
505 MPASS(((uintptr_t)addr & (sizeof(uint32_t) - 1)) == 0);
506 val = bits;
507#if BYTE_ORDER == BIG_ENDIAN
508 val <<= 24;
509#endif
510 atomic_clear_32(addr, val);
511}
512
513void
514vm_page_reference(vm_page_t m)
515{
516
517 vm_page_aflag_set(m, PGA_REFERENCED);
518}
519
520void
521vm_page_busy(vm_page_t m)
522{
523
524 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
525 KASSERT((m->oflags & VPO_BUSY) == 0,
526 ("vm_page_busy: page already busy!!!"));
527 m->oflags |= VPO_BUSY;
528}
529
530/*
531 * vm_page_flash:
532 *
533 * wakeup anyone waiting for the page.
534 */
535void
536vm_page_flash(vm_page_t m)
537{
538
539 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
540 if (m->oflags & VPO_WANTED) {
541 m->oflags &= ~VPO_WANTED;
542 wakeup(m);
543 }
544}
545
546/*
547 * vm_page_wakeup:
548 *
549 * clear the VPO_BUSY flag and wakeup anyone waiting for the
550 * page.
551 *
552 */
553void
554vm_page_wakeup(vm_page_t m)
555{
556
557 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
558 KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!"));
559 m->oflags &= ~VPO_BUSY;
560 vm_page_flash(m);
561}
562
563void
564vm_page_io_start(vm_page_t m)
565{
566
567 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
568 m->busy++;
569}
570
571void
572vm_page_io_finish(vm_page_t m)
573{
574
575 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
576 KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m));
577 m->busy--;
578 if (m->busy == 0)
579 vm_page_flash(m);
580}
581
582/*
583 * Keep page from being freed by the page daemon
584 * much of the same effect as wiring, except much lower
585 * overhead and should be used only for *very* temporary
586 * holding ("wiring").
587 */
588void
589vm_page_hold(vm_page_t mem)
590{
591
592 vm_page_lock_assert(mem, MA_OWNED);
593 mem->hold_count++;
594}
595
596void
597vm_page_unhold(vm_page_t mem)
598{
599
600 vm_page_lock_assert(mem, MA_OWNED);
601 --mem->hold_count;
602 KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!"));
603 if (mem->hold_count == 0 && mem->queue == PQ_HOLD)
604 vm_page_free_toq(mem);
605}
606
607/*
608 * vm_page_unhold_pages:
609 *
610 * Unhold each of the pages that is referenced by the given array.
611 */
612void
613vm_page_unhold_pages(vm_page_t *ma, int count)
614{
615 struct mtx *mtx, *new_mtx;
616
617 mtx = NULL;
618 for (; count != 0; count--) {
619 /*
620 * Avoid releasing and reacquiring the same page lock.
621 */
622 new_mtx = vm_page_lockptr(*ma);
623 if (mtx != new_mtx) {
624 if (mtx != NULL)
625 mtx_unlock(mtx);
626 mtx = new_mtx;
627 mtx_lock(mtx);
628 }
629 vm_page_unhold(*ma);
630 ma++;
631 }
632 if (mtx != NULL)
633 mtx_unlock(mtx);
634}
635
636/*
637 * vm_page_getfake:
638 *
639 * Create a fictitious page with the specified physical address and
640 * memory attribute. The memory attribute is the only the machine-
641 * dependent aspect of a fictitious page that must be initialized.
642 */
643vm_page_t
644vm_page_getfake(vm_paddr_t paddr, vm_memattr_t memattr)
645{
646 vm_page_t m;
647
648 m = uma_zalloc(fakepg_zone, M_WAITOK | M_ZERO);
649 m->phys_addr = paddr;
650 m->queue = PQ_NONE;
651 /* Fictitious pages don't use "segind". */
652 m->flags = PG_FICTITIOUS;
653 /* Fictitious pages don't use "order" or "pool". */
654 m->oflags = VPO_BUSY | VPO_UNMANAGED;
655 m->wire_count = 1;
656 pmap_page_set_memattr(m, memattr);
657 return (m);
658}
659
660/*
661 * vm_page_putfake:
662 *
663 * Release a fictitious page.
664 */
665void
666vm_page_putfake(vm_page_t m)
667{
668
669 KASSERT((m->flags & PG_FICTITIOUS) != 0,
670 ("vm_page_putfake: bad page %p", m));
671 uma_zfree(fakepg_zone, m);
672}
673
674/*
675 * vm_page_updatefake:
676 *
677 * Update the given fictitious page to the specified physical address and
678 * memory attribute.
679 */
680void
681vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr)
682{
683
684 KASSERT((m->flags & PG_FICTITIOUS) != 0,
685 ("vm_page_updatefake: bad page %p", m));
686 m->phys_addr = paddr;
687 pmap_page_set_memattr(m, memattr);
688}
689
690/*
691 * vm_page_free:
692 *
693 * Free a page.
694 */
695void
696vm_page_free(vm_page_t m)
697{
698
699 m->flags &= ~PG_ZERO;
700 vm_page_free_toq(m);
701}
702
703/*
704 * vm_page_free_zero:
705 *
706 * Free a page to the zerod-pages queue
707 */
708void
709vm_page_free_zero(vm_page_t m)
710{
711
712 m->flags |= PG_ZERO;
713 vm_page_free_toq(m);
714}
715
716/*
717 * vm_page_sleep:
718 *
719 * Sleep and release the page and page queues locks.
720 *
721 * The object containing the given page must be locked.
722 */
723void
724vm_page_sleep(vm_page_t m, const char *msg)
725{
726
727 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
728 if (mtx_owned(&vm_page_queue_mtx))
729 vm_page_unlock_queues();
730 if (mtx_owned(vm_page_lockptr(m)))
731 vm_page_unlock(m);
732
733 /*
734 * It's possible that while we sleep, the page will get
735 * unbusied and freed. If we are holding the object
736 * lock, we will assume we hold a reference to the object
737 * such that even if m->object changes, we can re-lock
738 * it.
739 */
740 m->oflags |= VPO_WANTED;
741 msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0);
742}
743
744/*
745 * vm_page_dirty:
746 *
747 * Set all bits in the page's dirty field.
748 *
749 * The object containing the specified page must be locked if the
750 * call is made from the machine-independent layer.
751 *
752 * See vm_page_clear_dirty_mask().
753 */
754void
755vm_page_dirty(vm_page_t m)
756{
757
758 KASSERT((m->flags & PG_CACHED) == 0,
759 ("vm_page_dirty: page in cache!"));
760 KASSERT(!VM_PAGE_IS_FREE(m),
761 ("vm_page_dirty: page is free!"));
762 KASSERT(m->valid == VM_PAGE_BITS_ALL,
763 ("vm_page_dirty: page is invalid!"));
764 m->dirty = VM_PAGE_BITS_ALL;
765}
766
767/*
768 * vm_page_splay:
769 *
770 * Implements Sleator and Tarjan's top-down splay algorithm. Returns
771 * the vm_page containing the given pindex. If, however, that
772 * pindex is not found in the vm_object, returns a vm_page that is
773 * adjacent to the pindex, coming before or after it.
774 */
775vm_page_t
776vm_page_splay(vm_pindex_t pindex, vm_page_t root)
777{
778 struct vm_page dummy;
779 vm_page_t lefttreemax, righttreemin, y;
780
781 if (root == NULL)
782 return (root);
783 lefttreemax = righttreemin = &dummy;
784 for (;; root = y) {
785 if (pindex < root->pindex) {
786 if ((y = root->left) == NULL)
787 break;
788 if (pindex < y->pindex) {
789 /* Rotate right. */
790 root->left = y->right;
791 y->right = root;
792 root = y;
793 if ((y = root->left) == NULL)
794 break;
795 }
796 /* Link into the new root's right tree. */
797 righttreemin->left = root;
798 righttreemin = root;
799 } else if (pindex > root->pindex) {
800 if ((y = root->right) == NULL)
801 break;
802 if (pindex > y->pindex) {
803 /* Rotate left. */
804 root->right = y->left;
805 y->left = root;
806 root = y;
807 if ((y = root->right) == NULL)
808 break;
809 }
810 /* Link into the new root's left tree. */
811 lefttreemax->right = root;
812 lefttreemax = root;
813 } else
814 break;
815 }
816 /* Assemble the new root. */
817 lefttreemax->right = root->left;
818 righttreemin->left = root->right;
819 root->left = dummy.right;
820 root->right = dummy.left;
821 return (root);
822}
823
824/*
825 * vm_page_insert: [ internal use only ]
826 *
827 * Inserts the given mem entry into the object and object list.
828 *
829 * The pagetables are not updated but will presumably fault the page
830 * in if necessary, or if a kernel page the caller will at some point
831 * enter the page into the kernel's pmap. We are not allowed to block
832 * here so we *can't* do this anyway.
833 *
834 * The object and page must be locked.
835 * This routine may not block.
836 */
837void
838vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex)
839{
840 vm_page_t root;
841
842 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
843 if (m->object != NULL)
844 panic("vm_page_insert: page already inserted");
845
846 /*
847 * Record the object/offset pair in this page
848 */
849 m->object = object;
850 m->pindex = pindex;
851
852 /*
853 * Now link into the object's ordered list of backed pages.
854 */
855 root = object->root;
856 if (root == NULL) {
857 m->left = NULL;
858 m->right = NULL;
859 TAILQ_INSERT_TAIL(&object->memq, m, listq);
860 } else {
861 root = vm_page_splay(pindex, root);
862 if (pindex < root->pindex) {
863 m->left = root->left;
864 m->right = root;
865 root->left = NULL;
866 TAILQ_INSERT_BEFORE(root, m, listq);
867 } else if (pindex == root->pindex)
868 panic("vm_page_insert: offset already allocated");
869 else {
870 m->right = root->right;
871 m->left = root;
872 root->right = NULL;
873 TAILQ_INSERT_AFTER(&object->memq, root, m, listq);
874 }
875 }
876 object->root = m;
877
878 /*
879 * show that the object has one more resident page.
880 */
881 object->resident_page_count++;
882 /*
883 * Hold the vnode until the last page is released.
884 */
885 if (object->resident_page_count == 1 && object->type == OBJT_VNODE)
886 vhold((struct vnode *)object->handle);
887
888 /*
889 * Since we are inserting a new and possibly dirty page,
890 * update the object's OBJ_MIGHTBEDIRTY flag.
891 */
892 if (m->aflags & PGA_WRITEABLE)
893 vm_object_set_writeable_dirty(object);
894}
895
896/*
897 * vm_page_remove:
898 * NOTE: used by device pager as well -wfj
899 *
900 * Removes the given mem entry from the object/offset-page
901 * table and the object page list, but do not invalidate/terminate
902 * the backing store.
903 *
904 * The object and page must be locked.
905 * The underlying pmap entry (if any) is NOT removed here.
906 * This routine may not block.
907 */
908void
909vm_page_remove(vm_page_t m)
910{
911 vm_object_t object;
912 vm_page_t next, prev, root;
913
914 if ((m->oflags & VPO_UNMANAGED) == 0)
915 vm_page_lock_assert(m, MA_OWNED);
916 if ((object = m->object) == NULL)
917 return;
918 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
919 if (m->oflags & VPO_BUSY) {
920 m->oflags &= ~VPO_BUSY;
921 vm_page_flash(m);
922 }
923
924 /*
925 * Now remove from the object's list of backed pages.
926 */
927 if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
928 /*
929 * Since the page's successor in the list is also its parent
930 * in the tree, its right subtree must be empty.
931 */
932 next->left = m->left;
933 KASSERT(m->right == NULL,
934 ("vm_page_remove: page %p has right child", m));
935 } else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
936 prev->right == m) {
937 /*
938 * Since the page's predecessor in the list is also its parent
939 * in the tree, its left subtree must be empty.
940 */
941 KASSERT(m->left == NULL,
942 ("vm_page_remove: page %p has left child", m));
943 prev->right = m->right;
944 } else {
945 if (m != object->root)
946 vm_page_splay(m->pindex, object->root);
947 if (m->left == NULL)
948 root = m->right;
949 else if (m->right == NULL)
950 root = m->left;
951 else {
952 /*
953 * Move the page's successor to the root, because
954 * pages are usually removed in ascending order.
955 */
956 if (m->right != next)
957 vm_page_splay(m->pindex, m->right);
958 next->left = m->left;
959 root = next;
960 }
961 object->root = root;
962 }
963 TAILQ_REMOVE(&object->memq, m, listq);
964
965 /*
966 * And show that the object has one fewer resident page.
967 */
968 object->resident_page_count--;
969 /*
970 * The vnode may now be recycled.
971 */
972 if (object->resident_page_count == 0 && object->type == OBJT_VNODE)
973 vdrop((struct vnode *)object->handle);
974
975 m->object = NULL;
976}
977
978/*
979 * vm_page_lookup:
980 *
981 * Returns the page associated with the object/offset
982 * pair specified; if none is found, NULL is returned.
983 *
984 * The object must be locked.
985 * This routine may not block.
986 * This is a critical path routine
987 */
988vm_page_t
989vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
990{
991 vm_page_t m;
992
993 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
994 if ((m = object->root) != NULL && m->pindex != pindex) {
995 m = vm_page_splay(pindex, m);
996 if ((object->root = m)->pindex != pindex)
997 m = NULL;
998 }
999 return (m);
1000}
1001
1002/*
1003 * vm_page_find_least:
1004 *
1005 * Returns the page associated with the object with least pindex
1006 * greater than or equal to the parameter pindex, or NULL.
1007 *
1008 * The object must be locked.
1009 * The routine may not block.
1010 */
1011vm_page_t
1012vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
1013{
1014 vm_page_t m;
1015
1016 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1017 if ((m = TAILQ_FIRST(&object->memq)) != NULL) {
1018 if (m->pindex < pindex) {
1019 m = vm_page_splay(pindex, object->root);
1020 if ((object->root = m)->pindex < pindex)
1021 m = TAILQ_NEXT(m, listq);
1022 }
1023 }
1024 return (m);
1025}
1026
1027/*
1028 * Returns the given page's successor (by pindex) within the object if it is
1029 * resident; if none is found, NULL is returned.
1030 *
1031 * The object must be locked.
1032 */
1033vm_page_t
1034vm_page_next(vm_page_t m)
1035{
1036 vm_page_t next;
1037
1038 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1039 if ((next = TAILQ_NEXT(m, listq)) != NULL &&
1040 next->pindex != m->pindex + 1)
1041 next = NULL;
1042 return (next);
1043}
1044
1045/*
1046 * Returns the given page's predecessor (by pindex) within the object if it is
1047 * resident; if none is found, NULL is returned.
1048 *
1049 * The object must be locked.
1050 */
1051vm_page_t
1052vm_page_prev(vm_page_t m)
1053{
1054 vm_page_t prev;
1055
1056 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1057 if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
1058 prev->pindex != m->pindex - 1)
1059 prev = NULL;
1060 return (prev);
1061}
1062
1063/*
1064 * vm_page_rename:
1065 *
1066 * Move the given memory entry from its
1067 * current object to the specified target object/offset.
1068 *
1069 * The object must be locked.
1070 * This routine may not block.
1071 *
1072 * Note: swap associated with the page must be invalidated by the move. We
1073 * have to do this for several reasons: (1) we aren't freeing the
1074 * page, (2) we are dirtying the page, (3) the VM system is probably
1075 * moving the page from object A to B, and will then later move
1076 * the backing store from A to B and we can't have a conflict.
1077 *
1078 * Note: we *always* dirty the page. It is necessary both for the
1079 * fact that we moved it, and because we may be invalidating
1080 * swap. If the page is on the cache, we have to deactivate it
1081 * or vm_page_dirty() will panic. Dirty pages are not allowed
1082 * on the cache.
1083 */
1084void
1085vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
1086{
1087
1088 vm_page_remove(m);
1089 vm_page_insert(m, new_object, new_pindex);
1090 vm_page_dirty(m);
1091}
1092
1093/*
1094 * Convert all of the given object's cached pages that have a
1095 * pindex within the given range into free pages. If the value
1096 * zero is given for "end", then the range's upper bound is
1097 * infinity. If the given object is backed by a vnode and it
1098 * transitions from having one or more cached pages to none, the
1099 * vnode's hold count is reduced.
1100 */
1101void
1102vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
1103{
1104 vm_page_t m, m_next;
1105 boolean_t empty;
1106
1107 mtx_lock(&vm_page_queue_free_mtx);
1108 if (__predict_false(object->cache == NULL)) {
1109 mtx_unlock(&vm_page_queue_free_mtx);
1110 return;
1111 }
1112 m = object->cache = vm_page_splay(start, object->cache);
1113 if (m->pindex < start) {
1114 if (m->right == NULL)
1115 m = NULL;
1116 else {
1117 m_next = vm_page_splay(start, m->right);
1118 m_next->left = m;
1119 m->right = NULL;
1120 m = object->cache = m_next;
1121 }
1122 }
1123
1124 /*
1125 * At this point, "m" is either (1) a reference to the page
1126 * with the least pindex that is greater than or equal to
1127 * "start" or (2) NULL.
1128 */
1129 for (; m != NULL && (m->pindex < end || end == 0); m = m_next) {
1130 /*
1131 * Find "m"'s successor and remove "m" from the
1132 * object's cache.
1133 */
1134 if (m->right == NULL) {
1135 object->cache = m->left;
1136 m_next = NULL;
1137 } else {
1138 m_next = vm_page_splay(start, m->right);
1139 m_next->left = m->left;
1140 object->cache = m_next;
1141 }
1142 /* Convert "m" to a free page. */
1143 m->object = NULL;
1144 m->valid = 0;
1145 /* Clear PG_CACHED and set PG_FREE. */
1146 m->flags ^= PG_CACHED | PG_FREE;
1147 KASSERT((m->flags & (PG_CACHED | PG_FREE)) == PG_FREE,
1148 ("vm_page_cache_free: page %p has inconsistent flags", m));
1149 cnt.v_cache_count--;
1150 cnt.v_free_count++;
1151 }
1152 empty = object->cache == NULL;
1153 mtx_unlock(&vm_page_queue_free_mtx);
1154 if (object->type == OBJT_VNODE && empty)
1155 vdrop(object->handle);
1156}
1157
1158/*
1159 * Returns the cached page that is associated with the given
1160 * object and offset. If, however, none exists, returns NULL.
1161 *
1162 * The free page queue must be locked.
1163 */
1164static inline vm_page_t
1165vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex)
1166{
1167 vm_page_t m;
1168
1169 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1170 if ((m = object->cache) != NULL && m->pindex != pindex) {
1171 m = vm_page_splay(pindex, m);
1172 if ((object->cache = m)->pindex != pindex)
1173 m = NULL;
1174 }
1175 return (m);
1176}
1177
1178/*
1179 * Remove the given cached page from its containing object's
1180 * collection of cached pages.
1181 *
1182 * The free page queue must be locked.
1183 */
1184void
1185vm_page_cache_remove(vm_page_t m)
1186{
1187 vm_object_t object;
1188 vm_page_t root;
1189
1190 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1191 KASSERT((m->flags & PG_CACHED) != 0,
1192 ("vm_page_cache_remove: page %p is not cached", m));
1193 object = m->object;
1194 if (m != object->cache) {
1195 root = vm_page_splay(m->pindex, object->cache);
1196 KASSERT(root == m,
1197 ("vm_page_cache_remove: page %p is not cached in object %p",
1198 m, object));
1199 }
1200 if (m->left == NULL)
1201 root = m->right;
1202 else if (m->right == NULL)
1203 root = m->left;
1204 else {
1205 root = vm_page_splay(m->pindex, m->left);
1206 root->right = m->right;
1207 }
1208 object->cache = root;
1209 m->object = NULL;
1210 cnt.v_cache_count--;
1211}
1212
1213/*
1214 * Transfer all of the cached pages with offset greater than or
1215 * equal to 'offidxstart' from the original object's cache to the
1216 * new object's cache. However, any cached pages with offset
1217 * greater than or equal to the new object's size are kept in the
1218 * original object. Initially, the new object's cache must be
1219 * empty. Offset 'offidxstart' in the original object must
1220 * correspond to offset zero in the new object.
1221 *
1222 * The new object must be locked.
1223 */
1224void
1225vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
1226 vm_object_t new_object)
1227{
1228 vm_page_t m, m_next;
1229
1230 /*
1231 * Insertion into an object's collection of cached pages
1232 * requires the object to be locked. In contrast, removal does
1233 * not.
1234 */
1235 VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED);
1236 KASSERT(new_object->cache == NULL,
1237 ("vm_page_cache_transfer: object %p has cached pages",
1238 new_object));
1239 mtx_lock(&vm_page_queue_free_mtx);
1240 if ((m = orig_object->cache) != NULL) {
1241 /*
1242 * Transfer all of the pages with offset greater than or
1243 * equal to 'offidxstart' from the original object's
1244 * cache to the new object's cache.
1245 */
1246 m = vm_page_splay(offidxstart, m);
1247 if (m->pindex < offidxstart) {
1248 orig_object->cache = m;
1249 new_object->cache = m->right;
1250 m->right = NULL;
1251 } else {
1252 orig_object->cache = m->left;
1253 new_object->cache = m;
1254 m->left = NULL;
1255 }
1256 while ((m = new_object->cache) != NULL) {
1257 if ((m->pindex - offidxstart) >= new_object->size) {
1258 /*
1259 * Return all of the cached pages with
1260 * offset greater than or equal to the
1261 * new object's size to the original
1262 * object's cache.
1263 */
1264 new_object->cache = m->left;
1265 m->left = orig_object->cache;
1266 orig_object->cache = m;
1267 break;
1268 }
1269 m_next = vm_page_splay(m->pindex, m->right);
1270 /* Update the page's object and offset. */
1271 m->object = new_object;
1272 m->pindex -= offidxstart;
1273 if (m_next == NULL)
1274 break;
1275 m->right = NULL;
1276 m_next->left = m;
1277 new_object->cache = m_next;
1278 }
1279 KASSERT(new_object->cache == NULL ||
1280 new_object->type == OBJT_SWAP,
1281 ("vm_page_cache_transfer: object %p's type is incompatible"
1282 " with cached pages", new_object));
1283 }
1284 mtx_unlock(&vm_page_queue_free_mtx);
1285}
1286
1287/*
1288 * vm_page_alloc:
1289 *
1290 * Allocate and return a page that is associated with the specified
1291 * object and offset pair. By default, this page has the flag VPO_BUSY
1292 * set.
1293 *
1294 * The caller must always specify an allocation class.
1295 *
1296 * allocation classes:
1297 * VM_ALLOC_NORMAL normal process request
1298 * VM_ALLOC_SYSTEM system *really* needs a page
1299 * VM_ALLOC_INTERRUPT interrupt time request
1300 *
1301 * optional allocation flags:
1302 * VM_ALLOC_COUNT(number) the number of additional pages that the caller
1303 * intends to allocate
1304 * VM_ALLOC_IFCACHED return page only if it is cached
1305 * VM_ALLOC_IFNOTCACHED return NULL, do not reactivate if the page
1306 * is cached
1307 * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page
|
| 1308 * VM_ALLOC_NODUMP do not include the page in a kernel core dump
|
1308 * VM_ALLOC_NOOBJ page is not associated with an object and
1309 * should not have the flag VPO_BUSY set
1310 * VM_ALLOC_WIRED wire the allocated page
1311 * VM_ALLOC_ZERO prefer a zeroed page
1312 *
1313 * This routine may not sleep.
1314 */
1315vm_page_t
1316vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
1317{
1318 struct vnode *vp = NULL;
1319 vm_object_t m_object;
1320 vm_page_t m;
1321 int flags, req_class;
1322
1323 KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
1324 ("vm_page_alloc: inconsistent object/req"));
1325 if (object != NULL)
1326 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1327
1328 req_class = req & VM_ALLOC_CLASS_MASK;
1329
1330 /*
1331 * The page daemon is allowed to dig deeper into the free page list.
1332 */
1333 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1334 req_class = VM_ALLOC_SYSTEM;
1335
1336 mtx_lock(&vm_page_queue_free_mtx);
1337 if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1338 (req_class == VM_ALLOC_SYSTEM &&
1339 cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1340 (req_class == VM_ALLOC_INTERRUPT &&
1341 cnt.v_free_count + cnt.v_cache_count > 0)) {
1342 /*
1343 * Allocate from the free queue if the number of free pages
1344 * exceeds the minimum for the request class.
1345 */
1346 if (object != NULL &&
1347 (m = vm_page_cache_lookup(object, pindex)) != NULL) {
1348 if ((req & VM_ALLOC_IFNOTCACHED) != 0) {
1349 mtx_unlock(&vm_page_queue_free_mtx);
1350 return (NULL);
1351 }
1352 if (vm_phys_unfree_page(m))
1353 vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, 0);
1354#if VM_NRESERVLEVEL > 0
1355 else if (!vm_reserv_reactivate_page(m))
1356#else
1357 else
1358#endif
1359 panic("vm_page_alloc: cache page %p is missing"
1360 " from the free queue", m);
1361 } else if ((req & VM_ALLOC_IFCACHED) != 0) {
1362 mtx_unlock(&vm_page_queue_free_mtx);
1363 return (NULL);
1364#if VM_NRESERVLEVEL > 0
1365 } else if (object == NULL || object->type == OBJT_DEVICE ||
1366 object->type == OBJT_SG ||
1367 (object->flags & OBJ_COLORED) == 0 ||
1368 (m = vm_reserv_alloc_page(object, pindex)) == NULL) {
1369#else
1370 } else {
1371#endif
1372 m = vm_phys_alloc_pages(object != NULL ?
1373 VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
1374#if VM_NRESERVLEVEL > 0
1375 if (m == NULL && vm_reserv_reclaim_inactive()) {
1376 m = vm_phys_alloc_pages(object != NULL ?
1377 VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
1378 0);
1379 }
1380#endif
1381 }
1382 } else {
1383 /*
1384 * Not allocatable, give up.
1385 */
1386 mtx_unlock(&vm_page_queue_free_mtx);
1387 atomic_add_int(&vm_pageout_deficit,
1388 max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
1389 pagedaemon_wakeup();
1390 return (NULL);
1391 }
1392
1393 /*
1394 * At this point we had better have found a good page.
1395 */
1396 KASSERT(m != NULL, ("vm_page_alloc: missing page"));
1397 KASSERT(m->queue == PQ_NONE,
1398 ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue));
1399 KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m));
1400 KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m));
1401 KASSERT(m->busy == 0, ("vm_page_alloc: page %p is busy", m));
1402 KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m));
1403 KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1404 ("vm_page_alloc: page %p has unexpected memattr %d", m,
1405 pmap_page_get_memattr(m)));
1406 if ((m->flags & PG_CACHED) != 0) {
1407 KASSERT((m->flags & PG_ZERO) == 0,
1408 ("vm_page_alloc: cached page %p is PG_ZERO", m));
1409 KASSERT(m->valid != 0,
1410 ("vm_page_alloc: cached page %p is invalid", m));
1411 if (m->object == object && m->pindex == pindex)
1412 cnt.v_reactivated++;
1413 else
1414 m->valid = 0;
1415 m_object = m->object;
1416 vm_page_cache_remove(m);
1417 if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1418 vp = m_object->handle;
1419 } else {
1420 KASSERT(VM_PAGE_IS_FREE(m),
1421 ("vm_page_alloc: page %p is not free", m));
1422 KASSERT(m->valid == 0,
1423 ("vm_page_alloc: free page %p is valid", m));
1424 cnt.v_free_count--;
1425 }
1426
1427 /*
1428 * Only the PG_ZERO flag is inherited. The PG_CACHED or PG_FREE flag
1429 * must be cleared before the free page queues lock is released.
1430 */
1431 flags = 0;
| 1309 * VM_ALLOC_NOOBJ page is not associated with an object and
1310 * should not have the flag VPO_BUSY set
1311 * VM_ALLOC_WIRED wire the allocated page
1312 * VM_ALLOC_ZERO prefer a zeroed page
1313 *
1314 * This routine may not sleep.
1315 */
1316vm_page_t
1317vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
1318{
1319 struct vnode *vp = NULL;
1320 vm_object_t m_object;
1321 vm_page_t m;
1322 int flags, req_class;
1323
1324 KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
1325 ("vm_page_alloc: inconsistent object/req"));
1326 if (object != NULL)
1327 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1328
1329 req_class = req & VM_ALLOC_CLASS_MASK;
1330
1331 /*
1332 * The page daemon is allowed to dig deeper into the free page list.
1333 */
1334 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1335 req_class = VM_ALLOC_SYSTEM;
1336
1337 mtx_lock(&vm_page_queue_free_mtx);
1338 if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1339 (req_class == VM_ALLOC_SYSTEM &&
1340 cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1341 (req_class == VM_ALLOC_INTERRUPT &&
1342 cnt.v_free_count + cnt.v_cache_count > 0)) {
1343 /*
1344 * Allocate from the free queue if the number of free pages
1345 * exceeds the minimum for the request class.
1346 */
1347 if (object != NULL &&
1348 (m = vm_page_cache_lookup(object, pindex)) != NULL) {
1349 if ((req & VM_ALLOC_IFNOTCACHED) != 0) {
1350 mtx_unlock(&vm_page_queue_free_mtx);
1351 return (NULL);
1352 }
1353 if (vm_phys_unfree_page(m))
1354 vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m, 0);
1355#if VM_NRESERVLEVEL > 0
1356 else if (!vm_reserv_reactivate_page(m))
1357#else
1358 else
1359#endif
1360 panic("vm_page_alloc: cache page %p is missing"
1361 " from the free queue", m);
1362 } else if ((req & VM_ALLOC_IFCACHED) != 0) {
1363 mtx_unlock(&vm_page_queue_free_mtx);
1364 return (NULL);
1365#if VM_NRESERVLEVEL > 0
1366 } else if (object == NULL || object->type == OBJT_DEVICE ||
1367 object->type == OBJT_SG ||
1368 (object->flags & OBJ_COLORED) == 0 ||
1369 (m = vm_reserv_alloc_page(object, pindex)) == NULL) {
1370#else
1371 } else {
1372#endif
1373 m = vm_phys_alloc_pages(object != NULL ?
1374 VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
1375#if VM_NRESERVLEVEL > 0
1376 if (m == NULL && vm_reserv_reclaim_inactive()) {
1377 m = vm_phys_alloc_pages(object != NULL ?
1378 VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
1379 0);
1380 }
1381#endif
1382 }
1383 } else {
1384 /*
1385 * Not allocatable, give up.
1386 */
1387 mtx_unlock(&vm_page_queue_free_mtx);
1388 atomic_add_int(&vm_pageout_deficit,
1389 max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
1390 pagedaemon_wakeup();
1391 return (NULL);
1392 }
1393
1394 /*
1395 * At this point we had better have found a good page.
1396 */
1397 KASSERT(m != NULL, ("vm_page_alloc: missing page"));
1398 KASSERT(m->queue == PQ_NONE,
1399 ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue));
1400 KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m));
1401 KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m));
1402 KASSERT(m->busy == 0, ("vm_page_alloc: page %p is busy", m));
1403 KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m));
1404 KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1405 ("vm_page_alloc: page %p has unexpected memattr %d", m,
1406 pmap_page_get_memattr(m)));
1407 if ((m->flags & PG_CACHED) != 0) {
1408 KASSERT((m->flags & PG_ZERO) == 0,
1409 ("vm_page_alloc: cached page %p is PG_ZERO", m));
1410 KASSERT(m->valid != 0,
1411 ("vm_page_alloc: cached page %p is invalid", m));
1412 if (m->object == object && m->pindex == pindex)
1413 cnt.v_reactivated++;
1414 else
1415 m->valid = 0;
1416 m_object = m->object;
1417 vm_page_cache_remove(m);
1418 if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1419 vp = m_object->handle;
1420 } else {
1421 KASSERT(VM_PAGE_IS_FREE(m),
1422 ("vm_page_alloc: page %p is not free", m));
1423 KASSERT(m->valid == 0,
1424 ("vm_page_alloc: free page %p is valid", m));
1425 cnt.v_free_count--;
1426 }
1427
1428 /*
1429 * Only the PG_ZERO flag is inherited. The PG_CACHED or PG_FREE flag
1430 * must be cleared before the free page queues lock is released.
1431 */
1432 flags = 0;
|
| 1433 if (req & VM_ALLOC_NODUMP)
1434 flags |= PG_NODUMP;
|
1432 if (m->flags & PG_ZERO) {
1433 vm_page_zero_count--;
1434 if (req & VM_ALLOC_ZERO)
1435 flags = PG_ZERO;
1436 }
1437 m->flags = flags;
1438 mtx_unlock(&vm_page_queue_free_mtx);
1439 m->aflags = 0;
1440 if (object == NULL || object->type == OBJT_PHYS)
1441 m->oflags = VPO_UNMANAGED;
1442 else
1443 m->oflags = 0;
1444 if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ)) == 0)
1445 m->oflags |= VPO_BUSY;
1446 if (req & VM_ALLOC_WIRED) {
1447 /*
1448 * The page lock is not required for wiring a page until that
1449 * page is inserted into the object.
1450 */
1451 atomic_add_int(&cnt.v_wire_count, 1);
1452 m->wire_count = 1;
1453 }
1454 m->act_count = 0;
1455
1456 if (object != NULL) {
1457 /* Ignore device objects; the pager sets "memattr" for them. */
1458 if (object->memattr != VM_MEMATTR_DEFAULT &&
1459 object->type != OBJT_DEVICE && object->type != OBJT_SG)
1460 pmap_page_set_memattr(m, object->memattr);
1461 vm_page_insert(m, object, pindex);
1462 } else
1463 m->pindex = pindex;
1464
1465 /*
1466 * The following call to vdrop() must come after the above call
1467 * to vm_page_insert() in case both affect the same object and
1468 * vnode. Otherwise, the affected vnode's hold count could
1469 * temporarily become zero.
1470 */
1471 if (vp != NULL)
1472 vdrop(vp);
1473
1474 /*
1475 * Don't wakeup too often - wakeup the pageout daemon when
1476 * we would be nearly out of memory.
1477 */
1478 if (vm_paging_needed())
1479 pagedaemon_wakeup();
1480
1481 return (m);
1482}
1483
1484/*
1485 * vm_page_alloc_contig:
1486 *
1487 * Allocate a contiguous set of physical pages of the given size "npages"
1488 * from the free lists. All of the physical pages must be at or above
1489 * the given physical address "low" and below the given physical address
1490 * "high". The given value "alignment" determines the alignment of the
1491 * first physical page in the set. If the given value "boundary" is
1492 * non-zero, then the set of physical pages cannot cross any physical
1493 * address boundary that is a multiple of that value. Both "alignment"
1494 * and "boundary" must be a power of two.
1495 *
1496 * If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT,
1497 * then the memory attribute setting for the physical pages is configured
1498 * to the object's memory attribute setting. Otherwise, the memory
1499 * attribute setting for the physical pages is configured to "memattr",
1500 * overriding the object's memory attribute setting. However, if the
1501 * object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the
1502 * memory attribute setting for the physical pages cannot be configured
1503 * to VM_MEMATTR_DEFAULT.
1504 *
1505 * The caller must always specify an allocation class.
1506 *
1507 * allocation classes:
1508 * VM_ALLOC_NORMAL normal process request
1509 * VM_ALLOC_SYSTEM system *really* needs a page
1510 * VM_ALLOC_INTERRUPT interrupt time request
1511 *
1512 * optional allocation flags:
1513 * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page
1514 * VM_ALLOC_NOOBJ page is not associated with an object and
1515 * should not have the flag VPO_BUSY set
1516 * VM_ALLOC_WIRED wire the allocated page
1517 * VM_ALLOC_ZERO prefer a zeroed page
1518 *
1519 * This routine may not sleep.
1520 */
1521vm_page_t
1522vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
1523 u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
1524 vm_paddr_t boundary, vm_memattr_t memattr)
1525{
1526 struct vnode *drop;
1527 vm_page_t deferred_vdrop_list, m, m_ret;
1528 u_int flags, oflags;
1529 int req_class;
1530
1531 KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
1532 ("vm_page_alloc_contig: inconsistent object/req"));
1533 if (object != NULL) {
1534 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1535 KASSERT(object->type == OBJT_PHYS,
1536 ("vm_page_alloc_contig: object %p isn't OBJT_PHYS",
1537 object));
1538 }
1539 KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
1540 req_class = req & VM_ALLOC_CLASS_MASK;
1541
1542 /*
1543 * The page daemon is allowed to dig deeper into the free page list.
1544 */
1545 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1546 req_class = VM_ALLOC_SYSTEM;
1547
1548 deferred_vdrop_list = NULL;
1549 mtx_lock(&vm_page_queue_free_mtx);
1550 if (cnt.v_free_count + cnt.v_cache_count >= npages +
1551 cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM &&
1552 cnt.v_free_count + cnt.v_cache_count >= npages +
1553 cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT &&
1554 cnt.v_free_count + cnt.v_cache_count >= npages)) {
1555#if VM_NRESERVLEVEL > 0
1556retry:
1557 if (object == NULL || (object->flags & OBJ_COLORED) == 0 ||
1558 (m_ret = vm_reserv_alloc_contig(object, pindex, npages,
1559 low, high, alignment, boundary)) == NULL)
1560#endif
1561 m_ret = vm_phys_alloc_contig(npages, low, high,
1562 alignment, boundary);
1563 } else {
1564 mtx_unlock(&vm_page_queue_free_mtx);
1565 atomic_add_int(&vm_pageout_deficit, npages);
1566 pagedaemon_wakeup();
1567 return (NULL);
1568 }
1569 if (m_ret != NULL)
1570 for (m = m_ret; m < &m_ret[npages]; m++) {
1571 drop = vm_page_alloc_init(m);
1572 if (drop != NULL) {
1573 /*
1574 * Enqueue the vnode for deferred vdrop().
1575 *
1576 * Once the pages are removed from the free
1577 * page list, "pageq" can be safely abused to
1578 * construct a short-lived list of vnodes.
1579 */
1580 m->pageq.tqe_prev = (void *)drop;
1581 m->pageq.tqe_next = deferred_vdrop_list;
1582 deferred_vdrop_list = m;
1583 }
1584 }
1585 else {
1586#if VM_NRESERVLEVEL > 0
1587 if (vm_reserv_reclaim_contig(npages, low, high, alignment,
1588 boundary))
1589 goto retry;
1590#endif
1591 }
1592 mtx_unlock(&vm_page_queue_free_mtx);
1593 if (m_ret == NULL)
1594 return (NULL);
1595
1596 /*
1597 * Initialize the pages. Only the PG_ZERO flag is inherited.
1598 */
1599 flags = 0;
1600 if ((req & VM_ALLOC_ZERO) != 0)
1601 flags = PG_ZERO;
| 1435 if (m->flags & PG_ZERO) {
1436 vm_page_zero_count--;
1437 if (req & VM_ALLOC_ZERO)
1438 flags = PG_ZERO;
1439 }
1440 m->flags = flags;
1441 mtx_unlock(&vm_page_queue_free_mtx);
1442 m->aflags = 0;
1443 if (object == NULL || object->type == OBJT_PHYS)
1444 m->oflags = VPO_UNMANAGED;
1445 else
1446 m->oflags = 0;
1447 if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ)) == 0)
1448 m->oflags |= VPO_BUSY;
1449 if (req & VM_ALLOC_WIRED) {
1450 /*
1451 * The page lock is not required for wiring a page until that
1452 * page is inserted into the object.
1453 */
1454 atomic_add_int(&cnt.v_wire_count, 1);
1455 m->wire_count = 1;
1456 }
1457 m->act_count = 0;
1458
1459 if (object != NULL) {
1460 /* Ignore device objects; the pager sets "memattr" for them. */
1461 if (object->memattr != VM_MEMATTR_DEFAULT &&
1462 object->type != OBJT_DEVICE && object->type != OBJT_SG)
1463 pmap_page_set_memattr(m, object->memattr);
1464 vm_page_insert(m, object, pindex);
1465 } else
1466 m->pindex = pindex;
1467
1468 /*
1469 * The following call to vdrop() must come after the above call
1470 * to vm_page_insert() in case both affect the same object and
1471 * vnode. Otherwise, the affected vnode's hold count could
1472 * temporarily become zero.
1473 */
1474 if (vp != NULL)
1475 vdrop(vp);
1476
1477 /*
1478 * Don't wakeup too often - wakeup the pageout daemon when
1479 * we would be nearly out of memory.
1480 */
1481 if (vm_paging_needed())
1482 pagedaemon_wakeup();
1483
1484 return (m);
1485}
1486
1487/*
1488 * vm_page_alloc_contig:
1489 *
1490 * Allocate a contiguous set of physical pages of the given size "npages"
1491 * from the free lists. All of the physical pages must be at or above
1492 * the given physical address "low" and below the given physical address
1493 * "high". The given value "alignment" determines the alignment of the
1494 * first physical page in the set. If the given value "boundary" is
1495 * non-zero, then the set of physical pages cannot cross any physical
1496 * address boundary that is a multiple of that value. Both "alignment"
1497 * and "boundary" must be a power of two.
1498 *
1499 * If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT,
1500 * then the memory attribute setting for the physical pages is configured
1501 * to the object's memory attribute setting. Otherwise, the memory
1502 * attribute setting for the physical pages is configured to "memattr",
1503 * overriding the object's memory attribute setting. However, if the
1504 * object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the
1505 * memory attribute setting for the physical pages cannot be configured
1506 * to VM_MEMATTR_DEFAULT.
1507 *
1508 * The caller must always specify an allocation class.
1509 *
1510 * allocation classes:
1511 * VM_ALLOC_NORMAL normal process request
1512 * VM_ALLOC_SYSTEM system *really* needs a page
1513 * VM_ALLOC_INTERRUPT interrupt time request
1514 *
1515 * optional allocation flags:
1516 * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page
1517 * VM_ALLOC_NOOBJ page is not associated with an object and
1518 * should not have the flag VPO_BUSY set
1519 * VM_ALLOC_WIRED wire the allocated page
1520 * VM_ALLOC_ZERO prefer a zeroed page
1521 *
1522 * This routine may not sleep.
1523 */
1524vm_page_t
1525vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
1526 u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment,
1527 vm_paddr_t boundary, vm_memattr_t memattr)
1528{
1529 struct vnode *drop;
1530 vm_page_t deferred_vdrop_list, m, m_ret;
1531 u_int flags, oflags;
1532 int req_class;
1533
1534 KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0),
1535 ("vm_page_alloc_contig: inconsistent object/req"));
1536 if (object != NULL) {
1537 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
1538 KASSERT(object->type == OBJT_PHYS,
1539 ("vm_page_alloc_contig: object %p isn't OBJT_PHYS",
1540 object));
1541 }
1542 KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero"));
1543 req_class = req & VM_ALLOC_CLASS_MASK;
1544
1545 /*
1546 * The page daemon is allowed to dig deeper into the free page list.
1547 */
1548 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1549 req_class = VM_ALLOC_SYSTEM;
1550
1551 deferred_vdrop_list = NULL;
1552 mtx_lock(&vm_page_queue_free_mtx);
1553 if (cnt.v_free_count + cnt.v_cache_count >= npages +
1554 cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM &&
1555 cnt.v_free_count + cnt.v_cache_count >= npages +
1556 cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT &&
1557 cnt.v_free_count + cnt.v_cache_count >= npages)) {
1558#if VM_NRESERVLEVEL > 0
1559retry:
1560 if (object == NULL || (object->flags & OBJ_COLORED) == 0 ||
1561 (m_ret = vm_reserv_alloc_contig(object, pindex, npages,
1562 low, high, alignment, boundary)) == NULL)
1563#endif
1564 m_ret = vm_phys_alloc_contig(npages, low, high,
1565 alignment, boundary);
1566 } else {
1567 mtx_unlock(&vm_page_queue_free_mtx);
1568 atomic_add_int(&vm_pageout_deficit, npages);
1569 pagedaemon_wakeup();
1570 return (NULL);
1571 }
1572 if (m_ret != NULL)
1573 for (m = m_ret; m < &m_ret[npages]; m++) {
1574 drop = vm_page_alloc_init(m);
1575 if (drop != NULL) {
1576 /*
1577 * Enqueue the vnode for deferred vdrop().
1578 *
1579 * Once the pages are removed from the free
1580 * page list, "pageq" can be safely abused to
1581 * construct a short-lived list of vnodes.
1582 */
1583 m->pageq.tqe_prev = (void *)drop;
1584 m->pageq.tqe_next = deferred_vdrop_list;
1585 deferred_vdrop_list = m;
1586 }
1587 }
1588 else {
1589#if VM_NRESERVLEVEL > 0
1590 if (vm_reserv_reclaim_contig(npages, low, high, alignment,
1591 boundary))
1592 goto retry;
1593#endif
1594 }
1595 mtx_unlock(&vm_page_queue_free_mtx);
1596 if (m_ret == NULL)
1597 return (NULL);
1598
1599 /*
1600 * Initialize the pages. Only the PG_ZERO flag is inherited.
1601 */
1602 flags = 0;
1603 if ((req & VM_ALLOC_ZERO) != 0)
1604 flags = PG_ZERO;
|
| 1605 if ((req & VM_ALLOC_NODUMP) != 0)
1606 flags |= PG_NODUMP;
|
1602 if ((req & VM_ALLOC_WIRED) != 0)
1603 atomic_add_int(&cnt.v_wire_count, npages);
1604 oflags = VPO_UNMANAGED;
1605 if (object != NULL) {
1606 if ((req & VM_ALLOC_NOBUSY) == 0)
1607 oflags |= VPO_BUSY;
1608 if (object->memattr != VM_MEMATTR_DEFAULT &&
1609 memattr == VM_MEMATTR_DEFAULT)
1610 memattr = object->memattr;
1611 }
1612 for (m = m_ret; m < &m_ret[npages]; m++) {
1613 m->aflags = 0;
1614 m->flags &= flags;
1615 if ((req & VM_ALLOC_WIRED) != 0)
1616 m->wire_count = 1;
1617 /* Unmanaged pages don't use "act_count". */
1618 m->oflags = oflags;
1619 if (memattr != VM_MEMATTR_DEFAULT)
1620 pmap_page_set_memattr(m, memattr);
1621 if (object != NULL)
1622 vm_page_insert(m, object, pindex);
1623 else
1624 m->pindex = pindex;
1625 pindex++;
1626 }
1627 while (deferred_vdrop_list != NULL) {
1628 vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
1629 deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next;
1630 }
1631 if (vm_paging_needed())
1632 pagedaemon_wakeup();
1633 return (m_ret);
1634}
1635
1636/*
1637 * Initialize a page that has been freshly dequeued from a freelist.
1638 * The caller has to drop the vnode returned, if it is not NULL.
1639 *
1640 * This function may only be used to initialize unmanaged pages.
1641 *
1642 * To be called with vm_page_queue_free_mtx held.
1643 */
1644static struct vnode *
1645vm_page_alloc_init(vm_page_t m)
1646{
1647 struct vnode *drop;
1648 vm_object_t m_object;
1649
1650 KASSERT(m->queue == PQ_NONE,
1651 ("vm_page_alloc_init: page %p has unexpected queue %d",
1652 m, m->queue));
1653 KASSERT(m->wire_count == 0,
1654 ("vm_page_alloc_init: page %p is wired", m));
1655 KASSERT(m->hold_count == 0,
1656 ("vm_page_alloc_init: page %p is held", m));
1657 KASSERT(m->busy == 0,
1658 ("vm_page_alloc_init: page %p is busy", m));
1659 KASSERT(m->dirty == 0,
1660 ("vm_page_alloc_init: page %p is dirty", m));
1661 KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1662 ("vm_page_alloc_init: page %p has unexpected memattr %d",
1663 m, pmap_page_get_memattr(m)));
1664 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1665 drop = NULL;
1666 if ((m->flags & PG_CACHED) != 0) {
1667 KASSERT((m->flags & PG_ZERO) == 0,
1668 ("vm_page_alloc_init: cached page %p is PG_ZERO", m));
1669 m->valid = 0;
1670 m_object = m->object;
1671 vm_page_cache_remove(m);
1672 if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1673 drop = m_object->handle;
1674 } else {
1675 KASSERT(VM_PAGE_IS_FREE(m),
1676 ("vm_page_alloc_init: page %p is not free", m));
1677 KASSERT(m->valid == 0,
1678 ("vm_page_alloc_init: free page %p is valid", m));
1679 cnt.v_free_count--;
1680 if ((m->flags & PG_ZERO) != 0)
1681 vm_page_zero_count--;
1682 }
1683 /* Don't clear the PG_ZERO flag; we'll need it later. */
1684 m->flags &= PG_ZERO;
1685 return (drop);
1686}
1687
1688/*
1689 * vm_page_alloc_freelist:
1690 *
1691 * Allocate a physical page from the specified free page list.
1692 *
1693 * The caller must always specify an allocation class.
1694 *
1695 * allocation classes:
1696 * VM_ALLOC_NORMAL normal process request
1697 * VM_ALLOC_SYSTEM system *really* needs a page
1698 * VM_ALLOC_INTERRUPT interrupt time request
1699 *
1700 * optional allocation flags:
1701 * VM_ALLOC_COUNT(number) the number of additional pages that the caller
1702 * intends to allocate
1703 * VM_ALLOC_WIRED wire the allocated page
1704 * VM_ALLOC_ZERO prefer a zeroed page
1705 *
1706 * This routine may not sleep.
1707 */
1708vm_page_t
1709vm_page_alloc_freelist(int flind, int req)
1710{
1711 struct vnode *drop;
1712 vm_page_t m;
1713 u_int flags;
1714 int req_class;
1715
1716 req_class = req & VM_ALLOC_CLASS_MASK;
1717
1718 /*
1719 * The page daemon is allowed to dig deeper into the free page list.
1720 */
1721 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1722 req_class = VM_ALLOC_SYSTEM;
1723
1724 /*
1725 * Do not allocate reserved pages unless the req has asked for it.
1726 */
1727 mtx_lock(&vm_page_queue_free_mtx);
1728 if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1729 (req_class == VM_ALLOC_SYSTEM &&
1730 cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1731 (req_class == VM_ALLOC_INTERRUPT &&
1732 cnt.v_free_count + cnt.v_cache_count > 0))
1733 m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0);
1734 else {
1735 mtx_unlock(&vm_page_queue_free_mtx);
1736 atomic_add_int(&vm_pageout_deficit,
1737 max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
1738 pagedaemon_wakeup();
1739 return (NULL);
1740 }
1741 if (m == NULL) {
1742 mtx_unlock(&vm_page_queue_free_mtx);
1743 return (NULL);
1744 }
1745 drop = vm_page_alloc_init(m);
1746 mtx_unlock(&vm_page_queue_free_mtx);
1747
1748 /*
1749 * Initialize the page. Only the PG_ZERO flag is inherited.
1750 */
1751 m->aflags = 0;
1752 flags = 0;
1753 if ((req & VM_ALLOC_ZERO) != 0)
1754 flags = PG_ZERO;
1755 m->flags &= flags;
1756 if ((req & VM_ALLOC_WIRED) != 0) {
1757 /*
1758 * The page lock is not required for wiring a page that does
1759 * not belong to an object.
1760 */
1761 atomic_add_int(&cnt.v_wire_count, 1);
1762 m->wire_count = 1;
1763 }
1764 /* Unmanaged pages don't use "act_count". */
1765 m->oflags = VPO_UNMANAGED;
1766 if (drop != NULL)
1767 vdrop(drop);
1768 if (vm_paging_needed())
1769 pagedaemon_wakeup();
1770 return (m);
1771}
1772
1773/*
1774 * vm_wait: (also see VM_WAIT macro)
1775 *
1776 * Block until free pages are available for allocation
1777 * - Called in various places before memory allocations.
1778 */
1779void
1780vm_wait(void)
1781{
1782
1783 mtx_lock(&vm_page_queue_free_mtx);
1784 if (curproc == pageproc) {
1785 vm_pageout_pages_needed = 1;
1786 msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
1787 PDROP | PSWP, "VMWait", 0);
1788 } else {
1789 if (!vm_pages_needed) {
1790 vm_pages_needed = 1;
1791 wakeup(&vm_pages_needed);
1792 }
1793 msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
1794 "vmwait", 0);
1795 }
1796}
1797
1798/*
1799 * vm_waitpfault: (also see VM_WAITPFAULT macro)
1800 *
1801 * Block until free pages are available for allocation
1802 * - Called only in vm_fault so that processes page faulting
1803 * can be easily tracked.
1804 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
1805 * processes will be able to grab memory first. Do not change
1806 * this balance without careful testing first.
1807 */
1808void
1809vm_waitpfault(void)
1810{
1811
1812 mtx_lock(&vm_page_queue_free_mtx);
1813 if (!vm_pages_needed) {
1814 vm_pages_needed = 1;
1815 wakeup(&vm_pages_needed);
1816 }
1817 msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
1818 "pfault", 0);
1819}
1820
1821/*
1822 * vm_page_requeue:
1823 *
1824 * Move the given page to the tail of its present page queue.
1825 *
1826 * The page queues must be locked.
1827 */
1828void
1829vm_page_requeue(vm_page_t m)
1830{
1831 struct vpgqueues *vpq;
1832 int queue;
1833
1834 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1835 queue = m->queue;
1836 KASSERT(queue != PQ_NONE,
1837 ("vm_page_requeue: page %p is not queued", m));
1838 vpq = &vm_page_queues[queue];
1839 TAILQ_REMOVE(&vpq->pl, m, pageq);
1840 TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1841}
1842
1843/*
1844 * vm_page_queue_remove:
1845 *
1846 * Remove the given page from the specified queue.
1847 *
1848 * The page and page queues must be locked.
1849 */
1850static __inline void
1851vm_page_queue_remove(int queue, vm_page_t m)
1852{
1853 struct vpgqueues *pq;
1854
1855 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1856 vm_page_lock_assert(m, MA_OWNED);
1857 pq = &vm_page_queues[queue];
1858 TAILQ_REMOVE(&pq->pl, m, pageq);
1859 (*pq->cnt)--;
1860}
1861
1862/*
1863 * vm_pageq_remove:
1864 *
1865 * Remove a page from its queue.
1866 *
1867 * The given page must be locked.
1868 * This routine may not block.
1869 */
1870void
1871vm_pageq_remove(vm_page_t m)
1872{
1873 int queue;
1874
1875 vm_page_lock_assert(m, MA_OWNED);
1876 if ((queue = m->queue) != PQ_NONE) {
1877 vm_page_lock_queues();
1878 m->queue = PQ_NONE;
1879 vm_page_queue_remove(queue, m);
1880 vm_page_unlock_queues();
1881 }
1882}
1883
1884/*
1885 * vm_page_enqueue:
1886 *
1887 * Add the given page to the specified queue.
1888 *
1889 * The page queues must be locked.
1890 */
1891static void
1892vm_page_enqueue(int queue, vm_page_t m)
1893{
1894 struct vpgqueues *vpq;
1895
1896 vpq = &vm_page_queues[queue];
1897 m->queue = queue;
1898 TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1899 ++*vpq->cnt;
1900}
1901
1902/*
1903 * vm_page_activate:
1904 *
1905 * Put the specified page on the active list (if appropriate).
1906 * Ensure that act_count is at least ACT_INIT but do not otherwise
1907 * mess with it.
1908 *
1909 * The page must be locked.
1910 * This routine may not block.
1911 */
1912void
1913vm_page_activate(vm_page_t m)
1914{
1915 int queue;
1916
1917 vm_page_lock_assert(m, MA_OWNED);
1918 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1919 if ((queue = m->queue) != PQ_ACTIVE) {
1920 if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
1921 if (m->act_count < ACT_INIT)
1922 m->act_count = ACT_INIT;
1923 vm_page_lock_queues();
1924 if (queue != PQ_NONE)
1925 vm_page_queue_remove(queue, m);
1926 vm_page_enqueue(PQ_ACTIVE, m);
1927 vm_page_unlock_queues();
1928 } else
1929 KASSERT(queue == PQ_NONE,
1930 ("vm_page_activate: wired page %p is queued", m));
1931 } else {
1932 if (m->act_count < ACT_INIT)
1933 m->act_count = ACT_INIT;
1934 }
1935}
1936
1937/*
1938 * vm_page_free_wakeup:
1939 *
1940 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
1941 * routine is called when a page has been added to the cache or free
1942 * queues.
1943 *
1944 * The page queues must be locked.
1945 * This routine may not block.
1946 */
1947static inline void
1948vm_page_free_wakeup(void)
1949{
1950
1951 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1952 /*
1953 * if pageout daemon needs pages, then tell it that there are
1954 * some free.
1955 */
1956 if (vm_pageout_pages_needed &&
1957 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1958 wakeup(&vm_pageout_pages_needed);
1959 vm_pageout_pages_needed = 0;
1960 }
1961 /*
1962 * wakeup processes that are waiting on memory if we hit a
1963 * high water mark. And wakeup scheduler process if we have
1964 * lots of memory. this process will swapin processes.
1965 */
1966 if (vm_pages_needed && !vm_page_count_min()) {
1967 vm_pages_needed = 0;
1968 wakeup(&cnt.v_free_count);
1969 }
1970}
1971
1972/*
1973 * vm_page_free_toq:
1974 *
1975 * Returns the given page to the free list,
1976 * disassociating it with any VM object.
1977 *
1978 * Object and page must be locked prior to entry.
1979 * This routine may not block.
1980 */
1981
1982void
1983vm_page_free_toq(vm_page_t m)
1984{
1985
1986 if ((m->oflags & VPO_UNMANAGED) == 0) {
1987 vm_page_lock_assert(m, MA_OWNED);
1988 KASSERT(!pmap_page_is_mapped(m),
1989 ("vm_page_free_toq: freeing mapped page %p", m));
1990 }
1991 PCPU_INC(cnt.v_tfree);
1992
1993 if (VM_PAGE_IS_FREE(m))
1994 panic("vm_page_free: freeing free page %p", m);
1995 else if (m->busy != 0)
1996 panic("vm_page_free: freeing busy page %p", m);
1997
1998 /*
1999 * unqueue, then remove page. Note that we cannot destroy
2000 * the page here because we do not want to call the pager's
2001 * callback routine until after we've put the page on the
2002 * appropriate free queue.
2003 */
2004 if ((m->oflags & VPO_UNMANAGED) == 0)
2005 vm_pageq_remove(m);
2006 vm_page_remove(m);
2007
2008 /*
2009 * If fictitious remove object association and
2010 * return, otherwise delay object association removal.
2011 */
2012 if ((m->flags & PG_FICTITIOUS) != 0) {
2013 return;
2014 }
2015
2016 m->valid = 0;
2017 vm_page_undirty(m);
2018
2019 if (m->wire_count != 0)
2020 panic("vm_page_free: freeing wired page %p", m);
2021 if (m->hold_count != 0) {
2022 m->flags &= ~PG_ZERO;
2023 vm_page_lock_queues();
2024 vm_page_enqueue(PQ_HOLD, m);
2025 vm_page_unlock_queues();
2026 } else {
2027 /*
2028 * Restore the default memory attribute to the page.
2029 */
2030 if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
2031 pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
2032
2033 /*
2034 * Insert the page into the physical memory allocator's
2035 * cache/free page queues.
2036 */
2037 mtx_lock(&vm_page_queue_free_mtx);
2038 m->flags |= PG_FREE;
2039 cnt.v_free_count++;
2040#if VM_NRESERVLEVEL > 0
2041 if (!vm_reserv_free_page(m))
2042#else
2043 if (TRUE)
2044#endif
2045 vm_phys_free_pages(m, 0);
2046 if ((m->flags & PG_ZERO) != 0)
2047 ++vm_page_zero_count;
2048 else
2049 vm_page_zero_idle_wakeup();
2050 vm_page_free_wakeup();
2051 mtx_unlock(&vm_page_queue_free_mtx);
2052 }
2053}
2054
2055/*
2056 * vm_page_wire:
2057 *
2058 * Mark this page as wired down by yet
2059 * another map, removing it from paging queues
2060 * as necessary.
2061 *
2062 * If the page is fictitious, then its wire count must remain one.
2063 *
2064 * The page must be locked.
2065 * This routine may not block.
2066 */
2067void
2068vm_page_wire(vm_page_t m)
2069{
2070
2071 /*
2072 * Only bump the wire statistics if the page is not already wired,
2073 * and only unqueue the page if it is on some queue (if it is unmanaged
2074 * it is already off the queues).
2075 */
2076 vm_page_lock_assert(m, MA_OWNED);
2077 if ((m->flags & PG_FICTITIOUS) != 0) {
2078 KASSERT(m->wire_count == 1,
2079 ("vm_page_wire: fictitious page %p's wire count isn't one",
2080 m));
2081 return;
2082 }
2083 if (m->wire_count == 0) {
2084 if ((m->oflags & VPO_UNMANAGED) == 0)
2085 vm_pageq_remove(m);
2086 atomic_add_int(&cnt.v_wire_count, 1);
2087 }
2088 m->wire_count++;
2089 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
2090}
2091
2092/*
2093 * vm_page_unwire:
2094 *
2095 * Release one wiring of the specified page, potentially enabling it to be
2096 * paged again. If paging is enabled, then the value of the parameter
2097 * "activate" determines to which queue the page is added. If "activate" is
2098 * non-zero, then the page is added to the active queue. Otherwise, it is
2099 * added to the inactive queue.
2100 *
2101 * However, unless the page belongs to an object, it is not enqueued because
2102 * it cannot be paged out.
2103 *
2104 * If a page is fictitious, then its wire count must alway be one.
2105 *
2106 * A managed page must be locked.
2107 */
2108void
2109vm_page_unwire(vm_page_t m, int activate)
2110{
2111
2112 if ((m->oflags & VPO_UNMANAGED) == 0)
2113 vm_page_lock_assert(m, MA_OWNED);
2114 if ((m->flags & PG_FICTITIOUS) != 0) {
2115 KASSERT(m->wire_count == 1,
2116 ("vm_page_unwire: fictitious page %p's wire count isn't one", m));
2117 return;
2118 }
2119 if (m->wire_count > 0) {
2120 m->wire_count--;
2121 if (m->wire_count == 0) {
2122 atomic_subtract_int(&cnt.v_wire_count, 1);
2123 if ((m->oflags & VPO_UNMANAGED) != 0 ||
2124 m->object == NULL)
2125 return;
2126 vm_page_lock_queues();
2127 if (activate)
2128 vm_page_enqueue(PQ_ACTIVE, m);
2129 else {
2130 m->flags &= ~PG_WINATCFLS;
2131 vm_page_enqueue(PQ_INACTIVE, m);
2132 }
2133 vm_page_unlock_queues();
2134 }
2135 } else
2136 panic("vm_page_unwire: page %p's wire count is zero", m);
2137}
2138
2139/*
2140 * Move the specified page to the inactive queue.
2141 *
2142 * Many pages placed on the inactive queue should actually go
2143 * into the cache, but it is difficult to figure out which. What
2144 * we do instead, if the inactive target is well met, is to put
2145 * clean pages at the head of the inactive queue instead of the tail.
2146 * This will cause them to be moved to the cache more quickly and
2147 * if not actively re-referenced, reclaimed more quickly. If we just
2148 * stick these pages at the end of the inactive queue, heavy filesystem
2149 * meta-data accesses can cause an unnecessary paging load on memory bound
2150 * processes. This optimization causes one-time-use metadata to be
2151 * reused more quickly.
2152 *
2153 * Normally athead is 0 resulting in LRU operation. athead is set
2154 * to 1 if we want this page to be 'as if it were placed in the cache',
2155 * except without unmapping it from the process address space.
2156 *
2157 * This routine may not block.
2158 */
2159static inline void
2160_vm_page_deactivate(vm_page_t m, int athead)
2161{
2162 int queue;
2163
2164 vm_page_lock_assert(m, MA_OWNED);
2165
2166 /*
2167 * Ignore if already inactive.
2168 */
2169 if ((queue = m->queue) == PQ_INACTIVE)
2170 return;
2171 if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
2172 vm_page_lock_queues();
2173 m->flags &= ~PG_WINATCFLS;
2174 if (queue != PQ_NONE)
2175 vm_page_queue_remove(queue, m);
2176 if (athead)
2177 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m,
2178 pageq);
2179 else
2180 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
2181 pageq);
2182 m->queue = PQ_INACTIVE;
2183 cnt.v_inactive_count++;
2184 vm_page_unlock_queues();
2185 }
2186}
2187
2188/*
2189 * Move the specified page to the inactive queue.
2190 *
2191 * The page must be locked.
2192 */
2193void
2194vm_page_deactivate(vm_page_t m)
2195{
2196
2197 _vm_page_deactivate(m, 0);
2198}
2199
2200/*
2201 * vm_page_try_to_cache:
2202 *
2203 * Returns 0 on failure, 1 on success
2204 */
2205int
2206vm_page_try_to_cache(vm_page_t m)
2207{
2208
2209 vm_page_lock_assert(m, MA_OWNED);
2210 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2211 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
2212 (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0)
2213 return (0);
2214 pmap_remove_all(m);
2215 if (m->dirty)
2216 return (0);
2217 vm_page_cache(m);
2218 return (1);
2219}
2220
2221/*
2222 * vm_page_try_to_free()
2223 *
2224 * Attempt to free the page. If we cannot free it, we do nothing.
2225 * 1 is returned on success, 0 on failure.
2226 */
2227int
2228vm_page_try_to_free(vm_page_t m)
2229{
2230
2231 vm_page_lock_assert(m, MA_OWNED);
2232 if (m->object != NULL)
2233 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2234 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
2235 (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0)
2236 return (0);
2237 pmap_remove_all(m);
2238 if (m->dirty)
2239 return (0);
2240 vm_page_free(m);
2241 return (1);
2242}
2243
2244/*
2245 * vm_page_cache
2246 *
2247 * Put the specified page onto the page cache queue (if appropriate).
2248 *
2249 * This routine may not block.
2250 */
2251void
2252vm_page_cache(vm_page_t m)
2253{
2254 vm_object_t object;
2255 vm_page_t next, prev, root;
2256
2257 vm_page_lock_assert(m, MA_OWNED);
2258 object = m->object;
2259 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2260 if ((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) || m->busy ||
2261 m->hold_count || m->wire_count)
2262 panic("vm_page_cache: attempting to cache busy page");
2263 pmap_remove_all(m);
2264 if (m->dirty != 0)
2265 panic("vm_page_cache: page %p is dirty", m);
2266 if (m->valid == 0 || object->type == OBJT_DEFAULT ||
2267 (object->type == OBJT_SWAP &&
2268 !vm_pager_has_page(object, m->pindex, NULL, NULL))) {
2269 /*
2270 * Hypothesis: A cache-elgible page belonging to a
2271 * default object or swap object but without a backing
2272 * store must be zero filled.
2273 */
2274 vm_page_free(m);
2275 return;
2276 }
2277 KASSERT((m->flags & PG_CACHED) == 0,
2278 ("vm_page_cache: page %p is already cached", m));
2279 PCPU_INC(cnt.v_tcached);
2280
2281 /*
2282 * Remove the page from the paging queues.
2283 */
2284 vm_pageq_remove(m);
2285
2286 /*
2287 * Remove the page from the object's collection of resident
2288 * pages.
2289 */
2290 if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
2291 /*
2292 * Since the page's successor in the list is also its parent
2293 * in the tree, its right subtree must be empty.
2294 */
2295 next->left = m->left;
2296 KASSERT(m->right == NULL,
2297 ("vm_page_cache: page %p has right child", m));
2298 } else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
2299 prev->right == m) {
2300 /*
2301 * Since the page's predecessor in the list is also its parent
2302 * in the tree, its left subtree must be empty.
2303 */
2304 KASSERT(m->left == NULL,
2305 ("vm_page_cache: page %p has left child", m));
2306 prev->right = m->right;
2307 } else {
2308 if (m != object->root)
2309 vm_page_splay(m->pindex, object->root);
2310 if (m->left == NULL)
2311 root = m->right;
2312 else if (m->right == NULL)
2313 root = m->left;
2314 else {
2315 /*
2316 * Move the page's successor to the root, because
2317 * pages are usually removed in ascending order.
2318 */
2319 if (m->right != next)
2320 vm_page_splay(m->pindex, m->right);
2321 next->left = m->left;
2322 root = next;
2323 }
2324 object->root = root;
2325 }
2326 TAILQ_REMOVE(&object->memq, m, listq);
2327 object->resident_page_count--;
2328
2329 /*
2330 * Restore the default memory attribute to the page.
2331 */
2332 if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
2333 pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
2334
2335 /*
2336 * Insert the page into the object's collection of cached pages
2337 * and the physical memory allocator's cache/free page queues.
2338 */
2339 m->flags &= ~PG_ZERO;
2340 mtx_lock(&vm_page_queue_free_mtx);
2341 m->flags |= PG_CACHED;
2342 cnt.v_cache_count++;
2343 root = object->cache;
2344 if (root == NULL) {
2345 m->left = NULL;
2346 m->right = NULL;
2347 } else {
2348 root = vm_page_splay(m->pindex, root);
2349 if (m->pindex < root->pindex) {
2350 m->left = root->left;
2351 m->right = root;
2352 root->left = NULL;
2353 } else if (__predict_false(m->pindex == root->pindex))
2354 panic("vm_page_cache: offset already cached");
2355 else {
2356 m->right = root->right;
2357 m->left = root;
2358 root->right = NULL;
2359 }
2360 }
2361 object->cache = m;
2362#if VM_NRESERVLEVEL > 0
2363 if (!vm_reserv_free_page(m)) {
2364#else
2365 if (TRUE) {
2366#endif
2367 vm_phys_set_pool(VM_FREEPOOL_CACHE, m, 0);
2368 vm_phys_free_pages(m, 0);
2369 }
2370 vm_page_free_wakeup();
2371 mtx_unlock(&vm_page_queue_free_mtx);
2372
2373 /*
2374 * Increment the vnode's hold count if this is the object's only
2375 * cached page. Decrement the vnode's hold count if this was
2376 * the object's only resident page.
2377 */
2378 if (object->type == OBJT_VNODE) {
2379 if (root == NULL && object->resident_page_count != 0)
2380 vhold(object->handle);
2381 else if (root != NULL && object->resident_page_count == 0)
2382 vdrop(object->handle);
2383 }
2384}
2385
2386/*
2387 * vm_page_dontneed
2388 *
2389 * Cache, deactivate, or do nothing as appropriate. This routine
2390 * is typically used by madvise() MADV_DONTNEED.
2391 *
2392 * Generally speaking we want to move the page into the cache so
2393 * it gets reused quickly. However, this can result in a silly syndrome
2394 * due to the page recycling too quickly. Small objects will not be
2395 * fully cached. On the otherhand, if we move the page to the inactive
2396 * queue we wind up with a problem whereby very large objects
2397 * unnecessarily blow away our inactive and cache queues.
2398 *
2399 * The solution is to move the pages based on a fixed weighting. We
2400 * either leave them alone, deactivate them, or move them to the cache,
2401 * where moving them to the cache has the highest weighting.
2402 * By forcing some pages into other queues we eventually force the
2403 * system to balance the queues, potentially recovering other unrelated
2404 * space from active. The idea is to not force this to happen too
2405 * often.
2406 */
2407void
2408vm_page_dontneed(vm_page_t m)
2409{
2410 int dnw;
2411 int head;
2412
2413 vm_page_lock_assert(m, MA_OWNED);
2414 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2415 dnw = PCPU_GET(dnweight);
2416 PCPU_INC(dnweight);
2417
2418 /*
2419 * Occasionally leave the page alone.
2420 */
2421 if ((dnw & 0x01F0) == 0 || m->queue == PQ_INACTIVE) {
2422 if (m->act_count >= ACT_INIT)
2423 --m->act_count;
2424 return;
2425 }
2426
2427 /*
2428 * Clear any references to the page. Otherwise, the page daemon will
2429 * immediately reactivate the page.
2430 *
2431 * Perform the pmap_clear_reference() first. Otherwise, a concurrent
2432 * pmap operation, such as pmap_remove(), could clear a reference in
2433 * the pmap and set PGA_REFERENCED on the page before the
2434 * pmap_clear_reference() had completed. Consequently, the page would
2435 * appear referenced based upon an old reference that occurred before
2436 * this function ran.
2437 */
2438 pmap_clear_reference(m);
2439 vm_page_aflag_clear(m, PGA_REFERENCED);
2440
2441 if (m->dirty == 0 && pmap_is_modified(m))
2442 vm_page_dirty(m);
2443
2444 if (m->dirty || (dnw & 0x0070) == 0) {
2445 /*
2446 * Deactivate the page 3 times out of 32.
2447 */
2448 head = 0;
2449 } else {
2450 /*
2451 * Cache the page 28 times out of every 32. Note that
2452 * the page is deactivated instead of cached, but placed
2453 * at the head of the queue instead of the tail.
2454 */
2455 head = 1;
2456 }
2457 _vm_page_deactivate(m, head);
2458}
2459
2460/*
2461 * Grab a page, waiting until we are waken up due to the page
2462 * changing state. We keep on waiting, if the page continues
2463 * to be in the object. If the page doesn't exist, first allocate it
2464 * and then conditionally zero it.
2465 *
2466 * The caller must always specify the VM_ALLOC_RETRY flag. This is intended
2467 * to facilitate its eventual removal.
2468 *
2469 * This routine may block.
2470 */
2471vm_page_t
2472vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
2473{
2474 vm_page_t m;
2475
2476 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2477 KASSERT((allocflags & VM_ALLOC_RETRY) != 0,
2478 ("vm_page_grab: VM_ALLOC_RETRY is required"));
2479retrylookup:
2480 if ((m = vm_page_lookup(object, pindex)) != NULL) {
2481 if ((m->oflags & VPO_BUSY) != 0 ||
2482 ((allocflags & VM_ALLOC_IGN_SBUSY) == 0 && m->busy != 0)) {
2483 /*
2484 * Reference the page before unlocking and
2485 * sleeping so that the page daemon is less
2486 * likely to reclaim it.
2487 */
2488 vm_page_aflag_set(m, PGA_REFERENCED);
2489 vm_page_sleep(m, "pgrbwt");
2490 goto retrylookup;
2491 } else {
2492 if ((allocflags & VM_ALLOC_WIRED) != 0) {
2493 vm_page_lock(m);
2494 vm_page_wire(m);
2495 vm_page_unlock(m);
2496 }
2497 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
2498 vm_page_busy(m);
2499 return (m);
2500 }
2501 }
2502 m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY |
2503 VM_ALLOC_IGN_SBUSY));
2504 if (m == NULL) {
2505 VM_OBJECT_UNLOCK(object);
2506 VM_WAIT;
2507 VM_OBJECT_LOCK(object);
2508 goto retrylookup;
2509 } else if (m->valid != 0)
2510 return (m);
2511 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
2512 pmap_zero_page(m);
2513 return (m);
2514}
2515
2516/*
2517 * Mapping function for valid bits or for dirty bits in
2518 * a page. May not block.
2519 *
2520 * Inputs are required to range within a page.
2521 */
2522vm_page_bits_t
2523vm_page_bits(int base, int size)
2524{
2525 int first_bit;
2526 int last_bit;
2527
2528 KASSERT(
2529 base + size <= PAGE_SIZE,
2530 ("vm_page_bits: illegal base/size %d/%d", base, size)
2531 );
2532
2533 if (size == 0) /* handle degenerate case */
2534 return (0);
2535
2536 first_bit = base >> DEV_BSHIFT;
2537 last_bit = (base + size - 1) >> DEV_BSHIFT;
2538
2539 return (((vm_page_bits_t)2 << last_bit) -
2540 ((vm_page_bits_t)1 << first_bit));
2541}
2542
2543/*
2544 * vm_page_set_valid_range:
2545 *
2546 * Sets portions of a page valid. The arguments are expected
2547 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2548 * of any partial chunks touched by the range. The invalid portion of
2549 * such chunks will be zeroed.
2550 *
2551 * (base + size) must be less then or equal to PAGE_SIZE.
2552 */
2553void
2554vm_page_set_valid_range(vm_page_t m, int base, int size)
2555{
2556 int endoff, frag;
2557
2558 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2559 if (size == 0) /* handle degenerate case */
2560 return;
2561
2562 /*
2563 * If the base is not DEV_BSIZE aligned and the valid
2564 * bit is clear, we have to zero out a portion of the
2565 * first block.
2566 */
2567 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2568 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
2569 pmap_zero_page_area(m, frag, base - frag);
2570
2571 /*
2572 * If the ending offset is not DEV_BSIZE aligned and the
2573 * valid bit is clear, we have to zero out a portion of
2574 * the last block.
2575 */
2576 endoff = base + size;
2577 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2578 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
2579 pmap_zero_page_area(m, endoff,
2580 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2581
2582 /*
2583 * Assert that no previously invalid block that is now being validated
2584 * is already dirty.
2585 */
2586 KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0,
2587 ("vm_page_set_valid_range: page %p is dirty", m));
2588
2589 /*
2590 * Set valid bits inclusive of any overlap.
2591 */
2592 m->valid |= vm_page_bits(base, size);
2593}
2594
2595/*
2596 * Clear the given bits from the specified page's dirty field.
2597 */
2598static __inline void
2599vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits)
2600{
2601 uintptr_t addr;
2602#if PAGE_SIZE < 16384
2603 int shift;
2604#endif
2605
2606 /*
2607 * If the object is locked and the page is neither VPO_BUSY nor
2608 * PGA_WRITEABLE, then the page's dirty field cannot possibly be
2609 * set by a concurrent pmap operation.
2610 */
2611 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2612 if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
2613 m->dirty &= ~pagebits;
2614 else {
2615 /*
2616 * The pmap layer can call vm_page_dirty() without
2617 * holding a distinguished lock. The combination of
2618 * the object's lock and an atomic operation suffice
2619 * to guarantee consistency of the page dirty field.
2620 *
2621 * For PAGE_SIZE == 32768 case, compiler already
2622 * properly aligns the dirty field, so no forcible
2623 * alignment is needed. Only require existence of
2624 * atomic_clear_64 when page size is 32768.
2625 */
2626 addr = (uintptr_t)&m->dirty;
2627#if PAGE_SIZE == 32768
2628 atomic_clear_64((uint64_t *)addr, pagebits);
2629#elif PAGE_SIZE == 16384
2630 atomic_clear_32((uint32_t *)addr, pagebits);
2631#else /* PAGE_SIZE <= 8192 */
2632 /*
2633 * Use a trick to perform a 32-bit atomic on the
2634 * containing aligned word, to not depend on the existence
2635 * of atomic_clear_{8, 16}.
2636 */
2637 shift = addr & (sizeof(uint32_t) - 1);
2638#if BYTE_ORDER == BIG_ENDIAN
2639 shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
2640#else
2641 shift *= NBBY;
2642#endif
2643 addr &= ~(sizeof(uint32_t) - 1);
2644 atomic_clear_32((uint32_t *)addr, pagebits << shift);
2645#endif /* PAGE_SIZE */
2646 }
2647}
2648
2649/*
2650 * vm_page_set_validclean:
2651 *
2652 * Sets portions of a page valid and clean. The arguments are expected
2653 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2654 * of any partial chunks touched by the range. The invalid portion of
2655 * such chunks will be zero'd.
2656 *
2657 * This routine may not block.
2658 *
2659 * (base + size) must be less then or equal to PAGE_SIZE.
2660 */
2661void
2662vm_page_set_validclean(vm_page_t m, int base, int size)
2663{
2664 vm_page_bits_t oldvalid, pagebits;
2665 int endoff, frag;
2666
2667 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2668 if (size == 0) /* handle degenerate case */
2669 return;
2670
2671 /*
2672 * If the base is not DEV_BSIZE aligned and the valid
2673 * bit is clear, we have to zero out a portion of the
2674 * first block.
2675 */
2676 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2677 (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0)
2678 pmap_zero_page_area(m, frag, base - frag);
2679
2680 /*
2681 * If the ending offset is not DEV_BSIZE aligned and the
2682 * valid bit is clear, we have to zero out a portion of
2683 * the last block.
2684 */
2685 endoff = base + size;
2686 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2687 (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0)
2688 pmap_zero_page_area(m, endoff,
2689 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2690
2691 /*
2692 * Set valid, clear dirty bits. If validating the entire
2693 * page we can safely clear the pmap modify bit. We also
2694 * use this opportunity to clear the VPO_NOSYNC flag. If a process
2695 * takes a write fault on a MAP_NOSYNC memory area the flag will
2696 * be set again.
2697 *
2698 * We set valid bits inclusive of any overlap, but we can only
2699 * clear dirty bits for DEV_BSIZE chunks that are fully within
2700 * the range.
2701 */
2702 oldvalid = m->valid;
2703 pagebits = vm_page_bits(base, size);
2704 m->valid |= pagebits;
2705#if 0 /* NOT YET */
2706 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
2707 frag = DEV_BSIZE - frag;
2708 base += frag;
2709 size -= frag;
2710 if (size < 0)
2711 size = 0;
2712 }
2713 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
2714#endif
2715 if (base == 0 && size == PAGE_SIZE) {
2716 /*
2717 * The page can only be modified within the pmap if it is
2718 * mapped, and it can only be mapped if it was previously
2719 * fully valid.
2720 */
2721 if (oldvalid == VM_PAGE_BITS_ALL)
2722 /*
2723 * Perform the pmap_clear_modify() first. Otherwise,
2724 * a concurrent pmap operation, such as
2725 * pmap_protect(), could clear a modification in the
2726 * pmap and set the dirty field on the page before
2727 * pmap_clear_modify() had begun and after the dirty
2728 * field was cleared here.
2729 */
2730 pmap_clear_modify(m);
2731 m->dirty = 0;
2732 m->oflags &= ~VPO_NOSYNC;
2733 } else if (oldvalid != VM_PAGE_BITS_ALL)
2734 m->dirty &= ~pagebits;
2735 else
2736 vm_page_clear_dirty_mask(m, pagebits);
2737}
2738
2739void
2740vm_page_clear_dirty(vm_page_t m, int base, int size)
2741{
2742
2743 vm_page_clear_dirty_mask(m, vm_page_bits(base, size));
2744}
2745
2746/*
2747 * vm_page_set_invalid:
2748 *
2749 * Invalidates DEV_BSIZE'd chunks within a page. Both the
2750 * valid and dirty bits for the effected areas are cleared.
2751 *
2752 * May not block.
2753 */
2754void
2755vm_page_set_invalid(vm_page_t m, int base, int size)
2756{
2757 vm_page_bits_t bits;
2758
2759 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2760 KASSERT((m->oflags & VPO_BUSY) == 0,
2761 ("vm_page_set_invalid: page %p is busy", m));
2762 bits = vm_page_bits(base, size);
2763 if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
2764 pmap_remove_all(m);
2765 KASSERT(!pmap_page_is_mapped(m),
2766 ("vm_page_set_invalid: page %p is mapped", m));
2767 m->valid &= ~bits;
2768 m->dirty &= ~bits;
2769}
2770
2771/*
2772 * vm_page_zero_invalid()
2773 *
2774 * The kernel assumes that the invalid portions of a page contain
2775 * garbage, but such pages can be mapped into memory by user code.
2776 * When this occurs, we must zero out the non-valid portions of the
2777 * page so user code sees what it expects.
2778 *
2779 * Pages are most often semi-valid when the end of a file is mapped
2780 * into memory and the file's size is not page aligned.
2781 */
2782void
2783vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
2784{
2785 int b;
2786 int i;
2787
2788 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2789 /*
2790 * Scan the valid bits looking for invalid sections that
2791 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
2792 * valid bit may be set ) have already been zerod by
2793 * vm_page_set_validclean().
2794 */
2795 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
2796 if (i == (PAGE_SIZE / DEV_BSIZE) ||
2797 (m->valid & ((vm_page_bits_t)1 << i))) {
2798 if (i > b) {
2799 pmap_zero_page_area(m,
2800 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
2801 }
2802 b = i + 1;
2803 }
2804 }
2805
2806 /*
2807 * setvalid is TRUE when we can safely set the zero'd areas
2808 * as being valid. We can do this if there are no cache consistancy
2809 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
2810 */
2811 if (setvalid)
2812 m->valid = VM_PAGE_BITS_ALL;
2813}
2814
2815/*
2816 * vm_page_is_valid:
2817 *
2818 * Is (partial) page valid? Note that the case where size == 0
2819 * will return FALSE in the degenerate case where the page is
2820 * entirely invalid, and TRUE otherwise.
2821 *
2822 * May not block.
2823 */
2824int
2825vm_page_is_valid(vm_page_t m, int base, int size)
2826{
2827 vm_page_bits_t bits;
2828
2829 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2830 bits = vm_page_bits(base, size);
2831 if (m->valid && ((m->valid & bits) == bits))
2832 return 1;
2833 else
2834 return 0;
2835}
2836
2837/*
2838 * update dirty bits from pmap/mmu. May not block.
2839 */
2840void
2841vm_page_test_dirty(vm_page_t m)
2842{
2843
2844 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2845 if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m))
2846 vm_page_dirty(m);
2847}
2848
2849void
2850vm_page_lock_KBI(vm_page_t m, const char *file, int line)
2851{
2852
2853 mtx_lock_flags_(vm_page_lockptr(m), 0, file, line);
2854}
2855
2856void
2857vm_page_unlock_KBI(vm_page_t m, const char *file, int line)
2858{
2859
2860 mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line);
2861}
2862
2863int
2864vm_page_trylock_KBI(vm_page_t m, const char *file, int line)
2865{
2866
2867 return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line));
2868}
2869
2870#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
2871void
2872vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line)
2873{
2874
2875 mtx_assert_(vm_page_lockptr(m), a, file, line);
2876}
2877#endif
2878
2879int so_zerocp_fullpage = 0;
2880
2881/*
2882 * Replace the given page with a copy. The copied page assumes
2883 * the portion of the given page's "wire_count" that is not the
2884 * responsibility of this copy-on-write mechanism.
2885 *
2886 * The object containing the given page must have a non-zero
2887 * paging-in-progress count and be locked.
2888 */
2889void
2890vm_page_cowfault(vm_page_t m)
2891{
2892 vm_page_t mnew;
2893 vm_object_t object;
2894 vm_pindex_t pindex;
2895
2896 mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
2897 vm_page_lock_assert(m, MA_OWNED);
2898 object = m->object;
2899 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2900 KASSERT(object->paging_in_progress != 0,
2901 ("vm_page_cowfault: object %p's paging-in-progress count is zero.",
2902 object));
2903 pindex = m->pindex;
2904
2905 retry_alloc:
2906 pmap_remove_all(m);
2907 vm_page_remove(m);
2908 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
2909 if (mnew == NULL) {
2910 vm_page_insert(m, object, pindex);
2911 vm_page_unlock(m);
2912 VM_OBJECT_UNLOCK(object);
2913 VM_WAIT;
2914 VM_OBJECT_LOCK(object);
2915 if (m == vm_page_lookup(object, pindex)) {
2916 vm_page_lock(m);
2917 goto retry_alloc;
2918 } else {
2919 /*
2920 * Page disappeared during the wait.
2921 */
2922 return;
2923 }
2924 }
2925
2926 if (m->cow == 0) {
2927 /*
2928 * check to see if we raced with an xmit complete when
2929 * waiting to allocate a page. If so, put things back
2930 * the way they were
2931 */
2932 vm_page_unlock(m);
2933 vm_page_lock(mnew);
2934 vm_page_free(mnew);
2935 vm_page_unlock(mnew);
2936 vm_page_insert(m, object, pindex);
2937 } else { /* clear COW & copy page */
2938 if (!so_zerocp_fullpage)
2939 pmap_copy_page(m, mnew);
2940 mnew->valid = VM_PAGE_BITS_ALL;
2941 vm_page_dirty(mnew);
2942 mnew->wire_count = m->wire_count - m->cow;
2943 m->wire_count = m->cow;
2944 vm_page_unlock(m);
2945 }
2946}
2947
2948void
2949vm_page_cowclear(vm_page_t m)
2950{
2951
2952 vm_page_lock_assert(m, MA_OWNED);
2953 if (m->cow) {
2954 m->cow--;
2955 /*
2956 * let vm_fault add back write permission lazily
2957 */
2958 }
2959 /*
2960 * sf_buf_free() will free the page, so we needn't do it here
2961 */
2962}
2963
2964int
2965vm_page_cowsetup(vm_page_t m)
2966{
2967
2968 vm_page_lock_assert(m, MA_OWNED);
2969 if ((m->flags & PG_FICTITIOUS) != 0 ||
2970 (m->oflags & VPO_UNMANAGED) != 0 ||
2971 m->cow == USHRT_MAX - 1 || !VM_OBJECT_TRYLOCK(m->object))
2972 return (EBUSY);
2973 m->cow++;
2974 pmap_remove_write(m);
2975 VM_OBJECT_UNLOCK(m->object);
2976 return (0);
2977}
2978
2979#ifdef INVARIANTS
2980void
2981vm_page_object_lock_assert(vm_page_t m)
2982{
2983
2984 /*
2985 * Certain of the page's fields may only be modified by the
2986 * holder of the containing object's lock or the setter of the
2987 * page's VPO_BUSY flag. Unfortunately, the setter of the
2988 * VPO_BUSY flag is not recorded, and thus cannot be checked
2989 * here.
2990 */
2991 if (m->object != NULL && (m->oflags & VPO_BUSY) == 0)
2992 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2993}
2994#endif
2995
2996#include "opt_ddb.h"
2997#ifdef DDB
2998#include <sys/kernel.h>
2999
3000#include <ddb/ddb.h>
3001
3002DB_SHOW_COMMAND(page, vm_page_print_page_info)
3003{
3004 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
3005 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
3006 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
3007 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
3008 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
3009 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
3010 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
3011 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
3012 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
3013 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
3014}
3015
3016DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
3017{
3018
3019 db_printf("PQ_FREE:");
3020 db_printf(" %d", cnt.v_free_count);
3021 db_printf("\n");
3022
3023 db_printf("PQ_CACHE:");
3024 db_printf(" %d", cnt.v_cache_count);
3025 db_printf("\n");
3026
3027 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
3028 *vm_page_queues[PQ_ACTIVE].cnt,
3029 *vm_page_queues[PQ_INACTIVE].cnt);
3030}
3031#endif /* DDB */
| 1607 if ((req & VM_ALLOC_WIRED) != 0)
1608 atomic_add_int(&cnt.v_wire_count, npages);
1609 oflags = VPO_UNMANAGED;
1610 if (object != NULL) {
1611 if ((req & VM_ALLOC_NOBUSY) == 0)
1612 oflags |= VPO_BUSY;
1613 if (object->memattr != VM_MEMATTR_DEFAULT &&
1614 memattr == VM_MEMATTR_DEFAULT)
1615 memattr = object->memattr;
1616 }
1617 for (m = m_ret; m < &m_ret[npages]; m++) {
1618 m->aflags = 0;
1619 m->flags &= flags;
1620 if ((req & VM_ALLOC_WIRED) != 0)
1621 m->wire_count = 1;
1622 /* Unmanaged pages don't use "act_count". */
1623 m->oflags = oflags;
1624 if (memattr != VM_MEMATTR_DEFAULT)
1625 pmap_page_set_memattr(m, memattr);
1626 if (object != NULL)
1627 vm_page_insert(m, object, pindex);
1628 else
1629 m->pindex = pindex;
1630 pindex++;
1631 }
1632 while (deferred_vdrop_list != NULL) {
1633 vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev);
1634 deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next;
1635 }
1636 if (vm_paging_needed())
1637 pagedaemon_wakeup();
1638 return (m_ret);
1639}
1640
1641/*
1642 * Initialize a page that has been freshly dequeued from a freelist.
1643 * The caller has to drop the vnode returned, if it is not NULL.
1644 *
1645 * This function may only be used to initialize unmanaged pages.
1646 *
1647 * To be called with vm_page_queue_free_mtx held.
1648 */
1649static struct vnode *
1650vm_page_alloc_init(vm_page_t m)
1651{
1652 struct vnode *drop;
1653 vm_object_t m_object;
1654
1655 KASSERT(m->queue == PQ_NONE,
1656 ("vm_page_alloc_init: page %p has unexpected queue %d",
1657 m, m->queue));
1658 KASSERT(m->wire_count == 0,
1659 ("vm_page_alloc_init: page %p is wired", m));
1660 KASSERT(m->hold_count == 0,
1661 ("vm_page_alloc_init: page %p is held", m));
1662 KASSERT(m->busy == 0,
1663 ("vm_page_alloc_init: page %p is busy", m));
1664 KASSERT(m->dirty == 0,
1665 ("vm_page_alloc_init: page %p is dirty", m));
1666 KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT,
1667 ("vm_page_alloc_init: page %p has unexpected memattr %d",
1668 m, pmap_page_get_memattr(m)));
1669 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1670 drop = NULL;
1671 if ((m->flags & PG_CACHED) != 0) {
1672 KASSERT((m->flags & PG_ZERO) == 0,
1673 ("vm_page_alloc_init: cached page %p is PG_ZERO", m));
1674 m->valid = 0;
1675 m_object = m->object;
1676 vm_page_cache_remove(m);
1677 if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
1678 drop = m_object->handle;
1679 } else {
1680 KASSERT(VM_PAGE_IS_FREE(m),
1681 ("vm_page_alloc_init: page %p is not free", m));
1682 KASSERT(m->valid == 0,
1683 ("vm_page_alloc_init: free page %p is valid", m));
1684 cnt.v_free_count--;
1685 if ((m->flags & PG_ZERO) != 0)
1686 vm_page_zero_count--;
1687 }
1688 /* Don't clear the PG_ZERO flag; we'll need it later. */
1689 m->flags &= PG_ZERO;
1690 return (drop);
1691}
1692
1693/*
1694 * vm_page_alloc_freelist:
1695 *
1696 * Allocate a physical page from the specified free page list.
1697 *
1698 * The caller must always specify an allocation class.
1699 *
1700 * allocation classes:
1701 * VM_ALLOC_NORMAL normal process request
1702 * VM_ALLOC_SYSTEM system *really* needs a page
1703 * VM_ALLOC_INTERRUPT interrupt time request
1704 *
1705 * optional allocation flags:
1706 * VM_ALLOC_COUNT(number) the number of additional pages that the caller
1707 * intends to allocate
1708 * VM_ALLOC_WIRED wire the allocated page
1709 * VM_ALLOC_ZERO prefer a zeroed page
1710 *
1711 * This routine may not sleep.
1712 */
1713vm_page_t
1714vm_page_alloc_freelist(int flind, int req)
1715{
1716 struct vnode *drop;
1717 vm_page_t m;
1718 u_int flags;
1719 int req_class;
1720
1721 req_class = req & VM_ALLOC_CLASS_MASK;
1722
1723 /*
1724 * The page daemon is allowed to dig deeper into the free page list.
1725 */
1726 if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT)
1727 req_class = VM_ALLOC_SYSTEM;
1728
1729 /*
1730 * Do not allocate reserved pages unless the req has asked for it.
1731 */
1732 mtx_lock(&vm_page_queue_free_mtx);
1733 if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved ||
1734 (req_class == VM_ALLOC_SYSTEM &&
1735 cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) ||
1736 (req_class == VM_ALLOC_INTERRUPT &&
1737 cnt.v_free_count + cnt.v_cache_count > 0))
1738 m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0);
1739 else {
1740 mtx_unlock(&vm_page_queue_free_mtx);
1741 atomic_add_int(&vm_pageout_deficit,
1742 max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
1743 pagedaemon_wakeup();
1744 return (NULL);
1745 }
1746 if (m == NULL) {
1747 mtx_unlock(&vm_page_queue_free_mtx);
1748 return (NULL);
1749 }
1750 drop = vm_page_alloc_init(m);
1751 mtx_unlock(&vm_page_queue_free_mtx);
1752
1753 /*
1754 * Initialize the page. Only the PG_ZERO flag is inherited.
1755 */
1756 m->aflags = 0;
1757 flags = 0;
1758 if ((req & VM_ALLOC_ZERO) != 0)
1759 flags = PG_ZERO;
1760 m->flags &= flags;
1761 if ((req & VM_ALLOC_WIRED) != 0) {
1762 /*
1763 * The page lock is not required for wiring a page that does
1764 * not belong to an object.
1765 */
1766 atomic_add_int(&cnt.v_wire_count, 1);
1767 m->wire_count = 1;
1768 }
1769 /* Unmanaged pages don't use "act_count". */
1770 m->oflags = VPO_UNMANAGED;
1771 if (drop != NULL)
1772 vdrop(drop);
1773 if (vm_paging_needed())
1774 pagedaemon_wakeup();
1775 return (m);
1776}
1777
1778/*
1779 * vm_wait: (also see VM_WAIT macro)
1780 *
1781 * Block until free pages are available for allocation
1782 * - Called in various places before memory allocations.
1783 */
1784void
1785vm_wait(void)
1786{
1787
1788 mtx_lock(&vm_page_queue_free_mtx);
1789 if (curproc == pageproc) {
1790 vm_pageout_pages_needed = 1;
1791 msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
1792 PDROP | PSWP, "VMWait", 0);
1793 } else {
1794 if (!vm_pages_needed) {
1795 vm_pages_needed = 1;
1796 wakeup(&vm_pages_needed);
1797 }
1798 msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PVM,
1799 "vmwait", 0);
1800 }
1801}
1802
1803/*
1804 * vm_waitpfault: (also see VM_WAITPFAULT macro)
1805 *
1806 * Block until free pages are available for allocation
1807 * - Called only in vm_fault so that processes page faulting
1808 * can be easily tracked.
1809 * - Sleeps at a lower priority than vm_wait() so that vm_wait()ing
1810 * processes will be able to grab memory first. Do not change
1811 * this balance without careful testing first.
1812 */
1813void
1814vm_waitpfault(void)
1815{
1816
1817 mtx_lock(&vm_page_queue_free_mtx);
1818 if (!vm_pages_needed) {
1819 vm_pages_needed = 1;
1820 wakeup(&vm_pages_needed);
1821 }
1822 msleep(&cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
1823 "pfault", 0);
1824}
1825
1826/*
1827 * vm_page_requeue:
1828 *
1829 * Move the given page to the tail of its present page queue.
1830 *
1831 * The page queues must be locked.
1832 */
1833void
1834vm_page_requeue(vm_page_t m)
1835{
1836 struct vpgqueues *vpq;
1837 int queue;
1838
1839 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1840 queue = m->queue;
1841 KASSERT(queue != PQ_NONE,
1842 ("vm_page_requeue: page %p is not queued", m));
1843 vpq = &vm_page_queues[queue];
1844 TAILQ_REMOVE(&vpq->pl, m, pageq);
1845 TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1846}
1847
1848/*
1849 * vm_page_queue_remove:
1850 *
1851 * Remove the given page from the specified queue.
1852 *
1853 * The page and page queues must be locked.
1854 */
1855static __inline void
1856vm_page_queue_remove(int queue, vm_page_t m)
1857{
1858 struct vpgqueues *pq;
1859
1860 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1861 vm_page_lock_assert(m, MA_OWNED);
1862 pq = &vm_page_queues[queue];
1863 TAILQ_REMOVE(&pq->pl, m, pageq);
1864 (*pq->cnt)--;
1865}
1866
1867/*
1868 * vm_pageq_remove:
1869 *
1870 * Remove a page from its queue.
1871 *
1872 * The given page must be locked.
1873 * This routine may not block.
1874 */
1875void
1876vm_pageq_remove(vm_page_t m)
1877{
1878 int queue;
1879
1880 vm_page_lock_assert(m, MA_OWNED);
1881 if ((queue = m->queue) != PQ_NONE) {
1882 vm_page_lock_queues();
1883 m->queue = PQ_NONE;
1884 vm_page_queue_remove(queue, m);
1885 vm_page_unlock_queues();
1886 }
1887}
1888
1889/*
1890 * vm_page_enqueue:
1891 *
1892 * Add the given page to the specified queue.
1893 *
1894 * The page queues must be locked.
1895 */
1896static void
1897vm_page_enqueue(int queue, vm_page_t m)
1898{
1899 struct vpgqueues *vpq;
1900
1901 vpq = &vm_page_queues[queue];
1902 m->queue = queue;
1903 TAILQ_INSERT_TAIL(&vpq->pl, m, pageq);
1904 ++*vpq->cnt;
1905}
1906
1907/*
1908 * vm_page_activate:
1909 *
1910 * Put the specified page on the active list (if appropriate).
1911 * Ensure that act_count is at least ACT_INIT but do not otherwise
1912 * mess with it.
1913 *
1914 * The page must be locked.
1915 * This routine may not block.
1916 */
1917void
1918vm_page_activate(vm_page_t m)
1919{
1920 int queue;
1921
1922 vm_page_lock_assert(m, MA_OWNED);
1923 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
1924 if ((queue = m->queue) != PQ_ACTIVE) {
1925 if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
1926 if (m->act_count < ACT_INIT)
1927 m->act_count = ACT_INIT;
1928 vm_page_lock_queues();
1929 if (queue != PQ_NONE)
1930 vm_page_queue_remove(queue, m);
1931 vm_page_enqueue(PQ_ACTIVE, m);
1932 vm_page_unlock_queues();
1933 } else
1934 KASSERT(queue == PQ_NONE,
1935 ("vm_page_activate: wired page %p is queued", m));
1936 } else {
1937 if (m->act_count < ACT_INIT)
1938 m->act_count = ACT_INIT;
1939 }
1940}
1941
1942/*
1943 * vm_page_free_wakeup:
1944 *
1945 * Helper routine for vm_page_free_toq() and vm_page_cache(). This
1946 * routine is called when a page has been added to the cache or free
1947 * queues.
1948 *
1949 * The page queues must be locked.
1950 * This routine may not block.
1951 */
1952static inline void
1953vm_page_free_wakeup(void)
1954{
1955
1956 mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
1957 /*
1958 * if pageout daemon needs pages, then tell it that there are
1959 * some free.
1960 */
1961 if (vm_pageout_pages_needed &&
1962 cnt.v_cache_count + cnt.v_free_count >= cnt.v_pageout_free_min) {
1963 wakeup(&vm_pageout_pages_needed);
1964 vm_pageout_pages_needed = 0;
1965 }
1966 /*
1967 * wakeup processes that are waiting on memory if we hit a
1968 * high water mark. And wakeup scheduler process if we have
1969 * lots of memory. this process will swapin processes.
1970 */
1971 if (vm_pages_needed && !vm_page_count_min()) {
1972 vm_pages_needed = 0;
1973 wakeup(&cnt.v_free_count);
1974 }
1975}
1976
1977/*
1978 * vm_page_free_toq:
1979 *
1980 * Returns the given page to the free list,
1981 * disassociating it with any VM object.
1982 *
1983 * Object and page must be locked prior to entry.
1984 * This routine may not block.
1985 */
1986
1987void
1988vm_page_free_toq(vm_page_t m)
1989{
1990
1991 if ((m->oflags & VPO_UNMANAGED) == 0) {
1992 vm_page_lock_assert(m, MA_OWNED);
1993 KASSERT(!pmap_page_is_mapped(m),
1994 ("vm_page_free_toq: freeing mapped page %p", m));
1995 }
1996 PCPU_INC(cnt.v_tfree);
1997
1998 if (VM_PAGE_IS_FREE(m))
1999 panic("vm_page_free: freeing free page %p", m);
2000 else if (m->busy != 0)
2001 panic("vm_page_free: freeing busy page %p", m);
2002
2003 /*
2004 * unqueue, then remove page. Note that we cannot destroy
2005 * the page here because we do not want to call the pager's
2006 * callback routine until after we've put the page on the
2007 * appropriate free queue.
2008 */
2009 if ((m->oflags & VPO_UNMANAGED) == 0)
2010 vm_pageq_remove(m);
2011 vm_page_remove(m);
2012
2013 /*
2014 * If fictitious remove object association and
2015 * return, otherwise delay object association removal.
2016 */
2017 if ((m->flags & PG_FICTITIOUS) != 0) {
2018 return;
2019 }
2020
2021 m->valid = 0;
2022 vm_page_undirty(m);
2023
2024 if (m->wire_count != 0)
2025 panic("vm_page_free: freeing wired page %p", m);
2026 if (m->hold_count != 0) {
2027 m->flags &= ~PG_ZERO;
2028 vm_page_lock_queues();
2029 vm_page_enqueue(PQ_HOLD, m);
2030 vm_page_unlock_queues();
2031 } else {
2032 /*
2033 * Restore the default memory attribute to the page.
2034 */
2035 if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
2036 pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
2037
2038 /*
2039 * Insert the page into the physical memory allocator's
2040 * cache/free page queues.
2041 */
2042 mtx_lock(&vm_page_queue_free_mtx);
2043 m->flags |= PG_FREE;
2044 cnt.v_free_count++;
2045#if VM_NRESERVLEVEL > 0
2046 if (!vm_reserv_free_page(m))
2047#else
2048 if (TRUE)
2049#endif
2050 vm_phys_free_pages(m, 0);
2051 if ((m->flags & PG_ZERO) != 0)
2052 ++vm_page_zero_count;
2053 else
2054 vm_page_zero_idle_wakeup();
2055 vm_page_free_wakeup();
2056 mtx_unlock(&vm_page_queue_free_mtx);
2057 }
2058}
2059
2060/*
2061 * vm_page_wire:
2062 *
2063 * Mark this page as wired down by yet
2064 * another map, removing it from paging queues
2065 * as necessary.
2066 *
2067 * If the page is fictitious, then its wire count must remain one.
2068 *
2069 * The page must be locked.
2070 * This routine may not block.
2071 */
2072void
2073vm_page_wire(vm_page_t m)
2074{
2075
2076 /*
2077 * Only bump the wire statistics if the page is not already wired,
2078 * and only unqueue the page if it is on some queue (if it is unmanaged
2079 * it is already off the queues).
2080 */
2081 vm_page_lock_assert(m, MA_OWNED);
2082 if ((m->flags & PG_FICTITIOUS) != 0) {
2083 KASSERT(m->wire_count == 1,
2084 ("vm_page_wire: fictitious page %p's wire count isn't one",
2085 m));
2086 return;
2087 }
2088 if (m->wire_count == 0) {
2089 if ((m->oflags & VPO_UNMANAGED) == 0)
2090 vm_pageq_remove(m);
2091 atomic_add_int(&cnt.v_wire_count, 1);
2092 }
2093 m->wire_count++;
2094 KASSERT(m->wire_count != 0, ("vm_page_wire: wire_count overflow m=%p", m));
2095}
2096
2097/*
2098 * vm_page_unwire:
2099 *
2100 * Release one wiring of the specified page, potentially enabling it to be
2101 * paged again. If paging is enabled, then the value of the parameter
2102 * "activate" determines to which queue the page is added. If "activate" is
2103 * non-zero, then the page is added to the active queue. Otherwise, it is
2104 * added to the inactive queue.
2105 *
2106 * However, unless the page belongs to an object, it is not enqueued because
2107 * it cannot be paged out.
2108 *
2109 * If a page is fictitious, then its wire count must alway be one.
2110 *
2111 * A managed page must be locked.
2112 */
2113void
2114vm_page_unwire(vm_page_t m, int activate)
2115{
2116
2117 if ((m->oflags & VPO_UNMANAGED) == 0)
2118 vm_page_lock_assert(m, MA_OWNED);
2119 if ((m->flags & PG_FICTITIOUS) != 0) {
2120 KASSERT(m->wire_count == 1,
2121 ("vm_page_unwire: fictitious page %p's wire count isn't one", m));
2122 return;
2123 }
2124 if (m->wire_count > 0) {
2125 m->wire_count--;
2126 if (m->wire_count == 0) {
2127 atomic_subtract_int(&cnt.v_wire_count, 1);
2128 if ((m->oflags & VPO_UNMANAGED) != 0 ||
2129 m->object == NULL)
2130 return;
2131 vm_page_lock_queues();
2132 if (activate)
2133 vm_page_enqueue(PQ_ACTIVE, m);
2134 else {
2135 m->flags &= ~PG_WINATCFLS;
2136 vm_page_enqueue(PQ_INACTIVE, m);
2137 }
2138 vm_page_unlock_queues();
2139 }
2140 } else
2141 panic("vm_page_unwire: page %p's wire count is zero", m);
2142}
2143
2144/*
2145 * Move the specified page to the inactive queue.
2146 *
2147 * Many pages placed on the inactive queue should actually go
2148 * into the cache, but it is difficult to figure out which. What
2149 * we do instead, if the inactive target is well met, is to put
2150 * clean pages at the head of the inactive queue instead of the tail.
2151 * This will cause them to be moved to the cache more quickly and
2152 * if not actively re-referenced, reclaimed more quickly. If we just
2153 * stick these pages at the end of the inactive queue, heavy filesystem
2154 * meta-data accesses can cause an unnecessary paging load on memory bound
2155 * processes. This optimization causes one-time-use metadata to be
2156 * reused more quickly.
2157 *
2158 * Normally athead is 0 resulting in LRU operation. athead is set
2159 * to 1 if we want this page to be 'as if it were placed in the cache',
2160 * except without unmapping it from the process address space.
2161 *
2162 * This routine may not block.
2163 */
2164static inline void
2165_vm_page_deactivate(vm_page_t m, int athead)
2166{
2167 int queue;
2168
2169 vm_page_lock_assert(m, MA_OWNED);
2170
2171 /*
2172 * Ignore if already inactive.
2173 */
2174 if ((queue = m->queue) == PQ_INACTIVE)
2175 return;
2176 if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) {
2177 vm_page_lock_queues();
2178 m->flags &= ~PG_WINATCFLS;
2179 if (queue != PQ_NONE)
2180 vm_page_queue_remove(queue, m);
2181 if (athead)
2182 TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m,
2183 pageq);
2184 else
2185 TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
2186 pageq);
2187 m->queue = PQ_INACTIVE;
2188 cnt.v_inactive_count++;
2189 vm_page_unlock_queues();
2190 }
2191}
2192
2193/*
2194 * Move the specified page to the inactive queue.
2195 *
2196 * The page must be locked.
2197 */
2198void
2199vm_page_deactivate(vm_page_t m)
2200{
2201
2202 _vm_page_deactivate(m, 0);
2203}
2204
2205/*
2206 * vm_page_try_to_cache:
2207 *
2208 * Returns 0 on failure, 1 on success
2209 */
2210int
2211vm_page_try_to_cache(vm_page_t m)
2212{
2213
2214 vm_page_lock_assert(m, MA_OWNED);
2215 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2216 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
2217 (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0)
2218 return (0);
2219 pmap_remove_all(m);
2220 if (m->dirty)
2221 return (0);
2222 vm_page_cache(m);
2223 return (1);
2224}
2225
2226/*
2227 * vm_page_try_to_free()
2228 *
2229 * Attempt to free the page. If we cannot free it, we do nothing.
2230 * 1 is returned on success, 0 on failure.
2231 */
2232int
2233vm_page_try_to_free(vm_page_t m)
2234{
2235
2236 vm_page_lock_assert(m, MA_OWNED);
2237 if (m->object != NULL)
2238 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2239 if (m->dirty || m->hold_count || m->busy || m->wire_count ||
2240 (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0)
2241 return (0);
2242 pmap_remove_all(m);
2243 if (m->dirty)
2244 return (0);
2245 vm_page_free(m);
2246 return (1);
2247}
2248
2249/*
2250 * vm_page_cache
2251 *
2252 * Put the specified page onto the page cache queue (if appropriate).
2253 *
2254 * This routine may not block.
2255 */
2256void
2257vm_page_cache(vm_page_t m)
2258{
2259 vm_object_t object;
2260 vm_page_t next, prev, root;
2261
2262 vm_page_lock_assert(m, MA_OWNED);
2263 object = m->object;
2264 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2265 if ((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) || m->busy ||
2266 m->hold_count || m->wire_count)
2267 panic("vm_page_cache: attempting to cache busy page");
2268 pmap_remove_all(m);
2269 if (m->dirty != 0)
2270 panic("vm_page_cache: page %p is dirty", m);
2271 if (m->valid == 0 || object->type == OBJT_DEFAULT ||
2272 (object->type == OBJT_SWAP &&
2273 !vm_pager_has_page(object, m->pindex, NULL, NULL))) {
2274 /*
2275 * Hypothesis: A cache-elgible page belonging to a
2276 * default object or swap object but without a backing
2277 * store must be zero filled.
2278 */
2279 vm_page_free(m);
2280 return;
2281 }
2282 KASSERT((m->flags & PG_CACHED) == 0,
2283 ("vm_page_cache: page %p is already cached", m));
2284 PCPU_INC(cnt.v_tcached);
2285
2286 /*
2287 * Remove the page from the paging queues.
2288 */
2289 vm_pageq_remove(m);
2290
2291 /*
2292 * Remove the page from the object's collection of resident
2293 * pages.
2294 */
2295 if ((next = TAILQ_NEXT(m, listq)) != NULL && next->left == m) {
2296 /*
2297 * Since the page's successor in the list is also its parent
2298 * in the tree, its right subtree must be empty.
2299 */
2300 next->left = m->left;
2301 KASSERT(m->right == NULL,
2302 ("vm_page_cache: page %p has right child", m));
2303 } else if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
2304 prev->right == m) {
2305 /*
2306 * Since the page's predecessor in the list is also its parent
2307 * in the tree, its left subtree must be empty.
2308 */
2309 KASSERT(m->left == NULL,
2310 ("vm_page_cache: page %p has left child", m));
2311 prev->right = m->right;
2312 } else {
2313 if (m != object->root)
2314 vm_page_splay(m->pindex, object->root);
2315 if (m->left == NULL)
2316 root = m->right;
2317 else if (m->right == NULL)
2318 root = m->left;
2319 else {
2320 /*
2321 * Move the page's successor to the root, because
2322 * pages are usually removed in ascending order.
2323 */
2324 if (m->right != next)
2325 vm_page_splay(m->pindex, m->right);
2326 next->left = m->left;
2327 root = next;
2328 }
2329 object->root = root;
2330 }
2331 TAILQ_REMOVE(&object->memq, m, listq);
2332 object->resident_page_count--;
2333
2334 /*
2335 * Restore the default memory attribute to the page.
2336 */
2337 if (pmap_page_get_memattr(m) != VM_MEMATTR_DEFAULT)
2338 pmap_page_set_memattr(m, VM_MEMATTR_DEFAULT);
2339
2340 /*
2341 * Insert the page into the object's collection of cached pages
2342 * and the physical memory allocator's cache/free page queues.
2343 */
2344 m->flags &= ~PG_ZERO;
2345 mtx_lock(&vm_page_queue_free_mtx);
2346 m->flags |= PG_CACHED;
2347 cnt.v_cache_count++;
2348 root = object->cache;
2349 if (root == NULL) {
2350 m->left = NULL;
2351 m->right = NULL;
2352 } else {
2353 root = vm_page_splay(m->pindex, root);
2354 if (m->pindex < root->pindex) {
2355 m->left = root->left;
2356 m->right = root;
2357 root->left = NULL;
2358 } else if (__predict_false(m->pindex == root->pindex))
2359 panic("vm_page_cache: offset already cached");
2360 else {
2361 m->right = root->right;
2362 m->left = root;
2363 root->right = NULL;
2364 }
2365 }
2366 object->cache = m;
2367#if VM_NRESERVLEVEL > 0
2368 if (!vm_reserv_free_page(m)) {
2369#else
2370 if (TRUE) {
2371#endif
2372 vm_phys_set_pool(VM_FREEPOOL_CACHE, m, 0);
2373 vm_phys_free_pages(m, 0);
2374 }
2375 vm_page_free_wakeup();
2376 mtx_unlock(&vm_page_queue_free_mtx);
2377
2378 /*
2379 * Increment the vnode's hold count if this is the object's only
2380 * cached page. Decrement the vnode's hold count if this was
2381 * the object's only resident page.
2382 */
2383 if (object->type == OBJT_VNODE) {
2384 if (root == NULL && object->resident_page_count != 0)
2385 vhold(object->handle);
2386 else if (root != NULL && object->resident_page_count == 0)
2387 vdrop(object->handle);
2388 }
2389}
2390
2391/*
2392 * vm_page_dontneed
2393 *
2394 * Cache, deactivate, or do nothing as appropriate. This routine
2395 * is typically used by madvise() MADV_DONTNEED.
2396 *
2397 * Generally speaking we want to move the page into the cache so
2398 * it gets reused quickly. However, this can result in a silly syndrome
2399 * due to the page recycling too quickly. Small objects will not be
2400 * fully cached. On the otherhand, if we move the page to the inactive
2401 * queue we wind up with a problem whereby very large objects
2402 * unnecessarily blow away our inactive and cache queues.
2403 *
2404 * The solution is to move the pages based on a fixed weighting. We
2405 * either leave them alone, deactivate them, or move them to the cache,
2406 * where moving them to the cache has the highest weighting.
2407 * By forcing some pages into other queues we eventually force the
2408 * system to balance the queues, potentially recovering other unrelated
2409 * space from active. The idea is to not force this to happen too
2410 * often.
2411 */
2412void
2413vm_page_dontneed(vm_page_t m)
2414{
2415 int dnw;
2416 int head;
2417
2418 vm_page_lock_assert(m, MA_OWNED);
2419 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2420 dnw = PCPU_GET(dnweight);
2421 PCPU_INC(dnweight);
2422
2423 /*
2424 * Occasionally leave the page alone.
2425 */
2426 if ((dnw & 0x01F0) == 0 || m->queue == PQ_INACTIVE) {
2427 if (m->act_count >= ACT_INIT)
2428 --m->act_count;
2429 return;
2430 }
2431
2432 /*
2433 * Clear any references to the page. Otherwise, the page daemon will
2434 * immediately reactivate the page.
2435 *
2436 * Perform the pmap_clear_reference() first. Otherwise, a concurrent
2437 * pmap operation, such as pmap_remove(), could clear a reference in
2438 * the pmap and set PGA_REFERENCED on the page before the
2439 * pmap_clear_reference() had completed. Consequently, the page would
2440 * appear referenced based upon an old reference that occurred before
2441 * this function ran.
2442 */
2443 pmap_clear_reference(m);
2444 vm_page_aflag_clear(m, PGA_REFERENCED);
2445
2446 if (m->dirty == 0 && pmap_is_modified(m))
2447 vm_page_dirty(m);
2448
2449 if (m->dirty || (dnw & 0x0070) == 0) {
2450 /*
2451 * Deactivate the page 3 times out of 32.
2452 */
2453 head = 0;
2454 } else {
2455 /*
2456 * Cache the page 28 times out of every 32. Note that
2457 * the page is deactivated instead of cached, but placed
2458 * at the head of the queue instead of the tail.
2459 */
2460 head = 1;
2461 }
2462 _vm_page_deactivate(m, head);
2463}
2464
2465/*
2466 * Grab a page, waiting until we are waken up due to the page
2467 * changing state. We keep on waiting, if the page continues
2468 * to be in the object. If the page doesn't exist, first allocate it
2469 * and then conditionally zero it.
2470 *
2471 * The caller must always specify the VM_ALLOC_RETRY flag. This is intended
2472 * to facilitate its eventual removal.
2473 *
2474 * This routine may block.
2475 */
2476vm_page_t
2477vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
2478{
2479 vm_page_t m;
2480
2481 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2482 KASSERT((allocflags & VM_ALLOC_RETRY) != 0,
2483 ("vm_page_grab: VM_ALLOC_RETRY is required"));
2484retrylookup:
2485 if ((m = vm_page_lookup(object, pindex)) != NULL) {
2486 if ((m->oflags & VPO_BUSY) != 0 ||
2487 ((allocflags & VM_ALLOC_IGN_SBUSY) == 0 && m->busy != 0)) {
2488 /*
2489 * Reference the page before unlocking and
2490 * sleeping so that the page daemon is less
2491 * likely to reclaim it.
2492 */
2493 vm_page_aflag_set(m, PGA_REFERENCED);
2494 vm_page_sleep(m, "pgrbwt");
2495 goto retrylookup;
2496 } else {
2497 if ((allocflags & VM_ALLOC_WIRED) != 0) {
2498 vm_page_lock(m);
2499 vm_page_wire(m);
2500 vm_page_unlock(m);
2501 }
2502 if ((allocflags & VM_ALLOC_NOBUSY) == 0)
2503 vm_page_busy(m);
2504 return (m);
2505 }
2506 }
2507 m = vm_page_alloc(object, pindex, allocflags & ~(VM_ALLOC_RETRY |
2508 VM_ALLOC_IGN_SBUSY));
2509 if (m == NULL) {
2510 VM_OBJECT_UNLOCK(object);
2511 VM_WAIT;
2512 VM_OBJECT_LOCK(object);
2513 goto retrylookup;
2514 } else if (m->valid != 0)
2515 return (m);
2516 if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
2517 pmap_zero_page(m);
2518 return (m);
2519}
2520
2521/*
2522 * Mapping function for valid bits or for dirty bits in
2523 * a page. May not block.
2524 *
2525 * Inputs are required to range within a page.
2526 */
2527vm_page_bits_t
2528vm_page_bits(int base, int size)
2529{
2530 int first_bit;
2531 int last_bit;
2532
2533 KASSERT(
2534 base + size <= PAGE_SIZE,
2535 ("vm_page_bits: illegal base/size %d/%d", base, size)
2536 );
2537
2538 if (size == 0) /* handle degenerate case */
2539 return (0);
2540
2541 first_bit = base >> DEV_BSHIFT;
2542 last_bit = (base + size - 1) >> DEV_BSHIFT;
2543
2544 return (((vm_page_bits_t)2 << last_bit) -
2545 ((vm_page_bits_t)1 << first_bit));
2546}
2547
2548/*
2549 * vm_page_set_valid_range:
2550 *
2551 * Sets portions of a page valid. The arguments are expected
2552 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2553 * of any partial chunks touched by the range. The invalid portion of
2554 * such chunks will be zeroed.
2555 *
2556 * (base + size) must be less then or equal to PAGE_SIZE.
2557 */
2558void
2559vm_page_set_valid_range(vm_page_t m, int base, int size)
2560{
2561 int endoff, frag;
2562
2563 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2564 if (size == 0) /* handle degenerate case */
2565 return;
2566
2567 /*
2568 * If the base is not DEV_BSIZE aligned and the valid
2569 * bit is clear, we have to zero out a portion of the
2570 * first block.
2571 */
2572 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2573 (m->valid & (1 << (base >> DEV_BSHIFT))) == 0)
2574 pmap_zero_page_area(m, frag, base - frag);
2575
2576 /*
2577 * If the ending offset is not DEV_BSIZE aligned and the
2578 * valid bit is clear, we have to zero out a portion of
2579 * the last block.
2580 */
2581 endoff = base + size;
2582 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2583 (m->valid & (1 << (endoff >> DEV_BSHIFT))) == 0)
2584 pmap_zero_page_area(m, endoff,
2585 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2586
2587 /*
2588 * Assert that no previously invalid block that is now being validated
2589 * is already dirty.
2590 */
2591 KASSERT((~m->valid & vm_page_bits(base, size) & m->dirty) == 0,
2592 ("vm_page_set_valid_range: page %p is dirty", m));
2593
2594 /*
2595 * Set valid bits inclusive of any overlap.
2596 */
2597 m->valid |= vm_page_bits(base, size);
2598}
2599
2600/*
2601 * Clear the given bits from the specified page's dirty field.
2602 */
2603static __inline void
2604vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits)
2605{
2606 uintptr_t addr;
2607#if PAGE_SIZE < 16384
2608 int shift;
2609#endif
2610
2611 /*
2612 * If the object is locked and the page is neither VPO_BUSY nor
2613 * PGA_WRITEABLE, then the page's dirty field cannot possibly be
2614 * set by a concurrent pmap operation.
2615 */
2616 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2617 if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0)
2618 m->dirty &= ~pagebits;
2619 else {
2620 /*
2621 * The pmap layer can call vm_page_dirty() without
2622 * holding a distinguished lock. The combination of
2623 * the object's lock and an atomic operation suffice
2624 * to guarantee consistency of the page dirty field.
2625 *
2626 * For PAGE_SIZE == 32768 case, compiler already
2627 * properly aligns the dirty field, so no forcible
2628 * alignment is needed. Only require existence of
2629 * atomic_clear_64 when page size is 32768.
2630 */
2631 addr = (uintptr_t)&m->dirty;
2632#if PAGE_SIZE == 32768
2633 atomic_clear_64((uint64_t *)addr, pagebits);
2634#elif PAGE_SIZE == 16384
2635 atomic_clear_32((uint32_t *)addr, pagebits);
2636#else /* PAGE_SIZE <= 8192 */
2637 /*
2638 * Use a trick to perform a 32-bit atomic on the
2639 * containing aligned word, to not depend on the existence
2640 * of atomic_clear_{8, 16}.
2641 */
2642 shift = addr & (sizeof(uint32_t) - 1);
2643#if BYTE_ORDER == BIG_ENDIAN
2644 shift = (sizeof(uint32_t) - sizeof(m->dirty) - shift) * NBBY;
2645#else
2646 shift *= NBBY;
2647#endif
2648 addr &= ~(sizeof(uint32_t) - 1);
2649 atomic_clear_32((uint32_t *)addr, pagebits << shift);
2650#endif /* PAGE_SIZE */
2651 }
2652}
2653
2654/*
2655 * vm_page_set_validclean:
2656 *
2657 * Sets portions of a page valid and clean. The arguments are expected
2658 * to be DEV_BSIZE aligned but if they aren't the bitmap is inclusive
2659 * of any partial chunks touched by the range. The invalid portion of
2660 * such chunks will be zero'd.
2661 *
2662 * This routine may not block.
2663 *
2664 * (base + size) must be less then or equal to PAGE_SIZE.
2665 */
2666void
2667vm_page_set_validclean(vm_page_t m, int base, int size)
2668{
2669 vm_page_bits_t oldvalid, pagebits;
2670 int endoff, frag;
2671
2672 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2673 if (size == 0) /* handle degenerate case */
2674 return;
2675
2676 /*
2677 * If the base is not DEV_BSIZE aligned and the valid
2678 * bit is clear, we have to zero out a portion of the
2679 * first block.
2680 */
2681 if ((frag = base & ~(DEV_BSIZE - 1)) != base &&
2682 (m->valid & ((vm_page_bits_t)1 << (base >> DEV_BSHIFT))) == 0)
2683 pmap_zero_page_area(m, frag, base - frag);
2684
2685 /*
2686 * If the ending offset is not DEV_BSIZE aligned and the
2687 * valid bit is clear, we have to zero out a portion of
2688 * the last block.
2689 */
2690 endoff = base + size;
2691 if ((frag = endoff & ~(DEV_BSIZE - 1)) != endoff &&
2692 (m->valid & ((vm_page_bits_t)1 << (endoff >> DEV_BSHIFT))) == 0)
2693 pmap_zero_page_area(m, endoff,
2694 DEV_BSIZE - (endoff & (DEV_BSIZE - 1)));
2695
2696 /*
2697 * Set valid, clear dirty bits. If validating the entire
2698 * page we can safely clear the pmap modify bit. We also
2699 * use this opportunity to clear the VPO_NOSYNC flag. If a process
2700 * takes a write fault on a MAP_NOSYNC memory area the flag will
2701 * be set again.
2702 *
2703 * We set valid bits inclusive of any overlap, but we can only
2704 * clear dirty bits for DEV_BSIZE chunks that are fully within
2705 * the range.
2706 */
2707 oldvalid = m->valid;
2708 pagebits = vm_page_bits(base, size);
2709 m->valid |= pagebits;
2710#if 0 /* NOT YET */
2711 if ((frag = base & (DEV_BSIZE - 1)) != 0) {
2712 frag = DEV_BSIZE - frag;
2713 base += frag;
2714 size -= frag;
2715 if (size < 0)
2716 size = 0;
2717 }
2718 pagebits = vm_page_bits(base, size & (DEV_BSIZE - 1));
2719#endif
2720 if (base == 0 && size == PAGE_SIZE) {
2721 /*
2722 * The page can only be modified within the pmap if it is
2723 * mapped, and it can only be mapped if it was previously
2724 * fully valid.
2725 */
2726 if (oldvalid == VM_PAGE_BITS_ALL)
2727 /*
2728 * Perform the pmap_clear_modify() first. Otherwise,
2729 * a concurrent pmap operation, such as
2730 * pmap_protect(), could clear a modification in the
2731 * pmap and set the dirty field on the page before
2732 * pmap_clear_modify() had begun and after the dirty
2733 * field was cleared here.
2734 */
2735 pmap_clear_modify(m);
2736 m->dirty = 0;
2737 m->oflags &= ~VPO_NOSYNC;
2738 } else if (oldvalid != VM_PAGE_BITS_ALL)
2739 m->dirty &= ~pagebits;
2740 else
2741 vm_page_clear_dirty_mask(m, pagebits);
2742}
2743
2744void
2745vm_page_clear_dirty(vm_page_t m, int base, int size)
2746{
2747
2748 vm_page_clear_dirty_mask(m, vm_page_bits(base, size));
2749}
2750
2751/*
2752 * vm_page_set_invalid:
2753 *
2754 * Invalidates DEV_BSIZE'd chunks within a page. Both the
2755 * valid and dirty bits for the effected areas are cleared.
2756 *
2757 * May not block.
2758 */
2759void
2760vm_page_set_invalid(vm_page_t m, int base, int size)
2761{
2762 vm_page_bits_t bits;
2763
2764 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2765 KASSERT((m->oflags & VPO_BUSY) == 0,
2766 ("vm_page_set_invalid: page %p is busy", m));
2767 bits = vm_page_bits(base, size);
2768 if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
2769 pmap_remove_all(m);
2770 KASSERT(!pmap_page_is_mapped(m),
2771 ("vm_page_set_invalid: page %p is mapped", m));
2772 m->valid &= ~bits;
2773 m->dirty &= ~bits;
2774}
2775
2776/*
2777 * vm_page_zero_invalid()
2778 *
2779 * The kernel assumes that the invalid portions of a page contain
2780 * garbage, but such pages can be mapped into memory by user code.
2781 * When this occurs, we must zero out the non-valid portions of the
2782 * page so user code sees what it expects.
2783 *
2784 * Pages are most often semi-valid when the end of a file is mapped
2785 * into memory and the file's size is not page aligned.
2786 */
2787void
2788vm_page_zero_invalid(vm_page_t m, boolean_t setvalid)
2789{
2790 int b;
2791 int i;
2792
2793 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2794 /*
2795 * Scan the valid bits looking for invalid sections that
2796 * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the
2797 * valid bit may be set ) have already been zerod by
2798 * vm_page_set_validclean().
2799 */
2800 for (b = i = 0; i <= PAGE_SIZE / DEV_BSIZE; ++i) {
2801 if (i == (PAGE_SIZE / DEV_BSIZE) ||
2802 (m->valid & ((vm_page_bits_t)1 << i))) {
2803 if (i > b) {
2804 pmap_zero_page_area(m,
2805 b << DEV_BSHIFT, (i - b) << DEV_BSHIFT);
2806 }
2807 b = i + 1;
2808 }
2809 }
2810
2811 /*
2812 * setvalid is TRUE when we can safely set the zero'd areas
2813 * as being valid. We can do this if there are no cache consistancy
2814 * issues. e.g. it is ok to do with UFS, but not ok to do with NFS.
2815 */
2816 if (setvalid)
2817 m->valid = VM_PAGE_BITS_ALL;
2818}
2819
2820/*
2821 * vm_page_is_valid:
2822 *
2823 * Is (partial) page valid? Note that the case where size == 0
2824 * will return FALSE in the degenerate case where the page is
2825 * entirely invalid, and TRUE otherwise.
2826 *
2827 * May not block.
2828 */
2829int
2830vm_page_is_valid(vm_page_t m, int base, int size)
2831{
2832 vm_page_bits_t bits;
2833
2834 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2835 bits = vm_page_bits(base, size);
2836 if (m->valid && ((m->valid & bits) == bits))
2837 return 1;
2838 else
2839 return 0;
2840}
2841
2842/*
2843 * update dirty bits from pmap/mmu. May not block.
2844 */
2845void
2846vm_page_test_dirty(vm_page_t m)
2847{
2848
2849 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2850 if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m))
2851 vm_page_dirty(m);
2852}
2853
2854void
2855vm_page_lock_KBI(vm_page_t m, const char *file, int line)
2856{
2857
2858 mtx_lock_flags_(vm_page_lockptr(m), 0, file, line);
2859}
2860
2861void
2862vm_page_unlock_KBI(vm_page_t m, const char *file, int line)
2863{
2864
2865 mtx_unlock_flags_(vm_page_lockptr(m), 0, file, line);
2866}
2867
2868int
2869vm_page_trylock_KBI(vm_page_t m, const char *file, int line)
2870{
2871
2872 return (mtx_trylock_flags_(vm_page_lockptr(m), 0, file, line));
2873}
2874
2875#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
2876void
2877vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line)
2878{
2879
2880 mtx_assert_(vm_page_lockptr(m), a, file, line);
2881}
2882#endif
2883
2884int so_zerocp_fullpage = 0;
2885
2886/*
2887 * Replace the given page with a copy. The copied page assumes
2888 * the portion of the given page's "wire_count" that is not the
2889 * responsibility of this copy-on-write mechanism.
2890 *
2891 * The object containing the given page must have a non-zero
2892 * paging-in-progress count and be locked.
2893 */
2894void
2895vm_page_cowfault(vm_page_t m)
2896{
2897 vm_page_t mnew;
2898 vm_object_t object;
2899 vm_pindex_t pindex;
2900
2901 mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
2902 vm_page_lock_assert(m, MA_OWNED);
2903 object = m->object;
2904 VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
2905 KASSERT(object->paging_in_progress != 0,
2906 ("vm_page_cowfault: object %p's paging-in-progress count is zero.",
2907 object));
2908 pindex = m->pindex;
2909
2910 retry_alloc:
2911 pmap_remove_all(m);
2912 vm_page_remove(m);
2913 mnew = vm_page_alloc(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY);
2914 if (mnew == NULL) {
2915 vm_page_insert(m, object, pindex);
2916 vm_page_unlock(m);
2917 VM_OBJECT_UNLOCK(object);
2918 VM_WAIT;
2919 VM_OBJECT_LOCK(object);
2920 if (m == vm_page_lookup(object, pindex)) {
2921 vm_page_lock(m);
2922 goto retry_alloc;
2923 } else {
2924 /*
2925 * Page disappeared during the wait.
2926 */
2927 return;
2928 }
2929 }
2930
2931 if (m->cow == 0) {
2932 /*
2933 * check to see if we raced with an xmit complete when
2934 * waiting to allocate a page. If so, put things back
2935 * the way they were
2936 */
2937 vm_page_unlock(m);
2938 vm_page_lock(mnew);
2939 vm_page_free(mnew);
2940 vm_page_unlock(mnew);
2941 vm_page_insert(m, object, pindex);
2942 } else { /* clear COW & copy page */
2943 if (!so_zerocp_fullpage)
2944 pmap_copy_page(m, mnew);
2945 mnew->valid = VM_PAGE_BITS_ALL;
2946 vm_page_dirty(mnew);
2947 mnew->wire_count = m->wire_count - m->cow;
2948 m->wire_count = m->cow;
2949 vm_page_unlock(m);
2950 }
2951}
2952
2953void
2954vm_page_cowclear(vm_page_t m)
2955{
2956
2957 vm_page_lock_assert(m, MA_OWNED);
2958 if (m->cow) {
2959 m->cow--;
2960 /*
2961 * let vm_fault add back write permission lazily
2962 */
2963 }
2964 /*
2965 * sf_buf_free() will free the page, so we needn't do it here
2966 */
2967}
2968
2969int
2970vm_page_cowsetup(vm_page_t m)
2971{
2972
2973 vm_page_lock_assert(m, MA_OWNED);
2974 if ((m->flags & PG_FICTITIOUS) != 0 ||
2975 (m->oflags & VPO_UNMANAGED) != 0 ||
2976 m->cow == USHRT_MAX - 1 || !VM_OBJECT_TRYLOCK(m->object))
2977 return (EBUSY);
2978 m->cow++;
2979 pmap_remove_write(m);
2980 VM_OBJECT_UNLOCK(m->object);
2981 return (0);
2982}
2983
2984#ifdef INVARIANTS
2985void
2986vm_page_object_lock_assert(vm_page_t m)
2987{
2988
2989 /*
2990 * Certain of the page's fields may only be modified by the
2991 * holder of the containing object's lock or the setter of the
2992 * page's VPO_BUSY flag. Unfortunately, the setter of the
2993 * VPO_BUSY flag is not recorded, and thus cannot be checked
2994 * here.
2995 */
2996 if (m->object != NULL && (m->oflags & VPO_BUSY) == 0)
2997 VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
2998}
2999#endif
3000
3001#include "opt_ddb.h"
3002#ifdef DDB
3003#include <sys/kernel.h>
3004
3005#include <ddb/ddb.h>
3006
3007DB_SHOW_COMMAND(page, vm_page_print_page_info)
3008{
3009 db_printf("cnt.v_free_count: %d\n", cnt.v_free_count);
3010 db_printf("cnt.v_cache_count: %d\n", cnt.v_cache_count);
3011 db_printf("cnt.v_inactive_count: %d\n", cnt.v_inactive_count);
3012 db_printf("cnt.v_active_count: %d\n", cnt.v_active_count);
3013 db_printf("cnt.v_wire_count: %d\n", cnt.v_wire_count);
3014 db_printf("cnt.v_free_reserved: %d\n", cnt.v_free_reserved);
3015 db_printf("cnt.v_free_min: %d\n", cnt.v_free_min);
3016 db_printf("cnt.v_free_target: %d\n", cnt.v_free_target);
3017 db_printf("cnt.v_cache_min: %d\n", cnt.v_cache_min);
3018 db_printf("cnt.v_inactive_target: %d\n", cnt.v_inactive_target);
3019}
3020
3021DB_SHOW_COMMAND(pageq, vm_page_print_pageq_info)
3022{
3023
3024 db_printf("PQ_FREE:");
3025 db_printf(" %d", cnt.v_free_count);
3026 db_printf("\n");
3027
3028 db_printf("PQ_CACHE:");
3029 db_printf(" %d", cnt.v_cache_count);
3030 db_printf("\n");
3031
3032 db_printf("PQ_ACTIVE: %d, PQ_INACTIVE: %d\n",
3033 *vm_page_queues[PQ_ACTIVE].cnt,
3034 *vm_page_queues[PQ_INACTIVE].cnt);
3035}
3036#endif /* DDB */
|