phys_pager.c revision 327785 
1/*-
2 * Copyright (c) 2000 Peter Wemm
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: stable/11/sys/vm/phys_pager.c 327785 2018-01-10 20:39:26Z markj $");
28
29#include <sys/param.h>
30#include <sys/systm.h>
31#include <sys/conf.h>
32#include <sys/kernel.h>
33#include <sys/lock.h>
34#include <sys/proc.h>
35#include <sys/mutex.h>
36#include <sys/mman.h>
37#include <sys/rwlock.h>
38#include <sys/sysctl.h>
39
40#include <vm/vm.h>
41#include <vm/vm_param.h>
42#include <vm/vm_object.h>
43#include <vm/vm_page.h>
44#include <vm/vm_pageout.h>
45#include <vm/vm_pager.h>
46
47/* list of phys pager objects */
48static struct pagerlst phys_pager_object_list;
49/* protect access to phys_pager_object_list */
50static struct mtx phys_pager_mtx;
51
52static void
53phys_pager_init(void)
54{
55
56	TAILQ_INIT(&phys_pager_object_list);
57	mtx_init(&phys_pager_mtx, "phys_pager list", NULL, MTX_DEF);
58}
59
60static vm_object_t
61phys_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
62    vm_ooffset_t foff, struct ucred *cred)
63{
64	vm_object_t object, object1;
65	vm_pindex_t pindex;
66
67	/*
68	 * Offset should be page aligned.
69	 */
70	if (foff & PAGE_MASK)
71		return (NULL);
72
73	pindex = OFF_TO_IDX(foff + PAGE_MASK + size);
74
75	if (handle != NULL) {
76		mtx_lock(&phys_pager_mtx);
77		/*
78		 * Look up pager, creating as necessary.
79		 */
80		object1 = NULL;
81		object = vm_pager_object_lookup(&phys_pager_object_list, handle);
82		if (object == NULL) {
83			/*
84			 * Allocate object and associate it with the pager.
85			 */
86			mtx_unlock(&phys_pager_mtx);
87			object1 = vm_object_allocate(OBJT_PHYS, pindex);
88			mtx_lock(&phys_pager_mtx);
89			object = vm_pager_object_lookup(&phys_pager_object_list,
90			    handle);
91			if (object != NULL) {
92				/*
93				 * We raced with other thread while
94				 * allocating object.
95				 */
96				if (pindex > object->size)
97					object->size = pindex;
98			} else {
99				object = object1;
100				object1 = NULL;
101				object->handle = handle;
102				vm_object_set_flag(object, OBJ_POPULATE);
103				TAILQ_INSERT_TAIL(&phys_pager_object_list,
104				    object, pager_object_list);
105			}
106		} else {
107			if (pindex > object->size)
108				object->size = pindex;
109		}
110		mtx_unlock(&phys_pager_mtx);
111		vm_object_deallocate(object1);
112	} else {
113		object = vm_object_allocate(OBJT_PHYS, pindex);
114		vm_object_set_flag(object, OBJ_POPULATE);
115	}
116
117	return (object);
118}
119
120static void
121phys_pager_dealloc(vm_object_t object)
122{
123
124	if (object->handle != NULL) {
125		VM_OBJECT_WUNLOCK(object);
126		mtx_lock(&phys_pager_mtx);
127		TAILQ_REMOVE(&phys_pager_object_list, object, pager_object_list);
128		mtx_unlock(&phys_pager_mtx);
129		VM_OBJECT_WLOCK(object);
130	}
131	object->handle = NULL;
132	object->type = OBJT_DEAD;
133}
134
135/*
136 * Fill as many pages as vm_fault has allocated for us.
137 */
138static int
139phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind,
140    int *rahead)
141{
142	int i;
143
144	VM_OBJECT_ASSERT_WLOCKED(object);
145	for (i = 0; i < count; i++) {
146		if (m[i]->valid == 0) {
147			if ((m[i]->flags & PG_ZERO) == 0)
148				pmap_zero_page(m[i]);
149			m[i]->valid = VM_PAGE_BITS_ALL;
150		}
151		KASSERT(m[i]->valid == VM_PAGE_BITS_ALL,
152		    ("phys_pager_getpages: partially valid page %p", m[i]));
153		KASSERT(m[i]->dirty == 0,
154		    ("phys_pager_getpages: dirty page %p", m[i]));
155	}
156	if (rbehind)
157		*rbehind = 0;
158	if (rahead)
159		*rahead = 0;
160	return (VM_PAGER_OK);
161}
162
163/*
164 * Implement a pretty aggressive clustered getpages strategy.  Hint that
165 * everything in an entire 4MB window should be prefaulted at once.
166 *
167 * 4MB (1024 slots per page table page) is convenient for x86,
168 * but may not be for other arches.
169 */
170#ifndef PHYSCLUSTER
171#define PHYSCLUSTER 1024
172#endif
173static int phys_pager_cluster = PHYSCLUSTER;
174SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN,
175    &phys_pager_cluster, 0,
176    "prefault window size for phys pager");
177
178/*
179 * Max hint to vm_page_alloc() about the further allocation needs
180 * inside the phys_pager_populate() loop.  The number of bits used to
181 * implement VM_ALLOC_COUNT() determines the hard limit on this value.
182 * That limit is currently 65535.
183 */
184#define	PHYSALLOC	16
185
186static int
187phys_pager_populate(vm_object_t object, vm_pindex_t pidx,
188    int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first,
189    vm_pindex_t *last)
190{
191	vm_page_t m;
192	vm_pindex_t base, end, i;
193	int ahead;
194
195	base = rounddown(pidx, phys_pager_cluster);
196	end = base + phys_pager_cluster - 1;
197	if (end >= object->size)
198		end = object->size - 1;
199	if (*first > base)
200		base = *first;
201	if (end > *last)
202		end = *last;
203	*first = base;
204	*last = end;
205
206	for (i = base; i <= end; i++) {
207retry:
208		m = vm_page_lookup(object, i);
209		if (m == NULL) {
210			ahead = MIN(end - i, PHYSALLOC);
211			m = vm_page_alloc(object, i, VM_ALLOC_NORMAL |
212			    VM_ALLOC_ZERO | VM_ALLOC_WAITFAIL |
213			    VM_ALLOC_COUNT(ahead));
214			if (m == NULL)
215				goto retry;
216			if ((m->flags & PG_ZERO) == 0)
217				pmap_zero_page(m);
218			m->valid = VM_PAGE_BITS_ALL;
219		} else if (vm_page_xbusied(m)) {
220			vm_page_lock(m);
221			VM_OBJECT_WUNLOCK(object);
222			vm_page_busy_sleep(m, "physb", true);
223			VM_OBJECT_WLOCK(object);
224			goto retry;
225		} else {
226			vm_page_xbusy(m);
227			if (m->valid != VM_PAGE_BITS_ALL)
228				vm_page_zero_invalid(m, TRUE);
229		}
230
231		KASSERT(m->valid == VM_PAGE_BITS_ALL,
232		    ("phys_pager_populate: partially valid page %p", m));
233		KASSERT(m->dirty == 0,
234		    ("phys_pager_populate: dirty page %p", m));
235	}
236	return (VM_PAGER_OK);
237}
238
239static void
240phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync,
241    int *rtvals)
242{
243
244	panic("phys_pager_putpage called");
245}
246
247static boolean_t
248phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,
249    int *after)
250{
251	vm_pindex_t base, end;
252
253	base = rounddown(pindex, phys_pager_cluster);
254	end = base + phys_pager_cluster - 1;
255	if (before != NULL)
256		*before = pindex - base;
257	if (after != NULL)
258		*after = end - pindex;
259	return (TRUE);
260}
261
262struct pagerops physpagerops = {
263	.pgo_init =	phys_pager_init,
264	.pgo_alloc =	phys_pager_alloc,
265	.pgo_dealloc = 	phys_pager_dealloc,
266	.pgo_getpages =	phys_pager_getpages,
267	.pgo_putpages =	phys_pager_putpages,
268	.pgo_haspage =	phys_pager_haspage,
269	.pgo_populate =	phys_pager_populate,
270};
271