uvm_anon.c revision 1.53 
1/*	$NetBSD: uvm_anon.c,v 1.53 2011/04/23 18:14:12 rmind Exp $	*/
2
3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28/*
29 * uvm_anon.c: uvm anon ops
30 */
31
32#include <sys/cdefs.h>
33__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.53 2011/04/23 18:14:12 rmind Exp $");
34
35#include "opt_uvmhist.h"
36
37#include <sys/param.h>
38#include <sys/systm.h>
39#include <sys/pool.h>
40#include <sys/kernel.h>
41
42#include <uvm/uvm.h>
43#include <uvm/uvm_swap.h>
44#include <uvm/uvm_pdpolicy.h>
45
46static struct pool_cache uvm_anon_cache;
47
48static int uvm_anon_ctor(void *, void *, int);
49static void uvm_anon_dtor(void *, void *);
50
51/*
52 * allocate anons
53 */
54void
55uvm_anon_init(void)
56{
57
58	pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0,
59	    PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor,
60	    uvm_anon_dtor, NULL);
61}
62
63static int
64uvm_anon_ctor(void *arg, void *object, int flags)
65{
66	struct vm_anon *anon = object;
67
68	anon->an_ref = 0;
69	mutex_init(&anon->an_lock, MUTEX_DEFAULT, IPL_NONE);
70	anon->an_page = NULL;
71#if defined(VMSWAP)
72	anon->an_swslot = 0;
73#endif /* defined(VMSWAP) */
74
75	return 0;
76}
77
78static void
79uvm_anon_dtor(void *arg, void *object)
80{
81	struct vm_anon *anon = object;
82
83	mutex_destroy(&anon->an_lock);
84}
85
86/*
87 * allocate an anon
88 *
89 * => new anon is returned locked!
90 */
91struct vm_anon *
92uvm_analloc(void)
93{
94	struct vm_anon *anon;
95
96	anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
97	if (anon) {
98		KASSERT(anon->an_ref == 0);
99		KASSERT(anon->an_page == NULL);
100#if defined(VMSWAP)
101		KASSERT(anon->an_swslot == 0);
102#endif /* defined(VMSWAP) */
103		anon->an_ref = 1;
104		mutex_enter(&anon->an_lock);
105	}
106	return anon;
107}
108
109/*
110 * uvm_anfree: free a single anon structure
111 *
112 * => caller must remove anon from its amap before calling (if it was in
113 *	an amap).
114 * => anon must be unlocked and have a zero reference count.
115 * => we may lock the pageq's.
116 */
117
118void
119uvm_anfree(struct vm_anon *anon)
120{
121	struct vm_page *pg;
122	UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist);
123	UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0);
124
125	KASSERT(anon->an_ref == 0);
126	KASSERT(!mutex_owned(&anon->an_lock));
127
128	/*
129	 * get page
130	 */
131
132	pg = anon->an_page;
133
134	/*
135	 * if there is a resident page and it is loaned, then anon may not
136	 * own it.   call out to uvm_anon_lockpage() to ensure the real owner
137 	 * of the page has been identified and locked.
138	 */
139
140	if (pg && pg->loan_count) {
141		mutex_enter(&anon->an_lock);
142		pg = uvm_anon_lockloanpg(anon);
143		mutex_exit(&anon->an_lock);
144	}
145
146	/*
147	 * if we have a resident page, we must dispose of it before freeing
148	 * the anon.
149	 */
150
151	if (pg) {
152
153		/*
154		 * if the page is owned by a uobject (now locked), then we must
155		 * kill the loan on the page rather than free it.
156		 */
157
158		if (pg->uobject) {
159			mutex_enter(&uvm_pageqlock);
160			KASSERT(pg->loan_count > 0);
161			pg->loan_count--;
162			pg->uanon = NULL;
163			mutex_exit(&uvm_pageqlock);
164			mutex_exit(&pg->uobject->vmobjlock);
165		} else {
166
167			/*
168			 * page has no uobject, so we must be the owner of it.
169			 */
170
171			KASSERT((pg->flags & PG_RELEASED) == 0);
172			mutex_enter(&anon->an_lock);
173			pmap_page_protect(pg, VM_PROT_NONE);
174
175			/*
176			 * if the page is busy, mark it as PG_RELEASED
177			 * so that uvm_anon_release will release it later.
178			 */
179
180			if (pg->flags & PG_BUSY) {
181				pg->flags |= PG_RELEASED;
182				mutex_exit(&anon->an_lock);
183				return;
184			}
185			mutex_enter(&uvm_pageqlock);
186			uvm_pagefree(pg);
187			mutex_exit(&uvm_pageqlock);
188			mutex_exit(&anon->an_lock);
189			UVMHIST_LOG(maphist, "anon 0x%x, page 0x%x: "
190				    "freed now!", anon, pg, 0, 0);
191		}
192	}
193#if defined(VMSWAP)
194	if (pg == NULL && anon->an_swslot > 0) {
195		/* this page is no longer only in swap. */
196		mutex_enter(&uvm_swap_data_lock);
197		KASSERT(uvmexp.swpgonly > 0);
198		uvmexp.swpgonly--;
199		mutex_exit(&uvm_swap_data_lock);
200	}
201#endif /* defined(VMSWAP) */
202
203	/*
204	 * free any swap resources.
205	 */
206
207	uvm_anon_dropswap(anon);
208
209	/*
210	 * give a page replacement hint.
211	 */
212
213	uvmpdpol_anfree(anon);
214
215	/*
216	 * now that we've stripped the data areas from the anon,
217	 * free the anon itself.
218	 */
219
220	KASSERT(anon->an_page == NULL);
221#if defined(VMSWAP)
222	KASSERT(anon->an_swslot == 0);
223#endif /* defined(VMSWAP) */
224
225	pool_cache_put(&uvm_anon_cache, anon);
226	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
227}
228
229#if defined(VMSWAP)
230
231/*
232 * uvm_anon_dropswap:  release any swap resources from this anon.
233 *
234 * => anon must be locked or have a reference count of 0.
235 */
236void
237uvm_anon_dropswap(struct vm_anon *anon)
238{
239	UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
240
241	if (anon->an_swslot == 0)
242		return;
243
244	UVMHIST_LOG(maphist,"freeing swap for anon %p, paged to swslot 0x%x",
245		    anon, anon->an_swslot, 0, 0);
246	uvm_swap_free(anon->an_swslot, 1);
247	anon->an_swslot = 0;
248}
249
250#endif /* defined(VMSWAP) */
251
252/*
253 * uvm_anon_lockloanpg: given a locked anon, lock its resident page
254 *
255 * => anon is locked by caller
256 * => on return: anon is locked
257 *		 if there is a resident page:
258 *			if it has a uobject, it is locked by us
259 *			if it is ownerless, we take over as owner
260 *		 we return the resident page (it can change during
261 *		 this function)
262 * => note that the only time an anon has an ownerless resident page
263 *	is if the page was loaned from a uvm_object and the uvm_object
264 *	disowned it
265 * => this only needs to be called when you want to do an operation
266 *	on an anon's resident page and that page has a non-zero loan
267 *	count.
268 */
269struct vm_page *
270uvm_anon_lockloanpg(struct vm_anon *anon)
271{
272	struct vm_page *pg;
273	bool locked = false;
274
275	KASSERT(mutex_owned(&anon->an_lock));
276
277	/*
278	 * loop while we have a resident page that has a non-zero loan count.
279	 * if we successfully get our lock, we will "break" the loop.
280	 * note that the test for pg->loan_count is not protected -- this
281	 * may produce false positive results.   note that a false positive
282	 * result may cause us to do more work than we need to, but it will
283	 * not produce an incorrect result.
284	 */
285
286	while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
287
288		/*
289		 * quickly check to see if the page has an object before
290		 * bothering to lock the page queues.   this may also produce
291		 * a false positive result, but that's ok because we do a real
292		 * check after that.
293		 */
294
295		if (pg->uobject) {
296			mutex_enter(&uvm_pageqlock);
297			if (pg->uobject) {
298				locked =
299				    mutex_tryenter(&pg->uobject->vmobjlock);
300			} else {
301				/* object disowned before we got PQ lock */
302				locked = true;
303			}
304			mutex_exit(&uvm_pageqlock);
305
306			/*
307			 * if we didn't get a lock (try lock failed), then we
308			 * toggle our anon lock and try again
309			 */
310
311			if (!locked) {
312				mutex_exit(&anon->an_lock);
313
314				/*
315				 * someone locking the object has a chance to
316				 * lock us right now
317				 */
318				/* XXX Better than yielding but inadequate. */
319				kpause("livelock", false, 1, NULL);
320
321				mutex_enter(&anon->an_lock);
322				continue;
323			}
324		}
325
326		/*
327		 * if page is un-owned [i.e. the object dropped its ownership],
328		 * then we can take over as owner!
329		 */
330
331		if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
332			mutex_enter(&uvm_pageqlock);
333			pg->pqflags |= PQ_ANON;
334			pg->loan_count--;
335			mutex_exit(&uvm_pageqlock);
336		}
337		break;
338	}
339	return(pg);
340}
341
342#if defined(VMSWAP)
343
344/*
345 * fetch an anon's page.
346 *
347 * => anon must be locked, and is unlocked upon return.
348 * => returns true if pagein was aborted due to lack of memory.
349 */
350
351bool
352uvm_anon_pagein(struct vm_anon *anon)
353{
354	struct vm_page *pg;
355	struct uvm_object *uobj;
356	int rv;
357
358	/* locked: anon */
359	KASSERT(mutex_owned(&anon->an_lock));
360
361	rv = uvmfault_anonget(NULL, NULL, anon);
362
363	/*
364	 * if rv == 0, anon is still locked, else anon
365	 * is unlocked
366	 */
367
368	switch (rv) {
369	case 0:
370		break;
371
372	case EIO:
373	case ERESTART:
374
375		/*
376		 * nothing more to do on errors.
377		 * ERESTART can only mean that the anon was freed,
378		 * so again there's nothing to do.
379		 */
380
381		return false;
382
383	default:
384		return true;
385	}
386
387	/*
388	 * ok, we've got the page now.
389	 * mark it as dirty, clear its swslot and un-busy it.
390	 */
391
392	pg = anon->an_page;
393	uobj = pg->uobject;
394	if (anon->an_swslot > 0)
395		uvm_swap_free(anon->an_swslot, 1);
396	anon->an_swslot = 0;
397	pg->flags &= ~(PG_CLEAN);
398
399	/*
400	 * deactivate the page (to put it on a page queue)
401	 */
402
403	mutex_enter(&uvm_pageqlock);
404	if (pg->wire_count == 0)
405		uvm_pagedeactivate(pg);
406	mutex_exit(&uvm_pageqlock);
407
408	if (pg->flags & PG_WANTED) {
409		wakeup(pg);
410		pg->flags &= ~(PG_WANTED);
411	}
412
413	/*
414	 * unlock the anon and we're done.
415	 */
416
417	mutex_exit(&anon->an_lock);
418	if (uobj) {
419		mutex_exit(&uobj->vmobjlock);
420	}
421	return false;
422}
423
424#endif /* defined(VMSWAP) */
425
426/*
427 * uvm_anon_release: release an anon and its page.
428 *
429 * => caller must lock the anon.
430 */
431
432void
433uvm_anon_release(struct vm_anon *anon)
434{
435	struct vm_page *pg = anon->an_page;
436
437	KASSERT(mutex_owned(&anon->an_lock));
438	KASSERT(pg != NULL);
439	KASSERT((pg->flags & PG_RELEASED) != 0);
440	KASSERT((pg->flags & PG_BUSY) != 0);
441	KASSERT(pg->uobject == NULL);
442	KASSERT(pg->uanon == anon);
443	KASSERT(pg->loan_count == 0);
444	KASSERT(anon->an_ref == 0);
445
446	mutex_enter(&uvm_pageqlock);
447	uvm_pagefree(pg);
448	mutex_exit(&uvm_pageqlock);
449	mutex_exit(&anon->an_lock);
450
451	KASSERT(anon->an_page == NULL);
452
453	uvm_anfree(anon);
454}
455