uvm_anon.c revision 1.47 
1/*	$NetBSD: uvm_anon.c,v 1.47 2007/11/07 00:23:46 ad Exp $	*/
2
3/*
4 *
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 *    must display the following acknowledgement:
18 *      This product includes software developed by Charles D. Cranor and
19 *      Washington University.
20 * 4. The name of the author may not be used to endorse or promote products
21 *    derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35/*
36 * uvm_anon.c: uvm anon ops
37 */
38
39#include <sys/cdefs.h>
40__KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.47 2007/11/07 00:23:46 ad Exp $");
41
42#include "opt_uvmhist.h"
43
44#include <sys/param.h>
45#include <sys/systm.h>
46#include <sys/proc.h>
47#include <sys/malloc.h>
48#include <sys/pool.h>
49#include <sys/kernel.h>
50
51#include <uvm/uvm.h>
52#include <uvm/uvm_swap.h>
53#include <uvm/uvm_pdpolicy.h>
54
55static POOL_INIT(uvm_anon_pool, sizeof(struct vm_anon), 0, 0, 0, "anonpl",
56    &pool_allocator_nointr, IPL_NONE);
57static struct pool_cache uvm_anon_cache;
58
59static int uvm_anon_ctor(void *, void *, int);
60static void uvm_anon_dtor(void *, void *);
61
62/*
63 * allocate anons
64 */
65void
66uvm_anon_init(void)
67{
68
69	pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0, 0,
70	    "anonpl", NULL, IPL_NONE, uvm_anon_ctor, uvm_anon_dtor, NULL);
71}
72
73static int
74uvm_anon_ctor(void *arg, void *object, int flags)
75{
76	struct vm_anon *anon = object;
77
78	anon->an_ref = 0;
79	simple_lock_init(&anon->an_lock);
80	anon->an_page = NULL;
81#if defined(VMSWAP)
82	anon->an_swslot = 0;
83#endif /* defined(VMSWAP) */
84
85	return 0;
86}
87
88static void
89uvm_anon_dtor(void *arg, void *object)
90{
91
92	/* nothing yet */
93}
94
95/*
96 * allocate an anon
97 *
98 * => new anon is returned locked!
99 */
100struct vm_anon *
101uvm_analloc(void)
102{
103	struct vm_anon *anon;
104
105	anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
106	if (anon) {
107		KASSERT(anon->an_ref == 0);
108		LOCK_ASSERT(simple_lock_held(&anon->an_lock) == 0);
109		KASSERT(anon->an_page == NULL);
110#if defined(VMSWAP)
111		KASSERT(anon->an_swslot == 0);
112#endif /* defined(VMSWAP) */
113		anon->an_ref = 1;
114		simple_lock(&anon->an_lock);
115	}
116	return anon;
117}
118
119/*
120 * uvm_anfree: free a single anon structure
121 *
122 * => caller must remove anon from its amap before calling (if it was in
123 *	an amap).
124 * => anon must be unlocked and have a zero reference count.
125 * => we may lock the pageq's.
126 */
127
128void
129uvm_anfree(struct vm_anon *anon)
130{
131	struct vm_page *pg;
132	UVMHIST_FUNC("uvm_anfree"); UVMHIST_CALLED(maphist);
133	UVMHIST_LOG(maphist,"(anon=0x%x)", anon, 0,0,0);
134
135	KASSERT(anon->an_ref == 0);
136
137	/*
138	 * get page
139	 */
140
141	pg = anon->an_page;
142
143	/*
144	 * if there is a resident page and it is loaned, then anon may not
145	 * own it.   call out to uvm_anon_lockpage() to ensure the real owner
146 	 * of the page has been identified and locked.
147	 */
148
149	if (pg && pg->loan_count) {
150		simple_lock(&anon->an_lock);
151		pg = uvm_anon_lockloanpg(anon);
152		simple_unlock(&anon->an_lock);
153	}
154
155	/*
156	 * if we have a resident page, we must dispose of it before freeing
157	 * the anon.
158	 */
159
160	if (pg) {
161
162		/*
163		 * if the page is owned by a uobject (now locked), then we must
164		 * kill the loan on the page rather than free it.
165		 */
166
167		if (pg->uobject) {
168			uvm_lock_pageq();
169			KASSERT(pg->loan_count > 0);
170			pg->loan_count--;
171			pg->uanon = NULL;
172			uvm_unlock_pageq();
173			simple_unlock(&pg->uobject->vmobjlock);
174		} else {
175
176			/*
177			 * page has no uobject, so we must be the owner of it.
178			 */
179
180			KASSERT((pg->flags & PG_RELEASED) == 0);
181			simple_lock(&anon->an_lock);
182			pmap_page_protect(pg, VM_PROT_NONE);
183
184			/*
185			 * if the page is busy, mark it as PG_RELEASED
186			 * so that uvm_anon_release will release it later.
187			 */
188
189			if (pg->flags & PG_BUSY) {
190				pg->flags |= PG_RELEASED;
191				simple_unlock(&anon->an_lock);
192				return;
193			}
194			uvm_lock_pageq();
195			uvm_pagefree(pg);
196			uvm_unlock_pageq();
197			simple_unlock(&anon->an_lock);
198			UVMHIST_LOG(maphist, "anon 0x%x, page 0x%x: "
199				    "freed now!", anon, pg, 0, 0);
200		}
201	}
202#if defined(VMSWAP)
203	if (pg == NULL && anon->an_swslot > 0) {
204		/* this page is no longer only in swap. */
205		mutex_enter(&uvm_swap_data_lock);
206		KASSERT(uvmexp.swpgonly > 0);
207		uvmexp.swpgonly--;
208		mutex_exit(&uvm_swap_data_lock);
209	}
210#endif /* defined(VMSWAP) */
211
212	/*
213	 * free any swap resources.
214	 */
215
216	uvm_anon_dropswap(anon);
217
218	/*
219	 * give a page replacement hint.
220	 */
221
222	uvmpdpol_anfree(anon);
223
224	/*
225	 * now that we've stripped the data areas from the anon,
226	 * free the anon itself.
227	 */
228
229	KASSERT(anon->an_page == NULL);
230#if defined(VMSWAP)
231	KASSERT(anon->an_swslot == 0);
232#endif /* defined(VMSWAP) */
233
234	pool_cache_put(&uvm_anon_cache, anon);
235	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
236}
237
238#if defined(VMSWAP)
239
240/*
241 * uvm_anon_dropswap:  release any swap resources from this anon.
242 *
243 * => anon must be locked or have a reference count of 0.
244 */
245void
246uvm_anon_dropswap(struct vm_anon *anon)
247{
248	UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
249
250	if (anon->an_swslot == 0)
251		return;
252
253	UVMHIST_LOG(maphist,"freeing swap for anon %p, paged to swslot 0x%x",
254		    anon, anon->an_swslot, 0, 0);
255	uvm_swap_free(anon->an_swslot, 1);
256	anon->an_swslot = 0;
257}
258
259#endif /* defined(VMSWAP) */
260
261/*
262 * uvm_anon_lockloanpg: given a locked anon, lock its resident page
263 *
264 * => anon is locked by caller
265 * => on return: anon is locked
266 *		 if there is a resident page:
267 *			if it has a uobject, it is locked by us
268 *			if it is ownerless, we take over as owner
269 *		 we return the resident page (it can change during
270 *		 this function)
271 * => note that the only time an anon has an ownerless resident page
272 *	is if the page was loaned from a uvm_object and the uvm_object
273 *	disowned it
274 * => this only needs to be called when you want to do an operation
275 *	on an anon's resident page and that page has a non-zero loan
276 *	count.
277 */
278struct vm_page *
279uvm_anon_lockloanpg(struct vm_anon *anon)
280{
281	struct vm_page *pg;
282	bool locked = false;
283
284	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
285
286	/*
287	 * loop while we have a resident page that has a non-zero loan count.
288	 * if we successfully get our lock, we will "break" the loop.
289	 * note that the test for pg->loan_count is not protected -- this
290	 * may produce false positive results.   note that a false positive
291	 * result may cause us to do more work than we need to, but it will
292	 * not produce an incorrect result.
293	 */
294
295	while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
296
297		/*
298		 * quickly check to see if the page has an object before
299		 * bothering to lock the page queues.   this may also produce
300		 * a false positive result, but that's ok because we do a real
301		 * check after that.
302		 */
303
304		if (pg->uobject) {
305			uvm_lock_pageq();
306			if (pg->uobject) {
307				locked =
308				    simple_lock_try(&pg->uobject->vmobjlock);
309			} else {
310				/* object disowned before we got PQ lock */
311				locked = true;
312			}
313			uvm_unlock_pageq();
314
315			/*
316			 * if we didn't get a lock (try lock failed), then we
317			 * toggle our anon lock and try again
318			 */
319
320			if (!locked) {
321				simple_unlock(&anon->an_lock);
322
323				/*
324				 * someone locking the object has a chance to
325				 * lock us right now
326				 */
327
328				simple_lock(&anon->an_lock);
329				continue;
330			}
331		}
332
333		/*
334		 * if page is un-owned [i.e. the object dropped its ownership],
335		 * then we can take over as owner!
336		 */
337
338		if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
339			uvm_lock_pageq();
340			pg->pqflags |= PQ_ANON;
341			pg->loan_count--;
342			uvm_unlock_pageq();
343		}
344		break;
345	}
346	return(pg);
347}
348
349#if defined(VMSWAP)
350
351/*
352 * fetch an anon's page.
353 *
354 * => anon must be locked, and is unlocked upon return.
355 * => returns true if pagein was aborted due to lack of memory.
356 */
357
358bool
359uvm_anon_pagein(struct vm_anon *anon)
360{
361	struct vm_page *pg;
362	struct uvm_object *uobj;
363	int rv;
364
365	/* locked: anon */
366	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
367
368	rv = uvmfault_anonget(NULL, NULL, anon);
369
370	/*
371	 * if rv == 0, anon is still locked, else anon
372	 * is unlocked
373	 */
374
375	switch (rv) {
376	case 0:
377		break;
378
379	case EIO:
380	case ERESTART:
381
382		/*
383		 * nothing more to do on errors.
384		 * ERESTART can only mean that the anon was freed,
385		 * so again there's nothing to do.
386		 */
387
388		return false;
389
390	default:
391		return true;
392	}
393
394	/*
395	 * ok, we've got the page now.
396	 * mark it as dirty, clear its swslot and un-busy it.
397	 */
398
399	pg = anon->an_page;
400	uobj = pg->uobject;
401	if (anon->an_swslot > 0)
402		uvm_swap_free(anon->an_swslot, 1);
403	anon->an_swslot = 0;
404	pg->flags &= ~(PG_CLEAN);
405
406	/*
407	 * deactivate the page (to put it on a page queue)
408	 */
409
410	pmap_clear_reference(pg);
411	uvm_lock_pageq();
412	if (pg->wire_count == 0)
413		uvm_pagedeactivate(pg);
414	uvm_unlock_pageq();
415
416	if (pg->flags & PG_WANTED) {
417		wakeup(pg);
418		pg->flags &= ~(PG_WANTED);
419	}
420
421	/*
422	 * unlock the anon and we're done.
423	 */
424
425	simple_unlock(&anon->an_lock);
426	if (uobj) {
427		simple_unlock(&uobj->vmobjlock);
428	}
429	return false;
430}
431
432#endif /* defined(VMSWAP) */
433
434/*
435 * uvm_anon_release: release an anon and its page.
436 *
437 * => caller must lock the anon.
438 */
439
440void
441uvm_anon_release(struct vm_anon *anon)
442{
443	struct vm_page *pg = anon->an_page;
444
445	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
446
447	KASSERT(pg != NULL);
448	KASSERT((pg->flags & PG_RELEASED) != 0);
449	KASSERT((pg->flags & PG_BUSY) != 0);
450	KASSERT(pg->uobject == NULL);
451	KASSERT(pg->uanon == anon);
452	KASSERT(pg->loan_count == 0);
453	KASSERT(anon->an_ref == 0);
454
455	uvm_lock_pageq();
456	uvm_pagefree(pg);
457	uvm_unlock_pageq();
458	simple_unlock(&anon->an_lock);
459
460	KASSERT(anon->an_page == NULL);
461
462	uvm_anfree(anon);
463}
464